Search code examples
pythonpython-3.xbeautifulsouppython-itertoolswebrequest

Adding data item data into grouping using itertools in Python Beautifulsoup


The snippet works already with the help of @Andrej Kesely but needs some improvements. I am not able to comprehend how to add the improvements I wanted in the snippet.

import requests
from bs4 import BeautifulSoup
from itertools import groupby
from time import sleep

url = "https://bscscan.com/tokentxns"
soup = BeautifulSoup(requests.get(url).content, "html.parser")

data = []
for tr in soup.select("tr:has(td)"):
    tds = [td.get_text(strip=True) for td in tr.select("td")]
    _, txn_hash, tm, age, from_, _, to_, value, token = tds
    a = tr.select("a")[-1]["href"][7:]
    data.append((a, value, token))

data = sorted(data)
for _, g in groupby(data, lambda k: k[0]):
    g = list(map(list, g))
    for subl in g[1:]:
        subl[0] = ""

    for subl in g:
        print("{:<45} {:<30} {:<20}".format(*subl) )
    print()
sleep(60)
    
        

Current Output:

0xe9e7cea3dedca5984780bafc599bd69add087d56    1.8                   Binance-Peg ...(BUSD)
                                              112                   Binance-Peg ...(BUSD)
                                              64.5                  Binance-Peg ...(BUSD)
0xacfc95585d80ab62f67a14c566c1b7a49fe91167    36,659,510.5          FEGtoken (FEG)      
                                              6,667,905             FEGtoken (FEG)      
0xeb32ca0d96602176f177178a8c3a37fd557b6e5f    1,000,000             SpaceEthereu...(SPACEE...)
                                              1,951.2               SpaceEthereu...(SPACEE...)
                                              39.82                 SpaceEthereu...(SPACEE...)

Wanted Output: TRANS / TOTAL (wanted data to be added to the display)

-> 3 TRANS  0xe9e7cea3dedca5984780bafc599bd69add087d56  Binance-Peg ...(BUSD)      1.8            -> 178.3 TOTAL
                                                                                   112           
                                                                                   64.5          
-> 2 TRANS  0xacfc95585d80ab62f67a14c566c1b7a49fe91167  FEGtoken (FEG)             36,659,510.5   -> 43,327,415.5 TOTAL    
                                                                                   6,667,905           
-> 3 TRANS  0xeb32ca0d96602176f177178a8c3a37fd557b6e5f  SpaceEthereu...(SPACEE...) 1,000,000      -> 1,001,991.02 TOTAL
                                                                                   1,951.2       
                                                                                   39.82         

Solution

  • Try:

    import requests
    from bs4 import BeautifulSoup
    from itertools import groupby
    
    url = "https://bscscan.com/tokentxns"
    soup = BeautifulSoup(requests.get(url).content, "html.parser")
    
    data = []
    for tr in soup.select("tr:has(td)"):
        tds = [td.get_text(strip=True) for td in tr.select("td")]
        _, txn_hash, tm, age, from_, _, to_, value, token = tds
        a = tr.select("a")[-1]["href"][7:]
        data.append((a, value, token))
    
    data = sorted(data)
    for _, g in groupby(data, lambda k: k[0]):
        g = list(map(list, g))
        total = sum(float(s.replace(",", "")) for _, s, *_ in g)
        total = [f"{total} TOTAL", *[""] * (len(g) - 1)]
        trans = [f"{len(g)} TRANS", *[""] * (len(g) - 1)]
        for subl in g[1:]:
            subl[0] = ""
    
        for tr, t, subl in zip(trans, total, g):
            print("{:<10} {:<45} {:<35} {:<30} {:<10}".format(tr, *subl, t))
        print()
    

    Prints:

    1 TRANS    0x009cf7bc57584b7998236eff51b98a168dcea9b0    0                                   SyrupBar Tok...(SYRUP)         0.0 TOTAL 
    
    5 TRANS    0x0e09fabb73bd3ade0a17ecc321fd13a19e81ce82    0                                   PancakeSwap ...(Cake)          29.477368351122603 TOTAL
                                                             0                                   PancakeSwap ...(Cake)                    
                                                             1.145747074846704429                PancakeSwap ...(Cake)                    
                                                             2.575601934206900007                PancakeSwap ...(Cake)                    
                                                             25.756019342069000075               PancakeSwap ...(Cake)                    
    
    13 TRANS   0x154a9f9cbd3449ad22fdae23044319d6ef2a1fab    0                                   CryptoBlades...(SKILL)         34.27483551127292 TOTAL
                                                             0.002499999999999999                CryptoBlades...(SKILL)                   
                                                             0.025                               CryptoBlades...(SKILL)                   
                                                             0.029999999999999999                CryptoBlades...(SKILL)                   
                                                             0.032352133402011616                CryptoBlades...(SKILL)                   
                                                             0.04                                CryptoBlades...(SKILL)                   
                                                             0.14                                CryptoBlades...(SKILL)                   
                                                             0.17                                CryptoBlades...(SKILL)                   
                                                             0.176755050266694096                CryptoBlades...(SKILL)                   
                                                             0.3                                 CryptoBlades...(SKILL)                   
                                                             1.1                                 CryptoBlades...(SKILL)                   
                                                             1.281535918881612748                CryptoBlades...(SKILL)                   
                                                             30.976692408722603577               CryptoBlades...(SKILL)                   
    
    1 TRANS    0x1796ae0b0fa4862485106a0de9b654efe301d0b2    1.723939371273915917                Polkamon (PMON)                1.723939371273916 TOTAL
    
    1 TRANS    0x1b2016ccad83172f149f470457104e6659b411ab    9                                   PowerDEX (POWERD...)           9.0 TOTAL 
    
    
    ...