How to get table data from Wikipedia page using python flask

Question

I tried scraping a wiki table from IBM using BeautifulSoup and Pandas. but I am unable to show the result using flask. What could be the reason?

Here is my code:

    from flask import Flask
    app = Flask(__name__)

    import pandas as pd
    import requests
    from bs4 import BeautifulSoup

    @app.route('/')
    def table():
        url = "https://en.wikipedia.org/wiki/IBM"
        page = requests.get(url)
        soup = BeautifulSoup(page.content, 'html.parser')
        table=soup.find('table',{'class':'wikitable float-left'}).tbody
        rows=table.find_all('tr')
        columns=[v.text.replace('\n', '')for v in rows [0].find_all('th')]
        return(columns)
        df= pd.DataFrame(columns=columns)

    for i in range(1, len(rows)):
        tds=rows[i].find_all('td')
        if len(tds)==6:
            values=[tds[0].text.replace('\n',''),tds[1].text.replace('\n',''),tds[2].text.replace('\n',''), tds[3].text.replace('\n',''),tds[4].text.replace('\n',''),tds[5].text.replace('\n', ' ').replace('\n', ' ')]
        df=df.append(pd.Series(values,index=columns),ignore_index=True)
         return(df)
    if __name__ == '__main__':
        app.run(debug=True)

Answer 1

Here is your python file:

from flask import Flask, render_template
app = Flask(__name__)

import pandas as pd
import requests
from bs4 import BeautifulSoup

@app.route('/')
def table():
    url = "https://en.wikipedia.org/wiki/IBM"
    page = requests.get(url)
    soup = BeautifulSoup(page.content, 'html.parser')
    table=soup.find('table',{'class':'wikitable float-left'}).tbody
    rows=table.find_all('tr')
    columns=[v.text.replace('\n', '')for v in rows [0].find_all('th')]
    # return(columns)
    df= pd.DataFrame(columns=columns)

    for i in range(1, len(rows)):
        tds=rows[i].find_all('td')
        if len(tds)==6:
            values=[tds[0].text.replace('\n',''),tds[1].text.replace('\n',''),tds[2].text.replace('\n',''), tds[3].text.replace('\n',''),tds[4].text.replace('\n',''),tds[5].text.replace('\n', ' ').replace('\n', ' ')]
            df=df.append(pd.Series(values,index=columns),ignore_index=True)
    return render_template('index.html', tables=[df.to_html(classes='data')], titles=df.columns.values)


if __name__ == '__main__':
    app.run(debug=True)

and here is your html file which should be located in your project/templates path:

{% for table in tables %}
            {{titles[loop.index]}}
            {{ table|safe }}
{% endfor %}