Python Requests module - Access multiple URL's at once

Question

I need to validate services and their dependencies using around 500+ URL's and I already have a python code that does it. But the problem is that some of the URL's take a minute each to give a response [due to some known dependencies being down]. As each URL is hosted on a different server, is there a way to access multiple URL's at once using the requests module?

Below is my entire code I use in pycharm:

import requests
import json
import pandas
import datetime
from requests.auth import HTTPBasicAuth
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


def validate():

    line = "---------------------------------------------------------------------------------------------------"
    print("Validation started for:", datetime.datetime.now().strftime("%d-%B-%Y at %H:%M:%S"), "\n" + line)
    username = 'username'
    password = 'password'
    mydata = pandas.read_excel(r'C:\mydata.xlsx', sheet_name='Sheet1')

    for i in mydata.index:
        srno = str(mydata['Sr No'][i])
        service = mydata['Service Name'][i]
        machine = mydata['Machine Name'][i]
        url = mydata['Node'][i]

        alwaysdownservice = ['service1', 'service2']
        paydown = ['dependency1', 'dependency2', 'dependency3']
        otherdown = ['dependency3']

        def get():
            response = requests.get(url, verify=False, auth=HTTPBasicAuth(username, password))
            data = json.loads(response.text)
            status = data['Success']
            if not status:
                response = requests.get(url, verify=False, auth=HTTPBasicAuth(username, password))
                data = json.loads(response.text)
                status = data['Success']
                if not status:
                    for j in list(data['Dependencies']):
                        dependency = j['DependencyName']
                        d_status = j['Success']
                        if not d_status:
                            if service in alwaysdownservice:
                                if dependency not in paydown:
                                    print(Dependency, "down on", machine, "for", service.)
                            else:
                                if dependency not in otherdown:
                                    print(Dependency, "down on", machine, "for", service.)
                                    
        try:
            get()
            
        except Exception as e:
                        print(line, "\n", e, "\n", srno, "| Below URL is not accessible: \n", url, "\n" + line)



validate()

Answer 1

For people who need the solution. I found this from @Yurii Kramarenko. Which worked perfectly and now my script finishes its run in 30 seconds instead of 10-11 minutes.

My Script -

def validate():
    alwaysdownservice = ['service1', 'service2']
    paydown = ['dependency1', 'dependency2', 'dependency3']
    otherdown = ['dependency3']
    username = 'username'
    password = 'password'
    mydata = pandas.read_excel(r'C:\mydata.xlsx', sheet_name='Sheet1')
    urls = mydata['urls']
    line = "---------------------------------------------------------------------------------------------------"
    print("Validation started for:", datetime.datetime.now().strftime("%d-%B-%Y at %H:%M:%S"), "\n" + line)

    async def fetch(session, url):
        async with session.get(url, auth=aiohttp.BasicAuth(username, password), ssl=False) as response:

            data = await response.text()
            data = json.loads(data)
            status = data['Success']

            if not status:
                for j in list(data['Dependencies']):
                    dependency = j['DependencyName']
                    d_status = j['Success']
                    if not d_status:
                        if service in alwaysdownservice:
                            if dependency not in paydown:
                                print("Dependency -",
                                      "\'" + dependency + "\'", "down on", "\nURL -", url, "\n" + line)
                        else:
                            if dependency not in otherdown:
                                ("Dependency -",
                                      "\'" + dependency + "\'", "down on", "\nURL -", url, "\n" + line)

            print(url, "validated at:", datetime.datetime.now().strftime("%H:%M:%S"))

    async def fetch_all(urls, loop):
        async with aiohttp.ClientSession(loop=loop) as session:
            results = await asyncio.gather(*[fetch(session, url) for url in urls], return_exceptions=True)

    if __name__ == '__main__':
        loop = asyncio.get_event_loop()
        htmls = loop.run_until_complete(fetch_all(urls, loop))

    print("Validation completed for:",
          datetime.datetime.now().strftime("%d-%B-%Y at %H:%M:%S"), "\n" + line, "\n" + line,)


validate()