I have currently been working where I create my own exception etc:
# -*- coding: utf-8 -*-
# ------------------------------------------------------------------------------- #
"""
exceptions
~~~~~~~~~~~~~~~~~~~
This module contains the set of multiple exceptions.
"""
# ------------------------------------------------------------------------------- #
class RequestsException(Exception):
"""
Base exception class
"""
class TooManyFailedRequests(RequestsException):
"""
Raise an exception for FailedRequests
"""
class TooManyTimedOut(RequestsException):
"""
Raise an exception for TimedOut
"""
and I have created a script that has a counter:
import time
import requests
from requests.exceptions import ConnectionError, ReadTimeout, Timeout
from lib.exceptions import (
TooManyFailedRequests,
TooManyTimedOut
)
def send_notification(msg):
print(f"Sending notification to discord later on! {msg}")
class simpleExceptionWay:
"""
Counter to check if we get exceptions x times in a row.
"""
def __init__(self):
self.failedCnt = 0
def check(self, exception, msg):
self.failedCnt += 1
if self.failedCnt > 2:
send_notification(msg)
raise exception(msg)
def reset(self):
self.failedCnt = 0
# Call this function where we want to check
# simpleException.check(exception, msg)
simpleException = simpleExceptionWay()
def test():
"""
We also want to count in here if we ever reach in here.
"""
return "Hello world"
def setup_scraper(site_url):
session = requests.session()
while True:
try:
response = session.post(site_url, timeout=5)
if response.ok:
simpleException.reset()
exit()
if response.status_code in {429, 403, 405}:
print(f"Status -> {response.status_code}")
simpleException.check(
TooManyFailedRequests,
{
'Title': f'Too many {response.status_code} requests in a row',
'Reason': str(),
'URL': str(site_url),
'Proxies': str(
session.proxies
if session.proxies
else "No proxies used"
),
'Headers': str(session.headers),
'Cookies': str(session.cookies)
}
)
time.sleep(3)
continue
except (ReadTimeout, Timeout, ConnectionError) as err:
simpleException.check(
TooManyTimedOut,
{
'Title': f'Timed out',
'Reason': "Too many timed out",
'URL': str(site_url),
'Proxies': str(
session.proxies
if session.proxies
else "No proxies used"
),
'Headers': str(session.headers),
'Cookies': str(session.cookies)
}
)
continue
if __name__ == "__main__":
setup_scraper("https://www.google.se/")
My problem currently is that I only have 1 counter that counts all together and I wonder how can I do for example if we reach TooManyTimedOut 100 times in a row, then we want to raise and when we reach TooManyFailedRequests 5 times in a row then we should raise exception. Is that possible to do it?
You could use the type() function to get the type of the exception and then use the name property of that type to get it's name as a string. Then you can store this in a dictionary that will keep track of exception count.
class simpleExceptionWay:
"""
Counter to check if we get exceptions x times in a row.
"""
def __init__(self):
self.exception_count = {}
def check(self, exception, msg):
exception_name = exception.__name__
# the dict.get() method will return the value from a dict if exists, else it will return the value provided
self.exception_count[exception_name] = self.exception_count.get(exception_name, 0) + 1
if self.exception_count[exception_name] > 2:
send_notification(msg)
raise exception(msg)
def reset(self):
self.exception_count.clear()