passing custom data to exceptions in request_futures

Question

I'm running the following code to make asynchronous "get" requests. The CustomSession class is used to add support for timing each request.

If an exception occurs or the request runs fine, I want to be able to access the test_id which is appended to the futures list, along with the URL to request. In other words, when a request runs or an exception is thrown, I want to find the test_id associated with the call to session.get.

from datetime import datetime
from requests_futures.sessions import FuturesSession

class CustomSession(FuturesSession):

    def __init__(self, *args, **kwargs):
        super(CustomSession, self).__init__(*args, **kwargs)
        self.timing = {}

    def request(self, method, url, *args, **kwargs):
        background_callback = kwargs.pop('background_callback', None)
        test_id = kwargs.pop('test_id', None)

        # start counting
        self.timing[test_id] = datetime.now()

        def time_it(sess, resp):
            # here if you want to time the server stuff only
            self.timing[test_id] = datetime.now() - self.timing[test_id]
            if background_callback:
                background_callback(sess, resp)
            # here if you want to include any time in the callback

        return super(CustomSession, self).request(method, url, *args,
                                                  background_callback=time_it,
                                                  **kwargs)


session = CustomSession()

futures = []
for url in ('http://httpbin.org/get?key=val',
            'http://httpasdfasfsadfasdfasdfbin.org/get?key2=val2'):

    futures.append(session.get(url, test_id=1))
for future in futures:
    try:
        r = future.result()
        print(r.status_code)
    except Exception as e:
        print(e)

Answer 1

I created a decorator for the result() function of the future object:

def mark_exception(fn, id, url):
    def new_fn(*args, **kwargs):
        try:
            return fn(*args, **kwargs)
        except:
            raise Exception("test id %d with url %s threw exception" % (id, url))
    return new_fn

And applied it at the end of your CustomSession.request() function, replacing the original return statement:

future =  super(CustomSession, self).request(method, url, *args,
                                          background_callback=time_it,
                                          **kwargs)
future.result = mark_exception(future.result, test_id, url)
return future

Output:

200
test id 1 with url http://httpasdfasfsadfasdfasdfbin.org/get?key2=val2 threw exception

I hope this helps.

EDIT:

If you want to get the test id for every future, here are two ways you can do it:

futures = []
for url in ('http://httpbin.org/get?key=val',
            'http://httpasdfasfsadfasdfasdfbin.org/get?key2=val2'):
    tid = 1
    future = session.get(url, test_id=tid)
    # option 1: set test_id as an attrib of the future object
    future.test_id = tid
    # option 2: put test_id and future object in a tuple before appending to the list
    futures.append((tid, future))
for tid, future in futures:
    try:
        r = future.result()
        print("tracked test_id is %d" % tid) #option 2
        print("status for test_id %d is %d" % (future.test_id, r.status_code)) #option 1
    except Exception as e:
        print(e)