Search code examples
pythonfile-iomemoizationshelve

Can Python's shelve.open be called in a nested fashion?


I am trying to write a memoization library that uses shelve to store the return values persistently. If I have memoized functions calling other memoized functions, I am wondering about how to correctly open the shelf file.

import shelve
import functools


def cache(filename):
    def decorating_function(user_function):
        def wrapper(*args, **kwds):
            key = str(hash(functools._make_key(args, kwds, typed=False)))
            with shelve.open(filename, writeback=True) as cache:
                if key in cache:
                    return cache[key]
                else:
                    result = user_function(*args, **kwds)
                    cache[key] = result
                    return result

        return functools.update_wrapper(wrapper, user_function)

    return decorating_function


@cache(filename='cache')
def expensive_calculation():
    print('inside function')
    return


@cache(filename='cache')
def other_expensive_calculation():
    print('outside function')
    return expensive_calculation()

other_expensive_calculation()

Except this doesn't work

$ python3 shelve_test.py
outside function
Traceback (most recent call last):
  File "shelve_test.py", line 33, in <module>
    other_expensive_calculation()
  File "shelve_test.py", line 13, in wrapper
    result = user_function(*args, **kwds)
  File "shelve_test.py", line 31, in other_expensive_calculation
    return expensive_calculation()
  File "shelve_test.py", line 9, in wrapper
    with shelve.open(filename, writeback=True) as cache:
  File "/usr/local/Cellar/python3/3.4.1/Frameworks/Python.framework/Versions/3.4/lib/python3.4/shelve.py", line 239, in open
    return DbfilenameShelf(filename, flag, protocol, writeback)
  File "/usr/local/Cellar/python3/3.4.1/Frameworks/Python.framework/Versions/3.4/lib/python3.4/shelve.py", line 223, in __init__
    Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback)
  File "/usr/local/Cellar/python3/3.4.1/Frameworks/Python.framework/Versions/3.4/lib/python3.4/dbm/__init__.py", line 94, in open
    return mod.open(file, flag, mode)
_gdbm.error: [Errno 35] Resource temporarily unavailable

What you recommend for a solution to this sort of problem.


Solution

  • Rather than trying to nest calls to open (which as you have discovered, does not work), you could make your decorator maintain a reference to the handle returned by shelve.open, and then if it exists and is still open, re-use that for subsequent calls:

    import shelve
    import functools
    
    def _check_cache(cache_, key, func, args, kwargs):
        if key in cache_:
            print("Using cached results")
            return cache_[key]
        else:
            print("No cached results, calling function")
            result = func(*args, **kwargs)
            cache_[key] = result
            return result
    
    def cache(filename):
        def decorating_function(user_function):
            def wrapper(*args, **kwds):
                args_key = str(hash(functools._make_key(args, kwds, typed=False)))
                func_key = '.'.join([user_function.__module__, user_function.__name__])
                key = func_key + args_key
                handle_name = "{}_handle".format(filename)
                if (hasattr(cache, handle_name) and
                    not hasattr(getattr(cache, handle_name).dict, "closed")
                   ):
                    print("Using open handle")
                    return _check_cache(getattr(cache, handle_name), key, 
                                        user_function, args, kwds)
                else:
                    print("Opening handle")
                    with shelve.open(filename, writeback=True) as c:
                        setattr(cache, handle_name, c)  # Save a reference to the open handle
                        return _check_cache(c, key, user_function, args, kwds)
    
            return functools.update_wrapper(wrapper, user_function)
        return decorating_function
    
    
    @cache(filename='cache')
    def expensive_calculation():
        print('inside function')
        return
    
    
    @cache(filename='cache')
    def other_expensive_calculation():
        print('outside function')
        return expensive_calculation()
    
    other_expensive_calculation()
    print("Again")
    other_expensive_calculation()
    

    Output:

    Opening handle
    No cached results, calling function
    outside function
    Using open handle
    No cached results, calling function
    inside function
    Again
    Opening handle
    Using cached results
    

    Edit:

    You could also implement the decorator using a WeakValueDictionary, which looks a bit more readable:

    from weakref import WeakValueDictionary
    
    _handle_dict = WeakValueDictionary()
    def cache(filename):
        def decorating_function(user_function):
            def wrapper(*args, **kwds):
                args_key = str(hash(functools._make_key(args, kwds, typed=False)))
                func_key = '.'.join([user_function.__module__, user_function.__name__])
                key = func_key + args_key
                handle_name = "{}_handle".format(filename)
                if handle_name in _handle_dict:
                    print("Using open handle")
                    return _check_cache(_handle_dict[handle_name], key, 
                                        user_function, args, kwds)
                else:
                    print("Opening handle")
                    with shelve.open(filename, writeback=True) as c:
                        _handle_dict[handle_name] = c
                        return _check_cache(c, key, user_function, args, kwds)
    
            return functools.update_wrapper(wrapper, user_function)
        return decorating_function
    

    As soon as there are no other references to a handle, it will be deleted from the dictionary. Since our handle only goes out of scope when the outer-most call to a decorated function ends, we'll always have an entry in the dict while a handle is open, and no entry right after it closes.