Sharing dictionary over multiprocesses (TypeError: cannot pickle 'weakref' object)

Question

I want to create a class Storage where each object has a dictionary orderbooks as a property. I want to write on orderbooks from the main process by invoking the method write, but I want to defer this action to another process and ensuring that the dictionary orderbooks is accessible from the main process.

To do so, I create a Mananger() that I pass during the definition of the object and that is used to notify the processes about the changes of the dictionary. My code is the following:

from multiprocessing import Process, Manager

class Storage():
    def __init__(self,manager):
        self.manager = manager
        self.orderbooks = self.manager.dict()

    def store_value(self,el):
        self.orderbooks[el[0]] = el[1]

    def write(self,el:list):
        p = Process(target=self.store_value,args=(el,))
        p.start()


if __name__ == '__main__':

    manager=Manager()
    book1 = Storage(manager)
    book1.write([0,1])

However, when I run this code, I get the following error

Traceback (most recent call last):
  File "/Users/main_user/PycharmProjects/handle_queue/main.py", line 21, in <module>
    book1.write([0,1])
  File "/Users/main_user/PycharmProjects/handle_queue/main.py", line 13, in write
    p.start()
  File "/Users/main_user/opt/anaconda3/envs/handle_queue/lib/python3.10/multiprocessing/process.py", line 121, in start
    self._popen = self._Popen(self)
  File "/Users/main_user/opt/anaconda3/envs/handle_queue/lib/python3.10/multiprocessing/context.py", line 224, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "/Users/main_user/opt/anaconda3/envs/handle_queue/lib/python3.10/multiprocessing/context.py", line 284, in _Popen
    return Popen(process_obj)
  File "/Users/main_user/opt/anaconda3/envs/handle_queue/lib/python3.10/multiprocessing/popen_spawn_posix.py", line 32, in __init__
    super().__init__(process_obj)
  File "/Users/main_user/opt/anaconda3/envs/handle_queue/lib/python3.10/multiprocessing/popen_fork.py", line 19, in __init__
    self._launch(process_obj)
  File "/Users/main_user/opt/anaconda3/envs/handle_queue/lib/python3.10/multiprocessing/popen_spawn_posix.py", line 47, in _launch
    reduction.dump(process_obj, fp)
  File "/Users/main_user/opt/anaconda3/envs/handle_queue/lib/python3.10/multiprocessing/reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref' object

What is wrong with my code?

Answer 1

Per Aaron's posted comment:

from multiprocessing import Process, Manager

class Storage():
    def __init__(self, orderbooks):
        self.orderbooks = orderbooks

    def store_value(self, el):
        self.orderbooks[el[0]] = el[1]

    def write(self, el: list):
        p = Process(target=self.store_value, args=(el,))
        p.start()
        # Ensure we do not return until store_value has
        # completed updating the dictionary:
        p.join()


if __name__ == '__main__':
    manager = Manager()
    orderbooks = manager.dict()
    book1 = Storage(orderbooks)
    book1.write([0, 1])
    print(orderbooks)

Prints:

{0: 1}