How do I share a list across processes with SyncManager in Python without passing references

Question

In python, the multiprocessing module provides managers that can generate shared lists/dicts between processes.

However, I'm having an issue using these shared objects if the processes accessing the manager are not child processes, but are instead connecting to the manager via Manager.connect.

Here's a very basic example: I'm trying to create a shared list that can be accessed by a group of processes. For this example, I'm just launching this same code twice in two terminal windows:

import os, time
from multiprocessing.managers import SyncManager
            
def main() -> None:
    print(f"I am process {os.getpid()}")
    print(f"Starting proxy server...")
 
    manager = SyncManager(address=("127.0.0.1", 8000), authkey=b"noauth")
    try:
        manager.start() # will start the sync process if it doesn't exist
    except:
        manager.connect() # if it does already exist, connect to it instead
 
    print(f"Proxy server started/connected")
 
    # would like to generate a shared list that each process can access.
    sharedList = manager.list() # this generates a new list, so each process gets their own, which is not what I want
 
    sharedList.append(os.getpid())
 
    time.sleep(20)
 
if __name__ == '__main__':
    main()

Pythons documentation on using remote managers seems similar to what I'm looking for, but there's no information on how to get a Manager.list or Manager.dict shared.

Note: I'd also be perfectly happy sharing a namespace object.

Answer 1

Here's how I ended up solving the problem. You need to spawn a manager process, that is in possession of the shared list, manually.

import multiprocessing
from multiprocessing import process
import os, time, sys
from multiprocessing.managers import SyncManager, ListProxy
from queue import Queue

class SharedStorage(SyncManager):
    pass

def ManagerProcess():
    sys.stdout = open(os.devnull, 'w') 
    sys.stderr = open(os.devnull, 'w') 

    l = list()

    SharedStorage.register('get_list', lambda: l, ListProxy)
    try: 
        ss = SharedStorage(address=("127.0.0.1", 8000), authkey=b"noauth")
        ss.get_server().serve_forever()
    except OSError:
        # failed to listen on port - already in use.
        pass

def main() -> None:
    print(f"I am process {os.getpid()}")
    print(f"Starting proxy server...")

    mainProcess = multiprocessing.Process(target=ManagerProcess, daemon=True)
    mainProcess.start()

    SharedStorage.register('get_list')
    manager = SharedStorage(address=("127.0.0.1", 8000), authkey=b"noauth")
    manager.connect()

    print(f"Proxy server started/connected")

    # required - see https://bugs.python.org/issue7503
    multiprocessing.current_process().authkey = b"noauth"

    # get reference to the shared list object
    shared_list = manager.get_list()

    shared_list.append(os.getpid())

    for i in shared_list:
        print(i)

    time.sleep(20)

if __name__ == '__main__':
    main()

This can be run several times safely, as manager processes spawned by later processes will exit after they cannot listen on the port.