Search code examples
pythonmultithreadingalgorithmsearchprocessing-efficiency

Efficient search algorithm in python to search strings in all sheets of an excel workbook and return matching sheet numbers


How do I search a string/pattern in all the sheets of a workbook and return all matching sheet numbers of the workbook?

I can traverse all the sheets in an Excel workbook, one by one, and search the string in each sheet (like a linear search) but it is inefficient and takes a long time, and I have to process hundreds of workbooks or even more.

Update 1: Sample code

from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool

def searchSheets(fnames):
    #Search Logic here
    #Loop over each Sheet
    #Search for string 'Balance' in each Sheet
    #Return matching Sheet Number

if __name__ == '__main__':
    __spec__ = None

    folder = "C://AB//"
    if os.path.exists(folder):
        files = glob.glob(folder + "*.xlsx")


    #Multi threading   
    pool = Pool()
    pool=ThreadPool(processes=10)
    #Suggested by @Dan D
    pool.map(searchSheets,files) # It did not work
    pool.close()    

Update 2:Error

multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
  File "C:\ProgramData\Anaconda3\lib\multiprocessing\pool.py", line 119, in work
er
    result = (True, func(*args, **kwds))
  File "C:\ProgramData\Anaconda3\lib\multiprocessing\pool.py", line 44, in mapst
ar
    return list(map(*args))
  File "C:\temp3.py", line 36, in searchSheet
    wb = xl_wb(f)
  File "C:\ProgramData\Anaconda3\lib\site-packages\xlrd\__init__.py", line 116,
in open_workbook
    with open(filename, "rb") as f:
FileNotFoundError: [Errno 2] No such file or directory: 'C'
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "C:\temp3.py", line 167, in <module>
    pool.map(searchSheet,files)
  File "C:\ProgramData\Anaconda3\lib\multiprocessing\pool.py", line 266, in map
    return self._map_async(func, iterable, mapstar, chunksize).get()
  File "C:\ProgramData\Anaconda3\lib\multiprocessing\pool.py", line 644, in get
    raise self._value
FileNotFoundError: [Errno 2] No such file or directory: 'C'
>>>

Solution

  • Solution

    from multiprocessing import Pool
    from multiprocessing.dummy import Pool as ThreadPool
    
    def searchSheets(fnames):
        #Search Logic here
        #Loop over each Sheet
        #Search for string 'Balance' in each Sheet
        #Return matching Sheet Number
    
    if __name__ == '__main__':
        __spec__ = None
    
        folder = "C://AB//"
        if os.path.exists(folder):
            files = glob.glob(folder + "*.xlsx")
    
    
        #Multi threading   
        pool = Pool()
        pool=ThreadPool(processes=10)
        #Suggested by @Dan D
        #pool.map(searchSheets,files) # It did not work
        pool.map(searchSheets,[workbook for workbook in files],)
        multiprocessing.freeze_support() # this line is needed on window 
        #only,found it in may other posts
        pool.close()    
        #pool.join() #Removed this from code as it made all the workers to wait