Converting multiple gz file within subdirectories into csv

Question

I have many subdirectories in my main directory and would like to write a script to unzip and convert all the files within it. If possible, I would also like to combine all the CSV within a single directory into a single CSV. But more importantly, I need help with my nested loop.

import gzip
import csv
import os

subdirlist = os.listdir('/home/user/Desktop/testloop')
subtotal = len(subdirlist)
subcounter = 0
for dirlist in subdirlist:
    print "Working On " + dirlist
    total = len(dirlist)
    counter = 0
    for dir in dirlist:
        print "Working On " + dir
        f = gzip.open('/' + str(subdirlist) + '/' + dir, 'rb')
        file_content = f.read()
        f.close()       
        print "25% Complete"    
        filename = '/' + str(subdirlist) + '/temp.txt'
        target = open(filename, 'w')
        target.write(file_content)
        target.close()
        print "50% Complete!"
        csv_file = '/' + str(subdirlist) + '/' + str(dir) + '.csv'
        in_txt = csv.reader(open(filename, "rb"), delimiter = '\t')
        out_csv = csv.writer(open(csv_file, 'wb'))
        out_csv.writerows(in_txt)
        os.remove(filename)
        os.remove('/' + str(subdirlist) + '/' + dir)
        counter+=1
        print str(counter) + "/" + str(total) + " " + str(dir) + " Complete!"
    print "SubDirectory Converted!"
    print str(subcounter) + "/" + str(subtotal) + " " + str(subdirlist) + " Complete!"
    subcounter+=1
print "All Files Converted!"

Thanks in advance

Answer 1

To get lists of files and subdirectories, you can use os.walk. Below is an implementation I wrote to get all files (optionally, of certain type(s)) in arbitrarily nested subdirectories:

from os import walk, sep
from functools import reduce # in Python 3.x only

def get_filelist(root, extensions=None):
    """Return a list of files (path and name) within a supplied root directory.

    To filter by extension(s), provide a list of strings, e.g.

        get_filelist(root, ["zip", "csv"])

    """
    return reduce(lambda x, y: x+y,
                  [[sep.join([item[0], name]) for name in item[2]
                    if (extensions is None or
                        name.split(".")[-1] in extensions)]
                   for item in walk(root)])