looking for speed-up ideas

[Ram Bhamidipaty]

Ok. Here are two scripts:

script1.py:
---------------------------------------------------------------------
#!/remote/TMhome/ramb/tools/bin/python

import sys, xreadlines, pdb, os.path, imp
import profile

m = imp.find_module("heapqc", ["/remote/TMhome/ramb/src/Downloaded/py_heap"])
imp.load_module("heapqc", m[0], m[1], m[2])
import heapqc

class FileSize:
    def __init__(self,name,size):
        self.name = name
        self.size = size
    def __le__(self,other):
        if self.size <= other.size:
            return 1
        else:
            return 0

def read_diskinfo_file(f,fout):
    global dirs

    big_files = []

    num_in_list = 0
    sys.stdout.flush()
    for l in xreadlines.xreadlines(f):
        if l[0] == "F":
            tup = l.split("/")
            try:
                fsize = int(tup[1])
            except ValueError:
                fsize = long(tup[1])
            
            if num_in_list < 200:
                big_files.append(FileSize((this_dir_num, tup[2]), fsize))
                num_in_list += 1
                if num_in_list == 200:
                    heapqc.heapify(big_files)
            else:
                if fsize < big_files[0].size:
                    continue
                heapqc.heapreplace(big_files, \
                                   FileSize((this_dir_num, tup[2]), fsize))

        elif l[0] == "S":
            tup = l.split("/")

            name         = tup[1]
            par_num      = int(tup[2])
            this_dir_num = int(tup[3])

            flist = []
            dirs[this_dir_num] = (name, [], flist, this_dir_num, par_num, 0L)

        elif l[0] == "T":
            tup = l.split()
            flist = []
            dirs[0] = (tup[1], [], flist, 0, None, 0L)

                
    final_order = []
    heapqc.heapify(big_files)
    while big_files:
        o = heapqc.heappop(big_files)
        final_order.append(o)

    final_order.reverse()
    for o in final_order:
        name = get_dir_name(o.name[0]) + o.name[1]
        print >>fout, o.size, name,

    return

def get_dir_name(num):
    global dirs

    cur_dir_num = num
    cur_name = ""
    while cur_dir_num != 0:
        dir_tup = dirs[cur_dir_num]
        cur_name = os.path.join(dir_tup[0], cur_name)
        cur_dir_num = dir_tup[4]
    cur_name = os.path.join(dirs[0][0], cur_name)
    return cur_name


dirs = {}
in_fname = sys.argv[1]
f_in = file(in_fname, "r")
out_fname = sys.argv[2]
f_out = file(out_fname, "w")
profile.run("read_diskinfo_file(f_in, f_out)")
f_in.close()
f_out.close()
---------------------------------------------------------------------

script2.py:
---------------------------------------------------------------------
#!/remote/espring/ramb/tools/bin/python

import sys, profile

def process(infile,outfile):
     dirid_info = {}

     line = infile.readline()
     assert line[:1] == "T"
     ignore, dirname, dirid = line.split()
     dirid_info[dirid] = (None, dirname)

     fileinfo = []

     for line in infile:
         if line[:1] == "F":
             ignore, size, name = line.split("/")
             # negate size so 'largest' is sorted first
             fileinfo.append( (-long(size), dirid, name) )
             if len(fileinfo) > 10000:
                 # Could use a heapq....
                 fileinfo.sort()
                 fileinfo = fileinfo[:200]
         else:
             ignore, dirname, parent_id, dirid = line[:-1].split("/")
             dirid_info[dirid] = (parent_id, dirname)

     fileinfo.sort()
     fileinfo = fileinfo[:200]

     for size, dirid, name in fileinfo:
         size = -size
         components = [name[:-1]]  # need to chop newline
         while dirid != None:
             dirid, dirname = dirid_info[dirid]
             components.append(dirname)
         components.reverse()
         print >>outfile, size, "/".join(components)

f = file(sys.argv[1], "r")
f2 = file(sys.argv[2], "w")
profile.run("process(f, f2)")
---------------------------------------------------------------------

Below are the timings from the two scripts.

espring> ~/script1.py /tmp/foo /tmp/foo_1 ; ~/script2.py /tmp/foo /tmp/foo_2
         18760 function calls in 17.830 CPU seconds

   Ordered by: standard name

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
        1    0.000    0.000   17.820   17.820 <string>:1(?)
     1488    0.110    0.000    0.110    0.000 posixpath.py:44(join)
        0    0.000             0.000          profile:0(profiler)
        1    0.010    0.010   17.830   17.830 profile:0(read_diskinfo_file(f_in, f_out))
     1752    0.080    0.000    0.080    0.000 script1.py:11(__init__)
    15317    0.580    0.000    0.580    0.000 script1.py:14(__le__)
        1   16.950   16.950   17.820   17.820 script1.py:20(read_diskinfo_file)
      200    0.100    0.000    0.210    0.001 script1.py:75(get_dir_name)


         3 function calls in 32.760 CPU seconds

   Ordered by: standard name

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
        1    0.220    0.220   32.690   32.690 <string>:1(?)
        1    0.070    0.070   32.760   32.760 profile:0(process(f, f2))
        0    0.000             0.000          profile:0(profiler)
        1   32.470   32.470   32.470   32.470 script2.py:5(process)

Both scripts generated the same output files.

espring> ls -l /tmp/foo_1 /tmp/foo_2
-rw-r-----   1 ramb     src        16253 Feb  4 10:55 /tmp/foo_1
-rw-r-----   1 ramb     src        16253 Feb  4 10:55 /tmp/foo_2


-Ram