old_contents: #!/usr/bin/python
  
  '''
  Program to process a whole directory full of compressed source code archives
  to create a knowledgebase. Needs a file LIST in the directory it is passed as
  a parameter.
  
  package version filename
  
  seperated by whitespace.
  
  Compression is determined using magic
  '''
  
  import sys, os, magic
  import tempfile, bz2, tarfile, gzip
  from optparse import OptionParser
  import extractprogramstrings
  
  tarmagic = ['POSIX tar archive (GNU)'
             , 'tar archive'
             ]
  
  def unpack(dir, filename):
          ms = magic.open(magic.MAGIC_NONE)
          ms.load()
          filemagic = ms.file("%s/%s" % (dir, filename))
  
          ## just assume if it is bz2 or gzip that we are looking at tar files with compression
  
          if 'bzip2 compressed data' in filemagic:
  	        tar = tarfile.open("%s/%s" % (dir, filename), 'r:bz2')
         		tmpdir = tempfile.mkdtemp()
         		tar.extractall(path=tmpdir)
          	tar.close()
  		return tmpdir
          elif 'gzip compressed data' in filemagic:
  	        tar = tarfile.open("%s/%s" % (dir, filename), 'r:gz')
         		tmpdir = tempfile.mkdtemp()
         		tar.extractall(path=tmpdir)
          	tar.close()
  		return tmpdir
  
  def main(argv):
          parser = OptionParser()
          parser.add_option("-d", "--database", action="store", dest="db", help="path to Lucene database)", metavar="DIR")
          parser.add_option("-f", "--files", action="store", dest="filedir", help="path to directory containing files to unpack)", metavar="DIR")
          parser.add_option("-z", "--cleanup", action="store_true", dest="cleanup", help="cleanup after unpacking? (default: true)")
          (options, args) = parser.parse_args()
  
  
          ## TODO: do all kinds of checks here
          filelist = open(options.filedir + "/LIST").readlines()
          for unpackfile in filelist:
                  (package, version, filename) = unpackfile.strip().split()
                  temporarydir = unpack(options.filedir, filename)
  		extractprogramstrings.main(["-i", "/tmp/lucene", "-d", temporarydir, "-p", package, "-v", version])
  
  if __name__ == "__main__":
          main(sys.argv)
  
new_contents: #!/usr/bin/python
  
  '''
  Program to process a whole directory full of compressed source code archives
  to create a knowledgebase. Needs a file LIST in the directory it is passed as
  a parameter.
  
  package version filename
  
  seperated by whitespace.
  
  Compression is determined using magic
  '''
  
  import sys, os, magic
  import tempfile, bz2, tarfile, gzip
  from optparse import OptionParser
  import extractprogramstrings
  
  tarmagic = ['POSIX tar archive (GNU)'
             , 'tar archive'
             ]
  
  def unpack(dir, filename):
          ms = magic.open(magic.MAGIC_NONE)
          ms.load()
          filemagic = ms.file("%s/%s" % (dir, filename))
  
          ## just assume if it is bz2 or gzip that we are looking at tar files with compression
  
          if 'bzip2 compressed data' in filemagic:
  	        tar = tarfile.open("%s/%s" % (dir, filename), 'r:bz2')
         		tmpdir = tempfile.mkdtemp()
         		tar.extractall(path=tmpdir)
          	tar.close()
  		return tmpdir
          elif 'gzip compressed data' in filemagic:
  	        tar = tarfile.open("%s/%s" % (dir, filename), 'r:gz')
         		tmpdir = tempfile.mkdtemp()
         		tar.extractall(path=tmpdir)
          	tar.close()
  		return tmpdir
  
  def main(argv):
          parser = OptionParser()
          parser.add_option("-d", "--database", action="store", dest="db", help="path to Lucene database)", metavar="DIR")
          parser.add_option("-f", "--files", action="store", dest="filedir", help="path to directory containing files to unpack)", metavar="DIR")
          parser.add_option("-v", "--verify", action="store_true", dest="verify", help="verify files, don't process (default: false)")
  	# implement later
          #parser.add_option("-z", "--cleanup", action="store_true", dest="cleanup", help="cleanup after unpacking? (default: true)")
          (options, args) = parser.parse_args()
  
  
          ## TODO: do all kinds of checks here
          filelist = open(options.filedir + "/LIST").readlines()
          for unpackfile in filelist:
                  (package, version, filename) = unpackfile.strip().split()
  		if options.verify:
  			try:
  				os.stat("%s/%s" % (options.filedir, filename))
  			except:
  				print >>sys.stderr, "Can't find %s" % filename
  		else:
                  	temporarydir = unpack(options.filedir, filename)
  			extractprogramstrings.main(["-i", "/tmp/lucene", "-d", temporarydir, "-p", package, "-v", version])
  
  if __name__ == "__main__":
          main(sys.argv)
  
subject: add verify option to make sure the files specified in the batch list actually exist