import os, sys, argparse
import subprocess as sp
import json

###########################
#
# This requires installing gsutil, see the instructions here:
#  + https://cloud.google.com/storage/docs/gsutil_install
#
# Then you can either change the default for the gsUtilHome directory below, or
# pass it in in the command line
#
###########################

if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  parser.add_argument("--outputDir", required=True, type=str)
  parser.add_argument("--gsLocation", required=True, type=str)
  parser.add_argument("--start", required=True, type=str)
  parser.add_argument("--end",   default="99999999", type=str)
  parser.add_argument("--gsUtilHome", default="/Users/ThatcherClay/google-cloud-sdk", type=str)

  args = parser.parse_args()

  if not os.path.exists(args.outputDir):
    os.makedirs(args.outputDir)

  print("Running a ls of %s"%args.gsLocation) 
  proc = sp.Popen("%s/bin/gsutil ls %s"%(args.gsUtilHome, args.gsLocation), shell=True, stdout=sp.PIPE)
  data = proc.stdout.read()
  available_data = [x for x in data.decode("utf-8").split("\n") if x != '']
  
  files_to_download = []
  for f in available_data:
    filename = f.split('/')[-1]
    date = filename.split('.')[0]
    if date >= args.start and date <= args.end:
      files_to_download.append(f)
  print("%s files to download, starting now..."%(len(files_to_download)))

  for f in files_to_download:
    sp.run("%s/bin/gsutil cp %s %s/"%(args.gsUtilHome, f, args.outputDir), shell=True, check=True)
  
  print("Finished!")
