import pandas as pd
import numpy as np
from time import time
from collections import OrderedDict


def fill_df(main_df):

  for i,val in enumerate(main_df['Id']):

    start = time()

    try:

      f = open('/Users/Arvind/Downloads/bytes/' + val + '.bytes', 'r')
      data = f.readlines()
      f.close()

      for line in data:
        for word in line.split()[1:]:
          if word == "00" or word == "FF" or word == "??":
            try:
              main_df.at[i, word] += 1
            except:
              continue

      print("finished: " + val + " in " + str(time() - start) + " seconds. \t" + str(i + 1) + " done so far.")

    except FileNotFoundError:

      print("The file " + val + " is not there")

  return main_df


def generate_df():

  df = pd.read_csv('crunchedTrainLabels.csv')

  df[str(format(0, '02X'))] = [0]*442
  df[str(format(255, '02X'))] = [0]*442
  df['??'] = [0]*442

  return df

if __name__ == '__main__':

  start = time()

  main_df = generate_df()
  main_df = fill_df(main_df)
  main_df.to_csv("byte_frequency.csv", index=False)

  print("This took " + str(time() - start) + " seconds.")
