import pandas as pd
reader = pd.read_csv('/home/malware/tc/train.csv',iterator=True)

loop = True
chunkSize = 100000
leftChunk = None


def saveData(df, pre_index, index, pre_file_id):
    save_data = df.iloc[pre_index:index, :]
    file_name = '/home/malware/tc/test/file_%s.txt' % pre_file_id
    save_data.to_csv(file_name,index=0)

def splitChunk(df):
    pre_file_id = -1
    pre_index = 0
    for index,data in df.iterrows():
        if data['file_id'] != pre_file_id and pre_file_id>-1:
            saveData(df,pre_index,index,pre_file_id)
            pre_index = index
        pre_file_id = data['file_id']
    return df.iloc[pre_index:, :]


# while loop:
#     try:
#         chunk = reader.get_chunk(chunkSize)
#         if leftChunk is not None:
#             chunk = pd.concat([leftChunk,chunk], ignore_index=True)
#         leftChunk = splitChunk(chunk)
#     except StopIteration:
#         loop = False
#         print("Iteration is stopped.")
# file_name = '/home/malware/tc/train/file_%s.txt' % leftChunk['file_id'][0]
#


if __name__=='__main__':
    df = pd.read_csv('/home/malware/tc/train.csv')
    leftChunk = df[df['file_id'] == 116623]
    leftChunk.to_csv('/home/malware/tc/train/file_116623.txt', index=0)


