#!/usr/bin/python
# Program to extract and Analyze Hexadecimal file type. The system classified for executable windows files. 
import sys
reload(sys)
sys.setdefaultencoding('utf8')

import os
import shutil
import requests
import hashlib
import urllib2
import csv
from datetime import datetime
import cPickle as Pickle
from subprocess import Popen, PIPE
from argparse import ArgumentParser
from bs4 import BeautifulSoup
from pyunpack import Archive
from os.path import basename

address = '<do not forget>'                 # Path of the list of Hexafiles kind
source = '<do not forget>'                  # Path of the directory for zip-files that will be analise
file_win = '<do not forget>'                # Path of the directory for files that my hexadecimal classifier identified like windows
file_others = '<do not forget>'             # Path of the directory for unknown files in our hexadecimal classifier
tempo_file = '<do not forget>'              # Path of the directory when the zips files will be extracted
error = '<do not forget>'                   # Path of the directory Error for others files that can not be identified
analisadozip = '<do not forget>'            # Path of the directory for zip-files were analyzed 

# the {} will be used to dynamically enter different ints with .format()
URL = "http://www.filesignatures.net/index.php?page=all&currentpage={}"
#PATH = os.path.expanduser('~/file_sigs.pickle')
PATH = os.path.expanduser(address+'file_sigs.pickle')
signatures = [] # contains all (signatures, descriptions)

def compile_sigs():
    ''' Compile the list of file signatures '''
    global signatures, PATH

    if not os.path.exists(PATH):
        for i in range(19): # 19 pages of signatures on the site
            response = urllib2.urlopen(URL.format(i))
            html = response.read() # get the html as a string

            soup = BeautifulSoup(html, "lxml") # parse the source

            t_cells = soup.find_all("td", {"width": 236}) # find td elements with width=236
            for td in t_cells:
                # append (signature, description) to signatures
                sig = str(td.get_text()).replace(' ', '').lower() # strip spaces, lowercase
                desc = str(td.find_next_sibling("td").get_text())
                signatures.append([sig, desc])
            # print("print singatures"+signatures)
        # pickle them sigs
        with open(PATH, 'wb') as f:
            Pickle.dump(signatures, f)
    else:
        with open(PATH, 'rb') as f:
            signatures = Pickle.load(f)
            #print(signatures)

def check_sig(fn):
    ''' Hex dump the file and search for signatures '''
    p = Popen(['xxd', '-p', fn], stdout=PIPE) # get plain(-p) hex dump of the file
    dump = p.communicate()[0]           # execute and extract stdout
    res = []
    for sig, desc in signatures:
        if sig in dump:
            res.append([sig, desc, dump.find(sig)])
    return res # [(sig, desc, offset), (sig, desc, offset), ... etc.]

# this function will convert bytes to MB.... GB... etc
def convert_bytes(num):
    for x in ['bytes', 'KB']:
        num /= 1024.0
    #return "%3.1f %s" % (num, x)
    return num

# this function will return the file size
def file_size(file_path):
    if os.path.isfile(file_path):
        file_info = os.stat(file_path)
        return convert_bytes(file_info.st_size)

def lists_files(root):
    the_roots = []
    for path, subdirs, files in os.walk(root):
        for name in files:
            the_roots.append(os.path.join(path, name))
    return the_roots
        	#return os.path.join(name)

# SHA256 Hash Funtion:
def sha256_checksum(filename, block_size=65536):
	sha256 = hashlib.sha256()
	with open(filename,'rb') as f:
		for block in iter(lambda: f.read(block_size), b''): 
			sha256.update(block)
	return sha256.hexdigest()

def FileCheck(fn):
	try:
		open(fn, "r")
		return 1
	except IOError:
		print "Error: File does not appear to exist."
		return 0

# The programa starts here
if __name__ == "__main__":
    print("Begining the signature classification files.\n")
    print("[*] Checking File for Known Signatures")
    print("[*] This may take a moment...")
    compile_sigs()
    print("[*] Files classification starts.\n")
    all_files = os.listdir(source)
    n_files = len([name for name in os.listdir(source) if os.path.isfile(os.path.join(source, name))])
    j = 0
    # Windows Executables signatures (.exe, .dll, etc...)
    win_signatures = ['25504446','908100000060500','006e1ef0','0f00e803','4c00000001140200','4d5a','504b030414000600','5f27a889','a0461df0','d0cf11e0a1b11ae1','d0cf11e0a1b11ae1','d0cf11e0a1b11ae1','d0cf11e0a1b11ae1','dba52d00','eca5c100','fdffffff0e000000','fdffffff10','fdffffff1c000000','fdffffff1f','fdffffff22','fdffffff23','fdffffff28','fdffffff29','fdffffff43000000','fffe']

    # Send one by one file to classify
    for file in all_files:
        print file
        j += 1
        print j,'/',n_files
        result = FileCheck(source+file)
        if result == 1:
	        size = file_size(source+file)
	        base_name = os.path.splitext(os.path.basename(source+file))[0]
	        address_tempofiles = tempo_file+base_name+'/'
	        # Create a new directory
	        if not os.path.exists(address_tempofiles):
	            os.makedirs(address_tempofiles)
	        # Extract zip file in a temporal dir
	        Archive(source+file).extractall(address_tempofiles)
	        # Obtain all subfiles in a tempofiles
	        subfiles = lists_files(address_tempofiles)
	        print subfiles,'\n'

	        for subfile in subfiles:
	            print subfile
	            result = FileCheck(subfile)
	            if result == 1:
		            results = check_sig(subfile)
		            subfile_name = os.path.basename(subfile)
		            if results:
		                results.sort(key=lambda x: x[2]) # sort results by offset in file
			            # find longest signature, and desc for output formatting purposes
		                big_sig = len(max([i[0] for i in results], key=lambda x: len(x)))
		                big_desc = len(max([i[1] for i in results], key=lambda x: len(x)))
		                sig = results[0][0]
		                desc = results[0][1]
		                print "[*] File Signature(s) detected:",subfile_name,sig,desc
		                if sig in win_signatures:

		                    checksum = sha256_checksum(subfile)
		                    print 'os.rename(',subfile,',',file_win+subfile_name,')\n'
		                    os.rename(subfile,file_win+checksum)
		            else:
		                print("[!] No File Signature Detected.\n")
		                subfile_name = os.path.basename(subfile)
		                os.rename(subfile,error+subfile_name)
	        print 'os.rename(',source+file,',',analisadozip+file,')\n'
	        os.rename(source+file,analisadozip+file)
	        shutil.rmtree(address_tempofiles)
	        print 'The file ',file,' was moved...'

    print("done!")
    print("fin")
