#!/usr/bin/env python3

# 
# Copyright (C) 2014i, Lorenzo Mureu <mureulor@gmail.com>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, 
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, 
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this 
# list of conditions and the following disclaimer in the documentation and/or other 
# materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors 
# may be used to endorse or promote products derived from this software without 
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, 
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

## This files search for duplicate files (i.e. files with different names
## but same content) and does the following operations:
## 1. removes all the duplicate files (leaving just one copy - i.e. all files will be different - i.e. no more duplicates)
## 2. renames each file with its md5sum - e.g. if the md5sum of file foo.bar is d41d8cd98f00b204e9800998ecf8427e
##                                            (empty file) then it will be renamed as d41d8cd98f00b204e9800998ecf8427e.bar

import hashlib, os, posix, re, sys

def getpath(parent, son):
    return os.path.join(parent,son)

def remdups(path, recursive=True):
    dic={}
    print("Scanning {0}...".format(path))
    
    for file in posix.listdir(path):
        fpath=getpath(path, file) 
        if os.path.isdir(fpath) and recursive: remdups(fpath, True)
        if os.path.isfile(fpath):
            dic[file]=md5sum(fpath)
    
    while len(dic) > 0: 
        file, digest = dic.popitem()
        fpath=getpath(path, file) 
        dups=[]
        for key in dic:
            if dic[key] == digest:
                print ( "*\tDuplicate found: {0} of {1}".format(key,file) )
                dups.append( key )

        for key in dups:
            del dic[key]
            p=getpath(path,key)
            print ( "D\tRemoving {0}".format(p))
            posix.remove(p)

        n=digest+getext(file)
        newname=getpath(path,n)
        
        if fpath != newname:
            print( "N\tRenaming {0} to {1}".format(file,n) )
            posix.rename(fpath, newname)

def getext(filename):
    return re.match('.+(\.\w+$)',filename).group(1)

def md5sum(file):
    f=open(file,"rb")
    h=hashlib.md5(f.raw.readall()).hexdigest()
    f.close()
    return h

def usage():
    print( "Usage: {0} path [...]".format(sys.argv[0] or "remdup" ))


if __name__ == '__main__':
    if len(sys.argv) > 1:
        for arg in sys.argv[1:]:
            remdups(os.path.abspath(arg))
    else:
        usage()
