import os
import sys
import string
import difflib
import hashlib
NAME_ACCURACY = 0.75
def get_disk_drives_list():
    disk_list = ['%s:' % letter for letter in string.ascii_lowercase if os.path.exists('%s:' % letter)]
    return disk_list
def get_full_filenames(path):
    for root, dirnames, filenames in os.walk(path):
        for filename in filenames:
            yield os.path.join(root, filename)
def get_md5(filename):
    """ Open, close, read file and calculate MD5 on its contents
    """
    with open(filename) as file_to_check:
        data = file_to_check.read()    
        md5 = hashlib.md5(data).hexdigest()
    return md5
def compare_files(filename1, filename2):
    """ Compare files by md5 sum, if don't match - compare by names
    """
    if filename1 != filename2:
        if get_md5(filename1) == get_md5(filename2):
	    with open('md5_duplicates.txt', 'a') as md5_duplicates:
	        md5_duplicates.write('%s\n%s\n---\n' % (filename1, filename2))
	elif difflib.SequenceMatcher(None, os.path.basename(filename1), os.path.basename(filename2)).ratio() >= NAME_ACCURACY:
	    with open('name_duplicates.txt', 'a') as name_duplicates:
	        name_duplicates.write('%s\n%s\n---\n' % (filename1, filename2))
def find_duplicates(path):
    """ Search for duplicates in set path
        if path == '/', it means main root - search on all disk drives
    """
    if path == '/':
        for disk_drive1 in get_disk_drives_list():
	    for filename1 in get_full_filenames(disk_drive1):
                for disk_drive2 in get_disk_drives_list():
	            for filename2 in get_full_filenames(disk_drive2):
	                compare_files(filename1, filename2)
    elif os.path.exists(path):
        for filename1 in get_full_filenames(path):
	    for filename2 in get_full_filenames(path):
	        compare_files(filename1, filename2)
    else:
        '%s is wrong path' % path
if __name__ == '__main__':
    """ Example usage:
        duplicates.py /
	duplicates.py c:/temp/
    """
    path = sys.argv[1]
    print 'Searching for duplicates in %s' % path
    print 'Please, wait...'
    find_duplicates(path)
    print '--- Job done ---'

>>> import hashlib
>>> m = hashlib.md5()
>>> m.update("Nobody inspects")
>>> m.update(" the spammish repetition")
>>> m.digest()
'\xbbd\x9c\x83\xdd\x1e\xa5\xc9\xd9\xde\xc9\xa1\x8d\xf0\xff\xe9'
>>> m.digest_size
16
>>> m.block_size
64

difflib.SequenceMatcher(None, os.path.basename(filename1), os.path.basename(filename2)).ratio() >= NAME_ACCURACY