#!/usr/bin/python
# Upgrades:
# 1) Would be nice to be able to restart a backup.  Is this possible?
# 2) Why on earth does it take so long, even for stuff that's already there?
#    read up on rsync
# 3) Better handling of stderr.  Spawn two threads to read it
#    separately?  Or maybe I don't really need popen2 at all, if I
#    want output to just inherit stdout and stderr

#
# Accounts not on Crashplan
#
# time backup usb_key 
# time backup rubicon
# time backup armstrong
# time backup tigressdata
# time backup hecate
# time backup lerma90
# time backup nfsn_greymonster
# time backup nfsn_zugasaurus
# time backup nfsn_greg
# time backup aaa 


# Jen's Accts
#
# time backup clio_jen 
# time backup thalia_jen 
# time backup france_jen_net 

# All accts - long
# time backup thalia 
# time backup clio 
# time backup aaa 
# time backup aramis 

# All accts - short
#
# time backup usb_key 
# time backup france_greg_net 
# time backup clio_shared 
# time backup rubicon
# time backup armstrong
# time backup tigressdata
# time backup hecate
# time backup lerma90

# Don't need to backup so often...
# time backup nfsn_greymonster
# time backup nfsn_zugasaurus
# time backup nfsn_greg

# Evaluating who's taking up space on disk
# Get a list of 'tags'
# ls | sed s/\\..*//g | sort | uniq
# Get one key in reverse sorted order
# ls -d -1 usb_key* | sort -t. -k 2 -n -r
# get ordered summary for one tag
# ls -d -1 usb_key* | sort -t. -k 2 -n -r | xargs du -h -s -c
# Do something on sets of reverse-ordered tags
# for tag in `ls | sed 's/\\..*//g' | sort | uniq` ; do ls -d -1 $tag* | sort -t. -k 2 -n -r | xargs echo ; done
# Get du info on sets of reverse ordered tags
# for tag in `ls | sed 's/\\..*//g' | sort | uniq` ; do ls -d -1 $tag* | sort -t. -k 2 -n -r | xargs echo ; done
###### Big money command line below
# Get du info on sets of reverse ordered tags, saving output.
# for tag in `ls | sed 's/\\..*//g' | sort | uniq` ; do ls -d -1 $tag* | sort -t. -k 2 -n -r | xargs du -h -s -c ; done > ~/backup-disk.txt

import re

base=4

default_destination = r'/Volumes/My Passport/'

rsync_command = "rsync"
# the options are equivalent to --archive, except without the -g to
# preserve groups.  The requirement to preserve groups causes problems
# on some remote machines when rsync is not run as root, in that it
# causes files to be spuriously transferred instead of hard linked.
rsync_options="-rlptoD --exclude-from=-"

config = dict(
    ######################
    ## Home Computers
    
    clio = dict(source='clio.local:/Users/novak/',
                destination=default_destination,
                exclude="""
/.Trash
/Desktop/Orig-ucsc-files/euterpe/novak/Pictures
/Desktop/Orig-ucsc-files/dionysus/Projects/Thesis/rudd
/Desktop/Orig-ucsc-files/dionysus/Projects/Thesis/nv-data
"""),

    clio_jen = dict(source='jen@clio.local:/Users/jen/',
                    destination=default_destination,
                    exclude="""
/.Trash
"""),

    clio_shared = dict(source='clio.local:/Users/Shared/',
                  destination=default_destination,
                  exclude="""
/.Trash
"""),

# eMusic: Verify that these are duplicated.              
#    thalia = dict(source='thalia.local:/Users/novak/',
    thalia = dict(source='thalia:/Users/novak/',
                  destination=default_destination,
                  exclude="""
/.Trash
"""),

#    thalia_jen = dict(source='jen@thalia.local:/Users/jen/',
    thalia_jen = dict(source='jen@thalia:/Users/jen/',
                  destination=default_destination,
                  exclude="""
/.Trash
"""),

    aaa = dict(source=r'clio.local:/Volumes/WD-300G/.aaa/',
                  destination=default_destination,
                  exclude="""
"""),

    france_jen_net = dict(source='jen@zuggernaut.local:/Users/jen/',
                          destination=default_destination,
                          exclude=
"""/.Trash
"""),

    france_greg_net = dict(source='zuggernaut.local:/Users/novak/',
                           destination=default_destination,
                           exclude=
"""/.Trash
"""),

    ######################
    ## External disks

    usb_key = dict(source='/Volumes/USB-Disk/',
                   destination=default_destination,
                   exclude=
"""
/.Trashes
"""),

    wd_300g_disk = dict(source=r'/Volumes/WD\ 300G/',
                        destination=default_destination,
                        exclude= """ """),
    
    wd_80g_disk = dict(source=r'/Volumes/Extra\ Storage\ Space/',
                       destination=default_destination,
                       exclude= """ """),

    ######################
    ## Web space

    nfsn_jen = dict(source='jen_web:/home/',
                    destination=default_destination,
                    exclude=""" """),

    nfsn_greymonster = dict(source='greymonster:/home/',
                            destination=default_destination,
                            exclude=""" """),

    nfsn_zugasaurus = dict(source='zugasaurus:/home/',
                           destination=default_destination,
                           exclude=""" """),

    nfsn_greg = dict(source='gregnovak:/home/',
                     destination=default_destination,
                     exclude=""" """),

    web = dict(source='jenandgreg@jenandgreg.org:/home/jenandgreg/',
               destination=default_destination,
               exclude=
"""
/.cpan
/.wusage
"""),

    web_www = dict(source='jenandgreg@jenandgreg.org:/home/jenandgreg/www/',
                   destination=default_destination,
                   rsync_options = '--copy-links',
                   exclude=""),

    web_sql = dict(source='jenandgreg@jenandgreg.org:/var/lib/mysql/weddingwikidb/',
                   destination=default_destination,
                   exclude=""),
    
    ######################
    ## Paris computers

    rubicon= dict(source='rubicon:',
                  destination=default_destination,
                  exclude= """
/bin/local
/bin/build
/.Trash
"""),


    aramis= dict(source='aramis:',
                  destination=default_destination,
                  exclude= """
/bin/local
/bin/build
/.Trash
"""),


    lerma90= dict(source='lerma90:',
                  destination=default_destination,
                  exclude= """
/bin/local
/bin/build
/.Trash
"""),


    ######################
    ## Princeton computers

    armstrong= dict(source='armstrong:',
                    destination=default_destination,
                    exclude= """
/bin/local
/bin/build
"""),
              
    sesame= dict(source='sesame1:',
                 destination=default_destination,
                 exclude= """
/bin/local
/bin/build
"""),

    hecate= dict(source='hecate:',
                 destination=default_destination,
                 exclude= """
/bin/local
/bin/build
"""),

    tigressdata= dict(source='tigressdata:',
                      destination=default_destination,
                      exclude= """
/bin/local
/bin/build
"""),

    columbia = dict(source='cfe2:',
                    destination="/Users/novak/Desktop/columbia-backup/",
                    #destination=default_destination,
                      exclude= """
/bin/local
/bin/build
"""),


    ######################
    ## UCSC Computers
    
    dionysus= dict(source='dionysus:/home/',
                   destination=default_destination,
                   exclude=
"""/novak/.Trash
/novak/bin/build
/novak/bin/build-rpm
/novak/bin/build-fink
/novak/bin/build-ubuntu
/novak/waiting-to-delete
/novak/.python23_compiled
/novak/.python24_compiled
/novak/.python25_compiled
/novak/Projects/Thesis/big-endian-data
/novak/Projects/Thesis/little-endian-data
/novak/Projects/Thesis/data
/novak/Projects/Thesis/Plots-old
/novak/Projects/Thesis/old-stuff/Plots-07-29-05
/novak/Projects/Thesis/Avishai-data
/novak/Projects/Thesis/rudd
/david/SPECTRUM
/novak/Desktop/ubuntu
"""),

    dionysus_etc= dict(source='root@dionysus:/etc/',
                       destination=default_destination,
                       exclude=""),
    )

import sys
import threading
import os
             
def main():
    system = sys.argv[1]
    
    backup(system)
    # GSN Dec 2011: disable deletions -- do this manually when I run out of space.
    #prune_backup_series(system)
    
def backup(system):
    starting_dir = os.getcwd()    
    os.chdir(config[system]['destination'])
    
    last_fn = get_last_filename(system)
    next_fn = get_next_filename(system)
    source = config[system]['source']
    # 'echo ' + ' ' + \

    options = rsync_options

    if config[system].has_key('rsync_options'):
        options = options  + ' ' + config[system]['rsync_options']
    
    command = rsync_command + ' ' + \
              options + ' ' + \
              '--link-dest=../' + last_fn + ' ' + \
              source + ' ' + \
              next_fn

    inf, outf = os.popen2(command)

    inf.write(config[system]['exclude'])
    inf.close()
    
    for line in outf:
        print line, 
    outf.close()
    
    os.chdir(starting_dir)

def largest_exponent(n):
    """Return largest i such that base**i is less than n"""
    assert n >= 0
    
    # handle zero as special case
    if n==0: return 0

    i=0
    while base**i <= n:
        i += 1
    return i-1

def keep_numbers(n):
    "Return a list of backup numbers to keep, given current number n"

    # handle 0 as a special case
    result = [ [i for i in range(n-base+1, n+1) if i >= 0] ]

    for i in range(1, largest_exponent(n)+1):
        last_multiple = n - (n % base**i)

        result.append([i for i in range(last_multiple-base**(i+1),
                                        last_multiple, base**i)
                       if i >= 0])

    return flatten(result)

def keep_filenames(base, n):
    """return a list of filenames that should not be deleted"""
    ns = keep_numbers(n)
    return [base + "." + str(n) for n in ns]

def prune_backup_series(base):
    """Prune the current backup series"""
    n = get_last_number(base)
    keep_fns = keep_filenames(base, n)

    fns = backup_filenames(base)
    for fn in fns:
        if not fn in keep_fns:            
            print "Deleting ", fn
            os.system('rm -rf ' + fn)
            # os.removedirs(fn)
            
def backup_filenames(base):
    """Return a list of all filenames in this backup series"""
    return [f for f in os.listdir('.')
            if re.match('^%s\.[0-9]+$' % base, f)] 

def get_next_filename(base):
    """get the next filename this backup series."""
    num = get_next_number(base)
    return base + '.' + str(num)

def get_last_filename(base):
    """get the next filename this backup series."""
    num = get_last_number(base)
    return base + '.' + str(num)

def get_last_number(base):
    """get the number of the last backup in this series."""
    fns = backup_filenames(base)
    nums = [int(fn.replace(base+'.', '')) for fn in fns]

    if len(nums) is 0: return 0
    else: return max(nums)

def get_next_number(base):
    """get the number of the next backup in this series."""
    return get_last_number(base) + 1

def flatten(L):
    """Make nested lists into a single list of atoms"""
    if type(L) != type([]): return [L]
    if L == []: return L
    return reduce(lambda x,y: x+y, map(flatten, L))

    
main()
