#this file contains functions for generating annotations from spectrogram clips

#filesystem
import os
import glob
from tqdm import tqdm

#plotting
#import seaborn as sns 
#import matplotlib.pyplot as plt

#data
import umap
import json
import random
import numpy as np
import pandas as pd
from scipy.io import wavfile
from scipy.signal import stft

#custom modules
from src.spectrogramming import get_spectrogram

def sample(seed, num_to_sample, annotated_df):
    """
    Sample annotations from the output of annotation.get(). Note this is 
    not intended to be a versatile function, just a helper to reproduce figures.

    Arguments:
        seed (int): seed for reproducible sampling
        num_to_sample (list): number of rows to sample for each vocalization type ('cry' or 'USV')
        annotated_df (dataframe): the dataframe to sample from, eg output of annotation.get()
        
    Returns:
       downsampled_df (dataframe): a dataframe of the annotations downsampled to have num_to_sample vocs from each voc type from each species
       
    """
    
    downsampled_list = []

    for species in annotated_df['species'].unique():

        #separate the voc types
        temp_cry = annotated_df.loc[annotated_df['species'] == species].loc[annotated_df['human_label'] == 'cry']
        temp_USV = annotated_df.loc[annotated_df['species'] == species].loc[annotated_df['human_label'] == 'USV']

        #sample
        temp_cry = temp_cry.sample(n=num_to_sample, random_state = seed)
        temp_USV = temp_USV.sample(n=num_to_sample, random_state = seed)

        #reasemble
        downsampled_species = pd.concat([temp_cry, temp_USV])

        #update downsampled_list
        downsampled_list.append(downsampled_species)

    #assemble all the species
    downsampled_df = pd.concat(downsampled_list).reset_index(drop=True)
    
    return downsampled_df
    
def get(annotations_root, species_list = ['BW', 'BK', 'NB', 'SW', 'PO', 'LO', 'GO', 'LL']):
    """
    Retrieve a dataframe of annotated vocalizations with labels from the annotations root directory. Note this is 
    not intended to be a versatile function, just a helper to reproduce figures.

    Arguments:
        annotations_root (str): path to the directory containing the annotations with one subdirectory for each species
        species_list (list): a list of the species annotations to retrieve using 2-letter code, eg ['BW', 'BK', 'NB', 'SW', 'PO', 'LO', 'GO', 'LL']
        
    Returns:
       all_combined (dataframe): a dataframe of the annotations excluding those that are 'scratch' or 'none'
       
    """
    
    annotated_files = []

    for species in species_list:
        print(species)

        #get the label_0 files
        label_0_voc_path = os.path.join(annotations_root,species,'annotations','hdbscan_label_0',species+'_hdbscan_label0_all.feather')

        #load the data
        label_0_voc_df = pd.read_feather(label_0_voc_path)

        #drop the spectrogram pixels and append
        label_0_voc_df = label_0_voc_df[['source_file', 'umap1', 'umap2', 'hdbscan_label', 'human_label']]
        annotated_files.append(label_0_voc_df)

        #get the label_1 files 
        label_1_voc_path = os.path.join(annotations_root,species,'annotations','hdbscan_label_1',species+'_hdbscan_label1_all.feather')

        #load the data
        label_1_voc_df = pd.read_feather(label_1_voc_path)

        #drop the spectrogram pixels and append
        label_1_voc_df = label_1_voc_df[['source_file', 'umap1', 'umap2', 'hdbscan_label', 'human_label']]
        annotated_files.append(label_1_voc_df)

    #combine the annotations and remove the non-vocal sounds
    print('combining files...')
    all_combined = pd.concat(annotated_files)
    all_combined['species'] = [i.split('_')[0] for i in all_combined['source_file']]
    all_combined = all_combined.loc[~all_combined['human_label'].isin(['scratch', 'none'])]
    all_combined = all_combined.reset_index(drop=True)
    
    return all_combined
    
    
def get_noise_clip(pup, audio_dir, seg_csv, save_dir, margin=0, min_dur=2, max_dur=3, units = 's'):
    """
    Interactive function to choose silent intervals from a recording for which an annotation or prediction exists. Useful for
    defining noise floors for each raw recording.

    Arguments:
        pup (string): the name of the raw recording with the '.wav' extension removed
        audio_dir (string): the path to the directory containing the raw recordings for which you want to get noise
        seg_df (string): the pah to the csv file containing the segments csv for each raw recording (eg generated by segment_wavs())
        save_dir (string): the path to the directory where the noise clips should be saved (as .wav)
        margin (float): a margin in seconds to add before the start and after the end of each noise clip
        min_dur (float): the minimum duration in seconds of the desired noise clips
        max_dur (float): the maximum duration in seconds of the desired noise clips
        units ('s' or 'ms'): units in which time is measured

    Returns:
       None
    """
    #check if you already have a noise clip
    print('getting starts and stops from...')
    print('\t',pup)
    if pup+'_noiseclip'+'.wav' in os.listdir(save_dir):
        print('\tnoise clip already exists for this recording')
        return

    #get the segments
    seg_df = pd.read_csv(seg_csv)
    seg_df = seg_df.loc[seg_df['source_file'] == pup+'.wav']

    #get the silent intervals
    start_column = [i for i in seg_df.columns if 'start' in i][0]
    stop_column = [i for i in seg_df.columns if 'end' in i or 'stop' in i][0]

    noise_stops = [float(i) for i in seg_df[start_column][1:]]
    noise_starts = [float(i) for i in seg_df[stop_column][:-1]]
    print('there are', len(noise_starts), 'clips to peruse...')

    #get the audio
    print('\tgetting raw audio..')
    wav_name = ('.').join([pup,'wav'])
    fs, audio = wavfile.read(os.path.join(audio_dir,wav_name))

    print('\tgetting clips from the audio...')
    if len(seg_df) > 0: #if there are vocalizations, looks through the silent periods for good ones
        counter = 0
        for stop, start in zip(noise_stops, noise_starts):

            noise_length = stop-start


            if min_dur < noise_length < max_dur:
                counter += 1
                #clip the clip
                clip_name = pup+'_noiseclip'+'.wav'

                if units == 's':
                    start= int((start+margin)*fs)
                    stop =  int((stop-margin)*fs)
                    clip = audio[start:stop] #get the clip
                elif units == 'ms':
                    start= int((start+margin)*(fs/1000))
                    stop =  int((stop-margin)*(fs/1000))
                    clip = audio[start:stop] #get the clip

                print(start/fs, ':', stop/fs, units)
                t,f,spec = stft(clip, noverlap=256, nperseg=1024, fs=fs)
                spec = np.log(np.abs(spec))
                plt.figure(figsize=[5,5])
                plt.imshow(spec, origin='lower')
                plt.show()

                #get input 
                val = input("looks ok? (y/n/next to go to the next pup/x to clip the wav)")
                if val == 'y':
                    if clip_name not in os.listdir(save_dir):
                        print('saving clip...')
                        wavfile.write(os.path.join(save_dir,clip_name), fs, clip) #write the clip to a wav
                        return
                    else:
                        print('clip already exists...')
                        continue
                elif val == 'n':
                    continue
                elif val =='next':
                    return
                elif val == 'x':
                    start_change = input("how many seconds into this clip do you want it to start?")
                    stop_change = input("how many seconds before the end of this clip do you want it to end?")

                    if units == 's':
                        new_start = start/fs + float(start_change)
                        new_stop = stop/fs - float(stop_change)

                    elif units == 'ms':
                        new_start = start/fs + float(start_change)/1000
                        new_stop = stop/fs - float(stop_change)/1000

                    if units == 's':
                        start= int((float(new_start)+margin)*fs)
                        stop =  int((float(new_stop)-margin)*fs)
                        clip = audio[start:stop] #get the clip

                    elif units == 'ms':
                        start= int((float(new_start)/1000+margin)*(fs/1000))
                        stop =  int((float(new_stop)/1000-margin)*(fs/1000))
                        clip = audio[start:stop] #get the clip

                    t,f,spec = stft(clip, noverlap=512, nperseg=1024, fs=fs)
                    spec = np.log(np.abs(spec))
                    plt.figure(figsize=[5,5])
                    plt.imshow(spec, origin='lower')
                    plt.show()
                    if clip_name not in os.listdir(save_dir):
                        print('saving this clip...')
                        wavfile.write(os.path.join(save_dir,clip_name), fs, clip) #write the clip to a wav
                        return
                    else:
                        print('clip already exists...')
                        continue

                else:
                    print('type y (looks good) or n (see the next one)')

        if counter == 0:
            print('No clips found in the range specified by min_dur and max_dur. Are your time units correct?')

    else: #if no vocs, just pick a random 1 s clip between seconds 3 and 4
        #clip the clip
        print('no vocs, saving this clip...')
        clip_name = pup+'_noiseclip'+'.wav'
        start = 3*fs
        stop = 4*fs
        clip = audio[start:stop] #get the clip
        t,f,spec = stft(clip, noverlap=256, nperseg=1024, fs=fs)
        spec = np.log(np.abs(spec))
        print(clip_name)
        plt.figure(figsize=[5,5])
        plt.imshow(spec, origin='lower')
        plt.show()
        wavfile.write(os.path.join(save_dir,clip_name), fs, clip) #write the clip to a wav

    print('done.')

#iterate through all of the noise clips generated with get_noise() and calculate a noise floor from each (2 standard deviations above the median)
#noise_dir is the directory holding all of these clips
#if verbose, show and save plots for each noise clip
#thresh is number of standard deviations about noise spectrogram median value to set the noise_floor
def get_noise_floor(noise_dir, thresh, species, spec_params, save_dir, verbose=False, save = False):
    """
    Calculate noise thresholds from the wav files generated by get_noise_clip()

    Arguments:
        noise_dir (string): the path to the directory containing the noise clips generated by get_noise()
        thresh (int): standard deviations above the median spectrogram pixel value of the noise clip above which a signal will no longer be noise
        spec_param (dict): parameters for generating spectrograms of noise clips with these keys: 'nperseg', 'noverlap', 'fs', 'min_freq', 'max_freq'
        verbose (bool): if True, histograms of the spetrogram pixel values with the thresold drawn as a verticle line will be saved to noise_dir
        save (bool): if true, a dataframe of the noise_floors for each noise clip will be saved to noise_dir

    Returns:
       floor_df (dataframe): a dataframe of the noise_floors for each noise clip will be saved

    """

    pups = [i.split('_noise')[0] for i in os.listdir(noise_dir) if i.startswith(species)]
    all_thresholds = []
    all_pups = []
    noise_wavs = []

    for pup in pups:
        #get the noise clip
        noise_wav = os.path.join(noise_dir,pup+'_noiseclip.wav')

        #make a spectrogram (modified from Goffinet et al. Elife 2022 https://autoencoded-vocal-analysis.readthedocs.io/)

        EPSILON = 1e-9
        fs, noise_audio = wavfile.read(noise_wav)
        f,t,noise_spec = stft(noise_audio, 
                                nperseg=spec_params['nperseg'], 
                                noverlap=spec_params['noverlap'],
                                fs=spec_params['fs'])

        i1 = np.searchsorted(f, spec_params['min_freq'])
        i2 = np.searchsorted(f, spec_params['max_freq'])
        f, noise_spec = f[i1:i2], noise_spec[i1:i2]
        noise_spec = np.log(np.abs(noise_spec)+EPSILON)

        #calculate the threshold
        noise_spec_vals = noise_spec.flatten()
        median_value = np.median(noise_spec_vals)
        std_value = np.nanstd(noise_spec_vals)
        floor = median_value + thresh*std_value

        #update
        all_pups.append(pup+'.wav')
        all_thresholds.append(floor)
        noise_wavs.append(noise_wav.split('/')[-1])

        if verbose:
            #plot
            fig = plt.figure(figsize=[20,5])
            ax1= fig.add_subplot(1, 2, 1)
            ax1.imshow(noise_spec, origin='lower')
            ax2= fig.add_subplot(1, 2, 2)
            sns.histplot(noise_spec_vals,ax=ax2, binwidth=.01, color='black')
            plt.axvline(x=floor,color='red')
            plt.show()

            #save
            if not os.path.exists(os.path.join(save_dir,'00_plots')):
                os.mkdir(os.path.join(save_dir,'00_plots'))
                print('made a directory:', os.path.join(save_dir,'00_plots'))
            hist_name = pup+'_noise_hist.jpeg'
            plt.savefig(os.path.join(save_dir,'00_plots',hist_name), dpi=300)

    floor_df = pd.DataFrame()
    floor_df['source_file'] = all_pups
    floor_df['noise_source'] = noise_wavs
    floor_df['noise_floor'] = all_thresholds

    if save:
        floor_df.to_csv(os.path.join(save_dir,'all_noise_floors.csv'), index=False)
        print('saved a csv of noise floors to...', save_dir+'all_noise_floors.csv')
    return floor_df

def check_test_set(frame, test_frame):
    """
    Check whether, which, and how many vocs in a sampling come from a test set (ie, any vocalizations that will be used to evaluate models trained on the vocalizations annotated from UMAP)

    Arguments:

        frame (data frame): data frame containing the spectrogram images, umap_coordinates, source_files, and hdbscan labels for a single species

        test_frame (data frame): data frame of the hand annotated recordings that will be used to evaluate models

    Returns:
        in_test_set (list): a list of source file names that come from pups in the test set
    """
    #get the source files in the test set (and remove file extension)
    test_set = [i.split('.wav')[0] for i in test_frame['source_file']]

    #initialize list
    in_test_set = []

    #check each vocalization from downsampled set
    #TODO: deal with MZ better than this
    if set([i.split('_')[0] for i in frame['source_file']]) == {'MZ'}:
        for spec_num in range(len(frame	)):
            spec_name = frame['source_file'].iloc[spec_num]
            short_name = spec_name.split('_na_na_na_na_na_na_na_na')[0]+spec_name.split('_na_na_na_na_na_na_na_na')[1].split('_clip')[0]+'_clip'
            if short_name in test_set: 
                in_test_set.append(spec_name) 
    else:
        for spec_num in range(len(frame)):
            spec_name = frame['source_file'].iloc[spec_num]
            if spec_name.split('_clip')[0]+'_clip' in test_set: 
                in_test_set.append(spec_name)  

    return in_test_set

def sample_vocs(frame, num_to_sample, label_to_sample, random_state):
	"""
	Take an labeled umap embedding spectrogram data frame and sample from within each cluster

	Parameters
	----------
	frame (data frame): data frame containing the spectrogram images, umap_coordinates, source_files, and hdbscan labels for a single species	
	num_to_sample (int): the number of vocalizations to sample	
	label_to_sample (0 or 1): the label to sample (0 corresponds to "USV" cluster, 1 to "cry" cluster)	
	random_state (int): random seed for reproducible sampling
	
	Returns
	-------
	downsampled_df (df): a downsampled dataframe containing the spectrogram images, umap_coordinates, source_files, and hdbscan labels for annotating

	"""
	
	# get the data for the given label
	df = frame.loc[frame['label'] == label_to_sample]
	
	#downsample
	downsampled_df = df.sample(n=num_to_sample, random_state =  random_state)
	
	return downsampled_df

def from_umap(downsampled_frame, num_freq_bins, num_time_bins, non_spec_columns, sampling_params, clips_dir, in_progress_dir, df_save_dir, df_save_name, spec_params, spec_type='from_wav'):
    """
    1. Take a data frame where each row is a vocalization, and columns spectrogram pixel values and a UMAP cluster label (eg from HDBSCAN)
    2. Display each spectrogram one at a time and ask for user input to label it
    3. Check whether the pup each vocalization came from is in the annotation set
    4. Save the dataframe with a column for user label and a column for whether or not this pup is in the annotation set

    Arguments:
        downsampled_frame (data frame): data frame containing the spectrogram images, umap_coordinates, source_files, and hdbscan labels for annotation.
        random_state (int): random seed for reproducible sampling
        num_freq_bins (int): the number of frequency bins in the spectrogram to be displayed
        num_time_bins (int): the number of time bins in the spectrogram to be displayed 
        non_spec_columns (list): the column names that do not correspond to spectrogram pixels (so they can be dropped before making the spectrogram)
        sampling_params_path (str): path to dictionary with the sampling parameters including keys: species, hdbscan label, random seed, and iteration 
        clips_dir (str): path to the directory containing all of the wav clips for each species
        in_progress_dir (str): path to the directroy to keep track of whcih vocalizations of each type you have annotated for a given iteration
        df_save_dir (str): path to a directory where hand annotations will be saved as .feather
        df_save_name (str): name of the .feather file to save including file extension
        spec_params (dict): a dictionary of spectrogram parameters used to make spectrograms if spec_type is 'from_wav'
        spec_type (str): If 'from_embedding', show the spectrogram that went into the embedding. If 'from_wav' (default) make a new spectrogram with                      spec_params - this is the better option for annotation.

    Returns:
       labeled_df (df): a dataframe containing spectrogram images, umap_coordinates, source_files, hdbscan labels, and user labels for each annotated voc

    """

    #check inputs
    assert spec_type in ['from_embedding', 'from_wav']

    #reset index
    ds_df = downsampled_frame.reset_index(drop=True)

    #check if there are completed source files for this species, label, and iteration and if so pick up from where you left off  
    source_file_save_name='completed_source_files.npy' #this file keeps track of paths to wav files for each vocalization that have been annotated
    human_save_name='completed_human_labels.npy' #this file keeps track of the labels those wav files were given

    source_file_save_path = os.path.join(in_progress_dir,source_file_save_name)
    human_save_path = os.path.join(in_progress_dir, human_save_name)

    if len(os.listdir(in_progress_dir)) != 0:

        print('loading saved annotations...')
        done_source_files = list(np.load(source_file_save_path))
        human_labels = list(np.load(human_save_path))
        print('you already annotated', len(done_source_files), 'vocalizations from this species, label, and iteration...')

    else:
        print('starting from scratch - no previously annotated vocalizations from this sampling params dictionary')
        done_source_files = []
        human_labels = []

    #get the source files to be processed and begin
    not_done_source_files = [i for i in ds_df['source_file'] if i not in done_source_files]

    for spec_name in not_done_source_files:

        #save so you can recover if interrupted
        print('saving progress...')
        np.save(file=source_file_save_path, arr=np.array(done_source_files))
        np.save(file=human_save_path, arr=np.array(human_labels))

        #progress input
        print('vocalization', len(done_source_files),  'of', len(ds_df), '...')

        #collect meta-data

        #display the source file and make the spectrogram
        print(spec_name)
        if spec_type == 'from_embedding':
            to_plot = downsampled_frame.loc[downsampled_frame['source_file']==spec_name]
            img = to_plot.drop(columns=non_spec_columns)
            img = np.array(img).reshape((num_freq_bins, num_time_bins))
            plt.imshow(img, origin = 'lower', extent = (num_freq_bins, 0, num_time_bins, 0 ))
            plt.show()

        elif spec_type == 'from_wav':
            fs, wav = wavfile.read(clips_dir+spec_name)
            t,f,spec = get_spectrogram(data = wav, 
                          fs=spec_params['fs'], 
                          nperseg=spec_params['nperseg'], 
                          noverlap=spec_params['noverlap'], 
                          num_freq_bins=spec_params['num_freq_bins'], 
                          num_time_bins=spec_params['num_time_bins'] , 
                          min_freq=spec_params['min_freq'], 
                          max_freq=spec_params['max_freq'], 
                          fill_value=spec_params['fill_value'], 
                          max_dur=spec_params['max_dur'], 
                          spec_min_val=spec_params['spec_min_val'], 
                          spec_max_val=spec_params['spec_max_val'])
            plt.figure(figsize=[5,5])
            plt.imshow(spec, origin='lower')
            plt.show()

        #get input 
        val = input("what does this look like?"+"\n"+"type c for 'cry' | w for 'whistle' | s for 'scratch' | . for none of the above |"+"\n"+ "type u for undo | 'exit' to save and return a df of what you've annotated")

        if val == 'c':
            human_labels.append('cry')
            print('ok - cry')
            done_source_files.append(spec_name)

            continue 

        elif val == 'w':
            human_labels.append('whistle')
            print('ok - whistle')
            done_source_files.append(spec_name)

            continue

        elif val == 's':
            human_labels.append('scratch')
            print('ok - scratch')
            done_source_files.append(spec_name)

            continue

        elif val == '.':
            human_labels.append('none')
            print('ok - none')
            done_source_files.append(spec_name)

            continue

        elif val == 'u':
            count_val = input("how many annotations back to you you want to go?")

            if count_val in [str(i) for i in range(1,100,1)]:
                count_val=int(count_val)
                done_source_files = done_source_files[:-count_val]
                human_labels = human_labels[:-count_val]

                np.save(file=source_file_save_path, arr=np.array(done_source_files))
                np.save(file=human_save_path, arr=np.array(human_labels))
                print('Removed the last', str(count_val), 'annotations and saved the list of completed source files. Start over to re-label')
                return
            else:
                print('Type a number between 1 and 100. Start over to try again.')
                return

        elif val == 'exit':
            print("exiting...you've annotated", len(done_source_files), "wav clips.")
            print('returning the labeled vocalizations to you...')
            #compile the data you just generated
            temp = pd.DataFrame()
            temp['source_file'] = done_source_files
            temp['human_label'] = human_labels

            #merge on 'source_file' with the spectrograms dataframe
            labeled_df = ds_df.merge(temp, on=['source_file'])
            labeled_df = labeled_df.rename(columns={'label':'hdbscan_label'})

            #save
            labeled_df.to_feather(df_save_dir+df_save_name)

            print('done.')
            return labeled_df

        else:
            print("You pressed a key that doesn't make sense - exiting. Just re-run the cell.")
            return

    #compile the data you just generated
    temp = pd.DataFrame()
    temp['source_file'] = done_source_files
    temp['human_label'] = human_labels

    #merge on 'source_file' with the spectrograms dataframe
    labeled_df = ds_df.merge(temp, on=['source_file'])
    labeled_df = labeled_df.rename(columns={'label':'hdbscan_label'})

    #save
    print("you've annotated all the vocalizations in the data frame...saving")
    labeled_df.to_feather(df_save_dir+df_save_name)
    print('done.')
    return labeled_df
	
def from_background(species, pup, bg_labeling_params, bg_clips_dir, df_save_dir, df_save_name, inprogress_dir):
	"""
	1. Display clips of background (ie nonvocal) sounds for each pup in species
	2. User labels the ones that don't contain vocalizations
	3. Concatenate those clips that don't contain vocalizations and save them.
    
    Useful for generating training data of nonvocal sound.
	
	Arguments:
        pup (str):pup to get background clips from
        species (str): 2-letter code for the species to process
        bg_labeling_params (dict): dictionary of metadata and sepctrogramming parameters for background visualization
        bg_clips_dir (str): path to the directory containing background wav clips (within wich there are subdirectories labeled by species 2 letter code)
        df_save_dir (str): path to the directory where the datafame with the annotated background source files should be saved
        df_save_name (str): name of the dataframe to be saved
        inprogress_dir (str): path to the directory where labeling progress will be saved
        other_iterations (bool): if true find the other iterations, get the files that have already been annotated, and make sure you don't sample these

	Returns:
	   validated_df (df): a dataframe containing the source_files of the background clips that do not contain vocalizations

	"""
	
	#print pup
	print('\n################################\nlabeling background from pup', pup,'\n################################\n')
	
	#get the paths to the clips
	clips = [bg_clips_dir+i for i in os.listdir(bg_clips_dir) if i.endswith('.wav') and pup in i]
	
	#check if there are completed source files for this species, label, and iteration and if so pick up from where you left off
	
	source_file_save_path = inprogress_dir+pup+'_bg_completed_source_files'
	human_save_path = inprogress_dir+pup+'_bg_completed_human_labels'
	durations_save_path = inprogress_dir+pup+'_bg_completed_durations'
	newstarts_save_path = inprogress_dir+pup+'_bg_completed_newstarts'
	newstops_save_path = inprogress_dir+pup+'_bg_completed_newstop'

	if pup in set([i.split('_bg_completed')[0] for i in os.listdir(inprogress_dir)]):
		
		print('loading saved annotations...')
		done_source_files = list(np.load(source_file_save_path+'.npy'))
		human_labels = list(np.load(human_save_path+'.npy'))
		durations = list(np.load(durations_save_path+'.npy'))
		newstarts = list(np.load(newstarts_save_path+'.npy'))
		newstops =  list(np.load(newstops_save_path+'.npy'))

		print('you already annotated', len(done_source_files), 'clips...amounting to', sum(durations), 'seconds of audio from this pup...\n')
		
	else:
		print('starting from scratch - no previously annotated vocalizations from this sampling params dictionary')
		done_source_files = []
		human_labels = []
		durations = []
		newstarts= []
		newstops = []
		
	#check if there are other labeling iterations and if so make sure you don't sample the same clips for this one
	inprogress_root = os.path.split(inprogress_dir[:-1])[0]
	other_iterations = [i for i in os.listdir(inprogress_root) if i!='iteration'+str(bg_labeling_params['sampling_iteration']) and 'iteration' in i]
	print(other_iterations)
	other_source_files = []
	if len(other_iterations) != 0:	
		for iteration in other_iterations:
			other_pups = [i for i in os.listdir(inprogress_root+'/'+iteration+'/')]
			if pup+'_bg_completed_source_files.npy' in other_pups:
				print('This pup has background clips annotated in another iteration:')
				other_source_files_npy = os.path.join(inprogress_root, iteration, pup+'_bg_completed_source_files.npy')
				other_source_files.extend(list(np.load(other_source_files_npy)))
				
	print(len(other_source_files),'have been annotated from other iterations')
	
	#get the source files to be processed from this pup and begin - shuffle so that you're not just getting background from the start of recordings
	not_done_source_files = sorted([i.split('/')[-1] for i in clips if i.split('/')[-1] not in done_source_files and i.split('/')[-1] not in other_source_files], key = lambda x: random.random())

	for clip in not_done_source_files:
	
		#display cumulative length
		print(sum(durations), "seconds of audio annotated from this pup...")

		#save so you can recover if interrupted
		print('saving progress...\n')
		np.save(file=source_file_save_path, arr=np.array(done_source_files))
		np.save(file=human_save_path, arr=np.array(human_labels))
		np.save(file=durations_save_path, arr=np.array(durations))
		np.save(file=newstarts_save_path, arr=np.array(newstarts))
		np.save(file=newstops_save_path, arr=np.array(newstops))

		#read it
		fs, wav = wavfile.read(bg_clips_dir+clip)
		
		#trim the first 10 ms off because this is very often the trailing end of a cry
		margin = int(.01*fs)
		if len(wav) > margin:
			wav = wav[margin:]

		#print the clip name
		print(clip)

		#get its duration in seconds
		dur = len(wav)/fs
		total_samples = int(dur*fs)
		print('		clip duration (s)', dur)
		
		#make a spectrogram and show it
		t,f,spec = stft(wav, 
						noverlap=bg_labeling_params['noverlap'], 
						nperseg=bg_labeling_params['nperseg'], fs=fs)
		spec = np.log(np.abs(spec))
		plt.figure(figsize=[5,5])
		plt.imshow(spec, origin='lower')
		plt.show()

		#get input
		val = input("Are there vocalizations in this image?\nq for yes | a for no | u for undo | s to skip to the next clip | x to clip | e to save and exit")

		if val=='q':
			print("		ok - ignoring")
			done_source_files.append(clip.split('/')[-1])
			human_labels.append('voc')
			durations.append(0)
			newstarts.append(0)
			newstops.append(0)
			continue
	
		elif val=='a':
			print("		ok - recording")
			done_source_files.append(clip.split('/')[-1])
			human_labels.append('no_voc')
			durations.append(dur)
			newstarts.append(margin)
			newstops.append(total_samples)
			
			continue

		elif val=='s':
			print('		SKIPPING to the next CLIP...')
			done_source_files.append(clip.split('/')[-1])
			human_labels.append('NOT_ANNOTATED')
			durations.append(0)
			newstarts.append(0)
			newstops.append(0)
			continue
			

		elif val == 'x':
					start_change = input("how many seconds into this clip do you want it to start?")
					stop_change = input("how many seconds before the end of this clip do you want it to end?")
					
					total_samples = int(dur*fs)
					new_start =  margin+int(float(start_change)*fs)
					new_stop = total_samples - int(float(stop_change)*fs)
					short_clip = wav[new_start:new_stop] #get the clip
					new_dur = len(short_clip)/fs
					print('		new duration is', new_dur)
					
					t,f,spec = stft(short_clip, 
									noverlap=bg_labeling_params['noverlap'], 
									nperseg=bg_labeling_params['nperseg'], fs=fs)
									
					spec = np.log(np.abs(spec))
					plt.figure(figsize=[5,5])
					plt.imshow(spec, origin='lower')
					plt.show()
					
					new_val = input("Are there vocalizations in this image? q for yes | a for no | s for skip")
					
					if new_val=='a':
						print("		ok - recording")
						done_source_files.append(clip.split('/')[-1])
						human_labels.append('no_voc')
						durations.append(new_dur)
						newstarts.append(new_start)
						newstops.append(new_stop)
						continue
						
					elif new_val=='q':
						print("		ok - ignoring")
						done_source_files.append(clip.split('/')[-1])
						human_labels.append('voc')
						durations.append(0)
						newstarts.append(0)
						newstops.append(0)
						continue
						
					elif new_val=='s':
						print('		SKIPPING to the next CLIP...')
						done_source_files.append(clip.split('/')[-1])
						human_labels.append('NOT_ANNOTATED')
						durations.append(0)
						newstarts.append(0)
						newstops.append(0)
						continue
						
					else:
						print("You pressed a key that doesn't make sense - exiting. Current clip not counted as done.")
						return
						
		elif val == 'u':
			count_val = input("how many annotations back to you you want to go?")
	
			if count_val in [str(i) for i in range(1,100,1)]:
				count_val=int(count_val)
				done_source_files = done_source_files[:-count_val]
				human_labels = human_labels[:-count_val]
				durations = durations[:-count_val]
				newstarts = newstarts[:-count_val]
				newstops = newstops[:-count_val]
		
				np.save(file=source_file_save_path, arr=np.array(done_source_files))
				np.save(file=human_save_path, arr=np.array(human_labels))
				np.save(file=durations_save_path, arr=np.array(durations))
				np.save(file=newstarts_save_path, arr=np.array(newstarts))
				np.save(file=newstops_save_path, arr=np.array(newstops))
			
				print('Removed the last', str(count_val), 'annotations and saved the list of completed source files.')
				return 
				
			else:
				print('Type a number between 1 and 100. Start over to try again.')
				return
	
		elif val=='e':
			print("exiting...")
			print('you annotated', len(done_source_files),'clips amounting to', sum(durations), 'seconds of background audio...')
			print('returning the labeled vocalizations to you...')
			#compile the data you just generated
			temp = pd.DataFrame()
			temp['source_file'] = done_source_files
			temp['vocalizations_present?'] = human_labels
			temp['duration_s'] = durations
			temp['start_frame'] = newstarts
			temp['stop_frame'] = newstops
			temp = temp.reset_index(drop=True)
	
			#save
			temp.to_feather(df_save_dir+df_save_name)
			return temp
	
		else:
			print("You pressed a key that doesn't make sense - exiting. Current clip not counted as done.")
			return
			
	#compile the data you just generated
	print('you annotated', len(done_source_files),'clips amounting to', sum(durations), 'seconds of background audio...')
	temp = pd.DataFrame()
	temp['source_file'] = done_source_files
	temp['vocalizations_present?'] = human_labels
	temp['duration_s'] = durations
	temp['start_frame'] = newstarts
	temp['stop_frame'] = newstops
	temp = temp.reset_index(drop=True)
	
	#save
	temp.to_feather(df_save_dir+df_save_name)
	return temp
	
#Use concat_for_das instead of this
# def wav_from_voc_annotations(frame, clips_dir, save_dir, save_name):
# 	"""
# 	take a dataframe of human labeled spectrograms and their source files
# 	load the audio for those source files and concatenate it all together
# 	save that big wav file
# 
# 	Parameters
# 	----------
# 
# 	frame (dataframe): the cleaned dataframe of human labeled vocalizations
# 
# 	clips_dir (str): the path to the directory containing the final wav clips for the development dataset
# 
# 	save_dir (str): the path to the place where the big wav file will be stored
# 
# 	save_name (str): the name of the big wav file
# 
# 
# 	Returns
# 	-------
# 	None
# 
# 	"""
# 
# 	#get the source files
# 	source_files = frame['source_file']
# 
# 	#collect the audio
# 	wavs = []
# 	for file in tqdm(source_files):
# 		fs, wav = wavfile.read(clips_dir+file)
# 		wavs.extend(wav)
# 	
# 	#write the audio
# 	wavfile.write(save_dir+save_name, fs, np.array(wavs))
# 	print('done.')
	
def wav_from_background_annotations(frame, clips_dir, save_dir, save_name):
	"""
	1. Take a dataframe of human labeled background clips and their source files
	2. Load the audio for those source files, trim them if they had been stimmed during annotation, and concatenate it all together
	3. Save that big wav file

	Arguments:
        frame (dataframe): the cleaned dataframe of human labeled vocalizations
        clips_dir (str): the path to the directory containing the final wav clips for the development dataset
        save_dir (str): the path to the place where the big wav file will be stored
        save_name (str): the name of the big wav file

	Returns:
	   None

	"""

	#get the source files
	source_files = frame['source_file']

	#collect the audio
	wavs = []
	for file in tqdm(source_files):
		fs, wav = wavfile.read(clips_dir+file)
		start = int(list(frame['start_frame'].loc[frame['source_file'] == file])[0])
		stop = int(list(frame['stop_frame'].loc[frame['source_file'] == file])[0])
		wav=wav[start:stop]
		wavs.extend(wav)
	
	#write the audio
	wavfile.write(save_dir+save_name, fs, np.array(wavs))
	print('done.')
	
	
def wav_from_nonvocal_annotations(frame, clips_dir, save_dir, save_name):
	"""
	1. Take a dataframe of human labeled non_vocal clips from a given cluster and their source files
	2. Load the audio for those source files and concatenate it all together
	3. Save that big wav file

	Arguments:
        frame (dataframe): the cleaned dataframe of human labeled vocalizations
        clips_dir (str): the path to the directory containing the final wav clips for the development dataset
        save_dir (str): the path to the place where the big wav file will be stored
        save_name (str): the name of the big wav file

	Returns:
	   None

	"""

	#get the source files
	source_files = frame['source_file']

	#collect the audio
	wavs = []
	for file in tqdm(source_files):
		fs, wav = wavfile.read(clips_dir+file)
		wavs.extend(wav)
	
	#write the audio
	wavfile.write(save_dir+save_name, fs, np.array(wavs))
	print('done.')
	
def clean_voc_labels(frame, save_dir, voc_save_name, nonvoc_save_name):
	"""
	1. Take a dataframe of human labeled spectrograms 
	2. If hdbscan label = 0, drop everything that does not have a human label of 'whistle'
	3. Else if hdbscan label = 1, drop everything that does not have a human label of 'cry'
	4. In either case also save a df of just nonvocal sounds
	5. Retrun the dataframe and save

	Arguments:
        frame (dataframe): the uncleaned original dataframe of human labeled vocalizations
        save_dir (str): the path to the place where the cleaned dataframes will be stored
        nonvoc_save_name (str): the name of the cleaned nonvocalization dataframe
        voc_save_name (str): the name of the cleaned vocalization dataframe

	Returns:
	   cleaned_df (dataframe): a data frame that is just whistle or cry

	"""
	
	if frame['hdbscan_label'].unique()[0]==0:
		cleaned_df = frame.loc[frame['human_label'] == 'whistle']
		cleaned_df = cleaned_df.reset_index(drop=True)
		
		non_vocal_df = frame.loc[frame['human_label'] == 'scratch']
		non_vocal_df = non_vocal_df.reset_index(drop=True)
		
	elif frame['hdbscan_label'].unique()[0]==1:
		cleaned_df = frame.loc[frame['human_label'] == 'cry']
		cleaned_df = cleaned_df.reset_index(drop=True)
		
		non_vocal_df = frame.loc[frame['human_label'] == 'scratch']
		non_vocal_df = non_vocal_df.reset_index(drop=True)
	
	if voc_save_name not in os.listdir(save_dir):
		cleaned_df.to_feather(save_dir+voc_save_name)
		print('done.')
	else:
		print('Clean vocalization annotation file appears to already exist-not saving: ', voc_save_name)
		
	if nonvoc_save_name not in os.listdir(save_dir):
		non_vocal_df.to_feather(save_dir+nonvoc_save_name)
		print('done.')
	else:
		print('Clean non-vocal annotation file appears to already exist-not saving: ', nonvoc_save_name)
	
	return cleaned_df, non_vocal_df
	
def clean_background_labels(frame, save_dir, save_name):
	"""
	take a dataframe of human labeled background (non-vocalization) clips
	remove rows where ['vocalizations_present?'] equal to 'no_voc'
	retrun the dataframe and save

	Parameters
	----------

	frame (dataframe): the uncleaned original dataframe of human labeled vocalizations

	save_dir (str): the path to the place where the big wav file will be stored

	save_name (str): the name of the big wav file


	Returns
	-------
	cleaned_df (dataframe): a data frame that is just whistle or cry

	"""
	
	cleaned_df = frame.loc[frame['vocalizations_present?'] == 'no_voc']
	cleaned_df = cleaned_df.reset_index(drop=True)

	if save_name not in os.listdir(save_dir):
		cleaned_df.to_feather(save_dir+save_name)
		print('saved cleaned annotations file to', save_dir+save_name	)
	else:
		print('This file appears to already exist...not saving.')
		val = input('overwrite? y/n')
		if val == 'y':
			cleaned_df.to_feather(save_dir+save_name)
			print('saved cleaned annotations file to', save_dir+save_name)
		elif val=='n':
			print('ok - no file saved.')
			
	
	return cleaned_df
	
	
def concat_annotated_spacer(voc_clips_dir, voc_clips_list, spacer_wav, save_dir, save_name, label, margin):
    """
    1. Interleve vocalizations annotated from umap with a background nonvocal clip to which noise is randomly added.
    2. Generate an annotations file of the vocalizations start and stop times
    3. Optionally include a margin to trim the annotated start and stop time by some amount.
    
    Useful for organizing annotations into audio files in a way that makes training models easier.
    The format of the annotation csv saved is formatted for use by DAS (https://github.com/janclemenslab/das)

    Arguments:
        voc_clips_dir (str): path to the vocalization clips to be concatenated
        voc_clips_list (str): list of wav files to be concatenated from annotation df
        spacer_wav (str): optional path to a ~100 ms non_vocal clip to act as a spacer between each vocalization
        save_dir (str): the path to the place where the big wav file will be stored
        save_name (str): the name of the annotations csv (no file extension, must end in '_annotations') - this determines the name of the wav file 
        margin (float): time in seconds to put the annotation start after the clip start and the annotation stop before the clip  stop
        
    Returns
    -------
        annotation (dataframe): the annotation csv

    """
    assert save_name.endswith('_annotations')
    
    clips = [voc_clips_dir+i for i in voc_clips_list if i.endswith('.wav') and 'background' and 'test' not in i]
    #print('processing', len(clips), 'clips')
    labels = []
    start_times = []
    stop_times = []
    clips_list = []
    start_time = 0 #first start time is 0
    durations = []
    total_duration = 0 #starting duration of the big wav is 0

    if spacer_wav != None:
        
        for clip in tqdm(clips):

            #read the filler wav
            fs, spacer = wavfile.read(spacer_wav)

            #get its duration
            spacer_dur = float(len(spacer)/fs)

            #read the voc wav
            fs, wav = wavfile.read(clip)

            #get its duration
            dur = float(len(wav)/fs)

            #update duration
            durations.append(dur + spacer_dur)
            total_duration = sum(durations)

            #reset start and stop times
            start_time = start_time + margin #update this below
            stop_time = total_duration - spacer_dur - margin #stop time for this clip in the big wav

            #add some random noise to the spacer
            noise = np.random.normal(0, 10, np.array(spacer).shape)
            spacer = spacer + noise
            spacer_noise = [int(i) for i in spacer]

            #update lists
            start_times.append(start_time)
            stop_times.append(stop_time)
            clips_list.extend(wav)
            clips_list.extend(spacer_noise)
            labels.append(label)

            #new start time is the current total duration
            start_time = total_duration

        #add the other label
        if label=='whistle':
            labels.append('cry')
            start_times.append('')
            stop_times.append(0)

        if label=='cry':
            labels.append('whistle')
            start_times.append('')
            stop_times.append(0)

        #compile the dataframe and write 
        annotation = pd.DataFrame()
        annotation['name'] = labels
        annotation['start_seconds'] = start_times
        annotation['stop_seconds'] = stop_times
        annotation.to_csv(save_dir+save_name+'.csv', index=False)

        #save the audio
        wav_save_name = save_name.split('_annotations')[0]+'.wav'


        clips_list = np.array(clips_list)
        clips_list = clips_list.astype('int16')
        wavfile.write(save_dir+wav_save_name, fs, np.array(clips_list))
        print('done.')

        return annotation

    else:

        for clip in tqdm(clips):

            #read the voc wav
            fs, wav = wavfile.read(clip)

            #get its duration
            dur = float(len(wav)/fs)
            print('voc_dur:', dur)

            #update duration
            durations.append(dur)
            total_duration = sum(durations)

            #reset start and stop times
            start_time = start_time + margin #update this below
            stop_time = total_duration - margin #stop time for this clip in the big wav

            #update lists
            start_times.append(start_time)
            stop_times.append(stop_time)
            clips_list.extend(wav)
            labels.append(label)

            #new start time is the current total duration
            start_time = total_duration

        #add the other label
        if label=='whistle':
            labels.append('cry')
            start_times.append('')
            stop_times.append(0)

        if label=='cry':
            labels.append('whistle')
            start_times.append('')
            stop_times.append(0)

        #compile the dataframe and write 
        annotation = pd.DataFrame()
        annotation['name'] = labels
        annotation['start_seconds'] = start_times
        annotation['stop_seconds'] = stop_times
        annotation.to_csv(save_dir+save_name+'.csv', index=False)

        #save the audio
        wav_save_name = save_name.split('_annotations')[0]+'.wav'

        print('test print')
        print('writing big wav')
        
        print('clips_list:',clips_list)
        clips_list = np.array(clips_list)
        clips_list = clips_list.astype('int16')
        print('int16 clips_list:',clips_list)
        wavfile.write(save_dir+wav_save_name, fs, np.array(clips_list))
        print('done.')

        return annotation

def get_random_durations(num_to_generate,random_seed, min_dur, max_dur):
	"""
	Generate random durations between min_dur, and max_dur
	Useful for generating background clips to inerleve with vocalization clips.

	Arguments:
	   num_to_generate (int): the number of durations to generate - should be 1 less than the number of vocalization clips you have
	   random_seed (int): seed for random.uniform()
	   min_dur (str): the minimum duration in seconds you want to sample 
	   max_dur (str): the maximum duration in seconds you want to sample

	Returns:
	   durations (list): a list of the durations

	"""
    
	#set the seed
	random.seed(a=random_seed, version=2)

	#load te background audio and get its length
	fs,wav=wavfile.read(background_wav_path)
	dur = float(len(wav)/fs)

	durations = []
	for num in range(num_to_generate):
		clip_dur = random.uniform(min_dur, max_dur)
		durations.append(clip_dur)

	return durations

def starts_stops_from_random_durations(durations):
	"""
	Generate start and stop times from a list of durations (eg output of get_random_durations)
	Useful for generating background clips to inerleve with vocalization clips or 
	Intended to be used after get_random_durations()

	Arguments:
	   durations (list): a list of durations, eg the output of get_random_durations()


	Returns:
	   segments (df): a dataframe with consecutive start and stop times with a column called 'start' and a column called 'stop'

	"""
    
	start_times = []
	stop_times = []
	cumulative_dur = 0
	start_time = 0

	for dur in durations:
	
		#update cumulative duration and stop time
		cumulative_dur += dur
		stop_time = cumulative_dur
	
		#update lists
		start_times.append(start_time)
		stop_times.append(stop_time)
	
		#start of next clip is stop of previous
		start_time = stop_time

	segments = pd.DataFrame(data=zip(start_times, stop_times), columns = ['start', 'stop'])
	
	return segments
    
def random_clips_from_background(background_wav_path, save_dir, clip_name_root, voc_clips_dir, random_seed, min_dur, max_dur):
	"""
	Generate clips of background from a longer background file eg, the output of wav_from_background_annotations()
	Inteneded to be used to get background clips for interleving by concat annotated clips

	Arguments:
        background_wav_path (str): the path to the big background wav file you are going to split up
        save_dir (str): the directory where the clips will be saved
        clip_name_root (str): the clips will all share this part of their file name and have different clip numbers appended
        voc_clips_dir (str): the path to the directory containing the vocalization labeled clips you want interleve
        random_seed (int): random seed for get_random_durations()
        min_dur (float): min_dir for get_random_durations() - clips will all be longer than this
        max_dur (float): min_dir for get_random_durations() - clips will all be shorter than this


	Returns:
	   random_durations (list): the durations of the clips
	   segments (df): a dataframe with consecutive start and stop times of the clips with a column called 'start' and a column called 'stop'

	"""
    
	#load the background wav and get its length
	fs, wav = wavfile.read(background_wav_path)
	full_duration = float(len(wav)/fs)

	#get the durations
	num_to_generate = len([i for i in os.listdir(voc_clips_dir) if i.endswith('.wav')]) - 1

	random_durations = get_random_durations(num_to_generate = num_to_generate, 
										  random_seed = random_seed, 
										  min_dur = min_dur, 
										  max_dur = max_dur)

	#get start and stop times
	segments = starts_stops_from_random_durations(random_durations)

	#check that sum of the random durations is less than the length of the background wav file
	if sum(random_durations) > full_duration:
		print('The random durations are longer than the background audio - annotate more background or reduce max_dur')
		return
	else: 
		last_clip_start = int(sum(random_durations)*fs)

	#write all the wav clips except the last
	count = 0
	for row, _ in segments.iterrows():

		#clip the clip
		start = int(segments['start'].iloc[row]*fs)
		stop = int(segments['stop'].iloc[row]*fs)

		background_clip = wav[start:stop]
		#print('length:', len(background_clip)/fs)
	
		#write the clip
		clip_name = save_dir+clip_name_root+'_clip_'+str(count)+'.wav'
		wavfile.write(clip_name, fs, np.array(background_clip))
	
		#update count
		count+=1
	
	#make the last background clip (from whatever is left over)
	last_clip = wav[last_clip_start:]
	clip_name = save_dir+clip_name_root+'_clip_'+str(count)+'.wav'
	wavfile.write(clip_name, fs, np.array(last_clip))
	
	return random_durations, segments
    
def concat_annotated_random(voc_clips_dir,random_bg_clips_dir, save_dir, save_name, label):
	"""
	1. Interleve vocalizations in voc_clips_dir with clips in random_bg_clips_dir generated from wav file annotations_from_background makes
	2. Save the audio
	3. Save an annotation file
    
    NOTE: Does the same thing as concat_annotated_spacer but at the risk of including vocalizations that were missed during annotation. Use concat_annotated_spacer unless you are 100% confident there are no vocalizations in the annotate background.

    Arguments:
        voc_clips_dir (str): the path to the directory containing the vocalization labeled clips you want interleve 
        random_bg_clips_dir (str):  the path to the directory containing the background clips generated by random_clips_from_background
        save_dir (str): the directory where the interleved wav and the annotations csv will be saved.
        save_name (str): the root name that will be shared between the wav and the annotation csv
        label (str): the label for the vocalization to be interleved (either 'cry' or 'USV' )


	Returns
	-------
	   annotation (df): the das ready annotation file
	"""
    
	#get the clips to interleve
	voc_clips = [voc_clips_dir+i for i in os.listdir(voc_clips_dir) if i.endswith('.wav') and 'back' not in i]
	bg_clips = [random_bg_clips_dir+i for i in os.listdir(random_bg_clips_dir) if i.endswith('.wav')]

	#check they are the same length
	if len(voc_clips) != len(bg_clips):
		print('different number of voc clips and background clips...stopping.')
		return

	#interleve the two lists
	interleved = list(itertools.chain(*zip(voc_clips, bg_clips)))

	#print('processing', len(clips), 'clips')
	labels = []
	start_times = []
	stop_times = []
	clips_list = []
	durations = []

	start_time = 0.001 #first start time is .001
	total_duration = 0 #starting duration of the big wav is 0

	for clip, count in zip(interleved, range(len(interleved))):
	
		if not count%2: #The wav will start with a vocalization
			print(clip.split('/')[-1])
		
			#read the wav
			fs, wav = wavfile.read(clip)

			#get its duration
			dur = float(len(wav)/fs)

			#update duration
			durations.append(dur)
			total_duration += dur
			print('voc total_dur', total_duration)
			print('durations', durations)
	
			#reset start and stop times
			start_time = start_time #update this below
			stop_time = total_duration #stop time for this clip in the big wav

			#update lists for the annotations .csv
			start_times.append(start_time)
			stop_times.append(stop_time)
			clips_list.extend(wav)
			labels.append(label)

			#new start time is the current total duration
			print(start_time)
			start_time = total_duration
			print(start_time)
		
		elif count%2: #for the background clips just update the clips list and start/stop_times
			print(clip.split('/')[-1])
		
			#read the wav
			fs, wav = wavfile.read(clip)

			#get its duration
			dur = float(len(wav)/fs)

			#update duration
			durations.append(dur)
			total_duration += dur
			print('bg total_dur', total_duration)
			print('durations', durations)
		
			#reset start and stop times
			start_time = start_time #update this below
			stop_time = total_duration #stop time for this clip in the big wav

			#update lists
			clips_list.extend(wav)

			#new start time is the current total duration
			print(start_time)
			start_time = total_duration
			print(start_time)

	#add the other label
	if label=='USV':
		labels.append('cry')
		start_times.append('')
		stop_times.append(0)
	
	if label=='cry':
		labels.append('USV')
		start_times.append('')
		stop_times.append(0)
	
	#compile the dataframe and write 
	annotation = pd.DataFrame()
	annotation['name'] = labels
	annotation['start_seconds'] = start_times
	annotation['stop_seconds'] = stop_times
	annotation.to_csv(save_dir+save_name+'_annotation.csv', index=False)

	#save the audio
	wav_save_name = save_name.split('_annotations')[0]+'.wav'

	print('writing big wav')
	wavfile.write(save_dir+wav_save_name, fs, np.array(clips_list))
	print('done.')

	return annotation

	
def make_directories(save_dir, iteration):
	"""
	Prepare the directories needed for annotation for a single species

	Arguments:

	   save_dir (str): the path to the directory containing the annotations for a single species set in the 3rd cell of annotations_from_umap.ipynb
        iteration (str): which iteration of hand labeling is this ('iteration1', 'iteration2', etc.)


	Returns:	
	   None

	"""

	#get an embedding with hdbscan labels
	if 'annotations' not in os.listdir(save_dir):
		os.mkdir(save_dir+'annotations')
		print('made annotations directory at', save_dir+'annotations\n')
	else:
		print('annotations directory already exists at', save_dir+'annotations\n')
	
	if '00_params' not in os.listdir(save_dir+'annotations'):
		os.mkdir(save_dir+'annotations/'+'00_params')
		print('made 00_params directory at', save_dir+'annotations/'+'00_params\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'00_params\n')
   
	if 'in_progress' not in os.listdir(save_dir+'annotations'):
		os.mkdir(save_dir+'annotations/'+'in_progress')
		print('made 00_params directory at', save_dir+'annotations/'+'in_progress\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'in_progress\n')

	if 'hdbscanlabel0' not in os.listdir(save_dir+'annotations/in_progress'):
		os.mkdir(save_dir+'annotations/'+'in_progress/hdbscanlabel0')
		print('made 00_params directory at', save_dir+'annotations/'+'in_progress/hdbscanlabel0\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'in_progress/hdbscanlabel0\n')
	
	if iteration not in os.listdir(save_dir+'annotations/in_progress/hdbscanlabel0'):
		os.mkdir(save_dir+'annotations/'+'in_progress/hdbscanlabel0/'+iteration)
		print('made 00_params directory at', save_dir+'annotations/'+'in_progress/hdbscanlabel0/'+iteration+'\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'in_progress/hdbscanlabel0/'+iteration+'\n')
	
	if 'hdbscanlabel1' not in os.listdir(save_dir+'annotations/in_progress'):
		os.mkdir(save_dir+'annotations/'+'in_progress/hdbscanlabel1')
		print('made 00_params directory at', save_dir+'annotations/'+'in_progress/hdbscanlabel1\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'in_progress/hdbscanlabel1\n')
	
	if iteration not in os.listdir(save_dir+'annotations/in_progress/hdbscanlabel1'):
		os.mkdir(save_dir+'annotations/'+'in_progress/hdbscanlabel1/'+iteration)
		print('made 00_params directory at', save_dir+'annotations/'+'in_progress/hdbscanlabel1/'+iteration+'\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'in_progress/hdbscanlabel1/'+iteration+'\n')
	
	if 'hdbscan_label_0' not in os.listdir(save_dir+'annotations'):
		os.mkdir(save_dir+'annotations/'+'hdbscan_label_0')
		print('made 00_params directory at', save_dir+'annotations/'+'hdbscan_label_0\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'hdbscan_label_0\n')
	
	if 'hdbscan_label_1' not in os.listdir(save_dir+'annotations'):
		os.mkdir(save_dir+'annotations/'+'hdbscan_label_1')
		print('made 00_params directory at', save_dir+'annotations/'+'hdbscan_label_1\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'hdbscan_label_1\n')
		
	if 'background' not in os.listdir(save_dir+'annotations'):
		os.mkdir(save_dir+'annotations/'+'background')
		print('made 00_params directory at', save_dir+'annotations/'+'background\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'background\n')
    
	if 'background' not in os.listdir(save_dir+'annotations/in_progress'):
		os.mkdir(save_dir+'annotations/'+'in_progress/background')
		print('made 00_params directory at', save_dir+'annotations/'+'in_progress/background\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'in_progress/background\n')
		
	if iteration not in os.listdir(save_dir+'annotations/in_progress/background'):
		os.mkdir(save_dir+'annotations/'+'in_progress/background/'+iteration)
		print('made 00_params directory at', save_dir+'annotations/'+'in_progress/background/'+iteration+'\n')
	else:
		print('00_params directory already exists at', save_dir+'annotations/'+'in_progress/background/'+iteration+'\n')
		
		

