import pandas as pd
import urllib2
from bs4 import BeautifulSoup
import re
from multiprocessing import Pool
import re
import numpy as np


# The prices are often listed as price + fee. This takes the string and adds those numbers, returns the number as a float type.
def add_prices(price_str):
    price_str = re.sub("[\\\][x|a|c]",'',price_str) 
    price_str = re.sub("a",'', price_str)
    if price_str is None:
        return None
    if price_str.find('+') != -1:
	if price_str.isdigit() == True:
        	return np.float(price_str.split('+')[0]) + np.float(price_str.split('+')[1])
    	else:
		return 0
    else:
	if price_str.isdigit() == True:
        	return np.float(price_str)
	else:
		return 0

# Given the raw scraper output generated by RAEventPageScraper.py, breaks up the information and returns price, lineup, promoters, venue, url.
def read_event_data():

    tickets_available = re.sub("[(]",'',sample_data.split(',')[0])
    release = re.sub("[(')]", '', sample_data.split(',')[1])
    if re.sub("[ ']", '', sample_data.split(',')[1]) == 'None':
        price = 0
        istart = 3
    else:
        price = add_prices(re.sub("[u'$)( ]", '', sample_data.split(',')[2]))
        istart = 4
    split_data = sample_data.split(',')
    lineup = []
    done = False
   
    if re.sub("[ \['\]]",'',sample_data.split(',')[istart])[0:]=='':
        lineupExists=False
        istart += 1
    else:
        for i in range(istart,10):
            lineup.append(re.sub("[ \['\]]",'',sample_data.split(',')[i])[0:])
            if split_data[i][-2:] == ']]':
                done = True
                break
            else:
                pass
        istart = i+1
    promoters = []
    
    if re.sub("[ \['\]]", '', sample_data.split(',')[istart]) == '':
            promotersExist = False
            istart += 1
    else:
            promotersExist = True
            istart -=1
    if promotersExist == True:        
        for j in range(istart+1,istart+5):
            promoters.append(re.sub("[ \['\]]",'',sample_data.split(',')[j])[0:])
            if split_data[j][-2:] == ']]':
                done = True
                break
            else:
                pass
        istart = j+1
    venue = re.sub("[' \[\]]",'',split_data[istart])
    url = re.sub(' ', '', sample_data[sample_data.find('http://www.residentadvisor.net/event.aspx?'):])
    return price, lineup, promoters, venue, url


user_id_list = []
def get_user_ids():
	id_index = [m.start() for m in re.finditer("Id", sample_data)]
	user_ids = []
	for i in id_index:
    		user_ids.append(re.sub("[^\\d]",'',sample_data[i+3:i+15]))
	return user_ids

# This is the raw output file generated by RAEventPageScraper.py. 
file = open('RAScraperDumpSept22_1')
lines = file.readlines()

# Format the columsn so pandas gives column headers later.
print 'userid cost dj1 dj2 dj3 promoter1 promoter2 promoter3 venue url'


# For each line of raw output
for line in lines:
	data = []
	sample_data = line
	start = sample_data.find('ai=8')
	end = sample_data.find('dy=')
	print sample_data[start:end+5]
	user_list = get_user_ids()
	for user_id in user_list:
	    event_data_list = read_event_data()		
    	    data.append(user_id)
	    data.append(event_data_list[0])
	    dj_list = event_data_list[1][0:min(3,len(event_data_list[1]))]
	    for dj in dj_list:
		data.append(dj)		
	    for i in range(0, max(3 - len(event_data_list[1]), 0)):
		data.append('None')
	    promoter_list = event_data_list[2][0:min(3,len(event_data_list[2]))]
	    for promoter in promoter_list:
		data.append(promoter)
	    for i in range(0, max(3 - len(event_data_list[2]), 0)):
		data.append('None')
	    data.append(event_data_list[3])    
	    data.append(event_data_list[-1])
	    for item in data:
		pass
		print item,
	    print ''
	    data = []
