# Scraper for Religous Freedom Reports from 2011-2013

import time
import os
from compiler.ast import flatten
import json

# My helper functions
os.chdir('/Users/janus829/Desktop/Research/WardProjects/regimeClassif/Scrapers')
from scrapeCrew import *

# Data storage
os.chdir('/Users/janus829/Dropbox/Research/WardProjects/regimeClassif/Data')

# Building Web Address for State Department Reports
base = 'http://www.state.gov/j/drl/rls/irf/'
yrURLs = [base+x+'religiousfreedom' for x in ['2011','']]

# Run through years
year=2011
for addressYr in yrURLs:
	print 'Collecting data from ' + addressYr +'\n'	

	# Pull out links for countries
	ctryYrData=subLinks(addressYr, 'div', 'id', 'countries', 
			'<li>', '<a href="', '" title=',
			True, 'title="', '" target="')

	# Scrape info
	for ctry in ctryYrData:

		# load webpage
		ctry['source']='StateRF'
		ctry['year']=str(year)
		ctry['data']=getText(ctry['url'], 'div', 'id', 'centerblock')
		time.sleep(3)
		print '   ...data for ' + ctry['name'] + ' collected'		

	# Save to json
	print '\n Data for ' + ctry['year'] + ' collected \n'	
	filename=ctry['source'] + '_' + ctry['year'] + '.json'
	f=open(filename, 'wb')
	json.dump(ctryYrData, f, sort_keys=True)
	f.close()
	year+=1	