#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Mon Apr  3 10:49:43 2017

@author: emg
"""

import pandas as pd
from bs4 import BeautifulSoup
from prawini import *

params = get_params()
headers={'User-agent':params['user_agent']}

def make_soup(url):
    r = requests.get(url, headers=headers)
    soup = BeautifulSoup(r.text, "html5lib")
    return soup

def scrape_mod_table(sub): 
    '''pulls mod info from soup object of /about/moderators snapshot
       dumps info into a dataframe
       sub should be in format 'r/subname' '''
    url = 'https://www.reddit.com/{}/about/moderators'.format(sub)
    soup = make_soup(url)
    table = soup.find('div', attrs={'class':'moderator-table'})
    table_body = table.find('tbody')
    rows = table_body.find_all('tr')
    name, href, permissions, date = [], [], [], []
    for row in rows:
        #info = mod.parent.parent.parent
        #children = info.findChildren()
        name.append(row.a.text)
        date.append(row.time['datetime'])
        href.append(row.a['href'])
        permissions.append(row.find('input',{'name':'permissions'})['value'])
    columns = {'name':name, 'date':date, 'useraccount':href, 'permissions':permissions, 'pubdate':url[27:41]}
    df = pd.DataFrame(columns)
    df['sub'] = sub
    return df

# FOR CMV  
df = scrape_mod_table('r/changemyview')
df.to_csv('/Users/emg/Programming/GitHub/cmv/raw_data/current-mod-table.csv')

# FOR TD 
df = scrape_mod_table('r/The_Donald')
df.to_csv('/Users/emg/Programming/GitHub/the_donald_project/raw_data/current-mod-table.csv')

# RUN GET_MOD_SUBS
# REPEAT FOR SUBREDDITS ONE STEP OUT

#mod_subs = pd.read_csv('/Users/emg/Programming/GitHub/cmv/raw_data/current-sub-mod-matrix.csv', index_col=0)
mod_subs = pd.read_csv('/Users/emg/Programming/GitHub/the_donald_project/raw_data/current-sub-mod-matrix.csv', index_col=0)

subs = list(mod_subs.index)

dfs = []
errors = []
n = 1
for sub in subs:
    print n, sub
    try:
        df = scrape_mod_table(sub)
    except:
        errors.append(sub)
        pass
    dfs.append(df)
    n += 1

all_sub_mods = pd.concat(dfs)
#all_sub_mods.to_csv('/Users/emg/Programming/GitHub/cmv/raw_data/all_sub_mods.csv')
all_sub_mods.to_csv('/Users/emg/Programming/GitHub/the_donald_project/raw_data/all_sub_mods.csv')

counts = all_sub_mods.groupby('name')['name'].count()
doubles = counts[counts>1]
subset = all_sub_mods[all_sub_mods['name'].isin(doubles.index)]
    