
# Status
col_meta_status = 2

# Progress
col_meta_progress = 4

# Finished 
col_meta_finished = 6

# Agree 
col_meta_agree = 17

# Have you participated in any of these hackathons this school year?\n\n"This school year" refers to the 2017-2018 school year.
col_hack_listAttended  = 18

# What other student hackathons have you participated in this school year? \n\nYou may also enter hackathons you are planning to participate in.\n\n(Ex: HackCU, BrickHacks, etc)
col_hack_listAttendedText = 19

# How many student hackathons have you participated in before this school year?
col_hack_numParticipated = 20

# Q5          If you were available, would you want to participate in a female or non-binary-focused hackathon student hackathon in the future? - Selected Choice
col_hack_partFemaleHack = 21

# Q5_7_TEXT   If you were available, would you want to participate in a female or non-binary-focused hackathon student hackathon in the future? - Other:  - Text
col_hack_partFemaleHackOther = 22

# Q6          If you were available, would you want to participate a non-gendered-focused hackathon in the future? - Selected Choice
col_hack_partNonGenderedHackathon = 23

# Q6_7_TEXT   If you were available, would you want to participate a non-gendered-focused hackathon in the future? - Other:\n  - Text
col_hack_partNongenderedHackathonText = 24


# Q8          Please type the name of the most recent hackathon you attended.
col_attend_name = 25

# Q9          Did you stay for most of the hackathon? - Selected Choice
col_attend_stay = 26

# Q9_5_TEXT   Did you stay for most of the hackathon? - Other: - Text
col_attend_stayText = 27

# Q10         Did you attend any of the technical workshops? - Selected Choice   
col_attend_workshops = 28

# Q10_4_TEXT  Did you attend any of the technical workshops? - Other:  - Text
col_attend_workshopstext = 29

# Q11         At any time (outside of the workshops) did you use the mentors as a resource to help you with your project? - Selected Choice
col_attend_mentors = 30

# Q12         Did you participate with a team? - Selected Choice      
col_attend_team = 32

# Q12_3_TEXT  Did you participate with a team? - Other:  - Text
col_attend_teamText = 33

# Q13         Did you demo your project at the end of the hackathon? - Selected Choice
col_attend_demo = 34

# Q13_3_TEXT  Did you demo your project at the end of the hackathon? - Other:  - Text 
col_attend_demoText = 35

# Q14         Did you win any prizes? - Selected Choice
col_attend_prizes = 36

# Q14_3_TEXT  Did you win any prizes? - Other:  - Text
col_attend_prizesText = 37



# Q16         I see myself as a "hacker" after participating at the hackathon.
col_bel_spirit_hacker = 38

# Q17         I felt like I belonged at the hackathon
col_bel_spirit_belonging = 39

# Q18         I felt like I fit in at the hackathon
col_bel_spirit_fit = 40

# Q20         I felt a sense of community at the hackathon
col_bel_spirit_community = 42

# Q21         I enjoyed myself at the hackathon
col_bel_spirit_enjoyment = 43

# Q22         I felt a sense of companionship with my team members
col_bel_spirit_compainship = 44


# Q23         I felt comfortable talking with the other participants
col_bel_trust_talkParticipants = 45

# Q24         I felt comfortable talking with my team members
col_bel_trust_talkteam = 46

# Q25         I felt comfortable talking to the mentors and sponsors
col_bel_trust_talkMentors = 47

# Q26         I felt like everyone at the hackathon was there to support each other
col_bel_trust_support = 48

# Q27         I felt like other participants were interested in the project I was building
col_bel_trust_interestParticipants = 49

# Q28         I felt like the mentors and sponsors were interested in the project I was building
col_bel_trust_interestMentors = 50

# Q36         I was interested in seeing the projects other participants built
col_bel_trust_interestSelf = 58


# Q29         When I had a question about my project, I asked other hackathon participants (who were not on my team)
col_bel_interaction_participants = 51

# Q30         When I had a question about my project, I asked my team members
col_bel_interaction_team = 52

# Q31         When I had a question about my project, I asked the mentors
col_bel_interaction_mentors = 53


# Q19         I felt like the hackathon was competitive
col_bel_expectations_competitive = 41

# Q32         I enjoyed the fun activities at the hackathon
col_bel_expectations_fun = 54

# Q33         I felt that my team and I had the same goals for our project
col_bel_expectations_goals = 55

# Q34         I understood the relevance of the hackathon workshops
col_bel_expectations_relavantWorkshops = 56

# Q35         I understand how to apply what I learned at the hackathon in my major or in the workplace
col_bel_expectations_apply = 57

# Q37         I believe I learned relevant skills
col_bel_expectations_relavantSkills = 59


# Q38         I believe it is important to host female and non-binary-focused hackathons
col_bel_other_femImp = 61

# Q39         How was your overall hackathon experience?
col_bel_other_overallExp = 60

col_belonging = [
    {
        "position": col_bel_spirit_hacker,      "constructType": 1,     "belongingGroup": 1,    "label": "Belonging - Spirit - Hacker"
    },
    {
        "position": col_bel_spirit_belonging,   "constructType": 1,     "belongingGroup": 1,    "label": "Belonging - Spirit - Belonging"
    },
    {
        "position": col_bel_spirit_fit,         "constructType": 1,     "belongingGroup": 1,    "label": "Belonging - Spirit - Fit"
    },
    {
        "position": col_bel_spirit_community,   "constructType": 1,     "belongingGroup": 1,    "label": "Belonging - Spirit - Community"
    },
    {
        "position": col_bel_spirit_enjoyment,   "constructType": 1,     "belongingGroup": 1,    "label": "Belonging - Spirit - Enjoyment"
    },
    {
        "position": col_bel_spirit_compainship, "constructType": 1,     "belongingGroup": 1,    "label": "Belonging - Spirit - Compainship"
    },
    
    {
        "position": col_bel_trust_talkParticipants,     "constructType": 1,     "belongingGroup": 2,    "label": "Belonging - Trust - Talk Participants"
    },
    {
        "position": col_bel_trust_talkteam,             "constructType": 1,     "belongingGroup": 2,    "label": "Belonging - Trust - Talk Team"
    },
    {
        "position": col_bel_trust_talkMentors,          "constructType": 1,     "belongingGroup": 2,    "label": "Belonging - Trust - Talk Mentors"
    },
    {
        "position": col_bel_trust_support,              "constructType": 1,     "belongingGroup": 2,    "label": "Belonging - Trust - Support"
    },
    {
        "position": col_bel_trust_interestParticipants, "constructType": 1,     "belongingGroup": 2,    "label": "Belonging - Trust - Interest Participants"
    },
    {
        "position": col_bel_trust_interestMentors,      "constructType": 1,     "belongingGroup": 2,    "label": "Belonging - Trust - Interest Mentors"
    },
    {
        "position": col_bel_trust_interestSelf,         "constructType": 1,     "belongingGroup": 2,    "label": "Belonging - Trust - Interest Self"
    },
    
    {
        "position": col_bel_interaction_participants,   "constructType": 1,     "belongingGroup": 3,    "label": "Belonging - Interaction - Participants"
    },
    {
        "position": col_bel_interaction_team,           "constructType": 1,     "belongingGroup": 3,    "label": "Belonging - Interaction - Team"
    },
    {
        "position": col_bel_interaction_mentors,         "constructType": 1,     "belongingGroup": 3,    "label": "Belonging - Interaction - Mentors"
    },
    
    {
        "position": col_bel_expectations_competitive,       "constructType": 0,     "belongingGroup": 4,    "label": "Belonging - Expectations - Competitive"
    },
    {
        "position": col_bel_expectations_fun,               "constructType": 1,     "belongingGroup": 4,    "label": "Belonging - Expectations - Fun"
    },
    {
        "position": col_bel_expectations_goals,             "constructType": 1,     "belongingGroup": 4,    "label": "Belonging - Expectations - Goals"
    },
    {
        "position": col_bel_expectations_relavantWorkshops, "constructType": 1,     "belongingGroup": 4,    "label": "Belonging - Expectations - Relevant Workshops"
    },
    {
        "position": col_bel_expectations_apply,             "constructType": 1,     "belongingGroup": 4,    "label": "Belonging - Expectations - Apply"
    },
    {
        "position": col_bel_expectations_relavantSkills,    "constructType": 1,     "belongingGroup": 4,    "label": "Belonging - Expectations - Relevant Skills"
    },
    
    {
        "position": col_bel_other_femImp,        "constructType": 1,     "belongingGroup": 5,    "label": "Belonging - Other - Female and Non-Binary Events Important"
    },
    {
        "position": col_bel_other_overallExp,    "constructType": 3,     "belongingGroup": 5,    "label": "Belonging - Other - Overall Experience Skills"
    },
]



# Q40         What is your gender? - Selected Choice
col_demo_gender = 62

# Q40_6_TEXT  What is your gender? - Not listed:  - Text
col_demo_genderText = 63

# Q41         What is your race/ethnicity?
col_demo_raceText = 64

# Q42         What is your major?
col_demo_majorText = 65

# Q43         What is your academic year?
col_demo_year = 66

col_demographics = [
    col_demo_gender,
    col_demo_genderText,
    col_demo_raceText,
    col_demo_majorText,
    col_demo_year
]

# Q44         Please make any comments or suggestions for the hackathon or this survey here:
col_other_commments = 67



##########################################################
import json # needed for json stuff
import os # for os.walk
#import metatree 
from bs4 import BeautifulSoup
from urllib.request import Request, urlopen
import urllib.request
import datetime
import re
import csv
import numpy as np



###########################################################################
# Read csv file and Clean data

# Read
#from numpy import *
csvFileName = "data/belonging_survey_2018-03-02.csv"
reader = csv.reader(open(csvFileName, "rU"), delimiter=",")
dataRaw = []
for row in reader:
    dataRaw.append(row)

# Print raw data
#print("dataRaw: ")
#print(dataRaw)

def saveListToCsv (mylist) :
    csvfile = "data/testCSV.csv"
    with open(csvfile, "w") as output:
        writer = csv.writer(output, lineterminator='\n')
        writer.writerows(mylist)



######################
# Clean Data
#print("---------- Data Raw ----------")
#print(dataRaw[0:9])

dataClean = dataRaw

# Remove first few rows, they are extra headers and the test responses
#del dataClean[0:3]
#print("---------- Data Clean - removed first few rows ----------")
#print(dataClean[0:9])

def nestedRemove(data, value, position):
    data[:] = [surveyResponse for surveyResponse in data if not surveyResponse[position] == value]

# Remove Survey Tests
nestedRemove(dataClean, "Status", col_meta_status)
nestedRemove(dataClean, "Response Type", col_meta_status)
nestedRemove(dataClean, '{"ImportId":"status"}', col_meta_status)
nestedRemove(dataClean, "Survey Test", col_meta_status)
nestedRemove(dataClean, "Survey Preview", col_meta_status)

# Remove first two test responses
dataClean.pop(0)
dataClean.pop(0)

# Remove Unfinished surveys
# don't blank remove the false finished surveys, they may have answered all the questions, but jusst didn't click to the end of the survey
# only remove the 2% and 13% finished surveys, they really didn't complete the survey
#nestedRemove(dataClean, "FALSE", col_meta_finished)
#nestedRemove(dataClean, "83", col_meta_progress)
nestedRemove(dataClean, "2", col_meta_progress)
nestedRemove(dataClean, "13", col_meta_progress)
nestedRemove(dataClean, "30", col_meta_progress)

# Remove Unauthorized surveys
nestedRemove(dataClean, "I disagree", col_meta_agree)


# Format Data 
for i, surveyResponse in enumerate(dataClean):
    for j, s in enumerate(surveyResponse):
        if j == col_bel_other_overallExp:
            r = -100
            if s == "1 Really Bad":     r = 0
            elif s == "2":              r = 1
            elif s == "3":              r = 2
            elif s == "4":              r = 3
            elif s == "5 Really Good" : r = 4
            elif s is None or s == "":  r = -2
            dataClean[i][j] = r
        elif j == col_bel_expectations_competitive:
            r = -200
            if s == "Strongly Disagree":                        r = 4
            elif s == "Disagree":                               r = 3
            elif s == "Agree":                                  r = 2
            elif s == "Strongly Agree":                         r = 1
            elif s == "Don't Know" or s == "N/A or Don't Know": r = -1
            elif s is None or s == "":                          r = -2
            dataClean[i][j] = r
        elif (j >= col_bel_spirit_hacker and j <= col_bel_other_femImp):
            r = -300
            if s == "Strongly Disagree":                        r = 0
            elif s == "Disagree":                               r = 1
            elif s == "Agree":                                  r = 2
            elif s == "Strongly Agree":                         r = 3
            elif s == "Don't Know" or s == "N/A or Don't Know": r = -1
            elif s is None or s == "":                          r = -2
            dataClean[i][j] = r

#save to CSV!
saveListToCsv(dataClean)
    
#print("-------------------------------------------------------\n---------- Data Clean ---------- \n-------------------------------------------------------")
#print(dataClean[0:3])



##################################
# 




#############################################################################3
# Descriptive statistics

def printDictionary(listToPrint):
    print("-------------------------------------------------------------------")
    for element in listToPrint:
        print(element, ":", listToPrint[element])
        
def countCategoricalData(dataClean, position):
    categories = {}
    for surveyResponse in dataClean:
        itemResponse = surveyResponse[position]
        if itemResponse in categories:
            categories[itemResponse]+=1
        else:
            categories[itemResponse] = 1
    return categories
    
def doSimpleStats(belongingDictionary, dataClean):
    for i, belongingItem in enumerate(belongingDictionary):
        # set variables from the belonging item
        position = belongingItem["position"]
        constructType = belongingItem["constructType"]
        belongingGroup = belongingItem["belongingGroup"]
        label = belongingItem["label"]
        
        # creat variables that will be set
        sumItems = 0
        count = 0
        # loop through data, set some of the stats variables
        for surveyResponse in dataClean:
            surveyItemResponse = surveyResponse[position]
            #print(surveyItemResponse)
            if surveyItemResponse >= 0:
                sumItems += surveyItemResponse
                count += 1
        # set the stats variables with the belonging item
        belongingItem["count"] = count 
        mean = (sumItems / count)
        belongingItem["mean"] = mean 
        
        # set more stats variables
        sumvar = 0;
        # loop through the data to calc more stats variables
        for surveyResponse in dataClean:
            surveyItemResponse = surveyResponse[position]
            if surveyItemResponse >= 0:
                s = (surveyItemResponse-mean) ** 2
                sumvar += s
        variance = sumvar / (count-1)
        belongingItem["variance"] = variance


# Number of responses per hackathon
counts_attendedHackathons = countCategoricalData(dataClean, col_hack_listAttended)
#printDictionary(counts_attendedHackathons)
#print("--------------")

counts_gender = countCategoricalData(dataClean, col_demo_gender)
#printDictionary(counts_gender)
#print("--------------")


####################
# Calculate Belonging Score
# count each 
doSimpleStats(col_belonging, dataClean)
#for i, belongingItem in enumerate(col_belonging):
#    printDictionary(belongingItem)


####################
# Cronbach’s Alpha

# α = (k/k–1) * (1 – ( (∑[k,i=1]σ2i) / σ2x))
# k refers to the number of scale items
# σ2i refers to the variance associated with item i
# σ2x refers to the variance associated with the observed total scores
ca_k = len(dataClean)
print("k refers to the number of scale items: " + str(ca_k))

ca_sumvars = 0
for belongingItem in col_belonging:
    ca_sumvars += belongingItem["variance"]
print("σ2i refers to the variance associated with item i: " + str(ca_sumvars))

ca_varp = 0
responseTotals = []
for i, surveyResponse in enumerate(dataClean):
    responseTotal = 0
    for j, surveyItemResponse in enumerate(surveyResponse):
        if j >= col_bel_spirit_hacker and j <= col_bel_other_femImp:
            if surveyItemResponse >= 0:
                responseTotal += surveyItemResponse
    responseTotals.append(responseTotal)
responseTotals_count = len(responseTotals)
responseTotals_mean = sum(responseTotals)/responseTotals_count
responseTotals_sumvar = 0
for i, r in enumerate(responseTotals):
    responseTotals_sumvar += ( (r-responseTotals_mean) ** 2 )
ca_varp = responseTotals_sumvar / responseTotals_count
ca_varp1 = np.var(responseTotals)        
print("σ2x refers to the variance associated with the observed total scores: " + str(ca_varp))
print("σ2x refers to the variance associated with the observed total scores: " + str(ca_varp1))

cronbachAlpha = (ca_k/(ca_k-1)) * (1 - (ca_sumvars/ca_varp) )
print("cronbachAlpha: " + str(cronbachAlpha))








