import ast
import os
import re
import sys
import json
import math
import collections

import nbconvert
import nbformat

PASS = "PASS"
TEXT_FORMAT = "text"

Question = collections.namedtuple("Question", ["number", "weight", "format"])

questions = [
    Question(number=1, weight=1, format=TEXT_FORMAT),
    Question(number=2, weight=1, format=TEXT_FORMAT),
    Question(number=3, weight=1, format=TEXT_FORMAT),
    Question(number=4, weight=1, format=TEXT_FORMAT),
    Question(number=5, weight=1, format=TEXT_FORMAT),
    Question(number=6, weight=1, format=TEXT_FORMAT),
    Question(number=7, weight=1, format=TEXT_FORMAT),
    Question(number=8, weight=1, format=TEXT_FORMAT),
    Question(number=9, weight=1, format=TEXT_FORMAT),
    Question(number=10, weight=1, format=TEXT_FORMAT),
    Question(number=11, weight=1, format=TEXT_FORMAT),
    Question(number=12, weight=1, format=TEXT_FORMAT),
    Question(number=13, weight=1, format=TEXT_FORMAT),
    Question(number=14, weight=1, format=TEXT_FORMAT),
    Question(number=15, weight=1, format=TEXT_FORMAT),
    Question(number=16, weight=1, format=TEXT_FORMAT),
    Question(number=17, weight=1, format=TEXT_FORMAT),
    Question(number=18, weight=1, format=TEXT_FORMAT),
    Question(number=19, weight=1, format=TEXT_FORMAT),
    Question(number=20, weight=1, format=TEXT_FORMAT),
]
question_nums = set([q.number for q in questions])

# JSON and plaintext values
expected_json = {
    "1": 'C. Muñoz',
    "2": 'L. Messi',
    "3": 'Neymar Jr',
    "4": 'Brazil',
    "5": ['Argentina', 'Portugal', 'Brazil', 'Slovenia', 'Belgium'],
    "6": ['A. Abdallah', 'A. Abdellaoui', 'A. Abdennour', 'A. Abdi', 'A. Abdu'],
    "7": 2484037.64,
    "8": 181.36,
    "9": 1146,
    "10": {'Argentina': 886,
		 'Portugal': 344,
		 'Brazil': 824,
		 'Slovenia': 61,
		 'Belgium': 268,
		 'Germany': 1216,
		 'Netherlands': 416,
		 'Croatia': 126,
		 'Egypt': 30,
		 'France': 984,
		 'Senegal': 127,
		 'England': 1667,
		 'Spain': 1035,
		 'Italy': 732,
		 'Uruguay': 164,
		 'Poland': 324,
		 'Denmark': 345,
		 'Gabon': 16,
		 'Korea Republic': 322,
		 'Costa Rica': 30,
		 'Slovakia': 54,
		 'Bosnia Herzegovina': 66,
		 'Serbia': 139,
		 'Scotland': 277,
		 'Hungary': 35,
		 'Switzerland': 229,
		 'Greece': 96,
		 'Austria': 319,
		 'Morocco': 94,
		 'Sweden': 358,
		 'Wales': 117,
		 'Colombia': 591,
		 'Czech Republic': 102,
		 'Chile': 370,
		 'Algeria': 50,
		 'Ivory Coast': 105,
		 'Togo': 13,
		 'Norway': 350,
		 'Mexico': 340,
		 'Iceland': 46,
		 'Finland': 72,
		 'Jamaica': 29,
		 'Albania': 43,
		 'Guinea': 35,
		 'Cameroon': 78,
		 'Ghana': 130,
		 'Montenegro': 33,
		 'Ukraine': 69,
		 'Russia': 81,
		 'DR Congo': 54,
		 'Central African Rep.': 4,
		 'Venezuela': 66,
		 'Nigeria': 126,
		 'Armenia': 8,
		 'Israel': 16,
		 'Ecuador': 53,
		 'Paraguay': 80,
		 'Australia': 196,
		 'Turkey': 294,
		 'Romania': 287,
		 'Japan': 453,
		 'Mali': 55,
		 'United States': 347,
		 'Kosovo': 40,
		 'Dominican Republic': 4,
		 'Tanzania': 4,
		 'China PR': 373,
		 'Northern Ireland': 81,
		 'Republic of Ireland': 348,
		 'Tunisia': 35,
		 'Cape Verde': 20,
		 'FYR Macedonia': 20,
		 'Burkina Faso': 16,
		 'Kenya': 7,
		 'Angola': 16,
		 'South Africa': 72,
		 'Peru': 35,
		 'Syria': 4,
		 'Gambia': 22,
		 'New Zealand': 35,
		 'Equatorial Guinea': 6,
		 'Zimbabwe': 12,
		 'Georgia': 25,
		 'Canada': 61,
		 'Estonia': 6,
		 'Benin': 15,
		 'Bulgaria': 41,
		 'Mozambique': 4,
		 'Honduras': 13,
		 'Guinea Bissau': 21,
		 'Iran': 15,
		 'Philippines': 2,
		 'Cyprus': 11,
		 'Madagascar': 8,
		 'Uzbekistan': 3,
		 'Moldova': 12,
		 'Cuba': 4,
		 'Sierra Leone': 10,
		 'Curacao': 16,
		 'Zambia': 10,
		 'Congo': 18,
		 'Bolivia': 23,
		 'Comoros': 9,
		 'Iraq': 5,
		 'Chad': 1,
		 'Lithuania': 10,
		 'Saudi Arabia': 310,
		 'Panama': 12,
		 'Libya': 4,
		 'Bahrain': 1,
		 'St Kitts Nevis': 4,
		 'New Caledonia': 2,
		 'Luxembourg': 9,
		 'Trinidad & Tobago': 6,
		 'Thailand': 4,
		 'United Arab Emirates': 22,
		 'Eritrea': 1,
		 'Korea DPR': 4,
		 'El Salvador': 4,
		 'Azerbaijan': 6,
		 'Latvia': 6,
		 'Montserrat': 3,
		 'Puerto Rico': 1,
		 'Bermuda': 3,
		 'São Tomé & Príncipe': 1,
		 'Antigua & Barbuda': 7,
		 'Burundi': 4,
		 'Kazakhstan': 2,
		 'Liberia': 1,
		 'Guyana': 4,
		 'Haiti': 7,
		 'Jordan': 1,
		 'Faroe Islands': 5,
		 'Mauritania': 5,
		 'Namibia': 2,
		 'Rwanda': 2,
		 'Uganda': 3,
		 'Hong Kong': 1,
		 'Chinese Taipei': 1,
		 'Belize': 1,
		 'Palestine': 4,
		 'Mauritius': 1,
		 'Guam': 1,
		 'Suriname': 2,
		 'Lebanon': 3,
		 'Guatemala': 2,
		 'Sudan': 3,
		 'Liechtenstein': 2,
		 'Grenada': 2,
		 'St Lucia': 1,
		 'Afghanistan': 2,
		 'Ethiopia': 1,
		 'Barbados': 1,
		 'India': 23,
		 'Malta': 2,
		 'Niger': 3,
		 'Vietnam': 1,
		 'Malawi': 1,
		 'Gibraltar': 1,
		 'Macau': 1,
		 'South Sudan': 1,
		 'Indonesia': 1},
    "11": 'England',
    "12": 886,
    "13": {'ID': '242444',
		 'Name': 'João Félix',
		 'Age': 19,
		 'Height(cm)': 181,
		 'Weight(kg)': 70,
		 'Nationality': 'Portugal',
		 'Club': 'Atlético Madrid',
		 'Value': 28000000,
		 'Wage': 38000,
		 'Player_Position': 'CF',
		 'Preferred_Foot': 'Right',
		 'Body_Type': 'Lean',
		 'Stamina': 79},
    "14": {'ID': '211300',
		 'Name': 'A. Martial',
		 'Age': 23,
		 'Height(cm)': 184,
		 'Weight(kg)': 76,
		 'Nationality': 'France',
		 'Club': 'Manchester United',
		 'Value': 34500000,
		 'Wage': 140000,
		 'Player_Position': 'LW',
		 'Preferred_Foot': 'Right',
		 'Body_Type': 'Normal',
		 'Stamina': 75},
    "15": {'ID': '198717',
		 'Name': 'W. Zaha',
		 'Age': 26,
		 'Height(cm)': 180,
		 'Weight(kg)': 66,
		 'Nationality': 'Ivory Coast',
		 'Club': 'Crystal Palace',
		 'Value': 32000000,
		 'Wage': 89000,
		 'Player_Position': 'CF',
		 'Preferred_Foot': 'Right',
		 'Body_Type': 'Lean',
		 'Stamina': 76},
    "16": {'ID': '200536',
		 'Name': 'N. Schulz',
		 'Age': 26,
		 'Height(cm)': 181,
		 'Weight(kg)': 83,
		 'Nationality': 'Germany',
		 'Club': 'Borussia Dortmund',
		 'Value': 22500000,
		 'Wage': 82000,
		 'Player_Position': 'LWB',
		 'Preferred_Foot': 'Left',
		 'Body_Type': 'Normal',
		 'Stamina': 79},
    "17": {'Messi': 1,
		 'C. Ronaldo': 1,
		 'Neymar': 1,
		 'Normal': 10750,
		 'Lean': 6505,
		 'PLAYER_BODY_TYPE_25': 1,
		 'Stocky': 1016,
		 'Courtois': 1,
		 'Shaqiri': 1,
		 'Akinfenwa': 1},
    "18": 4318,
    "19": 70.61,
    "20": {'FC Barcelona': 150000.0,
		 'Juventus': 113636.36,
		 'Paris Saint-Germain': 72606.06,
		 'Atlético Madrid': 44848.48,
		 'Real Madrid': 162242.42,
		 'Manchester City': 120727.27,
		 'Liverpool': 80818.18,
		 'Napoli': 59838.71,
		 'Tottenham Hotspur': 78878.79,
		 'Manchester United': 87090.91,
		 'Chelsea': 85030.3,
		 'FC Bayern München': 109391.3,
		 'Inter': 53733.33,
		 'Borussia Dortmund': 57806.45,
		 'Arsenal': 61393.94,
		 'Valencia CF': 29363.64,
		 'Lazio': 40454.55,
		 'Milan': 32482.76,
		 'Sporting CP': 11233.33,
		 'Olympique Lyonnais': 43655.17,
		 'RB Leipzig': 33636.36,
		 'Ajax': 15000.0,
		 'LA Galaxy': 4185.19,
		 'Atalanta': 38392.86,
		 'RC Celta': 14575.76,
		 'Bayer 04 Leverkusen': 50240.0,
		 'Real Betis': 18774.19,
		 'FC Porto': 11633.33,
		 'SV Werder Bremen': 18312.5,
		 'West Ham United': 52939.39,
		 'Wolverhampton Wanderers': 41969.7,
		 'AS Saint-Étienne': 17875.0,
		 'Torino': 28793.1,
		 'Dalian YiFang FC': 4928.57,
		 'Borussia Mönchengladbach': 20121.21,
		 'Roma': 31516.13,
		 'Guangzhou Evergrande Taobao FC': 10000.0,
		 'SL Benfica': 11833.33,
		 'Medipol Başakşehir FK': 27066.67,
		 'Everton': 56606.06,
		 'VfL Wolfsburg': 31424.24,
		 'Crystal Palace': 33939.39,
		 'Getafe CF': 18600.0,
		 'Shanghai SIPG FC': 6892.86,
		 'Eintracht Frankfurt': 24363.64,
		 'Olympique de Marseille': 23592.59,
		 'Hertha BSC': 17151.52,
		 'RSC Anderlecht': 12000.0,
		 'Villarreal CF': 21937.5,
		 'Sampdoria': 18781.25,
		 'Leicester City': 53000.0,
		 'AS Monaco': 24818.18,
		 'Jiangsu Suning FC': 7520.0,
		 'Los Angeles FC': 4217.39,
		 'Cagliari': 20322.58,
		 'Sevilla FC': 14939.39,
		 'Fenerbahçe SK': 38862.07,
		 'Real Sociedad': 18433.33,
		 'TSG 1899 Hoffenheim': 28266.67,
		 'Atlético Mineiro': 24750.0,
		 'Grêmio': 20800.0,
		 'PSV': 12233.33,
		 'Athletic Club de Bilbao': 17848.48,
		 'Deportivo Alavés': 15424.24,
		 'Boca Juniors': 16285.71,
		 'Lokomotiv Moscow': 1000.0,
		 'Al Nassr': 15133.33,
		 'Brescia': 7464.29,
		 'Shakhtar Donetsk': 1000.0,
		 'Shanghai Greenland Shenhua FC': 5571.43,
		 'PFC CSKA Moscow': 1000.0,
		 'Beijing Sinobo Guoan FC': 7538.46,
		 'Levante UD': 17531.25,
		 'Cruzeiro': 17050.0,
		 'Uruguay': 0.0,
		 'Montpellier HSC': 18222.22,
		 'Atlanta United': 3833.33,
		 'Watford': 47787.88,
		 '1. FC Köln': 17387.1,
		 'Bournemouth': 38060.61,
		 'Beşiktaş JK': 38433.33,
		 'Real Valladolid CF': 12151.52,
		 'Racing Club': 13928.57,
		 'Al Hilal': 20068.97,
		 'Guangzhou R&F FC': 5407.41,
		 'Sassuolo': 27392.86,
		 'FC Girondins de Bordeaux': 18781.25,
		 'LOSC Lille': 19483.87,
		 'Galatasaray SK': 41433.33,
		 'Chicago Fire': 3740.74,
		 'Fluminense': 16450.0,
		 'Ecuador': 0.0,
		 'RCD Espanyol': 18400.0,
		 'Dinamo Zagreb': 1000.0,
		 'FC Nantes': 12909.09,
		 'River Plate': 15785.71,
		 'OGC Nice': 16034.48,
		 'Newcastle United': 29393.94,
		 'Brighton & Hove Albion': 32060.61,
		 'Club Brugge KV': 14888.89,
		 'FC Schalke 04': 19600.0,
		 'SD Eibar': 18642.86,
		 'DC United': 3962.96,
		 'Orlando City SC': 3214.29,
		 'Hebei China Fortune FC': 4730.77,
		 'Tigres U.A.N.L.': 39769.23,
		 'Aston Villa': 28939.39,
		 'Montreal Impact': 3153.85,
		 'Olympiacos CFP': 1000.0,
		 'Norwich City': 23333.33,
		 'Feyenoord': 8100.0,
		 'Toronto FC': 3466.67,
		 'KRC Genk': 9321.43,
		 'Fiorentina': 26566.67,
		 'Spartak Moscow': 1000.0,
		 'Dynamo Kyiv': 1000.0,
		 'SK Slavia Praha': 1000.0,
		 'Southampton': 33272.73,
		 'Burnley': 23939.39,
		 'SC Braga': 11392.86,
		 'Russia': 0.0,
		 'RC Strasbourg Alsace': 15642.86,
		 'Wuhan Zall': 3250.0,
		 'Vissel Kobe': 3310.34,
		 'Portland Timbers': 3777.78,
		 'Genoa': 10656.25,
		 'Beijing Renhe FC': 3607.14,
		 'Toulouse Football Club': 13206.9,
		 'Girona FC': 6960.0,
		 'Real Zaragoza': 4241.38,
		 'CD Leganés': 16151.52,
		 'Shenzhen FC': 3851.85,
		 'Internacional': 18150.0,
		 'CSA - AL': 7900.0,
		 'Santos': 18550.0,
		 '1. FSV Mainz 05': 14515.15,
		 'Stoke City': 20533.33,
		 'Udinese': 11060.61,
		 'Colombia': 0.0,
		 'Angers SCO': 12290.32,
		 'FC Augsburg': 13515.15,
		 'Netherlands': 0.0,
		 'Fulham': 24266.67,
		 'FC København': 9888.89,
		 'KAA Gent': 12535.71,
		 'SC Freiburg': 14161.29,
		 'Stade Rennais FC': 24869.57,
		 'Club América': 29681.82,
		 'Trabzonspor': 12466.67,
		 'BSC Young Boys': 10037.04,
		 'Helsingborgs IF': 1259.26,
		 'Kaizer Chiefs': 1000.0,
		 'Parma': 22272.73,
		 'Mexico': 0.0,
		 'Royal Antwerp FC': 6535.71,
		 'Tianjin TEDA FC': 4678.57,
		 'Hannover 96': 15214.29,
		 'Tianjin Quanjian FC': 5962.96,
		 'Al Ahli': 14866.67,
		 'Bologna': 20387.1,
		 'VfB Stuttgart': 13200.0,
		 'Seattle Sounders FC': 4074.07,
		 'Godoy Cruz': 3035.71,
		 'Sparta Praha': 1000.0,
		 'Independiente Medellín': 1928.57,
		 'Sivasspor': 5923.08,
		 'Independiente': 13214.29,
		 'Nîmes Olympique': 9782.61,
		 'Club Tijuana': 9142.86,
		 'SPAL': 8740.74,
		 'Monterrey': 23344.83,
		 'CD Tondela': 2925.93,
		 'Fortuna Düsseldorf': 15212.12,
		 'Vitória Guimarães': 6433.33,
		 'AZ Alkmaar': 5481.48,
		 'Atlético de San Luis': 5111.11,
		 'PAOK': 1000.0,
		 'Nagoya Grampus': 2655.17,
		 'Club Atlético Banfield': 6678.57,
		 'Shandong Luneng TaiShan FC': 6000.0,
		 'New York Red Bulls': 3827.59,
		 'FC Red Bull Salzburg': 16000.0,
		 'Sweden': 0.0,
		 'Amiens SC': 9562.5,
		 'Hellas Verona': 9484.85,
		 'Cruz Azul': 13571.43,
		 'Gençlerbirliği SK': 7541.67,
		 '1. FC Union Berlin': 17606.06,
		 'Standard de Liège': 9785.71,
		 'Chongqing Dangdai Lifan FC SWM Team': 2720.0,
		 'New England Revolution': 3423.08,
		 'Club Atlético Colón': 7214.29,
		 'Celtic': 25678.57,
		 'Club Atlas': 7892.86,
		 'Botafogo': 12550.0,
		 'En Avant de Guingamp': 3966.67,
		 'West Bromwich Albion': 19366.67,
		 'Pachuca': 11560.0,
		 'AEK Athens': 1000.0,
		 'Portimonense SC': 3966.67,
		 'Real Salt Lake': 3034.48,
		 'FC Utrecht': 8703.7,
		 'Sheffield United': 17909.09,
		 "Newell's Old Boys": 7928.57,
		 'Club Atlético Talleres': 5392.86,
		 'Philadelphia Union': 3740.74,
		 'Rosenborg BK': 3111.11,
		 'FC Basel 1893': 11629.63,
		 'Brentford': 14866.67,
		 'Club León': 13964.29,
		 'Unión de Santa Fe': 6120.0,
		 'Deportivo de La Coruña': 5916.67,
		 'Rangers FC': 25892.86,
		 'Turkey': 0.0,
		 'Atiker Konyaspor': 8481.48,
		 'Granada CF': 11444.44,
		 'Perth Glory': 2478.26,
		 'Club Atlético Lanús': 9214.29,
		 'Hamburger SV': 7800.0,
		 'Al Ittihad': 13833.33,
		 'Santos Laguna': 10320.0,
		 'Western United FC': 1300.0,
		 'Columbus Crew SC': 3607.14,
		 'Deportivo Toluca': 11760.0,
		 'Cardiff City': 14700.0,
		 'CA Osasuna': 16166.67,
		 'Swansea City': 11833.33,
		 'Melbourne Victory': 3000.0,
		 'Leeds United': 25166.67,
		 'Göztepe SK': 10178.57,
		 'Hungary': 0.0,
		 'New York City FC': 4000.0,
		 'Bulgaria': 0.0,
		 'Kayserispor': 5758.62,
		 'Minnesota United FC': 3791.67,
		 'Guadalajara': 16148.15,
		 'FC Groningen': 3333.33,
		 'Paraguay': 0.0,
		 'Junior FC': 2392.86,
		 'Al Taawoun': 7653.85,
		 'Huddersfield Town': 14100.0,
		 'Ettifaq FC': 8037.04,
		 'Stade de Reims': 13153.85,
		 'Rayo Vallecano': 4333.33,
		 'San Lorenzo de Almagro': 11428.57,
		 'Bahia': 7400.0,
		 'Atlético Paranaense': 7200.0,
		 'Goiás': 7400.0,
		 'Avaí FC': 8050.0,
		 'Fortaleza': 5700.0,
		 'Kawasaki Frontale': 4172.41,
		 'Vélez Sarsfield': 7571.43,
		 'Hull City': 5300.0,
		 'Houston Dynamo': 2965.52,
		 'Birmingham City': 6300.0,
		 'Al Wehda': 7964.29,
		 'Nottingham Forest': 23500.0,
		 'CD Tenerife': 3900.0,
		 'Aalborg BK': 4041.67,
		 'Preston North End': 11000.0,
		 'Bristol City': 15866.67,
		 'Lecce': 7575.76,
		 'Gimnasia y Esgrima La Plata': 5392.86,
		 'CD Aves': 4440.0,
		 'Viktoria Plzeň': 1000.0,
		 '1. FC Nürnberg': 7200.0,
		 'Slovenia': 0.0,
		 'FC Metz': 8296.3,
		 'FC Midtjylland': 8296.3,
		 'Molde FK': 2814.81,
		 'Colo-Colo': 4160.0,
		 'RCD Mallorca': 12185.19,
		 'FC Sion': 9185.19,
		 'Wisła Kraków': 2000.0,
		 'Denizlispor': 9913.04,
		 'Middlesbrough': 11366.67,
		 'Universidad Católica': 6107.14,
		 'Alanyaspor': 7862.07,
		 'Moreirense FC': 6192.31,
		 "Côte d'Ivoire": 0.0,
		 'Stade Brestois 29': 8960.0,
		 'Chievo Verona': 2400.0,
		 'Boavista FC': 3482.76,
		 'MKE Ankaragücü': 8952.38,
		 'Sydney FC': 3666.67,
		 'Al Ain FC': 1000.0,
		 'Yeni Malatyaspor': 6321.43,
		 'Urawa Red Diamonds': 4733.33,
		 'Querétaro': 4142.86,
		 'Sporting Kansas City': 3964.29,
		 'Málaga CF': 5600.0,
		 '1. FC Heidenheim 1846': 5206.9,
		 'Atlético Tucumán': 5481.48,
		 'Gazişehir Gaziantep F.K.': 6533.33,
		 'Clube Sport Marítimo': 4655.17,
		 'Chapecoense': 8100.0,
		 'Atlético Nacional': 2571.43,
		 'Vitesse': 8153.85,
		 'Australia': 0.0,
		 'Henan Jianye FC': 3444.44,
		 'Panathinaikos FC': 1000.0,
		 'Blackburn Rovers': 7433.33,
		 'Santa Clara': 3148.15,
		 'Cameroon': 0.0,
		 'Puebla FC': 3851.85,
		 'U.N.A.M.': 9120.0,
		 'SD Huesca': 4038.46,
		 'Estudiantes de La Plata': 6307.69,
		 'Austria': 0.0,
		 'KAS Eupen': 4222.22,
		 'LASK Linz': 4000.0,
		 'Sporting de Charleroi': 6074.07,
		 'Daegu FC': 1750.0,
		 'Real Sporting de Gijón': 4000.0,
		 'Derby County': 11633.33,
		 'Rio Ave FC': 4592.59,
		 'South Africa': 0.0,
		 'Famalicão': 3916.67,
		 'Neuchâtel Xamax': 1615.38,
		 'Benevento': 2913.04,
		 'Cádiz CF': 5233.33,
		 'AIK': 4000.0,
		 'Sheffield Wednesday': 12166.67,
		 'Empoli': 2466.67,
		 'Colorado Rapids': 2518.52,
		 'Os Belenenses': 3233.33,
		 'Unión Magdalena': 1035.71,
		 'Real Oviedo': 5200.0,
		 'Peru': 0.0,
		 'Antalyaspor': 6785.71,
		 'Dijon FCO': 6166.67,
		 'SV Sandhausen': 2833.33,
		 'Rosario Central': 8923.08,
		 'Reading': 8666.67,
		 'DSC Arminia Bielefeld': 6666.67,
		 'Malmö FF': 4000.0,
		 'Jeonbuk Hyundai Motors': 4714.29,
		 'Frosinone': 2480.0,
		 'FC Tokyo': 3333.33,
		 'Canada': 0.0,
		 'Çaykur Rizespor': 6478.26,
		 'FCSB (Steaua)': 6333.33,
		 'Defensa y Justicia': 6142.86,
		 'Monarcas Morelia': 4392.86,
		 'Club Atlético Huracán': 7200.0,
		 'Ceará Sporting Club': 5400.0,
		 'Argentinos Juniors': 4428.57,
		 'Al Shabab': 10033.33,
		 'Legia Warszawa': 4555.56,
		 'Shimizu S-Pulse': 1533.33,
		 'Millonarios FC': 1714.29,
		 'Lechia Gdańsk': 3148.15,
		 'Brøndby IF': 6481.48,
		 'Albacete BP': 3964.29,
		 'FC Lorient': 3275.86,
		 'Universitatea Craiova': 3592.59,
		 'Deportivo Cali': 1392.86,
		 'SK Rapid Wien': 6640.0,
		 'Kashima Antlers': 3233.33,
		 'Poland': 0.0,
		 'Elche CF': 3238.1,
		 'Club Atlético Aldosivi': 4423.08,
		 'Deportes Tolima': 1821.43,
		 'Cúcuta Deportivo': 1035.71,
		 'Club Necaxa': 4357.14,
		 'Piast Gliwice': 2185.19,
		 'Pescara': 1833.33,
		 'Kasimpaşa SK': 6148.15,
		 'Egypt': 0.0,
		 'Holstein Kiel': 4714.29,
		 'Livorno': 1333.33,
		 'Coquimbo Unido': 1461.54,
		 'UD Las Palmas': 4433.33,
		 'Górnik Zabrze': 1444.44,
		 'FC Twente': 3692.31,
		 'Paris FC': 2615.38,
		 'Racing Club de Lens': 3633.33,
		 'VfL Bochum 1848': 6344.83,
		 'Sunderland': 4357.14,
		 'Aberdeen': 2807.69,
		 'Heart of Midlothian': 3107.14,
		 'Romania': 0.0,
		 'Crotone': 1653.85,
		 'Iceland': 0.0,
		 'CFR Cluj': 5703.7,
		 'FK Austria Wien': 5407.41,
		 'UD Almería': 2785.71,
		 'SK Brann': 2304.35,
		 'BK Häcken': 1592.59,
		 'Al Fateh': 6464.29,
		 'CD Everton de Viña del Mar': 1538.46,
		 'FC St. Pauli': 6666.67,
		 'Gamba Osaka': 4000.0,
		 'Yokohama F. Marinos': 2222.22,
		 'Djurgårdens IF': 2115.38,
		 'FC Dallas': 2785.71,
		 'Sagan Tosu': 1733.33,
		 'Al Fayha': 5384.62,
		 'Chile': 0.0,
		 'Once Caldas': 1285.71,
		 'Atlético Bucaramanga': 1071.43,
		 'FC Cincinnati': 2931.03,
		 'San Jose Earthquakes': 3103.45,
		 'América de Cali': 1214.29,
		 'Perugia': 1923.08,
		 'La Equidad': 1035.71,
		 'SV Darmstadt 98': 7366.67,
		 'FC Zürich': 3884.62,
		 'SC Paderborn 07': 8272.73,
		 'SV Zulte-Waregem': 5370.37,
		 'IFK Norrköping': 2807.69,
		 'FC Viitorul': 2703.7,
		 'Pordenone': 1240.0,
		 'AD Alcorcón': 4440.0,
		 'Sint-Truidense VV': 3760.0,
		 'KV Kortrijk': 5384.62,
		 'Royal Excel Mouscron': 3214.29,
		 'FC Lugano': 2444.44,
		 'Hokkaido Consadole Sapporo': 1958.33,
		 'Spezia': 1533.33,
		 'FC Paços de Ferreira': 3714.29,
		 'Servette FC': 3000.0,
		 'US Salernitana 1919': 1766.67,
		 'Lech Poznań': 3074.07,
		 'ESTAC Troyes': 2821.43,
		 'Fortuna Sittard': 2760.0,
		 'KV Oostende': 6307.69,
		 'VVV-Venlo': 2370.37,
		 'KSV Cercle Brugge': 4111.11,
		 'FC Seoul': 2821.43,
		 'Cerezo Osaka': 2866.67,
		 'Stade Malherbe Caen': 3520.0,
		 'Universidad de Chile': 5038.46,
		 'Kalmar FF': 1222.22,
		 'KV Mechelen': 4464.29,
		 'Deportes Iquique': 1269.23,
		 'US Cremonese': 1916.67,
		 'Bolivia': 0.0,
		 'Valenciennes FC': 1851.85,
		 'Ulsan Hyundai FC': 3535.71,
		 'Vegalta Sendai': 1413.79,
		 'ADO Den Haag': 2880.0,
		 'CD Palestino': 1520.0,
		 'Cittadella': 1346.15,
		 'Vitória de Setúbal': 2833.33,
		 'FC Nordsjælland': 2653.85,
		 'Charlton Athletic': 5034.48,
		 'Al Raed': 5318.18,
		 'Jagiellonia Białystok': 3259.26,
		 'Odense Boldklub': 4181.82,
		 'FC Thun': 3037.04,
		 'SG Dynamo Dresden': 6333.33,
		 'Hammarby IF': 2115.38,
		 'Central Córdoba': 3142.86,
		 'Queens Park Rangers': 3500.0,
		 ' SSV Jahn Regensburg': 4120.0,
		 'Tiburones Rojos de Veracruz': 2888.89,
		 'Patronato': 4821.43,
		 'Virtus Entella': 1100.0,
		 'SK Sturm Graz': 7840.0,
		 'Kilmarnock': 2363.64,
		 'Extremadura UD': 2576.92,
		 'Willem II': 3000.0,
		 'Gil Vicente FC': 3038.46,
		 'Randers FC': 2640.0,
		 'Ascoli': 1620.69,
		 'Vancouver Whitecaps FC': 3571.43,
		 'Millwall': 5100.0,
		 'Pohang Steelers': 2178.57,
		 'Suwon Samsung Bluewings': 2642.86,
		 'Heracles Almelo': 2259.26,
		 'Al Hazem': 3222.22,
		 'FC Juárez': 5461.54,
		 'SC Heerenveen': 2521.74,
		 'Dinamo Bucureşti': 2814.81,
		 'Gyeongnam FC': 1714.29,
		 'KFC Uerdingen 05': 2785.71,
		 'PEC Zwolle': 3370.37,
		 'Al Faisaly': 7050.0,
		 'Arsenal de Sarandí': 3750.0,
		 'FC Ingolstadt 04': 2423.08,
		 'Wolfsberger AC': 3423.08,
		 'Wigan Athletic': 6033.33,
		 'Júbilo Iwata': 2133.33,
		 'CD Lugo': 2862.07,
		 'SpVgg Greuther Fürth': 3344.83,
		 'FC Emmen': 1857.14,
		 'Cosenza': 1454.55,
		 'AJ Auxerre': 2466.67,
		 'Cracovia': 2074.07,
		 'FC Erzgebirge Aue': 2689.66,
		 'Grenoble Foot 38': 2000.0,
		 'Korona Kielce': 1518.52,
		 'Alianza Petrolera': 1142.86,
		 'CD Universidad de Concepción': 1461.54,
		 'Melbourne City FC': 3380.95,
		 'CD Antofagasta': 1500.0,
		 'Independiente Santa Fe': 1071.43,
		 'Sanfrecce Hiroshima': 4041.67,
		 'Damac FC': 4875.0,
		 'Chamois Niortais Football Club': 1777.78,
		 'Atlético Huila': 1000.0,
		 'CD Numancia': 3518.52,
		 'Luton Town': 5333.33,
		 'Unión La Calera': 1375.0,
		 'Le Havre AC': 2384.62,
		 'AS Nancy Lorraine': 1966.67,
		 'Audax Italiano': 1833.33,
		 'Hibernian': 2875.0,
		 'Barnsley': 3133.33,
		 'Pogoń Szczecin': 2000.0,
		 'Sarpsborg 08 FF': 1592.59,
		 'Kristiansund BK': 1238.1,
		 'FC Luzern': 3240.0,
		 'Unión Española': 1814.81,
		 'IFK Göteborg': 1608.7,
		 'Envigado FC': 1000.0,
		 'Jeju United FC': 1928.57,
		 'Patriotas Boyacá FC': 1000.0,
		 'Peterborough United': 3071.43,
		 'Deportivo Pasto': 1035.71,
		 'Wales': 0.0,
		 'Clermont Foot 63': 1851.85,
		 'Northern Ireland': 0.0,
		 'Astra Giurgiu': 2703.7,
		 'Racing Santander': 2692.31,
		 'Pisa': 1133.33,
		 'Newcastle Jets': 1181.82,
		 'Castellammare di Stabia': 1250.0,
		 'Rionegro Águilas': 1000.0,
		 'Vålerenga Fotball': 1666.67,
		 "CD O'Higgins": 1925.93,
		 'AC Ajaccio': 1583.33,
		 'CF Fuenlabrada': 1862.07,
		 'VfL Osnabrück': 4500.0,
		 'Arka Gdynia': 1555.56,
		 'FC Sochaux-Montbéliard': 1533.33,
		 'SpVgg Unterhaching': 1607.14,
		 'Central Coast Mariners': 1100.0,
		 'SV Wehen Wiesbaden': 4266.67,
		 'Fleetwood Town': 2392.86,
		 'Venezia FC': 1344.83,
		 'Karlsruher SC': 4321.43,
		 'Wisła Płock': 1592.59,
		 'Gangwon FC': 1500.0,
		 'IF Elfsborg': 1521.74,
		 'Sangju Sangmu FC': 1285.71,
		 'Zagłębie Lubin': 1560.0,
		 'Abha Club': 4285.71,
		 'Jaguares de Córdoba': 1000.0,
		 'FC Botoşani': 2444.44,
		 'Motherwell': 1833.33,
		 'FC St. Gallen': 3037.04,
		 'Bayern München II': 1250.0,
		 'CD Mirandés': 2476.19,
		 'St. Johnstone FC': 1833.33,
		 'Lillestrøm SK': 1347.83,
		 'Odds BK': 1545.45,
		 'SD Ponferradina': 2615.38,
		 'Doncaster Rovers': 2250.0,
		 'Örebro SK': 1407.41,
		 'Shamrock Rovers': 1045.45,
		 'FSV Zwickau': 1000.0,
		 '1. FC Kaiserslautern': 1607.14,
		 'Portsmouth': 3321.43,
		 'Waasland-Beveren': 2846.15,
		 'Ipswich Town': 4000.0,
		 'Dundalk': 1086.96,
		 'CD Huachipato': 2071.43,
		 'Le Mans FC': 1416.67,
		 'Strømsgodset IF': 1259.26,
		 'Sepsi OSK': 2888.89,
		 'Sparta Rotterdam': 2809.52,
		 'FC Admira Wacker Mödling': 2769.23,
		 'SønderjyskE': 3333.33,
		 'Hallescher FC': 1500.0,
		 'SKN St. Pölten': 3625.0,
		 'MSV Duisburg': 1125.0,
		 'La Berrichonne de Châteauroux': 1615.38,
		 'FC Hansa Rostock': 1400.0,
		 'Incheon United FC': 1214.29,
		 'Shonan Bellmare': 2000.0,
		 'Orlando Pirates': 1000.0,
		 'FK Bodø/Glimt': 1148.15,
		 'Lincoln City': 3000.0,
		 'Brisbane Roar': 1291.67,
		 'Eintracht Braunschweig': 1615.38,
		 'Curicó Unido': 1259.26,
		 'Falkenbergs FF': 1041.67,
		 'Venezuela': 0.0,
		 '1. FC Magdeburg': 1916.67,
		 'CD Cobresal': 1107.14,
		 'Aarhus GF': 3160.0,
		 'Salford City': 3285.71,
		 'HJK Helsinki': 1000.0,
		 'Oxford United': 2428.57,
		 'Hobro IK': 2375.0,
		 'Esbjerg fB': 2791.67,
		 'Bolton Wanderers': 1217.39,
		 'SV Mattersburg': 3920.0,
		 'Rotherham United': 3090.91,
		 'SCR Altach': 2629.63,
		 'Gaz Metan Mediaş': 2259.26,
		 'Tromsø IL': 1074.07,
		 'WSG Tirol': 2360.0,
		 'IK Sirius': 1185.19,
		 'Shrewsbury': 2142.86,
		 'Oldham Athletic': 2285.71,
		 'Southend United': 2321.43,
		 'Blackpool': 2285.71,
		 'Coventry City': 2214.29,
		 'Adelaide United': 1190.48,
		 'Ranheim Fotball': 1043.48,
		 'SG Sonnenhof Großaspach': 1280.0,
		 'Matsumoto Yamaga': 1233.33,
		 'Burton Albion': 2115.38,
		 'FC Hermannstadt': 2037.04,
		 'TSV 1860 München': 1000.0,
		 'LKS Lodz': 1111.11,
		 'FC Würzburger Kickers': 1571.43,
		 'TSV Hartberg': 1769.23,
		 'Politehnica Iaşi': 1740.74,
		 'Wellington Phoenix': 1000.0,
		 'Milton Keynes Dons': 2384.62,
		 'Stabæk Fotball': 1074.07,
		 'Wycombe Wanderers': 1875.0,
		 'SV Waldhof Mannheim': 1000.0,
		 'AC Horsens': 2238.1,
		 'Scunthorpe United': 3357.14,
		 'RKC Waalwijk': 1680.0,
		 'Oita Trinita': 1300.0,
		 'St. Mirren': 1565.22,
		 'Bristol Rovers': 2071.43,
		 'Rodez Aveyron Football': 1269.23,
		 'SV Meppen': 1000.0,
		 'Viking FK': 1160.0,
		 'Östersunds FK': 1259.26,
		 'FK Haugesund': 1037.04,
		 'Rochdale': 1240.0,
		 'Colchester United': 2892.86,
		 'Trapani': 1000.0,
		 'Stevenage': 2035.71,
		 'Bradford City': 3480.0,
		 'Livingston FC': 1607.14,
		 'FC Chambly Oise': 1366.67,
		 'Mansfield Town': 3678.57,
		 'SC Preußen Münster': 1250.0,
		 'FC Voluntari': 1666.67,
		 'Accrington Stanley': 1214.29,
		 'Al Adalah': 3640.0,
		 'Lyngby BK': 1958.33,
		 'FC Carl Zeiss Jena': 1000.0,
		 'Viktoria Köln': 1107.14,
		 'Tranmere Rovers': 1714.29,
		 'Silkeborg IF': 1814.81,
		 'Gillingham': 2000.0,
		 'Plymouth Argyle': 3440.0,
		 'Chemnitzer FC': 1000.0,
		 'Mjøndalen IF': 1090.91,
		 'Walsall': 3115.38,
		 'Northampton Town': 3000.0,
		 'Hamilton Academical FC': 1285.71,
		 'Grimsby Town': 1642.86,
		 'Exeter City': 2464.29,
		 'Swindon Town': 2535.71,
		 'Raków Częstochowa': 1041.67,
		 'Chindia Târgovişte': 1304.35,
		 'US Orléans Loiret Football': 1000.0,
		 'Forest Green Rovers': 3375.0,
		 'AFC Wimbledon': 1464.29,
		 'Carlisle United': 1920.0,
		 'Morecambe': 1629.63,
		 'Port Vale': 2360.0,
		 'Cheltenham Town': 1958.33,
		 'Academica Clinceni': 1076.92,
		 'Crawley Town': 1714.29,
		 'Ross County FC': 1520.0,
		 'AFC Eskilstuna': 1000.0,
		 'Macclesfield Town': 1541.67,
		 'Cork City': 1000.0,
		 'Newport County': 2038.46,
		 'Crewe Alexandra': 1960.0,
		 'Leyton Orient': 2222.22,
		 'Cambridge United': 1875.0,
		 "St. Patrick's Athletic": 1000.0,
		 'Bohemian FC': 1000.0,
		 'India': 0.0,
		 'Finland': 0.0,
		 'Waterford FC': 1000.0,
		 'Derry City': 1000.0,
		 'Bury': 1000.0,
		 'New Zealand': 0.0,
		 'GIF Sundsvall': 1000.0,
		 'Sligo Rovers': 1000.0,
		 'Finn Harps': 1000.0,
		 'Seongnam FC': 1000.0,
		 'UCD AFC': 1000.0,
		 'Śląsk Wrocław': 1000.0},
}

# find a comment something like this: #q10
def extract_question_num(cell):
    for line in cell.get('source', []):
        line = line.strip().replace(' ', '').lower()
        m = re.match(r'\#q(\d+)', line)
        if m:
            return int(m.group(1))
    return None


# rerun notebook and return parsed JSON
def rerun_notebook(orig_notebook):
    new_notebook = 'cs-220-test.ipynb'

    # re-execute it from the beginning
    with open(orig_notebook, encoding='utf-8') as f:
        nb = nbformat.read(f, as_version=nbformat.NO_CONVERT)
    ep = nbconvert.preprocessors.ExecutePreprocessor(timeout=120, kernel_name='python3')
    try:
        out = ep.preprocess(nb, {'metadata': {'path': os.getcwd()}})
    except nbconvert.preprocessors.CellExecutionError:
        out = None
        msg = 'Error executing the notebook "%s".\n\n' % orig_notebook
        msg += 'See notebook "%s" for the traceback.' % new_notebook
        print(msg)
        raise
    finally:
        with open(new_notebook, mode='w', encoding='utf-8') as f:
            nbformat.write(nb, f)

    # Note: Here we are saving and reloading, this isn't needed but can help student's debug

    # parse notebook
    with open(new_notebook, encoding='utf-8') as f:
        nb = json.load(f)
    return nb


def normalize_json(orig):
    try:
        return json.dumps(json.loads(orig.strip("'")), indent=2, sort_keys=True)
    except:
        return 'not JSON'


def check_cell_text(qnum, cell):
    outputs = cell.get('outputs', [])
    if len(outputs) == 0:
        return 'no outputs in an Out[N] cell'
    actual_lines = None
    for out in outputs:
        lines = out.get('data', {}).get('text/plain', [])
        if lines:
            actual_lines = lines
            break
    if actual_lines == None:
        return 'no Out[N] output found for cell (note: printing the output does not work)'
    actual = ''.join(actual_lines)
    actual = ast.literal_eval(actual)
    expected = expected_json[str(qnum)]

    expected_mismatch = False

    if type(expected) != type(actual):
        return "expected an answer of type %s but found one of type %s" % (type(expected), type(actual))
    elif type(expected) == float:
        if not math.isclose(actual, expected, rel_tol=1e-06, abs_tol=1e-06):
            expected_mismatch = True
    elif type(expected) == list:
        extra = set(actual) - set(expected)
        missing = set(expected) - set(actual)
        if extra:
            return "found unexpected entry in list: %s" % repr(list(extra)[0])
        elif missing:
            return "missing %d entries list, such as: %s" % (len(missing), repr(list(missing)[0]))
        elif len(actual) != len(expected):
            return "expected %d entries in the list but found %d" % (len(expected), len(actual))
    else:
        if expected != actual:
            expected_mismatch = True

    if expected_mismatch:
        return "found {} in cell {} but expected {}".format(actual, qnum, expected)

    return PASS


def check_cell(question, cell):
    print('Checking question %d' % question.number)
    if question.format == TEXT_FORMAT:
        return check_cell_text(question.number, cell)

    raise Exception("invalid question type")


def grade_answers(cells):
    results = {'score':0, 'tests': []}

    for question in questions:
        cell = cells.get(question.number, None)
        status = "not found"

        if question.number in cells:
            status = check_cell(question, cells[question.number])

        row = {"test": question.number, "result": status, "weight": question.weight}
        results['tests'].append(row)

    return results


def main():
    # rerun everything
    orig_notebook = 'main.ipynb'
    if len(sys.argv) > 2:
        print("Usage: test.py main.ipynb")
        return
    elif len(sys.argv) == 2:
        orig_notebook = sys.argv[1]
    nb = rerun_notebook(orig_notebook)

    # extract cells that have answers
    answer_cells = {}
    for cell in nb['cells']:
        q = extract_question_num(cell)
        if q == None:
            continue
        if not q in question_nums:
            print('no question %d' % q)
            continue
        answer_cells[q] = cell

    # do grading on extracted answers and produce results.json
    results = grade_answers(answer_cells)
    passing = sum(t['weight'] for t in results['tests'] if t['result'] == PASS)
    total = sum(t['weight'] for t in results['tests'])
    results['score'] = 100.0 * passing / total

    print("\nSummary:")
    for test in results["tests"]:
        print("  Test %d: %s" % (test["test"], test["result"]))

    print('\nTOTAL SCORE: %.2f%%' % results['score'])
    with open('result.json', 'w', encoding='utf-8') as f:
        f.write(json.dumps(results, indent=2))


if __name__ == '__main__':
    main()
