# ==============  SemEval-2015 Task 1  ==============
#  Paraphrase and Semantic Similarity in Twitter
# ===================================================
#
# Author: Wei Xu (UPenn xwe@cis.upenn.edu)
#
# use this script to check output format before submitting
#
# To run:
# > python pit2015_checkformat.py sampletestdata samplesystemoutput

import sys
import re


# return True if pass the check
def CheckFileFormat(testlabelfile, outputfile):
    anyerror = False
    rfilename = re.compile(r'PIT2015_[A-Z\-]{2,8}_0[1-2]_[a-z0-9]{2,10}.output')

    if not rfilename.match(outputfile):
        anyerror = True
        return anyerror

    ### check the content of the output file ###

    rdecimal = re.compile("^-?[0-9]\.[0-9][0-9][0-9][0-9]$")

    ntline = 0
    with open(testlabelfile) as tf:
        for tline in tf:
            tline = tline.strip()
            if len(tline.split('\t')) == 2:
                ntline += 1

    noline = 0
    oline_count = 0
    with open(outputfile) as of:
        for oline in of:
            oline = oline.strip()
            oline_count += 1
            ocolumns = oline.split('\t')
            if len(ocolumns) == 2:
                noline += 1
                if ocolumns[0] != "true" and ocolumns[0] != "false":
                    anyerror = True
                if not rdecimal.match(ocolumns[1]):
                    anyerror = True
                else:
                    score = float(ocolumns[1])
                    if score < 0 or score > 1:
                        anyerror = True
            else:
                anyerror = True

    if ntline != noline:
        anyerror = True

    return not anyerror


# print messages of format checking
def CheckFormat(testdatafile, outputfile):
    anyerror = False

    ###  check the name of the output file  ###
    # it has to be named as PIT2015_XXXXX_dd_xxxxxx.output
    # XXXXX is the team name,  2-8 characters of English letters in uppercase or '-'
    # dd is the index of the runs, can only be 01 or 02 or 03
    # xxxxx is the name of run, 2-10 characters of English letters in lowercase or digits

    rfilename = re.compile(r'PIT2015_[A-Z\-]{2,8}_0[1-3]_[a-z0-9]{2,10}.output')

    if not rfilename.match(outputfile):
        anyerror = True
        print("The name of the output file doesn't meet the requirements:")
        print()
        print("    it must be named as PIT2015_XXXXX_dd_xxxxxx.output")
        print(
            "    XXXXX is your team name, must be 2-8 characters of English letters in uppercase or '-'")
        print("    dd is the index of the runs, can only be 01 or 02")
        print(
            "    xxxxx is the name of this run, must be 2-10 characters of English letters in lowercase or digits")
        print()
        print("Sorry! The output file didn't pass the format check.")
        print()
        return anyerror

    ### check the content of the output file ###

    rdecimal = re.compile("^[0-9]\.[0-9][0-9][0-9][0-9]$")

    ntline = 0
    with open(testdatafile) as tf:
        for tline in tf:
            tline = tline.strip()
            if len(tline.split('\t')) == 6 or len(tline.split('\t')) == 2:
                ntline += 1

    noline = 0
    oline_count = 0
    with open(outputfile) as of:
        for oline in of:
            oline = oline.strip()
            oline_count += 1
            ocolumns = oline.split('\t')
            if len(ocolumns) == 2:
                noline += 1
                if ocolumns[0] != "true" and ocolumns[0] != "false":
                    anyerror = True
                    print("Error in line " + str(
                        oline_count) + " (1st column must be \"true\" or \"false\")")
                if not rdecimal.match(ocolumns[1]):
                    anyerror = True
                    print("Error in line " + str(
                        oline_count) + " (2nd column must be a decimal in X.XXXX format)")
                else:
                    score = float(ocolumns[1])
                    if score < 0 or score > 1:
                        anyerror = True
                        print("Error in line " + str(
                            oline_count) + " (2nd column must be a decimal between 0 and 1)")

            else:
                anyerror = True
                print("Error in line " + str(
                    oline_count) + " (must have 2 columns seperated by tab)")

    if ntline != noline:
        anyerror = True
        print(
            "Error (the total number of lines in the output file does not match with test data)")

    print()
    if anyerror == False:
        print("Congratulations! The output file has passed the format check.")
    else:
        print("Sorry! The output file didn't pass the format check.")
    print()

    return anyerror


if __name__ == "__main__":
    CheckFormat(sys.argv[1], sys.argv[2])
