from io import open
from conllu import parse_incr
from conllu import parse


file_list = [
"agr_adj_noun.conllu",
"agr_spg.conllu",
"agr_spn.conllu",
"agr_svn_obj-rel-clause.conllu",
"agr_svn_suj-rel-clause.conllu",
"agr_svp.conllu",
#"agr_svn.conllu",

"center_embedding.conllu",
"center_embedding_pairs_w_modifier.conllu",


"gss-subordination.conllu",
"gss-subordination-pp.conllu",
"gss-subordination-subj-rel-clause.conllu",
"gss-subordination-w-mod.conllu",

#"licensing_subj_neg_belief.conllu",
"licensing-npi.conllu",
#"licensing_subj_neg_belief-w-mod.conllu",
"licensing_subj_wish.conllu",

#"ldd-cleft.conllu",
#"ldd-cleft-w-mods.conllu",
"ldd-filler-gap-3stembed.conllu",
#"ldd-filler-gap-4stembed.conllu",
    "ldd-filler-gap-hierarchy.conllu",
"ldd-filler-gap-obj-extraction.conllu",
"ldd-filler-gap-subj-extraction.conllu",
]

lemmas = {}
#lemma_file = open("VERB.corrected", 'r', encoding='utf8')
lemma_file = open("NON_VERB.corrected", 'r', encoding='utf8')
for line in lemma_file:
    print(line)
    tokens = line.split()
    form_lemma = tokens[0] + tokens[2]
    lemmas[form_lemma] = tokens[1]
lemma_file.close()


for file in file_list:
    print("#####################")
    print(file)
    print("#####################")
    data_file = open(file, "r", encoding="utf-8")
    token_lists = []
    for token_list in parse_incr(data_file):
        for token in token_list:
            if token["form"]+token["upos"] in lemmas and lemmas[token["form"]+token["upos"]] != token["lemma"]:
                print("ERROR, lemma of ", token["form"]+token["upos"], "should be", lemmas[token["form"]+token["upos"]], "not" , token["lemma"])
                token["lemma"] = lemmas[token["form"]+token["upos"]]
        token_lists.append(token_list)
    data_file.close()
    for token_list in token_lists:
        print(token_list.serialize(), end='')

#    new_file_name = file + ".new"
    new_file_name = file 
    with open(new_file_name, 'w') as f:
        f.writelines([token_list.serialize() for token_list in token_lists])
