
import argparse
import glob
import logging as log
import os
import random
import time
import torch.nn.functional as F

import numpy as np
import torch
from eval_utils import f1_score, precision_score, recall_score, classification_report, macro_score
from utils import gen_knn_mix_batch
from torch.nn import CrossEntropyLoss
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
from torch.utils.data.distributed import DistributedSampler
from tqdm import tqdm, trange
import pickle
from transformers import *
from read_data import *

from tensorboardX import SummaryWriter

from bert_models import BertModel4Mix

logger = log.getLogger(__name__)

use_cuda = torch.cuda.is_available()
#CUDA_VISIBLE_DEVICES=6,7
#os.environ["CUDA_VISIBLE_DEVICES"] = "6,7"
MODEL_CLASSES = {"bert": (BertConfig, BertForTokenClassification, BertTokenizer)}

parser = argparse.ArgumentParser(description='PyTorch BaseNER')
parser.add_argument("--data-dir", default = './data', type = str, required = True)
parser.add_argument("--model-type", default = 'bert', type = str)
parser.add_argument("--model-name", default = 'bert-base-multilingual-cased', type = str)
parser.add_argument("--output-dir", default = './german_eval', type = str)
parser.add_argument('--gpu', default='0,1,2,3', type=str, help='id(s) for CUDA_VISIBLE_DEVICES')
parser.add_argument('--train-examples', default = -1, type = int)

parser.add_argument("--labels", default = "", type = str)
parser.add_argument('--config-name', default = '', type = str)
parser.add_argument("--tokenizer-name", default = '', type = str)
parser.add_argument("--max-seq-length", default = 128, type = int)

parser.add_argument("--do-train", action="store_true", help="Whether to run training.")
parser.add_argument("--do-eval", action="store_true", help="Whether to run eval on the dev set.")
parser.add_argument("--do-predict", action="store_true", help="Whether to run predictions on the test set.")
parser.add_argument("--evaluate-during-training", action="store_true", help="Whether to run evaluation during training at each logging step.")
parser.add_argument("--do-lower-case", action="store_true", help="Set this flag if you are using an uncased model.")

parser.add_argument("--batch-size", default = 16, type = int)
parser.add_argument('--eval-batch-size', default = 128, type = int)

parser.add_argument("--gradient-accumulation-steps", type=int, default=1, help="Number of updates steps to accumulate before performing a backward/update pass.")

parser.add_argument("--learning-rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
parser.add_argument("--weight-decay", default=0.0, type=float, help="Weight decay if we apply some.")
parser.add_argument("--adam-epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
parser.add_argument("--max-grad-norm", default=1.0, type=float, help="Max gradient norm.")

parser.add_argument("--num-train-epochs", default=20, type=float, help="Total number of training epochs to perform.")
parser.add_argument("--max-steps", default=-1, type=int, help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
parser.add_argument('--warmup-steps', default = 0, type = int,  help="Linear warmup over warmup_steps.")

parser.add_argument('--logging-steps', default = 150, type = int, help="Log every X updates steps.")
parser.add_argument("--save-steps", type=int, default=0, help="Save checkpoint every X updates steps.")
parser.add_argument("--eval-all-checkpoints", action="store_true", help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number")
parser.add_argument("--overwrite-output-dir", action="store_true", help="Overwrite the content of the output directory")

parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")

parser.add_argument("--pad-subtoken-with-real-label", action="store_true", help="give real label to the padded token instead of `-100` ")
parser.add_argument("--subtoken-label-type",default='real', type=str,help="[real|repeat|O] three ways to do pad subtoken with real label. [real] give the subtoken a real label e.g., B -> B I. [repeat] simply repeat the label e.g., B -> B B. [O] give it a O label. B -> B O")


parser.add_argument("--eval-pad-subtoken-with-first-subtoken-only", action="store_true", help="only works when --pad-subtoken-with-real-label is true, in this mode, we only test the prediction of the first subtoken of each word (if the word could be tokenized into multiple subtoken)")
parser.add_argument("--label-sep-cls", action="store_true", help="label [SEP] [CLS] with special labels, but not [PAD]") 



parser.add_argument("--log-file", default = "results.csv", type = str,help="the file to store resutls")

parser.add_argument("--optimizer", default = "adam", type = str,help='optimizer')
parser.add_argument('--special-label-weight', default=0, type=float, help='the special_label_weight in training . default 0')



# intra-mix




args = parser.parse_args()

os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
args.device = device
args.n_gpu = torch.cuda.device_count()
print("gpu num: ", args.n_gpu)

best_f1 = 0





def set_seed(args):
    logger.info("random seed %s", args.seed)
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if len(args.gpu) > 0:
        torch.cuda.manual_seed_all(args.seed)

def count_rule_based_aug(args, tokenizer, labels, pad_token_label_id, mode,  
              omit_sep_cls_token=False,
              pad_subtoken_with_real_label=False):

    
    examples = read_examples_from_file_excel("data/conll2003",mode)  #examples = read_examples_from_file_excel("data/conll2003",mode)
    augexamples=[]
    totalexamplelist=copy.deepcopy(examples)
    examples_for_rule_based_aug_training=copy.deepcopy(examples)
    
    if mode =='train':

    
        
        
        import pickle 
        

       
        
        #Count all eligible examples
        num_of_aug_examples_to_generate_for_aug_type_1=10000000000000000000000000000
        num_of_aug_examples_to_generate_for_aug_type_3=10000000000000000000000000000
        num_of_aug_examples_to_generate_for_aug_type_4=10000000000000000000000000000
        augruniter=0
       
        num_of_aug_examples_generated=0
        

         
        #Pattern 1
    
        data4 = pd.read_excel ('data/conll2003/Sports.xlsx') 
        sportnames=list(data4['Names'])
        data = pd.read_excel ('data/conll2003/ORG Entity Phrases for Hold Out Experiments.xlsx') 
        df=data['Pattern1']
        placement=list(data['Placement'])
        phrasedomains=list(data['Domain'])
        compatibility=list(data['Compatibility'])
        labelsdf=list(data['Labels'])
        phrases=[]
        labels2=[]
        dfindex=0
        exampleindex=0
        while dfindex< len(df.index):
            phrases.append(df[dfindex].split(","))
            dfindex=dfindex+1

        for label in labelsdf:
            labels2.append(label.replace(" ", "").split("|"))


        examplesguids=[]
        selectedwords=[]
        exampleguidsalreadyaugmented=[]
        iccands=[]
        iccandbool=False

        i=0
        selectedword=''
        pattern1orgtransitioncounter=0
    

        for ex in examples:
        
            e=random.choice(examples)
            for word,label in zip(e.words,e.labels):
                if 'B-PER' in label or 'B-LOC' in label and 'B-MISC' not in label:
                    iccandbool=True
                    selectedword=word            
                
                elif iccandbool==True and label=='O' and i==1:
                    
                    if e.guid not in examplesguids:

                        examplesguids.append(e.guid)
                        selectedwords.append(selectedword)
                

                    break 
                
                if iccandbool==True:                
                    i=i+1        
            pattern1orgtransitioncounter=pattern1orgtransitioncounter+1
            iccandbool=False
            i=0



        index=0
        
        for ex in examples:
            
            if ex.guid in examplesguids and ex.guid not in exampleguidsalreadyaugmented:
            
                for guid2,selword in zip(examplesguids,selectedwords):
                    if ex.guid==guid2:
                        
                    

                        randomindex=random.randint(0,len(phrases)-1)
                    
                        
                        if placement[randomindex]=='before':
                            currentexample=copy.deepcopy(examples[index])
                            if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                    if currentexample.words.index(selword)-1==0:
                                        currentexample.words[currentexample.words.index(selword)-1]="At"
                                    else:
                                        currentexample.words[currentexample.words.index(selword)-1]="at"
                                        o=0
                                    o=0
                            if currentexample.words.index(selword)==0:
                                currentexample.words.insert(currentexample.words.index(selword),"The")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                            else:
                                currentexample.words.insert(currentexample.words.index(selword),"the")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                            
                        
                            #Sport phrase code
                            if  "<x>" in phrases[randomindex]:
                                exwords = [word.lower() for word in currentexample.words]
                                insports=False
                                
                                for sport in sportnames:                               
                                    if sport.lower() in exwords :
                                        insports=True
                                        sportphrase1=phrases[randomindex]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=sportphrase.index("<x>")
                                        
                                        sportphrase[pindex]=sport
                                        
                                    
                                        break
                                if insports==False:
                                    randomsportindex=random.randint(0,len(sportnames)-1) 
                                    sportphrase1=phrases[randomindex]
                                    sportphrase=copy.deepcopy(sportphrase1)
                                    pindex=phrases[randomindex].index("<x>")
                                    sportphrase[pindex]=sportnames[randomsportindex]

                                for word,label in zip(sportphrase,labels2[randomindex]):
                                

                                    currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                    currentexample.words.insert(currentexample.words.index(selword),word)
                                    currentexample.labels.insert(currentexample.words.index(selword),label)
                                firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex])
                                currentexample.labels[firstwordindex]='B-ORG'
                                r=0
                            #Regular insertion 
                            else: 
                            
                                for word,label in zip(phrases[randomindex],labels2[randomindex]):
                                    

                                    currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                    currentexample.words.insert(currentexample.words.index(selword),word)
                                    currentexample.labels.insert(currentexample.words.index(selword),label)
                                firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex])
                                currentexample.labels[firstwordindex]='B-ORG'
                            
                                

                        elif placement[randomindex]=='after':
                            currentexample=copy.deepcopy(examples[index])
                            if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                    if currentexample.words.index(selword)-1==0:
                                        currentexample.words[currentexample.words.index(selword)-1]="At"
                                    else:
                                        currentexample.words[currentexample.words.index(selword)-1]="at"
                            if currentexample.words.index(selword)==0:
                                currentexample.words.insert(currentexample.words.index(selword),"The")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                            else:
                                currentexample.words.insert(currentexample.words.index(selword),"the")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")


                            #Sport phrase code
                            if  "<x>" in phrases[randomindex]:
                                exwords = [word.lower() for word in currentexample.words]
                                insports=False
                                
                                for sport in sportnames:                               
                                    if sport.lower() in exwords :
                                        insports=True
                                        sportphrase1=phrases[randomindex]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=sportphrase.index("<x>")
                                        
                                        sportphrase[pindex]=sport
                            
                                    
                                        break
                                if insports==False:
                                    randomsportindex=random.randint(0,len(sportnames)-1) 
                                    sportphrase1=phrases[randomindex]
                                    sportphrase=copy.deepcopy(sportphrase1)
                                    pindex=phrases[randomindex].index("<x>")
                                    sportphrase[pindex]=sportnames[randomsportindex]

                                for word,label in zip(sportphrase,labels2[randomindex]):
                                

                                    currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                    currentexample.words.insert(currentexample.words.index(selword),word)
                                    currentexample.labels.insert(currentexample.words.index(selword),label)
                                firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex])
                                currentexample.labels[firstwordindex]='B-ORG'
                                

                            #Regular insertion 
                            else:                        
                    
                            
                                #Start inserting words
                                k=1
                                for word,label in zip(phrases[randomindex],labels2[randomindex]):
                                    currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                    currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                    currentexample.labels[currentexample.words.index(selword)]='B-ORG'
                                    k=k+1
                        augexamples.append(currentexample)
                        exampleguidsalreadyaugmented.append(ex.guid)
                        num_of_aug_examples_generated=num_of_aug_examples_generated+1
                        break
                        
            if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_1:
                num_of_aug_examples_generated=0
                break
                        

            index=index+1
            
           
    
   
        
        dataloc = pd.read_excel ('data/conll2003/Transition To LOC.xlsx') 
        dfloc=list(dataloc['Pattern1'])
        dataloc2 = pd.read_excel ('data/conll2003/LOC Context Phrases.xlsx') 
        phrasesdfloc2=list(dataloc2['Pattern2'])
        labelsloc2=list(dataloc2['EntityType'])
        
        loclabels=[]
        phrasesloc=[]
        phrasesloc2=[]
        for locword in phrasesdfloc2:
            phrasesloc2.append(locword.split("|"))

        for label in labelsloc2:
            loclabels.append(label.replace(" ", "").split("|"))     
        
        for loc in dfloc:
            phrasesloc.append(loc.split(","))

        placementloc=list(dataloc['Placement'])

        labels3=[]
        labelset=[]
        for phrase, place in zip(phrasesloc, placementloc):
            if place=='before':
                first=True
                for word in phrase:
                    if first==True:
                        labelset.append('B-ORG')
                    else:
                        labelset.append('I-ORG')
            elif place=='after':
                
                for word in phrase:
                    labelset.append('I-ORG')
            labels3.append(labelset)
            labelset=[]
        index=0
        toloccands=[]
        iccandbool=False
        toloccounter=0
        selectedword=''
        pattern1loctransitioncounter=0
        #while  pattern1loctransitioncounter <30:
            
        for e in examples:
            if toloccounter==70:
                break

            if e.guid not in exampleguidsalreadyaugmented:
                for phrase,placementlociter in zip(phrasesloc,placementloc):
                    
                    if(set(phrase).issubset(set( e.words))):
                        
                        if placementlociter=='after':
                            
                            entityindex=e.words.index(phrase[-1])+1

                            if e.labels[entityindex]=='I-ORG':
                                #match e guid with example guid
                                index=0
                                
                                
                                for ex in examples:
                                    if ex.guid==e.guid:
                                        currentexample=copy.deepcopy(examples[index])
                                        
                                        currentexample.labels[entityindex]='B-LOC'
                                        k=1
                                        
                                        for word in phrase:
                                            
                                            currentexample.words.pop(entityindex-k)
                                            currentexample.labels.pop(entityindex-k)
                                            
                                            k=k+1
                                        
                                        if currentexample.labels[currentexample.labels.index('B-LOC')+1]=='I-ORG':
                                            currentexample.labels[currentexample.labels.index('B-LOC')+1]='I-LOC'
                                        
                                        #pattern 2
                                        randomindex=random.randint(0,len(phrasesdfloc2)-1)
                                        selword=currentexample.labels.index('B-LOC')

                                        k=1
                                        
                                        for word,label in zip(phrasesloc2[randomindex],loclabels[randomindex]):
                                            currentexample.words.insert(selword+k,word)
                                            currentexample.labels.insert(selword+k,label)
                                            
                                            k=k+1

                                        augexamples.append(currentexample)
                                        #num_of_aug_examples_generated=num_of_aug_examples_generated+1
                                        exampleguidsalreadyaugmented.append(ex.guid)
                                        toloccounter=toloccounter+1
                                        break
                                        
                                        
                                    index=index+1
                            else:
                                break

                        elif placementlociter=='before':
                            entityindex=e.words.index(phrase[0])-1
                            if e.labels[entityindex]=='B-ORG':
                                #match e guid with example guid
                                index=0
                                
                                
                                for ex in examples:
                                    if ex.guid==e.guid:
                                        currentexample=copy.deepcopy(examples[index])
                                        
                                        currentexample.labels[entityindex]='B-LOC'
                                        
                                        for word in phrase:
                                            
                                            currentexample.words.pop(entityindex+1)
                                            currentexample.labels.pop(entityindex+1)
                                        
                                        if currentexample.labels[currentexample.labels.index('B-LOC')+1]=='I-ORG':
                                            currentexample.labels[currentexample.labels.index('B-LOC')+1]='I-LOC'
                                        
                                        #pattern 2
                                        randomindex=random.randint(0,len(phrasesdfloc2)-1)
                                        selword=currentexample.labels.index('B-LOC')

                                        k=1
                                        
                                        for word,label in zip(phrasesloc2[randomindex],loclabels[randomindex]):
                                            currentexample.words.insert(selword+k,word)
                                            currentexample.labels.insert(selword+k,label)
                                            
                                            k=k+1

                                        augexamples.append(currentexample)
                                        
                                        exampleguidsalreadyaugmented.append(ex.guid)
                                        #num_of_aug_examples_generated=num_of_aug_examples_generated+1
                                        toloccounter=toloccounter+1
                                        break
                                        
                                        
                                    index=index+1

                            else:
                                break


          

                    if e.guid in exampleguidsalreadyaugmented:
                        break
                        
     

                            
      
        dataPER = pd.read_excel ('data/conll2003/PER Context Phrases for Hold Out Experiments.xlsx') 
        dataPER2 = pd.read_excel ('data/conll2003/PER Headline Phrases for Hold Out Experiments.xlsx') 
        dataORG = pd.read_excel ('data/conll2003/ORG Context Phrases for Hold Out Experiments.xlsx') 
        dfPER2=list(dataPER2['Pattern2'])
        labelsPER2=list(dataPER2['EntityType'])
        compatibility2=list(dataORG['Compatibility'])

        dataNames = pd.read_excel ('data/conll2003/Names.xlsx') 
        dfNames=list(dataNames['Names'])
        dfPER=list(dataPER['Pattern2'])
        perphrasedomains=list(dataPER['Domain'])
        orgphrasedomains=list(dataORG['Domain'])
        dfORG=list(dataORG['Pattern2'])
        labelsPER=list(dataPER['EntityType'])
        labelsORG=list(dataORG['EntityType'])
        perphrases=[] 
        orgphrases=[]
        perlabels=[]  
        orglabels=[]
        perphrases2=[] 
        perlabels2=[] 


        for per2 in dfPER2:
            perphrases2.append(per2.split("|"))

        for perlabel2 in labelsPER2:
            perlabels2.append(perlabel2.replace(" ", "").split("|"))

        for per in dfPER:
            perphrases.append(per.split("|"))
        
        for org in dfORG:
            orgphrases.append(org.split("|"))

        for label in labelsPER:
            perlabels.append(label.replace(" ", "").split("|"))
        for label in labelsORG:
            orglabels.append(label.replace(" ", "").split("|"))

        percands=[]
        orgcands=[]
        iccandbool=False
        i=0
        selectedword=''
        numofexamples=0
        pertransitioncounter=0
        orgtransitioncounter=0
        originallabel=''
        examplesguids=[]
        selectedwords=[]
        #while  pertransitioncounter <30:
        for ex in examples:
            #if len(examplesguids)==400:
            #    break
            e=random.choice(examples)
            if  e.guid not in exampleguidsalreadyaugmented:

                for word,label in zip(e.words,e.labels):
                                
                    nextwordinddex=e.words.index(word)+1
                    if nextwordinddex<len(e.words):
                        w=e.words[nextwordinddex]
                    else:
                        w=''

                    if 'B-ORG' in label or 'B-LOC' in label  and 'B-MISC' not in label and ')' not in w and '.' not in word:
                        iccandbool=True
                        selectedword=word
                        originallabel=label
                    elif iccandbool==True and label=='O' and i==1:
                        if e.guid not in examplesguids:
                        
                            examplesguids.append(e.guid)
                            selectedwords.append(selectedword)

                        break 
                    
                    if iccandbool==True:                
                        i=i+1        
                
                iccandbool=False
                i=0
                pertransitioncounter=pertransitioncounter+1
        index=0
        for ex in examples:
                
                if ex.guid in examplesguids and ex.guid not in exampleguidsalreadyaugmented:
                
                    for guid2,selword in zip(examplesguids,selectedwords):
                        if ex.guid==guid2:
                                            
                            
                            currentexample=copy.deepcopy(examples[index])
                            numbercount = sum(entry.isdigit() for entry in currentexample.words)
                            if len(currentexample.words)<4 and 'AT' not in currentexample.words or numbercount>3:
                                randomindex=random.randint(0,len(dfPER2)-1)
                                k=0
                                for word,label in zip(perphrases2[randomindex],perlabels2[randomindex]):
                                    currentexample.words.insert(0+k,word)
                                    currentexample.labels.insert(0+k,label)
                                    currentexample.labels[currentexample.words.index(selword)]='B-PER'
                                    k=k+1
                            else:
                                randomindex=random.randint(0,len(dfPER)-1)        
                                k=1
                                for word,label in zip(perphrases[randomindex],perlabels[randomindex]):
                                    currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                    currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                    currentexample.labels[currentexample.words.index(selword)]='B-PER'
                                    k=k+1

                            #Pattern 1
                            randomindex3=random.randint(0,len(dfNames)-1)
                            currentexample.words.insert(currentexample.words.index(selword)+1,dfNames[randomindex3])
                            currentexample.labels.insert(currentexample.words.index(selword)+1,'I-PER')
                            augexamples.append(currentexample)
                            exampleguidsalreadyaugmented.append(currentexample.guid)
                            num_of_aug_examples_generated=num_of_aug_examples_generated+1
                            
                            break

                index=index+1
                if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_3:
                    num_of_aug_examples_generated=0
                    break
        examplesguids=[]    
        selectedwords=[]

        #if num_of_aug_examples_generated<num_of_aug_examples_to_generate:
            
        
        for ex in examples:
            #if len(examplesguids)==400:
            #     break
            e=random.choice(examples)
            if  e.guid not in exampleguidsalreadyaugmented:
                for word,label in zip(e.words,e.labels):
                    if 'B-PER' in label or 'B-LOC' in label  and 'B-MISC' not in label:
                        iccandbool=True
                        selectedword=word
                        originallabel=label
                    elif iccandbool==True and label=='O' and i==1:
                    
                        if e.guid not in examplesguids:
                            examplesguids.append(e.guid)
                            selectedwords.append(selectedword)

                        break 
                    
                    if iccandbool==True:                
                        i=i+1        
                
                iccandbool=False
                i=0
                orgtransitioncounter=orgtransitioncounter+1

        j=0
        index=0
        for ex in examples:
                
                if ex.guid in examplesguids and ex.guid not in exampleguidsalreadyaugmented:
                    
                    for guid2,selword in zip(examplesguids,selectedwords):
                        if ex.guid==guid2:
                        

                            
                            currentexample=copy.deepcopy(examples[index])
                                                        
                            randomindex2=random.randint(0,len(phrases)-1)
                            randomindex=random.randint(0,len(orgphrases)-1)

                                    
                            k=1
                            for word,label in zip(orgphrases[randomindex],orglabels[randomindex]):
                                currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                currentexample.labels[currentexample.words.index(selword)]='B-ORG'
                                k=k+1
                      
                            
                            if placement[randomindex2]=='before':

                                if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                    if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                        if currentexample.words.index(selword)-1==0:
                                            currentexample.words[currentexample.words.index(selword)-1]="At"
                                        else:
                                            currentexample.words[currentexample.words.index(selword)-1]="at"
                                if currentexample.words.index(selword)==0:
                                    currentexample.words.insert(currentexample.words.index(selword),"The")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                                else:
                                    currentexample.words.insert(currentexample.words.index(selword),"the")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")



                                
                                #Sport phrase code
                                if  "<x>" in phrases[randomindex2]:
                                    exwords = [word.lower() for word in currentexample.words]
                                    insports=False
                                    
                                    for sport in sportnames:                               
                                        if sport.lower() in exwords :
                                            insports=True
                                            sportphrase1=phrases[randomindex2]
                                            sportphrase=copy.deepcopy(sportphrase1)
                                            pindex=sportphrase.index("<x>")
                                            
                                            sportphrase[pindex]=sport
                                    
                                        
                                            break
                                    if insports==False:
                                        randomsportindex=random.randint(0,len(sportnames)-1) 
                                        sportphrase1=phrases[randomindex2]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=phrases[randomindex2].index("<x>")
                                        sportphrase[pindex]=sportnames[randomsportindex]

                                    for word,label in zip(sportphrase,labels2[randomindex2]):
                                    

                                        currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                        currentexample.words.insert(currentexample.words.index(selword),word)
                                        currentexample.labels.insert(currentexample.words.index(selword),label)
                                    firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex2])
                                    currentexample.labels[firstwordindex]='B-ORG'
                                    

                                #Regular insertion 
                                else:
                                        
                                
                                    for word,label in zip(phrases[randomindex2],labels2[randomindex2]):
                                        currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                        currentexample.words.insert(currentexample.words.index(selword),word)
                                        currentexample.labels.insert(currentexample.words.index(selword),label)
                                    firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex2])
                                    currentexample.labels[firstwordindex]='B-ORG'


                            elif placement[randomindex2]=='after':
                                if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                    if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                        if currentexample.words.index(selword)-1==0:
                                            currentexample.words[currentexample.words.index(selword)-1]="At"
                                        else:
                                            currentexample.words[currentexample.words.index(selword)-1]="at"
                                if currentexample.words.index(selword)==0:
                                    currentexample.words.insert(currentexample.words.index(selword),"The")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                                else:
                                    currentexample.words.insert(currentexample.words.index(selword),"the")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")


                                
            
                                #Sport phrase code
                                if  "<x>" in phrases[randomindex2]:
                                    exwords = [word.lower() for word in currentexample.words]
                                    insports=False
                            
                                    for sport in sportnames:                               
                                        if sport.lower() in exwords :
                                            insports=True
                                            sportphrase1=phrases[randomindex2]
                                            sportphrase=copy.deepcopy(sportphrase1)
                                            pindex=sportphrase.index("<x>")
                                            
                                            sportphrase[pindex]=sport
                                        
                                        
                                            break
                                    if insports==False:
                                        randomsportindex=random.randint(0,len(sportnames)-1) 
                                        sportphrase1=phrases[randomindex2]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=phrases[randomindex2].index("<x>")
                                        sportphrase[pindex]=sportnames[randomsportindex]

                                    for word,label in zip(sportphrase,labels2[randomindex2]):
                                    

                                        currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                        currentexample.words.insert(currentexample.words.index(selword),word)
                                        currentexample.labels.insert(currentexample.words.index(selword),label)
                                    firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex2])
                                    currentexample.labels[firstwordindex]='B-ORG'
                                    

                                #Regular insertion 
                                else:
                        
                                    k=1
                                    for word,label in zip(phrases[randomindex2],labels2[randomindex2]):
                                        currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                        currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                        currentexample.labels[currentexample.words.index(selword)]='B-ORG'
                                        k=k+1
                                        
                            exampleguidsalreadyaugmented.append(currentexample.guid)
                            augexamples.append(currentexample)
                            num_of_aug_examples_generated=num_of_aug_examples_generated+1
                            

                            break

                index=index+1
                
                if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_4:
                    num_of_aug_examples_generated=0
                    break
        logger.info("Number of augmented examples: %s", len(augexamples))
        print("Number of augmented examples",len(augexamples))
        file_name="augexamples100percent.pkl"
        file_path = os.path.join(args.data_dir, file_name)
        
        open_file = open(file_path, "wb")
        pickle.dump(augexamples, open_file)
        open_file.close()
        return len(augexamples)




#just append ontonotes training examples to training set
def read_examples_from_ontonotes_file(args, tokenizer, labelslist, pad_token_label_id, mode, 
              omit_sep_cls_token=False,
              pad_subtoken_with_real_label=False):
    file_path = os.path.join(args.data_dir, 'ontotrain.ner')
    guid_index = 0
    examples = []
    
    label_map = {"PER": "PER",
             "person": "PER",
             "PERSON": "PER",
             "DOCTOR": "PER",
             "PATIENT": "PER",
             "LOC": "LOC",
             "location": "LOC",
             "GPE": "LOC",
             "FAC": "LOC",
             "HOSPITAL": "LOC", 
             "CITY": "LOC",
             "STATE": "LOC",
             "COUNTRY": "LOC",
             "LOCATION_OTHER": "LOC",
             "ORG": "ORG",
             "corporation": "ORG",
             "group": "ORG",
             "ORGANIZATION": "ORG",
             "MISC": "MISC",
             "product": "MISC",
             "creative-work": "MISC",
             "PRODUCT": "MISC", 
             "NORP": "MISC",
             "EVENT": "MISC",
             "LANGUAGE": "MISC",
             "LAW": "MISC",
             "WORK_OF_ART": "MISC"}
        
    
    with open(file_path, encoding="utf-8") as f:
        words = []
        labels = []
        for line in f:
            if line.startswith("-DOCSTART-") or line == "" or line == "\n":
                if words:
                    examples.append(InputExample(
                        guid="{}-{}".format(mode, guid_index), words=words, labels=labels))
                    guid_index += 1
                    words = []
                    labels = []
                    
            else:
                elems = line.strip().split()
                label = elems[-1].replace("\n", "")
                words.append(elems[0])
               
                if label != "O":
                    iob_prefix = label[:2]
                    etype = label[2:]
                    if etype in label_map:
                        labels.append(iob_prefix + label_map[etype])
                    else:
                        labels.append("O")
                        
                else:
                    labels.append("O")
        if words:
            examples.append(InputExample(
                guid="{}-{}".format(mode, guid_index), words=words, labels=labels))
  

        print(mode)
        print('data num: {}'.format(len(examples)))

    features = convert_examples_to_features(examples, labelslist, args.max_seq_length, tokenizer, 
                                cls_token = tokenizer.cls_token, sep_token =  tokenizer.sep_token, pad_token = tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
                                cls_token_segment_id = 2 if args.model_type in ["xlnet"] else 0, 
                                sequence_a_segment_id = 0, pad_token_segment_id=4 if args.model_type in ["xlnet"] else 0, 
                                pad_token_label_id = pad_token_label_id,
                                omit_sep_cls_token=omit_sep_cls_token,
                                pad_subtoken_with_real_label=pad_subtoken_with_real_label,
                                subtoken_label_type=args.subtoken_label_type,
                                label_sep_cls=args.label_sep_cls)
    
    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
    all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
    all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
    all_label_ids = torch.tensor([f.label_ids for f in features], dtype=torch.long)
    all_subtoken_ids = torch.tensor([f.subtoken_ids for f in features], dtype=torch.long)
    all_sent_id = torch.tensor([f.sent_id for f in features], dtype=torch.long)
    dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids, all_subtoken_ids,all_sent_id)

    
    return dataset

def read_data_rule_based_aug_for_zero_shot(args,amount, percentagename,tokenizer, labels, pad_token_label_id, mode,  
              omit_sep_cls_token=False,
              pad_subtoken_with_real_label=False):
    examples=read_examples_from_ontonotes_file(args, tokenizer, labels, pad_token_label_id, mode = 'train', pad_subtoken_with_real_label=args.pad_subtoken_with_real_label)
    #examples = read_examples_from_file_excel("data/conll2003",mode)  #examples = read_examples_from_file_excel("data/conll2003",mode)
    augexamples=[]
    totalexamplelist=copy.deepcopy(examples)
    examples_for_rule_based_aug_training=copy.deepcopy(examples)
    
    if mode =='train':

    
        
        
        import pickle 
        

        
       

        num_of_aug_examples_to_generate_minus_loc_transition=amount-70
        num_of_aug_examples_to_generate=num_of_aug_examples_to_generate_minus_loc_transition/3
        num_of_aug_examples_to_generate=round(num_of_aug_examples_to_generate)

        num_of_aug_examples_to_generate_for_aug_type_1=5
        num_of_aug_examples_to_generate_for_aug_type_3=10
        num_of_aug_examples_to_generate_for_aug_type_4=5

        augruniter=0
       
        num_of_aug_examples_generated=0
        

         
        #Pattern 1
    
        data4 = pd.read_excel ('data/conll2003/Sports.xlsx') 
        sportnames=list(data4['Names'])
        data = pd.read_excel ('data/conll2003/ORG Entity Phrases for Hold Out Experiments.xlsx') 
        df=data['Pattern1']
        placement=list(data['Placement'])
        phrasedomains=list(data['Domain'])
        compatibility=list(data['Compatibility'])
        labelsdf=list(data['Labels'])
        phrases=[]
        labels2=[]
        dfindex=0
        exampleindex=0
        while dfindex< len(df.index):
            phrases.append(df[dfindex].split(","))
            dfindex=dfindex+1

        for label in labelsdf:
            labels2.append(label.replace(" ", "").split("|"))


        examplesguids=[]
        selectedwords=[]
        exampleguidsalreadyaugmented=[]
        iccands=[]
        iccandbool=False

        i=0
        selectedword=''
        pattern1orgtransitioncounter=0
    

        for ex in examples:
      
            e=random.choice(examples)
            for word,label in zip(e.words,e.labels):
                if 'B-PER' in label or 'B-LOC' in label and 'B-MISC' not in label:
                    iccandbool=True
                    selectedword=word            
                
                elif iccandbool==True and label=='O' and i==1:
                    
                    if e.guid not in examplesguids:

                        examplesguids.append(e.guid)
                        selectedwords.append(selectedword)
                

                    break 
                
                if iccandbool==True:                
                    i=i+1        
            pattern1orgtransitioncounter=pattern1orgtransitioncounter+1
            iccandbool=False
            i=0


        #Code for putting data into examples instead of cand list of tuples
        index=0
        
        for ex in examples:
            
            if ex.guid in examplesguids and ex.guid not in exampleguidsalreadyaugmented:
            
                for guid2,selword in zip(examplesguids,selectedwords):
                    if ex.guid==guid2:
                        
                    

                        randomindex=random.randint(0,len(phrases)-1)
                    
                        
                        if placement[randomindex]=='before':
                            currentexample=copy.deepcopy(examples[index])
                            if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                    if currentexample.words.index(selword)-1==0:
                                        currentexample.words[currentexample.words.index(selword)-1]="At"
                                    else:
                                        currentexample.words[currentexample.words.index(selword)-1]="at"
                                        o=0
                                    o=0
                            if currentexample.words.index(selword)==0:
                                currentexample.words.insert(currentexample.words.index(selword),"The")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                            else:
                                currentexample.words.insert(currentexample.words.index(selword),"the")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                            
                        
                            #Sport phrase code
                            if  "<x>" in phrases[randomindex]:
                                exwords = [word.lower() for word in currentexample.words]
                                insports=False
                                
                                for sport in sportnames:                               
                                    if sport.lower() in exwords :
                                        insports=True
                                        sportphrase1=phrases[randomindex]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=sportphrase.index("<x>")
                                        
                                        sportphrase[pindex]=sport
                                        
                                    
                                        break
                                if insports==False:
                                    randomsportindex=random.randint(0,len(sportnames)-1) 
                                    sportphrase1=phrases[randomindex]
                                    sportphrase=copy.deepcopy(sportphrase1)
                                    pindex=phrases[randomindex].index("<x>")
                                    sportphrase[pindex]=sportnames[randomsportindex]

                                for word,label in zip(sportphrase,labels2[randomindex]):
                                

                                    currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                    currentexample.words.insert(currentexample.words.index(selword),word)
                                    currentexample.labels.insert(currentexample.words.index(selword),label)
                                firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex])
                                currentexample.labels[firstwordindex]='B-ORG'
                                r=0
                            #Regular insertion 
                            else: 
                            
                                for word,label in zip(phrases[randomindex],labels2[randomindex]):
                                    

                                    currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                    currentexample.words.insert(currentexample.words.index(selword),word)
                                    currentexample.labels.insert(currentexample.words.index(selword),label)
                                firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex])
                                currentexample.labels[firstwordindex]='B-ORG'
                            
                                

                        elif placement[randomindex]=='after':
                            currentexample=copy.deepcopy(examples[index])
                            if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                    if currentexample.words.index(selword)-1==0:
                                        currentexample.words[currentexample.words.index(selword)-1]="At"
                                    else:
                                        currentexample.words[currentexample.words.index(selword)-1]="at"
                            if currentexample.words.index(selword)==0:
                                currentexample.words.insert(currentexample.words.index(selword),"The")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                            else:
                                currentexample.words.insert(currentexample.words.index(selword),"the")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")


                            #Sport phrase code
                            if  "<x>" in phrases[randomindex]:
                                exwords = [word.lower() for word in currentexample.words]
                                insports=False
                                
                                for sport in sportnames:                               
                                    if sport.lower() in exwords :
                                        insports=True
                                        sportphrase1=phrases[randomindex]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=sportphrase.index("<x>")
                                        
                                        sportphrase[pindex]=sport
                            
                                    
                                        break
                                if insports==False:
                                    randomsportindex=random.randint(0,len(sportnames)-1) 
                                    sportphrase1=phrases[randomindex]
                                    sportphrase=copy.deepcopy(sportphrase1)
                                    pindex=phrases[randomindex].index("<x>")
                                    sportphrase[pindex]=sportnames[randomsportindex]

                                for word,label in zip(sportphrase,labels2[randomindex]):
                                

                                    currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                    currentexample.words.insert(currentexample.words.index(selword),word)
                                    currentexample.labels.insert(currentexample.words.index(selword),label)
                                firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex])
                                currentexample.labels[firstwordindex]='B-ORG'
                                

                            #Regular insertion 
                            else:                        
                    
                            
                                #Start inserting words
                                k=1
                                for word,label in zip(phrases[randomindex],labels2[randomindex]):
                                    currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                    currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                    currentexample.labels[currentexample.words.index(selword)]='B-ORG'
                                    k=k+1
                        augexamples.append(currentexample)
                        exampleguidsalreadyaugmented.append(ex.guid)
                        if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_1:
                            num_of_aug_examples_generated=0
                            break
                        num_of_aug_examples_generated=num_of_aug_examples_generated+1
                        break
                        
            if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_1:
                num_of_aug_examples_generated=0
                break
                        

            index=index+1
            

        AugLOCExamples = read_examples_from_file_excel_LOC("data/conll2003/OntoNotes LOC Aug.xlsx",mode)
        for e in AugLOCExamples:
            augexamples.append(e)
            exampleguidsalreadyaugmented.append(e.guid)
                            
        
   
      
   
        dataPER = pd.read_excel ('data/conll2003/PER Context Phrases for Hold Out Experiments.xlsx') 
        dataPER2 = pd.read_excel ('data/conll2003/PER Headline Phrases for Hold Out Experiments.xlsx') 
        dataORG = pd.read_excel ('data/conll2003/ORG Context Phrases for Hold Out Experiments.xlsx') 
        dfPER2=list(dataPER2['Pattern2'])
        labelsPER2=list(dataPER2['EntityType'])
        compatibility2=list(dataORG['Compatibility'])

        dataNames = pd.read_excel ('data/conll2003/Names.xlsx') 
        dfNames=list(dataNames['Names'])
        dfPER=list(dataPER['Pattern2'])
        perphrasedomains=list(dataPER['Domain'])
        orgphrasedomains=list(dataORG['Domain'])
        dfORG=list(dataORG['Pattern2'])
        labelsPER=list(dataPER['EntityType'])
        labelsORG=list(dataORG['EntityType'])
        perphrases=[] 
        orgphrases=[]
        perlabels=[]  
        orglabels=[]
        perphrases2=[] 
        perlabels2=[] 


        for per2 in dfPER2:
            perphrases2.append(per2.split("|"))

        for perlabel2 in labelsPER2:
            perlabels2.append(perlabel2.replace(" ", "").split("|"))

        for per in dfPER:
            perphrases.append(per.split("|"))
        
        for org in dfORG:
            orgphrases.append(org.split("|"))

        for label in labelsPER:
            perlabels.append(label.replace(" ", "").split("|"))
        for label in labelsORG:
            orglabels.append(label.replace(" ", "").split("|"))

        percands=[]
        orgcands=[]
        iccandbool=False
        i=0
        selectedword=''
        numofexamples=0
        pertransitioncounter=0
        orgtransitioncounter=0
        originallabel=''
        examplesguids=[]
        selectedwords=[]

        for ex in examples:
        
            e=random.choice(examples)
            if  e.guid not in exampleguidsalreadyaugmented:

                for word,label in zip(e.words,e.labels):
                                
                    nextwordinddex=e.words.index(word)+1
                    if nextwordinddex<len(e.words):
                        w=e.words[nextwordinddex]
                    else:
                        w=''

                    if 'B-ORG' in label or 'B-LOC' in label  and 'B-MISC' not in label and ')' not in w and '.' not in word:
                        iccandbool=True
                        selectedword=word
                        originallabel=label
                    elif iccandbool==True and label=='O' and i==1:
                        if e.guid not in examplesguids:
                        
                            examplesguids.append(e.guid)
                            selectedwords.append(selectedword)

                        break 
                    
                    if iccandbool==True:                
                        i=i+1        
                
                iccandbool=False
                i=0
                pertransitioncounter=pertransitioncounter+1
        index=0
        for ex in examples:
                
                if ex.guid in examplesguids and ex.guid not in exampleguidsalreadyaugmented:
                
                    for guid2,selword in zip(examplesguids,selectedwords):
                        if ex.guid==guid2:
                                            
                            
                            currentexample=copy.deepcopy(examples[index])
                            numbercount = sum(entry.isdigit() for entry in currentexample.words)
                            if len(currentexample.words)<4 and 'AT' not in currentexample.words or numbercount>3:
                                randomindex=random.randint(0,len(dfPER2)-1)
                                k=0
                                for word,label in zip(perphrases2[randomindex],perlabels2[randomindex]):
                                    currentexample.words.insert(0+k,word)
                                    currentexample.labels.insert(0+k,label)
                                    currentexample.labels[currentexample.words.index(selword)]='B-PER'
                                    k=k+1
                            else:
                                randomindex=random.randint(0,len(dfPER)-1)        
                                k=1
                                for word,label in zip(perphrases[randomindex],perlabels[randomindex]):
                                    currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                    currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                    currentexample.labels[currentexample.words.index(selword)]='B-PER'
                                    k=k+1

                            #Pattern 1
                            randomindex3=random.randint(0,len(dfNames)-1)
                            currentexample.words.insert(currentexample.words.index(selword)+1,dfNames[randomindex3])
                            currentexample.labels.insert(currentexample.words.index(selword)+1,'I-PER')
                            augexamples.append(currentexample)
                            exampleguidsalreadyaugmented.append(currentexample.guid)
                            if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_3:
                                num_of_aug_examples_generated=0
                                break
                            num_of_aug_examples_generated=num_of_aug_examples_generated+1
                            
                            break

                index=index+1
                if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_3:
                    num_of_aug_examples_generated=0
                    break
        examplesguids=[]    
        selectedwords=[]

 
            
        
        for ex in examples:
           
            e=random.choice(examples)
            if  e.guid not in exampleguidsalreadyaugmented:
                for word,label in zip(e.words,e.labels):
                    if 'B-PER' in label or 'B-LOC' in label  and 'B-MISC' not in label:
                        iccandbool=True
                        selectedword=word
                        originallabel=label
                    elif iccandbool==True and label=='O' and i==1:
                    
                        if e.guid not in examplesguids:
                            examplesguids.append(e.guid)
                            selectedwords.append(selectedword)

                        break 
                    
                    if iccandbool==True:                
                        i=i+1        
                
                iccandbool=False
                i=0
                orgtransitioncounter=orgtransitioncounter+1

        j=0
        index=0
        for ex in examples:
                
                if ex.guid in examplesguids and ex.guid not in exampleguidsalreadyaugmented:
                    
                    for guid2,selword in zip(examplesguids,selectedwords):
                        if ex.guid==guid2:
                        

                            
                            currentexample=copy.deepcopy(examples[index])
                                                        
                            randomindex2=random.randint(0,len(phrases)-1)
                            randomindex=random.randint(0,len(orgphrases)-1)

                                    
                            k=1
                            for word,label in zip(orgphrases[randomindex],orglabels[randomindex]):
                                currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                currentexample.labels[currentexample.words.index(selword)]='B-ORG'
                                k=k+1
                            #Pattern 1 do the fix
                            
                            if placement[randomindex2]=='before':

                                if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                    if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                        if currentexample.words.index(selword)-1==0:
                                            currentexample.words[currentexample.words.index(selword)-1]="At"
                                        else:
                                            currentexample.words[currentexample.words.index(selword)-1]="at"
                                if currentexample.words.index(selword)==0:
                                    currentexample.words.insert(currentexample.words.index(selword),"The")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                                else:
                                    currentexample.words.insert(currentexample.words.index(selword),"the")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")



                                
                                #Sport phrase code
                                if  "<x>" in phrases[randomindex2]:
                                    exwords = [word.lower() for word in currentexample.words]
                                    insports=False
                                    
                                    for sport in sportnames:                               
                                        if sport.lower() in exwords :
                                            insports=True
                                            sportphrase1=phrases[randomindex2]
                                            sportphrase=copy.deepcopy(sportphrase1)
                                            pindex=sportphrase.index("<x>")
                                            
                                            sportphrase[pindex]=sport
                                    
                                        
                                            break
                                    if insports==False:
                                        randomsportindex=random.randint(0,len(sportnames)-1) 
                                        sportphrase1=phrases[randomindex2]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=phrases[randomindex2].index("<x>")
                                        sportphrase[pindex]=sportnames[randomsportindex]

                                    for word,label in zip(sportphrase,labels2[randomindex2]):
                                    

                                        currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                        currentexample.words.insert(currentexample.words.index(selword),word)
                                        currentexample.labels.insert(currentexample.words.index(selword),label)
                                    firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex2])
                                    currentexample.labels[firstwordindex]='B-ORG'
                                    

                                #Regular insertion 
                                else:
                                        
                                
                                    for word,label in zip(phrases[randomindex2],labels2[randomindex2]):
                                        currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                        currentexample.words.insert(currentexample.words.index(selword),word)
                                        currentexample.labels.insert(currentexample.words.index(selword),label)
                                    firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex2])
                                    currentexample.labels[firstwordindex]='B-ORG'


                            elif placement[randomindex2]=='after':
                                if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                    if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                        if currentexample.words.index(selword)-1==0:
                                            currentexample.words[currentexample.words.index(selword)-1]="At"
                                        else:
                                            currentexample.words[currentexample.words.index(selword)-1]="at"
                                if currentexample.words.index(selword)==0:
                                    currentexample.words.insert(currentexample.words.index(selword),"The")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                                else:
                                    currentexample.words.insert(currentexample.words.index(selword),"the")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")


                                
            
                                #Sport phrase code
                                if  "<x>" in phrases[randomindex2]:
                                    exwords = [word.lower() for word in currentexample.words]
                                    insports=False
                            
                                    for sport in sportnames:                               
                                        if sport.lower() in exwords :
                                            insports=True
                                            sportphrase1=phrases[randomindex2]
                                            sportphrase=copy.deepcopy(sportphrase1)
                                            pindex=sportphrase.index("<x>")
                                            
                                            sportphrase[pindex]=sport
                                        
                                        
                                            break
                                    if insports==False:
                                        randomsportindex=random.randint(0,len(sportnames)-1) 
                                        sportphrase1=phrases[randomindex2]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=phrases[randomindex2].index("<x>")
                                        sportphrase[pindex]=sportnames[randomsportindex]

                                    for word,label in zip(sportphrase,labels2[randomindex2]):
                                    

                                        currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                        currentexample.words.insert(currentexample.words.index(selword),word)
                                        currentexample.labels.insert(currentexample.words.index(selword),label)
                                    firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex2])
                                    currentexample.labels[firstwordindex]='B-ORG'
                                    

                                #Regular insertion 
                                else:
                        
                                    k=1
                                    for word,label in zip(phrases[randomindex2],labels2[randomindex2]):
                                        currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                        currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                        currentexample.labels[currentexample.words.index(selword)]='B-ORG'
                                        k=k+1
                                        
                            exampleguidsalreadyaugmented.append(currentexample.guid)
                            augexamples.append(currentexample)
                            if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_4:
                                num_of_aug_examples_generated=0
                                break
                            num_of_aug_examples_generated=num_of_aug_examples_generated+1
                            

                            break

                index=index+1
                
                if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_4:
                    num_of_aug_examples_generated=0
                    break


        logger.info("Number of augmented examples: %s", len(augexamples))
        file_name="SupportSetAugmented2.pkl"
        file_path = os.path.join(args.data_dir, file_name)
        open_file = open(file_path, "wb")
        pickle.dump(augexamples, open_file)
        open_file.close()

        supportset=[]
        for e in examples:
            if e.guid in exampleguidsalreadyaugmented:
                supportset.append(e)

        file_name="SupportSet2.pkl"
        file_path = os.path.join(args.data_dir, file_name)
        open_file = open(file_path, "wb")
        pickle.dump(augexamples, open_file)
        open_file.close()


def read_data_rule_based_aug(args,amount, percentagename,tokenizer, labels, pad_token_label_id, mode,  
              omit_sep_cls_token=False,
              pad_subtoken_with_real_label=False):

    examples = read_examples_from_file_excel("data/conll2003",mode)  #examples = read_examples_from_file_excel("data/conll2003",mode)
    augexamples=[]
    totalexamplelist=copy.deepcopy(examples)
    examples_for_rule_based_aug_training=copy.deepcopy(examples)
    
    if mode =='train':

    
        
        
        import pickle 
        

        
       

        num_of_aug_examples_to_generate_minus_loc_transition=amount-70
        num_of_aug_examples_to_generate=num_of_aug_examples_to_generate_minus_loc_transition/3
        num_of_aug_examples_to_generate=round(num_of_aug_examples_to_generate)

        num_of_aug_examples_to_generate_for_aug_type_1=num_of_aug_examples_to_generate
        num_of_aug_examples_to_generate_for_aug_type_3=num_of_aug_examples_to_generate
        num_of_aug_examples_to_generate_for_aug_type_4=num_of_aug_examples_to_generate

        augruniter=0
       
        num_of_aug_examples_generated=0
        

         
        #Pattern 1
    
        data4 = pd.read_excel ('data/conll2003/Sports.xlsx') 
        sportnames=list(data4['Names'])
        data = pd.read_excel ('data/conll2003/ORG Entity Phrases for Hold Out Experiments.xlsx') 
        df=data['Pattern1']
        placement=list(data['Placement'])
        phrasedomains=list(data['Domain'])
        compatibility=list(data['Compatibility'])
        labelsdf=list(data['Labels'])
        phrases=[]
        labels2=[]
        dfindex=0
        exampleindex=0
        while dfindex< len(df.index):
            phrases.append(df[dfindex].split(","))
            dfindex=dfindex+1

        for label in labelsdf:
            labels2.append(label.replace(" ", "").split("|"))


        examplesguids=[]
        selectedwords=[]
        exampleguidsalreadyaugmented=[]
        iccands=[]
        iccandbool=False

        i=0
        selectedword=''
        pattern1orgtransitioncounter=0
    

        for ex in examples:
      
            e=random.choice(examples)
            for word,label in zip(e.words,e.labels):
                if 'B-PER' in label or 'B-LOC' in label and 'B-MISC' not in label:
                    iccandbool=True
                    selectedword=word            
                
                elif iccandbool==True and label=='O' and i==1:
                    
                    if e.guid not in examplesguids:

                        examplesguids.append(e.guid)
                        selectedwords.append(selectedword)
                

                    break 
                
                if iccandbool==True:                
                    i=i+1        
            pattern1orgtransitioncounter=pattern1orgtransitioncounter+1
            iccandbool=False
            i=0


        #Code for putting data into examples instead of cand list of tuples
        index=0
        
        for ex in examples:
            
            if ex.guid in examplesguids and ex.guid not in exampleguidsalreadyaugmented:
            
                for guid2,selword in zip(examplesguids,selectedwords):
                    if ex.guid==guid2:
                        
                    

                        randomindex=random.randint(0,len(phrases)-1)
                    
                        
                        if placement[randomindex]=='before':
                            currentexample=copy.deepcopy(examples[index])
                            if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                    if currentexample.words.index(selword)-1==0:
                                        currentexample.words[currentexample.words.index(selword)-1]="At"
                                    else:
                                        currentexample.words[currentexample.words.index(selword)-1]="at"
                                        o=0
                                    o=0
                            if currentexample.words.index(selword)==0:
                                currentexample.words.insert(currentexample.words.index(selword),"The")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                            else:
                                currentexample.words.insert(currentexample.words.index(selword),"the")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                            
                        
                            #Sport phrase code
                            if  "<x>" in phrases[randomindex]:
                                exwords = [word.lower() for word in currentexample.words]
                                insports=False
                                
                                for sport in sportnames:                               
                                    if sport.lower() in exwords :
                                        insports=True
                                        sportphrase1=phrases[randomindex]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=sportphrase.index("<x>")
                                        
                                        sportphrase[pindex]=sport
                                        
                                    
                                        break
                                if insports==False:
                                    randomsportindex=random.randint(0,len(sportnames)-1) 
                                    sportphrase1=phrases[randomindex]
                                    sportphrase=copy.deepcopy(sportphrase1)
                                    pindex=phrases[randomindex].index("<x>")
                                    sportphrase[pindex]=sportnames[randomsportindex]

                                for word,label in zip(sportphrase,labels2[randomindex]):
                                

                                    currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                    currentexample.words.insert(currentexample.words.index(selword),word)
                                    currentexample.labels.insert(currentexample.words.index(selword),label)
                                firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex])
                                currentexample.labels[firstwordindex]='B-ORG'
                                r=0
                            #Regular insertion 
                            else: 
                            
                                for word,label in zip(phrases[randomindex],labels2[randomindex]):
                                    

                                    currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                    currentexample.words.insert(currentexample.words.index(selword),word)
                                    currentexample.labels.insert(currentexample.words.index(selword),label)
                                firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex])
                                currentexample.labels[firstwordindex]='B-ORG'
                            
                                

                        elif placement[randomindex]=='after':
                            currentexample=copy.deepcopy(examples[index])
                            if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                    if currentexample.words.index(selword)-1==0:
                                        currentexample.words[currentexample.words.index(selword)-1]="At"
                                    else:
                                        currentexample.words[currentexample.words.index(selword)-1]="at"
                            if currentexample.words.index(selword)==0:
                                currentexample.words.insert(currentexample.words.index(selword),"The")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                            else:
                                currentexample.words.insert(currentexample.words.index(selword),"the")
                                currentexample.labels.insert(currentexample.words.index(selword)-1,"O")


                            #Sport phrase code
                            if  "<x>" in phrases[randomindex]:
                                exwords = [word.lower() for word in currentexample.words]
                                insports=False
                                
                                for sport in sportnames:                               
                                    if sport.lower() in exwords :
                                        insports=True
                                        sportphrase1=phrases[randomindex]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=sportphrase.index("<x>")
                                        
                                        sportphrase[pindex]=sport
                            
                                    
                                        break
                                if insports==False:
                                    randomsportindex=random.randint(0,len(sportnames)-1) 
                                    sportphrase1=phrases[randomindex]
                                    sportphrase=copy.deepcopy(sportphrase1)
                                    pindex=phrases[randomindex].index("<x>")
                                    sportphrase[pindex]=sportnames[randomsportindex]

                                for word,label in zip(sportphrase,labels2[randomindex]):
                                

                                    currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                    currentexample.words.insert(currentexample.words.index(selword),word)
                                    currentexample.labels.insert(currentexample.words.index(selword),label)
                                firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex])
                                currentexample.labels[firstwordindex]='B-ORG'
                                

                            #Regular insertion 
                            else:                        
                    
                            
                                #Start inserting words
                                k=1
                                for word,label in zip(phrases[randomindex],labels2[randomindex]):
                                    currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                    currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                    currentexample.labels[currentexample.words.index(selword)]='B-ORG'
                                    k=k+1
                        augexamples.append(currentexample)
                        exampleguidsalreadyaugmented.append(ex.guid)
                        if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_1:
                            num_of_aug_examples_generated=0
                            break
                        num_of_aug_examples_generated=num_of_aug_examples_generated+1
                        break
                        
            if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_1:
                num_of_aug_examples_generated=0
                break
                        

            index=index+1
            

        
        dataloc = pd.read_excel ('data/conll2003/Transition To LOC.xlsx') 
        dfloc=list(dataloc['Pattern1'])
        dataloc2 = pd.read_excel ('data/conll2003/LOC Context Phrases.xlsx') 
        phrasesdfloc2=list(dataloc2['Pattern2'])
        labelsloc2=list(dataloc2['EntityType'])
        
        loclabels=[]
        phrasesloc=[]
        phrasesloc2=[]
        for locword in phrasesdfloc2:
            phrasesloc2.append(locword.split("|"))

        for label in labelsloc2:
            loclabels.append(label.replace(" ", "").split("|"))     
        
        for loc in dfloc:
            phrasesloc.append(loc.split(","))

        placementloc=list(dataloc['Placement'])

        labels3=[]
        labelset=[]
        for phrase, place in zip(phrasesloc, placementloc):
            if place=='before':
                first=True
                for word in phrase:
                    if first==True:
                        labelset.append('B-ORG')
                    else:
                        labelset.append('I-ORG')
            elif place=='after':
                
                for word in phrase:
                    labelset.append('I-ORG')
            labels3.append(labelset)
            labelset=[]
        index=0
        toloccands=[]
        iccandbool=False
        toloccounter=0
        selectedword=''
        pattern1loctransitioncounter=0
        #while  pattern1loctransitioncounter <30:
            
        for e in examples:
            if toloccounter==70:
                break

            if e.guid not in exampleguidsalreadyaugmented:
                for phrase,placementlociter in zip(phrasesloc,placementloc):
                    
                    if(set(phrase).issubset(set( e.words))):
                        
                        if placementlociter=='after':
                            
                            entityindex=e.words.index(phrase[-1])+1

                            if e.labels[entityindex]=='I-ORG':
                                #match e guid with example guid
                                index=0
                                
                                
                                for ex in examples:
                                    if ex.guid==e.guid:
                                        currentexample=copy.deepcopy(examples[index])
                                        
                                        currentexample.labels[entityindex]='B-LOC'
                                        k=1
                                        
                                        for word in phrase:
                                            
                                            currentexample.words.pop(entityindex-k)
                                            currentexample.labels.pop(entityindex-k)
                                            
                                            k=k+1
                                        
                                        if currentexample.labels[currentexample.labels.index('B-LOC')+1]=='I-ORG':
                                            currentexample.labels[currentexample.labels.index('B-LOC')+1]='I-LOC'
                                        
                                        #pattern 2
                                        randomindex=random.randint(0,len(phrasesdfloc2)-1)
                                        selword=currentexample.labels.index('B-LOC')

                                        k=1
                                        
                                        for word,label in zip(phrasesloc2[randomindex],loclabels[randomindex]):
                                            currentexample.words.insert(selword+k,word)
                                            currentexample.labels.insert(selword+k,label)
                                            
                                            k=k+1

                                        augexamples.append(currentexample)
                                        #num_of_aug_examples_generated=num_of_aug_examples_generated+1
                                        exampleguidsalreadyaugmented.append(ex.guid)
                                        toloccounter=toloccounter+1
                                        break
                                        
                                        
                                    index=index+1
                            else:
                                break

                        elif placementlociter=='before':
                            entityindex=e.words.index(phrase[0])-1
                            if e.labels[entityindex]=='B-ORG':
                                #match e guid with example guid
                                index=0
                                
                                
                                for ex in examples:
                                    if ex.guid==e.guid:
                                        currentexample=copy.deepcopy(examples[index])
                                        
                                        currentexample.labels[entityindex]='B-LOC'
                                        
                                        for word in phrase:
                                            
                                            currentexample.words.pop(entityindex+1)
                                            currentexample.labels.pop(entityindex+1)
                                        
                                        if currentexample.labels[currentexample.labels.index('B-LOC')+1]=='I-ORG':
                                            currentexample.labels[currentexample.labels.index('B-LOC')+1]='I-LOC'
                                        
                                        #pattern 2
                                        randomindex=random.randint(0,len(phrasesdfloc2)-1)
                                        selword=currentexample.labels.index('B-LOC')

                                        k=1
                                        
                                        for word,label in zip(phrasesloc2[randomindex],loclabels[randomindex]):
                                            currentexample.words.insert(selword+k,word)
                                            currentexample.labels.insert(selword+k,label)
                                            
                                            k=k+1

                                        augexamples.append(currentexample)
                                        
                                        exampleguidsalreadyaugmented.append(ex.guid)
                                        #num_of_aug_examples_generated=num_of_aug_examples_generated+1
                                        toloccounter=toloccounter+1
                                        break
                                        
                                        
                                    index=index+1

                            else:
                                break


                        #if num_of_aug_examples_generated>=num_of_aug_examples_to_generate:
                        #    break

                    if e.guid in exampleguidsalreadyaugmented:
                        break
                        
                #if num_of_aug_examples_generated>=num_of_aug_examples_to_generate:
                #    break   

                            
        
   
      
   
        dataPER = pd.read_excel ('data/conll2003/PER Context Phrases for Hold Out Experiments.xlsx') 
        dataPER2 = pd.read_excel ('data/conll2003/PER Headline Phrases for Hold Out Experiments.xlsx') 
        dataORG = pd.read_excel ('data/conll2003/ORG Context Phrases for Hold Out Experiments.xlsx') 
        dfPER2=list(dataPER2['Pattern2'])
        labelsPER2=list(dataPER2['EntityType'])
        compatibility2=list(dataORG['Compatibility'])

        dataNames = pd.read_excel ('data/conll2003/Names.xlsx') 
        dfNames=list(dataNames['Names'])
        dfPER=list(dataPER['Pattern2'])
        perphrasedomains=list(dataPER['Domain'])
        orgphrasedomains=list(dataORG['Domain'])
        dfORG=list(dataORG['Pattern2'])
        labelsPER=list(dataPER['EntityType'])
        labelsORG=list(dataORG['EntityType'])
        perphrases=[] 
        orgphrases=[]
        perlabels=[]  
        orglabels=[]
        perphrases2=[] 
        perlabels2=[] 


        for per2 in dfPER2:
            perphrases2.append(per2.split("|"))

        for perlabel2 in labelsPER2:
            perlabels2.append(perlabel2.replace(" ", "").split("|"))

        for per in dfPER:
            perphrases.append(per.split("|"))
        
        for org in dfORG:
            orgphrases.append(org.split("|"))

        for label in labelsPER:
            perlabels.append(label.replace(" ", "").split("|"))
        for label in labelsORG:
            orglabels.append(label.replace(" ", "").split("|"))

        percands=[]
        orgcands=[]
        iccandbool=False
        i=0
        selectedword=''
        numofexamples=0
        pertransitioncounter=0
        orgtransitioncounter=0
        originallabel=''
        examplesguids=[]
        selectedwords=[]

        for ex in examples:
        
            e=random.choice(examples)
            if  e.guid not in exampleguidsalreadyaugmented:

                for word,label in zip(e.words,e.labels):
                                
                    nextwordinddex=e.words.index(word)+1
                    if nextwordinddex<len(e.words):
                        w=e.words[nextwordinddex]
                    else:
                        w=''

                    if 'B-ORG' in label or 'B-LOC' in label  and 'B-MISC' not in label and ')' not in w and '.' not in word:
                        iccandbool=True
                        selectedword=word
                        originallabel=label
                    elif iccandbool==True and label=='O' and i==1:
                        if e.guid not in examplesguids:
                        
                            examplesguids.append(e.guid)
                            selectedwords.append(selectedword)

                        break 
                    
                    if iccandbool==True:                
                        i=i+1        
                
                iccandbool=False
                i=0
                pertransitioncounter=pertransitioncounter+1
        index=0
        for ex in examples:
                
                if ex.guid in examplesguids and ex.guid not in exampleguidsalreadyaugmented:
                
                    for guid2,selword in zip(examplesguids,selectedwords):
                        if ex.guid==guid2:
                                            
                            
                            currentexample=copy.deepcopy(examples[index])
                            numbercount = sum(entry.isdigit() for entry in currentexample.words)
                            if len(currentexample.words)<4 and 'AT' not in currentexample.words or numbercount>3:
                                randomindex=random.randint(0,len(dfPER2)-1)
                                k=0
                                for word,label in zip(perphrases2[randomindex],perlabels2[randomindex]):
                                    currentexample.words.insert(0+k,word)
                                    currentexample.labels.insert(0+k,label)
                                    currentexample.labels[currentexample.words.index(selword)]='B-PER'
                                    k=k+1
                            else:
                                randomindex=random.randint(0,len(dfPER)-1)        
                                k=1
                                for word,label in zip(perphrases[randomindex],perlabels[randomindex]):
                                    currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                    currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                    currentexample.labels[currentexample.words.index(selword)]='B-PER'
                                    k=k+1

                            #Pattern 1
                            randomindex3=random.randint(0,len(dfNames)-1)
                            currentexample.words.insert(currentexample.words.index(selword)+1,dfNames[randomindex3])
                            currentexample.labels.insert(currentexample.words.index(selword)+1,'I-PER')
                            augexamples.append(currentexample)
                            exampleguidsalreadyaugmented.append(currentexample.guid)
                            if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_3:
                                num_of_aug_examples_generated=0
                                break
                            num_of_aug_examples_generated=num_of_aug_examples_generated+1
                            
                            break

                index=index+1
                if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_3:
                    num_of_aug_examples_generated=0
                    break
        examplesguids=[]    
        selectedwords=[]

 
            
        
        for ex in examples:
           
            e=random.choice(examples)
            if  e.guid not in exampleguidsalreadyaugmented:
                for word,label in zip(e.words,e.labels):
                    if 'B-PER' in label or 'B-LOC' in label  and 'B-MISC' not in label:
                        iccandbool=True
                        selectedword=word
                        originallabel=label
                    elif iccandbool==True and label=='O' and i==1:
                    
                        if e.guid not in examplesguids:
                            examplesguids.append(e.guid)
                            selectedwords.append(selectedword)

                        break 
                    
                    if iccandbool==True:                
                        i=i+1        
                
                iccandbool=False
                i=0
                orgtransitioncounter=orgtransitioncounter+1

        j=0
        index=0
        for ex in examples:
                
                if ex.guid in examplesguids and ex.guid not in exampleguidsalreadyaugmented:
                    
                    for guid2,selword in zip(examplesguids,selectedwords):
                        if ex.guid==guid2:
                        

                            
                            currentexample=copy.deepcopy(examples[index])
                                                        
                            randomindex2=random.randint(0,len(phrases)-1)
                            randomindex=random.randint(0,len(orgphrases)-1)

                                    
                            k=1
                            for word,label in zip(orgphrases[randomindex],orglabels[randomindex]):
                                currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                currentexample.labels[currentexample.words.index(selword)]='B-ORG'
                                k=k+1
                            #Pattern 1 do the fix
                            
                            if placement[randomindex2]=='before':

                                if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                    if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                        if currentexample.words.index(selword)-1==0:
                                            currentexample.words[currentexample.words.index(selword)-1]="At"
                                        else:
                                            currentexample.words[currentexample.words.index(selword)-1]="at"
                                if currentexample.words.index(selword)==0:
                                    currentexample.words.insert(currentexample.words.index(selword),"The")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                                else:
                                    currentexample.words.insert(currentexample.words.index(selword),"the")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")



                                
                                #Sport phrase code
                                if  "<x>" in phrases[randomindex2]:
                                    exwords = [word.lower() for word in currentexample.words]
                                    insports=False
                                    
                                    for sport in sportnames:                               
                                        if sport.lower() in exwords :
                                            insports=True
                                            sportphrase1=phrases[randomindex2]
                                            sportphrase=copy.deepcopy(sportphrase1)
                                            pindex=sportphrase.index("<x>")
                                            
                                            sportphrase[pindex]=sport
                                    
                                        
                                            break
                                    if insports==False:
                                        randomsportindex=random.randint(0,len(sportnames)-1) 
                                        sportphrase1=phrases[randomindex2]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=phrases[randomindex2].index("<x>")
                                        sportphrase[pindex]=sportnames[randomsportindex]

                                    for word,label in zip(sportphrase,labels2[randomindex2]):
                                    

                                        currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                        currentexample.words.insert(currentexample.words.index(selword),word)
                                        currentexample.labels.insert(currentexample.words.index(selword),label)
                                    firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex2])
                                    currentexample.labels[firstwordindex]='B-ORG'
                                    

                                #Regular insertion 
                                else:
                                        
                                
                                    for word,label in zip(phrases[randomindex2],labels2[randomindex2]):
                                        currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                        currentexample.words.insert(currentexample.words.index(selword),word)
                                        currentexample.labels.insert(currentexample.words.index(selword),label)
                                    firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex2])
                                    currentexample.labels[firstwordindex]='B-ORG'


                            elif placement[randomindex2]=='after':
                                if "B-LOC" == currentexample.labels[currentexample.words.index(selword)]:
                                    if currentexample.words[currentexample.words.index(selword)-1].lower()=="in":
                                        if currentexample.words.index(selword)-1==0:
                                            currentexample.words[currentexample.words.index(selword)-1]="At"
                                        else:
                                            currentexample.words[currentexample.words.index(selword)-1]="at"
                                if currentexample.words.index(selword)==0:
                                    currentexample.words.insert(currentexample.words.index(selword),"The")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")
                                else:
                                    currentexample.words.insert(currentexample.words.index(selword),"the")
                                    currentexample.labels.insert(currentexample.words.index(selword)-1,"O")


                                
            
                                #Sport phrase code
                                if  "<x>" in phrases[randomindex2]:
                                    exwords = [word.lower() for word in currentexample.words]
                                    insports=False
                            
                                    for sport in sportnames:                               
                                        if sport.lower() in exwords :
                                            insports=True
                                            sportphrase1=phrases[randomindex2]
                                            sportphrase=copy.deepcopy(sportphrase1)
                                            pindex=sportphrase.index("<x>")
                                            
                                            sportphrase[pindex]=sport
                                        
                                        
                                            break
                                    if insports==False:
                                        randomsportindex=random.randint(0,len(sportnames)-1) 
                                        sportphrase1=phrases[randomindex2]
                                        sportphrase=copy.deepcopy(sportphrase1)
                                        pindex=phrases[randomindex2].index("<x>")
                                        sportphrase[pindex]=sportnames[randomsportindex]

                                    for word,label in zip(sportphrase,labels2[randomindex2]):
                                    

                                        currentexample.labels[currentexample.words.index(selword)]='I-ORG'
                                        currentexample.words.insert(currentexample.words.index(selword),word)
                                        currentexample.labels.insert(currentexample.words.index(selword),label)
                                    firstwordindex=currentexample.words.index(selword)-len(phrases[randomindex2])
                                    currentexample.labels[firstwordindex]='B-ORG'
                                    

                                #Regular insertion 
                                else:
                        
                                    k=1
                                    for word,label in zip(phrases[randomindex2],labels2[randomindex2]):
                                        currentexample.words.insert(currentexample.words.index(selword)+k,word)
                                        currentexample.labels.insert(currentexample.words.index(selword)+k,label)
                                        currentexample.labels[currentexample.words.index(selword)]='B-ORG'
                                        k=k+1
                                        
                            exampleguidsalreadyaugmented.append(currentexample.guid)
                            augexamples.append(currentexample)
                            if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_4:
                                num_of_aug_examples_generated=0
                                break
                            num_of_aug_examples_generated=num_of_aug_examples_generated+1
                            

                            break

                index=index+1
                
                if num_of_aug_examples_generated>=num_of_aug_examples_to_generate_for_aug_type_4:
                    num_of_aug_examples_generated=0
                    break


        logger.info("Number of augmented examples: %s", len(augexamples))
        file_name=percentagename+".pkl"
        file_path = os.path.join(args.data_dir, file_name)
        open_file = open(file_path, "wb")
        pickle.dump(augexamples, open_file)
        open_file.close()
     

def generate_aug_percentage_categories(args, tokenizer, labels, pad_token_label_id, mode,  
              omit_sep_cls_token=False,
              pad_subtoken_with_real_label=False):   

    read_data_rule_based_aug_for_zero_shot(args, 1,"",tokenizer, labels, pad_token_label_id, mode = 'train', pad_subtoken_with_real_label=args.pad_subtoken_with_real_label) 

    #countaug=count_rule_based_aug(args, tokenizer, labels, pad_token_label_id, mode = 'train', pad_subtoken_with_real_label=args.pad_subtoken_with_real_label)

    #countaug5percent=countaug*0.05
    #file_name = "augexamples5percent"
    #read_data_rule_based_aug(args, countaug5percent,file_name,tokenizer, labels, pad_token_label_id, mode = 'train', pad_subtoken_with_real_label=args.pad_subtoken_with_real_label) 

    #countaug10percent=countaug*0.1
    #file_name = "augexamples10percent"
    #read_data_rule_based_aug(args, countaug10percent,file_name,tokenizer, labels, pad_token_label_id, mode = 'train', pad_subtoken_with_real_label=args.pad_subtoken_with_real_label) 

    #countaug30percent=countaug*0.3
    #file_name = "augexamples30percent"
    #read_data_rule_based_aug(args, countaug30percent,file_name,tokenizer, labels, pad_token_label_id, mode = 'train', pad_subtoken_with_real_label=args.pad_subtoken_with_real_label)

    #countaug50percent=countaug*0.5
    #file_name = "augexamples50percent"
    #read_data_rule_based_aug(args, countaug50percent,file_name,tokenizer, labels, pad_token_label_id, mode = 'train', pad_subtoken_with_real_label=args.pad_subtoken_with_real_label)


def read_examples_from_ontonotes_file(args, tokenizer, labelslist, pad_token_label_id, mode, 
              omit_sep_cls_token=False,
              pad_subtoken_with_real_label=False):
    file_path = os.path.join(args.data_dir, 'ontotrain.ner')
    guid_index = 0
    examples = []
    
    label_map = {"PER": "PER",
             "person": "PER",
             "PERSON": "PER",
             "DOCTOR": "PER",
             "PATIENT": "PER",
             "LOC": "LOC",
             "location": "LOC",
             "GPE": "LOC",
             "FAC": "LOC",
             "HOSPITAL": "LOC", 
             "CITY": "LOC",
             "STATE": "LOC",
             "COUNTRY": "LOC",
             "LOCATION_OTHER": "LOC",
             "ORG": "ORG",
             "corporation": "ORG",
             "group": "ORG",
             "ORGANIZATION": "ORG",
             "MISC": "MISC",
             "product": "MISC",
             "creative-work": "MISC",
             "PRODUCT": "MISC", 
             "NORP": "MISC",
             "EVENT": "MISC",
             "LANGUAGE": "MISC",
             "LAW": "MISC",
             "WORK_OF_ART": "MISC"}
        
    
    with open(file_path, encoding="utf-8") as f:
        words = []
        labels = []
        for line in f:
            if line.startswith("-DOCSTART-") or line == "" or line == "\n":
                if words:
                                      
                    examples.append(InputExample(
                        guid="{}-{}".format(mode, guid_index), words=words, labels=labels))
                    guid_index += 1
                    words = []
                    labels = []
                    
            else:
                elems = line.strip().split()
                label = elems[-1].replace("\n", "")
                words.append(elems[0])
               
                if label != "O":
                    iob_prefix = label[:2]
                    etype = label[2:]
                    if etype in label_map:
                        labels.append(iob_prefix + label_map[etype])
                    else:
                        labels.append("O")
                        
                else:
                    labels.append("O")
        if words:
            
            examples.append(InputExample(
                guid="{}-{}".format(mode, guid_index), words=words, labels=labels))
    

    
    return examples
          

def read_examples_from_file_excel_LOC(data_dir, mode):
    file_path = data_dir
    guid_index = 0
    examples = []

                

    
    
    import pandas as pd

    
    train = pd.read_excel(file_path)
    firstrun=True
    exampleindex=5
    examples = []
    guidloopstop=''
    guid_index=0
    while guidloopstop!='train-41737':

        if firstrun:
            guidloopstop='train-1'
            qualityr=train.iloc[0]     
            quality=qualityr.iloc[1]
            augtyper=train.iloc[1]     
            augtype=augtyper.iloc[1]
            wordr=train.iloc[2]  
            words=[]
            labelsr=train.iloc[3]  
            labels=[]
            i=1
            while 1==1:
                c=wordr.iloc[i]
                label=labelsr.iloc[i]
                if c!='^':
                    words.append(c)
                    labels.append(label)
                    i=i+1
                else:
                    break
            if    quality==1:
                examples.append(InputExample(
                    guid=guidloopstop, words=words, labels=labels))
                guid_index += 1
            firstrun=False
        else:
            
            guidr=train.iloc[exampleindex]     
            guidloopstop=guidr.iloc[1]
            
            
            qualityr=train.iloc[exampleindex+1]     
            quality=qualityr.iloc[1]
            augtyper=train.iloc[exampleindex+2]     
            augtype=augtyper.iloc[1]
            wordr=train.iloc[exampleindex+3]  
            words=[]
            labelsr=train.iloc[exampleindex+4]  
            labels=[]
            i=1
            while 1==1:
                c=wordr.iloc[i]
                label=labelsr.iloc[i]
                if c!='^':
                    if c=='s':
                        c='\'s'
                    words.append(c)
                    labels.append(label)
                    i=i+1
                else:
                    break
            if    quality==1:
                examples.append(InputExample(
                    guid=guidloopstop, words=words, labels=labels))
                guid_index += 1
            exampleindex=exampleindex+6
        

    print("Number of examples ",len(examples))
    return examples  
   
    
    
    
def linear_rampup(current, rampup_length=args.num_train_epochs):
    if rampup_length == 0:
        return 1.0
    else:
        current = np.clip(current / rampup_length, 0.0, 1.0)
        return float(current)



def main():
    global best_f1
    if (os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir):
        raise ValueError( "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir))
    
    logger.setLevel(log.INFO)
    formatter = log.Formatter("%(asctime)s - %(levelname)s - %(name)s -   %(message)s", datefmt="%m/%d/%Y %H:%M:%S")
    
    if not os.path.exists(args.output_dir):
            os.makedirs(args.output_dir)
            
            
    fh = log.FileHandler(args.output_dir  +'/' + str(args.train_examples)+'-' + 'log.txt')
    fh.setLevel(log.INFO)
    fh.setFormatter(formatter)

    ch = log.StreamHandler()
    ch.setLevel(log.INFO)
    ch.setFormatter(formatter)

    logger.addHandler(ch)
    logger.addHandler(fh)
    
    logger.info("------NEW RUN-----")

    logger.info("device: %s, n_gpu: %s", args.device, args.n_gpu)

    set_seed(args)

    labels = get_labels(args.labels)
    num_labels = len(labels)
    args.num_labels=num_labels

    pad_token_label_id = CrossEntropyLoss().ignore_index





    args.model_type = args.model_type.lower()
    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]

  
    tokenizer = tokenizer_class.from_pretrained(args.model_name, do_lower_case=args.do_lower_case)

    
    config = config_class.from_pretrained(
        args.config_name if args.config_name else args.model_name,
        num_labels=num_labels,
    )
    tokenizer = tokenizer_class.from_pretrained(
        args.tokenizer_name if args.tokenizer_name else args.model_name,
        do_lower_case=args.do_lower_case,
    )


    logger.info("Training/evaluation parameters %s", args)
    

    generate_aug_percentage_categories(args, tokenizer, labels, pad_token_label_id, mode = 'train', pad_subtoken_with_real_label=args.pad_subtoken_with_real_label)        

    




main()

    