{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = \"retina\"\n",
    "import transformers\n",
    "from tqdm import tqdm\n",
    "\n",
    "def preprocess_text(input_text):\n",
    "    input_text = str(input_text).strip()\n",
    "    input_text = input_text.replace(\"``\", \"''\").replace(\"‘‘\", '\"').replace(\"’’\", '\"').replace(\"''\", '\"')\n",
    "    input_text = input_text.replace(\"[\", \"\").replace(\"]\", \"\")\n",
    "    input_text = input_text.replace(\" .\", \".\").replace(\" ,\", \",\")\n",
    "    input_text = input_text.replace(\"’\", \"'\").replace(\"“\", '\"').replace(\"”\", '\"')\n",
    "    return input_text.replace(\"  \", \" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 128\n",
    "MODEL_NAME = \"microsoft/deberta-large\"\n",
    "LEARNING_RATE = 5e-6\n",
    "BATCH_SIZE = 128\n",
    "TRAIN_DATASET = \"MRPC_COR\"\n",
    "EPOCHS = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL_NAME)\n",
    "\n",
    "def tokenize_text(t1, t2):\n",
    "    o = tokenizer(t1, t2,\n",
    "                  padding=\"max_length\",\n",
    "                  truncation=\"longest_first\",\n",
    "                  max_length=MAX_SEQUENCE_LENGTH,\n",
    "                  return_token_type_ids=True,\n",
    "                  return_attention_mask=True)\n",
    "    return o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sent_1_list, train_sent_2_list, train_labels_raw = [], [], []\n",
    "\n",
    "if TRAIN_DATASET==\"PAWS\":\n",
    "    with open(\"../datasets/paws/train.tsv\", \"r\") as csvfile:\n",
    "        reader = csv.reader(csvfile, delimiter='\\t')\n",
    "        next(reader, None)  # skip the headers\n",
    "        for row in reader:\n",
    "            label = int(row[3])\n",
    "            train_sent_1_list.append(preprocess_text(row[1]))\n",
    "            train_sent_2_list.append(preprocess_text(row[2]))\n",
    "            train_labels_raw.append(label)\n",
    "if TRAIN_DATASET==\"MRPC\":\n",
    "    with open(\"../datasets/mrpc_train_processed.csv\", \"r\") as csvfile:\n",
    "        reader = csv.reader(csvfile)\n",
    "        next(reader, None)  # skip the headers\n",
    "        for row in reader:\n",
    "            label = int(row[2])\n",
    "            train_sent_1_list.append(preprocess_text(row[0]))\n",
    "            train_sent_2_list.append(preprocess_text(row[1]))\n",
    "            assert len(row[1]) > 5\n",
    "            train_labels_raw.append(label)\n",
    "if TRAIN_DATASET==\"MRPC_COR\":\n",
    "    with open(\"../datasets/mrpc_train_corrected.csv\", \"r\") as csvfile:\n",
    "        reader = csv.reader(csvfile)\n",
    "        next(reader, None)  # skip the headers\n",
    "        for row in reader:\n",
    "            label = int(row[-1])\n",
    "            train_sent_1_list.append(preprocess_text(row[0]))\n",
    "            train_sent_2_list.append(preprocess_text(row[1]))\n",
    "            assert len(row[1]) > 5\n",
    "            train_labels_raw.append(label)\n",
    "\n",
    "train_tokens = []\n",
    "train_type_ids = []\n",
    "train_attn_masks = []\n",
    "train_labels = []\n",
    "\n",
    "for t1, t2, l in zip(train_sent_1_list, train_sent_2_list, train_labels_raw):\n",
    "    t = tokenize_text(t1, t2)\n",
    "    train_tokens.append(t[\"input_ids\"])\n",
    "    train_type_ids.append(t[\"token_type_ids\"])\n",
    "    train_attn_masks.append(t[\"attention_mask\"])\n",
    "    train_labels.append(l)\n",
    "    t = tokenize_text(t2, t1)\n",
    "    train_tokens.append(t[\"input_ids\"])\n",
    "    train_type_ids.append(t[\"token_type_ids\"])\n",
    "    train_attn_masks.append(t[\"attention_mask\"])\n",
    "    train_labels.append(l)\n",
    "    \n",
    "train_tokens = np.asarray(train_tokens)\n",
    "train_type_ids = np.asarray(train_type_ids)\n",
    "train_attn_masks = np.asarray(train_attn_masks)\n",
    "train_labels = np.asarray(train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_sent_1_list, valid_sent_2_list, valid_labels_raw = [], [], []\n",
    "        \n",
    "with open(\"../datasets/paws/dev.tsv\", \"r\") as csvfile:\n",
    "    reader = csv.reader(csvfile, delimiter='\\t')\n",
    "    next(reader, None)  # skip the headers\n",
    "    for row in reader:\n",
    "        label = int(row[3])\n",
    "        valid_sent_1_list.append(preprocess_text(row[1]))\n",
    "        valid_sent_2_list.append(preprocess_text(row[2]))\n",
    "        valid_labels_raw.append(label)\n",
    "\n",
    "valid_tokens = []\n",
    "valid_type_ids = []\n",
    "valid_attn_masks = []\n",
    "valid_labels = []\n",
    "\n",
    "for t1, t2, l in zip(valid_sent_1_list, valid_sent_2_list, valid_labels_raw):\n",
    "    t = tokenize_text(t1, t2)\n",
    "    valid_tokens.append(t[\"input_ids\"])\n",
    "    valid_type_ids.append(t[\"token_type_ids\"])\n",
    "    valid_attn_masks.append(t[\"attention_mask\"])\n",
    "    valid_labels.append(l)\n",
    "    t = tokenize_text(t2, t1)\n",
    "    valid_tokens.append(t[\"input_ids\"])\n",
    "    valid_type_ids.append(t[\"token_type_ids\"])\n",
    "    valid_attn_masks.append(t[\"attention_mask\"])\n",
    "    valid_labels.append(l)\n",
    "    \n",
    "valid_tokens = np.asarray(valid_tokens)\n",
    "valid_type_ids = np.asarray(valid_type_ids)\n",
    "valid_attn_masks = np.asarray(valid_attn_masks)\n",
    "valid_labels = np.asarray(valid_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of label tensor: (8870,)\n"
     ]
    }
   ],
   "source": [
    "train_labels = np.asarray(train_labels)\n",
    "print('Shape of label tensor:', train_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of entries in each category:\n",
      "Training: 2680\n",
      "Validation: 7078\n"
     ]
    }
   ],
   "source": [
    "print('Number of entries in each category:')\n",
    "print(\"Training:\", train_labels.sum(axis=0))\n",
    "print(\"Validation:\", valid_labels.sum(axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "class TextDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, tokens, type_ids, attn_masks, labels):\n",
    "        self.tokens = tokens\n",
    "        self.type_ids = type_ids\n",
    "        self.attn_masks = attn_masks\n",
    "        self.labels = labels\n",
    "    def __getitem__(self, idx):\n",
    "        item = {\n",
    "            \"input_ids\": torch.tensor(self.tokens[idx], dtype=torch.int64),\n",
    "            \"token_type_ids\": torch.tensor(self.type_ids[idx], dtype=torch.int64),\n",
    "            \"attention_mask\": torch.tensor(self.attn_masks[idx], dtype=torch.int64),\n",
    "        }\n",
    "        item['labels'] = torch.tensor(self.labels[idx], dtype=torch.int64)\n",
    "        return item\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "train_dataset = TextDataset(train_tokens, train_type_ids, train_attn_masks, train_labels)\n",
    "valid_dataset = TextDataset(valid_tokens, valid_type_ids, valid_attn_masks, valid_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS]Amrozi accused his brother, whom he called \"the witness\", of deliberately distorting his evidence.[SEP]Referring to him as only \"the witness\", Amrozi accused his brother of deliberately distorting his evidence.[SEP][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD]\n",
      "tensor(1)\n"
     ]
    }
   ],
   "source": [
    "d0 = train_dataset[0]\n",
    "print(tokenizer.decode(d0[\"input_ids\"]))\n",
    "print(d0[\"labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS]Bradd Crellin represented BARLA Cumbria on a tour of Australia with 6 other players representing Britain, also on a tour of Australia.[SEP]Bradd Crellin also represented BARLA Great Britain on a tour through Australia on a tour through Australia with 6 other players representing Cumbria.[SEP][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD]\n",
      "tensor(0)\n"
     ]
    }
   ],
   "source": [
    "d0 = valid_dataset[0]\n",
    "print(tokenizer.decode(d0[\"input_ids\"]))\n",
    "print(d0[\"labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForSequenceClassification, Trainer, TrainingArguments\n",
    "from sklearn.metrics import precision_recall_fscore_support, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro')\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\n",
    "        'accuracy': acc,\n",
    "        'f1': f1,\n",
    "        'precision': precision,\n",
    "        'recall': recall\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at microsoft/deberta-large were not used when initializing DebertaForSequenceClassification: ['config', 'lm_predictions.lm_head.bias', 'lm_predictions.lm_head.LayerNorm.bias', 'lm_predictions.lm_head.dense.weight', 'lm_predictions.lm_head.LayerNorm.weight', 'lm_predictions.lm_head.dense.bias']\n",
      "- This IS expected if you are initializing DebertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DebertaForSequenceClassification were not initialized from the model checkpoint at microsoft/deberta-large and are newly initialized: ['classifier.bias', 'pooler.dense.bias', 'pooler.dense.weight', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n"
     ]
    }
   ],
   "source": [
    "LOGGING_STEP = 50\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    logging_dir='./logs',\n",
    "    overwrite_output_dir=True,\n",
    "    num_train_epochs=2,\n",
    "    per_device_train_batch_size=32,\n",
    "    per_device_eval_batch_size=128,\n",
    "    weight_decay=0.00,\n",
    "    warmup_ratio=0.4,\n",
    "    lr_scheduler_type=\"constant_with_warmup\",\n",
    "    eval_steps=LOGGING_STEP,\n",
    "    logging_steps=LOGGING_STEP,\n",
    "    save_steps=LOGGING_STEP,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    logging_strategy=\"steps\",\n",
    "    save_strategy=\"steps\",\n",
    "    save_total_limit=10,\n",
    "    learning_rate=1e-5,\n",
    "    fp16=True,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=\"eval_f1\",\n",
    "    greater_is_better=True,\n",
    "    disable_tqdm=False,\n",
    ")\n",
    "\n",
    "trainer = Trainer(model=model,\n",
    "                  args=training_args,\n",
    "                  train_dataset=train_dataset,\n",
    "                  eval_dataset=valid_dataset,\n",
    "                  compute_metrics=compute_metrics,\n",
    "                  callbacks=[transformers.EarlyStoppingCallback(early_stopping_patience=4,\n",
    "                                                                early_stopping_threshold=0.01),])\n",
    "trainer.remove_callback(transformers.integrations.TensorBoardCallback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 8870\n",
      "  Num Epochs = 2\n",
      "  Instantaneous batch size per device = 32\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 556\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='201' max='556' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [201/556 05:26 < 09:42, 0.61 it/s, Epoch 0.72/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.650000</td>\n",
       "      <td>0.784724</td>\n",
       "      <td>0.557562</td>\n",
       "      <td>0.357971</td>\n",
       "      <td>0.278799</td>\n",
       "      <td>0.499944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.581000</td>\n",
       "      <td>0.752105</td>\n",
       "      <td>0.469375</td>\n",
       "      <td>0.381191</td>\n",
       "      <td>0.580453</td>\n",
       "      <td>0.519786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>0.452700</td>\n",
       "      <td>1.087971</td>\n",
       "      <td>0.477875</td>\n",
       "      <td>0.392171</td>\n",
       "      <td>0.611344</td>\n",
       "      <td>0.528167</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "    <div>\n",
       "      \n",
       "      <progress value='85' max='125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [ 85/125 00:51 < 00:24, 1.64 it/s]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/transformers/trainer.py:1314: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  nn.utils.clip_grad_norm_(\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 16000\n",
      "  Batch size = 128\n",
      "Saving model checkpoint to ./results/checkpoint-50\n",
      "Configuration saved in ./results/checkpoint-50/config.json\n",
      "Model weights saved in ./results/checkpoint-50/pytorch_model.bin\n",
      "/opt/conda/lib/python3.8/site-packages/transformers/trainer.py:1314: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  nn.utils.clip_grad_norm_(\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 16000\n",
      "  Batch size = 128\n",
      "Saving model checkpoint to ./results/checkpoint-100\n",
      "Configuration saved in ./results/checkpoint-100/config.json\n",
      "Model weights saved in ./results/checkpoint-100/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 16000\n",
      "  Batch size = 128\n",
      "Saving model checkpoint to ./results/checkpoint-150\n",
      "Configuration saved in ./results/checkpoint-150/config.json\n",
      "Model weights saved in ./results/checkpoint-150/pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 16000\n",
      "  Batch size = 128\n",
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 3441, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"/tmp/ipykernel_1815/4032920361.py\", line 1, in <module>\n",
      "    trainer.train()\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\", line 1342, in train\n",
      "    self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\", line 1443, in _maybe_log_save_evaluate\n",
      "    metrics = self.evaluate(ignore_keys=ignore_keys_for_eval)\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\", line 2041, in evaluate\n",
      "    output = eval_loop(\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\", line 2210, in evaluation_loop\n",
      "    loss, logits, labels = self.prediction_step(model, inputs, prediction_loss_only, ignore_keys=ignore_keys)\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\", line 2414, in prediction_step\n",
      "    loss, outputs = self.compute_loss(model, inputs, return_outputs=True)\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\", line 1811, in compute_loss\n",
      "    outputs = model(**inputs)\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 1056, in _call_impl\n",
      "    return forward_call(*input, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/transformers/models/deberta/modeling_deberta.py\", line 1164, in forward\n",
      "    label_index = (labels >= 0).nonzero()\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/IPython/core/interactiveshell.py\", line 2061, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/IPython/core/ultratb.py\", line 1101, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/IPython/core/ultratb.py\", line 248, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.8/site-packages/IPython/core/ultratb.py\", line 281, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/opt/conda/lib/python3.8/inspect.py\", line 1515, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/opt/conda/lib/python3.8/inspect.py\", line 1473, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/opt/conda/lib/python3.8/inspect.py\", line 708, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/opt/conda/lib/python3.8/inspect.py\", line 754, in getmodule\n",
      "    os.path.realpath(f)] = module.__name__\n",
      "  File \"/opt/conda/lib/python3.8/posixpath.py\", line 391, in realpath\n",
      "    path, ok = _joinrealpath(filename[:0], filename, {})\n",
      "  File \"/opt/conda/lib/python3.8/posixpath.py\", line 425, in _joinrealpath\n",
      "    if not islink(newpath):\n",
      "  File \"/opt/conda/lib/python3.8/posixpath.py\", line 167, in islink\n",
      "    st = os.lstat(path)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "object of type 'NoneType' has no len()",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "    \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_1815/4032920361.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1342\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_log_save_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtr_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_keys_for_eval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36m_maybe_log_save_evaluate\u001b[0;34m(self, tr_loss, model, trial, epoch, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   1442\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_evaluate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1443\u001b[0;31m             \u001b[0mmetrics\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mignore_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mignore_keys_for_eval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1444\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_report_to_hp_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, eval_dataset, ignore_keys, metric_key_prefix)\u001b[0m\n\u001b[1;32m   2040\u001b[0m         \u001b[0meval_loop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprediction_loop\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_legacy_prediction_loop\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluation_loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2041\u001b[0;31m         output = eval_loop(\n\u001b[0m\u001b[1;32m   2042\u001b[0m             \u001b[0meval_dataloader\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mevaluation_loop\u001b[0;34m(self, dataloader, description, prediction_loss_only, ignore_keys, metric_key_prefix)\u001b[0m\n\u001b[1;32m   2209\u001b[0m             \u001b[0;31m# Prediction step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2210\u001b[0;31m             \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprediction_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprediction_loss_only\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mignore_keys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mprediction_step\u001b[0;34m(self, model, inputs, prediction_loss_only, ignore_keys)\u001b[0m\n\u001b[1;32m   2413\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mhas_labels\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2414\u001b[0;31m                     \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2415\u001b[0m                     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mcompute_loss\u001b[0;34m(self, model, inputs, return_outputs)\u001b[0m\n\u001b[1;32m   1810\u001b[0m             \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1811\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1812\u001b[0m         \u001b[0;31m# Save past state if it exists\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1055\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1056\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1057\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/models/deberta/modeling_deberta.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1163\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1164\u001b[0;31m                 \u001b[0mlabel_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnonzero\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1165\u001b[0m                 \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[1;32m   2060\u001b[0m                         \u001b[0;31m# in the engines. This should return a list of strings.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2061\u001b[0;31m                         \u001b[0mstb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_render_traceback_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2062\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'KeyboardInterrupt' object has no attribute '_render_traceback_'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "    \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[1;32m   2061\u001b[0m                         \u001b[0mstb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_render_traceback_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2062\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2063\u001b[0;31m                         stb = self.InteractiveTB.structured_traceback(etype,\n\u001b[0m\u001b[1;32m   2064\u001b[0m                                             value, tb, tb_offset=tb_offset)\n\u001b[1;32m   2065\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1365\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1366\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1367\u001b[0;31m         return FormattedTB.structured_traceback(\n\u001b[0m\u001b[1;32m   1368\u001b[0m             self, etype, value, tb, tb_offset, number_of_lines_of_context)\n\u001b[1;32m   1369\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1265\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose_modes\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1266\u001b[0m             \u001b[0;31m# Verbose modes need a full traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1267\u001b[0;31m             return VerboseTB.structured_traceback(\n\u001b[0m\u001b[1;32m   1268\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_offset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnumber_of_lines_of_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1269\u001b[0m             )\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1122\u001b[0m         \u001b[0;34m\"\"\"Return a nice text document describing the traceback.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1124\u001b[0;31m         formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,\n\u001b[0m\u001b[1;32m   1125\u001b[0m                                                                tb_offset)\n\u001b[1;32m   1126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mformat_exception_as_a_whole\u001b[0;34m(self, etype, evalue, etb, number_of_lines_of_context, tb_offset)\u001b[0m\n\u001b[1;32m   1080\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1081\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1082\u001b[0;31m         \u001b[0mlast_unique\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecursion_repeat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_recursion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morig_etype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1083\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1084\u001b[0m         \u001b[0mframes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat_records\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlast_unique\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecursion_repeat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.8/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mfind_recursion\u001b[0;34m(etype, value, records)\u001b[0m\n\u001b[1;32m    380\u001b[0m     \u001b[0;31m# first frame (from in to out) that looks different.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_recursion_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 382\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    383\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m     \u001b[0;31m# Select filename, lineno, func_name to track frames with\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: object of type 'NoneType' has no len()"
     ]
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name_short = MODEL_NAME.split(\"/\")[-1]\n",
    "model.save_pretrained(\"./\"+TRAIN_DATASET.lower()+\"_\"+model_name_short+\"/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
