{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = \"retina\"\n",
    "import transformers\n",
    "from tqdm import tqdm\n",
    "\n",
    "def preprocess_text(input_text):\n",
    "    input_text = str(input_text).strip()\n",
    "    input_text = input_text.replace(\"``\", \"''\").replace(\"‘‘\", '\"').replace(\"’’\", '\"').replace(\"''\", '\"')\n",
    "    input_text = input_text.replace(\"[\", \"\").replace(\"]\", \"\")\n",
    "    input_text = input_text.replace(\" .\", \".\").replace(\" ,\", \",\")\n",
    "    input_text = input_text.replace(\"’\", \"'\").replace(\"“\", '\"').replace(\"”\", '\"')\n",
    "    return input_text.replace(\"  \", \" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 128\n",
    "MODEL_NAME = \"microsoft/deberta-base\"\n",
    "LEARNING_RATE = 1e-5\n",
    "BATCH_SIZE = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL_NAME)\n",
    "\n",
    "def tokenize_text(t1, t2):\n",
    "    o = tokenizer(t1, t2,\n",
    "                  padding=\"max_length\",\n",
    "                  truncation=\"longest_first\",\n",
    "                  max_length=MAX_SEQUENCE_LENGTH,\n",
    "                  return_token_type_ids=True,\n",
    "                  return_attention_mask=True)\n",
    "    return o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_sent_1_list, valid_sent_2_list, valid_labels_raw = [], [], []\n",
    "        \n",
    "with open(\"../datasets/paws/test.tsv\", \"r\") as csvfile:\n",
    "    reader = csv.reader(csvfile, delimiter='\\t')\n",
    "    next(reader, None)  # skip the headers\n",
    "    for row in reader:\n",
    "        label = int(row[3])\n",
    "        valid_sent_1_list.append(preprocess_text(row[1]))\n",
    "        valid_sent_2_list.append(preprocess_text(row[2]))\n",
    "        valid_labels_raw.append(label)\n",
    "\n",
    "valid_tokens = []\n",
    "valid_type_ids = []\n",
    "valid_attn_masks = []\n",
    "valid_labels = []\n",
    "\n",
    "for t1, t2, l in zip(valid_sent_1_list, valid_sent_2_list, valid_labels_raw):\n",
    "    t = tokenize_text(t1, t2)\n",
    "    valid_tokens.append(t[\"input_ids\"])\n",
    "    valid_type_ids.append(t[\"token_type_ids\"])\n",
    "    valid_attn_masks.append(t[\"attention_mask\"])\n",
    "    valid_labels.append(l)\n",
    "    \n",
    "valid_tokens = np.asarray(valid_tokens)\n",
    "valid_type_ids = np.asarray(valid_type_ids)\n",
    "valid_attn_masks = np.asarray(valid_attn_masks)\n",
    "valid_labels = np.asarray(valid_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "class TextDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, tokens, type_ids, attn_masks, labels):\n",
    "        self.tokens = tokens\n",
    "        self.type_ids = type_ids\n",
    "        self.attn_masks = attn_masks\n",
    "        self.labels = labels\n",
    "    def __getitem__(self, idx):\n",
    "        item = {\n",
    "            \"input_ids\": torch.tensor(self.tokens[idx], dtype=torch.int64),\n",
    "            \"token_type_ids\": torch.tensor(self.type_ids[idx], dtype=torch.int64),\n",
    "            \"attention_mask\": torch.tensor(self.attn_masks[idx], dtype=torch.int64),\n",
    "        }\n",
    "        item['labels'] = torch.tensor(self.labels[idx], dtype=torch.int64)\n",
    "        return item\n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "test_dataset = TextDataset(valid_tokens, valid_type_ids, valid_attn_masks, valid_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS]This was a series of nested angular standards, so that measurements in azimuth and elevation could be done directly in polar coordinates relative to the ecliptic.[SEP]This was a series of nested polar scales, so that measurements in azimuth and elevation could be performed directly in angular coordinates relative to the ecliptic.[SEP][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD][PAD]\n",
      "tensor(0)\n"
     ]
    }
   ],
   "source": [
    "d0 = test_dataset[0]\n",
    "print(tokenizer.decode(d0[\"input_ids\"]))\n",
    "print(d0[\"labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForSequenceClassification, Trainer, TrainingArguments\n",
    "from sklearn.metrics import precision_recall_fscore_support, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = \"./model\"\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_path, num_labels=2).cuda()\n",
    "model.eval()\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset,\n",
    "                                          batch_size=BATCH_SIZE,\n",
    "                                          num_workers=1,\n",
    "                                          pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/125 [00:00<?, ?it/s][W pthreadpool-cpp.cc:99] Warning: Leaking Caffe2 thread-pool after fork. (function pthreadpool)\n",
      "100%|██████████| 125/125 [00:24<00:00,  5.13it/s]\n"
     ]
    }
   ],
   "source": [
    "acc_list = []\n",
    "\n",
    "for batch in tqdm(test_loader, total=len(test_loader)):\n",
    "    input_ids = batch['input_ids'].cuda(non_blocking=True)\n",
    "    attention_mask = batch['attention_mask'].cuda(non_blocking=True)\n",
    "    token_type_ids = batch['token_type_ids'].cuda(non_blocking=True)\n",
    "    outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids).logits\n",
    "    outputs = outputs.cpu().detach()\n",
    "    preds = torch.argmax(outputs, dim=1).numpy()\n",
    "    targets = batch[\"labels\"].numpy()\n",
    "    accuracy = accuracy_score(targets, preds)\n",
    "    acc_list.append(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.93425"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Accuracy\")\n",
    "sum(acc_list)/len(acc_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
