{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "from tqdm import tqdm\n",
    "import transformers\n",
    "import pandas as pd\n",
    "import metrics\n",
    "import json\n",
    "import seaborn\n",
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_lg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "s1_list = []\n",
    "s2_list = []\n",
    "source_lengths = []\n",
    "target_lengths = []\n",
    "split = \"train\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd90a6f0e4974b08a5313f0fd5c7793b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.20k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "448f8a09256b46eb9dafd7fb8c11b59b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/792k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a411e8add11452ebc488ab1f0e7d81b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.39M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = transformers.AutoTokenizer.from_pretrained(\"t5-large\")\n",
    "\n",
    "def get_len(text):\n",
    "    return len(tokenizer.encode(text))\n",
    "\n",
    "def create_pair(s, t):\n",
    "    line = {\n",
    "        \"translation\": {\n",
    "            \"en\": s,\n",
    "            \"pa\": t\n",
    "        }\n",
    "    }\n",
    "    source_lengths.append(get_len(s))\n",
    "    target_lengths.append(get_len(t))\n",
    "    with open(\"./\"+split+\"lines.json\", 'a+') as outfile:\n",
    "        json.dump(line, outfile)\n",
    "        outfile.write(\"\\n\")\n",
    "    line = {\n",
    "        \"translation\": {\n",
    "            \"en\": t,\n",
    "            \"pa\": s\n",
    "        }\n",
    "    }\n",
    "    with open(\"./\"+split+\"lines.json\", 'a+') as outfile:\n",
    "        json.dump(line, outfile)\n",
    "        outfile.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MRPC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2753, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Amrozi accused his brother, whom he called \"th...</td>\n",
       "      <td>Referring to him as only \"the witness\", Amrozi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>They had published an advertisement on the Int...</td>\n",
       "      <td>On June 10, the ship's owners had published an...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>The stock rose $2.11, or about 11 percent, to ...</td>\n",
       "      <td>PG&amp;E Corp. shares jumped $1.63 or 8 percent to...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>Revenue in the first quarter of the year dropp...</td>\n",
       "      <td>With the scandal hanging over Stewart's compan...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>The DVD-CCA then appealed to the state Supreme...</td>\n",
       "      <td>The DVD CCA appealed that decision to the U.S....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                                 s1  \\\n",
       "0           0  Amrozi accused his brother, whom he called \"th...   \n",
       "2           2  They had published an advertisement on the Int...   \n",
       "4           4  The stock rose $2.11, or about 11 percent, to ...   \n",
       "5           5  Revenue in the first quarter of the year dropp...   \n",
       "7           7  The DVD-CCA then appealed to the state Supreme...   \n",
       "\n",
       "                                                  s2  label  \n",
       "0  Referring to him as only \"the witness\", Amrozi...      1  \n",
       "2  On June 10, the ship's owners had published an...      1  \n",
       "4  PG&E Corp. shares jumped $1.63 or 8 percent to...      1  \n",
       "5  With the scandal hanging over Stewart's compan...      1  \n",
       "7  The DVD CCA appealed that decision to the U.S....      1  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(\"./mrpc/mrpc_\"+split+\".csv\")\n",
    "df = df_train[df_train[\"label\"]==1]\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, row in df.iterrows():\n",
    "    s1, s2 = row[\"s1\"], row[\"s2\"]\n",
    "    s1 = metrics.preprocess_text(s1)\n",
    "    s2 = metrics.preprocess_text(s2)\n",
    "    s1_list.append(s1)\n",
    "    s2_list.append(s2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PAWS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25368, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentence1</th>\n",
       "      <th>sentence2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>The NBA season of 1975 -- 76 was the 30th seas...</td>\n",
       "      <td>The 1975 -- 76 season of the National Basketba...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>When comparable rates of flow can be maintaine...</td>\n",
       "      <td>The results are high when comparable flow rate...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>It is the seat of Zerendi District in Akmola R...</td>\n",
       "      <td>It is the seat of the district of Zerendi in A...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>William Henry Henry Harman was born on 17 Febr...</td>\n",
       "      <td>William Henry Harman was born in Waynesboro , ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>With a discrete amount of probabilities Formul...</td>\n",
       "      <td>Given a discrete set of probabilities formula ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                          sentence1  \\\n",
       "1   2  The NBA season of 1975 -- 76 was the 30th seas...   \n",
       "3   4  When comparable rates of flow can be maintaine...   \n",
       "4   5  It is the seat of Zerendi District in Akmola R...   \n",
       "5   6  William Henry Henry Harman was born on 17 Febr...   \n",
       "7   8  With a discrete amount of probabilities Formul...   \n",
       "\n",
       "                                           sentence2  label  \n",
       "1  The 1975 -- 76 season of the National Basketba...      1  \n",
       "3  The results are high when comparable flow rate...      1  \n",
       "4  It is the seat of the district of Zerendi in A...      1  \n",
       "5  William Henry Harman was born in Waynesboro , ...      1  \n",
       "7  Given a discrete set of probabilities formula ...      1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if split==\"train\":\n",
    "    df_train = pd.read_csv(\"./paws/train.tsv\", sep=\"\\t\")\n",
    "    df_train = df_train[df_train[\"label\"]==1]\n",
    "    df_dev = pd.read_csv(\"./paws/dev.tsv\", sep=\"\\t\")\n",
    "    df_dev = df_dev[df_dev[\"label\"]==1]\n",
    "    df = pd.concat([df_train, df_dev])\n",
    "else:\n",
    "    df_dev = pd.read_csv(\"./paws/test.tsv\", sep=\"\\t\")\n",
    "    df = df_dev[df_dev[\"label\"]==1]\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, row in df.iterrows():\n",
    "    s1 = metrics.preprocess_text(row[\"sentence1\"])\n",
    "    s2 = metrics.preprocess_text(row[\"sentence2\"])\n",
    "    s1_list.append(s1)\n",
    "    s2_list.append(s2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28121/28121 [03:43<00:00, 125.75it/s]\n"
     ]
    }
   ],
   "source": [
    "for s, t in tqdm(zip(s1_list, s2_list), total=len(s1_list)):\n",
    "    s, t = nlp(s), nlp(t)\n",
    "    wpd = metrics.wpd(s, t)\n",
    "    if wpd > 0.2:\n",
    "        create_pair(str(s), str(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<seaborn.axisgrid.FacetGrid at 0x7f40657c66a0>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7604 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "seaborn.displot(source_lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<seaborn.axisgrid.FacetGrid at 0x7f4020789790>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 7784 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "seaborn.displot(target_lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
