{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f80e72cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2e9bc1ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_TO_TRAIN_FILE = \"./msr_paraphrase_train.txt\"\n",
    "PATH_TO_TEST_FILE = \"./msr_paraphrase_test.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ae31d75a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_and_correct(FILE_PATH):\n",
    "    lines = [line.rstrip(\"\\n\") for line in open(FILE_PATH)]\n",
    "    # skip header\n",
    "    lines = lines[1:]\n",
    "    s1_list = []\n",
    "    s2_list = []\n",
    "    label_list = []\n",
    "    for line in lines:\n",
    "        label, id1, id2, s1, s2 = line.split(\"\\t\")\n",
    "        assert len(s1) > 4 and len(s2) > 4\n",
    "        s1_list.append(s1.strip())\n",
    "        s2_list.append(s2.strip())\n",
    "        label_list.append(int(label))\n",
    "        df = pd.DataFrame(list(zip(s1_list, s2_list, label_list)),\n",
    "                          columns =[\"s1\", \"s2\", \"label\"])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5e3fcf69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4071</th>\n",
       "      <td>\"At this point, Mr. Brando announced: 'Somebod...</td>\n",
       "      <td>Brando said that \"somebody ought to put a bull...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4072</th>\n",
       "      <td>Martin, 58, will be freed today after serving ...</td>\n",
       "      <td>Martin served two thirds of a five-year senten...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4073</th>\n",
       "      <td>\"We have concluded that the outlook for price ...</td>\n",
       "      <td>In a statement, the ECB said the outlook for p...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4074</th>\n",
       "      <td>The notification was first reported Friday by ...</td>\n",
       "      <td>MSNBC.com first reported the CIA request on Fr...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4075</th>\n",
       "      <td>The 30-year bond US30YT=RR rose 22/32 for a yi...</td>\n",
       "      <td>The 30-year bond US30YT=RR grew 1-3/32 for a y...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     s1  \\\n",
       "4071  \"At this point, Mr. Brando announced: 'Somebod...   \n",
       "4072  Martin, 58, will be freed today after serving ...   \n",
       "4073  \"We have concluded that the outlook for price ...   \n",
       "4074  The notification was first reported Friday by ...   \n",
       "4075  The 30-year bond US30YT=RR rose 22/32 for a yi...   \n",
       "\n",
       "                                                     s2  label  \n",
       "4071  Brando said that \"somebody ought to put a bull...      1  \n",
       "4072  Martin served two thirds of a five-year senten...      0  \n",
       "4073  In a statement, the ECB said the outlook for p...      1  \n",
       "4074  MSNBC.com first reported the CIA request on Fr...      1  \n",
       "4075  The 30-year bond US30YT=RR grew 1-3/32 for a y...      0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = read_and_correct(PATH_TO_TRAIN_FILE)\n",
    "df.to_csv(\"./mrpc_train.csv\")\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3e7de4f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1720</th>\n",
       "      <td>After Hughes refused to rehire Hernandez, he c...</td>\n",
       "      <td>Hernandez filed an Equal Employment Opportunit...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1721</th>\n",
       "      <td>There are 103 Democrats in the Assembly and 47...</td>\n",
       "      <td>Democrats dominate the Assembly while Republic...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1722</th>\n",
       "      <td>Bethany Hamilton remained in stable condition ...</td>\n",
       "      <td>Bethany, who remained in stable condition afte...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1723</th>\n",
       "      <td>Last week the power station’s US owners, AES C...</td>\n",
       "      <td>The news comes after Drax's American owner, AE...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1724</th>\n",
       "      <td>Sobig.F spreads when unsuspecting computer use...</td>\n",
       "      <td>The virus spreads when unsuspecting computer u...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     s1  \\\n",
       "1720  After Hughes refused to rehire Hernandez, he c...   \n",
       "1721  There are 103 Democrats in the Assembly and 47...   \n",
       "1722  Bethany Hamilton remained in stable condition ...   \n",
       "1723  Last week the power station’s US owners, AES C...   \n",
       "1724  Sobig.F spreads when unsuspecting computer use...   \n",
       "\n",
       "                                                     s2  label  \n",
       "1720  Hernandez filed an Equal Employment Opportunit...      0  \n",
       "1721  Democrats dominate the Assembly while Republic...      0  \n",
       "1722  Bethany, who remained in stable condition afte...      0  \n",
       "1723  The news comes after Drax's American owner, AE...      1  \n",
       "1724  The virus spreads when unsuspecting computer u...      1  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = read_and_correct(PATH_TO_TEST_FILE)\n",
    "df.to_csv(\"./mrpc_test.csv\")\n",
    "df.tail()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
