{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import spacy\n",
    "import metrics\n",
    "nlp = spacy.load(\"en_core_web_lg\")\n",
    "\n",
    "def compute_wpd_ld(s1, s2):\n",
    "    s1, s2 = nlp(s1), nlp(s2)\n",
    "    wpd = metrics.wpd(s1, s2)\n",
    "    ld = metrics.ld(s1, s2)\n",
    "    return wpd, ld"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.027149321266968323, 0.38888888888888884)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"A conviction could bring a maximum penalty of 10 years in prison and a $250,000 fine.\"\n",
    "s2 = \"If convicted, he faces a maximum penalty of 10 years in prison and a $250,000 fine.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.4950980392156862, 0.33333333333333337)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"The top rate will go to 4.45 percent for all residents with taxable incomes above $500,000.\"\n",
    "s2 = \"For residents with incomes above $500,000, the income-tax rate will increase to 4.45 percent.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.06176470588235296, 0.2857142857142857)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"However, prosecutors have declined to take criminal action against guards, though Fine said his inquiry is not finished.\"\n",
    "s2 = \"Prosecutors have declined to take criminal action against corrections officers, although Fine said his inquiry was not finished.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.4422084623323013, 0.21052631578947367)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"In trading on the New York Stock Exchange, Kraft shares fell 25 cents to close at $32.30.\"\n",
    "s2 = \"Kraft's shares fell 25 cents to close at $32.30 yesterday on the New York Stock Exchange.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.30968858131487886, 0.0)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"An attempt last month in the Senate to keep the fund open for another year fell flat.\"\n",
    "s2 = \"An attempt to keep the fund open for another year fell flat in the Senate last month.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.08974358974358976, 0.4285714285714286)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"Prisoners were tortured and executed -- their ears and scalps severed for souvenirs.\"\n",
    "s2 = \"They frequently tortured and shot prisoners, severing ears and scalps for souvenirs.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Appendix - MRPC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.03793322062552832, 0.04166666666666663)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"However, the talk was downplayed by PBL which said it would focus only on smaller purchases that were immediately earnings and cash flow-accretive.\"\n",
    "s2 = \"The talk, however, has been downplayed by PBL which said it would focus only on smaller purchases that were immediately earnings and cash flow-accretive.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.45066137566137565, 0.04347826086956519)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"With an estimated net worth of $1.7 billion, Mrs. Kroc ranked No. 121 on Forbes magazine's latest list of the nation's wealthiest people.\"\n",
    "s2 = \"Kroc ranked No. 121 on Forbes magazine's latest list of the nation's wealthiest people, with an estimated net worth of $1.7 billion.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.4087301587301587, 0.5625)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"Federal Emergency Management Administration designated $20 million to establish the registry.\"\n",
    "s2 = \"The registry was launched with $20 million from the Federal Emergency Management Agency.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.136437908496732, 0.6521739130434783)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"As a result, Nelson now faces up to a 10 year jail term instead of life.\"\n",
    "s2 = \"The verdict means Nelson faces up to 10 years in prison rather than a life sentence.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Appendix - PAWS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.02951388888888889, 0.06666666666666665)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"Brockton is approximately 25 miles northeast of Providence, Rhode Island, and 30 miles south of Boston.\"\n",
    "s2 = \"Brockton is located approximately 25 miles northeast of Providence, Rhode Island and 30 miles south of Boston.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.3010416666666666, 0.0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"Wollstonecraft arrived in Grenada on board the ship 'Sydney' on 31 August 1819.\"\n",
    "s2 = \"Wollstonecraft arrived on August 31, 1819 on board the ship 'Sydney' in Grenada.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.015029761904761905, 0.07692307692307687)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"Based on the city of Baltimore, only mentioned, never visited in the show.\"\n",
    "s2 = \"Based on the city of Baltimore, only mentioned, has never visited in the show.\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.02794316107009604, 0.375)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = \"The dividends have increased the total return on the average equity to double, approximately 3.2 %.\"\n",
    "s2 = \"The dividends increased the real 'total return' of the average equity to double, about 3.2 % .\"\n",
    "compute_wpd_ld(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\ndf = pd.read_csv(\"./dataset_stats.csv\")\\ndf.head()\\nFILTER_FOR = \"edit distance\"\\ndf = pd.read_csv(\"./dataset_stats.csv\")\\ndf = df[df[\"dataset\"]==\"mrpc\"]\\ndf = df[df[\"type\"]==FILTER_FOR]\\ndf.head()\\ndf = df[df[\"value\"]>0.5][df[\"value\"]<0.9]\\ndf.head()\\n\\nfor index, row in df.iterrows():\\n    s1, s2, value = row[\\'s1\\'], row[\\'s2\\'], row[\"value\"]\\n    if len(s1.split(\" \")) < 20:\\n        s1, s2 = nlp(s1), nlp(s2)\\n        wpd = metrics.wpd(s1, s2)\\n        ld = metrics.ld(s1, s2)\\n        if wpd > 0.3 and ld < 0.1:\\n            print(s1)\\n            print(s2)\\n            print(FILTER_FOR, value)\\n            print(\"WPD\", wpd)\\n            print(\"LD\", ld)\\n            print(\" \")\\n            \\nfor index, row in df.iterrows():\\n    s1, s2, value = row[\\'s1\\'], row[\\'s2\\'], row[\"value\"]\\n    if len(s1.split(\" \")) < 20:\\n        s1, s2 = nlp(s1), nlp(s2)\\n        wpd = metrics.wpd(s1, s2)\\n        ld = metrics.ld(s1, s2)\\n        if wpd < 0.1 and ld > 0.3:\\n            print(s1)\\n            print(s2)\\n            print(FILTER_FOR, value)\\n            print(\"WPD\", wpd)\\n            print(\"LD\", ld)\\n            print(\" \")\\n'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "df = pd.read_csv(\"./dataset_stats.csv\")\n",
    "df.head()\n",
    "FILTER_FOR = \"edit distance\"\n",
    "df = pd.read_csv(\"./dataset_stats.csv\")\n",
    "df = df[df[\"dataset\"]==\"mrpc\"]\n",
    "df = df[df[\"type\"]==FILTER_FOR]\n",
    "df.head()\n",
    "df = df[df[\"value\"]>0.5][df[\"value\"]<0.9]\n",
    "df.head()\n",
    "\n",
    "for index, row in df.iterrows():\n",
    "    s1, s2, value = row['s1'], row['s2'], row[\"value\"]\n",
    "    if len(s1.split(\" \")) < 20:\n",
    "        s1, s2 = nlp(s1), nlp(s2)\n",
    "        wpd = metrics.wpd(s1, s2)\n",
    "        ld = metrics.ld(s1, s2)\n",
    "        if wpd > 0.3 and ld < 0.1:\n",
    "            print(s1)\n",
    "            print(s2)\n",
    "            print(FILTER_FOR, value)\n",
    "            print(\"WPD\", wpd)\n",
    "            print(\"LD\", ld)\n",
    "            print(\" \")\n",
    "            \n",
    "for index, row in df.iterrows():\n",
    "    s1, s2, value = row['s1'], row['s2'], row[\"value\"]\n",
    "    if len(s1.split(\" \")) < 20:\n",
    "        s1, s2 = nlp(s1), nlp(s2)\n",
    "        wpd = metrics.wpd(s1, s2)\n",
    "        ld = metrics.ld(s1, s2)\n",
    "        if wpd < 0.1 and ld > 0.3:\n",
    "            print(s1)\n",
    "            print(s2)\n",
    "            print(FILTER_FOR, value)\n",
    "            print(\"WPD\", wpd)\n",
    "            print(\"LD\", ld)\n",
    "            print(\" \")\n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
