{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e9adb96b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5006f42b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_runs = glob.glob(\"./results/*/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "96acd435",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_results = {\n",
    "    \"mrpc\": {},\n",
    "    \"mrpc_cor\": {},\n",
    "}\n",
    "\n",
    "model_name_list = []\n",
    "data_list = []\n",
    "f1_list = []\n",
    "\n",
    "train_runs = glob.glob(\"./results/*/\")\n",
    "\n",
    "for s in train_runs:\n",
    "    for r in [\"deberta-base\", \"deberta-large\"]:\n",
    "        data_results = glob.glob(s+r+\"/*/\")\n",
    "        for i in data_results:\n",
    "            try:\n",
    "                test_result = i+\"all_results.json\"\n",
    "                with open(test_result) as f:\n",
    "                      data = json.load(f)\n",
    "                test_f1 = data[\"test_f1\"]\n",
    "                model_name = i.split(\"/\")[3]\n",
    "                model_name = model_name.replace(\".\", \"/\")\n",
    "                dataset = i.split(\"/\")[2]\n",
    "                try:\n",
    "                    _ = test_results[dataset][model_name]\n",
    "                    test_results[dataset][model_name].append(test_f1*100)\n",
    "                except KeyError:\n",
    "                    test_results[dataset][model_name] = [test_f1*100]\n",
    "                short_model_name = model_name.split(\"/\")[-1]\n",
    "                if len(short_model_name) > 24:\n",
    "                    short_model_name = short_model_name[:20]+\"...\"\n",
    "                model_name_list.append(short_model_name)\n",
    "                data_list.append(dataset)\n",
    "                f1_list.append(test_f1*100)\n",
    "            except Exception as e:\n",
    "                print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "17055730",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mrpc': {'deberta-base': [30.651872399445217,\n",
       "   34.99924656025779,\n",
       "   33.38236523148467,\n",
       "   30.651872399445217,\n",
       "   30.651872399445217],\n",
       "  'deberta-large': [30.858830109071828,\n",
       "   30.82781119032309,\n",
       "   30.727004827505528,\n",
       "   31.65233018737682,\n",
       "   30.92149994500804]},\n",
       " 'mrpc_cor': {'deberta-base': [51.91433367122964,\n",
       "   50.62350636106743,\n",
       "   50.18168319594207,\n",
       "   51.21889899929324,\n",
       "   50.92234902982658],\n",
       "  'deberta-large': [47.7329151966549,\n",
       "   54.85379428572658,\n",
       "   52.33905334782787,\n",
       "   55.023573770880006,\n",
       "   52.14979298720822]}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "703fb1df",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(list(zip(model_name_list, data_list, f1_list)),\n",
    "                     columns =[\"model_name\", \"dataset\", \"test_f1\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "89739f29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 40124 characters]\n",
      "text/plain": [
       "<Figure size 1008x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 445,
       "width": 829
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(14,6))\n",
    "sns.boxplot(x=\"model_name\", y=\"test_f1\",\n",
    "            hue=\"dataset\", whis=20,\n",
    "            data=df.sort_values([\"model_name\"], ascending=False))\n",
    "plt.xticks(rotation=90)\n",
    "plt.title(\"SciCite Test F1 Score (All Models)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1b44d828",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mrpc deberta-base 32.07\n",
      "mrpc deberta-large 31.0\n",
      "mrpc_cor deberta-base 50.97\n",
      "mrpc_cor deberta-large 52.42\n"
     ]
    }
   ],
   "source": [
    "for d in [\"mrpc\", \"mrpc_cor\"]:\n",
    "    split = test_results[d]\n",
    "    model_name_list = split.keys()\n",
    "    for m in model_name_list:\n",
    "        result = split[m]\n",
    "        mean = round(sum(result)/len(result), 2)\n",
    "        print(d, m, mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2d585ec7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mrpc deberta-base 33.38\n",
      "mrpc deberta-large 30.92\n",
      "mrpc_cor deberta-base 51.22\n",
      "mrpc_cor deberta-large 54.85\n"
     ]
    }
   ],
   "source": [
    "for d in [\"mrpc\", \"mrpc_cor\"]:\n",
    "    split = test_results[d]\n",
    "    model_name_list = split.keys()\n",
    "    for m in model_name_list:\n",
    "        result = split[m]\n",
    "        result.sort()\n",
    "        #print(result)\n",
    "        median = round(result[3],2)\n",
    "        print(d, m, median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07eed802",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9cd477e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
