{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ded845f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "10a8729b",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_runs = glob.glob(\"./results/*/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "35cc7996",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_results = {\n",
    "    \"mrpc\": {},\n",
    "    \"mrpc_cor\": {},\n",
    "}\n",
    "\n",
    "model_name_list = []\n",
    "data_list = []\n",
    "f1_list = []\n",
    "\n",
    "train_runs = glob.glob(\"./results/*/\")\n",
    "\n",
    "for s in train_runs:\n",
    "    for r in [\"deberta-base\", \"deberta-large\"]:\n",
    "        data_results = glob.glob(s+r+\"/*/\")\n",
    "        for i in data_results:\n",
    "            try:\n",
    "                test_result = i+\"all_results.json\"\n",
    "                with open(test_result) as f:\n",
    "                      data = json.load(f)\n",
    "                test_f1 = data[\"valid_f1\"]\n",
    "                model_name = i.split(\"/\")[3]\n",
    "                model_name = model_name.replace(\".\", \"/\")\n",
    "                dataset = i.split(\"/\")[2]\n",
    "                try:\n",
    "                    _ = test_results[dataset][model_name]\n",
    "                    test_results[dataset][model_name].append(test_f1*100)\n",
    "                except KeyError:\n",
    "                    test_results[dataset][model_name] = [test_f1*100]\n",
    "                short_model_name = model_name.split(\"/\")[-1]\n",
    "                if len(short_model_name) > 24:\n",
    "                    short_model_name = short_model_name[:20]+\"...\"\n",
    "                model_name_list.append(short_model_name)\n",
    "                data_list.append(dataset)\n",
    "                f1_list.append(test_f1*100)\n",
    "            except Exception as e:\n",
    "                print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3742d20d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mrpc': {'deberta-base': [30.669902071236677,\n",
       "   37.55242460127764,\n",
       "   35.672076230282116,\n",
       "   31.811361221477704,\n",
       "   30.669902071236677],\n",
       "  'deberta-large': [30.808093970432644,\n",
       "   30.6889575495307,\n",
       "   30.739057476854853,\n",
       "   30.795154511543238,\n",
       "   30.789119147319465]},\n",
       " 'mrpc_cor': {'deberta-base': [53.256516874822815,\n",
       "   46.21856123569427,\n",
       "   50.88188457632702,\n",
       "   52.25054539359878,\n",
       "   51.45301752677981],\n",
       "  'deberta-large': [57.10744273529187,\n",
       "   55.98211735917019,\n",
       "   51.15057259286018,\n",
       "   56.03567794317386,\n",
       "   54.217306705892824]}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a823bf75",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(list(zip(model_name_list, data_list, f1_list)),\n",
    "                     columns =[\"model_name\", \"dataset\", \"test_f1\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "65a76d5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 40172 characters]\n",
      "text/plain": [
       "<Figure size 1008x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 445,
       "width": 829
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(14,6))\n",
    "sns.boxplot(x=\"model_name\", y=\"test_f1\",\n",
    "            hue=\"dataset\", whis=20,\n",
    "            data=df.sort_values([\"model_name\"], ascending=False))\n",
    "plt.xticks(rotation=90)\n",
    "plt.title(\"SciCite Test F1 Score (All Models)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4c7367cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mrpc deberta-base 33.28\n",
      "mrpc deberta-large 30.76\n",
      "mrpc_cor deberta-base 50.81\n",
      "mrpc_cor deberta-large 54.9\n"
     ]
    }
   ],
   "source": [
    "for d in [\"mrpc\", \"mrpc_cor\"]:\n",
    "    split = test_results[d]\n",
    "    model_name_list = split.keys()\n",
    "    for m in model_name_list:\n",
    "        result = split[m]\n",
    "        mean = round(sum(result)/len(result), 2)\n",
    "        print(d, m, mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b35f66d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mrpc deberta-base 35.67\n",
      "mrpc deberta-large 30.8\n",
      "mrpc_cor deberta-base 52.25\n",
      "mrpc_cor deberta-large 56.04\n"
     ]
    }
   ],
   "source": [
    "for d in [\"mrpc\", \"mrpc_cor\"]:\n",
    "    split = test_results[d]\n",
    "    model_name_list = split.keys()\n",
    "    for m in model_name_list:\n",
    "        result = split[m]\n",
    "        result.sort()\n",
    "        #print(result)\n",
    "        median = round(result[3],2)\n",
    "        print(d, m, median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca11da08",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "583697cd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
