{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "partial-piano",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "SEEDS = ['42', '43', '44', '46']\n",
    "STRATEGIES = ['random', 'dal', 'bald', 'mc-max-entropy']\n",
    "\n",
    "def aggregate_over_sources(experiment='normaal'):\n",
    "\n",
    "    results = {key:{} for key in STRATEGIES}\n",
    "\n",
    "    for seed in SEEDS:\n",
    "\n",
    "        df = pd.read_csv('csv_files/{}_{}.csv'.format(experiment, seed))\n",
    "\n",
    "        for strategy in STRATEGIES:\n",
    "\n",
    "            scores_over_sources = []\n",
    "            for dataset in ['ANLI', 'SNLI', 'WANLI']:\n",
    "\n",
    "                scores = list(df['acquisition_fn: {} - active_{}_val_acc_epoch'.format(strategy, dataset)])\n",
    "\n",
    "                scores_over_sources.append(scores)\n",
    "\n",
    "\n",
    "            mean_over_sources = np.mean(scores_over_sources, axis=0)\n",
    "\n",
    "            results[strategy][seed] = mean_over_sources\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "assigned-uganda",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def agg_over_seeds(_results):\n",
    "\n",
    "    agg_over_seed_results = {}\n",
    "\n",
    "    for strategy in STRATEGIES:\n",
    "\n",
    "        agg_over_seeds = []\n",
    "        for seed in SEEDS:\n",
    "\n",
    "            agg_over_seeds.append(_results[strategy][seed])\n",
    "\n",
    "        mean_over_seeds = np.mean(agg_over_seeds, axis=0)\n",
    "        std_over_seeds = np.std(agg_over_seeds, axis=0)\n",
    "\n",
    "        agg_over_seed_results[strategy] = {'mean': mean_over_seeds,\n",
    "                                           'std':  std_over_seeds} \n",
    "        \n",
    "    return agg_over_seed_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "rocky-turkey",
   "metadata": {},
   "outputs": [],
   "source": [
    "COLORDICT = {'random':'blue', 'mc-max-entropy':'red', 'bald':'orange', 'dal':'purple'}\n",
    "def plot_curves(agg_over_seed_results):\n",
    "    \n",
    "    plt.figure(figsize=(SIZE,SIZE))\n",
    "    for strategy in STRATEGIES:\n",
    "\n",
    "        Y = agg_over_seed_results[strategy]['mean']\n",
    "        std = agg_over_seed_results[strategy]['std']\n",
    "        X = [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000]\n",
    "        plt.plot(X,Y, label=strategy, color=COLORDICT[strategy])\n",
    "        plt.fill_between(X, Y-std, Y+std, alpha=0.10)\n",
    "        plt.xlabel(\"Labelled examples\")\n",
    "        plt.ylabel(\"Validation Accuracy\")\n",
    "        plt.title('Agg. Validation Accuracy')\n",
    "        plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "appropriate-blair",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 48352 characters]\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 46536 characters]\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "SIZE=6\n",
    "SEEDS = ['42', '43', '44', '46']\n",
    "EXPERIMENT = \n",
    "plot_curves(agg_over_seeds(aggregate_over_sources(experiment='normaal')))\n",
    "SEEDS = ['42', '43', '44', '45', '46']\n",
    "plot_curves(agg_over_seeds(aggregate_over_sources(experiment='ablated')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "green-annual",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
