{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "partial-piano",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math \n",
    "\n",
    "SEEDS = ['42', '43', '44', '46']\n",
    "STRATEGIES = ['random', 'dal', 'bald', 'mc-max-entropy']\n",
    "\n",
    "def aggregate_over_sources_agg(experiment='normaal'):\n",
    "\n",
    "    results = {key:{} for key in STRATEGIES}\n",
    "\n",
    "    for seed in SEEDS:\n",
    "\n",
    "        df = pd.read_csv('csv_files/{}_{}.csv'.format(experiment, seed))\n",
    "\n",
    "        for strategy in STRATEGIES:\n",
    "\n",
    "            scores_over_sources = []\n",
    "            for dataset in ['ANLI', 'SNLI', 'WANLI']:\n",
    "\n",
    "                scores = list(df['acquisition_fn: {} - active_{}_val_acc_epoch'.format(strategy, dataset)])\n",
    "\n",
    "                scores_over_sources.append(scores)\n",
    "\n",
    "\n",
    "            mean_over_sources = np.mean(scores_over_sources, axis=0)\n",
    "\n",
    "            results[strategy][seed] = mean_over_sources\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "assigned-uganda",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def agg_over_seeds(_results):\n",
    "\n",
    "    agg_over_seed_results = {}\n",
    "\n",
    "    for strategy in STRATEGIES:\n",
    "\n",
    "        agg_over_seeds = []\n",
    "        for seed in SEEDS:\n",
    "\n",
    "            agg_over_seeds.append(_results[strategy][seed])\n",
    "\n",
    "        mean_over_seeds = np.mean(agg_over_seeds, axis=0)\n",
    "        std_over_seeds = np.std(agg_over_seeds, axis=0)/math.sqrt(5)\n",
    "\n",
    "        agg_over_seed_results[strategy] = {'mean': mean_over_seeds,\n",
    "                                           'std':  std_over_seeds} \n",
    "        \n",
    "    return agg_over_seed_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "rocky-turkey",
   "metadata": {},
   "outputs": [],
   "source": [
    "COLORDICT = {'random':'blue', 'mc-max-entropy':'red', 'bald':'orange', 'dal':'purple'}\n",
    "def plot_curves(agg_over_seed_results, name):\n",
    "    \n",
    "    plt.figure(figsize=(SIZE,SIZE))\n",
    "    for strategy in STRATEGIES:\n",
    "\n",
    "        Y = agg_over_seed_results[strategy]['mean']\n",
    "        std = agg_over_seed_results[strategy]['std']\n",
    "        X = [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000]\n",
    "        plt.plot(X,Y, label=strategy, color=COLORDICT[strategy])\n",
    "        plt.fill_between(X, Y-std, Y+std, alpha=0.10)\n",
    "        ax = plt.gca()\n",
    "        ax.spines['top'].set_visible(False)\n",
    "        ax.spines['right'].set_visible(False)\n",
    "        ax.spines['bottom'].set_visible(False)\n",
    "        ax.spines['left'].set_visible(False)\n",
    "        ax.set_ylim([0.36, 0.42])\n",
    "    \n",
    "    plt.xlabel(\"Acquired Examples\", fontsize=FONTSIZE-3)\n",
    "    plt.xticks(fontsize=FONTSIZE-5)\n",
    "    plt.ylabel(\"Accuracy\", fontsize=FONTSIZE-3)\n",
    "    plt.yticks(fontsize=FONTSIZE-5)\n",
    "    plt.title('Agg. validation accuracy', fontsize=FONTSIZE)\n",
    "    plt.legend(loc='lower right')\n",
    "    plt.grid()\n",
    "    plt.savefig(name+'.pdf',format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "appropriate-blair",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'baselines/normaal_42.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-87-def8366ad847>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mSEEDS\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'42'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'43'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'44'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'46'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# EXPERIMENT =\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mplot_curves\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0magg_over_seeds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maggregate_over_sources\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexperiment\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'normaal'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'normaal'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mSEEDS\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'42'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'43'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'44'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'45'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'46'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mplot_curves\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0magg_over_seeds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maggregate_over_sources\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexperiment\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ablated'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ablated'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-84-3dc20047c5b3>\u001b[0m in \u001b[0;36maggregate_over_sources\u001b[0;34m(experiment)\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mseed\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mSEEDS\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'baselines/{}_{}.csv'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexperiment\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;31m#         print(df)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    686\u001b[0m     )\n\u001b[1;32m    687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 688\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    689\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    452\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    453\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 454\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    455\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    946\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    947\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 948\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    949\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    950\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1178\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"c\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1179\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"c\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1180\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1181\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"python\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   2008\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"usecols\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2009\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2010\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2011\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2012\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'baselines/normaal_42.csv'"
     ]
    }
   ],
   "source": [
    "FONTSIZE = 18\n",
    "import matplotlib as mpl\n",
    "mpl.rcParams['figure.dpi'] = 150\n",
    "SIZE=3\n",
    "SEEDS = ['42', '43', '44', '46']\n",
    "# EXPERIMENT = \n",
    "plot_curves(agg_over_seeds(aggregate_over_sources(experiment='normaal')), 'normaal')\n",
    "SEEDS = ['42', '43', '44', '45', '46']\n",
    "plot_curves(agg_over_seeds(aggregate_over_sources(experiment='ablated')), 'ablated')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "electronic-distribution",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   labelled_examples  project_dir: array - _step  \\\n",
      "0                500                      9904.4   \n",
      "1               1000                     19294.0   \n",
      "2               1500                     29555.6   \n",
      "3               2000                     44285.6   \n",
      "4               2500                     55817.2   \n",
      "5               3000                     71968.8   \n",
      "6               3500                     84772.4   \n",
      "7               4000                     99269.4   \n",
      "\n",
      "   project_dir: array - _step__MIN  project_dir: array - _step__MAX  \\\n",
      "0                             6962                            17046   \n",
      "1                            16456                            24960   \n",
      "2                            25896                            35476   \n",
      "3                            36350                            51908   \n",
      "4                            47777                            70418   \n",
      "5                            57833                            88032   \n",
      "6                            69193                           101273   \n",
      "7                            83624                           112265   \n",
      "\n",
      "   project_dir: array - active_ANLI_val_acc_epoch  \\\n",
      "0                                        0.360909   \n",
      "1                                        0.377364   \n",
      "2                                        0.374000   \n",
      "3                                        0.383909   \n",
      "4                                        0.389000   \n",
      "5                                        0.396182   \n",
      "6                                        0.405182   \n",
      "7                                        0.400091   \n",
      "\n",
      "   project_dir: array - active_ANLI_val_acc_epoch__MIN  \\\n",
      "0                                           0.340000     \n",
      "1                                           0.366364     \n",
      "2                                           0.365000     \n",
      "3                                           0.370909     \n",
      "4                                           0.378182     \n",
      "5                                           0.387273     \n",
      "6                                           0.401818     \n",
      "7                                           0.394545     \n",
      "\n",
      "   project_dir: array - active_ANLI_val_acc_epoch__MAX  \n",
      "0                                           0.374091    \n",
      "1                                           0.386364    \n",
      "2                                           0.390000    \n",
      "3                                           0.394091    \n",
      "4                                           0.405909    \n",
      "5                                           0.404545    \n",
      "6                                           0.412273    \n",
      "7                                           0.407727    \n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'sys' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-57-25725fb40e73>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m \u001b[0mresult_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maggregate_over_sources\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mplot_curves\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0magg_over_seed_results\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-57-25725fb40e73>\u001b[0m in \u001b[0;36maggregate_over_sources\u001b[0;34m(experiment, dataset)\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstrategy\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'acc'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'project_dir: array - active_ANLI_val_acc_epoch'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'sys' is not defined"
     ]
    }
   ],
   "source": [
    "# STRATEGIES_2 = ['random', 'dal', 'bald', 'mcme']\n",
    "# COLORDICT_2 = {'random':'blue', 'mcme':'red', 'bald':'orange', 'dal':'purple'}\n",
    "\n",
    "# def aggregate_over_sources(experiment='baselines', dataset='anli'):\n",
    "\n",
    "#     results = {key:{} for key in STRATEGIES_2}\n",
    "\n",
    "#     for strategy in STRATEGIES_2:\n",
    "        \n",
    "#         df = pd.read_csv('{}/{}_{}.csv'.format(experiment, dataset, strategy))\n",
    "        \n",
    "#         print(df)\n",
    "        \n",
    "#         sys.exit()\n",
    "        \n",
    "#         results[strategy]['acc'] = df['project_dir: array - active_ANLI_val_acc_epoch'].tolist()\n",
    "#         results[strategy]['min'] = df['project_dir: array - active_ANLI_val_acc_epoch__MIN'].tolist()\n",
    "#         results[strategy]['max'] = df['project_dir: array - active_ANLI_val_acc_epoch__MAX'].tolist()\n",
    "    \n",
    "#     return results\n",
    "\n",
    "# result_dict = aggregate_over_sources()\n",
    "\n",
    "# def plot_curves(agg_over_seed_results, name):\n",
    "    \n",
    "#     plt.figure(figsize=(SIZE,SIZE))\n",
    "#     for strategy in STRATEGIES_2:\n",
    "\n",
    "#         Y = agg_over_seed_results[strategy]['acc']\n",
    "#         Y_min = agg_over_seed_results[strategy]['min']\n",
    "#         Y_max = agg_over_seed_results[strategy]['max']\n",
    "#         X = [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000]\n",
    "#         plt.plot(X,Y, label=strategy, color=COLORDICT_2[strategy])\n",
    "#         plt.fill_between(X, Y_min, Y_max, alpha=0.10)\n",
    "#         ax = plt.gca()\n",
    "#         ax.spines['top'].set_visible(False)\n",
    "#         ax.spines['right'].set_visible(False)\n",
    "#         ax.spines['bottom'].set_visible(False)\n",
    "#         ax.spines['left'].set_visible(False)\n",
    "#         ax.set_ylim([0.33, 0.42])\n",
    "    \n",
    "#     plt.xlabel(\"Acquired Examples\", fontsize=FONTSIZE-3)\n",
    "#     plt.xticks(fontsize=FONTSIZE-5)\n",
    "#     plt.ylabel(\"Accuracy\", fontsize=FONTSIZE-3)\n",
    "#     plt.yticks(fontsize=FONTSIZE-5)\n",
    "#     plt.title('Agg. validation accuracy', fontsize=FONTSIZE)\n",
    "#     plt.legend(loc='lower right')\n",
    "#     plt.grid()\n",
    "# #     plt.savefig(name+'.pdf',format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "judicial-shuttle",
   "metadata": {},
   "outputs": [],
   "source": [
    "SEEDS = ['42', '43', '44', '45', '46']\n",
    "STRATEGIES = ['random', 'dal', 'bald', 'mc-max-entropy']\n",
    "\n",
    "def aggregate_over_sources(experiment='anli', mnli=None):\n",
    "    \n",
    "        \n",
    "    results = {key:{} for key in STRATEGIES}\n",
    "\n",
    "    for seed in SEEDS:\n",
    "\n",
    "        df = pd.read_csv('baselines/{}_{}.csv'.format(experiment, seed))\n",
    "        \n",
    "#         print(df)\n",
    "\n",
    "        for strategy in STRATEGIES:\n",
    "\n",
    "            scores_over_sources = []\n",
    "#             for dataset in ['ANLI', 'SNLI', 'WANLI']:\n",
    "\n",
    "            if mnli:\n",
    "                experiment_id = 'mnli'\n",
    "            \n",
    "            else:\n",
    "                experiment_id = experiment\n",
    "\n",
    "            scores = list(df['acquisition_fn: {} - active_{}_val_acc_epoch'.format(strategy, experiment_id.upper())])\n",
    "            \n",
    "            scores\n",
    "            \n",
    "            results[strategy][seed] = scores\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "hungry-colombia",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 80252 characters]\n",
      "text/plain": [
       "<Figure size 450x450 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 70808 characters]\n",
      "text/plain": [
       "<Figure size 450x450 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "SMALL_SIZE = 13\n",
    "MEDIUM_SIZE = 14\n",
    "BIGGER_SIZE = 5\n",
    "\n",
    "plt.rc('font', size=SMALL_SIZE)          # controls default text sizes\n",
    "plt.rc('axes', titlesize=SMALL_SIZE)     # fontsize of the axes title\n",
    "plt.rc('axes', labelsize=MEDIUM_SIZE)    # fontsize of the x and y labels\n",
    "plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels\n",
    "plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels\n",
    "plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize\n",
    "plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title\n",
    "\n",
    "plt.rcParams[\"axes.edgecolor\"] = \"dimgray\"\n",
    "def plot_curves(agg_over_seed_results, title, fname, lim, y_label=True, legend=True):\n",
    "    \n",
    "    plt.figure(figsize=(SIZE,SIZE))\n",
    "    for strategy in STRATEGIES:\n",
    "\n",
    "        Y = agg_over_seed_results[strategy]['mean']\n",
    "        std = agg_over_seed_results[strategy]['std']\n",
    "        X = [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000]\n",
    "        if strategy == 'mc-max-entropy':\n",
    "            strategy2 = 'mcme'\n",
    "        else:\n",
    "            strategy2 = strategy\n",
    "        plt.plot(X,Y, label=strategy2, color=COLORDICT[strategy])\n",
    "        plt.fill_between(X, Y-std, Y+std, alpha=0.10)\n",
    "        ax = plt.gca()\n",
    "#         ax.spines['top'].set_visible(True)\n",
    "#         ax.spines['right'].set_visible(False)\n",
    "#         ax.spines['bottom'].set_visible(False)\n",
    "#         ax.spines['left'].set_visible(False)\n",
    "        ax.set_ylim(lim)\n",
    "        ax.set_xlim([500,4000])\n",
    "    \n",
    "    plt.xlabel(\"Acquired Examples\", fontsize=MEDIUM_SIZE)\n",
    "    plt.xticks(fontsize=MEDIUM_SIZE)\n",
    "    if y_label:\n",
    "        plt.ylabel(\"Accuracy\", fontsize=MEDIUM_SIZE)\n",
    "    ax.yaxis.set_label_position(\"right\")\n",
    "    plt.yticks(fontsize=MEDIUM_SIZE)\n",
    "    plt.title('{} validation accuracy'.format(title), fontsize=MEDIUM_SIZE)\n",
    "    if legend:\n",
    "        plt.legend(loc='lower right', fontsize=SMALL_SIZE)\n",
    "    plt.grid()\n",
    "    plt.savefig(fname+'.pdf',format='pdf', bbox_inches='tight')\n",
    "\n",
    "\n",
    "# SEEDS = ['42', '43', '44', '46']\n",
    "# MEDIUM_SIZE = 14\n",
    "# # AGG BASELINES\n",
    "# plot_curves(agg_over_seeds(aggregate_over_sources_agg(experiment='normaal')), \n",
    "#             title='Agg.', \n",
    "#             fname='normaal', \n",
    "#             lim=[0.52, 0.64])\n",
    "# SEEDS = ['42', '43', '44', '45', '46']\n",
    "# plot_curves(agg_over_seeds(aggregate_over_sources_agg(experiment='ablated')), \n",
    "#             title='Agg.', \n",
    "#             fname='ablated', \n",
    "#             lim=[0.52, 0.64],\n",
    "#             legend=False)\n",
    "\n",
    "SEEDS = ['42', '43', '44', '46']\n",
    "plot_curves(agg_over_seeds(aggregate_over_sources(experiment='mnli_normaal', mnli=True)), \n",
    "            title='MNLI', \n",
    "            fname='mnli_normaal', \n",
    "            lim=[0.54, 0.8])\n",
    "SEEDS = ['42', '43', '44', '45', '46']\n",
    "plot_curves(agg_over_seeds(aggregate_over_sources(experiment='mnli_ablated', mnli=True)), \n",
    "            title='MNLI', \n",
    "            fname='mnli_ablated', \n",
    "            lim=[0.54, 0.8],\n",
    "            legend=False)\n",
    "\n",
    "# # SINGLE BASELINES\n",
    "# MEDIUM_SIZE = 14\n",
    "# SEEDS = ['42', '43', '44', '45', '46']\n",
    "# plot_curves(agg_over_seeds(aggregate_over_sources(experiment='anli')), title='ANLI', fname='ANLI', lim=[0.36, 0.42], legend=False)\n",
    "# plot_curves(agg_over_seeds(aggregate_over_sources(experiment='wanli')), title='WANLI', fname='WANLI', lim=[0.6, 0.72])\n",
    "# plot_curves(agg_over_seeds(aggregate_over_sources(experiment='snli')), title='SNLI', fname='SNLI', lim=[0.84, 0.9], legend=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "strange-zealand",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
