{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "wicked-sunday",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:70% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "from pprint import pprint\n",
    "import sys\n",
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "import matplotlib.patches as mpatches\n",
    "\n",
    "from scipy.stats import entropy\n",
    "from matplotlib.ticker import PercentFormatter\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "\n",
    "from scipy.stats import sem\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 1000)\n",
    "\n",
    "COL_WIDTH=250\n",
    "CMAP = 'seismic_r'\n",
    "_CMAP = 'viridis'\n",
    "pd.options.display.max_colwidth = COL_WIDTH\n",
    "\n",
    "EASY_THRESH = 0.75\n",
    "MEDIUM_THRESH = 0.5\n",
    "HARD_THRESH = 0.25\n",
    "IMPOSSIBLE_THRESH = 0.0\n",
    "ALPHA = 0.6\n",
    "\n",
    "SEEDS = [42, 43, 44, 45, 46]\n",
    "BASE_PATH = '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/datamaps/test/datamap_test_subepoch_inference_elke_0.5_epochs/roberta-large/random/SEED/confidences.pickle'\n",
    "# BASE_PATH = '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/datamap/test/datamap_op_test_set_subepoch_inference_elke_0.5_epochs_max_4_epochs/random/SEED/datamap.pickle'\n",
    "STRATEGY_PATH = \"/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/acquisition_IDs/array/ANLI_baseline_2/roberta-large/STRATEGY/SEED/acquisition_ids.csv\"\n",
    "\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:70% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unknown-supplement",
   "metadata": {},
   "source": [
    "# Loading pickled confidences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "postal-indication",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_confidences(path, predictions=False, exclude=1):\n",
    "\n",
    "    df = pd.read_pickle(path)\n",
    "    sample_ids = list(df.keys())\n",
    "\n",
    "    cartography_dict = {}\n",
    "    prediction_dict = {}\n",
    "\n",
    "    for sample_id in sample_ids:\n",
    "\n",
    "        times_correct = []\n",
    "        confidences = []\n",
    "\n",
    "        outputs = df[sample_id]\n",
    "        outputs = outputs[exclude:] # don't count the first 0.5 epoch!\n",
    "\n",
    "        for output in outputs:\n",
    "\n",
    "            # extract values from output tuple\n",
    "            prediction = output[0]\n",
    "            label = output[1]\n",
    "\n",
    "            # obtain GT confidence, determine correctness of prediction\n",
    "            confidence = prediction[label]\n",
    "            correct = 1 if prediction.index(max(prediction)) == label else 0\n",
    "\n",
    "            # track confidence and correctness\n",
    "            confidences.append(confidence)\n",
    "            times_correct.append(correct)\n",
    "\n",
    "\n",
    "        cartography_dict[sample_id] = {'confidence': np.mean(confidences),\n",
    "                                      'variability': np.std(confidences),\n",
    "                                      'correctness': np.mean(times_correct)}\n",
    "\n",
    "        prediction_dict[sample_id] = {'prediction': outputs[-1][0]}\n",
    "\n",
    "    confidences = [cartography_dict[key]['confidence'] for key in list(cartography_dict.keys())]\n",
    "    variability = [cartography_dict[key]['variability'] for key in list(cartography_dict.keys())]\n",
    "    correctness = [cartography_dict[key]['correctness'] for key in list(cartography_dict.keys())]\n",
    "\n",
    "    df = pd.DataFrame(list(zip(sample_ids, confidences, variability, correctness)), columns=['id', 'mean_conf', 'variability', 'correctness'])\n",
    "    \n",
    "    if predictions == False:\n",
    "        return df\n",
    "    else:\n",
    "        return df, prediction_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cordless-parts",
   "metadata": {},
   "source": [
    "# Code to read datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "hybrid-decimal",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def read_dataset(input_dir, dataset_id, split, seed, wanli_id):\n",
    "    \"\"\"\n",
    "    This function takes a dataset id and reads the corresponding .json split in an appropriate Pandas DataFrame\n",
    "    :param wanli_id: id key for wanli json file (pairID for metrics, id for non metric stuff)\n",
    "    :param seed:\n",
    "    :param input_dir:\n",
    "    :param dataset_id: str indicating which dataset should be read\n",
    "    :param split: str indicating which split should be read\n",
    "    :return: DataFrame with examples (Premise, Hypothesis, Label, ID)\n",
    "    \"\"\"\n",
    "    # anli_rounds = ['ANLI_R1', 'ANLI_R2', 'ANLI_R3']\n",
    "    anli_rounds = ['ANLI_R2', 'ANLI_R3']\n",
    "\n",
    "    def replace_labels(label):\n",
    "        label_conversions = {'e': 'entailment',\n",
    "                             'c': 'contradiction',\n",
    "                             'n': 'neutral'}\n",
    "\n",
    "        return label_conversions[label]\n",
    "\n",
    "    if dataset_id == 'SNLI':\n",
    "\n",
    "        # TODO only consider snli with gold labels?\n",
    "        data_path = '{}/snli_1.0/snli_1.0_{}.jsonl'.format(input_dir, split)\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['sentence1', 'sentence2', 'gold_label', 'pairID']]\n",
    "        dataset['dataset'] = 'SNLI'\n",
    "        dataset = dataset.drop(dataset[dataset.gold_label.str.contains('-')].index)  # drop examples with no gold label\n",
    "\n",
    "    # load separate ANLI rounds\n",
    "    elif dataset_id in anli_rounds:\n",
    "\n",
    "        data_round = dataset_id[-2:]  # extract the 'R#' substring from the provided ID\n",
    "        data_path = '{}/anli_v1.0/{}/{}.jsonl'.format(input_dir, data_round, split)\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['context', 'hypothesis', 'label', 'uid']]  # get rid of unnecessary columns\n",
    "        dataset['label'] = dataset['label'].apply(replace_labels)  # ensures consistently named labels\n",
    "        dataset['dataset'] = 'ANLI_{}'.format(data_round)\n",
    "        #\n",
    "        # print('Dataframe for {} - {}'.format(dataset_id, split), flush=True)\n",
    "\n",
    "    # compile all ANLI rounds\n",
    "    elif dataset_id == 'ANLI':\n",
    "\n",
    "        dataset_list = []\n",
    "        for data_round in anli_rounds:\n",
    "            data_round = data_round[-2:]\n",
    "            data_path = '{}/anli_v1.0/{}/{}.jsonl'.format(input_dir, data_round, split)\n",
    "            dataset = pd.read_json(data_path, lines=True)\n",
    "            dataset = dataset[['context', 'hypothesis', 'label', 'uid']]  # get rid of unnecessary columns\n",
    "            dataset['label'] = dataset['label'].apply(replace_labels)  # ensures consistently named labels\n",
    "            dataset['dataset'] = 'ANLI'\n",
    "            dataset_list.append(dataset)\n",
    "        dataset = pd.concat(dataset_list, axis=0)\n",
    "\n",
    "    # Load MNLI\n",
    "    elif dataset_id == 'MNLI':\n",
    "\n",
    "        if split == 'train':\n",
    "            data_path = '{}/multinli_1.0/multinli_1.0_{}.jsonl'.format(input_dir, split)\n",
    "        elif split == 'dev' or split == 'test':\n",
    "            data_path = '{}/multinli_1.0/multinli_1.0_dev_matched.jsonl'.format(input_dir)\n",
    "\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['sentence1', 'sentence2', 'gold_label', 'pairID']]\n",
    "        dataset['dataset'] = 'MNLI'\n",
    "        dataset = dataset.drop(dataset[dataset.gold_label.str.contains('-')].index)\n",
    "\n",
    "        # split dev into dev and test set\n",
    "        if split == 'dev':\n",
    "            dataset = dataset.sample(frac=1,  # reshuffle all rows in the dataframe prior to split\n",
    "                                     random_state=seed)  # seed is fixed such that dev and test set are always the same\n",
    "            dataset = dataset[:int((len(dataset)/2))]\n",
    "\n",
    "        elif split == 'test':\n",
    "            dataset = dataset.sample(frac=1,\n",
    "                                     random_state=seed)\n",
    "            dataset = dataset[int((len(dataset)/2)):]\n",
    "\n",
    "    # Load WANLI\n",
    "    elif dataset_id == 'WANLI':\n",
    "\n",
    "        if split == 'train':\n",
    "            data_path = '{}/wanli/train.jsonl'.format(input_dir)\n",
    "        elif split == 'dev' or split == 'test':\n",
    "            data_path = '{}/wanli/test.jsonl'.format(input_dir)\n",
    "\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['premise', 'hypothesis', 'gold', wanli_id]]\n",
    "        dataset['dataset'] = 'WANLI'\n",
    "        dataset = dataset.drop(dataset[dataset.gold.str.contains('-')].index)\n",
    "\n",
    "        # split dev into dev and test set\n",
    "        if split == 'dev':\n",
    "            dataset = dataset.sample(frac=1,  # reshuffle all rows in the dataframe prior to split\n",
    "                                     random_state=seed)  # seed is fixed such that dev and test set are always the same\n",
    "            dataset = dataset[:int((len(dataset) / 2))]\n",
    "\n",
    "        elif split == 'test':\n",
    "            dataset = dataset.sample(frac=1,\n",
    "                                     random_state=seed)\n",
    "            dataset = dataset[int((len(dataset) / 2)):]\n",
    "\n",
    "    else:\n",
    "        raise KeyError('No dataset found for \"{}\"'.format(dataset_id))\n",
    "\n",
    "    # ensure consistent headers per dataset DataFrame\n",
    "    dataset.columns = ['Premise', 'Hypothesis', 'Label', 'ID', 'Dataset']\n",
    "\n",
    "    dataset['ID'] = dataset['ID'].astype('string')\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "circular-orientation",
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIR='/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/resources/ard_data'\n",
    "SPLIT='train'\n",
    "WANLI_ID='pairID'\n",
    "\n",
    "def read_source(dataset_id):\n",
    "    dataset = read_dataset(input_dir=INPUT_DIR, \n",
    "                 dataset_id=dataset_id, \n",
    "                 split=SPLIT, \n",
    "                 seed=None, \n",
    "                 wanli_id=WANLI_ID)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "promising-receptor",
   "metadata": {},
   "source": [
    "# Loading all datasets into their own DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "legendary-estonia",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mnli = read_source('MNLI')\n",
    "# snli = read_source('SNLI')\n",
    "# anli = read_source('ANLI')\n",
    "wanli = read_source('WANLI')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sublime-finance",
   "metadata": {},
   "source": [
    "# Code for Generating Datamaps for Different Sources / Strategies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "third-homework",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_df(dataframe):\n",
    "    X = dataframe['variability'].tolist()\n",
    "    Y = dataframe['mean_conf'].tolist()\n",
    "    colors = dataframe['correctness'].tolist()\n",
    "\n",
    "    plt.figure(figsize=(30, 10))\n",
    "    plt.scatter(x=X, y=Y, c=colors, alpha=1, cmap=mpl.cm.get_cmap(CMAP))\n",
    "    plt.xlim(0,0.5)\n",
    "    plt.ylim(0,1)\n",
    "    plt.xlabel('Variability')\n",
    "    plt.ylabel('Confidence')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    \n",
    "def generate_source_map(base_df, source_df):\n",
    "\n",
    "    ids = source_df.ID\n",
    "\n",
    "    source_df = base_df[base_df.id.astype('string').isin(ids)]\n",
    "    \n",
    "    print('num of source in base: {}'.format(len(source_df)))\n",
    "    \n",
    "    generate_df(source_df)\n",
    "    \n",
    "def datamap_subset(base_df, source_df, name=None):\n",
    "    \n",
    "    source_ids = source_df.ID\n",
    "    \n",
    "    subset_df = base_df[base_df.id.astype('string').isin(source_ids)]\n",
    "\n",
    "    if name is not None:\n",
    "        subset_df['Name'] = name\n",
    "    else:\n",
    "        subset_df['Name'] = source_df.Dataset.tolist()[0]\n",
    "    \n",
    "    return subset_df\n",
    "\n",
    "def datamap_subset_multi(base_path, source_df, name=None):\n",
    "    \n",
    "    list_of_maps = []\n",
    "    \n",
    "    source_ids = source_df.ID\n",
    "    \n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        _base_path = base_path.replace('SEED', str(seed))\n",
    "        \n",
    "        base_df = load_confidences(_base_path, predictions=False)\n",
    "\n",
    "        subset_df = base_df[base_df.id.astype('string').isin(source_ids)]\n",
    "\n",
    "        if name is not None:\n",
    "            subset_df['Name'] = name\n",
    "        else:\n",
    "            subset_df['Name'] = source_df.Dataset.tolist()[0]\n",
    "            \n",
    "        list_of_maps.append(subset_df)\n",
    "    \n",
    "    return list_of_maps\n",
    "\n",
    "def generate_dfs_2(dataframes, markers=None, learnability_only=False, size=40, n_bins=10, font_size=20):\n",
    "    \n",
    "    plt.rcParams.update({'font.size': font_size})\n",
    "    \n",
    "    def generate_hist(axis, data, name, color, colormap, n_bins=10):\n",
    "        \n",
    "        cmap = False\n",
    "        if color == 'cmap':\n",
    "            cmap = True\n",
    "            color = 'green'\n",
    "        \n",
    "        n, bins, patches = axis.hist(data, \n",
    "                                     n_bins, \n",
    "                                     weights=np.ones(len(data)) / len(data), \n",
    "                                     density=False, \n",
    "                                     color=color)\n",
    "\n",
    "        bin_centers = 0.5 * (bins[:-1] + bins[1:])\n",
    "        \n",
    "        if cmap:\n",
    "        \n",
    "            # scale values to interval [0,1]\n",
    "            col = bin_centers - min(bin_centers)\n",
    "            col /= max(col)\n",
    "\n",
    "            for c, p in zip(col, patches):\n",
    "                plt.setp(p, 'facecolor', colormap(c))\n",
    "\n",
    "        axis.set_xlabel(name)\n",
    "        axis.set_ylim(0,1)\n",
    "\n",
    "        axis.yaxis.set_major_formatter(FuncFormatter('{0:.0%}'.format))\n",
    "    \n",
    "    # creating grid for subplots\n",
    "    fig = plt.figure()\n",
    "    fig.set_figheight(size)\n",
    "    fig.set_figwidth(size)\n",
    "    \n",
    "    ax1 = plt.subplot2grid(shape=(4, 4), loc=(0, 0), colspan=3, rowspan=4)\n",
    "    ax2 = plt.subplot2grid(shape=(4, 4), loc=(0, 3), rowspan=1)\n",
    "    ax3 = plt.subplot2grid(shape=(4, 4), loc=(1, 3), colspan=1)\n",
    "    ax4 = plt.subplot2grid(shape=(4, 4), loc=(2, 3), rowspan=1)\n",
    "    ax5 = plt.subplot2grid(shape=(4, 4), loc=(3, 3), rowspan=1)\n",
    "    \n",
    "    for dataframe in dataframes:\n",
    "\n",
    "        if markers is not None:\n",
    "\n",
    "            for dataset, marker in zip(['SNLI', 'ANLI', 'WANLI', 'MNLI'], ['s', 'x', 'o', 'v']):\n",
    "\n",
    "\n",
    "                sub_frame = dataframe[dataframe['Dataset'] == dataset]\n",
    "                X = sub_frame['variability'].tolist()\n",
    "                Y = sub_frame['mean_conf'].tolist()\n",
    "                colors = sub_frame['correctness'].tolist()\n",
    "                ax1.scatter(x=X, y=Y, c=colors, alpha=1, marker=marker, cmap=mpl.cm.get_cmap(CMAP), label=dataset, s=size)\n",
    "\n",
    "        else:\n",
    "\n",
    "\n",
    "            X = dataframe['variability'].tolist()\n",
    "            Y = dataframe['mean_conf'].tolist()\n",
    "            colors = dataframe['correctness'].tolist()\n",
    "\n",
    "            ax1.scatter(x=X, y=Y, c=colors, alpha=1, cmap=mpl.cm.get_cmap(CMAP), s=size)\n",
    "        \n",
    "        name = dataframe.Name.tolist()[0].upper()\n",
    "        ax1.set_title('Dataset Map for ' + name)\n",
    "        ax1.set_xlabel('Variability')\n",
    "        ax1.set_ylabel('Confidence')\n",
    "\n",
    "        if markers:\n",
    "            ax1.legend(loc='upper right', prop={'size': 30})\n",
    "            ax1.legend(markerscale=4)\n",
    "            \n",
    "        break # terminate loop for datamap after first seed since only one seed can be plotted\n",
    "    \n",
    "    \n",
    "    easy = []\n",
    "    medium = []\n",
    "    hard = []\n",
    "    impossible = []\n",
    "    X = []\n",
    "    Y = []\n",
    "    correctness = []\n",
    "    for dataframe in dataframes:\n",
    "    \n",
    "        X.extend( dataframe['variability'].tolist() )\n",
    "        Y.extend( dataframe['mean_conf'].tolist() )\n",
    "        correctness.extend( dataframe['correctness'].tolist() )\n",
    "\n",
    "        easy.append( len(dataframe[dataframe['mean_conf'] > EASY_THRESH])  )\n",
    "        medium.append( len(dataframe[(dataframe['mean_conf'] > MEDIUM_THRESH) & (dataframe['mean_conf'] < EASY_THRESH)])  )\n",
    "        hard.append( len(dataframe[(dataframe['mean_conf'] > HARD_THRESH) & (dataframe['mean_conf'] < MEDIUM_THRESH)])   )\n",
    "        impossible.append( len(dataframe[(dataframe['mean_conf'] > IMPOSSIBLE_THRESH) & (dataframe['mean_conf'] < HARD_THRESH)])  )\n",
    "    \n",
    "    # take the mean over difficulty brackets\n",
    "    # TODO: generate std bars?\n",
    "    def mean_and_std_e(array):\n",
    "        \n",
    "        return np.mean(array), np.std(array)\n",
    "    \n",
    "    easy, std_easy = mean_and_std_e(easy)\n",
    "    medium, std_medium = mean_and_std_e(medium)\n",
    "    hard, std_hard = mean_and_std_e(hard)\n",
    "    impossible, std_impossible = mean_and_std_e(impossible)\n",
    "    \n",
    "#     std_easy, std_medium, std_hard, std_impossible = 200,30,400,20\n",
    "    \n",
    "    # LEARNABILITY      \n",
    "    width = 1\n",
    "    cmap = mpl.cm.get_cmap(CMAP)\n",
    "    \n",
    "    categories = ['Easy', 'Med', 'Hard', 'Imp.']\n",
    "    x_pos = [0,1,2,3]\n",
    "    means = [easy, medium, hard, impossible]\n",
    "    \n",
    "    \n",
    "    y_err = [std_easy, std_medium, std_hard, std_impossible]\n",
    "    \n",
    "    normed_means = [mean / sum(means) for mean in means]\n",
    "    y_err = [std / sum(means) for std in y_err]\n",
    "    \n",
    "    colors = [cmap(value) for value in [0.85, 0.65, 0.35, 0.125]]\n",
    "    \n",
    "    labels = [('[p > {}]'.format(thresh, color=cmap(value), alpha=ALPHA)) for thresh, value in zip([EASY_THRESH, MEDIUM_THRESH, HARD_THRESH, IMPOSSIBLE_THRESH],[0.85, 0.65, 0.35, 0.125])]\n",
    "    ax2.bar(x_pos, normed_means, yerr=y_err, width=1, color=colors, capsize=10, alpha=ALPHA)\n",
    "\n",
    "    thresholds = [EASY_THRESH, MEDIUM_THRESH, HARD_THRESH, IMPOSSIBLE_THRESH]\n",
    "    patches = [mpatches.Patch(color=value, label='[p > {}]'.format(thresh), alpha=ALPHA) for thresh, value in zip(thresholds, colors)]\n",
    "    \n",
    "    ax2.legend(handles=patches, loc='upper right')\n",
    "    \n",
    "    ax2.set_xticks(x_pos)\n",
    "    ax2.set_xticklabels(categories)\n",
    "    ax2.set_ylim(0,1)\n",
    "    \n",
    "#     ylabels = ['{}'.format(round(x,1)) + 'K' for x in ax2.get_yticks()[::1]/1000]\n",
    "    ylabels = ['{}%'.format(int(x))  for x in ax2.get_yticks()[::1]*100]\n",
    "    ax2.set_yticklabels(ylabels)    \n",
    "    ax2.set_xlabel('Learnability')\n",
    "#     ax2.legend(loc='upper right')\n",
    "\n",
    "    \n",
    "    # CORRECTNESS      \n",
    "    generate_hist(axis=ax3, data=correctness, name='Correctness', color='cmap', colormap=mpl.cm.get_cmap(CMAP))\n",
    "\n",
    "    # CONFIDENCES      \n",
    "    generate_hist(axis=ax4, data=Y, name='Confidences', color='red', colormap=None)\n",
    "\n",
    "    # VARIABILITY      \n",
    "    generate_hist(axis=ax5, data=X, name='Variability', color='green', colormap=None)\n",
    "    \n",
    "    for ax in [ax2, ax3, ax4, ax5]:\n",
    "        ax.yaxis.tick_right()\n",
    "        ax.yaxis.set_label_position(\"right\")\n",
    "        ax.xaxis.set_label_position('top')\n",
    "    \n",
    "    plt.tight_layout()  \n",
    "    \n",
    "    if learnability_only:\n",
    "        print('learnability for {}'.format(name))\n",
    "        ax1.remove()\n",
    "        ax3.remove()\n",
    "        ax4.remove()\n",
    "        ax5.remove()\n",
    "        \n",
    "        plt.show(ax2)\n",
    "        # Save just the portion _inside_ the second axis's boundaries\n",
    "        extent = ax2.get_window_extent().transformed(fig.dpi_scale_trans.inverted())\n",
    "        fig.savefig('learnability_{}.pdf'.format(name), format='pdf', bbox_inches=extent.expanded(1.1, 1.2))\n",
    "#         plt.savefig('learnability_{}.pdf'.format(name), format='pdf')\n",
    "    else:\n",
    "        plt.savefig('datamap_{}.pdf'.format(name), format='pdf') \n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "numeric-speaker",
   "metadata": {},
   "source": [
    "# Datamaps per Source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "injured-float",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "# snli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=snli)\n",
    "# anli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=anli)\n",
    "wanli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=wanli)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "confidential-draft",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 823568 characters]\n",
      "text/plain": [
       "<Figure size 1200x1200 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.dpi'] = 80\n",
    "SIZE = 15\n",
    "FONT_SIZE = 19\n",
    "# generate_dfs_2(snli_maps, size=SIZE, font_size=FONT_SIZE)\n",
    "# generate_dfs_2(anli_maps, size=SIZE, font_size=FONT_SIZE)\n",
    "generate_dfs_2(wanli_maps, size=SIZE, font_size=FONT_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "requested-workstation",
   "metadata": {},
   "source": [
    "# Datamaps per Strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "laden-dover",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label distribution for random: {'neutral': 0.43, 'entailment': 0.32, 'contradiction': 0.26}\n",
      "label distribution for dal: {'neutral': 0.47, 'entailment': 0.3, 'contradiction': 0.23}\n",
      "label distribution for bald: {'neutral': 0.39, 'entailment': 0.3, 'contradiction': 0.31}\n",
      "label distribution for mc-max-entropy: {'neutral': 0.37, 'entailment': 0.31, 'contradiction': 0.31}\n"
     ]
    }
   ],
   "source": [
    "for strategy in ['random', 'dal', 'bald', 'mc-max-entropy']:\n",
    "    \n",
    "    label_dict = {'neutral': [], 'entailment': [], 'contradiction': []}\n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        _strategy_path = STRATEGY_PATH.replace('SEED', str(seed)).replace('STRATEGY', str(strategy))\n",
    "        strategy_df = load_strategy_acquisitions(path=_strategy_path, strategy=strategy)\n",
    "        strategy_df.ID = strategy_df.ID.astype('str')\n",
    "        \n",
    "        \n",
    "        label_counts = strategy_df.Label.value_counts(normalize=True)\n",
    "        \n",
    "        for key in list(label_dict.keys()):\n",
    "            label_dict[key].append(label_counts[key])\n",
    "            \n",
    "    \n",
    "    label_dict = {key:round(np.mean(label_dict[key]),2) for key in list(label_dict.keys())}\n",
    "    print('label distribution for {}: {}'.format(strategy, label_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "aquatic-bunch",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neutral          0.46775\n",
      "entailment       0.38000\n",
      "contradiction    0.15225\n",
      "Name: Label, dtype: float64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neutral          0.48225\n",
      "entailment       0.37300\n",
      "contradiction    0.14475\n",
      "Name: Label, dtype: float64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:40: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neutral          0.4635\n",
      "entailment       0.3870\n",
      "contradiction    0.1495\n",
      "Name: Label, dtype: float64\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-102-a87438c6df16>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     46\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0msubset_list\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m \u001b[0msubset_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_subset_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstrategy_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSTRATEGY_PATH\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbase_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mBASE_PATH\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrategy_list\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'random'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'bald'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'mc-max-entropy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'dal'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-102-a87438c6df16>\u001b[0m in \u001b[0;36mget_subset_list\u001b[0;34m(strategy_path, base_path, strategy_list)\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstrategy_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLabel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue_counts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnormalize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m             \u001b[0mbase_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_confidences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_base_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-2-76bd8d9f6beb>\u001b[0m in \u001b[0;36mload_confidences\u001b[0;34m(path, predictions, exclude)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         cartography_dict[sample_id] = {'confidence': np.mean(confidences),\n\u001b[0;32m---> 33\u001b[0;31m                                       \u001b[0;34m'variability'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfidences\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m                                       'correctness': np.mean(times_correct)}\n\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mstd\u001b[0;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36mstd\u001b[0;34m(a, axis, dtype, out, ddof, keepdims)\u001b[0m\n\u001b[1;32m   3495\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3496\u001b[0m     return _methods._std(a, axis=axis, dtype=dtype, out=out, ddof=ddof,\n\u001b[0;32m-> 3497\u001b[0;31m                          **kwargs)\n\u001b[0m\u001b[1;32m   3498\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3499\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/numpy/core/_methods.py\u001b[0m in \u001b[0;36m_std\u001b[0;34m(a, axis, dtype, out, ddof, keepdims)\u001b[0m\n\u001b[1;32m    232\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_std\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mddof\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m     ret = _var(a, axis=axis, dtype=dtype, out=out, ddof=ddof,\n\u001b[0;32m--> 234\u001b[0;31m                keepdims=keepdims)\n\u001b[0m\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/numpy/core/_methods.py\u001b[0m in \u001b[0;36m_var\u001b[0;34m(a, axis, dtype, out, ddof, keepdims)\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    204\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0missubclass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloating\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minteger\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 205\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mum\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmultiply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    206\u001b[0m     \u001b[0;31m# Fast-paths for built-in complex types\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    207\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_complex_to_float\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def load_strategy_acquisitions(path, strategy, cap=4000):\n",
    "    \n",
    "    acquired_df = pd.read_csv(path)\n",
    "    acquired_df = acquired_df[:cap]\n",
    "    acquired_df['Name'] = strategy.capitalize()\n",
    "    \n",
    "    return acquired_df\n",
    "\n",
    "def get_subset_list(strategy_path, base_path, strategy_list):\n",
    "    \n",
    "    subset_list = []\n",
    "    \n",
    "    for strategy in strategy_list:\n",
    "        \n",
    "        strategy_list = []\n",
    "        \n",
    "        for seed in SEEDS:\n",
    "            \n",
    "            _base_path = base_path.replace('SEED', str(seed))\n",
    "            _strategy_path = strategy_path.replace('SEED', str(seed)).replace('STRATEGY', str(strategy))\n",
    "            \n",
    "            strategy_df = load_strategy_acquisitions(path=_strategy_path, strategy=strategy)\n",
    "            strategy_df.ID = strategy_df.ID.astype('str')\n",
    "            \n",
    "            print(strategy_df.Label.value_counts(normalize=True))\n",
    "            \n",
    "            base_df = load_confidences(_base_path)\n",
    "            \n",
    "            \n",
    "            \n",
    "#             sys.exit()\n",
    "\n",
    "            id2dataset = {}\n",
    "            for i in range(len(strategy_df)):\n",
    "                row = strategy_df.iloc[i]\n",
    "                id2dataset[row.ID] = row.Dataset\n",
    "\n",
    "            strategy_subset = datamap_subset(base_df=base_df, source_df=strategy_df, name=strategy)\n",
    "            \n",
    "            strategy_subset['Dataset'] = strategy_subset['id'].apply(lambda x: id2dataset[x])\n",
    "            \n",
    "            strategy_list.append(strategy_subset)\n",
    "        \n",
    "        subset_list.append(strategy_list)\n",
    "\n",
    "    return subset_list \n",
    "\n",
    "subset_list = get_subset_list(strategy_path=STRATEGY_PATH, base_path=BASE_PATH, strategy_list=['random', 'bald', 'mc-max-entropy', 'dal'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "abroad-reproduction",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learnability for RANDOM\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 12360 characters]\n",
      "text/plain": [
       "<Figure size 880x880 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learnability for BALD\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 12496 characters]\n",
      "text/plain": [
       "<Figure size 880x880 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learnability for MC-MAX-ENTROPY\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 12520 characters]\n",
      "text/plain": [
       "<Figure size 880x880 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learnability for DAL\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 12596 characters]\n",
      "text/plain": [
       "<Figure size 880x880 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# generate_dfs(subset_list, markers=True, size=100, n_bins=10)\n",
    "\n",
    "plt.rcParams['figure.dpi'] = 80\n",
    "SIZE = 11\n",
    "FONT_SIZE = 12\n",
    "\n",
    "for subset_maps in subset_list:\n",
    "    generate_dfs_2(subset_maps, markers=True, size=SIZE, font_size=FONT_SIZE, learnability_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "medium-ecology",
   "metadata": {},
   "source": [
    "# Acquisitions by Difficulty over Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "recent-slave",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_stacked_barchart(base_dfs, strategy_dfs, return_axs=False, size=20, font_size=22, dpi=40, show_legend=True):\n",
    "    \n",
    "    plt.rcParams.update({'font.size': font_size})\n",
    "    # Call this fn for a single strategy!\n",
    "    \n",
    "    num_easy_aggregate = []\n",
    "    num_medium_aggregate = []\n",
    "    num_hard_aggregate = []\n",
    "    num_impossible_aggregate = []\n",
    "    \n",
    "    for strategy_df, base_df in zip(strategy_dfs, base_dfs):\n",
    "        \n",
    "        number_of_examples = 500\n",
    "\n",
    "        labels = []\n",
    "        num_easy = []\n",
    "        num_medium = []\n",
    "        num_hard = []\n",
    "        num_impossible = []\n",
    "\n",
    "        easy_thresh = 0.75\n",
    "        medium_thresh = 0.5\n",
    "        hard_thresh = 0.25\n",
    "        impossible_thresh = 0.0\n",
    "\n",
    "        for active_round in range(1, strategy_df['round'].max()+1):\n",
    "\n",
    "            acquisitions = strategy_df[strategy_df['round']==active_round]\n",
    "            number_of_examples += len(acquisitions)\n",
    "            labels.append(number_of_examples)\n",
    "\n",
    "            confidences = base_df[base_df.id.isin(acquisitions.ID)]\n",
    "\n",
    "            easy_mask = confidences.mean_conf > easy_thresh\n",
    "            medium_mask = (confidences.mean_conf > medium_thresh) & (confidences.mean_conf < easy_thresh)\n",
    "            hard_mask = (confidences.mean_conf > hard_thresh) & (confidences.mean_conf < medium_thresh)\n",
    "            impossible_mask = (confidences.mean_conf > impossible_thresh) & (confidences.mean_conf < hard_thresh)\n",
    "\n",
    "            num_easy.append(len(confidences[easy_mask]))\n",
    "            num_medium.append(len(confidences[medium_mask]))\n",
    "            num_hard.append(len(confidences[hard_mask]))\n",
    "            num_impossible.append(len(confidences[impossible_mask]))\n",
    "            \n",
    "        num_easy_aggregate.append(num_easy)\n",
    "        num_medium_aggregate.append(num_medium)\n",
    "        num_hard_aggregate.append(num_hard)\n",
    "        num_impossible_aggregate.append(num_impossible)\n",
    "        \n",
    "    def get_mean_and_std_e(array):\n",
    "        \n",
    "        return np.mean(array, axis=0), np.std(array, axis=0)\n",
    "    \n",
    "    num_easy, num_easy_std = get_mean_and_std_e(num_easy_aggregate)    \n",
    "    num_medium, num_medium_std = get_mean_and_std_e(num_medium_aggregate)\n",
    "    num_hard, num_hard_std = get_mean_and_std_e(num_hard_aggregate)\n",
    "    num_impossible, num_impossible_std = get_mean_and_std_e(num_impossible_aggregate)  \n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "\n",
    "    width = 210\n",
    "    alpha = ALPHA\n",
    "    labels = np.array(labels)\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(CMAP)\n",
    "\n",
    "    ax.bar(labels, num_easy, width, yerr=num_easy_std, label='Easy [p > {}]'.format(easy_thresh), color=cmap(0.85), alpha=alpha)\n",
    "    ax.bar(labels, num_medium, width, yerr=num_medium_std, bottom=num_easy,  label='Medium [p > {}]'.format(medium_thresh), color=cmap(0.65), alpha=alpha)\n",
    "    ax.bar(labels, num_hard, width, yerr=num_hard_std, bottom=num_easy + num_medium, label='Hard [p > {}]'.format(hard_thresh), color=cmap(0.35), alpha=alpha)\n",
    "    ax.bar(labels, num_impossible, width, yerr=num_impossible_std, bottom=num_easy + num_medium + num_hard, label='Impossible [p > {}]'.format(impossible_thresh), color=cmap(0.125), alpha=alpha)\n",
    "\n",
    "#     ax.set_xticks(ax.get_xticks()[2:])\n",
    "#     ax.set_ylabel('Acquisition by Difficulty')\n",
    "    ax.set_xlabel('Number of Training Examples')\n",
    "    ax.set_ylim(0, 510)\n",
    "    ax.set_title(strategy_df['Name'].tolist()[0])\n",
    "    \n",
    "    if show_legend:\n",
    "        ax.legend(loc='lower right')\n",
    "    \n",
    "    if return_axs:\n",
    "        \n",
    "        return ax\n",
    "    \n",
    "    else:\n",
    "        \n",
    "#         if impossible_only:\n",
    "#             impossible_str = '_impossible_'\n",
    "#         else:\n",
    "#             impossible_str = ''\n",
    "        \n",
    "        plt.plot(stacked=True)\n",
    "        name = strategy_df['Name'].tolist()[0]\n",
    "        plt.savefig('acquisitions_by_difficulty_{}.pdf'.format(name), format='pdf')\n",
    "        plt.show()\n",
    "        \n",
    "        return None\n",
    "    \n",
    "\n",
    "def generate_stacked_barchart_data(base_dfs, strategy_dfs, return_axs=False, impossible_only=False, size=20, font_size=22, dpi=40, show_legend=True):\n",
    "    \n",
    "    plt.rcParams.update({'font.size': font_size})\n",
    "    # Call this fn for a single strategy!\n",
    "    \n",
    "    total_counts = {'SNLI':[], 'ANLI':[], 'WANLI':[]}\n",
    "    \n",
    "    for strategy_df, base_df in zip(strategy_dfs, base_dfs):\n",
    "        \n",
    "        number_of_examples = 500\n",
    "\n",
    "        labels = []\n",
    "        results = {'SNLI':[], 'ANLI':[], 'WANLI':[]}\n",
    "\n",
    "        for active_round in range(1, strategy_df['round'].max()+1):\n",
    "\n",
    "            acquisitions = strategy_df[strategy_df['round']==active_round]\n",
    "            number_of_examples += len(acquisitions)\n",
    "            \n",
    "            if impossible_only:\n",
    "                \n",
    "                impossible_acquisitions = base_df[base_df.mean_conf < MEDIUM_THRESH]\n",
    "                acquisitions = acquisitions[acquisitions.ID.isin(impossible_acquisitions.id)]\n",
    "            \n",
    "            labels.append(number_of_examples)\n",
    "            \n",
    "            data_counts = acquisitions.Dataset.value_counts()\n",
    "            \n",
    "            for dataset, value in zip(data_counts.index, data_counts):\n",
    "                results[dataset].append(value)\n",
    "            \n",
    "            for dataset in list(results.keys()):\n",
    "                \n",
    "                if dataset not in data_counts.index:\n",
    "                    \n",
    "                    results[dataset].append(0)\n",
    "        \n",
    "        for key in list(total_counts.keys()):\n",
    "            total_counts[key].append(results[key])\n",
    "        \n",
    "    mean_and_std = {'SNLI':[], 'ANLI':[], 'WANLI':[]}\n",
    "    \n",
    "    for key in list(results.keys()):\n",
    "        mean_and_std[key].append(np.mean(total_counts[key], axis=0))\n",
    "        mean_and_std[key].append(np.std(total_counts[key], axis=0))\n",
    "        \n",
    "    def get_mean_and_std_e(array):\n",
    "        \n",
    "        return np.mean(array, axis=0), np.std(array, axis=0) \n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "    \n",
    "    if impossible_only:\n",
    "        ax.set_ylim(0, 250)\n",
    "    else:\n",
    "        ax.set_ylim(0, 510)\n",
    "        \n",
    "    width = 210\n",
    "    alpha = ALPHA\n",
    "    labels = np.array(labels)\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(_CMAP)\n",
    "    \n",
    "    ax.bar(labels, mean_and_std['SNLI'][0], width, yerr=mean_and_std['SNLI'][1], label='SNLI', color=cmap(0.85), alpha=alpha)\n",
    "    ax.bar(labels, mean_and_std['ANLI'][0], width, yerr=mean_and_std['ANLI'][1], bottom=mean_and_std['SNLI'][0],  label='ANLI', color=cmap(0.65), alpha=alpha)\n",
    "    ax.bar(labels, mean_and_std['WANLI'][0], width, yerr=mean_and_std['WANLI'][1], bottom=mean_and_std['SNLI'][0] + mean_and_std['ANLI'][0], label='WANLI', color=cmap(0.35), alpha=alpha)\n",
    "\n",
    "    ax.set_xticks(ax.get_xticks()[:])\n",
    "#     ax.set_ylabel('Acquisition by Dataset')\n",
    "    ax.set_xlabel('Number of Training Examples')\n",
    "    \n",
    "    if not impossible_only:     \n",
    "        ax.set_title(strategy_df['Name'].tolist()[0])\n",
    "        \n",
    "    else:\n",
    "        ax.set_title(strategy_df['Name'].tolist()[0])\n",
    "    \n",
    "    if show_legend:\n",
    "        ax.legend(loc='lower right')\n",
    "    \n",
    "    if return_axs:\n",
    "        \n",
    "        return ax\n",
    "    \n",
    "    else:\n",
    "        \n",
    "        if impossible_only:\n",
    "            impossible_str = '_impossible_'\n",
    "        else:\n",
    "            impossible_str = ''\n",
    "\n",
    "\n",
    "        \n",
    "        plt.plot(stacked=True)\n",
    "        name = strategy_df['Name'].tolist()[0]\n",
    "        plt.savefig('acquisitions_by_data_{}{}.pdf'.format(name,impossible_str), format='pdf')\n",
    "#         plt.tight_layout()\n",
    "        plt.show()\n",
    "        \n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "stock-forestry",
   "metadata": {},
   "outputs": [],
   "source": [
    "strategy_ids = ['random', 'mc-max-entropy', 'dal', 'bald']\n",
    "\n",
    "strategies_list = []\n",
    "for strategy_id in strategy_ids:\n",
    "    \n",
    "    strategy_list = []\n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        strategy_path = STRATEGY_PATH.replace('SEED', str(seed)).replace('STRATEGY', str(strategy_id))\n",
    "        strategy_df = load_strategy_acquisitions(strategy=strategy_id, path=strategy_path) \n",
    "        strategy_df.ID = strategy_df.ID.astype('str')\n",
    "        \n",
    "        strategy_list.append(strategy_df)\n",
    "    \n",
    "    strategies_list.append(strategy_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "qualified-stations",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dfs = [load_confidences(BASE_PATH.replace('SEED', str(seed)), predictions=False) for seed in SEEDS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "thorough-wound",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 26288 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 16584 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 16312 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 14484 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 13872 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 13656 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 12560 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 11988 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 11740 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 12808 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 12220 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 11964 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "SIZE = 4.5\n",
    "FONTSIZE = 14\n",
    "DPI = 80\n",
    "\n",
    "SHOW_LEGEND=True\n",
    "axs = []\n",
    "for strategy_df in strategies_list:\n",
    "     generate_stacked_barchart(base_dfs=base_dfs, strategy_dfs=strategy_df, return_axs=False, size=SIZE, font_size=FONTSIZE, dpi=DPI, show_legend=SHOW_LEGEND)\n",
    "     generate_stacked_barchart_data(base_dfs=base_dfs, strategy_dfs=strategy_df, impossible_only=False, return_axs=False, size=SIZE, font_size=FONTSIZE, dpi=DPI, show_legend=SHOW_LEGEND)\n",
    "     generate_stacked_barchart_data(base_dfs=base_dfs, strategy_dfs=strategy_df, impossible_only=True, return_axs=False, size=SIZE, font_size=FONTSIZE, dpi=DPI, show_legend=SHOW_LEGEND)\n",
    "     SHOW_LEGEND=False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "known-lighter",
   "metadata": {},
   "source": [
    "# Examples of various difficulties:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ranking-guatemala",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n",
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n",
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3437, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-145-7c0a6e9294f9>\", line 8, in <module>\n",
      "    _, prediction_dict = load_confidences(BASE_PATH.replace('SEED',str(42)), predictions=True)\n",
      "  File \"<ipython-input-2-76bd8d9f6beb>\", line 3, in load_confidences\n",
      "    df = pd.read_pickle(path)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/pickle.py\", line 169, in read_pickle\n",
      "    f, fh = get_handle(fp_or_buf, \"rb\", compression=compression, is_text=False)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/common.py\", line 418, in get_handle\n",
      "    from s3fs import S3File\n",
      "  File \"<frozen importlib._bootstrap>\", line 983, in _find_and_load\n",
      "  File \"<frozen importlib._bootstrap>\", line 963, in _find_and_load_unlocked\n",
      "  File \"<frozen importlib._bootstrap>\", line 906, in _find_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1280, in find_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1252, in _get_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1368, in find_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1408, in _fill_cache\n",
      "OSError: [Errno 6] Device not configured: '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/data_analysis'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2061, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'OSError' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1101, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 248, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 281, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 1495, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 1453, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 725, in getmodule\n",
      "    file = getabsfile(object, _filename)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 709, in getabsfile\n",
      "    return os.path.normcase(os.path.abspath(_filename))\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/posixpath.py\", line 376, in abspath\n",
      "    cwd = os.getcwd()\n",
      "OSError: [Errno 6] Device not configured\n",
      "ERROR! Session/line number was not unique in database. History logging moved to new session 686\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3437, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-145-7c0a6e9294f9>\", line 8, in <module>\n",
      "    _, prediction_dict = load_confidences(BASE_PATH.replace('SEED',str(42)), predictions=True)\n",
      "  File \"<ipython-input-2-76bd8d9f6beb>\", line 3, in load_confidences\n",
      "    df = pd.read_pickle(path)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/pickle.py\", line 169, in read_pickle\n",
      "    f, fh = get_handle(fp_or_buf, \"rb\", compression=compression, is_text=False)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/common.py\", line 418, in get_handle\n",
      "    from s3fs import S3File\n",
      "  File \"<frozen importlib._bootstrap>\", line 983, in _find_and_load\n",
      "  File \"<frozen importlib._bootstrap>\", line 963, in _find_and_load_unlocked\n",
      "  File \"<frozen importlib._bootstrap>\", line 906, in _find_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1280, in find_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1252, in _get_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1368, in find_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1408, in _fill_cache\n",
      "OSError: [Errno 6] Device not configured: '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/data_analysis'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2061, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'OSError' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3357, in run_ast_nodes\n",
      "    if (await self.run_code(code, result,  async_=asy)):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3454, in run_code\n",
      "    self.showtraceback(running_compiled_code=True)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2064, in showtraceback\n",
      "    value, tb, tb_offset=tb_offset)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1368, in structured_traceback\n",
      "    self, etype, value, tb, tb_offset, number_of_lines_of_context)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1268, in structured_traceback\n",
      "    self, etype, value, tb, tb_offset, number_of_lines_of_context\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1125, in structured_traceback\n",
      "    tb_offset)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1082, in format_exception_as_a_whole\n",
      "    last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 382, in find_recursion\n",
      "    return len(records), 0\n",
      "TypeError: object of type 'NoneType' has no len()\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2061, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'TypeError' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1101, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 248, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 281, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 1495, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 1453, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 725, in getmodule\n",
      "    file = getabsfile(object, _filename)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 709, in getabsfile\n",
      "    return os.path.normcase(os.path.abspath(_filename))\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/posixpath.py\", line 376, in abspath\n",
      "    cwd = os.getcwd()\n",
      "OSError: [Errno 6] Device not configured\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3437, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-145-7c0a6e9294f9>\", line 8, in <module>\n",
      "    _, prediction_dict = load_confidences(BASE_PATH.replace('SEED',str(42)), predictions=True)\n",
      "  File \"<ipython-input-2-76bd8d9f6beb>\", line 3, in load_confidences\n",
      "    df = pd.read_pickle(path)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/pickle.py\", line 169, in read_pickle\n",
      "    f, fh = get_handle(fp_or_buf, \"rb\", compression=compression, is_text=False)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/common.py\", line 418, in get_handle\n",
      "    from s3fs import S3File\n",
      "  File \"<frozen importlib._bootstrap>\", line 983, in _find_and_load\n",
      "  File \"<frozen importlib._bootstrap>\", line 963, in _find_and_load_unlocked\n",
      "  File \"<frozen importlib._bootstrap>\", line 906, in _find_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1280, in find_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1252, in _get_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1368, in find_spec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1408, in _fill_cache\n",
      "OSError: [Errno 6] Device not configured: '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/data_analysis'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2061, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'OSError' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3357, in run_ast_nodes\n",
      "    if (await self.run_code(code, result,  async_=asy)):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3454, in run_code\n",
      "    self.showtraceback(running_compiled_code=True)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2064, in showtraceback\n",
      "    value, tb, tb_offset=tb_offset)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1368, in structured_traceback\n",
      "    self, etype, value, tb, tb_offset, number_of_lines_of_context)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1268, in structured_traceback\n",
      "    self, etype, value, tb, tb_offset, number_of_lines_of_context\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1125, in structured_traceback\n",
      "    tb_offset)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1082, in format_exception_as_a_whole\n",
      "    last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 382, in find_recursion\n",
      "    return len(records), 0\n",
      "TypeError: object of type 'NoneType' has no len()\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2061, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'TypeError' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2940, in _run_cell\n",
      "    return runner(coro)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3166, in run_cell_async\n",
      "    interactivity=interactivity, compiler=compiler, result=result)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3376, in run_ast_nodes\n",
      "    self.showtraceback()\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2064, in showtraceback\n",
      "    value, tb, tb_offset=tb_offset)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1368, in structured_traceback\n",
      "    self, etype, value, tb, tb_offset, number_of_lines_of_context)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1268, in structured_traceback\n",
      "    self, etype, value, tb, tb_offset, number_of_lines_of_context\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1143, in structured_traceback\n",
      "    chained_exceptions_tb_offset)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1082, in format_exception_as_a_whole\n",
      "    last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 382, in find_recursion\n",
      "    return len(records), 0\n",
      "TypeError: object of type 'NoneType' has no len()\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2061, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'TypeError' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1101, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 248, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 281, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 1495, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 1453, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 725, in getmodule\n",
      "    file = getabsfile(object, _filename)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/inspect.py\", line 709, in getabsfile\n",
      "    return os.path.normcase(os.path.abspath(_filename))\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/posixpath.py\", line 376, in abspath\n",
      "    cwd = os.getcwd()\n",
      "OSError: [Errno 6] Device not configured\n"
     ]
    }
   ],
   "source": [
    "EASY_THRESH = 0.75\n",
    "MEDIUM_THRESH = 0.5\n",
    "HARD_THRESH = 0.25\n",
    "IMPOSSIBLE_THRESH = 0.0\n",
    "\n",
    "COL_WIDTH = 25\n",
    "\n",
    "_, prediction_dict = load_confidences(BASE_PATH.replace('SEED',str(42)), predictions=True)\n",
    "\n",
    "HARD_THRESH = 0.25\n",
    "def show_examples(source_df, source_map, base_df=None, num=5):\n",
    "    \n",
    "    source_map = source_map[0]\n",
    "    \n",
    "    difficulty_dict = {}\n",
    "    \n",
    "    easy_mask = source_map.mean_conf > EASY_THRESH\n",
    "    medium_mask = (source_map.mean_conf > MEDIUM_THRESH) & (source_map.mean_conf < EASY_THRESH)\n",
    "    hard_mask = (source_map.mean_conf > HARD_THRESH) & (source_map.mean_conf < MEDIUM_THRESH)\n",
    "    impossible_mask = (source_map.mean_conf > IMPOSSIBLE_THRESH) & (source_map.mean_conf < 0.01)\n",
    "    \n",
    "    difficulty_dict['Easy'] = source_map[easy_mask]\n",
    "    difficulty_dict['Medium'] = source_map[medium_mask]\n",
    "    difficulty_dict['Hard'] = source_map[hard_mask]\n",
    "    difficulty_dict['Impossible'] = source_map[impossible_mask]\n",
    "    \n",
    "    print(source_df[source_df.ID=='dfaed736-3361-4401-a646-3d7f501eaabd'])\n",
    "    \n",
    "    print('Examples from classes EASY, MEDIUM, HARD, and IMPOSSIBLE for {} \\n'.format(source_map.Name.tolist()[0]))\n",
    "    \n",
    "    print('\\nConfidence Thresholds:\\nEasy: {}\\nMedium: {}\\nHard: {}\\nImpossible: {}\\n'.format(EASY_THRESH, MEDIUM_THRESH, HARD_THRESH, IMPOSSIBLE_THRESH))\n",
    "    \n",
    "    id2label = {0: \"entailment\",\n",
    "                1: \"contradiction\",\n",
    "                2: \"neutral\"}\n",
    "    for key in difficulty_dict.keys():\n",
    "        \n",
    "        examples = source_df[source_df.ID.astype('str').isin(difficulty_dict[key].id.astype('string'))].head(num)\n",
    "        examples['Model Prediction'] = examples.ID.apply(lambda x: id2label[np.argmax(prediction_dict[x]['prediction'])])\n",
    "        \n",
    "        print(str(key).capitalize() + ' examples:')\n",
    "        examples = examples[['Premise', 'Hypothesis', 'Label', 'Model Prediction']]\n",
    "        examples['Hypothesis'] = examples['Hypothesis'].str.wrap(COL_WIDTH).astype('string')\n",
    "        examples['Premise'] = examples['Premise'].str.wrap(COL_WIDTH).astype('string')\n",
    "        \n",
    "        examples = examples.style.set_properties(**{'text-align': 'left'})\n",
    "        examples = examples.set_table_styles([dict(selector = 'th', props=[('text-align', 'left')])])\n",
    "        display(examples)\n",
    "   \n",
    "\n",
    "show_examples(source_df=anli, source_map=anli_maps, base_df=None, num=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "immune-bones",
   "metadata": {},
   "source": [
    "# Uncertainty metric:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "essential-homework",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing UNC for seed: 42\n",
      "Computing UNC for seed: 43\n",
      "Computing UNC for seed: 44\n",
      "Computing UNC for seed: 45\n",
      "Computing UNC for seed: 46\n"
     ]
    }
   ],
   "source": [
    "def compute_uncertainty(strategies, base_path, strategy_path):\n",
    "    \n",
    "    results = {key: [] for key in strategies}\n",
    "    \n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        print('Computing UNC for seed: {}'.format(seed))\n",
    "        \n",
    "        confidences = pd.read_pickle(base_path.replace('SEED', str(seed)))\n",
    "        \n",
    "        seed_entropies = {}\n",
    "        \n",
    "        sample_ids = list(confidences.keys())\n",
    "        for sample_id in sample_ids:\n",
    "    \n",
    "            times_correct = []\n",
    "\n",
    "            outputs = confidences[sample_id]\n",
    "            output = outputs[-1] # last epoch\n",
    "            \n",
    "            prediction = output[0]\n",
    "            pred_entropy = entropy(prediction)\n",
    "            seed_entropies[sample_id] = pred_entropy\n",
    "        \n",
    "        def get_entropies(x):\n",
    "            \n",
    "            if x in seed_entropies.keys():\n",
    "                return seed_entropies[x]\n",
    "            else:\n",
    "                return False\n",
    "        \n",
    "        for strategy in strategies:\n",
    "            \n",
    "            strategy_df = load_strategy_acquisitions(strategy=strategy, path=strategy_path.replace('SEED', str(seed)).replace('STRATEGY', str(strategy)))\n",
    "            \n",
    "            strategy_df['entropies'] = strategy_df.ID.apply(lambda x: get_entropies(x))\n",
    "            uncertainty = np.mean(strategy_df.entropies.tolist())\n",
    "            \n",
    "            results[strategy].append(uncertainty)\n",
    "        \n",
    "    return results\n",
    "\n",
    "uncertainty = compute_uncertainty(strategies=['random', 'bald', 'dal', 'mc-max-entropy'], base_path=BASE_PATH, strategy_path=STRATEGY_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "molecular-niger",
   "metadata": {},
   "outputs": [],
   "source": [
    "strats = ['random', 'bald', 'dal', 'mc-max-entropy']\n",
    "\n",
    "CMAP2 = 'viridis'\n",
    "def plot_uncertainty(uncertainty, size=15, dpi=60):\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "\n",
    "    width = 1\n",
    "    alpha = ALPHA\n",
    "#     labels = np.array(range(len(uncertainty.keys())))\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(CMAP2)\n",
    "    \n",
    "    for i, key in enumerate(strats):\n",
    "        ax.bar(key, np.mean(uncertainty[key]), yerr=np.std(uncertainty[key]), width=width, label='{}'.format(key), color=cmap(1-(i/4)), alpha=alpha)\n",
    "    \n",
    "    ax.set_ylabel('Mean predictive entropy')\n",
    "    ax.set_xlabel('Strategy')\n",
    "    ax.set_xticks([])\n",
    "    ax.set_ylim(0.3, 0.6)\n",
    "    ax.set_title('Uncertainty per Strategy')\n",
    "    ax.legend(loc='upper left')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('uncertainty.pdf', format='pdf')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "maritime-scratch",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 28468 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 21892 characters]\n",
      "text/plain": [
       "<Figure size 350x350 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_input_div(input_div, size=20, dpi=60):\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "\n",
    "    width = 1\n",
    "    alpha = ALPHA\n",
    "    labels = np.array(range(len(uncertainty.keys())))\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(CMAP2)\n",
    "    \n",
    "    for i, key in enumerate(strats):\n",
    "        ax.bar(key, input_div[key][0], yerr=input_div[key][1], width=width, label='{}'.format(key), color=cmap(1-(i/4)), alpha=alpha)\n",
    "\n",
    "    \n",
    "    ax.set_ylabel('Jaccard similarity')\n",
    "    ax.set_xlabel('Strategy')\n",
    "    ax.set_xticks([])\n",
    "    ax.set_ylim(0.2, 0.4)\n",
    "    ax.set_title('Input diversity per Strategy')\n",
    "    ax.legend(loc='upper left')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('input_div.pdf', format='pdf')\n",
    "    plt.show()\n",
    "\n",
    "input_div = {'dal':[0.2761, 0.004296],\n",
    "'max-entropy': [0.3308, 0.01008],\n",
    "'random': [0.2756, 0.00295],\n",
    "'bald': [0.2203, 0.01226],\n",
    "'mc-max-entropy':[0.3227, 0.00429]}\n",
    "\n",
    "SIZE = 5\n",
    "DPI = 70\n",
    "FONT_SIZE = 15\n",
    "plt.rcParams.update({'font.size': FONT_SIZE})\n",
    "plot_input_div(input_div, size=SIZE, dpi=DPI)\n",
    "plot_uncertainty(uncertainty, size=SIZE, dpi=DPI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "protective-abraham",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "miniature-drink",
   "metadata": {},
   "source": [
    "# TODO: find out if token overlap is a good enough signal to regularise towards target domain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "egyptian-backing",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "anli_dev = anli.sample(5000)\n",
    "\n",
    "anli_train = anli[-anli.ID.isin(anli_dev.ID)].sample(50000)\n",
    "snli_train = snli.sample(50000)\n",
    "\n",
    "def get_set_of_tokens(dataset):\n",
    "\n",
    "            # join sentences from Premise and Hypothesis column into single string\n",
    "            dataset['Combined'] = dataset['Premise'] + ' ' + dataset['Hypothesis']\n",
    "            list_of_tokens = []\n",
    "            # split each sentence into tokens and add to list\n",
    "            for sentence in dataset['Combined'].tolist():\n",
    "                for char in [',', '.', '?', '!', ':', ';',\"'\", '\"', ')', '(', '<br>']:\n",
    "                    sentence = sentence.replace(char, '').lower()\n",
    "#                 dataset['Split_set'] = sentence.split()\n",
    "                list_of_tokens.extend(sentence.split())\n",
    "            # return set of unique tokens\n",
    "            return dataset, set(list_of_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "careful-authority",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "anli_train, anli_train_tokens = get_set_of_tokens(anli_train)\n",
    "snli_train, snli_train_tokens = get_set_of_tokens(snli_train)\n",
    "anli_dev, anli_dev_tokens = get_set_of_tokens(anli_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "noted-survival",
   "metadata": {},
   "outputs": [],
   "source": [
    "# anli_intersection = anli_train_tokens.intersection(anli_dev_tokens)\n",
    "# snli_intersection = snli_train_tokens.intersection(anli_dev_tokens)\n",
    "\n",
    "# print(len(anli_intersection)/len(anli_train_tokens))\n",
    "# print(len(snli_intersection)/len(snli_train_tokens))\n",
    "\n",
    "def compute_mean_overlap(train_df, dev_set):\n",
    "    overlap = []\n",
    "    for sentence in train_df.Combined:\n",
    "        for char in [',', '.', '?', '!', ':', ';',\"'\", '\"', ')', '(', '<br>']:\n",
    "            sentence = sentence.replace(char, '').lower()\n",
    "\n",
    "            overlap.append(len(set(sentence).intersection(dev_set)))\n",
    "            \n",
    "    return np.mean(overlap)\n",
    "            \n",
    "compute_mean_overlap(train_df=anli_train, dev_set=anli_dev_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lyric-thanks",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/test_predictions/test/test_preds_1_epoch_op_versch_test_sets_2/random/46/SNLI/test_predictions.pickle'\n",
    "SNLI = load_confidences(path=path, predictions=False, exclude=0)\n",
    "\n",
    "path = '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/test_predictions/test/test_preds_1_epoch_op_versch_test_sets_2/random/46/MNLI/test_predictions.pickle'\n",
    "MNLI = load_confidences(path=path, predictions=False, exclude=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "british-michigan",
   "metadata": {},
   "outputs": [],
   "source": [
    "SNLI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "equipped-republican",
   "metadata": {},
   "outputs": [],
   "source": [
    "MNLI = MNLI[~MNLI.id.isin(SNLI.id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "opposed-glass",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(MNLI[MNLI.correctness == 1.0])/len(MNLI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "brief-tourism",
   "metadata": {},
   "outputs": [],
   "source": [
    "MNLI = MNLI[MNLI.id.isin(SNLI.id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "assured-bobby",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(SNLI[SNLI.correctness == 1])/len(SNLI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "informed-gender",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(MNLI[MNLI.correctness == 1])/len(MNLI)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
