{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "wicked-sunday",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:70% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "from pprint import pprint\n",
    "import sys\n",
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "import matplotlib.patches as mpatches\n",
    "\n",
    "from scipy.stats import entropy\n",
    "from matplotlib.ticker import PercentFormatter\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "\n",
    "from scipy.stats import sem\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 1000)\n",
    "\n",
    "COL_WIDTH=250\n",
    "CMAP = 'seismic_r'\n",
    "_CMAP = 'viridis'\n",
    "pd.options.display.max_colwidth = COL_WIDTH\n",
    "\n",
    "EASY_THRESH = 0.75\n",
    "MEDIUM_THRESH = 0.5\n",
    "HARD_THRESH = 0.25\n",
    "IMPOSSIBLE_THRESH = 0.0\n",
    "ALPHA = 0.6\n",
    "\n",
    "SEEDS = [42, 43, 44, 45, 46]\n",
    "# BASE_PATH = '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/datamaps/test/datamap_test_subepoch_inference_elke_0.5_epochs/roberta-large/random/SEED/confidences.pickle'\n",
    "BASE_PATH = '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/datamap/test/datamap_op_test_set_subepoch_inference_elke_0.5_epochs_max_4_epochs/random/SEED/datamap.pickle'\n",
    "STRATEGY_PATH = \"/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/acquisition_IDs/array/OOD_MNLI_active_learning_array_donderdag/roberta-large/STRATEGY/SEED/acquisition_ids.csv\"\n",
    "\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:70% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unknown-supplement",
   "metadata": {},
   "source": [
    "# Loading pickled confidences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "postal-indication",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_confidences(path, predictions=False, exclude=1):\n",
    "\n",
    "    df = pd.read_pickle(path)\n",
    "    sample_ids = list(df.keys())\n",
    "\n",
    "    cartography_dict = {}\n",
    "    prediction_dict = {}\n",
    "\n",
    "    for sample_id in sample_ids:\n",
    "\n",
    "        times_correct = []\n",
    "        confidences = []\n",
    "\n",
    "        outputs = df[sample_id]\n",
    "        outputs = outputs[exclude:] # don't count the first 0.5 epoch!\n",
    "\n",
    "        for output in outputs:\n",
    "\n",
    "            # extract values from output tuple\n",
    "            prediction = output[0]\n",
    "            label = output[1]\n",
    "\n",
    "            # obtain GT confidence, determine correctness of prediction\n",
    "            confidence = prediction[label]\n",
    "            correct = 1 if prediction.index(max(prediction)) == label else 0\n",
    "\n",
    "            # track confidence and correctness\n",
    "            confidences.append(confidence)\n",
    "            times_correct.append(correct)\n",
    "\n",
    "\n",
    "        cartography_dict[sample_id] = {'confidence': np.mean(confidences),\n",
    "                                      'variability': np.std(confidences),\n",
    "                                      'correctness': np.mean(times_correct)}\n",
    "\n",
    "        prediction_dict[sample_id] = {'prediction': outputs[-1][0]}\n",
    "\n",
    "    confidences = [cartography_dict[key]['confidence'] for key in list(cartography_dict.keys())]\n",
    "    variability = [cartography_dict[key]['variability'] for key in list(cartography_dict.keys())]\n",
    "    correctness = [cartography_dict[key]['correctness'] for key in list(cartography_dict.keys())]\n",
    "\n",
    "    df = pd.DataFrame(list(zip(sample_ids, confidences, variability, correctness)), columns=['id', 'mean_conf', 'variability', 'correctness'])\n",
    "    \n",
    "    if predictions == False:\n",
    "        return df\n",
    "    else:\n",
    "        return df, prediction_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cordless-parts",
   "metadata": {},
   "source": [
    "# Code to read datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "hybrid-decimal",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def read_dataset(input_dir, dataset_id, split, seed, wanli_id):\n",
    "    \"\"\"\n",
    "    This function takes a dataset id and reads the corresponding .json split in an appropriate Pandas DataFrame\n",
    "    :param wanli_id: id key for wanli json file (pairID for metrics, id for non metric stuff)\n",
    "    :param seed:\n",
    "    :param input_dir:\n",
    "    :param dataset_id: str indicating which dataset should be read\n",
    "    :param split: str indicating which split should be read\n",
    "    :return: DataFrame with examples (Premise, Hypothesis, Label, ID)\n",
    "    \"\"\"\n",
    "    # anli_rounds = ['ANLI_R1', 'ANLI_R2', 'ANLI_R3']\n",
    "    anli_rounds = ['ANLI_R2', 'ANLI_R3']\n",
    "\n",
    "    def replace_labels(label):\n",
    "        label_conversions = {'e': 'entailment',\n",
    "                             'c': 'contradiction',\n",
    "                             'n': 'neutral'}\n",
    "\n",
    "        return label_conversions[label]\n",
    "\n",
    "    if dataset_id == 'SNLI':\n",
    "\n",
    "        # TODO only consider snli with gold labels?\n",
    "        data_path = '{}/snli_1.0/snli_1.0_{}.jsonl'.format(input_dir, split)\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['sentence1', 'sentence2', 'gold_label', 'pairID']]\n",
    "        dataset['dataset'] = 'SNLI'\n",
    "        dataset = dataset.drop(dataset[dataset.gold_label.str.contains('-')].index)  # drop examples with no gold label\n",
    "\n",
    "    # load separate ANLI rounds\n",
    "    elif dataset_id in anli_rounds:\n",
    "\n",
    "        data_round = dataset_id[-2:]  # extract the 'R#' substring from the provided ID\n",
    "        data_path = '{}/anli_v1.0/{}/{}.jsonl'.format(input_dir, data_round, split)\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['context', 'hypothesis', 'label', 'uid']]  # get rid of unnecessary columns\n",
    "        dataset['label'] = dataset['label'].apply(replace_labels)  # ensures consistently named labels\n",
    "        dataset['dataset'] = 'ANLI_{}'.format(data_round)\n",
    "        #\n",
    "        # print('Dataframe for {} - {}'.format(dataset_id, split), flush=True)\n",
    "\n",
    "    # compile all ANLI rounds\n",
    "    elif dataset_id == 'ANLI':\n",
    "\n",
    "        dataset_list = []\n",
    "        for data_round in anli_rounds:\n",
    "            data_round = data_round[-2:]\n",
    "            data_path = '{}/anli_v1.0/{}/{}.jsonl'.format(input_dir, data_round, split)\n",
    "            dataset = pd.read_json(data_path, lines=True)\n",
    "            dataset = dataset[['context', 'hypothesis', 'label', 'uid']]  # get rid of unnecessary columns\n",
    "            dataset['label'] = dataset['label'].apply(replace_labels)  # ensures consistently named labels\n",
    "            dataset['dataset'] = 'ANLI'\n",
    "            dataset_list.append(dataset)\n",
    "        dataset = pd.concat(dataset_list, axis=0)\n",
    "\n",
    "    # Load MNLI\n",
    "    elif dataset_id == 'MNLI':\n",
    "\n",
    "        if split == 'train':\n",
    "            data_path = '{}/multinli_1.0/multinli_1.0_{}.jsonl'.format(input_dir, split)\n",
    "        elif split == 'dev' or split == 'test':\n",
    "            data_path = '{}/multinli_1.0/multinli_1.0_dev_matched.jsonl'.format(input_dir)\n",
    "\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['sentence1', 'sentence2', 'gold_label', 'pairID']]\n",
    "        dataset['dataset'] = 'MNLI'\n",
    "        dataset = dataset.drop(dataset[dataset.gold_label.str.contains('-')].index)\n",
    "\n",
    "        # split dev into dev and test set\n",
    "        if split == 'dev':\n",
    "            dataset = dataset.sample(frac=1,  # reshuffle all rows in the dataframe prior to split\n",
    "                                     random_state=seed)  # seed is fixed such that dev and test set are always the same\n",
    "            dataset = dataset[:int((len(dataset)/2))]\n",
    "\n",
    "        elif split == 'test':\n",
    "            dataset = dataset.sample(frac=1,\n",
    "                                     random_state=seed)\n",
    "            dataset = dataset[int((len(dataset)/2)):]\n",
    "\n",
    "    # Load WANLI\n",
    "    elif dataset_id == 'WANLI':\n",
    "\n",
    "        if split == 'train':\n",
    "            data_path = '{}/wanli/train.jsonl'.format(input_dir)\n",
    "        elif split == 'dev' or split == 'test':\n",
    "            data_path = '{}/wanli/test.jsonl'.format(input_dir)\n",
    "\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['premise', 'hypothesis', 'gold', wanli_id]]\n",
    "        dataset['dataset'] = 'WANLI'\n",
    "        dataset = dataset.drop(dataset[dataset.gold.str.contains('-')].index)\n",
    "\n",
    "        # split dev into dev and test set\n",
    "        if split == 'dev':\n",
    "            dataset = dataset.sample(frac=1,  # reshuffle all rows in the dataframe prior to split\n",
    "                                     random_state=seed)  # seed is fixed such that dev and test set are always the same\n",
    "            dataset = dataset[:int((len(dataset) / 2))]\n",
    "\n",
    "        elif split == 'test':\n",
    "            dataset = dataset.sample(frac=1,\n",
    "                                     random_state=seed)\n",
    "            dataset = dataset[int((len(dataset) / 2)):]\n",
    "\n",
    "    else:\n",
    "        raise KeyError('No dataset found for \"{}\"'.format(dataset_id))\n",
    "\n",
    "    # ensure consistent headers per dataset DataFrame\n",
    "    dataset.columns = ['Premise', 'Hypothesis', 'Label', 'ID', 'Dataset']\n",
    "\n",
    "    dataset['ID'] = dataset['ID'].astype('string')\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "circular-orientation",
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIR='/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/resources/ard_data'\n",
    "SPLIT='test'\n",
    "WANLI_ID='pairID'\n",
    "\n",
    "def read_source(dataset_id):\n",
    "    dataset = read_dataset(input_dir=INPUT_DIR, \n",
    "                 dataset_id=dataset_id, \n",
    "                 split=SPLIT, \n",
    "                 seed=None, \n",
    "                 wanli_id=WANLI_ID)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "promising-receptor",
   "metadata": {},
   "source": [
    "# Loading all datasets into their own DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "legendary-estonia",
   "metadata": {},
   "outputs": [],
   "source": [
    "mnli = read_source('MNLI')\n",
    "snli = read_source('SNLI')\n",
    "anli = read_source('ANLI')\n",
    "wanli = read_source('WANLI')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sublime-finance",
   "metadata": {},
   "source": [
    "# Code for Generating Datamaps for Different Sources / Strategies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "third-homework",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_df(dataframe):\n",
    "    X = dataframe['variability'].tolist()\n",
    "    Y = dataframe['mean_conf'].tolist()\n",
    "    colors = dataframe['correctness'].tolist()\n",
    "\n",
    "    plt.figure(figsize=(30, 10))\n",
    "    plt.scatter(x=X, y=Y, c=colors, alpha=1, cmap=mpl.cm.get_cmap(CMAP))\n",
    "    plt.xlim(0,0.5)\n",
    "    plt.ylim(0,1)\n",
    "    plt.xlabel('Variability')\n",
    "    plt.ylabel('Confidence')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    \n",
    "def generate_source_map(base_df, source_df):\n",
    "\n",
    "    ids = source_df.ID\n",
    "\n",
    "    source_df = base_df[base_df.id.astype('string').isin(ids)]\n",
    "    \n",
    "    print('num of source in base: {}'.format(len(source_df)))\n",
    "    \n",
    "    generate_df(source_df)\n",
    "    \n",
    "def datamap_subset(base_df, source_df, name=None):\n",
    "    \n",
    "    source_ids = source_df.ID\n",
    "    \n",
    "    subset_df = base_df[base_df.id.astype('string').isin(source_ids)]\n",
    "\n",
    "    if name is not None:\n",
    "        subset_df['Name'] = name\n",
    "    else:\n",
    "        subset_df['Name'] = source_df.Dataset.tolist()[0]\n",
    "    \n",
    "    return subset_df\n",
    "\n",
    "def datamap_subset_multi(base_path, source_df, name=None):\n",
    "    \n",
    "    list_of_maps = []\n",
    "    \n",
    "    source_ids = source_df.ID\n",
    "    \n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        _base_path = base_path.replace('SEED', str(seed))\n",
    "        \n",
    "        base_df = load_confidences(_base_path, predictions=False)\n",
    "\n",
    "        subset_df = base_df[base_df.id.astype('string').isin(source_ids)]\n",
    "\n",
    "        if name is not None:\n",
    "            subset_df['Name'] = name\n",
    "        else:\n",
    "            subset_df['Name'] = source_df.Dataset.tolist()[0]\n",
    "            \n",
    "        list_of_maps.append(subset_df)\n",
    "    \n",
    "    return list_of_maps\n",
    "\n",
    "def generate_dfs_2(dataframes, markers=None, size=40, n_bins=10, font_size=20):\n",
    "    \n",
    "    plt.rcParams.update({'font.size': font_size})\n",
    "    \n",
    "    def generate_hist(axis, data, name, color, colormap, n_bins=10):\n",
    "        \n",
    "        cmap = False\n",
    "        if color == 'cmap':\n",
    "            cmap = True\n",
    "            color = 'green'\n",
    "        \n",
    "        n, bins, patches = axis.hist(data, \n",
    "                                     n_bins, \n",
    "                                     weights=np.ones(len(data)) / len(data), \n",
    "                                     density=False, \n",
    "                                     color=color)\n",
    "\n",
    "        bin_centers = 0.5 * (bins[:-1] + bins[1:])\n",
    "        \n",
    "        if cmap:\n",
    "        \n",
    "            # scale values to interval [0,1]\n",
    "            col = bin_centers - min(bin_centers)\n",
    "            col /= max(col)\n",
    "\n",
    "            for c, p in zip(col, patches):\n",
    "                plt.setp(p, 'facecolor', colormap(c))\n",
    "\n",
    "        axis.set_xlabel(name)\n",
    "        axis.set_ylim(0,1)\n",
    "\n",
    "        axis.yaxis.set_major_formatter(FuncFormatter('{0:.0%}'.format))\n",
    "    \n",
    "    # creating grid for subplots\n",
    "    fig = plt.figure()\n",
    "    fig.set_figheight(size)\n",
    "    fig.set_figwidth(size)\n",
    "\n",
    "    ax1 = plt.subplot2grid(shape=(4, 4), loc=(0, 0), colspan=3, rowspan=4)\n",
    "    ax2 = plt.subplot2grid(shape=(4, 4), loc=(0, 3), rowspan=1)\n",
    "    ax3 = plt.subplot2grid(shape=(4, 4), loc=(1, 3), colspan=1)\n",
    "    ax4 = plt.subplot2grid(shape=(4, 4), loc=(2, 3), rowspan=1)\n",
    "    ax5 = plt.subplot2grid(shape=(4, 4), loc=(3, 3), rowspan=1)\n",
    "    \n",
    "    for dataframe in dataframes:\n",
    "\n",
    "        if markers is not None:\n",
    "\n",
    "            for dataset, marker in zip(['SNLI', 'ANLI', 'WANLI', 'MNLI'], ['s', 'x', 'o', 'v']):\n",
    "\n",
    "\n",
    "                sub_frame = dataframe[dataframe['Dataset'] == dataset]\n",
    "                X = sub_frame['variability'].tolist()\n",
    "                Y = sub_frame['mean_conf'].tolist()\n",
    "                colors = sub_frame['correctness'].tolist()\n",
    "                ax1.scatter(x=X, y=Y, c=colors, alpha=1, marker=marker, cmap=mpl.cm.get_cmap(CMAP), label=dataset, s=size)\n",
    "\n",
    "        else:\n",
    "\n",
    "\n",
    "            X = dataframe['variability'].tolist()\n",
    "            Y = dataframe['mean_conf'].tolist()\n",
    "            colors = dataframe['correctness'].tolist()\n",
    "\n",
    "            ax1.scatter(x=X, y=Y, c=colors, alpha=1, cmap=mpl.cm.get_cmap(CMAP), s=size)\n",
    "\n",
    "        \n",
    "        name = dataframe.Name.tolist()[0].upper()\n",
    "        ax1.set_title('Dataset Map for ' + name + ' Test Data')\n",
    "        ax1.set_xlabel('Variability')\n",
    "        ax1.set_ylabel('Confidence')\n",
    "\n",
    "        if markers:\n",
    "            ax1.legend(loc='upper right', prop={'size': 30})\n",
    "            ax1.legend(markerscale=4)\n",
    "            \n",
    "        break # terminate loop for datamap after first seed since only one seed can be plotted\n",
    "    \n",
    "    \n",
    "    easy = []\n",
    "    medium = []\n",
    "    hard = []\n",
    "    impossible = []\n",
    "    X = []\n",
    "    Y = []\n",
    "    correctness = []\n",
    "    for dataframe in dataframes:\n",
    "    \n",
    "        X.extend( dataframe['variability'].tolist() )\n",
    "        Y.extend( dataframe['mean_conf'].tolist() )\n",
    "        correctness.extend( dataframe['correctness'].tolist() )\n",
    "\n",
    "        easy.append( len(dataframe[dataframe['mean_conf'] > EASY_THRESH])  )\n",
    "        medium.append( len(dataframe[(dataframe['mean_conf'] > MEDIUM_THRESH) & (dataframe['mean_conf'] < EASY_THRESH)])  )\n",
    "        hard.append( len(dataframe[(dataframe['mean_conf'] > HARD_THRESH) & (dataframe['mean_conf'] < MEDIUM_THRESH)])   )\n",
    "        impossible.append( len(dataframe[(dataframe['mean_conf'] > IMPOSSIBLE_THRESH) & (dataframe['mean_conf'] < HARD_THRESH)])  )\n",
    "    \n",
    "    # take the mean over difficulty brackets\n",
    "    # TODO: generate std bars?\n",
    "    def mean_and_std_e(array):\n",
    "        \n",
    "        return np.mean(array), np.std(array)\n",
    "    \n",
    "    easy, std_easy = mean_and_std_e(easy)\n",
    "    medium, std_medium = mean_and_std_e(medium)\n",
    "    hard, std_hard = mean_and_std_e(hard)\n",
    "    impossible, std_impossible = mean_and_std_e(impossible)\n",
    "    \n",
    "#     std_easy, std_medium, std_hard, std_impossible = 200,30,400,20\n",
    "    \n",
    "    # LEARNABILITY      \n",
    "    width = 1\n",
    "    cmap = mpl.cm.get_cmap(CMAP)\n",
    "    \n",
    "    categories = ['Easy', 'Med', 'Hard', 'Imp.']\n",
    "    x_pos = [0,1,2,3]\n",
    "    means = [easy, medium, hard, impossible]\n",
    "    \n",
    "    \n",
    "    y_err = [std_easy, std_medium, std_hard, std_impossible]\n",
    "    \n",
    "    normed_means = [mean / sum(means) for mean in means]\n",
    "    y_err = [std / sum(means) for std in y_err]\n",
    "    \n",
    "    colors = [cmap(value) for value in [0.85, 0.65, 0.35, 0.125]]\n",
    "    \n",
    "    labels = [('[p > {}]'.format(thresh, color=cmap(value), alpha=ALPHA)) for thresh, value in zip([EASY_THRESH, MEDIUM_THRESH, HARD_THRESH, IMPOSSIBLE_THRESH],[0.85, 0.65, 0.35, 0.125])]\n",
    "    ax2.bar(x_pos, normed_means, yerr=y_err, width=1, color=colors, capsize=10, alpha=ALPHA)\n",
    "\n",
    "    thresholds = [EASY_THRESH, MEDIUM_THRESH, HARD_THRESH, IMPOSSIBLE_THRESH]\n",
    "    patches = [mpatches.Patch(color=value, label='[p > {}]'.format(thresh), alpha=ALPHA) for thresh, value in zip(thresholds, colors)]\n",
    "    \n",
    "    ax2.legend(handles=patches, loc='upper right')\n",
    "    \n",
    "    ax2.set_xticks(x_pos)\n",
    "    ax2.set_xticklabels(categories)\n",
    "    ax2.set_ylim(0,1)\n",
    "    \n",
    "#     ylabels = ['{}'.format(round(x,1)) + 'K' for x in ax2.get_yticks()[::1]/1000]\n",
    "    ylabels = ['{}%'.format(int(x))  for x in ax2.get_yticks()[::1]*100]\n",
    "    ax2.set_yticklabels(ylabels)    \n",
    "    ax2.set_xlabel('Learnability')\n",
    "#     ax2.legend(loc='upper right')\n",
    "\n",
    "    \n",
    "    # CORRECTNESS      \n",
    "    generate_hist(axis=ax3, data=correctness, name='Correctness', color='cmap', colormap=mpl.cm.get_cmap(CMAP))\n",
    "    \n",
    "    # CONFIDENCES      \n",
    "    generate_hist(axis=ax4, data=Y, name='Confidences', color='red', colormap=None)\n",
    "\n",
    "    # VARIABILITY      \n",
    "    generate_hist(axis=ax5, data=X, name='Variability', color='green', colormap=None)\n",
    "    \n",
    "    for ax in [ax2, ax3, ax4, ax5]:\n",
    "        ax.yaxis.tick_right()\n",
    "        ax.yaxis.set_label_position(\"right\")\n",
    "#         ax.xaxis.set_label_position('top')\n",
    "    \n",
    "    plt.savefig('datamap_{}'.format(name))    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "numeric-speaker",
   "metadata": {},
   "source": [
    "# Datamaps per Source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "injured-float",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "snli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=snli)\n",
    "anli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=anli)\n",
    "wanli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=wanli)\n",
    "mnli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=mnli)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "confidential-draft",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 309876 characters]\n",
      "text/plain": [
       "<Figure size 1000x1000 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 221524 characters]\n",
      "text/plain": [
       "<Figure size 1000x1000 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 166576 characters]\n",
      "text/plain": [
       "<Figure size 1000x1000 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 231612 characters]\n",
      "text/plain": [
       "<Figure size 1000x1000 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.dpi'] = 40\n",
    "SIZE = 25\n",
    "FONT_SIZE = 22\n",
    "generate_dfs_2(snli_maps, size=SIZE, font_size=FONT_SIZE)\n",
    "generate_dfs_2(anli_maps, size=SIZE, font_size=FONT_SIZE)\n",
    "generate_dfs_2(wanli_maps, size=SIZE, font_size=FONT_SIZE)\n",
    "generate_dfs_2(mnli_maps, size=SIZE, font_size=FONT_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "contemporary-sessions",
   "metadata": {},
   "source": [
    "# Performance per strategy per difficulty segment per dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "filled-lotus",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean accuracy, std on easy SNLI examples for random: 0.954 - 0.008\n",
      "mean accuracy, std on easy ANLI examples for random: 0.804 - 0.048\n",
      "mean accuracy, std on easy WANLI examples for random: 0.949 - 0.016\n",
      "mean accuracy, std on easy MNLI examples for random: 0.944 - 0.017\n",
      "\n",
      "random - mean accuracy, std over all easy examples: 0.913 - 0.068\n",
      "\n",
      "mean accuracy, std on medium SNLI examples for random: 0.696 - 0.026\n",
      "mean accuracy, std on medium ANLI examples for random: 0.622 - 0.027\n",
      "mean accuracy, std on medium WANLI examples for random: 0.727 - 0.019\n",
      "mean accuracy, std on medium MNLI examples for random: 0.728 - 0.047\n",
      "\n",
      "random - mean accuracy, std over all medium examples: 0.693 - 0.053\n",
      "\n",
      "mean accuracy, std on hard SNLI examples for random: 0.463 - 0.035\n",
      "mean accuracy, std on hard ANLI examples for random: 0.393 - 0.033\n",
      "mean accuracy, std on hard WANLI examples for random: 0.385 - 0.056\n",
      "mean accuracy, std on hard MNLI examples for random: 0.412 - 0.045\n",
      "\n",
      "random - mean accuracy, std over all hard examples: 0.413 - 0.053\n",
      "\n",
      "mean accuracy, std on impossible SNLI examples for random: 0.177 - 0.017\n",
      "mean accuracy, std on impossible ANLI examples for random: 0.144 - 0.024\n",
      "mean accuracy, std on impossible WANLI examples for random: 0.124 - 0.019\n",
      "mean accuracy, std on impossible MNLI examples for random: 0.122 - 0.029\n",
      "\n",
      "random - mean accuracy, std over all impossible examples: 0.141 - 0.032\n",
      "\n",
      "mean accuracy, std on easy SNLI examples for mc-max-entropy: 0.95 - 0.005\n",
      "mean accuracy, std on easy ANLI examples for mc-max-entropy: 0.77 - 0.041\n",
      "mean accuracy, std on easy WANLI examples for mc-max-entropy: 0.923 - 0.007\n",
      "mean accuracy, std on easy MNLI examples for mc-max-entropy: 0.95 - 0.008\n",
      "\n",
      "mc-max-entropy - mean accuracy, std over all easy examples: 0.898 - 0.078\n",
      "\n",
      "mean accuracy, std on medium SNLI examples for mc-max-entropy: 0.702 - 0.016\n",
      "mean accuracy, std on medium ANLI examples for mc-max-entropy: 0.617 - 0.035\n",
      "mean accuracy, std on medium WANLI examples for mc-max-entropy: 0.714 - 0.012\n",
      "mean accuracy, std on medium MNLI examples for mc-max-entropy: 0.784 - 0.026\n",
      "\n",
      "mc-max-entropy - mean accuracy, std over all medium examples: 0.704 - 0.064\n",
      "\n",
      "mean accuracy, std on hard SNLI examples for mc-max-entropy: 0.485 - 0.016\n",
      "mean accuracy, std on hard ANLI examples for mc-max-entropy: 0.429 - 0.032\n",
      "mean accuracy, std on hard WANLI examples for mc-max-entropy: 0.441 - 0.038\n",
      "mean accuracy, std on hard MNLI examples for mc-max-entropy: 0.517 - 0.047\n",
      "\n",
      "mc-max-entropy - mean accuracy, std over all hard examples: 0.468 - 0.05\n",
      "\n",
      "mean accuracy, std on impossible SNLI examples for mc-max-entropy: 0.203 - 0.024\n",
      "mean accuracy, std on impossible ANLI examples for mc-max-entropy: 0.157 - 0.02\n",
      "mean accuracy, std on impossible WANLI examples for mc-max-entropy: 0.161 - 0.029\n",
      "mean accuracy, std on impossible MNLI examples for mc-max-entropy: 0.212 - 0.029\n",
      "\n",
      "mc-max-entropy - mean accuracy, std over all impossible examples: 0.183 - 0.036\n",
      "\n",
      "mean accuracy, std on easy SNLI examples for bald: 0.955 - 0.01\n",
      "mean accuracy, std on easy ANLI examples for bald: 0.821 - 0.031\n",
      "mean accuracy, std on easy WANLI examples for bald: 0.945 - 0.009\n",
      "mean accuracy, std on easy MNLI examples for bald: 0.948 - 0.005\n",
      "\n",
      "bald - mean accuracy, std over all easy examples: 0.917 - 0.058\n",
      "\n",
      "mean accuracy, std on medium SNLI examples for bald: 0.705 - 0.026\n",
      "mean accuracy, std on medium ANLI examples for bald: 0.611 - 0.016\n",
      "mean accuracy, std on medium WANLI examples for bald: 0.752 - 0.023\n",
      "mean accuracy, std on medium MNLI examples for bald: 0.764 - 0.014\n",
      "\n",
      "bald - mean accuracy, std over all medium examples: 0.708 - 0.063\n",
      "\n",
      "mean accuracy, std on hard SNLI examples for bald: 0.471 - 0.032\n",
      "mean accuracy, std on hard ANLI examples for bald: 0.392 - 0.026\n",
      "mean accuracy, std on hard WANLI examples for bald: 0.458 - 0.034\n",
      "mean accuracy, std on hard MNLI examples for bald: 0.425 - 0.024\n",
      "\n",
      "bald - mean accuracy, std over all hard examples: 0.436 - 0.042\n",
      "\n",
      "mean accuracy, std on impossible SNLI examples for bald: 0.207 - 0.017\n",
      "mean accuracy, std on impossible ANLI examples for bald: 0.126 - 0.012\n",
      "mean accuracy, std on impossible WANLI examples for bald: 0.129 - 0.024\n",
      "mean accuracy, std on impossible MNLI examples for bald: 0.134 - 0.021\n",
      "\n",
      "bald - mean accuracy, std over all impossible examples: 0.149 - 0.039\n",
      "\n",
      "mean accuracy, std on easy SNLI examples for dal: 0.937 - 0.012\n",
      "mean accuracy, std on easy ANLI examples for dal: 0.818 - 0.035\n",
      "mean accuracy, std on easy WANLI examples for dal: 0.943 - 0.008\n",
      "mean accuracy, std on easy MNLI examples for dal: 0.94 - 0.009\n",
      "\n",
      "dal - mean accuracy, std over all easy examples: 0.909 - 0.056\n",
      "\n",
      "mean accuracy, std on medium SNLI examples for dal: 0.678 - 0.024\n",
      "mean accuracy, std on medium ANLI examples for dal: 0.631 - 0.023\n",
      "mean accuracy, std on medium WANLI examples for dal: 0.748 - 0.051\n",
      "mean accuracy, std on medium MNLI examples for dal: 0.739 - 0.034\n",
      "\n",
      "dal - mean accuracy, std over all medium examples: 0.699 - 0.059\n",
      "\n",
      "mean accuracy, std on hard SNLI examples for dal: 0.447 - 0.022\n",
      "mean accuracy, std on hard ANLI examples for dal: 0.39 - 0.02\n",
      "mean accuracy, std on hard WANLI examples for dal: 0.406 - 0.022\n",
      "mean accuracy, std on hard MNLI examples for dal: 0.427 - 0.035\n",
      "\n",
      "dal - mean accuracy, std over all hard examples: 0.417 - 0.033\n",
      "\n",
      "mean accuracy, std on impossible SNLI examples for dal: 0.184 - 0.02\n",
      "mean accuracy, std on impossible ANLI examples for dal: 0.142 - 0.017\n",
      "mean accuracy, std on impossible WANLI examples for dal: 0.129 - 0.027\n",
      "mean accuracy, std on impossible MNLI examples for dal: 0.127 - 0.022\n",
      "\n",
      "dal - mean accuracy, std over all impossible examples: 0.145 - 0.032\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "To do:\n",
    "Per seed, per strategy kijken hoeveel test examples ze goed hebben van:\n",
    "- makkelijk\n",
    "- medium\n",
    "- moeilijk\n",
    "- onmogelijk\n",
    "\"\"\"\n",
    "\n",
    "test_pred_path = '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/test_predictions/array/OOD_MNLI_active_learning_array_remove_bottom_025_poging_4/STRATEGY/SEED/DATASET/test_predictions.pickle'\n",
    "\n",
    "\n",
    "source_maps = [snli_maps, anli_maps, wanli_maps, mnli_maps]\n",
    "\n",
    "\n",
    "def compute_acc_per_difficulty(difficulty_str, strategy_str):\n",
    "    for difficulty in [difficulty_str]:\n",
    "\n",
    "        total_corr = []\n",
    "        total_ex = []\n",
    "\n",
    "        for datamaps in source_maps:\n",
    "\n",
    "            for strategy in [strategy_str]:\n",
    "\n",
    "                    tally = []\n",
    "                    for datamap, seed in zip(datamaps, SEEDS):\n",
    "\n",
    "                        dataset_id = (datamap.Name.iloc[0])\n",
    "\n",
    "                        pred_path = test_pred_path.replace('SEED',str(seed)).replace('STRATEGY', strategy).replace('DATASET', dataset_id)\n",
    "\n",
    "                        test_predictions = load_confidences(pred_path, predictions=False, exclude=0)\n",
    "\n",
    "                        if difficulty == 'easy':\n",
    "                            pred_segments = datamap[(datamap.mean_conf > EASY_THRESH)]\n",
    "\n",
    "                        if difficulty == 'medium':\n",
    "                            pred_segments = datamap[(datamap.mean_conf > MEDIUM_THRESH) & (datamap.mean_conf < EASY_THRESH)]\n",
    "\n",
    "                        if difficulty == 'hard':\n",
    "                            pred_segments = datamap[(datamap.mean_conf > HARD_THRESH) & (datamap.mean_conf < MEDIUM_THRESH)]\n",
    "\n",
    "                        if difficulty == 'impossible':\n",
    "                            pred_segments = datamap[(datamap.mean_conf < HARD_THRESH)]\n",
    "\n",
    "                        segment_acc = np.mean(test_predictions[test_predictions.id.isin(pred_segments.id)].correctness)\n",
    "                        tally.append(segment_acc)\n",
    "\n",
    "    #                     print(np.sum(test_predictions[test_predictions.id.isin(pred_segments.id)].correctness))\n",
    "\n",
    "                        total_corr.append( int(np.sum(test_predictions[test_predictions.id.isin(pred_segments.id)].correctness)) )\n",
    "                        total_ex.append( len(pred_segments) )\n",
    "\n",
    "\n",
    "                    print('mean accuracy, std on {} {} examples for {}: {} - {}'.format(difficulty, dataset_id, strategy, round(np.mean(tally),3), round(np.std(tally),3)))\n",
    "\n",
    "        result = [round(x/y,3) for x,y in zip(total_corr, total_ex)]\n",
    "        print('\\n{} - mean accuracy, std over all {} examples: {} - {}\\n'.format(strategy_str, difficulty,\n",
    "                                                                   round(np.mean(result),3),\n",
    "                                                                   round(np.std(result), 3)))\n",
    "\n",
    "for strategy in ['random', 'mc-max-entropy', 'bald', 'dal']:\n",
    "    compute_acc_per_difficulty('easy', strategy_str=strategy)\n",
    "    compute_acc_per_difficulty('medium', strategy_str=strategy)\n",
    "    compute_acc_per_difficulty('hard', strategy_str=strategy)\n",
    "    compute_acc_per_difficulty('impossible', strategy_str=strategy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "regulated-declaration",
   "metadata": {},
   "source": [
    "# Performance per strategy per difficulty segment for all data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "threaded-hacker",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "DATASETS = ['WANLI']\n",
    "\n",
    "def aggregate_predictions(strategies):\n",
    "    \n",
    "    preds_per_strategy = []\n",
    "    for strategy in strategies:\n",
    "\n",
    "        # aggregate predictions over all datasets over all seeds\n",
    "        agg_over_seeds = []\n",
    "        for seed in SEEDS:\n",
    "\n",
    "            # aggregate predictions over all datasets\n",
    "            agg_over_datasets = []\n",
    "            for dataset_id in DATASETS:\n",
    "\n",
    "                prediction_path = test_pred_path.replace('SEED', str(seed)).replace('STRATEGY', strategy).replace('DATASET', dataset_id)\n",
    "\n",
    "                test_predictions = load_confidences(prediction_path, predictions=False, exclude=0)\n",
    "                agg_over_datasets.append(test_predictions)\n",
    "\n",
    "            df_for_all_datasets = pd.concat(agg_over_datasets) # combine for all datasets,\n",
    "#             print(df_for_all_datasets)\n",
    "\n",
    "            agg_over_seeds.append(df_for_all_datasets)\n",
    "\n",
    "        all_predictions = pd.concat(agg_over_seeds)\n",
    "        preds_per_strategy.append(all_predictions)\n",
    "        \n",
    "    return preds_per_strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "loaded-onion",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4708"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "early-gates",
   "metadata": {},
   "outputs": [],
   "source": [
    "def aggregate_maps_over_seeds(maps_list):\n",
    "    \n",
    "    maps_per_source = []\n",
    "    for datamaps in maps_list:\n",
    "        \n",
    "        aggregate_map = pd.concat([datamap for datamap in datamaps]).groupby('id').mean()\n",
    "        maps_per_source.append(aggregate_map)\n",
    "        \n",
    "    return maps_per_source\n",
    "        \n",
    "def aggregate_maps_over_sources(maps_list):\n",
    "    \n",
    "    return pd.concat(aggregate_maps_over_seeds(maps_list))\n",
    "    \n",
    "complete_map = aggregate_maps_over_sources([snli_maps, anli_maps, wanli_maps, mnli_maps])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "sharp-mounting",
   "metadata": {},
   "outputs": [],
   "source": [
    "easy = complete_map[complete_map.mean_conf > EASY_THRESH]\n",
    "medium = complete_map[(complete_map.mean_conf > MEDIUM_THRESH) & (complete_map.mean_conf < EASY_THRESH)]\n",
    "hard = complete_map[(complete_map.mean_conf > HARD_THRESH) & (complete_map.mean_conf < MEDIUM_THRESH)]\n",
    "impossible = complete_map[complete_map.mean_conf < HARD_THRESH]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "alive-extension",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[          id  mean_conf  variability  correctness\n",
       " 0     276736   0.913072          0.0          1.0\n",
       " 1     152465   0.389430          0.0          0.0\n",
       " 2      33339   0.952567          0.0          1.0\n",
       " 3     195850   0.494612          0.0          0.0\n",
       " 4     284917   0.297490          0.0          0.0\n",
       " ...      ...        ...          ...          ...\n",
       " 2460  130537   0.987579          0.0          1.0\n",
       " 2461  333820   0.801645          0.0          1.0\n",
       " 2462  162086   0.009525          0.0          0.0\n",
       " 2463  371164   0.304923          0.0          0.0\n",
       " 2464  325724   0.995773          0.0          1.0\n",
       " \n",
       " [12319 rows x 4 columns]]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "requested-workstation",
   "metadata": {},
   "source": [
    "# Datamaps per Strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "aquatic-bunch",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:33: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "def load_strategy_acquisitions(path, strategy, cap=4000):\n",
    "    \n",
    "    acquired_df = pd.read_csv(path)\n",
    "    acquired_df = acquired_df[:cap]\n",
    "    acquired_df['Name'] = strategy.capitalize()\n",
    "    \n",
    "    return acquired_df\n",
    "\n",
    "def get_subset_list(strategy_path, base_path, strategy_list):\n",
    "    \n",
    "    subset_list = []\n",
    "    \n",
    "    for strategy in strategy_list:\n",
    "        \n",
    "        strategy_list = []\n",
    "        \n",
    "        for seed in SEEDS:\n",
    "            \n",
    "            _base_path = base_path.replace('SEED', str(seed))\n",
    "            _strategy_path = strategy_path.replace('SEED', str(seed)).replace('STRATEGY', str(strategy))\n",
    "            \n",
    "            strategy_df = load_strategy_acquisitions(path=_strategy_path, strategy=strategy)\n",
    "\n",
    "            \n",
    "            base_df = load_confidences(_base_path)\n",
    "\n",
    "            id2dataset = {}\n",
    "            for i in range(len(strategy_df)):\n",
    "                row = strategy_df.iloc[i]\n",
    "                id2dataset[row.ID] = row.Dataset\n",
    "\n",
    "            strategy_subset = datamap_subset(base_df=base_df, source_df=strategy_df, name=strategy)\n",
    "            strategy_subset['Dataset'] = strategy_subset['id'].apply(lambda x: id2dataset[x])\n",
    "            \n",
    "            strategy_list.append(strategy_subset)\n",
    "        \n",
    "        subset_list.append(strategy_list)\n",
    "\n",
    "    return subset_list \n",
    "\n",
    "subset_list = get_subset_list(strategy_path=STRATEGY_PATH, base_path=BASE_PATH, strategy_list=['random', 'bald', 'mc-max-entropy', 'dal'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "abroad-reproduction",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 48384 characters]\n",
      "text/plain": [
       "<Figure size 900x900 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 44116 characters]\n",
      "text/plain": [
       "<Figure size 900x900 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 47496 characters]\n",
      "text/plain": [
       "<Figure size 900x900 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 49160 characters]\n",
      "text/plain": [
       "<Figure size 900x900 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# generate_dfs(subset_list, markers=True, size=100, n_bins=10)\n",
    "\n",
    "plt.rcParams['figure.dpi'] = 60\n",
    "SIZE = 15\n",
    "FONT_SIZE = 12\n",
    "\n",
    "for subset_maps in subset_list:\n",
    "    generate_dfs_2(subset_maps, markers=True, size=SIZE, font_size=FONT_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "medium-ecology",
   "metadata": {},
   "source": [
    "# Acquisitions by Difficulty over Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "recent-slave",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_stacked_barchart(base_dfs, strategy_dfs, return_axs=False, size=20, font_size=22, dpi=40):\n",
    "    \n",
    "    plt.rcParams.update({'font.size': font_size})\n",
    "    # Call this fn for a single strategy!\n",
    "    \n",
    "    num_easy_aggregate = []\n",
    "    num_medium_aggregate = []\n",
    "    num_hard_aggregate = []\n",
    "    num_impossible_aggregate = []\n",
    "    \n",
    "    for strategy_df, base_df in zip(strategy_dfs, base_dfs):\n",
    "        \n",
    "        number_of_examples = 500\n",
    "\n",
    "        labels = []\n",
    "        num_easy = []\n",
    "        num_medium = []\n",
    "        num_hard = []\n",
    "        num_impossible = []\n",
    "\n",
    "        easy_thresh = 0.75\n",
    "        medium_thresh = 0.5\n",
    "        hard_thresh = 0.25\n",
    "        impossible_thresh = 0.0\n",
    "\n",
    "        for active_round in range(1, strategy_df['round'].max()+1):\n",
    "\n",
    "            acquisitions = strategy_df[strategy_df['round']==active_round]\n",
    "            number_of_examples += len(acquisitions)\n",
    "            labels.append(number_of_examples)\n",
    "\n",
    "            confidences = base_df[base_df.id.isin(acquisitions.ID)]\n",
    "\n",
    "            easy_mask = confidences.mean_conf > easy_thresh\n",
    "            medium_mask = (confidences.mean_conf > medium_thresh) & (confidences.mean_conf < easy_thresh)\n",
    "            hard_mask = (confidences.mean_conf > hard_thresh) & (confidences.mean_conf < medium_thresh)\n",
    "            impossible_mask = (confidences.mean_conf > impossible_thresh) & (confidences.mean_conf < hard_thresh)\n",
    "\n",
    "            num_easy.append(len(confidences[easy_mask]))\n",
    "            num_medium.append(len(confidences[medium_mask]))\n",
    "            num_hard.append(len(confidences[hard_mask]))\n",
    "            num_impossible.append(len(confidences[impossible_mask]))\n",
    "            \n",
    "        num_easy_aggregate.append(num_easy)\n",
    "        num_medium_aggregate.append(num_medium)\n",
    "        num_hard_aggregate.append(num_hard)\n",
    "        num_impossible_aggregate.append(num_impossible)\n",
    "        \n",
    "    def get_mean_and_std_e(array):\n",
    "        \n",
    "        return np.mean(array, axis=0), np.std(array, axis=0)\n",
    "    \n",
    "    num_easy, num_easy_std = get_mean_and_std_e(num_easy_aggregate)    \n",
    "    num_medium, num_medium_std = get_mean_and_std_e(num_medium_aggregate)\n",
    "    num_hard, num_hard_std = get_mean_and_std_e(num_hard_aggregate)\n",
    "    num_impossible, num_impossible_std = get_mean_and_std_e(num_impossible_aggregate)  \n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "\n",
    "    width = 210\n",
    "    alpha = ALPHA\n",
    "    labels = np.array(labels)\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(CMAP)\n",
    "\n",
    "    ax.bar(labels, num_easy, width, yerr=num_easy_std, label='Easy [p > {}]'.format(easy_thresh), color=cmap(0.85), alpha=alpha)\n",
    "    ax.bar(labels, num_medium, width, yerr=num_medium_std, bottom=num_easy,  label='Medium [p > {}]'.format(medium_thresh), color=cmap(0.65), alpha=alpha)\n",
    "    ax.bar(labels, num_hard, width, yerr=num_hard_std, bottom=num_easy + num_medium, label='Hard [p > {}]'.format(hard_thresh), color=cmap(0.35), alpha=alpha)\n",
    "    ax.bar(labels, num_impossible, width, yerr=num_impossible_std, bottom=num_easy + num_medium + num_hard, label='Impossible [p > {}]'.format(impossible_thresh), color=cmap(0.125), alpha=alpha)\n",
    "\n",
    "    ax.set_xticks(ax.get_xticks()[2:])\n",
    "    ax.set_ylabel('Acquisition by Difficulty')\n",
    "    ax.set_xlabel('Number of Training Examples')\n",
    "    ax.set_ylim(0, 260)\n",
    "    ax.set_title(strategy_df['Name'].tolist()[0]+' - Acquisitions by Difficulty')\n",
    "    ax.legend(loc='lower right')\n",
    "    \n",
    "    if return_axs:\n",
    "        \n",
    "        return ax\n",
    "    \n",
    "    else:\n",
    "        \n",
    "        plt.plot(stacked=True)\n",
    "        name = strategy_df['Name'].tolist()[0]\n",
    "        plt.savefig('acquisitions_by_difficulty_{}_seed_{}'.format(name,seed))\n",
    "        plt.show()\n",
    "        \n",
    "        return None\n",
    "    \n",
    "\n",
    "def generate_stacked_barchart_data(base_dfs, strategy_dfs, return_axs=False, impossible_only=False, size=20, font_size=22, dpi=40):\n",
    "    \n",
    "    plt.rcParams.update({'font.size': font_size})\n",
    "    # Call this fn for a single strategy!\n",
    "    \n",
    "    total_counts = {'SNLI':[], 'ANLI':[], 'WANLI':[]}\n",
    "    \n",
    "    for strategy_df, base_df in zip(strategy_dfs, base_dfs):\n",
    "        \n",
    "        number_of_examples = 500\n",
    "\n",
    "        labels = []\n",
    "        results = {'SNLI':[], 'ANLI':[], 'WANLI':[]}\n",
    "\n",
    "        for active_round in range(1, strategy_df['round'].max()+1):\n",
    "\n",
    "            acquisitions = strategy_df[strategy_df['round']==active_round]\n",
    "            number_of_examples += len(acquisitions)\n",
    "            \n",
    "            if impossible_only:\n",
    "                \n",
    "                impossible_acquisitions = base_df[base_df.mean_conf < HARD_THRESH]\n",
    "                acquisitions = acquisitions[acquisitions.ID.isin(impossible_acquisitions.id)]\n",
    "            \n",
    "            labels.append(number_of_examples)\n",
    "            \n",
    "            data_counts = acquisitions.Dataset.value_counts()\n",
    "            \n",
    "            for dataset, value in zip(data_counts.index, data_counts):\n",
    "                results[dataset].append(value)\n",
    "            \n",
    "            for dataset in list(results.keys()):\n",
    "                \n",
    "                if dataset not in data_counts.index:\n",
    "                    \n",
    "                    results[dataset].append(0)\n",
    "        \n",
    "        for key in list(total_counts.keys()):\n",
    "            total_counts[key].append(results[key])\n",
    "        \n",
    "    mean_and_std = {'SNLI':[], 'ANLI':[], 'WANLI':[]}\n",
    "    \n",
    "    for key in list(results.keys()):\n",
    "        mean_and_std[key].append(np.mean(total_counts[key], axis=0))\n",
    "        mean_and_std[key].append(np.std(total_counts[key], axis=0))\n",
    "        \n",
    "    def get_mean_and_std_e(array):\n",
    "        \n",
    "        return np.mean(array, axis=0), np.std(array, axis=0) \n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "    \n",
    "    if impossible_only:\n",
    "        ax.set_ylim(0, 50)\n",
    "    else:\n",
    "        ax.set_ylim(0, 260)\n",
    "        \n",
    "    width = 210\n",
    "    alpha = ALPHA\n",
    "    labels = np.array(labels)\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(_CMAP)\n",
    "    \n",
    "    ax.bar(labels, mean_and_std['SNLI'][0], width, yerr=mean_and_std['SNLI'][1], label='SNLI', color=cmap(0.85), alpha=alpha)\n",
    "    ax.bar(labels, mean_and_std['ANLI'][0], width, yerr=mean_and_std['ANLI'][1], bottom=mean_and_std['SNLI'][0],  label='ANLI', color=cmap(0.65), alpha=alpha)\n",
    "    ax.bar(labels, mean_and_std['WANLI'][0], width, yerr=mean_and_std['WANLI'][1], bottom=mean_and_std['SNLI'][0] + mean_and_std['ANLI'][0], label='WANLI', color=cmap(0.35), alpha=alpha)\n",
    "\n",
    "    ax.set_xticks(ax.get_xticks()[2:])\n",
    "    ax.set_ylabel('Acquisition by Dataset')\n",
    "    ax.set_xlabel('Number of Training Examples')\n",
    "    \n",
    "    if not impossible_only:     \n",
    "        ax.set_title(strategy_df['Name'].tolist()[0]+' - Acquisitions by Dataset')\n",
    "        \n",
    "    else:\n",
    "        ax.set_title(strategy_df['Name'].tolist()[0]+' - Imp. Acquisitions by Dataset')\n",
    "    \n",
    "    ax.legend(loc='lower right')\n",
    "    \n",
    "    if return_axs:\n",
    "        \n",
    "        return ax\n",
    "    \n",
    "    else:\n",
    "        \n",
    "        plt.plot(stacked=True)\n",
    "        name = strategy_df['Name'].tolist()[0]\n",
    "        plt.savefig('acquisitions_by_data_{}_seed_{}'.format(name,seed))\n",
    "        plt.show()\n",
    "        \n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "stock-forestry",
   "metadata": {},
   "outputs": [],
   "source": [
    "strategy_ids = ['random', 'max-entropy', 'mc-max-entropy', 'dal', 'bald']\n",
    "\n",
    "strategies_list = []\n",
    "for strategy_id in strategy_ids:\n",
    "    \n",
    "    strategy_list = []\n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        strategy_path = STRATEGY_PATH.replace('SEED', str(seed)).replace('STRATEGY', str(strategy_id))\n",
    "        strategy_df = load_strategy_acquisitions(strategy=strategy_id, path=strategy_path) \n",
    "        \n",
    "        strategy_list.append(strategy_df)\n",
    "    \n",
    "    strategies_list.append(strategy_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "qualified-stations",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dfs = [load_confidences(BASE_PATH.replace('SEED', str(seed)), predictions=False) for seed in SEEDS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "dental-airline",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>mean_conf</th>\n",
       "      <th>variability</th>\n",
       "      <th>correctness</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2677109430.jpg#1r1n</td>\n",
       "      <td>0.856797</td>\n",
       "      <td>0.129124</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2677109430.jpg#1r1e</td>\n",
       "      <td>0.661288</td>\n",
       "      <td>0.104505</td>\n",
       "      <td>0.857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2677109430.jpg#1r1c</td>\n",
       "      <td>0.970684</td>\n",
       "      <td>0.025171</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6160193920.jpg#4r1n</td>\n",
       "      <td>0.880483</td>\n",
       "      <td>0.092355</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6160193920.jpg#4r1e</td>\n",
       "      <td>0.409567</td>\n",
       "      <td>0.037043</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19392</th>\n",
       "      <td>19768e</td>\n",
       "      <td>0.693017</td>\n",
       "      <td>0.042896</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19393</th>\n",
       "      <td>117680c</td>\n",
       "      <td>0.771595</td>\n",
       "      <td>0.148959</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19394</th>\n",
       "      <td>11876e</td>\n",
       "      <td>0.141856</td>\n",
       "      <td>0.027361</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19395</th>\n",
       "      <td>72032c</td>\n",
       "      <td>0.909874</td>\n",
       "      <td>0.039028</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19396</th>\n",
       "      <td>42321n</td>\n",
       "      <td>0.148335</td>\n",
       "      <td>0.093685</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19397 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        id  mean_conf  variability  correctness\n",
       "0      2677109430.jpg#1r1n   0.856797     0.129124     1.000000\n",
       "1      2677109430.jpg#1r1e   0.661288     0.104505     0.857143\n",
       "2      2677109430.jpg#1r1c   0.970684     0.025171     1.000000\n",
       "3      6160193920.jpg#4r1n   0.880483     0.092355     1.000000\n",
       "4      6160193920.jpg#4r1e   0.409567     0.037043     0.000000\n",
       "...                    ...        ...          ...          ...\n",
       "19392               19768e   0.693017     0.042896     1.000000\n",
       "19393              117680c   0.771595     0.148959     1.000000\n",
       "19394               11876e   0.141856     0.027361     0.000000\n",
       "19395               72032c   0.909874     0.039028     1.000000\n",
       "19396               42321n   0.148335     0.093685     0.000000\n",
       "\n",
       "[19397 rows x 4 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_dfs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "thorough-wound",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 33220 characters]\n",
      "text/plain": [
       "<Figure size 480x480 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 18800 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 17476 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 24868 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 19764 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 18000 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 24980 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 19916 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 18304 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 24160 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 18592 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 16968 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 23540 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 18920 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 17200 characters]\n",
      "text/plain": [
       "<Figure size 400x400 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "axs = []\n",
    "for strategy_df in strategies_list:\n",
    "     ax = generate_stacked_barchart(base_dfs=base_dfs, strategy_dfs=strategy_df, return_axs=False, size=8, font_size=18, dpi=50)\n",
    "     ax = generate_stacked_barchart_data(base_dfs=base_dfs, strategy_dfs=strategy_df, impossible_only=False, return_axs=False, size=8, font_size=18, dpi=50)\n",
    "     ax = generate_stacked_barchart_data(base_dfs=base_dfs, strategy_dfs=strategy_df, impossible_only=True, return_axs=False, size=8, font_size=18, dpi=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "known-lighter",
   "metadata": {},
   "source": [
    "# Examples of various difficulties:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ranking-guatemala",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Examples from classes EASY, MEDIUM, HARD, and IMPOSSIBLE for WANLI \n",
      "\n",
      "\n",
      "Confidence Thresholds:\n",
      "Easy: 0.75\n",
      "Medium: 0.5\n",
      "Hard: 0.25\n",
      "Impossible: 0.0\n",
      "\n",
      "Easy examples:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_c3e76992_075d_11ed_ac98_acbc32902497 th {\n",
       "          text-align: left;\n",
       "    }#T_c3e76992_075d_11ed_ac98_acbc32902497row0_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row0_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row0_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row0_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row1_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row1_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row1_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row1_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row2_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row2_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row2_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row2_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row3_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row3_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row3_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row3_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row4_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row4_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row4_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row4_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row5_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row5_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row5_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row5_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row6_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row6_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row6_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row6_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row7_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row7_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row7_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row7_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row8_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row8_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row8_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row8_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row9_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row9_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row9_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row9_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row10_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row10_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row10_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row10_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row11_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row11_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row11_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row11_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row12_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row12_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row12_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row12_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row13_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row13_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row13_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row13_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row14_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row14_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row14_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row14_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row15_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row15_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row15_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row15_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row16_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row16_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row16_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row16_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row17_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row17_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row17_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row17_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row18_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row18_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row18_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row18_col3,#T_c3e76992_075d_11ed_ac98_acbc32902497row19_col0,#T_c3e76992_075d_11ed_ac98_acbc32902497row19_col1,#T_c3e76992_075d_11ed_ac98_acbc32902497row19_col2,#T_c3e76992_075d_11ed_ac98_acbc32902497row19_col3{\n",
       "            text-align:  left;\n",
       "        }</style><table id=\"T_c3e76992_075d_11ed_ac98_acbc32902497\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Premise</th>        <th class=\"col_heading level0 col1\" >Hypothesis</th>        <th class=\"col_heading level0 col2\" >Label</th>        <th class=\"col_heading level0 col3\" >Model Prediction</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row0\" class=\"row_heading level0 row0\" >3258</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row0_col0\" class=\"data row0 col0\" >A large marble fountain,\n",
       "designed by architect\n",
       "Paul Manship, features\n",
       "the city's coat of arms,\n",
       "the words \"San Francisco\"\n",
       "and the motto \"Oro en\n",
       "Paz\" (Gold in Peace).</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row0_col1\" class=\"data row0 col1\" >The city's motto is \"Gold\n",
       "in Peace\" but the\n",
       "residents are too lazy to\n",
       "clean up the fountain.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row0_col2\" class=\"data row0 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row0_col3\" class=\"data row0 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row1\" class=\"row_heading level0 row1\" >1715</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row1_col0\" class=\"data row1 col0\" >A long, long time ago, in\n",
       "a galaxy far, far away, a\n",
       "galactic senate of\n",
       "intergalactic lawgivers\n",
       "was faced with a crisis.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row1_col1\" class=\"data row1 col1\" >The Senate of the\n",
       "Galactic Republic was\n",
       "faced with a crisis.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row1_col2\" class=\"data row1 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row1_col3\" class=\"data row1 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row2\" class=\"row_heading level0 row2\" >3938</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row2_col0\" class=\"data row2 col0\" >Under the proposal, the\n",
       "Company would be required\n",
       "to adopt a compensation\n",
       "plan that would provide\n",
       "for annual cash bonuses\n",
       "for all of its employees.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row2_col1\" class=\"data row2 col1\" >The plan would provide\n",
       "for cash bonuses for all\n",
       "employees.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row2_col2\" class=\"data row2 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row2_col3\" class=\"data row2 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row3\" class=\"row_heading level0 row3\" >1792</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row3_col0\" class=\"data row3 col0\" >I'm not saying that the\n",
       "job is perfect, but it's\n",
       "not the end of the world\n",
       "either.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row3_col1\" class=\"data row3 col1\" >The job is not the end of\n",
       "the world.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row3_col2\" class=\"data row3 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row3_col3\" class=\"data row3 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row4\" class=\"row_heading level0 row4\" >3267</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row4_col0\" class=\"data row4 col0\" >The stranger's face was\n",
       "expressionless.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row4_col1\" class=\"data row4 col1\" >The stranger was bored.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row4_col2\" class=\"data row4 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row4_col3\" class=\"data row4 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row5\" class=\"row_heading level0 row5\" >4290</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row5_col0\" class=\"data row5 col0\" >When this shift occurs,\n",
       "it will be important to\n",
       "focus on the strengths of\n",
       "the organization and its\n",
       "people.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row5_col1\" class=\"data row5 col1\" >Strengths are important\n",
       "in an organization.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row5_col2\" class=\"data row5 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row5_col3\" class=\"data row5 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row6\" class=\"row_heading level0 row6\" >2821</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row6_col0\" class=\"data row6 col0\" >That's a very interesting\n",
       "point of view.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row6_col1\" class=\"data row6 col1\" >I agree with you.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row6_col2\" class=\"data row6 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row6_col3\" class=\"data row6 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row7\" class=\"row_heading level0 row7\" >2345</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row7_col0\" class=\"data row7 col0\" >A common example of this\n",
       "is the distinction\n",
       "between an \"option\" and\n",
       "an \"option contract\".</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row7_col1\" class=\"data row7 col1\" >The distinction is not\n",
       "valid.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row7_col2\" class=\"data row7 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row7_col3\" class=\"data row7 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row8\" class=\"row_heading level0 row8\" >2598</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row8_col0\" class=\"data row8 col0\" >This example is very\n",
       "close to the second\n",
       "example.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row8_col1\" class=\"data row8 col1\" >The examples are very\n",
       "close to each other.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row8_col2\" class=\"data row8 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row8_col3\" class=\"data row8 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row9\" class=\"row_heading level0 row9\" >3061</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row9_col0\" class=\"data row9 col0\" >\"The Times\" says the\n",
       "Prime Minister is being\n",
       "driven to distraction by\n",
       "the war.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row9_col1\" class=\"data row9 col1\" >The Prime Minister is\n",
       "very angry with the war.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row9_col2\" class=\"data row9 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row9_col3\" class=\"data row9 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row10\" class=\"row_heading level0 row10\" >505</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row10_col0\" class=\"data row10 col0\" >In a number of other\n",
       "countries, the notion of\n",
       "a \"special relationship\"\n",
       "between the United States\n",
       "and Britain has been a\n",
       "myth.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row10_col1\" class=\"data row10 col1\" >Other countries have a\n",
       "special relationship with\n",
       "Britain.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row10_col2\" class=\"data row10 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row10_col3\" class=\"data row10 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row11\" class=\"row_heading level0 row11\" >94</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row11_col0\" class=\"data row11 col0\" >In an effort to show that\n",
       "the threat of nuclear war\n",
       "is still real, the U.S.\n",
       "government has\n",
       "established a commission\n",
       "to determine the\n",
       "feasibility of a\n",
       "preemptive nuclear\n",
       "strike.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row11_col1\" class=\"data row11 col1\" >The commission will\n",
       "determine the feasibility\n",
       "of a preemptive nuclear\n",
       "strike.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row11_col2\" class=\"data row11 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row11_col3\" class=\"data row11 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row12\" class=\"row_heading level0 row12\" >4843</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row12_col0\" class=\"data row12 col0\" >In the mid-nineteenth\n",
       "century, the \"Free\n",
       "Church\" movement, a\n",
       "product of the\n",
       "Evangelical Revival, was\n",
       "a movement that wanted to\n",
       "replace the Church of\n",
       "England with a more pure\n",
       "form of Christianity.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row12_col1\" class=\"data row12 col1\" >The Free Church movement\n",
       "was a product of the\n",
       "Evangelical Revival.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row12_col2\" class=\"data row12 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row12_col3\" class=\"data row12 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row13\" class=\"row_heading level0 row13\" >4562</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row13_col0\" class=\"data row13 col0\" >He was known for his\n",
       "ability to get along with\n",
       "anyone, and his unique\n",
       "style of humor was not\n",
       "only funny, but it was\n",
       "also incredibly funny.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row13_col1\" class=\"data row13 col1\" >He was very funny.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row13_col2\" class=\"data row13 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row13_col3\" class=\"data row13 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row14\" class=\"row_heading level0 row14\" >3766</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row14_col0\" class=\"data row14 col0\" >The success of the\n",
       "British was attributed to\n",
       "the fact that they were\n",
       "the only people in the\n",
       "country who were not\n",
       "armed.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row14_col1\" class=\"data row14 col1\" >The British were armed.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row14_col2\" class=\"data row14 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row14_col3\" class=\"data row14 col3\" >contradiction</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row15\" class=\"row_heading level0 row15\" >3186</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row15_col0\" class=\"data row15 col0\" >This question is designed\n",
       "to ensure that the new\n",
       "program is likely to be\n",
       "successful.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row15_col1\" class=\"data row15 col1\" >The new program is\n",
       "designed to be\n",
       "successful.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row15_col2\" class=\"data row15 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row15_col3\" class=\"data row15 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row16\" class=\"row_heading level0 row16\" >4009</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row16_col0\" class=\"data row16 col0\" >I have no idea how I\n",
       "shall pass the\n",
       "examination.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row16_col1\" class=\"data row16 col1\" >Passing the examination\n",
       "is very important.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row16_col2\" class=\"data row16 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row16_col3\" class=\"data row16 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row17\" class=\"row_heading level0 row17\" >2761</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row17_col0\" class=\"data row17 col0\" >The book tells the story\n",
       "of a boy who falls in\n",
       "love with a girl who is\n",
       "engaged to another boy.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row17_col1\" class=\"data row17 col1\" >The book tells the story\n",
       "of a boy who falls in\n",
       "love with a girl who is\n",
       "not engaged to another\n",
       "boy.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row17_col2\" class=\"data row17 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row17_col3\" class=\"data row17 col3\" >contradiction</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row18\" class=\"row_heading level0 row18\" >3174</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row18_col0\" class=\"data row18 col0\" >A new venture in avant-\n",
       "garde music.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row18_col1\" class=\"data row18 col1\" >The music is avant-garde.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row18_col2\" class=\"data row18 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row18_col3\" class=\"data row18 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3e76992_075d_11ed_ac98_acbc32902497level0_row19\" class=\"row_heading level0 row19\" >2046</th>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row19_col0\" class=\"data row19 col0\" >In addition to the\n",
       "federal program, the\n",
       "state also provides a\n",
       "separate set of funds to\n",
       "the local program.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row19_col1\" class=\"data row19 col1\" >The local program is a\n",
       "separate program.</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row19_col2\" class=\"data row19 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3e76992_075d_11ed_ac98_acbc32902497row19_col3\" class=\"data row19 col3\" >entailment</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f97fb9c9b38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Medium examples:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_c3eaa18e_075d_11ed_ac98_acbc32902497 th {\n",
       "          text-align: left;\n",
       "    }#T_c3eaa18e_075d_11ed_ac98_acbc32902497row0_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row0_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row0_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row0_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row1_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row1_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row1_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row1_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row2_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row2_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row2_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row2_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row3_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row3_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row3_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row3_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row4_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row4_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row4_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row4_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row5_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row5_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row5_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row5_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row6_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row6_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row6_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row6_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row7_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row7_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row7_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row7_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row8_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row8_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row8_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row8_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row9_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row9_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row9_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row9_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row10_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row10_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row10_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row10_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row11_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row11_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row11_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row11_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row12_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row12_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row12_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row12_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row13_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row13_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row13_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row13_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row14_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row14_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row14_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row14_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row15_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row15_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row15_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row15_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row16_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row16_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row16_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row16_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row17_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row17_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row17_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row17_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row18_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row18_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row18_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row18_col3,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row19_col0,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row19_col1,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row19_col2,#T_c3eaa18e_075d_11ed_ac98_acbc32902497row19_col3{\n",
       "            text-align:  left;\n",
       "        }</style><table id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Premise</th>        <th class=\"col_heading level0 col1\" >Hypothesis</th>        <th class=\"col_heading level0 col2\" >Label</th>        <th class=\"col_heading level0 col3\" >Model Prediction</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row0\" class=\"row_heading level0 row0\" >57</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row0_col0\" class=\"data row0 col0\" >I thought you might be\n",
       "able to tell me something\n",
       "about the letters Mrs.\n",
       "Inglethorp wrote last\n",
       "night.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row0_col1\" class=\"data row0 col1\" >I thought Annie might be\n",
       "able to tell me something\n",
       "about the letters written\n",
       "by Mrs. Inglethorp last\n",
       "night.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row0_col2\" class=\"data row0 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row0_col3\" class=\"data row0 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row1\" class=\"row_heading level0 row1\" >1671</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row1_col0\" class=\"data row1 col0\" >A survey of the attitudes\n",
       "of students in high\n",
       "school found that the\n",
       "most common response to\n",
       "the question, \"What do\n",
       "you want to be when you\n",
       "grow up?\" was \"A doctor.\"</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row1_col1\" class=\"data row1 col1\" >A survey of the attitudes\n",
       "of students in high\n",
       "school found that the\n",
       "most common response to\n",
       "the question, \"What do\n",
       "you want to be when you\n",
       "grow up?\" was \"A\n",
       "teacher.\"</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row1_col2\" class=\"data row1 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row1_col3\" class=\"data row1 col3\" >contradiction</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row2\" class=\"row_heading level0 row2\" >3037</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row2_col0\" class=\"data row2 col0\" >If the two lines of the\n",
       "image are separated by a\n",
       "space, the lines will\n",
       "appear to be parallel.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row2_col1\" class=\"data row2 col1\" >The lines of the image\n",
       "are not parallel.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row2_col2\" class=\"data row2 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row2_col3\" class=\"data row2 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row3\" class=\"row_heading level0 row3\" >1396</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row3_col0\" class=\"data row3 col0\" >It's a matter of personal\n",
       "preference.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row3_col1\" class=\"data row3 col1\" >I think it's a matter of\n",
       "personal preference.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row3_col2\" class=\"data row3 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row3_col3\" class=\"data row3 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row4\" class=\"row_heading level0 row4\" >2820</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row4_col0\" class=\"data row4 col0\" >There are many people who\n",
       "feel that the time has\n",
       "come to start working on\n",
       "this.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row4_col1\" class=\"data row4 col1\" >It was not a fact that\n",
       "people should start\n",
       "working on this.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row4_col2\" class=\"data row4 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row4_col3\" class=\"data row4 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row5\" class=\"row_heading level0 row5\" >3287</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row5_col0\" class=\"data row5 col0\" >If he is found guilty, he\n",
       "will be executed.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row5_col1\" class=\"data row5 col1\" >He will be executed.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row5_col2\" class=\"data row5 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row5_col3\" class=\"data row5 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row6\" class=\"row_heading level0 row6\" >4596</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row6_col0\" class=\"data row6 col0\" >These findings suggest\n",
       "that the government has\n",
       "been successful in\n",
       "reducing its dependence\n",
       "on foreign oil.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row6_col1\" class=\"data row6 col1\" >The government has been\n",
       "successful in reducing\n",
       "its dependence on foreign\n",
       "oil.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row6_col2\" class=\"data row6 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row6_col3\" class=\"data row6 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row7\" class=\"row_heading level0 row7\" >1325</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row7_col0\" class=\"data row7 col0\" >I have always thought\n",
       "that the president should\n",
       "be required to certify\n",
       "that the actions he is\n",
       "taking are consistent\n",
       "with the Constitution.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row7_col1\" class=\"data row7 col1\" >The president should be\n",
       "required to do so.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row7_col2\" class=\"data row7 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row7_col3\" class=\"data row7 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row8\" class=\"row_heading level0 row8\" >4303</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row8_col0\" class=\"data row8 col0\" >I'll be there.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row8_col1\" class=\"data row8 col1\" >I'll be there soon.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row8_col2\" class=\"data row8 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row8_col3\" class=\"data row8 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row9\" class=\"row_heading level0 row9\" >2902</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row9_col0\" class=\"data row9 col0\" >The English version of\n",
       "the story is quite\n",
       "different from the French\n",
       "version.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row9_col1\" class=\"data row9 col1\" >The French version is\n",
       "quite different from the\n",
       "English version.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row9_col2\" class=\"data row9 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row9_col3\" class=\"data row9 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row10\" class=\"row_heading level0 row10\" >3231</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row10_col0\" class=\"data row10 col0\" >The father is a hard\n",
       "worker.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row10_col1\" class=\"data row10 col1\" >The mother is a hard\n",
       "worker.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row10_col2\" class=\"data row10 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row10_col3\" class=\"data row10 col3\" >contradiction</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row11\" class=\"row_heading level0 row11\" >2037</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row11_col0\" class=\"data row11 col0\" >The doctor's face fell.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row11_col1\" class=\"data row11 col1\" >The doctor's face turned\n",
       "red with anger.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row11_col2\" class=\"data row11 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row11_col3\" class=\"data row11 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row12\" class=\"row_heading level0 row12\" >409</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row12_col0\" class=\"data row12 col0\" >The new national policy\n",
       "is expected to help ease\n",
       "the problem.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row12_col1\" class=\"data row12 col1\" >The new national policy\n",
       "is expected to help solve\n",
       "the problem.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row12_col2\" class=\"data row12 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row12_col3\" class=\"data row12 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row13\" class=\"row_heading level0 row13\" >1280</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row13_col0\" class=\"data row13 col0\" >The teacher told the\n",
       "class that the missing\n",
       "tooth was a fake.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row13_col1\" class=\"data row13 col1\" >The teacher was a\n",
       "vampire.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row13_col2\" class=\"data row13 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row13_col3\" class=\"data row13 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row14\" class=\"row_heading level0 row14\" >2453</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row14_col0\" class=\"data row14 col0\" >A great leader must be\n",
       "able to inspire people to\n",
       "follow him, and he must\n",
       "be able to get them to do\n",
       "what he wants them to do.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row14_col1\" class=\"data row14 col1\" >Great leaders must be\n",
       "able to inspire people to\n",
       "follow them, and they\n",
       "must be able to get them\n",
       "to do what they want them\n",
       "to do.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row14_col2\" class=\"data row14 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row14_col3\" class=\"data row14 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row15\" class=\"row_heading level0 row15\" >574</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row15_col0\" class=\"data row15 col0\" >'The government is not in\n",
       "a position to provide any\n",
       "guarantees,' he said,\n",
       "'but we are committed to\n",
       "working with the\n",
       "international community\n",
       "to secure the release of\n",
       "the hostages.'</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row15_col1\" class=\"data row15 col1\" >The government did not\n",
       "provide any guarantees.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row15_col2\" class=\"data row15 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row15_col3\" class=\"data row15 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row16\" class=\"row_heading level0 row16\" >3302</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row16_col0\" class=\"data row16 col0\" >And so we must come to\n",
       "terms with the idea that\n",
       "the most precious of our\n",
       "treasures is not to be\n",
       "hoarded but shared.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row16_col1\" class=\"data row16 col1\" >We must share our wealth.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row16_col2\" class=\"data row16 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row16_col3\" class=\"data row16 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row17\" class=\"row_heading level0 row17\" >580</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row17_col0\" class=\"data row17 col0\" >We can't all be James\n",
       "Bond, but we can all be\n",
       "James Bond fans.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row17_col1\" class=\"data row17 col1\" >James Bond is a popular\n",
       "character.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row17_col2\" class=\"data row17 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row17_col3\" class=\"data row17 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row18\" class=\"row_heading level0 row18\" >2132</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row18_col0\" class=\"data row18 col0\" >The women in the town had\n",
       "made up their minds that\n",
       "he was a traitor, and\n",
       "they would not be\n",
       "reconciled to him until\n",
       "he was hanged.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row18_col1\" class=\"data row18 col1\" >The women in the town\n",
       "were not reconciled to\n",
       "him.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row18_col2\" class=\"data row18 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row18_col3\" class=\"data row18 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497level0_row19\" class=\"row_heading level0 row19\" >3</th>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row19_col0\" class=\"data row19 col0\" >If you're a good swimmer,\n",
       "it's a good idea to try\n",
       "the shallow end of the\n",
       "pool.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row19_col1\" class=\"data row19 col1\" >The shallow end of the\n",
       "pool is good for\n",
       "swimming.</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row19_col2\" class=\"data row19 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3eaa18e_075d_11ed_ac98_acbc32902497row19_col3\" class=\"data row19 col3\" >entailment</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f97fd7da160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hard examples:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_c3ee57f2_075d_11ed_ac98_acbc32902497 th {\n",
       "          text-align: left;\n",
       "    }#T_c3ee57f2_075d_11ed_ac98_acbc32902497row0_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row0_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row0_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row0_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row1_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row1_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row1_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row1_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row2_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row2_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row2_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row2_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row3_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row3_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row3_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row3_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row4_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row4_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row4_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row4_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row5_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row5_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row5_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row5_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row6_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row6_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row6_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row6_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row7_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row7_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row7_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row7_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row8_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row8_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row8_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row8_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row9_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row9_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row9_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row9_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row10_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row10_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row10_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row10_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row11_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row11_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row11_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row11_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row12_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row12_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row12_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row12_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row13_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row13_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row13_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row13_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row14_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row14_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row14_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row14_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row15_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row15_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row15_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row15_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row16_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row16_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row16_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row16_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row17_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row17_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row17_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row17_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row18_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row18_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row18_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row18_col3,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row19_col0,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row19_col1,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row19_col2,#T_c3ee57f2_075d_11ed_ac98_acbc32902497row19_col3{\n",
       "            text-align:  left;\n",
       "        }</style><table id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Premise</th>        <th class=\"col_heading level0 col1\" >Hypothesis</th>        <th class=\"col_heading level0 col2\" >Label</th>        <th class=\"col_heading level0 col3\" >Model Prediction</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row0\" class=\"row_heading level0 row0\" >684</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row0_col0\" class=\"data row0 col0\" >This would be the first\n",
       "federal agency to be led\n",
       "by a person of color.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row0_col1\" class=\"data row0 col1\" >Not all federal agencies\n",
       "are led by people of\n",
       "color.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row0_col2\" class=\"data row0 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row0_col3\" class=\"data row0 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row1\" class=\"row_heading level0 row1\" >2869</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row1_col0\" class=\"data row1 col0\" >The loss of our American\n",
       "jobs to other countries\n",
       "is a direct result of the\n",
       "trade policies of the\n",
       "Bush Administration.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row1_col1\" class=\"data row1 col1\" >The loss of American jobs\n",
       "to other countries is a\n",
       "direct result of the\n",
       "trade policies of the\n",
       "Clinton Administration.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row1_col2\" class=\"data row1 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row1_col3\" class=\"data row1 col3\" >contradiction</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row2\" class=\"row_heading level0 row2\" >2780</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row2_col0\" class=\"data row2 col0\" >A major purpose of the\n",
       "bill is to increase the\n",
       "transparency of\n",
       "government.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row2_col1\" class=\"data row2 col1\" >Transparency is\n",
       "important.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row2_col2\" class=\"data row2 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row2_col3\" class=\"data row2 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row3\" class=\"row_heading level0 row3\" >271</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row3_col0\" class=\"data row3 col0\" >He thought of his friend\n",
       "in the army, in a way he\n",
       "never had before, and he\n",
       "realized that he had been\n",
       "a little bit in love with\n",
       "him.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row3_col1\" class=\"data row3 col1\" >He was in love with his\n",
       "friend in the army.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row3_col2\" class=\"data row3 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row3_col3\" class=\"data row3 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row4\" class=\"row_heading level0 row4\" >4304</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row4_col0\" class=\"data row4 col0\" >An alternative, and\n",
       "perhaps more useful,\n",
       "approach is to focus on\n",
       "the interaction between\n",
       "the content of the\n",
       "argument and the way in\n",
       "which it is presented.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row4_col1\" class=\"data row4 col1\" >The argument is presented\n",
       "in a particular way.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row4_col2\" class=\"data row4 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row4_col3\" class=\"data row4 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row5\" class=\"row_heading level0 row5\" >4069</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row5_col0\" class=\"data row5 col0\" >In fact, I think that the\n",
       "music of the \"nuevo\n",
       "flamenco\" is as good as\n",
       "the music of the\n",
       "\"flamenco de gaita\" of\n",
       "the 19th century.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row5_col1\" class=\"data row5 col1\" >Flamenco is alive and\n",
       "well.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row5_col2\" class=\"data row5 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row5_col3\" class=\"data row5 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row6\" class=\"row_heading level0 row6\" >469</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row6_col0\" class=\"data row6 col0\" >In a way, the law can be\n",
       "seen as a public good, a\n",
       "good that is available to\n",
       "all, but its provision is\n",
       "contingent on the actions\n",
       "of a relatively small\n",
       "number of people.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row6_col1\" class=\"data row6 col1\" >The law is a public good,\n",
       "but it is provided by a\n",
       "small number of people.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row6_col2\" class=\"data row6 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row6_col3\" class=\"data row6 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row7\" class=\"row_heading level0 row7\" >4235</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row7_col0\" class=\"data row7 col0\" >There is no question that\n",
       "the US will be a\n",
       "superpower in the next\n",
       "century.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row7_col1\" class=\"data row7 col1\" >There is no question that\n",
       "the US will not be a\n",
       "superpower in the next\n",
       "century.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row7_col2\" class=\"data row7 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row7_col3\" class=\"data row7 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row8\" class=\"row_heading level0 row8\" >4399</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row8_col0\" class=\"data row8 col0\" >The best and most\n",
       "important step in\n",
       "preventing and\n",
       "controlling crime is to\n",
       "establish and maintain a\n",
       "well-organized, well-\n",
       "trained, and well-\n",
       "equipped police force.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row8_col1\" class=\"data row8 col1\" >The best and most\n",
       "important step in\n",
       "preventing and\n",
       "controlling crime is to\n",
       "establish and maintain a\n",
       "well-organized, well-\n",
       "trained, and well-\n",
       "equipped fire department.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row8_col2\" class=\"data row8 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row8_col3\" class=\"data row8 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row9\" class=\"row_heading level0 row9\" >1725</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row9_col0\" class=\"data row9 col0\" >Many people visit the\n",
       "city for its famous Roman\n",
       "ruins.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row9_col1\" class=\"data row9 col1\" >There are many ruins in\n",
       "the city.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row9_col2\" class=\"data row9 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row9_col3\" class=\"data row9 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row10\" class=\"row_heading level0 row10\" >2246</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row10_col0\" class=\"data row10 col0\" >I can't believe that\n",
       "she's really going to do\n",
       "it.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row10_col1\" class=\"data row10 col1\" >She is going to do it.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row10_col2\" class=\"data row10 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row10_col3\" class=\"data row10 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row11\" class=\"row_heading level0 row11\" >1287</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row11_col0\" class=\"data row11 col0\" >Who is this?</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row11_col1\" class=\"data row11 col1\" >Who is this person?</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row11_col2\" class=\"data row11 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row11_col3\" class=\"data row11 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row12\" class=\"row_heading level0 row12\" >1677</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row12_col0\" class=\"data row12 col0\" >He must have made a\n",
       "mistake.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row12_col1\" class=\"data row12 col1\" >He must have done\n",
       "something right.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row12_col2\" class=\"data row12 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row12_col3\" class=\"data row12 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row13\" class=\"row_heading level0 row13\" >390</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row13_col0\" class=\"data row13 col0\" >When the river freezes,\n",
       "the riverwalk becomes a\n",
       "skating rink.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row13_col1\" class=\"data row13 col1\" >The river is frozen when\n",
       "the riverwalk is not.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row13_col2\" class=\"data row13 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row13_col3\" class=\"data row13 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row14\" class=\"row_heading level0 row14\" >916</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row14_col0\" class=\"data row14 col0\" >A company is often a\n",
       "collection of people, not\n",
       "just a set of financial\n",
       "statements.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row14_col1\" class=\"data row14 col1\" >A company is a set of\n",
       "financial statements.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row14_col2\" class=\"data row14 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row14_col3\" class=\"data row14 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row15\" class=\"row_heading level0 row15\" >3272</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row15_col0\" class=\"data row15 col0\" >There is no good reason\n",
       "to believe that the time\n",
       "horizon is the most\n",
       "important determinant of\n",
       "the welfare effects of\n",
       "government policies.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row15_col1\" class=\"data row15 col1\" >The time horizon is not\n",
       "the most important\n",
       "determinant of the\n",
       "welfare effects of\n",
       "government policies.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row15_col2\" class=\"data row15 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row15_col3\" class=\"data row15 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row16\" class=\"row_heading level0 row16\" >2076</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row16_col0\" class=\"data row16 col0\" >On the other hand, in the\n",
       "second case, there is a\n",
       "great deal of evidence\n",
       "that the accused is\n",
       "innocent.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row16_col1\" class=\"data row16 col1\" >The evidence is\n",
       "inconclusive.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row16_col2\" class=\"data row16 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row16_col3\" class=\"data row16 col3\" >contradiction</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row17\" class=\"row_heading level0 row17\" >4018</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row17_col0\" class=\"data row17 col0\" >When a mistake is made, a\n",
       "report is made.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row17_col1\" class=\"data row17 col1\" >A mistake was made.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row17_col2\" class=\"data row17 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row17_col3\" class=\"data row17 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row18\" class=\"row_heading level0 row18\" >3180</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row18_col0\" class=\"data row18 col0\" >For a three-year-old, a\n",
       "toy with a built-in\n",
       "microphone and speaker is\n",
       "a wonderful thing.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row18_col1\" class=\"data row18 col1\" >A three-year-old's toy\n",
       "should not have a built-\n",
       "in microphone and\n",
       "speaker.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row18_col2\" class=\"data row18 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row18_col3\" class=\"data row18 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497level0_row19\" class=\"row_heading level0 row19\" >899</th>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row19_col0\" class=\"data row19 col0\" >The argument that if\n",
       "Congress can force people\n",
       "to buy health insurance,\n",
       "it can force them to buy\n",
       "broccoli, is a logical\n",
       "fallacy.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row19_col1\" class=\"data row19 col1\" >If Congress can force\n",
       "people to buy broccoli,\n",
       "it can force them to buy\n",
       "health insurance.</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row19_col2\" class=\"data row19 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3ee57f2_075d_11ed_ac98_acbc32902497row19_col3\" class=\"data row19 col3\" >neutral</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f97fd47b240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Impossible examples:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_c3f215d6_075d_11ed_ac98_acbc32902497 th {\n",
       "          text-align: left;\n",
       "    }#T_c3f215d6_075d_11ed_ac98_acbc32902497row0_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row0_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row0_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row0_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row1_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row1_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row1_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row1_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row2_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row2_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row2_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row2_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row3_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row3_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row3_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row3_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row4_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row4_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row4_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row4_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row5_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row5_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row5_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row5_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row6_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row6_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row6_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row6_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row7_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row7_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row7_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row7_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row8_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row8_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row8_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row8_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row9_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row9_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row9_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row9_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row10_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row10_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row10_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row10_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row11_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row11_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row11_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row11_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row12_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row12_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row12_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row12_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row13_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row13_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row13_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row13_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row14_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row14_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row14_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row14_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row15_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row15_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row15_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row15_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row16_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row16_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row16_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row16_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row17_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row17_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row17_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row17_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row18_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row18_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row18_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row18_col3,#T_c3f215d6_075d_11ed_ac98_acbc32902497row19_col0,#T_c3f215d6_075d_11ed_ac98_acbc32902497row19_col1,#T_c3f215d6_075d_11ed_ac98_acbc32902497row19_col2,#T_c3f215d6_075d_11ed_ac98_acbc32902497row19_col3{\n",
       "            text-align:  left;\n",
       "        }</style><table id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Premise</th>        <th class=\"col_heading level0 col1\" >Hypothesis</th>        <th class=\"col_heading level0 col2\" >Label</th>        <th class=\"col_heading level0 col3\" >Model Prediction</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row0\" class=\"row_heading level0 row0\" >389</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row0_col0\" class=\"data row0 col0\" >What's the worst that can\n",
       "happen?</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row0_col1\" class=\"data row0 col1\" >The worst that can happen\n",
       "is that nothing will\n",
       "happen.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row0_col2\" class=\"data row0 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row0_col3\" class=\"data row0 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row1\" class=\"row_heading level0 row1\" >3307</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row1_col0\" class=\"data row1 col0\" >It is a matter of record\n",
       "that during the period of\n",
       "his imprisonment,\n",
       "Jefferson wrote most of\n",
       "the Declaration of\n",
       "Independence.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row1_col1\" class=\"data row1 col1\" >The Declaration of\n",
       "Independence is a\n",
       "document that was written\n",
       "by Thomas Jefferson.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row1_col2\" class=\"data row1 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row1_col3\" class=\"data row1 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row2\" class=\"row_heading level0 row2\" >4965</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row2_col0\" class=\"data row2 col0\" >She is an excellent shot,\n",
       "but her record is better\n",
       "than that of any other\n",
       "woman in the country.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row2_col1\" class=\"data row2 col1\" >She is the best shot in\n",
       "the country.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row2_col2\" class=\"data row2 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row2_col3\" class=\"data row2 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row3\" class=\"row_heading level0 row3\" >3510</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row3_col0\" class=\"data row3 col0\" >He went on to explain\n",
       "that he had always\n",
       "believed that it was the\n",
       "duty of the people to\n",
       "govern themselves, and\n",
       "that they should not be\n",
       "governed by any man, or\n",
       "by any group of men, no\n",
       "matter how intelligent or\n",
       "powerful they might be.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row3_col1\" class=\"data row3 col1\" >The people should govern\n",
       "themselves.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row3_col2\" class=\"data row3 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row3_col3\" class=\"data row3 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row4\" class=\"row_heading level0 row4\" >3990</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row4_col0\" class=\"data row4 col0\" >The final exam will be a\n",
       "two-hour oral exam that\n",
       "will include a written\n",
       "component.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row4_col1\" class=\"data row4 col1\" >The final exam will be a\n",
       "two-hour written exam\n",
       "that will include a two-\n",
       "hour oral component.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row4_col2\" class=\"data row4 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row4_col3\" class=\"data row4 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row5\" class=\"row_heading level0 row5\" >3900</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row5_col0\" class=\"data row5 col0\" >The opportunity to make a\n",
       "statement in a manner\n",
       "that is completely\n",
       "truthful and in a way\n",
       "that will be remembered\n",
       "for a long time.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row5_col1\" class=\"data row5 col1\" >It is the only way to\n",
       "make a statement.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row5_col2\" class=\"data row5 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row5_col3\" class=\"data row5 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row6\" class=\"row_heading level0 row6\" >2310</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row6_col0\" class=\"data row6 col0\" >It is not clear whether\n",
       "the increase in\n",
       "international capital\n",
       "flows is due to the\n",
       "Federal Reserve's tight\n",
       "monetary policy or to the\n",
       "depreciation of the\n",
       "dollar.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row6_col1\" class=\"data row6 col1\" >The Federal Reserve's\n",
       "tight monetary policy is\n",
       "the only reason why the\n",
       "value of the dollar has\n",
       "declined.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row6_col2\" class=\"data row6 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row6_col3\" class=\"data row6 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row7\" class=\"row_heading level0 row7\" >571</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row7_col0\" class=\"data row7 col0\" >In a dark, damp basement\n",
       "in the eastern part of\n",
       "the city, a little-known\n",
       "museum is displaying a\n",
       "remarkable collection of\n",
       "paintings by a single\n",
       "artist, Ettore Tito.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row7_col1\" class=\"data row7 col1\" >The museum has a large\n",
       "collection of paintings\n",
       "by a single artist.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row7_col2\" class=\"data row7 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row7_col3\" class=\"data row7 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row8\" class=\"row_heading level0 row8\" >3236</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row8_col0\" class=\"data row8 col0\" >For example, in an\n",
       "economy that is growing\n",
       "at a rate of 2 percent, a\n",
       "2 percent increase in\n",
       "government spending will\n",
       "have a 1 percent effect\n",
       "on the economy.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row8_col1\" class=\"data row8 col1\" >The effect of government\n",
       "spending is a 1 percent\n",
       "increase.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row8_col2\" class=\"data row8 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row8_col3\" class=\"data row8 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row9\" class=\"row_heading level0 row9\" >3446</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row9_col0\" class=\"data row9 col0\" >He has the typical\n",
       "characteristics of a poor\n",
       "child, a fat face, a\n",
       "stammer, and a love of\n",
       "sweets.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row9_col1\" class=\"data row9 col1\" >He is a typical child.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row9_col2\" class=\"data row9 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row9_col3\" class=\"data row9 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row10\" class=\"row_heading level0 row10\" >4025</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row10_col0\" class=\"data row10 col0\" >\"You're a big-ideared\n",
       "person, you know,\" he\n",
       "said. \"I don't think\n",
       "you'd like living in a\n",
       "place like this.\"</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row10_col1\" class=\"data row10 col1\" >He thinks the person is\n",
       "too emotional and not\n",
       "logical.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row10_col2\" class=\"data row10 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row10_col3\" class=\"data row10 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row11\" class=\"row_heading level0 row11\" >3068</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row11_col0\" class=\"data row11 col0\" >The Japanese, of course,\n",
       "were determined to show\n",
       "that they were not merely\n",
       "a colony of the\n",
       "Europeans.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row11_col1\" class=\"data row11 col1\" >The Japanese were\n",
       "determined to be a colony\n",
       "of the Europeans.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row11_col2\" class=\"data row11 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row11_col3\" class=\"data row11 col3\" >contradiction</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row12\" class=\"row_heading level0 row12\" >3177</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row12_col0\" class=\"data row12 col0\" >Although the program is\n",
       "relatively inexpensive,\n",
       "the equipment is not\n",
       "cheap.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row12_col1\" class=\"data row12 col1\" >The program is not cheap.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row12_col2\" class=\"data row12 col2\" >contradiction</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row12_col3\" class=\"data row12 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row13\" class=\"row_heading level0 row13\" >142</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row13_col0\" class=\"data row13 col0\" >As he turned to leave, he\n",
       "found the girl sitting on\n",
       "the edge of the bed,\n",
       "staring at him.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row13_col1\" class=\"data row13 col1\" >The girl was sitting on\n",
       "the bed when he left.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row13_col2\" class=\"data row13 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row13_col3\" class=\"data row13 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row14\" class=\"row_heading level0 row14\" >4051</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row14_col0\" class=\"data row14 col0\" >The party of the first\n",
       "part of the sentence is\n",
       "the subject of the\n",
       "sentence.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row14_col1\" class=\"data row14 col1\" >The party of the first\n",
       "part of the sentence is\n",
       "the object of the\n",
       "sentence.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row14_col2\" class=\"data row14 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row14_col3\" class=\"data row14 col3\" >contradiction</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row15\" class=\"row_heading level0 row15\" >3216</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row15_col0\" class=\"data row15 col0\" >He was an old-fashioned\n",
       "person who liked to keep\n",
       "things neat and orderly.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row15_col1\" class=\"data row15 col1\" >He was a neat and orderly\n",
       "person.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row15_col2\" class=\"data row15 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row15_col3\" class=\"data row15 col3\" >entailment</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row16\" class=\"row_heading level0 row16\" >608</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row16_col0\" class=\"data row16 col0\" >The authors of the book\n",
       "say that their project is\n",
       "a great idea, but not a\n",
       "great book.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row16_col1\" class=\"data row16 col1\" >The book is a great idea,\n",
       "but not a great book.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row16_col2\" class=\"data row16 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row16_col3\" class=\"data row16 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row17\" class=\"row_heading level0 row17\" >966</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row17_col0\" class=\"data row17 col0\" >The ability to write well\n",
       "is not something that can\n",
       "be taught.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row17_col1\" class=\"data row17 col1\" >It is possible to teach\n",
       "writing.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row17_col2\" class=\"data row17 col2\" >neutral</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row17_col3\" class=\"data row17 col3\" >contradiction</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row18\" class=\"row_heading level0 row18\" >3032</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row18_col0\" class=\"data row18 col0\" >If the U.S. will pay $100\n",
       "million to build a\n",
       "bridge, what will the\n",
       "cost of the bridge be?</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row18_col1\" class=\"data row18 col1\" >The U.S. will pay $100\n",
       "million to build a\n",
       "bridge. The cost of the\n",
       "bridge will be $100\n",
       "million.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row18_col2\" class=\"data row18 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row18_col3\" class=\"data row18 col3\" >neutral</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497level0_row19\" class=\"row_heading level0 row19\" >503</th>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row19_col0\" class=\"data row19 col0\" >I remember when I was in\n",
       "the third grade, I wanted\n",
       "to be a ballerina.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row19_col1\" class=\"data row19 col1\" >I wanted to be a\n",
       "ballerina when I was in\n",
       "the first grade.</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row19_col2\" class=\"data row19 col2\" >entailment</td>\n",
       "                        <td id=\"T_c3f215d6_075d_11ed_ac98_acbc32902497row19_col3\" class=\"data row19 col3\" >neutral</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f97fbc857b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "EASY_THRESH = 0.75\n",
    "MEDIUM_THRESH = 0.5\n",
    "HARD_THRESH = 0.25\n",
    "IMPOSSIBLE_THRESH = 0.0\n",
    "\n",
    "COL_WIDTH = 25\n",
    "\n",
    "_, prediction_dict = load_confidences(BASE_PATH.replace('SEED',str(42)), predictions=True)\n",
    "\n",
    "def show_examples(source_df, source_map, base_df=None, num=5):\n",
    "    \n",
    "    source_map = source_map[0]\n",
    "    \n",
    "    difficulty_dict = {}\n",
    "    \n",
    "    easy_mask = source_map.mean_conf > EASY_THRESH\n",
    "    medium_mask = (source_map.mean_conf > MEDIUM_THRESH) & (source_map.mean_conf < EASY_THRESH)\n",
    "    hard_mask = (source_map.mean_conf > HARD_THRESH) & (source_map.mean_conf < MEDIUM_THRESH)\n",
    "    impossible_mask = (source_map.mean_conf > IMPOSSIBLE_THRESH) & (source_map.mean_conf < HARD_THRESH)\n",
    "    \n",
    "    difficulty_dict['Easy'] = source_map[easy_mask]\n",
    "    difficulty_dict['Medium'] = source_map[medium_mask]\n",
    "    difficulty_dict['Hard'] = source_map[hard_mask]\n",
    "    difficulty_dict['Impossible'] = source_map[impossible_mask]\n",
    "    \n",
    "    print('Examples from classes EASY, MEDIUM, HARD, and IMPOSSIBLE for {} \\n'.format(source_map.Name.tolist()[0]))\n",
    "    \n",
    "    print('\\nConfidence Thresholds:\\nEasy: {}\\nMedium: {}\\nHard: {}\\nImpossible: {}\\n'.format(EASY_THRESH, MEDIUM_THRESH, HARD_THRESH, IMPOSSIBLE_THRESH))\n",
    "    \n",
    "    id2label = {0: \"entailment\",\n",
    "                1: \"contradiction\",\n",
    "                2: \"neutral\"}\n",
    "    for key in difficulty_dict.keys():\n",
    "        \n",
    "        examples = source_df[source_df.ID.astype('str').isin(difficulty_dict[key].id.astype('string'))].head(num)\n",
    "        examples['Model Prediction'] = examples.ID.apply(lambda x: id2label[np.argmax(prediction_dict[x]['prediction'])])\n",
    "        \n",
    "        print(str(key).capitalize() + ' examples:')\n",
    "        examples = examples[['Premise', 'Hypothesis', 'Label', 'Model Prediction']]\n",
    "        examples['Hypothesis'] = examples['Hypothesis'].str.wrap(COL_WIDTH).astype('string')\n",
    "        examples['Premise'] = examples['Premise'].str.wrap(COL_WIDTH).astype('string')\n",
    "        \n",
    "        examples = examples.style.set_properties(**{'text-align': 'left'})\n",
    "        examples = examples.set_table_styles([dict(selector = 'th', props=[('text-align', 'left')])])\n",
    "        display(examples)\n",
    "   \n",
    "\n",
    "show_examples(source_df=wanli, source_map=wanli_maps, base_df=None, num=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "immune-bones",
   "metadata": {},
   "source": [
    "# Uncertainty metric:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "essential-homework",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing UNC for seed: 42\n",
      "Computing UNC for seed: 43\n",
      "Computing UNC for seed: 44\n",
      "Computing UNC for seed: 45\n",
      "Computing UNC for seed: 46\n"
     ]
    }
   ],
   "source": [
    "def compute_uncertainty(strategies, base_path, strategy_path):\n",
    "    \n",
    "    results = {key: [] for key in strategies}\n",
    "    \n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        print('Computing UNC for seed: {}'.format(seed))\n",
    "        \n",
    "        confidences = pd.read_pickle(base_path.replace('SEED', str(seed)))\n",
    "        \n",
    "        seed_entropies = {}\n",
    "        \n",
    "        sample_ids = list(confidences.keys())\n",
    "        for sample_id in sample_ids:\n",
    "    \n",
    "            times_correct = []\n",
    "\n",
    "            outputs = confidences[sample_id]\n",
    "            output = outputs[-1] # last epoch\n",
    "            \n",
    "            prediction = output[0]\n",
    "            pred_entropy = entropy(prediction)\n",
    "            seed_entropies[sample_id] = pred_entropy\n",
    "        \n",
    "        def get_entropies(x):\n",
    "            \n",
    "            if x in seed_entropies.keys():\n",
    "                return seed_entropies[x]\n",
    "            else:\n",
    "                return False\n",
    "        \n",
    "        for strategy in strategies:\n",
    "            \n",
    "            strategy_df = load_strategy_acquisitions(strategy=strategy, path=strategy_path.replace('SEED', str(seed)).replace('STRATEGY', str(strategy)))\n",
    "            \n",
    "            strategy_df['entropies'] = strategy_df.ID.apply(lambda x: get_entropies(x))\n",
    "            uncertainty = np.mean(strategy_df.entropies.tolist())\n",
    "            \n",
    "            results[strategy].append(uncertainty)\n",
    "        \n",
    "    return results\n",
    "\n",
    "uncertainty = compute_uncertainty(strategies=['random', 'bald', 'dal', 'mc-max-entropy'], base_path=BASE_PATH, strategy_path=STRATEGY_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "molecular-niger",
   "metadata": {},
   "outputs": [],
   "source": [
    "strats = ['random', 'bald', 'dal', 'mc-max-entropy']\n",
    "\n",
    "def plot_uncertainty(uncertainty, size=15, dpi=60):\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "\n",
    "    width = 1\n",
    "    alpha = ALPHA\n",
    "    labels = np.array(range(len(uncertainty.keys())))\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(CMAP)\n",
    "    \n",
    "    for i, key in enumerate(strats):\n",
    "        ax.bar(key, np.mean(uncertainty[key]), yerr=np.std(uncertainty[key]), width=width, label='{}'.format(key), color=cmap(i/len(labels)), alpha=alpha)\n",
    "\n",
    "    \n",
    "    ax.set_ylabel('Mean predictive entropy')\n",
    "    ax.set_xlabel('Strategy')\n",
    "    ax.set_ylim(0, 1)\n",
    "    ax.set_title('Uncertainty per Strategy')\n",
    "    ax.legend(loc='upper right')\n",
    "    \n",
    "    plt.savefig('uncertainty')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "maritime-scratch",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 15724 characters]\n",
      "text/plain": [
       "<Figure size 350x350 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 17164 characters]\n",
      "text/plain": [
       "<Figure size 350x350 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_input_div(input_div, size=20, dpi=60):\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "\n",
    "    width = 1\n",
    "    alpha = ALPHA\n",
    "    labels = np.array(range(len(uncertainty.keys())))\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(CMAP)\n",
    "    \n",
    "    for i, key in enumerate(strats):\n",
    "        ax.bar(key, input_div[key][0], yerr=input_div[key][1], width=width, label='{}'.format(key), color=cmap(i/len(labels)), alpha=alpha)\n",
    "\n",
    "    \n",
    "    ax.set_ylabel(' entropy')\n",
    "    ax.set_xlabel('Strategy')\n",
    "    ax.set_ylim(0, 1)\n",
    "    ax.set_title('Input diversity per Strategy')\n",
    "    ax.legend(loc='upper right')\n",
    "    \n",
    "    plt.savefig('input_div')\n",
    "    plt.show()\n",
    "\n",
    "input_div = {'dal':[0.2761, 0.004296],\n",
    "'max-entropy': [0.3308, 0.01008],\n",
    "'random': [0.2756, 0.00295],\n",
    "'bald': [0.2203, 0.01226],\n",
    "'mc-max-entropy':[0.3227, 0.00429]}\n",
    "\n",
    "SIZE = 5\n",
    "DPI = 70\n",
    "FONT_SIZE = 12\n",
    "plt.rcParams.update({'font.size': FONT_SIZE})\n",
    "plot_input_div(input_div, size=SIZE, dpi=DPI)\n",
    "plot_uncertainty(uncertainty, size=SIZE, dpi=DPI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "protective-abraham",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "miniature-drink",
   "metadata": {},
   "source": [
    "# TODO: find out if token overlap is a good enough signal to regularise towards target domain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "egyptian-backing",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Cannot take a larger sample than population when 'replace=False'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-618314727a07>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0manli_dev\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0manli\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0manli_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0manli\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0manli\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mID\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0manli_dev\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mID\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m50000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0msnli_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msnli\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m50000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36msample\u001b[0;34m(self, n, frac, replace, weights, random_state, axis)\u001b[0m\n\u001b[1;32m   4993\u001b[0m             )\n\u001b[1;32m   4994\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4995\u001b[0;31m         \u001b[0mlocs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchoice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mweights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4996\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlocs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4997\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mmtrand.pyx\u001b[0m in \u001b[0;36mnumpy.random.mtrand.RandomState.choice\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Cannot take a larger sample than population when 'replace=False'"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "anli_dev = anli.sample(5000)\n",
    "\n",
    "anli_train = anli[-anli.ID.isin(anli_dev.ID)].sample(50000)\n",
    "snli_train = snli.sample(50000)\n",
    "\n",
    "def get_set_of_tokens(dataset):\n",
    "\n",
    "            # join sentences from Premise and Hypothesis column into single string\n",
    "            dataset['Combined'] = dataset['Premise'] + ' ' + dataset['Hypothesis']\n",
    "            list_of_tokens = []\n",
    "            # split each sentence into tokens and add to list\n",
    "            for sentence in dataset['Combined'].tolist():\n",
    "                for char in [',', '.', '?', '!', ':', ';',\"'\", '\"', ')', '(', '<br>']:\n",
    "                    sentence = sentence.replace(char, '').lower()\n",
    "#                 dataset['Split_set'] = sentence.split()\n",
    "                list_of_tokens.extend(sentence.split())\n",
    "            # return set of unique tokens\n",
    "            return dataset, set(list_of_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "careful-authority",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "anli_train, anli_train_tokens = get_set_of_tokens(anli_train)\n",
    "snli_train, snli_train_tokens = get_set_of_tokens(snli_train)\n",
    "anli_dev, anli_dev_tokens = get_set_of_tokens(anli_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "noted-survival",
   "metadata": {},
   "outputs": [],
   "source": [
    "# anli_intersection = anli_train_tokens.intersection(anli_dev_tokens)\n",
    "# snli_intersection = snli_train_tokens.intersection(anli_dev_tokens)\n",
    "\n",
    "# print(len(anli_intersection)/len(anli_train_tokens))\n",
    "# print(len(snli_intersection)/len(snli_train_tokens))\n",
    "\n",
    "def compute_mean_overlap(train_df, dev_set):\n",
    "    overlap = []\n",
    "    for sentence in train_df.Combined:\n",
    "        for char in [',', '.', '?', '!', ':', ';',\"'\", '\"', ')', '(', '<br>']:\n",
    "            sentence = sentence.replace(char, '').lower()\n",
    "\n",
    "            overlap.append(len(set(sentence).intersection(dev_set)))\n",
    "            \n",
    "    return np.mean(overlap)\n",
    "            \n",
    "compute_mean_overlap(train_df=anli_train, dev_set=anli_dev_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lyric-thanks",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/test_predictions/test/test_preds_1_epoch_op_versch_test_sets_2/random/46/SNLI/test_predictions.pickle'\n",
    "SNLI = load_confidences(path=path, predictions=False, exclude=0)\n",
    "\n",
    "path = '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning/results/ard_output_dir_results/test_predictions/test/test_preds_1_epoch_op_versch_test_sets_2/random/46/MNLI/test_predictions.pickle'\n",
    "MNLI = load_confidences(path=path, predictions=False, exclude=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "british-michigan",
   "metadata": {},
   "outputs": [],
   "source": [
    "SNLI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "equipped-republican",
   "metadata": {},
   "outputs": [],
   "source": [
    "MNLI = MNLI[~MNLI.id.isin(SNLI.id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "opposed-glass",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(MNLI[MNLI.correctness == 1.0])/len(MNLI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "brief-tourism",
   "metadata": {},
   "outputs": [],
   "source": [
    "MNLI = MNLI[MNLI.id.isin(SNLI.id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "assured-bobby",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(SNLI[SNLI.correctness == 1])/len(SNLI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "informed-gender",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(MNLI[MNLI.correctness == 1])/len(MNLI)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
