{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "wicked-sunday",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:90% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "from pprint import pprint\n",
    "import sys\n",
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "import matplotlib.patches as mpatches\n",
    "\n",
    "from scipy.stats import entropy\n",
    "from matplotlib.ticker import PercentFormatter\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "\n",
    "from scipy.stats import sem\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 1000)\n",
    "\n",
    "COL_WIDTH=250\n",
    "CMAP = 'seismic_r'\n",
    "_CMAP = 'viridis'\n",
    "pd.options.display.max_colwidth = COL_WIDTH\n",
    "\n",
    "EASY_THRESH = 0.75\n",
    "MEDIUM_THRESH = 0.5\n",
    "HARD_THRESH = 0.25\n",
    "IMPOSSIBLE_THRESH = 0.0\n",
    "ALPHA = 0.6\n",
    "\n",
    "SEEDS = [42, 43, 44, 46]\n",
    "BASE_PATH = '/Users/ardsnijders/Desktop/AL_backup/results/ard_output_dir_results/datamaps/test/datamap_test_subepoch_inference_elke_0.5_epochs/roberta-large/random/SEED/confidences.pickle'\n",
    "# BASE_PATH = '/Users/ardsnijders/Documents/GitHub/Lisa/active_learning_backup/results/ard_output_dir_results/datamap/test/datamap_op_test_set_subepoch_inference_elke_0.5_epochs_max_4_epochs/random/SEED/datamap.pickle'\n",
    "STRATEGY_PATH = \"/Users/ardsnijders/Desktop/AL_backup/results/ard_output_dir_results/acquisition_IDs/array/main_experiment_herhaald_dubbele_acq_batch_size_woensdag/roberta-large/STRATEGY/SEED/acquisition_ids.csv\"\n",
    "\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:90% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unknown-supplement",
   "metadata": {},
   "source": [
    "# Loading pickled confidences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "postal-indication",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_confidences(path, predictions=False, exclude=1, up_to=-1):\n",
    "\n",
    "    df = pd.read_pickle(path)\n",
    "    sample_ids = list(df.keys())\n",
    "\n",
    "    cartography_dict = {}\n",
    "    prediction_dict = {}\n",
    "\n",
    "    for sample_id in sample_ids:\n",
    "\n",
    "        times_correct = []\n",
    "        confidences = []\n",
    "\n",
    "        outputs = df[sample_id]\n",
    "        outputs = outputs[exclude:]#[:up_to] # don't count the first 0.5 epoch!\n",
    "\n",
    "        for output in outputs:\n",
    "\n",
    "            # extract values from output tuple\n",
    "            prediction = output[0]\n",
    "            label = output[1]\n",
    "\n",
    "            # obtain GT confidence, determine correctness of prediction\n",
    "            confidence = prediction[label]\n",
    "            correct = 1 if prediction.index(max(prediction)) == label else 0\n",
    "\n",
    "            # track confidence and correctness\n",
    "            confidences.append(confidence)\n",
    "            times_correct.append(correct)\n",
    "\n",
    "\n",
    "        cartography_dict[sample_id] = {'confidence': np.mean(confidences),\n",
    "                                      'variability': np.std(confidences),\n",
    "                                      'correctness': np.mean(times_correct)}\n",
    "\n",
    "        prediction_dict[sample_id] = {'prediction': outputs[-1][0]}\n",
    "\n",
    "    confidences = [cartography_dict[key]['confidence'] for key in list(cartography_dict.keys())]\n",
    "    variability = [cartography_dict[key]['variability'] for key in list(cartography_dict.keys())]\n",
    "    correctness = [cartography_dict[key]['correctness'] for key in list(cartography_dict.keys())]\n",
    "\n",
    "    df = pd.DataFrame(list(zip(sample_ids, confidences, variability, correctness)), columns=['id', 'mean_conf', 'variability', 'correctness'])\n",
    "    \n",
    "    if predictions == False:\n",
    "        return df\n",
    "    else:\n",
    "        return df, prediction_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cordless-parts",
   "metadata": {},
   "source": [
    "# Code to read datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "hybrid-decimal",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def read_dataset(input_dir, dataset_id, split, seed, wanli_id):\n",
    "    \"\"\"\n",
    "    This function takes a dataset id and reads the corresponding .json split in an appropriate Pandas DataFrame\n",
    "    :param wanli_id: id key for wanli json file (pairID for metrics, id for non metric stuff)\n",
    "    :param seed:\n",
    "    :param input_dir:\n",
    "    :param dataset_id: str indicating which dataset should be read\n",
    "    :param split: str indicating which split should be read\n",
    "    :return: DataFrame with examples (Premise, Hypothesis, Label, ID)\n",
    "    \"\"\"\n",
    "    # anli_rounds = ['ANLI_R1', 'ANLI_R2', 'ANLI_R3']\n",
    "    anli_rounds = ['ANLI_R2', 'ANLI_R3']\n",
    "\n",
    "    def replace_labels(label):\n",
    "        label_conversions = {'e': 'entailment',\n",
    "                             'c': 'contradiction',\n",
    "                             'n': 'neutral'}\n",
    "\n",
    "        return label_conversions[label]\n",
    "\n",
    "    if dataset_id == 'SNLI':\n",
    "\n",
    "        # TODO only consider snli with gold labels?\n",
    "        data_path = '{}/snli_1.0/snli_1.0_{}.jsonl'.format(input_dir, split)\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['sentence1', 'sentence2', 'gold_label', 'pairID']]\n",
    "        dataset['dataset'] = 'SNLI'\n",
    "        dataset = dataset.drop(dataset[dataset.gold_label.str.contains('-')].index)  # drop examples with no gold label\n",
    "\n",
    "    # load separate ANLI rounds\n",
    "    elif dataset_id in anli_rounds:\n",
    "\n",
    "        data_round = dataset_id[-2:]  # extract the 'R#' substring from the provided ID\n",
    "        data_path = '{}/anli_v1.0/{}/{}.jsonl'.format(input_dir, data_round, split)\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['context', 'hypothesis', 'label', 'uid']]  # get rid of unnecessary columns\n",
    "        dataset['label'] = dataset['label'].apply(replace_labels)  # ensures consistently named labels\n",
    "        dataset['dataset'] = 'ANLI_{}'.format(data_round)\n",
    "        #\n",
    "        # print('Dataframe for {} - {}'.format(dataset_id, split), flush=True)\n",
    "\n",
    "    # compile all ANLI rounds\n",
    "    elif dataset_id == 'ANLI':\n",
    "\n",
    "        dataset_list = []\n",
    "        for data_round in anli_rounds:\n",
    "            data_round = data_round[-2:]\n",
    "            data_path = '{}/anli_v1.0/{}/{}.jsonl'.format(input_dir, data_round, split)\n",
    "            dataset = pd.read_json(data_path, lines=True)\n",
    "            dataset = dataset[['context', 'hypothesis', 'label', 'uid']]  # get rid of unnecessary columns\n",
    "            dataset['label'] = dataset['label'].apply(replace_labels)  # ensures consistently named labels\n",
    "            dataset['dataset'] = 'ANLI'\n",
    "            dataset_list.append(dataset)\n",
    "        dataset = pd.concat(dataset_list, axis=0)\n",
    "\n",
    "    # Load MNLI\n",
    "    elif dataset_id == 'MNLI':\n",
    "\n",
    "        if split == 'train':\n",
    "            data_path = '{}/multinli_1.0/multinli_1.0_{}.jsonl'.format(input_dir, split)\n",
    "        elif split == 'dev' or split == 'test':\n",
    "            data_path = '{}/multinli_1.0/multinli_1.0_dev_matched.jsonl'.format(input_dir)\n",
    "\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['sentence1', 'sentence2', 'gold_label', 'pairID']]\n",
    "        dataset['dataset'] = 'MNLI'\n",
    "        dataset = dataset.drop(dataset[dataset.gold_label.str.contains('-')].index)\n",
    "\n",
    "        # split dev into dev and test set\n",
    "        if split == 'dev':\n",
    "            dataset = dataset.sample(frac=1,  # reshuffle all rows in the dataframe prior to split\n",
    "                                     random_state=seed)  # seed is fixed such that dev and test set are always the same\n",
    "            dataset = dataset[:int((len(dataset)/2))]\n",
    "\n",
    "        elif split == 'test':\n",
    "            dataset = dataset.sample(frac=1,\n",
    "                                     random_state=seed)\n",
    "            dataset = dataset[int((len(dataset)/2)):]\n",
    "\n",
    "    # Load WANLI\n",
    "    elif dataset_id == 'WANLI':\n",
    "\n",
    "        if split == 'train':\n",
    "            data_path = '{}/wanli/train.jsonl'.format(input_dir)\n",
    "        elif split == 'dev' or split == 'test':\n",
    "            data_path = '{}/wanli/test.jsonl'.format(input_dir)\n",
    "\n",
    "        dataset = pd.read_json(data_path, lines=True)\n",
    "        dataset = dataset[['premise', 'hypothesis', 'gold', wanli_id]]\n",
    "        dataset['dataset'] = 'WANLI'\n",
    "        dataset = dataset.drop(dataset[dataset.gold.str.contains('-')].index)\n",
    "\n",
    "        # split dev into dev and test set\n",
    "        if split == 'dev':\n",
    "            dataset = dataset.sample(frac=1,  # reshuffle all rows in the dataframe prior to split\n",
    "                                     random_state=seed)  # seed is fixed such that dev and test set are always the same\n",
    "            dataset = dataset[:int((len(dataset) / 2))]\n",
    "\n",
    "        elif split == 'test':\n",
    "            dataset = dataset.sample(frac=1,\n",
    "                                     random_state=seed)\n",
    "            dataset = dataset[int((len(dataset) / 2)):]\n",
    "\n",
    "    else:\n",
    "        raise KeyError('No dataset found for \"{}\"'.format(dataset_id))\n",
    "\n",
    "    # ensure consistent headers per dataset DataFrame\n",
    "    dataset.columns = ['Premise', 'Hypothesis', 'Label', 'ID', 'Dataset']\n",
    "\n",
    "    dataset['ID'] = dataset['ID'].astype('string')\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "circular-orientation",
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_DIR='/Users/ardsnijders/Desktop/AL_backup/resources/ard_data'\n",
    "SPLIT='train'\n",
    "WANLI_ID='pairID'\n",
    "\n",
    "\n",
    "\n",
    "def read_source(dataset_id, split='train'):\n",
    "    dataset = read_dataset(input_dir=INPUT_DIR, \n",
    "                 dataset_id=dataset_id, \n",
    "                 split=split, \n",
    "                 seed=None, \n",
    "                 wanli_id=WANLI_ID)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "promising-receptor",
   "metadata": {},
   "source": [
    "# Loading all datasets into their own DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sublime-finance",
   "metadata": {},
   "source": [
    "# Code for Generating Datamaps for Different Sources / Strategies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "third-homework",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_df(dataframe):\n",
    "    X = dataframe['variability'].tolist()\n",
    "    Y = dataframe['mean_conf'].tolist()\n",
    "    colors = dataframe['correctness'].tolist()\n",
    "\n",
    "    plt.figure(figsize=(30, 10))\n",
    "    plt.scatter(x=X, y=Y, c=colors, alpha=1, cmap=mpl.cm.get_cmap(CMAP))\n",
    "    plt.xlim(0,0.5)\n",
    "    plt.ylim(0,1)\n",
    "    plt.xlabel('Variability')\n",
    "    plt.ylabel('Confidence')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    \n",
    "def generate_source_map(base_df, source_df):\n",
    "\n",
    "    ids = source_df.ID\n",
    "\n",
    "    source_df = base_df[base_df.id.astype('string').isin(ids)]\n",
    "    \n",
    "    print('num of source in base: {}'.format(len(source_df)))\n",
    "    \n",
    "    generate_df(source_df)\n",
    "    \n",
    "def datamap_subset(base_df, source_df, name=None):\n",
    "    \n",
    "    source_ids = source_df.ID\n",
    "    \n",
    "    subset_df = base_df[base_df.id.astype('string').isin(source_ids)]\n",
    "\n",
    "    if name is not None:\n",
    "        subset_df['Name'] = name\n",
    "    else:\n",
    "        subset_df['Name'] = source_df.Dataset.tolist()[0]\n",
    "    \n",
    "    return subset_df\n",
    "\n",
    "def datamap_subset_multi(base_path, source_df, name=None, up_to=False):\n",
    "    \n",
    "    list_of_maps = []\n",
    "    \n",
    "    source_ids = source_df.ID\n",
    "    \n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        _base_path = base_path.replace('SEED', str(seed))\n",
    "        \n",
    "        base_df = load_confidences(_base_path, predictions=False, up_to=up_to)\n",
    "\n",
    "        subset_df = base_df[base_df.id.astype('string').isin(source_ids)]\n",
    "\n",
    "        if name is not None:\n",
    "            subset_df['Name'] = name\n",
    "        else:\n",
    "            subset_df['Name'] = source_df.Dataset.tolist()[0]\n",
    "            \n",
    "        list_of_maps.append(subset_df)\n",
    "    \n",
    "    return list_of_maps\n",
    "\n",
    "def generate_dfs_2(dataframes, markers=None, learnability_only=False, map_only=False, size=40, n_bins=10, font_size=20, up_to='None'):\n",
    "    \n",
    "    plt.rcParams.update({'font.size': font_size})\n",
    "    \n",
    "    def generate_hist(axis, data, name, color, colormap, n_bins=10):\n",
    "        \n",
    "        cmap = False\n",
    "        if color == 'cmap':\n",
    "            cmap = True\n",
    "            color = 'green'\n",
    "        \n",
    "        n, bins, patches = axis.hist(data, \n",
    "                                     n_bins, \n",
    "                                     weights=np.ones(len(data)) / len(data), \n",
    "                                     density=False, \n",
    "                                     color=color)\n",
    "\n",
    "        bin_centers = 0.5 * (bins[:-1] + bins[1:])\n",
    "        \n",
    "        if cmap:\n",
    "        \n",
    "            # scale values to interval [0,1]\n",
    "            col = bin_centers - min(bin_centers)\n",
    "            col /= max(col)\n",
    "\n",
    "            for c, p in zip(col, patches):\n",
    "                plt.setp(p, 'facecolor', colormap(c))\n",
    "\n",
    "        axis.set_xlabel(name)\n",
    "        axis.set_ylim(0,1)\n",
    "\n",
    "        axis.yaxis.set_major_formatter(FuncFormatter('{0:.0%}'.format))\n",
    "    \n",
    "    # creating grid for subplots\n",
    "    fig = plt.figure()\n",
    "    fig.set_figheight(size)\n",
    "    fig.set_figwidth(size*1.5)\n",
    "    \n",
    "    ax1 = plt.subplot2grid(shape=(4, 4), loc=(0, 0), colspan=3, rowspan=4)\n",
    "    ax2 = plt.subplot2grid(shape=(4, 4), loc=(0, 3), rowspan=1)\n",
    "    ax3 = plt.subplot2grid(shape=(4, 4), loc=(1, 3), colspan=1)\n",
    "    ax4 = plt.subplot2grid(shape=(4, 4), loc=(2, 3), rowspan=1)\n",
    "    ax5 = plt.subplot2grid(shape=(4, 4), loc=(3, 3), rowspan=1)\n",
    "    \n",
    "    color_dict = {'SNLI':'red', 'ANLI':'green', 'WANLI':'blue'}\n",
    "    \n",
    "    for dataframe in dataframes:\n",
    "\n",
    "        if markers is not None:\n",
    "\n",
    "            for dataset, marker in zip(['ANLI', 'WANLI', 'SNLI'], ['s', 'x', 'o', 'v']):\n",
    "\n",
    "\n",
    "                sub_frame = dataframe[dataframe['Dataset'] == dataset]\n",
    "                \n",
    "                sub_frame = sub_frame.sample(800)\n",
    "                \n",
    "                X = sub_frame['variability'].tolist()\n",
    "                Y = sub_frame['mean_conf'].tolist()\n",
    "#                 colors = sub_frame['correctness'].tolist()\n",
    "#                 colors = [color_dict[dataset]]*len(sub_frame.tolist())\n",
    "                \n",
    "#                 ax1.scatter(x=X, y=Y, c=colors, alpha=1, marker=marker, cmap=mpl.cm.get_cmap(CMAP), \n",
    "#                             label=dataset, s=size*4)\n",
    "                ax1.scatter(x=X, y=Y, c=color_dict[dataset], alpha=0.2,\n",
    "                            label=dataset, s=size*4)\n",
    "\n",
    "        else:\n",
    "\n",
    "\n",
    "            X = dataframe['variability'].tolist()\n",
    "            Y = dataframe['mean_conf'].tolist()\n",
    "            colors = dataframe['correctness'].tolist()\n",
    "\n",
    "            ax1.scatter(x=X, y=Y, c=colors, alpha=1, cmap=mpl.cm.get_cmap(CMAP), s=size)\n",
    "        \n",
    "        name = dataframe.Name.tolist()[0].upper()\n",
    "        ax1.set_title('Dataset Map for ' + name)\n",
    "        ax1.set_xlabel('Variability')\n",
    "        ax1.set_ylabel('Confidence')\n",
    "\n",
    "        if markers:\n",
    "            ax1.legend(loc='upper right', prop={'size': 30})\n",
    "            ax1.legend(markerscale=2)\n",
    "            \n",
    "        break # terminate loop for datamap after first seed since only one seed can be plotted\n",
    "    \n",
    "    \n",
    "    easy = []\n",
    "    medium = []\n",
    "    hard = []\n",
    "    impossible = []\n",
    "    X = []\n",
    "    Y = []\n",
    "    correctness = []\n",
    "    for dataframe in dataframes:\n",
    "    \n",
    "        X.extend( dataframe['variability'].tolist() )\n",
    "        Y.extend( dataframe['mean_conf'].tolist() )\n",
    "        correctness.extend( dataframe['correctness'].tolist() )\n",
    "\n",
    "        easy.append( len(dataframe[dataframe['mean_conf'] > EASY_THRESH])  )\n",
    "        medium.append( len(dataframe[(dataframe['mean_conf'] > MEDIUM_THRESH) & (dataframe['mean_conf'] < EASY_THRESH)])  )\n",
    "        hard.append( len(dataframe[(dataframe['mean_conf'] > HARD_THRESH) & (dataframe['mean_conf'] < MEDIUM_THRESH)])   )\n",
    "        impossible.append( len(dataframe[(dataframe['mean_conf'] > IMPOSSIBLE_THRESH) & (dataframe['mean_conf'] < HARD_THRESH)])  )\n",
    "    \n",
    "    # take the mean over difficulty brackets\n",
    "    # TODO: generate std bars?\n",
    "    def mean_and_std_e(array):\n",
    "        \n",
    "        return np.mean(array), np.std(array)\n",
    "    \n",
    "    easy, std_easy = mean_and_std_e(easy)\n",
    "    medium, std_medium = mean_and_std_e(medium)\n",
    "    hard, std_hard = mean_and_std_e(hard)\n",
    "    impossible, std_impossible = mean_and_std_e(impossible)\n",
    "    \n",
    "#     std_easy, std_medium, std_hard, std_impossible = 200,30,400,20\n",
    "    \n",
    "    # LEARNABILITY      \n",
    "    width = 1\n",
    "    cmap = mpl.cm.get_cmap(CMAP)\n",
    "    \n",
    "    categories = ['E', 'M', 'H', 'I']\n",
    "    x_pos = [0,1,2,3]\n",
    "    means = [easy, medium, hard, impossible]\n",
    "    \n",
    "    \n",
    "    y_err = [std_easy, std_medium, std_hard, std_impossible]\n",
    "    \n",
    "    normed_means = [mean / sum(means) for mean in means]\n",
    "    y_err = [std / sum(means) for std in y_err]\n",
    "    \n",
    "    colors = [cmap(value) for value in [0.85, 0.65, 0.35, 0.125]]\n",
    "    \n",
    "    labels = [('[p > {}]'.format(thresh, color=cmap(value), alpha=ALPHA)) for thresh, value in zip([EASY_THRESH, MEDIUM_THRESH, HARD_THRESH, IMPOSSIBLE_THRESH],[0.85, 0.65, 0.35, 0.125])]\n",
    "    ax2.bar(x_pos, normed_means, yerr=y_err, width=1, color=colors, capsize=10, alpha=ALPHA)\n",
    "    \n",
    "    print('values for {}'.format(name))\n",
    "    for val, std, pctage in zip(normed_means, y_err, ['0.75', '0.5', '0.25', '0']):\n",
    "        print('val, std for p > {} = {}, {}'.format(pctage, val, std))\n",
    "\n",
    "    thresholds = [EASY_THRESH, MEDIUM_THRESH, HARD_THRESH, IMPOSSIBLE_THRESH]\n",
    "    patches = [mpatches.Patch(color=value, label='[p > {}]'.format(thresh), alpha=ALPHA) for thresh, value in zip(thresholds, colors)]\n",
    "    \n",
    "    ax2.legend(handles=patches, loc='upper right')\n",
    "    \n",
    "    ax2.set_xticks(x_pos)\n",
    "    ax2.set_xticklabels(categories)\n",
    "    ax2.set_ylim(0,1)\n",
    "    \n",
    "#     ylabels = ['{}'.format(round(x,1)) + 'K' for x in ax2.get_yticks()[::1]/1000]\n",
    "    ylabels = ['{}%'.format(int(x))  for x in ax2.get_yticks()[::1]*100]\n",
    "    ax2.set_yticklabels(ylabels)\n",
    "    if learnability_only:\n",
    "        ax2.set_xlabel('{}'.format(name))\n",
    "    else:\n",
    "        ax2.set_xlabel('Learnability')\n",
    "    ax2.legend(loc='upper right')\n",
    "\n",
    "    \n",
    "    # CORRECTNESS      \n",
    "    generate_hist(axis=ax3, data=correctness, name='Correctness', color='cmap', colormap=mpl.cm.get_cmap(CMAP))\n",
    "\n",
    "    # CONFIDENCES      \n",
    "    generate_hist(axis=ax4, data=Y, name='Confidences', color='red', colormap=None)\n",
    "\n",
    "    # VARIABILITY      \n",
    "    generate_hist(axis=ax5, data=X, name='Variability', color='green', colormap=None)\n",
    "    \n",
    "    for ax in [ax2, ax3, ax4, ax5]:\n",
    "        ax.yaxis.tick_right()\n",
    "        ax.yaxis.set_label_position(\"right\")\n",
    "        ax.xaxis.set_label_position('top')\n",
    "    \n",
    "    plt.tight_layout()  \n",
    "    \n",
    "    if learnability_only:\n",
    "        print('learnability {}'.format(name))\n",
    "        ax1.remove()\n",
    "        ax3.remove()\n",
    "        ax4.remove()\n",
    "        ax5.remove()\n",
    "        \n",
    "        plt.show(ax2)\n",
    "        # Save just the portion _inside_ the second axis's boundaries\n",
    "        extent = ax2.get_window_extent().transformed(fig.dpi_scale_trans.inverted())\n",
    "        fig.savefig('learnability_{}.pdf'.format(name), format='pdf', bbox_inches=extent.expanded(1.55, 1.3))\n",
    "#         plt.savefig('learnability_{}.pdf'.format(name), format='pdf')\n",
    "\n",
    "    if map_only:\n",
    "        ax2.remove()\n",
    "        ax3.remove()\n",
    "        ax4.remove()\n",
    "        ax5.remove()\n",
    "        \n",
    "        plt.show(ax1)\n",
    "        # Save just the portion _inside_ the second axis's boundaries\n",
    "        extent = ax1.get_window_extent().transformed(fig.dpi_scale_trans.inverted())\n",
    "        fig.savefig('map_only_{}_iter_{}.pdf'.format(name, up_to), format='pdf', bbox_inches=extent.expanded(1.55, 1.3))\n",
    "#         plt.savefig('learnability_{}.pdf'.format(name), format='pdf')\n",
    "    \n",
    "    else:\n",
    "        plt.savefig('datamap_{}_iter_{}.pdf'.format(name, up_to), format='pdf') \n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "numerical-riding",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mnli = read_source('MNLI')\n",
    "snli = read_source('SNLI')\n",
    "anli = read_source('ANLI')\n",
    "wanli = read_source('WANLI')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "atmospheric-theory",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "entailment       183416\n",
       "contradiction    183187\n",
       "neutral          182764\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "snli.Label.value_counts()\n",
    "# snli.Label.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "numeric-speaker",
   "metadata": {},
   "source": [
    "# Datamaps per Source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "natural-payment",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for ANLI\n",
      "val, std for p > 0.75 = 0.6018697692419331, 0.02816470267944927\n",
      "val, std for p > 0.5 = 0.20106671181848967, 0.018668407844484616\n",
      "val, std for p > 0.25 = 0.11315369358037831, 0.008161164765577083\n",
      "val, std for p > 0 = 0.08390982535919887, 0.0014393667985885385\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 548516 characters]\n",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for ANLI\n",
      "val, std for p > 0.75 = 0.6018697692419331, 0.02816470267944927\n",
      "val, std for p > 0.5 = 0.20106671181848967, 0.018668407844484616\n",
      "val, std for p > 0.25 = 0.11315369358037831, 0.008161164765577083\n",
      "val, std for p > 0 = 0.08390982535919887, 0.0014393667985885385\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 548516 characters]\n",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for ANLI\n",
      "val, std for p > 0.75 = 0.6018697692419331, 0.02816470267944927\n",
      "val, std for p > 0.5 = 0.20106671181848967, 0.018668407844484616\n",
      "val, std for p > 0.25 = 0.11315369358037831, 0.008161164765577083\n",
      "val, std for p > 0 = 0.08390982535919887, 0.0014393667985885385\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 548516 characters]\n",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for ANLI\n",
      "val, std for p > 0.75 = 0.6018697692419331, 0.02816470267944927\n",
      "val, std for p > 0.5 = 0.20106671181848967, 0.018668407844484616\n",
      "val, std for p > 0.25 = 0.11315369358037831, 0.008161164765577083\n",
      "val, std for p > 0 = 0.08390982535919887, 0.0014393667985885385\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 548516 characters]\n",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for ANLI\n",
      "val, std for p > 0.75 = 0.6018697692419331, 0.02816470267944927\n",
      "val, std for p > 0.5 = 0.20106671181848967, 0.018668407844484616\n",
      "val, std for p > 0.25 = 0.11315369358037831, 0.008161164765577083\n",
      "val, std for p > 0 = 0.08390982535919887, 0.0014393667985885385\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 548516 characters]\n",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.dpi'] = 80\n",
    "SIZE = 10\n",
    "FONT_SIZE = 12\n",
    "\n",
    "for iteration in range(2,7):\n",
    "    \n",
    "    anli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=anli, up_to=iteration)\n",
    "    generate_dfs_2(anli_maps, size=SIZE, font_size=FONT_SIZE, up_to=iteration, map_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "injured-float",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:55: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "snli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=snli)\n",
    "anli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=anli)\n",
    "wanli_maps = datamap_subset_multi(base_path=BASE_PATH, source_df=wanli)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "confidential-draft",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for ANLI\n",
      "val, std for p > 0.75 = 0.6018697692419331, 0.02816470267944927\n",
      "val, std for p > 0.5 = 0.20106671181848967, 0.018668407844484616\n",
      "val, std for p > 0.25 = 0.11315369358037831, 0.008161164765577083\n",
      "val, std for p > 0 = 0.08390982535919887, 0.0014393667985885385\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 488980 characters]\n",
      "text/plain": [
       "<Figure size 1080x720 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for SNLI\n",
      "val, std for p > 0.75 = 0.7828865979381443, 0.008913652019033469\n",
      "val, std for p > 0.5 = 0.09307459066100666, 0.007547397136051078\n",
      "val, std for p > 0.25 = 0.05069739235900546, 0.002605129454619063\n",
      "val, std for p > 0 = 0.07334141904184355, 0.0009825633079238024\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 457088 characters]\n",
      "text/plain": [
       "<Figure size 1080x720 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for WANLI\n",
      "val, std for p > 0.75 = 0.3680994317789053, 0.0382182233543834\n",
      "val, std for p > 0.5 = 0.32344361338845035, 0.028376783573517986\n",
      "val, std for p > 0.25 = 0.1935684561253128, 0.012692188276250642\n",
      "val, std for p > 0 = 0.11488849870733157, 0.006805265415679207\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 507852 characters]\n",
      "text/plain": [
       "<Figure size 1080x720 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.dpi'] = 80\n",
    "SIZE = 9\n",
    "FONT_SIZE = 13\n",
    "generate_dfs_2(anli_maps, size=SIZE, font_size=FONT_SIZE)\n",
    "generate_dfs_2(snli_maps, size=SIZE, font_size=FONT_SIZE)\n",
    "generate_dfs_2(wanli_maps, size=SIZE, font_size=FONT_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "requested-workstation",
   "metadata": {},
   "source": [
    "# Datamaps per Strategy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "laden-dover",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label distribution for random: {'neutral': 0.41, 'entailment': 0.34, 'contradiction': 0.25}\n",
      "label distribution for dal: {'neutral': 0.46, 'entailment': 0.34, 'contradiction': 0.2}\n",
      "label distribution for bald: {'neutral': 0.32, 'entailment': 0.32, 'contradiction': 0.36}\n",
      "label distribution for mc-max-entropy: {'neutral': 0.38, 'entailment': 0.31, 'contradiction': 0.32}\n"
     ]
    }
   ],
   "source": [
    "for strategy in ['random', 'dal', 'bald', 'mc-max-entropy']:\n",
    "    \n",
    "    label_dict = {'neutral': [], 'entailment': [], 'contradiction': []}\n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        _strategy_path = STRATEGY_PATH.replace('SEED', str(seed)).replace('STRATEGY', str(strategy))\n",
    "        strategy_df = load_strategy_acquisitions(path=_strategy_path, strategy=strategy)\n",
    "        strategy_df.ID = strategy_df.ID.astype('str')\n",
    "        \n",
    "        \n",
    "        label_counts = strategy_df.Label.value_counts(normalize=True)\n",
    "        \n",
    "        for key in list(label_dict.keys()):\n",
    "            label_dict[key].append(label_counts[key])\n",
    "            \n",
    "    \n",
    "    label_dict = {key:round(np.mean(label_dict[key]),2) for key in list(label_dict.keys())}\n",
    "    print('label distribution for {}: {}'.format(strategy, label_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "pacific-alaska",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_strategy_acquisitions(path, strategy, cap=4000):\n",
    "    \n",
    "    acquired_df = pd.read_csv(path)\n",
    "    acquired_df = acquired_df[:cap]\n",
    "    acquired_df['Name'] = strategy.capitalize()\n",
    "    \n",
    "    return acquired_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "aquatic-bunch",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:32: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "def get_subset_list(strategy_path, base_path, strategy_list):\n",
    "    \n",
    "    subset_list = []\n",
    "    \n",
    "    for strategy in strategy_list:\n",
    "        \n",
    "        strategy_list = []\n",
    "        \n",
    "        for seed in SEEDS:\n",
    "            \n",
    "            _base_path = base_path.replace('SEED', str(seed))\n",
    "            _strategy_path = strategy_path.replace('SEED', str(seed)).replace('STRATEGY', str(strategy))\n",
    "            \n",
    "            strategy_df = load_strategy_acquisitions(path=_strategy_path, strategy=strategy)\n",
    "            strategy_df.ID = strategy_df.ID.astype('str')\n",
    "            \n",
    "#             print(strategy_df.Label.value_counts(normalize=True))\n",
    "            \n",
    "            base_df = load_confidences(_base_path)\n",
    "            \n",
    "            \n",
    "            \n",
    "#             sys.exit()\n",
    "\n",
    "            id2dataset = {}\n",
    "            for i in range(len(strategy_df)):\n",
    "                row = strategy_df.iloc[i]\n",
    "                id2dataset[row.ID] = row.Dataset\n",
    "\n",
    "            strategy_subset = datamap_subset(base_df=base_df, source_df=strategy_df, name=strategy)\n",
    "            \n",
    "            strategy_subset['Dataset'] = strategy_subset['id'].apply(lambda x: id2dataset[x])\n",
    "            \n",
    "            strategy_list.append(strategy_subset)\n",
    "        \n",
    "        subset_list.append(strategy_list)\n",
    "\n",
    "    return subset_list \n",
    "\n",
    "subset_list = get_subset_list(strategy_path=STRATEGY_PATH, base_path=BASE_PATH, strategy_list=['random', 'bald', 'mc-max-entropy', 'dal'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "experienced-swiss",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "subset_list_sampled = []\n",
    "for subset_maps in subset_list:\n",
    "    subset_maps_sampled = [subset_map.sample(2000) for subset_map in subset_maps]\n",
    "    subset_list_sampled.append(subset_maps_sampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "abroad-reproduction",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for RANDOM\n",
      "val, std for p > 0.75 = 0.5766748135614463, 0.021206386362711622\n",
      "val, std for p > 0.5 = 0.2160807169267406, 0.01582622710300392\n",
      "val, std for p > 0.25 = 0.11919533746944914, 0.008194959472721702\n",
      "val, std for p > 0 = 0.08804913204236385, 0.005993604675802536\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 154796 characters]\n",
      "text/plain": [
       "<Figure size 720x480 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for BALD\n",
      "val, std for p > 0.75 = 0.5287967947915363, 0.018800019089840955\n",
      "val, std for p > 0.5 = 0.21403530737448354, 0.015848378832361024\n",
      "val, std for p > 0.25 = 0.1330912733191436, 0.009215008355906948\n",
      "val, std for p > 0 = 0.12407662451483661, 0.006358314223410353\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 173448 characters]\n",
      "text/plain": [
       "<Figure size 720x480 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for MC-MAX-ENTROPY\n",
      "val, std for p > 0.75 = 0.38860688099266777, 0.028936460727191422\n",
      "val, std for p > 0.5 = 0.27523970671178793, 0.01709642317020001\n",
      "val, std for p > 0.25 = 0.19590148524158676, 0.014361327126054712\n",
      "val, std for p > 0 = 0.1402519270539575, 0.0007200053451824314\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 186812 characters]\n",
      "text/plain": [
       "<Figure size 720x480 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "values for DAL\n",
      "val, std for p > 0.75 = 0.5082234777150031, 0.021212255552217653\n",
      "val, std for p > 0.5 = 0.25204017576898935, 0.017269644339850627\n",
      "val, std for p > 0.25 = 0.14030131826741996, 0.006466397255513896\n",
      "val, std for p > 0 = 0.09943502824858758, 0.0030804379526391205\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 167264 characters]\n",
      "text/plain": [
       "<Figure size 720x480 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# generate_dfs(subset_list, markers=True, size=100, n_bins=10)\n",
    "\n",
    "plt.rcParams['figure.dpi'] = 80\n",
    "SIZE = 6\n",
    "FONT_SIZE = 17.5\n",
    "\n",
    "for subset_maps in subset_list:\n",
    "    generate_dfs_2(subset_maps, markers=True, size=SIZE, font_size=FONT_SIZE) #map_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "medium-ecology",
   "metadata": {},
   "source": [
    "# Acquisitions by Difficulty over Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "recent-slave",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_stacked_barchart(base_dfs, strategy_dfs, return_axs=False, size=20, font_size=22, dpi=40, show_legend=True):\n",
    "    \n",
    "    plt.rcParams.update({'font.size': font_size})\n",
    "    # Call this fn for a single strategy!\n",
    "    \n",
    "    num_easy_aggregate = []\n",
    "    num_medium_aggregate = []\n",
    "    num_hard_aggregate = []\n",
    "    num_impossible_aggregate = []\n",
    "    \n",
    "    for strategy_df, base_df in zip(strategy_dfs, base_dfs):\n",
    "        \n",
    "        number_of_examples = 500\n",
    "\n",
    "        labels = []\n",
    "        num_easy = []\n",
    "        num_medium = []\n",
    "        num_hard = []\n",
    "        num_impossible = []\n",
    "\n",
    "        easy_thresh = 0.75\n",
    "        medium_thresh = 0.5\n",
    "        hard_thresh = 0.25\n",
    "        impossible_thresh = 0.0\n",
    "\n",
    "        for active_round in range(1, strategy_df['round'].max()+1):\n",
    "\n",
    "            acquisitions = strategy_df[strategy_df['round']==active_round]\n",
    "            number_of_examples += len(acquisitions)\n",
    "            labels.append(number_of_examples)\n",
    "\n",
    "            confidences = base_df[base_df.id.isin(acquisitions.ID)]\n",
    "\n",
    "            easy_mask = confidences.mean_conf > easy_thresh\n",
    "            medium_mask = (confidences.mean_conf > medium_thresh) & (confidences.mean_conf < easy_thresh)\n",
    "            hard_mask = (confidences.mean_conf > hard_thresh) & (confidences.mean_conf < medium_thresh)\n",
    "            impossible_mask = (confidences.mean_conf > impossible_thresh) & (confidences.mean_conf < hard_thresh)\n",
    "\n",
    "            num_easy.append(len(confidences[easy_mask]))\n",
    "            num_medium.append(len(confidences[medium_mask]))\n",
    "            num_hard.append(len(confidences[hard_mask]))\n",
    "            num_impossible.append(len(confidences[impossible_mask]))\n",
    "            \n",
    "        num_easy_aggregate.append(num_easy)\n",
    "        num_medium_aggregate.append(num_medium)\n",
    "        num_hard_aggregate.append(num_hard)\n",
    "        num_impossible_aggregate.append(num_impossible)\n",
    "        \n",
    "    def get_mean_and_std_e(array):\n",
    "        \n",
    "        return np.mean(array, axis=0), np.std(array, axis=0)\n",
    "    \n",
    "    num_easy, num_easy_std = get_mean_and_std_e(num_easy_aggregate)    \n",
    "    num_medium, num_medium_std = get_mean_and_std_e(num_medium_aggregate)\n",
    "    num_hard, num_hard_std = get_mean_and_std_e(num_hard_aggregate)\n",
    "    num_impossible, num_impossible_std = get_mean_and_std_e(num_impossible_aggregate)  \n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "\n",
    "    width = 210\n",
    "    alpha = ALPHA\n",
    "    labels = np.array(labels)\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(CMAP)\n",
    "\n",
    "    ax.bar(labels, num_easy, width, yerr=num_easy_std, label='Easy [{} < p < 1.0]'.format(easy_thresh), color=cmap(0.85), alpha=alpha)\n",
    "    ax.bar(labels, num_medium, width, yerr=num_medium_std, bottom=num_easy,  label='Med. [{} < p < {}]'.format(medium_thresh, easy_thresh), color=cmap(0.65), alpha=alpha)\n",
    "    ax.bar(labels, num_hard, width, yerr=num_hard_std, bottom=num_easy + num_medium, label='Hard [{} < p < {}]'.format(hard_thresh, medium_thresh), color=cmap(0.35), alpha=alpha)\n",
    "    ax.bar(labels, num_impossible, width, yerr=num_impossible_std, bottom=num_easy + num_medium + num_hard, label='Imp. [0 < p < {}]'.format(hard_thresh), color=cmap(0.125), alpha=alpha)\n",
    "\n",
    "#     ax.set_xticks(ax.get_xticks()[2:])\n",
    "#     ax.set_ylabel('Acquisition by Difficulty')\n",
    "    ax.set_xlabel('Labeled Ex.')\n",
    "    ax.set_ylim(0, 510)\n",
    "    ax.set_title(strategy_df['Name'].tolist()[0])\n",
    "    \n",
    "    if show_legend:\n",
    "        ax.legend(loc='lower right', prop={'size': 0.9*font_size})\n",
    "    \n",
    "    if return_axs:\n",
    "        \n",
    "        return ax\n",
    "    \n",
    "    else:\n",
    "        \n",
    "#         if impossible_only:\n",
    "#             impossible_str = '_impossible_'\n",
    "#         else:\n",
    "#             impossible_str = ''\n",
    "        \n",
    "        plt.plot(stacked=True)\n",
    "        name = strategy_df['Name'].tolist()[0]\n",
    "        plt.savefig('acquisitions_by_difficulty_{}.pdf'.format(name), format='pdf', bbox_inches='tight')\n",
    "        plt.show()\n",
    "        \n",
    "        return None\n",
    "    \n",
    "\n",
    "def generate_stacked_barchart_data(base_dfs, strategy_dfs, return_axs=False, impossible_only=False, size=20, font_size=22, dpi=40, show_legend=True):\n",
    "    \n",
    "    plt.rcParams.update({'font.size': font_size})\n",
    "    # Call this fn for a single strategy!\n",
    "    \n",
    "    total_counts = {'SNLI':[], 'ANLI':[], 'WANLI':[]}\n",
    "    \n",
    "    for strategy_df, base_df in zip(strategy_dfs, base_dfs):\n",
    "        \n",
    "        number_of_examples = 500\n",
    "\n",
    "        labels = []\n",
    "        results = {'SNLI':[], 'ANLI':[], 'WANLI':[]}\n",
    "\n",
    "        for active_round in range(1, strategy_df['round'].max()+1):\n",
    "\n",
    "            acquisitions = strategy_df[strategy_df['round']==active_round]\n",
    "            number_of_examples += len(acquisitions)\n",
    "            \n",
    "            if impossible_only:\n",
    "                \n",
    "                impossible_acquisitions = base_df[base_df.mean_conf < MEDIUM_THRESH]\n",
    "                acquisitions = acquisitions[acquisitions.ID.isin(impossible_acquisitions.id)]\n",
    "            \n",
    "            labels.append(number_of_examples)\n",
    "            \n",
    "            data_counts = acquisitions.Dataset.value_counts()\n",
    "            \n",
    "            for dataset, value in zip(data_counts.index, data_counts):\n",
    "                results[dataset].append(value)\n",
    "            \n",
    "            for dataset in list(results.keys()):\n",
    "                \n",
    "                if dataset not in data_counts.index:\n",
    "                    \n",
    "                    results[dataset].append(0)\n",
    "        \n",
    "        for key in list(total_counts.keys()):\n",
    "            total_counts[key].append(results[key])\n",
    "        \n",
    "    mean_and_std = {'SNLI':[], 'ANLI':[], 'WANLI':[]}\n",
    "    \n",
    "    for key in list(results.keys()):\n",
    "        mean_and_std[key].append(np.mean(total_counts[key], axis=0))\n",
    "        mean_and_std[key].append(np.std(total_counts[key], axis=0))\n",
    "        \n",
    "    def get_mean_and_std_e(array):\n",
    "        \n",
    "        return np.mean(array, axis=0), np.std(array, axis=0) \n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "    \n",
    "    if impossible_only:\n",
    "        ax.set_ylim(0, 250)\n",
    "    else:\n",
    "        ax.set_ylim(0, 510)\n",
    "        \n",
    "    width = 210\n",
    "    alpha = ALPHA\n",
    "    labels = np.array(labels)\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(_CMAP)\n",
    "    \n",
    "    ax.bar(labels, mean_and_std['SNLI'][0], width, yerr=mean_and_std['SNLI'][1], label='SNLI', color=cmap(0.85), alpha=alpha)\n",
    "    ax.bar(labels, mean_and_std['ANLI'][0], width, yerr=mean_and_std['ANLI'][1], bottom=mean_and_std['SNLI'][0],  label='ANLI', color=cmap(0.65), alpha=alpha)\n",
    "    ax.bar(labels, mean_and_std['WANLI'][0], width, yerr=mean_and_std['WANLI'][1], bottom=mean_and_std['SNLI'][0] + mean_and_std['ANLI'][0], label='WANLI', color=cmap(0.35), alpha=alpha)\n",
    "\n",
    "    ax.set_xticks(ax.get_xticks()[:])\n",
    "#     ax.set_ylabel('Acquisition by Dataset')\n",
    "    ax.set_xlabel('Labeled Ex.')\n",
    "    \n",
    "    if not impossible_only:     \n",
    "        ax.set_title(strategy_df['Name'].tolist()[0])\n",
    "        \n",
    "    else:\n",
    "        ax.set_title(strategy_df['Name'].tolist()[0])\n",
    "    \n",
    "    if show_legend:\n",
    "        ax.legend(loc='lower right')\n",
    "    \n",
    "    if return_axs:\n",
    "        \n",
    "        return ax\n",
    "    \n",
    "    else:\n",
    "        \n",
    "        if impossible_only:\n",
    "            impossible_str = '_impossible_'\n",
    "        else:\n",
    "            impossible_str = ''\n",
    "\n",
    "\n",
    "        \n",
    "        plt.plot(stacked=True)\n",
    "        name = strategy_df['Name'].tolist()[0]\n",
    "        plt.savefig('acquisitions_by_data_{}{}.pdf'.format(name,impossible_str), format='pdf')\n",
    "#         plt.tight_layout()\n",
    "        plt.show()\n",
    "        \n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "stock-forestry",
   "metadata": {},
   "outputs": [],
   "source": [
    "strategy_ids = ['random', 'mc-max-entropy', 'dal', 'bald']\n",
    "\n",
    "strategies_list = []\n",
    "for strategy_id in strategy_ids:\n",
    "    \n",
    "    strategy_list = []\n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        strategy_path = STRATEGY_PATH.replace('SEED', str(seed)).replace('STRATEGY', str(strategy_id))\n",
    "        strategy_df = load_strategy_acquisitions(strategy=strategy_id, path=strategy_path) \n",
    "        strategy_df.ID = strategy_df.ID.astype('str')\n",
    "        \n",
    "        strategy_list.append(strategy_df)\n",
    "    \n",
    "    strategies_list.append(strategy_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "qualified-stations",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dfs = [load_confidences(BASE_PATH.replace('SEED', str(seed)), predictions=False) for seed in SEEDS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "thorough-wound",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 29820 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 14688 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 12080 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 12468 characters]\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "SIZE = 4.5\n",
    "FONTSIZE = 17.5\n",
    "DPI = 80\n",
    "\n",
    "SHOW_LEGEND=True\n",
    "axs = []\n",
    "for strategy_df in strategies_list:\n",
    "     generate_stacked_barchart(base_dfs=base_dfs, strategy_dfs=strategy_df, return_axs=False, size=SIZE, font_size=FONTSIZE, dpi=DPI, show_legend=SHOW_LEGEND)\n",
    "#      generate_stacked_barchart_data(base_dfs=base_dfs, strategy_dfs=strategy_df, impossible_only=False, return_axs=False, size=SIZE, font_size=FONTSIZE, dpi=DPI, show_legend=SHOW_LEGEND)\n",
    "#      generate_stacked_barchart_data(base_dfs=base_dfs, strategy_dfs=strategy_df, impossible_only=True, return_axs=False, size=SIZE, font_size=FONTSIZE, dpi=DPI, show_legend=SHOW_LEGEND)\n",
    "     SHOW_LEGEND=False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "known-lighter",
   "metadata": {},
   "source": [
    "# Examples of various difficulties:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "sitting-theory",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "ranking-guatemala",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Impossible examples:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_374c12b4_a792_11ed_8264_acbc32902497 th {\n",
       "          text-align: left;\n",
       "    }#T_374c12b4_a792_11ed_8264_acbc32902497row0_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row0_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row0_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row0_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row0_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row1_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row1_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row1_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row1_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row1_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row2_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row2_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row2_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row2_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row2_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row3_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row3_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row3_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row3_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row3_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row4_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row4_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row4_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row4_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row4_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row5_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row5_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row5_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row5_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row5_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row6_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row6_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row6_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row6_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row6_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row7_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row7_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row7_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row7_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row7_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row8_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row8_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row8_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row8_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row8_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row9_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row9_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row9_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row9_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row9_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row10_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row10_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row10_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row10_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row10_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row11_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row11_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row11_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row11_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row11_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row12_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row12_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row12_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row12_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row12_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row13_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row13_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row13_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row13_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row13_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row14_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row14_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row14_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row14_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row14_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row15_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row15_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row15_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row15_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row15_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row16_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row16_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row16_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row16_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row16_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row17_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row17_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row17_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row17_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row17_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row18_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row18_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row18_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row18_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row18_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row19_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row19_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row19_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row19_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row19_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row20_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row20_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row20_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row20_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row20_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row21_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row21_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row21_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row21_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row21_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row22_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row22_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row22_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row22_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row22_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row23_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row23_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row23_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row23_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row23_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row24_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row24_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row24_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row24_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row24_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row25_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row25_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row25_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row25_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row25_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row26_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row26_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row26_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row26_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row26_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row27_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row27_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row27_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row27_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row27_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row28_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row28_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row28_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row28_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row28_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row29_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row29_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row29_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row29_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row29_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row30_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row30_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row30_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row30_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row30_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row31_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row31_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row31_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row31_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row31_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row32_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row32_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row32_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row32_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row32_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row33_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row33_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row33_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row33_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row33_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row34_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row34_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row34_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row34_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row34_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row35_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row35_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row35_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row35_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row35_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row36_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row36_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row36_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row36_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row36_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row37_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row37_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row37_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row37_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row37_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row38_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row38_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row38_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row38_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row38_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row39_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row39_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row39_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row39_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row39_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row40_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row40_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row40_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row40_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row40_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row41_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row41_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row41_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row41_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row41_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row42_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row42_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row42_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row42_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row42_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row43_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row43_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row43_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row43_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row43_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row44_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row44_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row44_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row44_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row44_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row45_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row45_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row45_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row45_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row45_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row46_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row46_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row46_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row46_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row46_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row47_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row47_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row47_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row47_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row47_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row48_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row48_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row48_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row48_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row48_col4,#T_374c12b4_a792_11ed_8264_acbc32902497row49_col0,#T_374c12b4_a792_11ed_8264_acbc32902497row49_col1,#T_374c12b4_a792_11ed_8264_acbc32902497row49_col2,#T_374c12b4_a792_11ed_8264_acbc32902497row49_col3,#T_374c12b4_a792_11ed_8264_acbc32902497row49_col4{\n",
       "            text-align:  left;\n",
       "        }</style><table id=\"T_374c12b4_a792_11ed_8264_acbc32902497\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Premise</th>        <th class=\"col_heading level0 col1\" >Hypothesis</th>        <th class=\"col_heading level0 col2\" >Label</th>        <th class=\"col_heading level0 col3\" >Model Prediction</th>        <th class=\"col_heading level0 col4\" >Model Confidence</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row0\" class=\"row_heading level0 row0\" >111</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row0_col0\" class=\"data row0 col0\" >A skier in electric green\n",
       "on the edge of a ramp\n",
       "made of metal bars.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row0_col1\" class=\"data row0 col1\" >The skier was on the edge\n",
       "of the ramp.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row0_col2\" class=\"data row0 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row0_col3\" class=\"data row0 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row0_col4\" class=\"data row0 col4\" >0.976000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row1\" class=\"row_heading level0 row1\" >112</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row1_col0\" class=\"data row1 col0\" >A skier in electric green\n",
       "on the edge of a ramp\n",
       "made of metal bars.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row1_col1\" class=\"data row1 col1\" >The brightly dressed\n",
       "skier slid down the race\n",
       "course.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row1_col2\" class=\"data row1 col2\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row1_col3\" class=\"data row1 col3\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row1_col4\" class=\"data row1 col4\" >0.770000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row2\" class=\"row_heading level0 row2\" >1070</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row2_col0\" class=\"data row2 col0\" >A man wearing a red\n",
       "sweater is sitting on a\n",
       "car bumper watching\n",
       "another person work.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row2_col1\" class=\"data row2 col1\" >people make speed fast at\n",
       "speed breaker.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row2_col2\" class=\"data row2 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row2_col3\" class=\"data row2 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row2_col4\" class=\"data row2 col4\" >0.799000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row3\" class=\"row_heading level0 row3\" >1584</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row3_col0\" class=\"data row3 col0\" >A blond child is pulling\n",
       "a wagon with a little\n",
       "blond boy in it.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row3_col1\" class=\"data row3 col1\" >A child is pulling a toy\n",
       "wagon.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row3_col2\" class=\"data row3 col2\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row3_col3\" class=\"data row3 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row3_col4\" class=\"data row3 col4\" >0.988000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row4\" class=\"row_heading level0 row4\" >2188</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row4_col0\" class=\"data row4 col0\" >Woman sitting in tree\n",
       "with dove.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row4_col1\" class=\"data row4 col1\" >The lady is touching a\n",
       "dove.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row4_col2\" class=\"data row4 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row4_col3\" class=\"data row4 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row4_col4\" class=\"data row4 col4\" >0.920000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row5\" class=\"row_heading level0 row5\" >2279</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row5_col0\" class=\"data row5 col0\" >A lady standing on the\n",
       "corner using her phone.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row5_col1\" class=\"data row5 col1\" >The lady has a smartphone</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row5_col2\" class=\"data row5 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row5_col3\" class=\"data row5 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row5_col4\" class=\"data row5 col4\" >0.753000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row6\" class=\"row_heading level0 row6\" >2367</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row6_col0\" class=\"data row6 col0\" >Young boy jogging through\n",
       "the forest.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row6_col1\" class=\"data row6 col1\" >Little Timmy was chased\n",
       "by goblins through the\n",
       "forest</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row6_col2\" class=\"data row6 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row6_col3\" class=\"data row6 col3\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row6_col4\" class=\"data row6 col4\" >0.697000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row7\" class=\"row_heading level0 row7\" >2950</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row7_col0\" class=\"data row7 col0\" >A woman in a pool looking\n",
       "down.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row7_col1\" class=\"data row7 col1\" >A woman is looking down\n",
       "into a pool of tears\n",
       "preparing to dive into\n",
       "them.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row7_col2\" class=\"data row7 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row7_col3\" class=\"data row7 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row7_col4\" class=\"data row7 col4\" >0.952000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row8\" class=\"row_heading level0 row8\" >3603</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row8_col0\" class=\"data row8 col0\" >Fellows writing on the\n",
       "back of a friend with\n",
       "black markers.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row8_col1\" class=\"data row8 col1\" >People are bullying a kid\n",
       "by writing cruel things\n",
       "on his back with\n",
       "permanant marker.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row8_col2\" class=\"data row8 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row8_col3\" class=\"data row8 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row8_col4\" class=\"data row8 col4\" >0.971000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row9\" class=\"row_heading level0 row9\" >4255</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row9_col0\" class=\"data row9 col0\" >Man with orange dyed hair\n",
       "playing an electric\n",
       "guitar.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row9_col1\" class=\"data row9 col1\" >The man is on stage.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row9_col2\" class=\"data row9 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row9_col3\" class=\"data row9 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row9_col4\" class=\"data row9 col4\" >0.724000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row10\" class=\"row_heading level0 row10\" >4274</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row10_col0\" class=\"data row10 col0\" >Man sitting in a beached\n",
       "canoe by a lake.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row10_col1\" class=\"data row10 col1\" >a man is sitting outside</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row10_col2\" class=\"data row10 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row10_col3\" class=\"data row10 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row10_col4\" class=\"data row10 col4\" >0.980000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row11\" class=\"row_heading level0 row11\" >4564</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row11_col0\" class=\"data row11 col0\" >Many cabs lining up at a\n",
       "station close to an\n",
       "elevator.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row11_col1\" class=\"data row11 col1\" >Taxis line up at a\n",
       "station.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row11_col2\" class=\"data row11 col2\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row11_col3\" class=\"data row11 col3\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row11_col4\" class=\"data row11 col4\" >0.687000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row12\" class=\"row_heading level0 row12\" >5257</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row12_col0\" class=\"data row12 col0\" >A train conductor looking\n",
       "out the window of a black\n",
       "train while the worker\n",
       "looks at the gauges.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row12_col1\" class=\"data row12 col1\" >The conductor is looking\n",
       "out the window at the\n",
       "train.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row12_col2\" class=\"data row12 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row12_col3\" class=\"data row12 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row12_col4\" class=\"data row12 col4\" >0.955000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row13\" class=\"row_heading level0 row13\" >5367</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row13_col0\" class=\"data row13 col0\" >Four children climbing up\n",
       "a snow covered hill.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row13_col1\" class=\"data row13 col1\" >Four children climb up an\n",
       "Icy palace in the\n",
       "Norwegian countryside</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row13_col2\" class=\"data row13 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row13_col3\" class=\"data row13 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row13_col4\" class=\"data row13 col4\" >0.875000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row14\" class=\"row_heading level0 row14\" >5368</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row14_col0\" class=\"data row14 col0\" >Four children climbing up\n",
       "a snow covered hill.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row14_col1\" class=\"data row14 col1\" >Four children are\n",
       "climbing in the snow of a\n",
       "covered hill</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row14_col2\" class=\"data row14 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row14_col3\" class=\"data row14 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row14_col4\" class=\"data row14 col4\" >0.903000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row15\" class=\"row_heading level0 row15\" >7246</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row15_col0\" class=\"data row15 col0\" >A motorcyclist drives up\n",
       "the side of a green\n",
       "mountain while another\n",
       "bike remains stationary.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row15_col1\" class=\"data row15 col1\" >One Biker chases another\n",
       "on a Mountain Road.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row15_col2\" class=\"data row15 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row15_col3\" class=\"data row15 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row15_col4\" class=\"data row15 col4\" >0.813000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row16\" class=\"row_heading level0 row16\" >7585</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row16_col0\" class=\"data row16 col0\" >Two modern buildings made\n",
       "of glass and stone accent\n",
       "the setting.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row16_col1\" class=\"data row16 col1\" >Two modern buildings\n",
       "complete the setting.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row16_col2\" class=\"data row16 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row16_col3\" class=\"data row16 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row16_col4\" class=\"data row16 col4\" >0.756000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row17\" class=\"row_heading level0 row17\" >7743</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row17_col0\" class=\"data row17 col0\" >A young girl jumps off of\n",
       "a couch and high into the\n",
       "air.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row17_col1\" class=\"data row17 col1\" >the young lady knows how\n",
       "to fly in sky</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row17_col2\" class=\"data row17 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row17_col3\" class=\"data row17 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row17_col4\" class=\"data row17 col4\" >0.765000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row18\" class=\"row_heading level0 row18\" >8170</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row18_col0\" class=\"data row18 col0\" >A old man blowing bubbles\n",
       "in the park.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row18_col1\" class=\"data row18 col1\" >An old man is ourdoors.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row18_col2\" class=\"data row18 col2\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row18_col3\" class=\"data row18 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row18_col4\" class=\"data row18 col4\" >0.957000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row19\" class=\"row_heading level0 row19\" >8372</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row19_col0\" class=\"data row19 col0\" >A woman is taking a\n",
       "picture.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row19_col1\" class=\"data row19 col1\" >A woman holds a camera.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row19_col2\" class=\"data row19 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row19_col3\" class=\"data row19 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row19_col4\" class=\"data row19 col4\" >0.971000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row20\" class=\"row_heading level0 row20\" >8865</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row20_col0\" class=\"data row20 col0\" >Two rickshaw operators\n",
       "waiting for a fare with\n",
       "traffic in the\n",
       "background.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row20_col1\" class=\"data row20 col1\" >Two rickshaw operators\n",
       "wait for a fair.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row20_col2\" class=\"data row20 col2\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row20_col3\" class=\"data row20 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row20_col4\" class=\"data row20 col4\" >0.954000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row21\" class=\"row_heading level0 row21\" >9202</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row21_col0\" class=\"data row21 col0\" >A woman is rollerblading\n",
       "as she hangs out with\n",
       "another woman.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row21_col1\" class=\"data row21 col1\" >Two women rollerblading\n",
       "together.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row21_col2\" class=\"data row21 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row21_col3\" class=\"data row21 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row21_col4\" class=\"data row21 col4\" >0.793000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row22\" class=\"row_heading level0 row22\" >9303</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row22_col0\" class=\"data row22 col0\" >The rowdy bull spits as\n",
       "the men attempt to lasso\n",
       "it.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row22_col1\" class=\"data row22 col1\" >The bull is angry.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row22_col2\" class=\"data row22 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row22_col3\" class=\"data row22 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row22_col4\" class=\"data row22 col4\" >0.874000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row23\" class=\"row_heading level0 row23\" >9724</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row23_col0\" class=\"data row23 col0\" >Several young people sit\n",
       "at a table playing poker.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row23_col1\" class=\"data row23 col1\" >Some young guys are\n",
       "hanging out playing\n",
       "cards.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row23_col2\" class=\"data row23 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row23_col3\" class=\"data row23 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row23_col4\" class=\"data row23 col4\" >0.870000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row24\" class=\"row_heading level0 row24\" >10889</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row24_col0\" class=\"data row24 col0\" >A man in a suit looking\n",
       "down at a podium in a\n",
       "dinning restaurant.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row24_col1\" class=\"data row24 col1\" >Went to dinner at a\n",
       "restaurant the other day.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row24_col2\" class=\"data row24 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row24_col3\" class=\"data row24 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row24_col4\" class=\"data row24 col4\" >0.815000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row25\" class=\"row_heading level0 row25\" >10903</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row25_col0\" class=\"data row25 col0\" >Two girls squatting with\n",
       "their backs turned on a\n",
       "beach, examining\n",
       "something that they have\n",
       "found.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row25_col1\" class=\"data row25 col1\" >Two girls on a beach are\n",
       "looking at tomatoes.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row25_col2\" class=\"data row25 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row25_col3\" class=\"data row25 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row25_col4\" class=\"data row25 col4\" >0.984000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row26\" class=\"row_heading level0 row26\" >11665</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row26_col0\" class=\"data row26 col0\" >A man wearing a jacket\n",
       "and hat rides his bicycle\n",
       "among bright rays of\n",
       "sunshine that is\n",
       "streaming between an\n",
       "array of Autumn leafs.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row26_col1\" class=\"data row26 col1\" >A bike rides a man.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row26_col2\" class=\"data row26 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row26_col3\" class=\"data row26 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row26_col4\" class=\"data row26 col4\" >0.968000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row27\" class=\"row_heading level0 row27\" >11761</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row27_col0\" class=\"data row27 col0\" >Middle-aged female\n",
       "wearing a white sunhat\n",
       "and white jacket, slips\n",
       "her hand inside a man's\n",
       "pants pocket.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row27_col1\" class=\"data row27 col1\" >The man and woman are\n",
       "playing together.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row27_col2\" class=\"data row27 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row27_col3\" class=\"data row27 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row27_col4\" class=\"data row27 col4\" >0.869000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row28\" class=\"row_heading level0 row28\" >12187</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row28_col0\" class=\"data row28 col0\" >Three people, two adult\n",
       "women and a young girl,\n",
       "talk about a clock the\n",
       "girl is holding.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row28_col1\" class=\"data row28 col1\" >The women are the same\n",
       "age.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row28_col2\" class=\"data row28 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row28_col3\" class=\"data row28 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row28_col4\" class=\"data row28 col4\" >0.690000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row29\" class=\"row_heading level0 row29\" >12233</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row29_col0\" class=\"data row29 col0\" >A construction crew is\n",
       "waiting around with\n",
       "yellow and orange\n",
       "jackets.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row29_col1\" class=\"data row29 col1\" >The crew is working hard.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row29_col2\" class=\"data row29 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row29_col3\" class=\"data row29 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row29_col4\" class=\"data row29 col4\" >0.892000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row30\" class=\"row_heading level0 row30\" >12583</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row30_col0\" class=\"data row30 col0\" >A group of boys and girls\n",
       "sit and talk together\n",
       "under a tree</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row30_col1\" class=\"data row30 col1\" >kids are sitting near a\n",
       "tree</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row30_col2\" class=\"data row30 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row30_col3\" class=\"data row30 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row30_col4\" class=\"data row30 col4\" >0.928000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row31\" class=\"row_heading level0 row31\" >13268</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row31_col0\" class=\"data row31 col0\" >A tennis player wearing\n",
       "white is jumping up to\n",
       "hit a ball.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row31_col1\" class=\"data row31 col1\" >A man is playing tennis.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row31_col2\" class=\"data row31 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row31_col3\" class=\"data row31 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row31_col4\" class=\"data row31 col4\" >0.926000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row32\" class=\"row_heading level0 row32\" >13522</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row32_col0\" class=\"data row32 col0\" >Young girl wearing gray\n",
       "and pink sweater and\n",
       "leather boots shields her\n",
       "face from the camera.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row32_col1\" class=\"data row32 col1\" >Young girl is in the\n",
       "middle of a photoshoot.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row32_col2\" class=\"data row32 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row32_col3\" class=\"data row32 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row32_col4\" class=\"data row32 col4\" >0.975000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row33\" class=\"row_heading level0 row33\" >13635</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row33_col0\" class=\"data row33 col0\" >Customers are waiting for\n",
       "service at a takeout\n",
       "window.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row33_col1\" class=\"data row33 col1\" >People are standing in\n",
       "line for food.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row33_col2\" class=\"data row33 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row33_col3\" class=\"data row33 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row33_col4\" class=\"data row33 col4\" >0.935000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row34\" class=\"row_heading level0 row34\" >13928</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row34_col0\" class=\"data row34 col0\" >An old man holding a\n",
       "large stick.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row34_col1\" class=\"data row34 col1\" >A old guy is clapping.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row34_col2\" class=\"data row34 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row34_col3\" class=\"data row34 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row34_col4\" class=\"data row34 col4\" >0.742000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row35\" class=\"row_heading level0 row35\" >13930</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row35_col0\" class=\"data row35 col0\" >A woman taking her wallet\n",
       "out of her purse at a\n",
       "vendor stand.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row35_col1\" class=\"data row35 col1\" >A woman buying something\n",
       "from a vendor.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row35_col2\" class=\"data row35 col2\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row35_col3\" class=\"data row35 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row35_col4\" class=\"data row35 col4\" >0.892000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row36\" class=\"row_heading level0 row36\" >13965</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row36_col0\" class=\"data row36 col0\" >The man is throwing a\n",
       "tennis ball for the brown\n",
       "dog to fetch.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row36_col1\" class=\"data row36 col1\" >A dog is chasing a ball\n",
       "around.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row36_col2\" class=\"data row36 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row36_col3\" class=\"data row36 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row36_col4\" class=\"data row36 col4\" >0.901000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row37\" class=\"row_heading level0 row37\" >13980</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row37_col0\" class=\"data row37 col0\" >A British concert band\n",
       "seated and playing under\n",
       "a canvas awning across\n",
       "from a crowd of\n",
       "spectators gathered on\n",
       "the lawn.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row37_col1\" class=\"data row37 col1\" >Members of the audience\n",
       "form a mosh pit after\n",
       "being energized by the\n",
       "rock band's performance.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row37_col2\" class=\"data row37 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row37_col3\" class=\"data row37 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row37_col4\" class=\"data row37 col4\" >0.974000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row38\" class=\"row_heading level0 row38\" >14273</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row38_col0\" class=\"data row38 col0\" >A shoe shiner waits for\n",
       "customers.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row38_col1\" class=\"data row38 col1\" >The shoe shiner is busy</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row38_col2\" class=\"data row38 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row38_col3\" class=\"data row38 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row38_col4\" class=\"data row38 col4\" >0.755000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row39\" class=\"row_heading level0 row39\" >15238</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row39_col0\" class=\"data row39 col0\" >A young boy, wearing\n",
       "suspenders and a red\n",
       "flannel shirt, hits a\n",
       "piece of metal on an\n",
       "anvil.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row39_col1\" class=\"data row39 col1\" >A boy is making something</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row39_col2\" class=\"data row39 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row39_col3\" class=\"data row39 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row39_col4\" class=\"data row39 col4\" >0.847000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row40\" class=\"row_heading level0 row40\" >17011</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row40_col0\" class=\"data row40 col0\" >A group of people\n",
       "standing on a rock path.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row40_col1\" class=\"data row40 col1\" >A group of people are\n",
       "hiking.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row40_col2\" class=\"data row40 col2\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row40_col3\" class=\"data row40 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row40_col4\" class=\"data row40 col4\" >0.972000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row41\" class=\"row_heading level0 row41\" >17276</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row41_col0\" class=\"data row41 col0\" >A woman in a tan tunic\n",
       "leans against a gate.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row41_col1\" class=\"data row41 col1\" >A woman is standing\n",
       "outside.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row41_col2\" class=\"data row41 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row41_col3\" class=\"data row41 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row41_col4\" class=\"data row41 col4\" >0.966000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row42\" class=\"row_heading level0 row42\" >17732</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row42_col0\" class=\"data row42 col0\" >A crowd is assembled and\n",
       "cheering for an on stage\n",
       "performance.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row42_col1\" class=\"data row42 col1\" >A crowd waits for a\n",
       "performance to start.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row42_col2\" class=\"data row42 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row42_col3\" class=\"data row42 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row42_col4\" class=\"data row42 col4\" >0.513000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row43\" class=\"row_heading level0 row43\" >17960</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row43_col0\" class=\"data row43 col0\" >A young boy jumps into\n",
       "the oncoming wave.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row43_col1\" class=\"data row43 col1\" >The boy is at a lake.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row43_col2\" class=\"data row43 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row43_col3\" class=\"data row43 col3\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row43_col4\" class=\"data row43 col4\" >0.951000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row44\" class=\"row_heading level0 row44\" >20268</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row44_col0\" class=\"data row44 col0\" >A group of men are\n",
       "planting a tree.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row44_col1\" class=\"data row44 col1\" >There are some female\n",
       "humans planting a tree.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row44_col2\" class=\"data row44 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row44_col3\" class=\"data row44 col3\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row44_col4\" class=\"data row44 col4\" >0.940000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row45\" class=\"row_heading level0 row45\" >20791</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row45_col0\" class=\"data row45 col0\" >A group of people are\n",
       "running in the streets in\n",
       "their under clothes.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row45_col1\" class=\"data row45 col1\" >A group od people are\n",
       "runnig their\n",
       "underrclithes.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row45_col2\" class=\"data row45 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row45_col3\" class=\"data row45 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row45_col4\" class=\"data row45 col4\" >0.973000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row46\" class=\"row_heading level0 row46\" >21104</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row46_col0\" class=\"data row46 col0\" >A person wearing a white\n",
       "helmet races in a black\n",
       "race car.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row46_col1\" class=\"data row46 col1\" >A person wearing a black\n",
       "helmet races in their\n",
       "car.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row46_col2\" class=\"data row46 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row46_col3\" class=\"data row46 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row46_col4\" class=\"data row46 col4\" >0.722000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row47\" class=\"row_heading level0 row47\" >21282</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row47_col0\" class=\"data row47 col0\" >A group of children have\n",
       "fun in sack race.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row47_col1\" class=\"data row47 col1\" >Some childrens are\n",
       "playing a sack game</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row47_col2\" class=\"data row47 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row47_col3\" class=\"data row47 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row47_col4\" class=\"data row47 col4\" >0.896000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row48\" class=\"row_heading level0 row48\" >21482</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row48_col0\" class=\"data row48 col0\" >A woman in pink is\n",
       "covering her face with a\n",
       "scarf.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row48_col1\" class=\"data row48 col1\" >A woman puts a scarf on\n",
       "the girl.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row48_col2\" class=\"data row48 col2\" >contradiction</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row48_col3\" class=\"data row48 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row48_col4\" class=\"data row48 col4\" >0.785000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_374c12b4_a792_11ed_8264_acbc32902497level0_row49\" class=\"row_heading level0 row49\" >21620</th>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row49_col0\" class=\"data row49 col0\" >A bird, wings\n",
       "outstreached, over the\n",
       "water.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row49_col1\" class=\"data row49 col1\" >The water has a bird.</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row49_col2\" class=\"data row49 col2\" >neutral</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row49_col3\" class=\"data row49 col3\" >entailment</td>\n",
       "                        <td id=\"T_374c12b4_a792_11ed_8264_acbc32902497row49_col4\" class=\"data row49 col4\" >0.929000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fe439e70438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Impossible examples:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_375a331c_a792_11ed_8264_acbc32902497 th {\n",
       "          text-align: left;\n",
       "    }#T_375a331c_a792_11ed_8264_acbc32902497row0_col0,#T_375a331c_a792_11ed_8264_acbc32902497row0_col1,#T_375a331c_a792_11ed_8264_acbc32902497row0_col2,#T_375a331c_a792_11ed_8264_acbc32902497row0_col3,#T_375a331c_a792_11ed_8264_acbc32902497row0_col4,#T_375a331c_a792_11ed_8264_acbc32902497row1_col0,#T_375a331c_a792_11ed_8264_acbc32902497row1_col1,#T_375a331c_a792_11ed_8264_acbc32902497row1_col2,#T_375a331c_a792_11ed_8264_acbc32902497row1_col3,#T_375a331c_a792_11ed_8264_acbc32902497row1_col4,#T_375a331c_a792_11ed_8264_acbc32902497row2_col0,#T_375a331c_a792_11ed_8264_acbc32902497row2_col1,#T_375a331c_a792_11ed_8264_acbc32902497row2_col2,#T_375a331c_a792_11ed_8264_acbc32902497row2_col3,#T_375a331c_a792_11ed_8264_acbc32902497row2_col4,#T_375a331c_a792_11ed_8264_acbc32902497row3_col0,#T_375a331c_a792_11ed_8264_acbc32902497row3_col1,#T_375a331c_a792_11ed_8264_acbc32902497row3_col2,#T_375a331c_a792_11ed_8264_acbc32902497row3_col3,#T_375a331c_a792_11ed_8264_acbc32902497row3_col4,#T_375a331c_a792_11ed_8264_acbc32902497row4_col0,#T_375a331c_a792_11ed_8264_acbc32902497row4_col1,#T_375a331c_a792_11ed_8264_acbc32902497row4_col2,#T_375a331c_a792_11ed_8264_acbc32902497row4_col3,#T_375a331c_a792_11ed_8264_acbc32902497row4_col4,#T_375a331c_a792_11ed_8264_acbc32902497row5_col0,#T_375a331c_a792_11ed_8264_acbc32902497row5_col1,#T_375a331c_a792_11ed_8264_acbc32902497row5_col2,#T_375a331c_a792_11ed_8264_acbc32902497row5_col3,#T_375a331c_a792_11ed_8264_acbc32902497row5_col4,#T_375a331c_a792_11ed_8264_acbc32902497row6_col0,#T_375a331c_a792_11ed_8264_acbc32902497row6_col1,#T_375a331c_a792_11ed_8264_acbc32902497row6_col2,#T_375a331c_a792_11ed_8264_acbc32902497row6_col3,#T_375a331c_a792_11ed_8264_acbc32902497row6_col4,#T_375a331c_a792_11ed_8264_acbc32902497row7_col0,#T_375a331c_a792_11ed_8264_acbc32902497row7_col1,#T_375a331c_a792_11ed_8264_acbc32902497row7_col2,#T_375a331c_a792_11ed_8264_acbc32902497row7_col3,#T_375a331c_a792_11ed_8264_acbc32902497row7_col4,#T_375a331c_a792_11ed_8264_acbc32902497row8_col0,#T_375a331c_a792_11ed_8264_acbc32902497row8_col1,#T_375a331c_a792_11ed_8264_acbc32902497row8_col2,#T_375a331c_a792_11ed_8264_acbc32902497row8_col3,#T_375a331c_a792_11ed_8264_acbc32902497row8_col4,#T_375a331c_a792_11ed_8264_acbc32902497row9_col0,#T_375a331c_a792_11ed_8264_acbc32902497row9_col1,#T_375a331c_a792_11ed_8264_acbc32902497row9_col2,#T_375a331c_a792_11ed_8264_acbc32902497row9_col3,#T_375a331c_a792_11ed_8264_acbc32902497row9_col4,#T_375a331c_a792_11ed_8264_acbc32902497row10_col0,#T_375a331c_a792_11ed_8264_acbc32902497row10_col1,#T_375a331c_a792_11ed_8264_acbc32902497row10_col2,#T_375a331c_a792_11ed_8264_acbc32902497row10_col3,#T_375a331c_a792_11ed_8264_acbc32902497row10_col4,#T_375a331c_a792_11ed_8264_acbc32902497row11_col0,#T_375a331c_a792_11ed_8264_acbc32902497row11_col1,#T_375a331c_a792_11ed_8264_acbc32902497row11_col2,#T_375a331c_a792_11ed_8264_acbc32902497row11_col3,#T_375a331c_a792_11ed_8264_acbc32902497row11_col4,#T_375a331c_a792_11ed_8264_acbc32902497row12_col0,#T_375a331c_a792_11ed_8264_acbc32902497row12_col1,#T_375a331c_a792_11ed_8264_acbc32902497row12_col2,#T_375a331c_a792_11ed_8264_acbc32902497row12_col3,#T_375a331c_a792_11ed_8264_acbc32902497row12_col4,#T_375a331c_a792_11ed_8264_acbc32902497row13_col0,#T_375a331c_a792_11ed_8264_acbc32902497row13_col1,#T_375a331c_a792_11ed_8264_acbc32902497row13_col2,#T_375a331c_a792_11ed_8264_acbc32902497row13_col3,#T_375a331c_a792_11ed_8264_acbc32902497row13_col4,#T_375a331c_a792_11ed_8264_acbc32902497row14_col0,#T_375a331c_a792_11ed_8264_acbc32902497row14_col1,#T_375a331c_a792_11ed_8264_acbc32902497row14_col2,#T_375a331c_a792_11ed_8264_acbc32902497row14_col3,#T_375a331c_a792_11ed_8264_acbc32902497row14_col4,#T_375a331c_a792_11ed_8264_acbc32902497row15_col0,#T_375a331c_a792_11ed_8264_acbc32902497row15_col1,#T_375a331c_a792_11ed_8264_acbc32902497row15_col2,#T_375a331c_a792_11ed_8264_acbc32902497row15_col3,#T_375a331c_a792_11ed_8264_acbc32902497row15_col4,#T_375a331c_a792_11ed_8264_acbc32902497row16_col0,#T_375a331c_a792_11ed_8264_acbc32902497row16_col1,#T_375a331c_a792_11ed_8264_acbc32902497row16_col2,#T_375a331c_a792_11ed_8264_acbc32902497row16_col3,#T_375a331c_a792_11ed_8264_acbc32902497row16_col4,#T_375a331c_a792_11ed_8264_acbc32902497row17_col0,#T_375a331c_a792_11ed_8264_acbc32902497row17_col1,#T_375a331c_a792_11ed_8264_acbc32902497row17_col2,#T_375a331c_a792_11ed_8264_acbc32902497row17_col3,#T_375a331c_a792_11ed_8264_acbc32902497row17_col4,#T_375a331c_a792_11ed_8264_acbc32902497row18_col0,#T_375a331c_a792_11ed_8264_acbc32902497row18_col1,#T_375a331c_a792_11ed_8264_acbc32902497row18_col2,#T_375a331c_a792_11ed_8264_acbc32902497row18_col3,#T_375a331c_a792_11ed_8264_acbc32902497row18_col4,#T_375a331c_a792_11ed_8264_acbc32902497row19_col0,#T_375a331c_a792_11ed_8264_acbc32902497row19_col1,#T_375a331c_a792_11ed_8264_acbc32902497row19_col2,#T_375a331c_a792_11ed_8264_acbc32902497row19_col3,#T_375a331c_a792_11ed_8264_acbc32902497row19_col4,#T_375a331c_a792_11ed_8264_acbc32902497row20_col0,#T_375a331c_a792_11ed_8264_acbc32902497row20_col1,#T_375a331c_a792_11ed_8264_acbc32902497row20_col2,#T_375a331c_a792_11ed_8264_acbc32902497row20_col3,#T_375a331c_a792_11ed_8264_acbc32902497row20_col4,#T_375a331c_a792_11ed_8264_acbc32902497row21_col0,#T_375a331c_a792_11ed_8264_acbc32902497row21_col1,#T_375a331c_a792_11ed_8264_acbc32902497row21_col2,#T_375a331c_a792_11ed_8264_acbc32902497row21_col3,#T_375a331c_a792_11ed_8264_acbc32902497row21_col4,#T_375a331c_a792_11ed_8264_acbc32902497row22_col0,#T_375a331c_a792_11ed_8264_acbc32902497row22_col1,#T_375a331c_a792_11ed_8264_acbc32902497row22_col2,#T_375a331c_a792_11ed_8264_acbc32902497row22_col3,#T_375a331c_a792_11ed_8264_acbc32902497row22_col4,#T_375a331c_a792_11ed_8264_acbc32902497row23_col0,#T_375a331c_a792_11ed_8264_acbc32902497row23_col1,#T_375a331c_a792_11ed_8264_acbc32902497row23_col2,#T_375a331c_a792_11ed_8264_acbc32902497row23_col3,#T_375a331c_a792_11ed_8264_acbc32902497row23_col4,#T_375a331c_a792_11ed_8264_acbc32902497row24_col0,#T_375a331c_a792_11ed_8264_acbc32902497row24_col1,#T_375a331c_a792_11ed_8264_acbc32902497row24_col2,#T_375a331c_a792_11ed_8264_acbc32902497row24_col3,#T_375a331c_a792_11ed_8264_acbc32902497row24_col4,#T_375a331c_a792_11ed_8264_acbc32902497row25_col0,#T_375a331c_a792_11ed_8264_acbc32902497row25_col1,#T_375a331c_a792_11ed_8264_acbc32902497row25_col2,#T_375a331c_a792_11ed_8264_acbc32902497row25_col3,#T_375a331c_a792_11ed_8264_acbc32902497row25_col4,#T_375a331c_a792_11ed_8264_acbc32902497row26_col0,#T_375a331c_a792_11ed_8264_acbc32902497row26_col1,#T_375a331c_a792_11ed_8264_acbc32902497row26_col2,#T_375a331c_a792_11ed_8264_acbc32902497row26_col3,#T_375a331c_a792_11ed_8264_acbc32902497row26_col4,#T_375a331c_a792_11ed_8264_acbc32902497row27_col0,#T_375a331c_a792_11ed_8264_acbc32902497row27_col1,#T_375a331c_a792_11ed_8264_acbc32902497row27_col2,#T_375a331c_a792_11ed_8264_acbc32902497row27_col3,#T_375a331c_a792_11ed_8264_acbc32902497row27_col4,#T_375a331c_a792_11ed_8264_acbc32902497row28_col0,#T_375a331c_a792_11ed_8264_acbc32902497row28_col1,#T_375a331c_a792_11ed_8264_acbc32902497row28_col2,#T_375a331c_a792_11ed_8264_acbc32902497row28_col3,#T_375a331c_a792_11ed_8264_acbc32902497row28_col4,#T_375a331c_a792_11ed_8264_acbc32902497row29_col0,#T_375a331c_a792_11ed_8264_acbc32902497row29_col1,#T_375a331c_a792_11ed_8264_acbc32902497row29_col2,#T_375a331c_a792_11ed_8264_acbc32902497row29_col3,#T_375a331c_a792_11ed_8264_acbc32902497row29_col4,#T_375a331c_a792_11ed_8264_acbc32902497row30_col0,#T_375a331c_a792_11ed_8264_acbc32902497row30_col1,#T_375a331c_a792_11ed_8264_acbc32902497row30_col2,#T_375a331c_a792_11ed_8264_acbc32902497row30_col3,#T_375a331c_a792_11ed_8264_acbc32902497row30_col4,#T_375a331c_a792_11ed_8264_acbc32902497row31_col0,#T_375a331c_a792_11ed_8264_acbc32902497row31_col1,#T_375a331c_a792_11ed_8264_acbc32902497row31_col2,#T_375a331c_a792_11ed_8264_acbc32902497row31_col3,#T_375a331c_a792_11ed_8264_acbc32902497row31_col4,#T_375a331c_a792_11ed_8264_acbc32902497row32_col0,#T_375a331c_a792_11ed_8264_acbc32902497row32_col1,#T_375a331c_a792_11ed_8264_acbc32902497row32_col2,#T_375a331c_a792_11ed_8264_acbc32902497row32_col3,#T_375a331c_a792_11ed_8264_acbc32902497row32_col4,#T_375a331c_a792_11ed_8264_acbc32902497row33_col0,#T_375a331c_a792_11ed_8264_acbc32902497row33_col1,#T_375a331c_a792_11ed_8264_acbc32902497row33_col2,#T_375a331c_a792_11ed_8264_acbc32902497row33_col3,#T_375a331c_a792_11ed_8264_acbc32902497row33_col4,#T_375a331c_a792_11ed_8264_acbc32902497row34_col0,#T_375a331c_a792_11ed_8264_acbc32902497row34_col1,#T_375a331c_a792_11ed_8264_acbc32902497row34_col2,#T_375a331c_a792_11ed_8264_acbc32902497row34_col3,#T_375a331c_a792_11ed_8264_acbc32902497row34_col4,#T_375a331c_a792_11ed_8264_acbc32902497row35_col0,#T_375a331c_a792_11ed_8264_acbc32902497row35_col1,#T_375a331c_a792_11ed_8264_acbc32902497row35_col2,#T_375a331c_a792_11ed_8264_acbc32902497row35_col3,#T_375a331c_a792_11ed_8264_acbc32902497row35_col4,#T_375a331c_a792_11ed_8264_acbc32902497row36_col0,#T_375a331c_a792_11ed_8264_acbc32902497row36_col1,#T_375a331c_a792_11ed_8264_acbc32902497row36_col2,#T_375a331c_a792_11ed_8264_acbc32902497row36_col3,#T_375a331c_a792_11ed_8264_acbc32902497row36_col4,#T_375a331c_a792_11ed_8264_acbc32902497row37_col0,#T_375a331c_a792_11ed_8264_acbc32902497row37_col1,#T_375a331c_a792_11ed_8264_acbc32902497row37_col2,#T_375a331c_a792_11ed_8264_acbc32902497row37_col3,#T_375a331c_a792_11ed_8264_acbc32902497row37_col4,#T_375a331c_a792_11ed_8264_acbc32902497row38_col0,#T_375a331c_a792_11ed_8264_acbc32902497row38_col1,#T_375a331c_a792_11ed_8264_acbc32902497row38_col2,#T_375a331c_a792_11ed_8264_acbc32902497row38_col3,#T_375a331c_a792_11ed_8264_acbc32902497row38_col4,#T_375a331c_a792_11ed_8264_acbc32902497row39_col0,#T_375a331c_a792_11ed_8264_acbc32902497row39_col1,#T_375a331c_a792_11ed_8264_acbc32902497row39_col2,#T_375a331c_a792_11ed_8264_acbc32902497row39_col3,#T_375a331c_a792_11ed_8264_acbc32902497row39_col4,#T_375a331c_a792_11ed_8264_acbc32902497row40_col0,#T_375a331c_a792_11ed_8264_acbc32902497row40_col1,#T_375a331c_a792_11ed_8264_acbc32902497row40_col2,#T_375a331c_a792_11ed_8264_acbc32902497row40_col3,#T_375a331c_a792_11ed_8264_acbc32902497row40_col4,#T_375a331c_a792_11ed_8264_acbc32902497row41_col0,#T_375a331c_a792_11ed_8264_acbc32902497row41_col1,#T_375a331c_a792_11ed_8264_acbc32902497row41_col2,#T_375a331c_a792_11ed_8264_acbc32902497row41_col3,#T_375a331c_a792_11ed_8264_acbc32902497row41_col4,#T_375a331c_a792_11ed_8264_acbc32902497row42_col0,#T_375a331c_a792_11ed_8264_acbc32902497row42_col1,#T_375a331c_a792_11ed_8264_acbc32902497row42_col2,#T_375a331c_a792_11ed_8264_acbc32902497row42_col3,#T_375a331c_a792_11ed_8264_acbc32902497row42_col4,#T_375a331c_a792_11ed_8264_acbc32902497row43_col0,#T_375a331c_a792_11ed_8264_acbc32902497row43_col1,#T_375a331c_a792_11ed_8264_acbc32902497row43_col2,#T_375a331c_a792_11ed_8264_acbc32902497row43_col3,#T_375a331c_a792_11ed_8264_acbc32902497row43_col4,#T_375a331c_a792_11ed_8264_acbc32902497row44_col0,#T_375a331c_a792_11ed_8264_acbc32902497row44_col1,#T_375a331c_a792_11ed_8264_acbc32902497row44_col2,#T_375a331c_a792_11ed_8264_acbc32902497row44_col3,#T_375a331c_a792_11ed_8264_acbc32902497row44_col4,#T_375a331c_a792_11ed_8264_acbc32902497row45_col0,#T_375a331c_a792_11ed_8264_acbc32902497row45_col1,#T_375a331c_a792_11ed_8264_acbc32902497row45_col2,#T_375a331c_a792_11ed_8264_acbc32902497row45_col3,#T_375a331c_a792_11ed_8264_acbc32902497row45_col4,#T_375a331c_a792_11ed_8264_acbc32902497row46_col0,#T_375a331c_a792_11ed_8264_acbc32902497row46_col1,#T_375a331c_a792_11ed_8264_acbc32902497row46_col2,#T_375a331c_a792_11ed_8264_acbc32902497row46_col3,#T_375a331c_a792_11ed_8264_acbc32902497row46_col4,#T_375a331c_a792_11ed_8264_acbc32902497row47_col0,#T_375a331c_a792_11ed_8264_acbc32902497row47_col1,#T_375a331c_a792_11ed_8264_acbc32902497row47_col2,#T_375a331c_a792_11ed_8264_acbc32902497row47_col3,#T_375a331c_a792_11ed_8264_acbc32902497row47_col4,#T_375a331c_a792_11ed_8264_acbc32902497row48_col0,#T_375a331c_a792_11ed_8264_acbc32902497row48_col1,#T_375a331c_a792_11ed_8264_acbc32902497row48_col2,#T_375a331c_a792_11ed_8264_acbc32902497row48_col3,#T_375a331c_a792_11ed_8264_acbc32902497row48_col4,#T_375a331c_a792_11ed_8264_acbc32902497row49_col0,#T_375a331c_a792_11ed_8264_acbc32902497row49_col1,#T_375a331c_a792_11ed_8264_acbc32902497row49_col2,#T_375a331c_a792_11ed_8264_acbc32902497row49_col3,#T_375a331c_a792_11ed_8264_acbc32902497row49_col4{\n",
       "            text-align:  left;\n",
       "        }</style><table id=\"T_375a331c_a792_11ed_8264_acbc32902497\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Premise</th>        <th class=\"col_heading level0 col1\" >Hypothesis</th>        <th class=\"col_heading level0 col2\" >Label</th>        <th class=\"col_heading level0 col3\" >Model Prediction</th>        <th class=\"col_heading level0 col4\" >Model Confidence</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row0\" class=\"row_heading level0 row0\" >14</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row0_col0\" class=\"data row0 col0\" >The legal difficulties\n",
       "were such that the case\n",
       "was not heard until\n",
       "January 9, 2000.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row0_col1\" class=\"data row0 col1\" >The case was so\n",
       "complicated that it took\n",
       "over a year to be heard.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row0_col2\" class=\"data row0 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row0_col3\" class=\"data row0 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row0_col4\" class=\"data row0 col4\" >0.867000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row1\" class=\"row_heading level0 row1\" >31</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row1_col0\" class=\"data row1 col0\" >We would like to see the\n",
       "two countries share more\n",
       "information on the issue\n",
       "of the smuggling of\n",
       "nuclear materials.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row1_col1\" class=\"data row1 col1\" >There are multiple ways\n",
       "to share information.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row1_col2\" class=\"data row1 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row1_col3\" class=\"data row1 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row1_col4\" class=\"data row1 col4\" >0.770000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row2\" class=\"row_heading level0 row2\" >42</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row2_col0\" class=\"data row2 col0\" >It was the only time he\n",
       "had seen her laugh.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row2_col1\" class=\"data row2 col1\" >He had never seen her\n",
       "laugh before.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row2_col2\" class=\"data row2 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row2_col3\" class=\"data row2 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row2_col4\" class=\"data row2 col4\" >0.502000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row3\" class=\"row_heading level0 row3\" >59</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row3_col0\" class=\"data row3 col0\" >It is a principle of law\n",
       "that a court will not\n",
       "accept a statement of a\n",
       "witness that is not\n",
       "supported by other\n",
       "evidence.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row3_col1\" class=\"data row3 col1\" >A witness's statement is\n",
       "not evidence.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row3_col2\" class=\"data row3 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row3_col3\" class=\"data row3 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row3_col4\" class=\"data row3 col4\" >0.597000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row4\" class=\"row_heading level0 row4\" >77</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row4_col0\" class=\"data row4 col0\" >The musician shook his\n",
       "head.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row4_col1\" class=\"data row4 col1\" >The musician moved his\n",
       "head up and down.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row4_col2\" class=\"data row4 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row4_col3\" class=\"data row4 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row4_col4\" class=\"data row4 col4\" >0.510000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row5\" class=\"row_heading level0 row5\" >98</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row5_col0\" class=\"data row5 col0\" >Would Higher Interest\n",
       "Rates Stimulate Saving?</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row5_col1\" class=\"data row5 col1\" >Higher interest rates\n",
       "would not stimulate\n",
       "saving.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row5_col2\" class=\"data row5 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row5_col3\" class=\"data row5 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row5_col4\" class=\"data row5 col4\" >0.929000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row6\" class=\"row_heading level0 row6\" >113</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row6_col0\" class=\"data row6 col0\" >In the future, human\n",
       "cloning will be legal,\n",
       "but it will be prohibited\n",
       "for personal use.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row6_col1\" class=\"data row6 col1\" >Human cloning will be\n",
       "legal for personal use.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row6_col2\" class=\"data row6 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row6_col3\" class=\"data row6 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row6_col4\" class=\"data row6 col4\" >0.769000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row7\" class=\"row_heading level0 row7\" >121</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row7_col0\" class=\"data row7 col0\" >A trial of brief\n",
       "interventions with heavy\n",
       "drinkers.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row7_col1\" class=\"data row7 col1\" >Testing how short\n",
       "interventions effect\n",
       "heavy drinkers.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row7_col2\" class=\"data row7 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row7_col3\" class=\"data row7 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row7_col4\" class=\"data row7 col4\" >0.728000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row8\" class=\"row_heading level0 row8\" >133</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row8_col0\" class=\"data row8 col0\" >Some students believe\n",
       "that to achieve their\n",
       "goals they must take the\n",
       "lead.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row8_col1\" class=\"data row8 col1\" >Some students believe\n",
       "that to achieve their\n",
       "goals they must follow\n",
       "the lead.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row8_col2\" class=\"data row8 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row8_col3\" class=\"data row8 col3\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row8_col4\" class=\"data row8 col4\" >0.630000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row9\" class=\"row_heading level0 row9\" >135</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row9_col0\" class=\"data row9 col0\" >An object is an entity\n",
       "that has a distinct\n",
       "identity and is separate\n",
       "from other objects.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row9_col1\" class=\"data row9 col1\" >The same object can be an\n",
       "entity and a part of\n",
       "another object.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row9_col2\" class=\"data row9 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row9_col3\" class=\"data row9 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row9_col4\" class=\"data row9 col4\" >0.744000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row10\" class=\"row_heading level0 row10\" >137</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row10_col0\" class=\"data row10 col0\" >The first principle of\n",
       "art is that art is not a\n",
       "way of life, but a means\n",
       "of life.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row10_col1\" class=\"data row10 col1\" >Art is a way of life.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row10_col2\" class=\"data row10 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row10_col3\" class=\"data row10 col3\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row10_col4\" class=\"data row10 col4\" >0.944000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row11\" class=\"row_heading level0 row11\" >139</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row11_col0\" class=\"data row11 col0\" >The Department of Justice\n",
       "is committed to\n",
       "protecting the civil\n",
       "rights of all Americans,\n",
       "and the Department of\n",
       "Justice is committed to\n",
       "protecting the civil\n",
       "rights of all Americans.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row11_col1\" class=\"data row11 col1\" >The Department of Justice\n",
       "is committed to\n",
       "protecting the civil\n",
       "rights of some Americans.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row11_col2\" class=\"data row11 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row11_col3\" class=\"data row11 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row11_col4\" class=\"data row11 col4\" >0.684000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row12\" class=\"row_heading level0 row12\" >196</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row12_col0\" class=\"data row12 col0\" >The second is the reason\n",
       "why the first is.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row12_col1\" class=\"data row12 col1\" >The first is the reason\n",
       "why the second is.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row12_col2\" class=\"data row12 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row12_col3\" class=\"data row12 col3\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row12_col4\" class=\"data row12 col4\" >0.440000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row13\" class=\"row_heading level0 row13\" >207</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row13_col0\" class=\"data row13 col0\" >The big, bulky one said,\n",
       "\"It's the science that\n",
       "gets me.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row13_col1\" class=\"data row13 col1\" >They were impressed with\n",
       "how it was crafted.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row13_col2\" class=\"data row13 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row13_col3\" class=\"data row13 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row13_col4\" class=\"data row13 col4\" >0.920000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row14\" class=\"row_heading level0 row14\" >214</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row14_col0\" class=\"data row14 col0\" >When I am a senior\n",
       "citizen, I want to be\n",
       "able to say, \"I am a\n",
       "senior citizen, and I am\n",
       "proud of it.\"</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row14_col1\" class=\"data row14 col1\" >She wants to be a senior\n",
       "citizen.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row14_col2\" class=\"data row14 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row14_col3\" class=\"data row14 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row14_col4\" class=\"data row14 col4\" >0.853000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row15\" class=\"row_heading level0 row15\" >221</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row15_col0\" class=\"data row15 col0\" >The very nature of the\n",
       "\"American Dream\" is that\n",
       "it is not always\n",
       "attainable.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row15_col1\" class=\"data row15 col1\" >The American Dream is\n",
       "attainable.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row15_col2\" class=\"data row15 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row15_col3\" class=\"data row15 col3\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row15_col4\" class=\"data row15 col4\" >0.817000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row16\" class=\"row_heading level0 row16\" >247</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row16_col0\" class=\"data row16 col0\" >The article, in which a\n",
       "researcher in a\n",
       "laboratory, in a cold\n",
       "room, saw a vision of a\n",
       "green-eyed woman,\n",
       "appeared in the New York\n",
       "Times.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row16_col1\" class=\"data row16 col1\" >The researcher saw a\n",
       "vision of a green-eyed\n",
       "woman in the New York\n",
       "Times.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row16_col2\" class=\"data row16 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row16_col3\" class=\"data row16 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row16_col4\" class=\"data row16 col4\" >0.912000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row17\" class=\"row_heading level0 row17\" >276</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row17_col0\" class=\"data row17 col0\" >After an appeal by the\n",
       "Center for Constitutional\n",
       "Rights, the Supreme Court\n",
       "ruled that the federal\n",
       "government could not hold\n",
       "terrorism suspects\n",
       "indefinitely without\n",
       "charging them.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row17_col1\" class=\"data row17 col1\" >The Supreme Court wanted\n",
       "to stop the government\n",
       "from holding suspects\n",
       "indefinitely.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row17_col2\" class=\"data row17 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row17_col3\" class=\"data row17 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row17_col4\" class=\"data row17 col4\" >0.861000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row18\" class=\"row_heading level0 row18\" >290</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row18_col0\" class=\"data row18 col0\" >The draft order, which\n",
       "has been circulating for\n",
       "several weeks, would\n",
       "create a \"global center\"\n",
       "for fighting terrorism,\n",
       "but it is unclear whether\n",
       "the center would be in\n",
       "the United States or\n",
       "overseas.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row18_col1\" class=\"data row18 col1\" >The draft order would\n",
       "create a \"global center\"\n",
       "for fighting terrorism.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row18_col2\" class=\"data row18 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row18_col3\" class=\"data row18 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row18_col4\" class=\"data row18 col4\" >0.903000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row19\" class=\"row_heading level0 row19\" >367</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row19_col0\" class=\"data row19 col0\" >There are several\n",
       "different approaches to\n",
       "studying and evaluating\n",
       "health care quality.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row19_col1\" class=\"data row19 col1\" >There are several\n",
       "different approaches to\n",
       "quality.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row19_col2\" class=\"data row19 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row19_col3\" class=\"data row19 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row19_col4\" class=\"data row19 col4\" >0.970000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row20\" class=\"row_heading level0 row20\" >371</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row20_col0\" class=\"data row20 col0\" >The first step in\n",
       "preparing an application\n",
       "for a grant is to\n",
       "identify the\n",
       "organization's mission\n",
       "and goals.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row20_col1\" class=\"data row20 col1\" >The mission and goals of\n",
       "the organization are\n",
       "always met.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row20_col2\" class=\"data row20 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row20_col3\" class=\"data row20 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row20_col4\" class=\"data row20 col4\" >0.796000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row21\" class=\"row_heading level0 row21\" >382</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row21_col0\" class=\"data row21 col0\" >In the long run, this\n",
       "approach will be more\n",
       "effective than the\n",
       "current approach of\n",
       "merely assessing the\n",
       "current status of each\n",
       "program and then\n",
       "evaluating it against the\n",
       "plan.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row21_col1\" class=\"data row21 col1\" >The current status of\n",
       "each program is always\n",
       "measured against the\n",
       "plan.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row21_col2\" class=\"data row21 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row21_col3\" class=\"data row21 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row21_col4\" class=\"data row21 col4\" >0.927000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row22\" class=\"row_heading level0 row22\" >383</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row22_col0\" class=\"data row22 col0\" >In addition, the court\n",
       "found that the city had a\n",
       "legitimate interest in\n",
       "reducing crime and had a\n",
       "legitimate reason for the\n",
       "ordinance.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row22_col1\" class=\"data row22 col1\" >The city had a legitimate\n",
       "interest in reducing\n",
       "crime.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row22_col2\" class=\"data row22 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row22_col3\" class=\"data row22 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row22_col4\" class=\"data row22 col4\" >0.839000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row23\" class=\"row_heading level0 row23\" >397</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row23_col0\" class=\"data row23 col0\" >On the drive to the\n",
       "house, you can stop at\n",
       "the cemetery to see the\n",
       "graves of two famous\n",
       "painters, J. M. W. Turner\n",
       "and James McNeill\n",
       "Whistler.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row23_col1\" class=\"data row23 col1\" >The graves of the two\n",
       "famous painters are worth\n",
       "seeing.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row23_col2\" class=\"data row23 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row23_col3\" class=\"data row23 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row23_col4\" class=\"data row23 col4\" >0.931000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row24\" class=\"row_heading level0 row24\" >401</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row24_col0\" class=\"data row24 col0\" >The analysis of data on\n",
       "work activities is an\n",
       "important component of\n",
       "the analysis of program\n",
       "effectiveness.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row24_col1\" class=\"data row24 col1\" >Data on work activities\n",
       "is important for\n",
       "evaluating program\n",
       "effectiveness.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row24_col2\" class=\"data row24 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row24_col3\" class=\"data row24 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row24_col4\" class=\"data row24 col4\" >0.871000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row25\" class=\"row_heading level0 row25\" >424</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row25_col0\" class=\"data row25 col0\" >In the first place, the\n",
       "fact that the mayor's\n",
       "speech was plagiarized\n",
       "from a newspaper article\n",
       "is a reason for not\n",
       "accepting it as a valid\n",
       "source.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row25_col1\" class=\"data row25 col1\" >The mayor's speech was a\n",
       "valid source.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row25_col2\" class=\"data row25 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row25_col3\" class=\"data row25 col3\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row25_col4\" class=\"data row25 col4\" >0.872000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row26\" class=\"row_heading level0 row26\" >434</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row26_col0\" class=\"data row26 col0\" >If you are to be an\n",
       "effective consultant, you\n",
       "must have a clear\n",
       "understanding of the\n",
       "process you are trying to\n",
       "influence.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row26_col1\" class=\"data row26 col1\" >The process can only be\n",
       "understood by an expert.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row26_col2\" class=\"data row26 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row26_col3\" class=\"data row26 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row26_col4\" class=\"data row26 col4\" >0.751000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row27\" class=\"row_heading level0 row27\" >452</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row27_col0\" class=\"data row27 col0\" >This led to the\n",
       "construction of the fort,\n",
       "which was completed in\n",
       "1540 to protect the town\n",
       "against the Ottoman\n",
       "threat.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row27_col1\" class=\"data row27 col1\" >This led to the\n",
       "construction of the wall,\n",
       "which was built to\n",
       "protect the Ottoman's\n",
       "from being conquered.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row27_col2\" class=\"data row27 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row27_col3\" class=\"data row27 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row27_col4\" class=\"data row27 col4\" >0.533000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row28\" class=\"row_heading level0 row28\" >467</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row28_col0\" class=\"data row28 col0\" >\"Must be right good\n",
       "stock,\" Fenner observed.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row28_col1\" class=\"data row28 col1\" >\"Must be pretty good\n",
       "stock,\" Fenner said.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row28_col2\" class=\"data row28 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row28_col3\" class=\"data row28 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row28_col4\" class=\"data row28 col4\" >0.646000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row29\" class=\"row_heading level0 row29\" >475</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row29_col0\" class=\"data row29 col0\" >The fact that the US\n",
       "Supreme Court's 5-4\n",
       "decision in \"Griswold v.\n",
       "Connecticut\" was issued\n",
       "by a justice nominated by\n",
       "a Republican president\n",
       "and confirmed by a\n",
       "Republican Senate\n",
       "suggests that the court\n",
       "is becoming more\n",
       "politically polarized.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row29_col1\" class=\"data row29 col1\" >The Supreme Court is\n",
       "becoming more politically\n",
       "polarized.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row29_col2\" class=\"data row29 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row29_col3\" class=\"data row29 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row29_col4\" class=\"data row29 col4\" >0.781000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row30\" class=\"row_heading level0 row30\" >483</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row30_col0\" class=\"data row30 col0\" >I think the fact that you\n",
       "were in the same class as\n",
       "the guy who did the\n",
       "original programming, and\n",
       "you got along with him,\n",
       "was a big factor.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row30_col1\" class=\"data row30 col1\" >I think it was because I\n",
       "was in the same class as\n",
       "the guy who did the\n",
       "original programming.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row30_col2\" class=\"data row30 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row30_col3\" class=\"data row30 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row30_col4\" class=\"data row30 col4\" >0.781000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row31\" class=\"row_heading level0 row31\" >500</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row31_col0\" class=\"data row31 col0\" >To get a good job, you\n",
       "need to be an expert in a\n",
       "certain field.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row31_col1\" class=\"data row31 col1\" >You can get a good job\n",
       "without being an expert.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row31_col2\" class=\"data row31 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row31_col3\" class=\"data row31 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row31_col4\" class=\"data row31 col4\" >0.761000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row32\" class=\"row_heading level0 row32\" >543</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row32_col0\" class=\"data row32 col0\" >After his father's death,\n",
       "Paul wrote to his mother,\n",
       "\"I feel I am being\n",
       "punished for not being\n",
       "able to give him the love\n",
       "he needed.\"</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row32_col1\" class=\"data row32 col1\" >Paul's father loved him.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row32_col2\" class=\"data row32 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row32_col3\" class=\"data row32 col3\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row32_col4\" class=\"data row32 col4\" >0.657000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row33\" class=\"row_heading level0 row33\" >605</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row33_col0\" class=\"data row33 col0\" >We do not want to go on\n",
       "the record saying that\n",
       "the solution is so simple\n",
       "that it can be written\n",
       "down on a single sheet of\n",
       "paper.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row33_col1\" class=\"data row33 col1\" >The solution is so simple\n",
       "that it can be written\n",
       "down on a single sheet of\n",
       "paper.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row33_col2\" class=\"data row33 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row33_col3\" class=\"data row33 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row33_col4\" class=\"data row33 col4\" >0.853000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row34\" class=\"row_heading level0 row34\" >655</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row34_col0\" class=\"data row34 col0\" >On the face of it, the\n",
       "English language is the\n",
       "most democratic of all\n",
       "languages.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row34_col1\" class=\"data row34 col1\" >The English language is\n",
       "the most democratic of\n",
       "all languages.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row34_col2\" class=\"data row34 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row34_col3\" class=\"data row34 col3\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row34_col4\" class=\"data row34 col4\" >0.875000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row35\" class=\"row_heading level0 row35\" >711</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row35_col0\" class=\"data row35 col0\" >He had a little pig with\n",
       "a red ribbon tied around\n",
       "its neck, and a little\n",
       "lamb with a blue ribbon\n",
       "tied around its neck.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row35_col1\" class=\"data row35 col1\" >Pig and lamb are the same\n",
       "color.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row35_col2\" class=\"data row35 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row35_col3\" class=\"data row35 col3\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row35_col4\" class=\"data row35 col4\" >0.782000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row36\" class=\"row_heading level0 row36\" >764</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row36_col0\" class=\"data row36 col0\" >She had her picture taken\n",
       "with her, her son, and\n",
       "her father, who is a\n",
       "distinguished\n",
       "businessman.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row36_col1\" class=\"data row36 col1\" >She had her picture taken\n",
       "with her, her son, and\n",
       "her father, who is a\n",
       "distinguished painter.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row36_col2\" class=\"data row36 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row36_col3\" class=\"data row36 col3\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row36_col4\" class=\"data row36 col4\" >0.889000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row37\" class=\"row_heading level0 row37\" >779</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row37_col0\" class=\"data row37 col0\" >I would like to be a part\n",
       "of the group that works\n",
       "to keep the park clean\n",
       "and beautiful.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row37_col1\" class=\"data row37 col1\" >The group that works to\n",
       "keep the park clean and\n",
       "beautiful is no longer\n",
       "around.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row37_col2\" class=\"data row37 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row37_col3\" class=\"data row37 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row37_col4\" class=\"data row37 col4\" >0.891000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row38\" class=\"row_heading level0 row38\" >795</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row38_col0\" class=\"data row38 col0\" >The table below shows the\n",
       "percent of the population\n",
       "that lives in poverty in\n",
       "each state.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row38_col1\" class=\"data row38 col1\" >All states have the same\n",
       "poverty rate.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row38_col2\" class=\"data row38 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row38_col3\" class=\"data row38 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row38_col4\" class=\"data row38 col4\" >0.955000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row39\" class=\"row_heading level0 row39\" >797</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row39_col0\" class=\"data row39 col0\" >It is not always possible\n",
       "to provide an accurate\n",
       "picture of the extent of\n",
       "fraud because of the lack\n",
       "of data.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row39_col1\" class=\"data row39 col1\" >The lack of data does not\n",
       "make it impossible to\n",
       "provide an accurate\n",
       "picture of the extent of\n",
       "fraud.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row39_col2\" class=\"data row39 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row39_col3\" class=\"data row39 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row39_col4\" class=\"data row39 col4\" >0.922000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row40\" class=\"row_heading level0 row40\" >834</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row40_col0\" class=\"data row40 col0\" >It's hard to know what to\n",
       "do with the money.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row40_col1\" class=\"data row40 col1\" >I have no idea what to do\n",
       "with the money.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row40_col2\" class=\"data row40 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row40_col3\" class=\"data row40 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row40_col4\" class=\"data row40 col4\" >0.872000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row41\" class=\"row_heading level0 row41\" >865</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row41_col0\" class=\"data row41 col0\" >This might be an issue\n",
       "for a company that is in\n",
       "the process of\n",
       "introducing a new\n",
       "product.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row41_col1\" class=\"data row41 col1\" >Every company is always\n",
       "in the process of\n",
       "introducing a new\n",
       "product.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row41_col2\" class=\"data row41 col2\" >contradiction</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row41_col3\" class=\"data row41 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row41_col4\" class=\"data row41 col4\" >0.894000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row42\" class=\"row_heading level0 row42\" >871</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row42_col0\" class=\"data row42 col0\" >'You're not supposed to\n",
       "be here,' said the\n",
       "Captain.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row42_col1\" class=\"data row42 col1\" >The captain wasn't sure\n",
       "how he would react to the\n",
       "situation.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row42_col2\" class=\"data row42 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row42_col3\" class=\"data row42 col3\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row42_col4\" class=\"data row42 col4\" >0.594000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row43\" class=\"row_heading level0 row43\" >898</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row43_col0\" class=\"data row43 col0\" >I'm not saying that the\n",
       "World Cup is the only\n",
       "event that should be\n",
       "covered by the media, but\n",
       "it is the one that is\n",
       "covered the most.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row43_col1\" class=\"data row43 col1\" >The World Cup is the one\n",
       "event that is covered the\n",
       "most.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row43_col2\" class=\"data row43 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row43_col3\" class=\"data row43 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row43_col4\" class=\"data row43 col4\" >0.943000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row44\" class=\"row_heading level0 row44\" >911</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row44_col0\" class=\"data row44 col0\" >For the most part,\n",
       "though, they can be found\n",
       "in public libraries,\n",
       "where they can be\n",
       "accessed by the public.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row44_col1\" class=\"data row44 col1\" >Some books can be found\n",
       "in public libraries.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row44_col2\" class=\"data row44 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row44_col3\" class=\"data row44 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row44_col4\" class=\"data row44 col4\" >0.925000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row45\" class=\"row_heading level0 row45\" >933</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row45_col0\" class=\"data row45 col0\" >The idea of the self is\n",
       "one of the most complex\n",
       "concepts in philosophy.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row45_col1\" class=\"data row45 col1\" >The concept of the self\n",
       "is complex.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row45_col2\" class=\"data row45 col2\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row45_col3\" class=\"data row45 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row45_col4\" class=\"data row45 col4\" >0.651000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row46\" class=\"row_heading level0 row46\" >948</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row46_col0\" class=\"data row46 col0\" >As a result, the FBI's\n",
       "ability to gather\n",
       "evidence on those charged\n",
       "was severely limited.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row46_col1\" class=\"data row46 col1\" >The FBI had limited\n",
       "resources.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row46_col2\" class=\"data row46 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row46_col3\" class=\"data row46 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row46_col4\" class=\"data row46 col4\" >0.775000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row47\" class=\"row_heading level0 row47\" >971</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row47_col0\" class=\"data row47 col0\" >For instance, some\n",
       "students have taken to\n",
       "wearing long hair, but it\n",
       "is more common for women\n",
       "to have short hair.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row47_col1\" class=\"data row47 col1\" >Women are more likely to\n",
       "have short hair than men.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row47_col2\" class=\"data row47 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row47_col3\" class=\"data row47 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row47_col4\" class=\"data row47 col4\" >0.768000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row48\" class=\"row_heading level0 row48\" >984</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row48_col0\" class=\"data row48 col0\" >In a market with a small\n",
       "number of sellers, a\n",
       "seller's willingness-to-\n",
       "pay is a good measure of\n",
       "the market price.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row48_col1\" class=\"data row48 col1\" >A seller's willingness-\n",
       "to-pay is a good measure\n",
       "of the market price.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row48_col2\" class=\"data row48 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row48_col3\" class=\"data row48 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row48_col4\" class=\"data row48 col4\" >0.843000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_375a331c_a792_11ed_8264_acbc32902497level0_row49\" class=\"row_heading level0 row49\" >1013</th>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row49_col0\" class=\"data row49 col0\" >In 1999, the Census\n",
       "Bureau estimated that 1.2\n",
       "million children were\n",
       "living in poverty.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row49_col1\" class=\"data row49 col1\" >More than 1.2 million\n",
       "children are living in\n",
       "poverty.</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row49_col2\" class=\"data row49 col2\" >neutral</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row49_col3\" class=\"data row49 col3\" >entailment</td>\n",
       "                        <td id=\"T_375a331c_a792_11ed_8264_acbc32902497row49_col4\" class=\"data row49 col4\" >0.944000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fe378939198>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Impossible examples:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_3777b158_a792_11ed_8264_acbc32902497 th {\n",
       "          text-align: left;\n",
       "    }#T_3777b158_a792_11ed_8264_acbc32902497row0_col0,#T_3777b158_a792_11ed_8264_acbc32902497row0_col1,#T_3777b158_a792_11ed_8264_acbc32902497row0_col2,#T_3777b158_a792_11ed_8264_acbc32902497row0_col3,#T_3777b158_a792_11ed_8264_acbc32902497row0_col4,#T_3777b158_a792_11ed_8264_acbc32902497row1_col0,#T_3777b158_a792_11ed_8264_acbc32902497row1_col1,#T_3777b158_a792_11ed_8264_acbc32902497row1_col2,#T_3777b158_a792_11ed_8264_acbc32902497row1_col3,#T_3777b158_a792_11ed_8264_acbc32902497row1_col4,#T_3777b158_a792_11ed_8264_acbc32902497row2_col0,#T_3777b158_a792_11ed_8264_acbc32902497row2_col1,#T_3777b158_a792_11ed_8264_acbc32902497row2_col2,#T_3777b158_a792_11ed_8264_acbc32902497row2_col3,#T_3777b158_a792_11ed_8264_acbc32902497row2_col4,#T_3777b158_a792_11ed_8264_acbc32902497row3_col0,#T_3777b158_a792_11ed_8264_acbc32902497row3_col1,#T_3777b158_a792_11ed_8264_acbc32902497row3_col2,#T_3777b158_a792_11ed_8264_acbc32902497row3_col3,#T_3777b158_a792_11ed_8264_acbc32902497row3_col4,#T_3777b158_a792_11ed_8264_acbc32902497row4_col0,#T_3777b158_a792_11ed_8264_acbc32902497row4_col1,#T_3777b158_a792_11ed_8264_acbc32902497row4_col2,#T_3777b158_a792_11ed_8264_acbc32902497row4_col3,#T_3777b158_a792_11ed_8264_acbc32902497row4_col4,#T_3777b158_a792_11ed_8264_acbc32902497row5_col0,#T_3777b158_a792_11ed_8264_acbc32902497row5_col1,#T_3777b158_a792_11ed_8264_acbc32902497row5_col2,#T_3777b158_a792_11ed_8264_acbc32902497row5_col3,#T_3777b158_a792_11ed_8264_acbc32902497row5_col4,#T_3777b158_a792_11ed_8264_acbc32902497row6_col0,#T_3777b158_a792_11ed_8264_acbc32902497row6_col1,#T_3777b158_a792_11ed_8264_acbc32902497row6_col2,#T_3777b158_a792_11ed_8264_acbc32902497row6_col3,#T_3777b158_a792_11ed_8264_acbc32902497row6_col4,#T_3777b158_a792_11ed_8264_acbc32902497row7_col0,#T_3777b158_a792_11ed_8264_acbc32902497row7_col1,#T_3777b158_a792_11ed_8264_acbc32902497row7_col2,#T_3777b158_a792_11ed_8264_acbc32902497row7_col3,#T_3777b158_a792_11ed_8264_acbc32902497row7_col4,#T_3777b158_a792_11ed_8264_acbc32902497row8_col0,#T_3777b158_a792_11ed_8264_acbc32902497row8_col1,#T_3777b158_a792_11ed_8264_acbc32902497row8_col2,#T_3777b158_a792_11ed_8264_acbc32902497row8_col3,#T_3777b158_a792_11ed_8264_acbc32902497row8_col4,#T_3777b158_a792_11ed_8264_acbc32902497row9_col0,#T_3777b158_a792_11ed_8264_acbc32902497row9_col1,#T_3777b158_a792_11ed_8264_acbc32902497row9_col2,#T_3777b158_a792_11ed_8264_acbc32902497row9_col3,#T_3777b158_a792_11ed_8264_acbc32902497row9_col4,#T_3777b158_a792_11ed_8264_acbc32902497row10_col0,#T_3777b158_a792_11ed_8264_acbc32902497row10_col1,#T_3777b158_a792_11ed_8264_acbc32902497row10_col2,#T_3777b158_a792_11ed_8264_acbc32902497row10_col3,#T_3777b158_a792_11ed_8264_acbc32902497row10_col4,#T_3777b158_a792_11ed_8264_acbc32902497row11_col0,#T_3777b158_a792_11ed_8264_acbc32902497row11_col1,#T_3777b158_a792_11ed_8264_acbc32902497row11_col2,#T_3777b158_a792_11ed_8264_acbc32902497row11_col3,#T_3777b158_a792_11ed_8264_acbc32902497row11_col4,#T_3777b158_a792_11ed_8264_acbc32902497row12_col0,#T_3777b158_a792_11ed_8264_acbc32902497row12_col1,#T_3777b158_a792_11ed_8264_acbc32902497row12_col2,#T_3777b158_a792_11ed_8264_acbc32902497row12_col3,#T_3777b158_a792_11ed_8264_acbc32902497row12_col4,#T_3777b158_a792_11ed_8264_acbc32902497row13_col0,#T_3777b158_a792_11ed_8264_acbc32902497row13_col1,#T_3777b158_a792_11ed_8264_acbc32902497row13_col2,#T_3777b158_a792_11ed_8264_acbc32902497row13_col3,#T_3777b158_a792_11ed_8264_acbc32902497row13_col4,#T_3777b158_a792_11ed_8264_acbc32902497row14_col0,#T_3777b158_a792_11ed_8264_acbc32902497row14_col1,#T_3777b158_a792_11ed_8264_acbc32902497row14_col2,#T_3777b158_a792_11ed_8264_acbc32902497row14_col3,#T_3777b158_a792_11ed_8264_acbc32902497row14_col4,#T_3777b158_a792_11ed_8264_acbc32902497row15_col0,#T_3777b158_a792_11ed_8264_acbc32902497row15_col1,#T_3777b158_a792_11ed_8264_acbc32902497row15_col2,#T_3777b158_a792_11ed_8264_acbc32902497row15_col3,#T_3777b158_a792_11ed_8264_acbc32902497row15_col4,#T_3777b158_a792_11ed_8264_acbc32902497row16_col0,#T_3777b158_a792_11ed_8264_acbc32902497row16_col1,#T_3777b158_a792_11ed_8264_acbc32902497row16_col2,#T_3777b158_a792_11ed_8264_acbc32902497row16_col3,#T_3777b158_a792_11ed_8264_acbc32902497row16_col4,#T_3777b158_a792_11ed_8264_acbc32902497row17_col0,#T_3777b158_a792_11ed_8264_acbc32902497row17_col1,#T_3777b158_a792_11ed_8264_acbc32902497row17_col2,#T_3777b158_a792_11ed_8264_acbc32902497row17_col3,#T_3777b158_a792_11ed_8264_acbc32902497row17_col4,#T_3777b158_a792_11ed_8264_acbc32902497row18_col0,#T_3777b158_a792_11ed_8264_acbc32902497row18_col1,#T_3777b158_a792_11ed_8264_acbc32902497row18_col2,#T_3777b158_a792_11ed_8264_acbc32902497row18_col3,#T_3777b158_a792_11ed_8264_acbc32902497row18_col4,#T_3777b158_a792_11ed_8264_acbc32902497row19_col0,#T_3777b158_a792_11ed_8264_acbc32902497row19_col1,#T_3777b158_a792_11ed_8264_acbc32902497row19_col2,#T_3777b158_a792_11ed_8264_acbc32902497row19_col3,#T_3777b158_a792_11ed_8264_acbc32902497row19_col4,#T_3777b158_a792_11ed_8264_acbc32902497row20_col0,#T_3777b158_a792_11ed_8264_acbc32902497row20_col1,#T_3777b158_a792_11ed_8264_acbc32902497row20_col2,#T_3777b158_a792_11ed_8264_acbc32902497row20_col3,#T_3777b158_a792_11ed_8264_acbc32902497row20_col4,#T_3777b158_a792_11ed_8264_acbc32902497row21_col0,#T_3777b158_a792_11ed_8264_acbc32902497row21_col1,#T_3777b158_a792_11ed_8264_acbc32902497row21_col2,#T_3777b158_a792_11ed_8264_acbc32902497row21_col3,#T_3777b158_a792_11ed_8264_acbc32902497row21_col4,#T_3777b158_a792_11ed_8264_acbc32902497row22_col0,#T_3777b158_a792_11ed_8264_acbc32902497row22_col1,#T_3777b158_a792_11ed_8264_acbc32902497row22_col2,#T_3777b158_a792_11ed_8264_acbc32902497row22_col3,#T_3777b158_a792_11ed_8264_acbc32902497row22_col4,#T_3777b158_a792_11ed_8264_acbc32902497row23_col0,#T_3777b158_a792_11ed_8264_acbc32902497row23_col1,#T_3777b158_a792_11ed_8264_acbc32902497row23_col2,#T_3777b158_a792_11ed_8264_acbc32902497row23_col3,#T_3777b158_a792_11ed_8264_acbc32902497row23_col4,#T_3777b158_a792_11ed_8264_acbc32902497row24_col0,#T_3777b158_a792_11ed_8264_acbc32902497row24_col1,#T_3777b158_a792_11ed_8264_acbc32902497row24_col2,#T_3777b158_a792_11ed_8264_acbc32902497row24_col3,#T_3777b158_a792_11ed_8264_acbc32902497row24_col4,#T_3777b158_a792_11ed_8264_acbc32902497row25_col0,#T_3777b158_a792_11ed_8264_acbc32902497row25_col1,#T_3777b158_a792_11ed_8264_acbc32902497row25_col2,#T_3777b158_a792_11ed_8264_acbc32902497row25_col3,#T_3777b158_a792_11ed_8264_acbc32902497row25_col4,#T_3777b158_a792_11ed_8264_acbc32902497row26_col0,#T_3777b158_a792_11ed_8264_acbc32902497row26_col1,#T_3777b158_a792_11ed_8264_acbc32902497row26_col2,#T_3777b158_a792_11ed_8264_acbc32902497row26_col3,#T_3777b158_a792_11ed_8264_acbc32902497row26_col4,#T_3777b158_a792_11ed_8264_acbc32902497row27_col0,#T_3777b158_a792_11ed_8264_acbc32902497row27_col1,#T_3777b158_a792_11ed_8264_acbc32902497row27_col2,#T_3777b158_a792_11ed_8264_acbc32902497row27_col3,#T_3777b158_a792_11ed_8264_acbc32902497row27_col4,#T_3777b158_a792_11ed_8264_acbc32902497row28_col0,#T_3777b158_a792_11ed_8264_acbc32902497row28_col1,#T_3777b158_a792_11ed_8264_acbc32902497row28_col2,#T_3777b158_a792_11ed_8264_acbc32902497row28_col3,#T_3777b158_a792_11ed_8264_acbc32902497row28_col4,#T_3777b158_a792_11ed_8264_acbc32902497row29_col0,#T_3777b158_a792_11ed_8264_acbc32902497row29_col1,#T_3777b158_a792_11ed_8264_acbc32902497row29_col2,#T_3777b158_a792_11ed_8264_acbc32902497row29_col3,#T_3777b158_a792_11ed_8264_acbc32902497row29_col4,#T_3777b158_a792_11ed_8264_acbc32902497row30_col0,#T_3777b158_a792_11ed_8264_acbc32902497row30_col1,#T_3777b158_a792_11ed_8264_acbc32902497row30_col2,#T_3777b158_a792_11ed_8264_acbc32902497row30_col3,#T_3777b158_a792_11ed_8264_acbc32902497row30_col4,#T_3777b158_a792_11ed_8264_acbc32902497row31_col0,#T_3777b158_a792_11ed_8264_acbc32902497row31_col1,#T_3777b158_a792_11ed_8264_acbc32902497row31_col2,#T_3777b158_a792_11ed_8264_acbc32902497row31_col3,#T_3777b158_a792_11ed_8264_acbc32902497row31_col4,#T_3777b158_a792_11ed_8264_acbc32902497row32_col0,#T_3777b158_a792_11ed_8264_acbc32902497row32_col1,#T_3777b158_a792_11ed_8264_acbc32902497row32_col2,#T_3777b158_a792_11ed_8264_acbc32902497row32_col3,#T_3777b158_a792_11ed_8264_acbc32902497row32_col4,#T_3777b158_a792_11ed_8264_acbc32902497row33_col0,#T_3777b158_a792_11ed_8264_acbc32902497row33_col1,#T_3777b158_a792_11ed_8264_acbc32902497row33_col2,#T_3777b158_a792_11ed_8264_acbc32902497row33_col3,#T_3777b158_a792_11ed_8264_acbc32902497row33_col4,#T_3777b158_a792_11ed_8264_acbc32902497row34_col0,#T_3777b158_a792_11ed_8264_acbc32902497row34_col1,#T_3777b158_a792_11ed_8264_acbc32902497row34_col2,#T_3777b158_a792_11ed_8264_acbc32902497row34_col3,#T_3777b158_a792_11ed_8264_acbc32902497row34_col4,#T_3777b158_a792_11ed_8264_acbc32902497row35_col0,#T_3777b158_a792_11ed_8264_acbc32902497row35_col1,#T_3777b158_a792_11ed_8264_acbc32902497row35_col2,#T_3777b158_a792_11ed_8264_acbc32902497row35_col3,#T_3777b158_a792_11ed_8264_acbc32902497row35_col4,#T_3777b158_a792_11ed_8264_acbc32902497row36_col0,#T_3777b158_a792_11ed_8264_acbc32902497row36_col1,#T_3777b158_a792_11ed_8264_acbc32902497row36_col2,#T_3777b158_a792_11ed_8264_acbc32902497row36_col3,#T_3777b158_a792_11ed_8264_acbc32902497row36_col4,#T_3777b158_a792_11ed_8264_acbc32902497row37_col0,#T_3777b158_a792_11ed_8264_acbc32902497row37_col1,#T_3777b158_a792_11ed_8264_acbc32902497row37_col2,#T_3777b158_a792_11ed_8264_acbc32902497row37_col3,#T_3777b158_a792_11ed_8264_acbc32902497row37_col4,#T_3777b158_a792_11ed_8264_acbc32902497row38_col0,#T_3777b158_a792_11ed_8264_acbc32902497row38_col1,#T_3777b158_a792_11ed_8264_acbc32902497row38_col2,#T_3777b158_a792_11ed_8264_acbc32902497row38_col3,#T_3777b158_a792_11ed_8264_acbc32902497row38_col4,#T_3777b158_a792_11ed_8264_acbc32902497row39_col0,#T_3777b158_a792_11ed_8264_acbc32902497row39_col1,#T_3777b158_a792_11ed_8264_acbc32902497row39_col2,#T_3777b158_a792_11ed_8264_acbc32902497row39_col3,#T_3777b158_a792_11ed_8264_acbc32902497row39_col4,#T_3777b158_a792_11ed_8264_acbc32902497row40_col0,#T_3777b158_a792_11ed_8264_acbc32902497row40_col1,#T_3777b158_a792_11ed_8264_acbc32902497row40_col2,#T_3777b158_a792_11ed_8264_acbc32902497row40_col3,#T_3777b158_a792_11ed_8264_acbc32902497row40_col4,#T_3777b158_a792_11ed_8264_acbc32902497row41_col0,#T_3777b158_a792_11ed_8264_acbc32902497row41_col1,#T_3777b158_a792_11ed_8264_acbc32902497row41_col2,#T_3777b158_a792_11ed_8264_acbc32902497row41_col3,#T_3777b158_a792_11ed_8264_acbc32902497row41_col4,#T_3777b158_a792_11ed_8264_acbc32902497row42_col0,#T_3777b158_a792_11ed_8264_acbc32902497row42_col1,#T_3777b158_a792_11ed_8264_acbc32902497row42_col2,#T_3777b158_a792_11ed_8264_acbc32902497row42_col3,#T_3777b158_a792_11ed_8264_acbc32902497row42_col4,#T_3777b158_a792_11ed_8264_acbc32902497row43_col0,#T_3777b158_a792_11ed_8264_acbc32902497row43_col1,#T_3777b158_a792_11ed_8264_acbc32902497row43_col2,#T_3777b158_a792_11ed_8264_acbc32902497row43_col3,#T_3777b158_a792_11ed_8264_acbc32902497row43_col4,#T_3777b158_a792_11ed_8264_acbc32902497row44_col0,#T_3777b158_a792_11ed_8264_acbc32902497row44_col1,#T_3777b158_a792_11ed_8264_acbc32902497row44_col2,#T_3777b158_a792_11ed_8264_acbc32902497row44_col3,#T_3777b158_a792_11ed_8264_acbc32902497row44_col4,#T_3777b158_a792_11ed_8264_acbc32902497row45_col0,#T_3777b158_a792_11ed_8264_acbc32902497row45_col1,#T_3777b158_a792_11ed_8264_acbc32902497row45_col2,#T_3777b158_a792_11ed_8264_acbc32902497row45_col3,#T_3777b158_a792_11ed_8264_acbc32902497row45_col4,#T_3777b158_a792_11ed_8264_acbc32902497row46_col0,#T_3777b158_a792_11ed_8264_acbc32902497row46_col1,#T_3777b158_a792_11ed_8264_acbc32902497row46_col2,#T_3777b158_a792_11ed_8264_acbc32902497row46_col3,#T_3777b158_a792_11ed_8264_acbc32902497row46_col4,#T_3777b158_a792_11ed_8264_acbc32902497row47_col0,#T_3777b158_a792_11ed_8264_acbc32902497row47_col1,#T_3777b158_a792_11ed_8264_acbc32902497row47_col2,#T_3777b158_a792_11ed_8264_acbc32902497row47_col3,#T_3777b158_a792_11ed_8264_acbc32902497row47_col4,#T_3777b158_a792_11ed_8264_acbc32902497row48_col0,#T_3777b158_a792_11ed_8264_acbc32902497row48_col1,#T_3777b158_a792_11ed_8264_acbc32902497row48_col2,#T_3777b158_a792_11ed_8264_acbc32902497row48_col3,#T_3777b158_a792_11ed_8264_acbc32902497row48_col4,#T_3777b158_a792_11ed_8264_acbc32902497row49_col0,#T_3777b158_a792_11ed_8264_acbc32902497row49_col1,#T_3777b158_a792_11ed_8264_acbc32902497row49_col2,#T_3777b158_a792_11ed_8264_acbc32902497row49_col3,#T_3777b158_a792_11ed_8264_acbc32902497row49_col4{\n",
       "            text-align:  left;\n",
       "        }</style><table id=\"T_3777b158_a792_11ed_8264_acbc32902497\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Premise</th>        <th class=\"col_heading level0 col1\" >Hypothesis</th>        <th class=\"col_heading level0 col2\" >Label</th>        <th class=\"col_heading level0 col3\" >Model Prediction</th>        <th class=\"col_heading level0 col4\" >Model Confidence</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row0\" class=\"row_heading level0 row0\" >116</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row0_col0\" class=\"data row0 col0\" >Glaiza Herradura-Agullo\n",
       "(born February 24, 1978)\n",
       "is a Filipino former\n",
       "child actress. She was\n",
       "the first-ever grand\n",
       "winner of the Little Miss\n",
       "Philippines segment of\n",
       "\"Eat Bulaga!\" in 1984.\n",
       "She starred in RPN-9's\n",
       "television series\n",
       "\"Heredero\" with Manilyn\n",
       "Reynes and Richard\n",
       "Arellano. She won the\n",
       "1988 FAMAS Best Child\n",
       "Actress award for her\n",
       "role in \"Batas Sa Aking\n",
       "Kamay\" starring Fernando\n",
       "Poe, Jr..</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row0_col1\" class=\"data row0 col1\" >Herradura-Agullo was born\n",
       "in the 80's</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row0_col2\" class=\"data row0 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row0_col3\" class=\"data row0 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row0_col4\" class=\"data row0 col4\" >0.941000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row1\" class=\"row_heading level0 row1\" >171</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row1_col0\" class=\"data row1 col0\" >The Whitechapel murders\n",
       "were committed in or near\n",
       "the impoverished\n",
       "Whitechapel district in\n",
       "the East End of London\n",
       "between 3 April 1888 and\n",
       "13 February 1891. At\n",
       "various points some or\n",
       "all of these eleven\n",
       "unsolved murders of women\n",
       "have been ascribed to the\n",
       "notorious unidentified\n",
       "serial killer known as\n",
       "Jack the Ripper.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row1_col1\" class=\"data row1 col1\" >The women killed in the\n",
       "Whitechapel murders were\n",
       "impoverished.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row1_col2\" class=\"data row1 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row1_col3\" class=\"data row1 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row1_col4\" class=\"data row1 col4\" >0.832000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row2\" class=\"row_heading level0 row2\" >300</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row2_col0\" class=\"data row2 col0\" >Departure of a Grand Old\n",
       "Man (Russian: Уход\n",
       "великого старца ,\n",
       "translit. Ukhod velikovo\n",
       "startza) is a 1912\n",
       "Russian silent film about\n",
       "the last days of author\n",
       "Leo Tolstoy. The film was\n",
       "directed by Yakov\n",
       "Protazanov and Elizaveta\n",
       "Thiman, and was actress\n",
       "Olga Petrova's first\n",
       "film.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row2_col1\" class=\"data row2 col1\" >Olga performed in many\n",
       "films before This one</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row2_col2\" class=\"data row2 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row2_col3\" class=\"data row2 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row2_col4\" class=\"data row2 col4\" >0.922000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row3\" class=\"row_heading level0 row3\" >565</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row3_col0\" class=\"data row3 col0\" >Gay Sex in the 70s is a\n",
       "2005 American documentary\n",
       "film about gay sexual\n",
       "culture in New York City\n",
       "in the 1970s. The film\n",
       "was directed by Joseph\n",
       "Lovett and encompasses\n",
       "the twelve years of\n",
       "sexual freedom bookended\n",
       "by the Stonewall riots of\n",
       "1969 and the recognition\n",
       "of AIDS in 1981, and\n",
       "features interviews with\n",
       "Larry Kramer, Tom\n",
       "Bianchi, Barton Lidice\n",
       "Beneš, Rodger McFarlane,\n",
       "and many others.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row3_col1\" class=\"data row3 col1\" >Gay Sex in the 70s was\n",
       "directed by a gay man.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row3_col2\" class=\"data row3 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row3_col3\" class=\"data row3 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row3_col4\" class=\"data row3 col4\" >0.907000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row4\" class=\"row_heading level0 row4\" >795</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row4_col0\" class=\"data row4 col0\" >Sophia Turkiewicz is an\n",
       "Australian film and\n",
       "television director known\n",
       "for her film \"Silver\n",
       "City\". \"Silver City\",\n",
       "which Turkiewicz began\n",
       "during a six-month stay\n",
       "in Poland, was released\n",
       "internationally and won 3\n",
       "AFI awards. Turkiewicz\n",
       "has also spent six years\n",
       "as a lecturer in the\n",
       "directing department of\n",
       "the Australian Film,\n",
       "Television and Radio\n",
       "School before leaving to\n",
       "direct \"Once My Mother\".</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row4_col1\" class=\"data row4 col1\" >The film Silver City won\n",
       "3 AFI awards six years\n",
       "after Once My Mother was\n",
       "directed by Turkiewicz in\n",
       "Poland.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row4_col2\" class=\"data row4 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row4_col3\" class=\"data row4 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row4_col4\" class=\"data row4 col4\" >0.705000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row5\" class=\"row_heading level0 row5\" >862</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row5_col0\" class=\"data row5 col0\" >The Doberman Gang is a\n",
       "1972 film about a\n",
       "talented animal trainer\n",
       "who uses a pack of\n",
       "Dobermans to commit a\n",
       "bank robbery. The six\n",
       "dogs were all named after\n",
       "famous bank robbers.\n",
       "Their names were\n",
       "Dillinger (John\n",
       "Dillinger), Bonnie\n",
       "(Bonnie Parker), Clyde\n",
       "(Clyde Barrow), Pretty\n",
       "Boy Floyd, Baby Face\n",
       "Nelson, and Ma Barker.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row5_col1\" class=\"data row5 col1\" >The Doberman Gang was\n",
       "released within the last\n",
       "100 years.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row5_col2\" class=\"data row5 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row5_col3\" class=\"data row5 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row5_col4\" class=\"data row5 col4\" >0.921000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row6\" class=\"row_heading level0 row6\" >1138</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row6_col0\" class=\"data row6 col0\" >SuperpinkyMandy is the\n",
       "debut studio album of\n",
       "British singer Beth\n",
       "Orton. Largely in the\n",
       "style of electronica, and\n",
       "produced closely with\n",
       "then boyfriend William\n",
       "Orbit, it was a limited\n",
       "Japan-only release, with\n",
       "about 5000 copies\n",
       "pressed. As such, it is\n",
       "very much sought after.\n",
       "Orton largely passes over\n",
       "the release when\n",
       "interviewed, citing\n",
       "1996's \"Trailer Park\" as\n",
       "her first release.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row6_col1\" class=\"data row6 col1\" >beth orton produced indie\n",
       "music</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row6_col2\" class=\"data row6 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row6_col3\" class=\"data row6 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row6_col4\" class=\"data row6 col4\" >0.785000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row7\" class=\"row_heading level0 row7\" >1169</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row7_col0\" class=\"data row7 col0\" >Dhanish Karthik (born 24\n",
       "July 1989) is an Indian\n",
       "actor. He made his debut\n",
       "as Sanjeev Menon in the\n",
       "Malayalam film \"Ivide\"\n",
       "(2015) directed by\n",
       "Shyamaprasad. He recently\n",
       "finished filming for the\n",
       "Bollywood film Chef (2017\n",
       "film) with Saif Ali Khan.\n",
       "The film, directed by\n",
       "Raja Krishna Menon, is\n",
       "slated to release in July\n",
       "2017. This will be\n",
       "Karthik's debut in\n",
       "Bollywood.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row7_col1\" class=\"data row7 col1\" >Saif Ali Khan was not\n",
       "disassociated with the\n",
       "movie Chef.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row7_col2\" class=\"data row7 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row7_col3\" class=\"data row7 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row7_col4\" class=\"data row7 col4\" >0.762000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row8\" class=\"row_heading level0 row8\" >1172</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row8_col0\" class=\"data row8 col0\" >There Was a Crooked\n",
       "Man... is a 1970 western\n",
       "starring Kirk Douglas and\n",
       "Henry Fonda and directed\n",
       "by Joseph L. Mankiewicz.\n",
       "This was the only western\n",
       "made by Mankiewicz,\n",
       "director of such notable\n",
       "films as \"All About Eve\",\n",
       "\"Guys and Dolls\" and\n",
       "\"Cleopatra\". It was\n",
       "written by David Newman\n",
       "and Robert Benton, their\n",
       "first script after\n",
       "\"Bonnie and Clyde\".</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row8_col1\" class=\"data row8 col1\" >There Was a Crooked Man\n",
       "was released after the\n",
       "year 1970 ended.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row8_col2\" class=\"data row8 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row8_col3\" class=\"data row8 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row8_col4\" class=\"data row8 col4\" >0.522000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row9\" class=\"row_heading level0 row9\" >1290</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row9_col0\" class=\"data row9 col0\" >Peter John Reynolds (6\n",
       "November 1939 – 26\n",
       "September 2001) was a\n",
       "British archaeologist\n",
       "known for his research in\n",
       "experimental archaeology\n",
       "and the British Iron Age\n",
       "and for being recruited\n",
       "as the first director of\n",
       "Butser Ancient Farm, a\n",
       "working replica of an\n",
       "Iron Age farmstead in\n",
       "Hampshire.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row9_col1\" class=\"data row9 col1\" >Peter John Reynolds was\n",
       "alive after the 9/11\n",
       "attacks.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row9_col2\" class=\"data row9 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row9_col3\" class=\"data row9 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row9_col4\" class=\"data row9 col4\" >0.748000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row10\" class=\"row_heading level0 row10\" >1302</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row10_col0\" class=\"data row10 col0\" >Héctor Canziani was an\n",
       "Argentine poet,\n",
       "screenwriter and film\n",
       "director who worked in\n",
       "Argentine cinema in the\n",
       "1940s and 1950s. Although\n",
       "his work was most\n",
       "abundant in screenwriting\n",
       "and poetry after his\n",
       "brief film career, he is\n",
       "best known for his\n",
       "directorship and\n",
       "production of the 1950\n",
       "tango dancing film Al\n",
       "Compás de tu Mentira\n",
       "based on a play by Oscar\n",
       "Wilde.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row10_col1\" class=\"data row10 col1\" >He did direct a movie\n",
       "after 1950</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row10_col2\" class=\"data row10 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row10_col3\" class=\"data row10 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row10_col4\" class=\"data row10 col4\" >0.944000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row11\" class=\"row_heading level0 row11\" >1470</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row11_col0\" class=\"data row11 col0\" >Marie Hedwig Auguste of\n",
       "Sulzbach (German: \"Marie\n",
       "Hedwig Auguste von\n",
       "Sulzbach\" ; born: 15\n",
       "April 1650 in Sulzbach;\n",
       "died: 23 November 1681 in\n",
       "Hamburg) was a Countess\n",
       "Palatine of Sulzbach by\n",
       "birth and by marriage,\n",
       "Archduchess of Austria\n",
       "and by her second\n",
       "marriage, Duchess of\n",
       "Saxe-Lauenburg.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row11_col1\" class=\"data row11 col1\" >Marie Hedwig Auguste of\n",
       "Sulzbach died at age 70.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row11_col2\" class=\"data row11 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row11_col3\" class=\"data row11 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row11_col4\" class=\"data row11 col4\" >0.515000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row12\" class=\"row_heading level0 row12\" >1545</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row12_col0\" class=\"data row12 col0\" >Marguerite Aimee Rosine\n",
       "Coppin (2 February 1867 –\n",
       "1931) was born in\n",
       "Brussels and became woman\n",
       "Poet Laureate of Belgium\n",
       "and a noted feminist and\n",
       "pioneer in female\n",
       "emancipation and equal\n",
       "rights for women. She was\n",
       "compared with women's\n",
       "rights activists Amelia\n",
       "Bloomer and Emmeline\n",
       "Pankhurst.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row12_col1\" class=\"data row12 col1\" >Marguerite Aimee Rosine\n",
       "Coppin died at age 100.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row12_col2\" class=\"data row12 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row12_col3\" class=\"data row12 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row12_col4\" class=\"data row12 col4\" >0.525000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row13\" class=\"row_heading level0 row13\" >1668</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row13_col0\" class=\"data row13 col0\" >Harold Buttleman,\n",
       "Daredevil Stuntman (also\n",
       "known as Buttleman) is a\n",
       "2003 film written and\n",
       "directed by Francis\n",
       "Stokes; the only movie he\n",
       "has directed. It won the\n",
       "Jury Prize at the Deep\n",
       "Ellum Film Festival in\n",
       "2003. It was awarded the\n",
       "audience award in the Had\n",
       "to Be Made Film Festival\n",
       "in 2005.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row13_col1\" class=\"data row13 col1\" >Daredevil Stuntman is the\n",
       "only title for the movie.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row13_col2\" class=\"data row13 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row13_col3\" class=\"data row13 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row13_col4\" class=\"data row13 col4\" >0.712000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row14\" class=\"row_heading level0 row14\" >1922</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row14_col0\" class=\"data row14 col0\" >\"Stagger Lee\", also known\n",
       "as \"Stagolee\" and other\n",
       "variants, is a popular\n",
       "American folk song about\n",
       "the murder of Billy Lyons\n",
       "by \"Stag\" Lee Shelton in\n",
       "St. Louis, Missouri at\n",
       "Christmas, 1895. The song\n",
       "was first published in\n",
       "1911, and was first\n",
       "recorded in 1923 by Fred\n",
       "Waring's Pennsylvanians.\n",
       "A version by Lloyd Price\n",
       "reached number one on the\n",
       "\"Billboard\" Hot 100 in\n",
       "1959.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row14_col1\" class=\"data row14 col1\" >The song \"Stagger Lee\"\n",
       "was popular in the 1890s.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row14_col2\" class=\"data row14 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row14_col3\" class=\"data row14 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row14_col4\" class=\"data row14 col4\" >0.665000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row15\" class=\"row_heading level0 row15\" >2109</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row15_col0\" class=\"data row15 col0\" >Concrete is a comic book\n",
       "series created and\n",
       "written by Paul Chadwick\n",
       "and published by Dark\n",
       "Horse Comics. His first\n",
       "appearance is \"Dark Horse\n",
       "Presents\" #1 (July,\n",
       "1986). The eponymous\n",
       "central character is a\n",
       "normal man whose brain\n",
       "was transplanted into a\n",
       "large, stone body by\n",
       "aliens, and who lives an\n",
       "extraordinary life on\n",
       "Earth following his\n",
       "escape.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row15_col1\" class=\"data row15 col1\" >Concrete was not written\n",
       "and published by the same\n",
       "entity.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row15_col2\" class=\"data row15 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row15_col3\" class=\"data row15 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row15_col4\" class=\"data row15 col4\" >0.869000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row16\" class=\"row_heading level0 row16\" >2225</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row16_col0\" class=\"data row16 col0\" >Nigel Edward Povah (born\n",
       "17 July 1952 in\n",
       "Wandworth, London) is a\n",
       "British chess player. He\n",
       "is an International\n",
       "Master at over-the-board\n",
       "chess and a grandmaster\n",
       "at correspondence chess.\n",
       "Povah is the author of\n",
       "\"Chess Training\". He is\n",
       "reckoned to be the UK's\n",
       "strongest correspondence\n",
       "chess player since\n",
       "Jonathan Penrose. Povah\n",
       "has one son, Jonathan\n",
       "Povah.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row16_col1\" class=\"data row16 col1\" >Nigel Edward Povah is a\n",
       "grandmaster at all kinds\n",
       "of chess.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row16_col2\" class=\"data row16 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row16_col3\" class=\"data row16 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row16_col4\" class=\"data row16 col4\" >0.432000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row17\" class=\"row_heading level0 row17\" >2227</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row17_col0\" class=\"data row17 col0\" >Denis Villeneuve (] ;\n",
       "born October 3, 1967) is\n",
       "a French Canadian film\n",
       "director and writer. He\n",
       "is a four-time recipient\n",
       "of the Canadian Screen\n",
       "Award (formerly Genie\n",
       "Award) for Best\n",
       "Direction, for\n",
       "\"Maelström\" in 2001,\n",
       "\"Polytechnique\" in 2010,\n",
       "\"Incendies\" in 2011, and\n",
       "\"Enemy\" in 2013. The\n",
       "first three films also\n",
       "won the Academy of\n",
       "Canadian Cinema and\n",
       "Television Award for Best\n",
       "Motion Picture.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row17_col1\" class=\"data row17 col1\" >Denis Villeneuve released\n",
       "a movie in 2008.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row17_col2\" class=\"data row17 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row17_col3\" class=\"data row17 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row17_col4\" class=\"data row17 col4\" >0.711000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row18\" class=\"row_heading level0 row18\" >2295</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row18_col0\" class=\"data row18 col0\" >The International\n",
       "University of Rabat or\n",
       "IUR is a semi-public\n",
       "university founded in\n",
       "2010 in Morocco. It\n",
       "delivers double-degrees,\n",
       "in collaboration with\n",
       "foreign universities, in\n",
       "law, engineering,\n",
       "aeronautics, energy\n",
       "engineering,\n",
       "architecture, business\n",
       "management and political\n",
       "sciences.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row18_col1\" class=\"data row18 col1\" >Science fiction is only\n",
       "written for the 21st\n",
       "century.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row18_col2\" class=\"data row18 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row18_col3\" class=\"data row18 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row18_col4\" class=\"data row18 col4\" >0.881000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row19\" class=\"row_heading level0 row19\" >2483</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row19_col0\" class=\"data row19 col0\" >OverBlood is a science\n",
       "fiction video game\n",
       "developed by\n",
       "Riverhillsoft and\n",
       "published by Electronic\n",
       "Arts for the PlayStation.\n",
       "It is considered one of\n",
       "the first survival horror\n",
       "games to make use of a\n",
       "fully three-dimensional\n",
       "virtual environment,\n",
       "second only to\n",
       "Riverhillsoft's own\n",
       "\"Doctor Hauzer\", a fully\n",
       "3D survival horror game\n",
       "released for the 3DO in\n",
       "1994.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row19_col1\" class=\"data row19 col1\" >OverBlood is a video game\n",
       "that has horror and\n",
       "survival elements that\n",
       "make use of the hard and\n",
       "talented work of video\n",
       "game designers to make a\n",
       "third dimensional world,\n",
       "never seen before in this\n",
       "genre.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row19_col2\" class=\"data row19 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row19_col3\" class=\"data row19 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row19_col4\" class=\"data row19 col4\" >0.855000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row20\" class=\"row_heading level0 row20\" >2507</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row20_col0\" class=\"data row20 col0\" >Luton Town Ladies\n",
       "Football Club was founded\n",
       "in 1997 and formed a\n",
       "partnership with its male\n",
       "counterpart, Luton Town\n",
       "F.C. in 2000. The club is\n",
       "currently a member of the\n",
       "FA Women's Premier League\n",
       "South East Division One\n",
       "and play home matches at\n",
       "The Carlsberg Stadium,\n",
       "home of Biggleswade Town\n",
       "F.C.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row20_col1\" class=\"data row20 col1\" >Luton Town Ladies\n",
       "Football Club was founded\n",
       "in the 90s.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row20_col2\" class=\"data row20 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row20_col3\" class=\"data row20 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row20_col4\" class=\"data row20 col4\" >0.956000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row21\" class=\"row_heading level0 row21\" >2561</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row21_col0\" class=\"data row21 col0\" >Americana Deluxe is the\n",
       "second studio album by\n",
       "Big Bad Voodoo Daddy.\n",
       "This album is also\n",
       "sometimes called \"Big Bad\n",
       "Voodoo Daddy\", as the\n",
       "album cover prominently\n",
       "displays a stylized \"Big\n",
       "Bad Voodoo Daddy\" logo\n",
       "and does not feature the\n",
       "phrase \"Americana Deluxe\"\n",
       "on it. However, the liner\n",
       "notes and the band's\n",
       "website clearly show that\n",
       "the true title is indeed\n",
       "\"Americana Deluxe\".</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row21_col1\" class=\"data row21 col1\" >Americana Deluxe's title\n",
       "is not Big Bad Voodoo\n",
       "daddy.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row21_col2\" class=\"data row21 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row21_col3\" class=\"data row21 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row21_col4\" class=\"data row21 col4\" >0.679000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row22\" class=\"row_heading level0 row22\" >2676</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row22_col0\" class=\"data row22 col0\" >Ernest R. Kroeger (August\n",
       "10, 1862 – April 7, 1934)\n",
       "was an American composer.\n",
       "He is mainly known for\n",
       "the pedagogical works he\n",
       "composed for piano; he\n",
       "also taught music in St.\n",
       "Louis, Missouri. Today\n",
       "his papers are held at\n",
       "the Missouri Historical\n",
       "Society.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row22_col1\" class=\"data row22 col1\" >Ernest R. Kroeger was\n",
       "born in the late Spring\n",
       "during the year 1862.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row22_col2\" class=\"data row22 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row22_col3\" class=\"data row22 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row22_col4\" class=\"data row22 col4\" >0.757000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row23\" class=\"row_heading level0 row23\" >2706</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row23_col0\" class=\"data row23 col0\" >John Davison Rockefeller\n",
       "III (March 21, 1906 –\n",
       "July 10, 1978) was a\n",
       "philanthropist and third-\n",
       "generation member of the\n",
       "prominent Rockefeller\n",
       "family. He was the eldest\n",
       "son of philanthropists\n",
       "John D. Rockefeller Jr.\n",
       "and Abby Aldrich\n",
       "Rockefeller. His siblings\n",
       "were Abby, Nelson,\n",
       "Laurance, Winthrop, and\n",
       "David.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row23_col1\" class=\"data row23 col1\" >John Rockefeller was a\n",
       "solider in the first\n",
       "world war.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row23_col2\" class=\"data row23 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row23_col3\" class=\"data row23 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row23_col4\" class=\"data row23 col4\" >0.877000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row24\" class=\"row_heading level0 row24\" >2769</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row24_col0\" class=\"data row24 col0\" >Hidden City Entertainment\n",
       "was a game publisher\n",
       "founded in 2004 (as\n",
       "Hidden City Games, Inc.)\n",
       "by Jesper Myrfors and\n",
       "Paul Peterson to develop\n",
       "and market the chip-\n",
       "throwing game, \"Clout\n",
       "Fantasy.\" After Clout was\n",
       "developed the company\n",
       "recruited Peter Adkison\n",
       "as CEO.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row24_col1\" class=\"data row24 col1\" >Peter Adkison was not the\n",
       "founder of Hidden City\n",
       "Entertainment.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row24_col2\" class=\"data row24 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row24_col3\" class=\"data row24 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row24_col4\" class=\"data row24 col4\" >0.547000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row25\" class=\"row_heading level0 row25\" >2778</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row25_col0\" class=\"data row25 col0\" >Rear Admiral Kevin John\n",
       "Scarce {'1': \", '2': \",\n",
       "'3': \", '4': \"} (born 4\n",
       "May 1952) is a retired\n",
       "Royal Australian Navy\n",
       "officer who was the 34th\n",
       "Governor of South\n",
       "Australia, serving from\n",
       "August 2007 to August\n",
       "2014. He was succeeded by\n",
       "Hieu Van Le, who had\n",
       "previously been his\n",
       "lieutenant governor.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row25_col1\" class=\"data row25 col1\" >Rear Admiral Kevin is\n",
       "still a navy officer\n",
       "today.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row25_col2\" class=\"data row25 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row25_col3\" class=\"data row25 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row25_col4\" class=\"data row25 col4\" >0.610000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row26\" class=\"row_heading level0 row26\" >2805</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row26_col0\" class=\"data row26 col0\" >Fast-moving consumer\n",
       "goods (FMCG) or consumer\n",
       "packaged goods (CPG) are\n",
       "products that are sold\n",
       "quickly and at relatively\n",
       "low cost. Examples\n",
       "include non-durable goods\n",
       "such as packaged foods,\n",
       "beverages, toiletries,\n",
       "over-the-counter drugs\n",
       "and many other\n",
       "consumables. In contrast,\n",
       "durable goods or major\n",
       "appliances such as\n",
       "kitchen appliances are\n",
       "generally replaced over a\n",
       "period of several years.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row26_col1\" class=\"data row26 col1\" >(FMCG) stands for\n",
       "Further-moving consumer\n",
       "goods.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row26_col2\" class=\"data row26 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row26_col3\" class=\"data row26 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row26_col4\" class=\"data row26 col4\" >0.865000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row27\" class=\"row_heading level0 row27\" >2914</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row27_col0\" class=\"data row27 col0\" >The Menu (), is a 2015\n",
       "television series\n",
       "produced by Hong Kong\n",
       "Television Network. The\n",
       "series is starred by Noel\n",
       "Leung, Catherine Chau,\n",
       "Kate Yeung and Gregory\n",
       "Wong, written by Pun Man-\n",
       "hung and directed by Ben\n",
       "Fong. The first episode\n",
       "premiered on 10 March 10\n",
       "2015. The plot revolves\n",
       "around the newspaper\n",
       "business.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row27_col1\" class=\"data row27 col1\" >The Menu is a chinese\n",
       "series.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row27_col2\" class=\"data row27 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row27_col3\" class=\"data row27 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row27_col4\" class=\"data row27 col4\" >0.793000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row28\" class=\"row_heading level0 row28\" >3479</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row28_col0\" class=\"data row28 col0\" >Ronald Mayorga Sánchez\n",
       "(born June 26, 1984,\n",
       "Yumbo, Valle del Cauca,\n",
       "Colombia) is an award-\n",
       "winning Colombian\n",
       "journalist and TV anchor\n",
       "of \"Red\" in Caracol\n",
       "Television in Colombia.\n",
       "As a radio journalist who\n",
       "works with \"Blue Radio\"\n",
       "one of the radio\n",
       "station's imported from\n",
       "Latin America as a host\n",
       "in \"Vox Populi\".</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row28_col1\" class=\"data row28 col1\" >Ronald Mayorga Sánchez\n",
       "won an award for\n",
       "journalism before 1980</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row28_col2\" class=\"data row28 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row28_col3\" class=\"data row28 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row28_col4\" class=\"data row28 col4\" >0.619000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row29\" class=\"row_heading level0 row29\" >3521</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row29_col0\" class=\"data row29 col0\" >Elizabeth \"Long Liz\"\n",
       "Stride (née\n",
       "Gustafsdotter) (27\n",
       "November 1843 – 30\n",
       "September 1888) is\n",
       "believed to be a victim\n",
       "of the notorious\n",
       "unidentified serial\n",
       "killer called Jack the\n",
       "Ripper, who killed and\n",
       "mutilated several women\n",
       "in the Whitechapel area\n",
       "of London from late\n",
       "August to early November\n",
       "1888.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row29_col1\" class=\"data row29 col1\" >Jack the Ripper was never\n",
       "identified.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row29_col2\" class=\"data row29 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row29_col3\" class=\"data row29 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row29_col4\" class=\"data row29 col4\" >0.843000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row30\" class=\"row_heading level0 row30\" >3537</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row30_col0\" class=\"data row30 col0\" >The North African ostrich\n",
       "or red-necked ostrich\n",
       "(\"Struthio camelus\n",
       "camelus\"), also known as\n",
       "the Barbary ostrich, is\n",
       "the nominate subspecies\n",
       "of the common ostrich\n",
       "from West and North\n",
       "Africa. It is the largest\n",
       "subspecies, making it the\n",
       "largest living bird.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row30_col1\" class=\"data row30 col1\" >The Barbary ostrich was\n",
       "also referred to as The\n",
       "North African ostrich by\n",
       "scientists.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row30_col2\" class=\"data row30 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row30_col3\" class=\"data row30 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row30_col4\" class=\"data row30 col4\" >0.783000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row31\" class=\"row_heading level0 row31\" >3900</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row31_col0\" class=\"data row31 col0\" >Hidden City Entertainment\n",
       "was a game publisher\n",
       "founded in 2004 (as\n",
       "Hidden City Games, Inc.)\n",
       "by Jesper Myrfors and\n",
       "Paul Peterson to develop\n",
       "and market the chip-\n",
       "throwing game, \"Clout\n",
       "Fantasy.\" After Clout was\n",
       "developed the company\n",
       "recruited Peter Adkison\n",
       "as CEO.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row31_col1\" class=\"data row31 col1\" >Hidden City Entertainment\n",
       "was founded by Jesper\n",
       "Myrfors</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row31_col2\" class=\"data row31 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row31_col3\" class=\"data row31 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row31_col4\" class=\"data row31 col4\" >0.509000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row32\" class=\"row_heading level0 row32\" >3967</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row32_col0\" class=\"data row32 col0\" >Louis Marx (August 11,\n",
       "1896 – February 5, 1982)\n",
       "was an American toy maker\n",
       "and businessman whose\n",
       "company, Louis Marx and\n",
       "Company, was the largest\n",
       "toy company in the world\n",
       "in the 1950s. Described\n",
       "by many as an experienced\n",
       "businessman with the mind\n",
       "of child; Louis Marx’s\n",
       "ability to see into the\n",
       "minds of children around\n",
       "the world guided his toy\n",
       "creations and advertising\n",
       "efforts.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row32_col1\" class=\"data row32 col1\" >Louis Marx lived to be 85\n",
       "years old.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row32_col2\" class=\"data row32 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row32_col3\" class=\"data row32 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row32_col4\" class=\"data row32 col4\" >0.830000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row33\" class=\"row_heading level0 row33\" >4022</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row33_col0\" class=\"data row33 col0\" >John Michael \"Mickey\"\n",
       "Arthur (born 17 May 1968)\n",
       "is a South African\n",
       "cricketer who played\n",
       "South African domestic\n",
       "cricket from 1986 to\n",
       "2001. He is the current\n",
       "coach of the Pakistan\n",
       "national cricket team. He\n",
       "coached the South African\n",
       "national team from 2005\n",
       "to 2010, and was the\n",
       "coach of the Australia\n",
       "national cricket team\n",
       "until his sacking on 23\n",
       "June 2013.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row33_col1\" class=\"data row33 col1\" >John Arthur played or\n",
       "coached Cricket\n",
       "consistently until 2013.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row33_col2\" class=\"data row33 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row33_col3\" class=\"data row33 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row33_col4\" class=\"data row33 col4\" >0.734000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row34\" class=\"row_heading level0 row34\" >4072</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row34_col0\" class=\"data row34 col0\" >Return to Paradise is a\n",
       "live album by Styx,\n",
       "released in 1997. It\n",
       "features songs from their\n",
       "successful reunion tour\n",
       "with Tommy Shaw, but\n",
       "without John Panozzo, who\n",
       "died in July 1996. It\n",
       "includes three new studio\n",
       "tracks, including \"Dear\n",
       "John\", which Shaw wrote\n",
       "as a tribute to Panozzo.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row34_col1\" class=\"data row34 col1\" >Tommy Shaw wrote the new\n",
       "songs before John\n",
       "Pannozzo died.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row34_col2\" class=\"data row34 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row34_col3\" class=\"data row34 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row34_col4\" class=\"data row34 col4\" >0.746000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row35\" class=\"row_heading level0 row35\" >4704</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row35_col0\" class=\"data row35 col0\" >The History Boys is a\n",
       "2006 British comedy-drama\n",
       "film adapted by Alan\n",
       "Bennett from his play of\n",
       "the same name, which won\n",
       "the 2005 Olivier Award\n",
       "for Best New Play and the\n",
       "2006 Tony Award for Best\n",
       "Play. It was directed by\n",
       "Nicholas Hytner, who\n",
       "directed the original\n",
       "production at the Royal\n",
       "National Theatre in\n",
       "London, and features the\n",
       "original cast of the\n",
       "play.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row35_col1\" class=\"data row35 col1\" >The History boys was\n",
       "founded in 2005</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row35_col2\" class=\"data row35 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row35_col3\" class=\"data row35 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row35_col4\" class=\"data row35 col4\" >0.906000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row36\" class=\"row_heading level0 row36\" >4724</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row36_col0\" class=\"data row36 col0\" >Boy Meets Girl is an ITV\n",
       "comedy-drama television\n",
       "miniseries starring\n",
       "Rachael Stirling and\n",
       "Martin Freeman. In the\n",
       "show, Danny Reed\n",
       "(Freeman) is struck by\n",
       "lightning. When he wakes\n",
       "up from the attack, he is\n",
       "inside the body of a\n",
       "woman, fashion journalist\n",
       "Veronica Burton\n",
       "(Stirling). Written by\n",
       "David Allison, the series\n",
       "began on 1 May 2009.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row36_col1\" class=\"data row36 col1\" >Veronica Burton and\n",
       "Martin Freeman were\n",
       "actors in the television\n",
       "miniseries Boy Meets\n",
       "Girl.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row36_col2\" class=\"data row36 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row36_col3\" class=\"data row36 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row36_col4\" class=\"data row36 col4\" >0.964000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row37\" class=\"row_heading level0 row37\" >4780</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row37_col0\" class=\"data row37 col0\" >Mickey Munday (born June\n",
       "29, 1945) is an American\n",
       "former drug trafficker\n",
       "and former associate of\n",
       "Colombia's Medellin\n",
       "Cartel during the growth\n",
       "phase in cocaine\n",
       "trafficking, 1975–1985.\n",
       "Munday was featured in\n",
       "the 2006 Rakontur\n",
       "documentary, \"Cocaine\n",
       "Cowboys.\"</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row37_col1\" class=\"data row37 col1\" >Cocaine Cowboys was about\n",
       "Colombia's Medellin\n",
       "Cartel</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row37_col2\" class=\"data row37 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row37_col3\" class=\"data row37 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row37_col4\" class=\"data row37 col4\" >0.844000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row38\" class=\"row_heading level0 row38\" >4801</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row38_col0\" class=\"data row38 col0\" >Alan R Duffy (born 1983)\n",
       "is a professional\n",
       "astronomer and science\n",
       "communicator. He was born\n",
       "in England, raised in\n",
       "Ireland, and is currently\n",
       "based in Australia. He is\n",
       "a Research Fellow and\n",
       "Associate Professor at\n",
       "the Centre for\n",
       "Astrophysics and\n",
       "Supercomputing at\n",
       "Swinburne University of\n",
       "Technology.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row38_col1\" class=\"data row38 col1\" >Duffy can't read English</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row38_col2\" class=\"data row38 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row38_col3\" class=\"data row38 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row38_col4\" class=\"data row38 col4\" >0.774000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row39\" class=\"row_heading level0 row39\" >4810</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row39_col0\" class=\"data row39 col0\" >\"Paradise\" is the only\n",
       "single release from\n",
       "Styx's 1997 live double\n",
       "album \"Return to\n",
       "Paradise\". The song was\n",
       "originally written and\n",
       "recorded by Dennis\n",
       "DeYoung for his musical\n",
       "\"The Hunchback of Notre\n",
       "Dame\". The song was re-\n",
       "recorded by Styx for\n",
       "inclusion as one of three\n",
       "new studio tracks on the\n",
       "live album.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row39_col1\" class=\"data row39 col1\" >Paradise is one of 5\n",
       "singles</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row39_col2\" class=\"data row39 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row39_col3\" class=\"data row39 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row39_col4\" class=\"data row39 col4\" >0.494000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row40\" class=\"row_heading level0 row40\" >4894</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row40_col0\" class=\"data row40 col0\" >Avani Modi is an Indian\n",
       "model and film actress, a\n",
       "well-known face in Indian\n",
       "movies and theatre plays\n",
       "in Gujarati theatre She\n",
       "made her Bollywood debut\n",
       "in Madhur Bhandarkar's\n",
       "drama film \"Calendar\n",
       "Girls\", which is\n",
       "scheduled to release on\n",
       "25 September 2015. The\n",
       "movie is based upon the\n",
       "story of five girls and\n",
       "their journey as an\n",
       "annual calendar girl.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row40_col1\" class=\"data row40 col1\" >Avani Modi be in a film\n",
       "that will be released in\n",
       "2015 on September 25</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row40_col2\" class=\"data row40 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row40_col3\" class=\"data row40 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row40_col4\" class=\"data row40 col4\" >0.970000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row41\" class=\"row_heading level0 row41\" >4913</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row41_col0\" class=\"data row41 col0\" >Vincent Edward \"Bo\"\n",
       "Jackson (born November\n",
       "30, 1962) is a former\n",
       "baseball and American\n",
       "football player. He is\n",
       "one of the few athletes\n",
       "to be named an All-Star\n",
       "in two major sports, and\n",
       "the only one to do so in\n",
       "both baseball and\n",
       "football. He is widely\n",
       "considered one of the\n",
       "greatest athletes of all\n",
       "time.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row41_col1\" class=\"data row41 col1\" >Bo Jackson played in the\n",
       "MLS</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row41_col2\" class=\"data row41 col2\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row41_col3\" class=\"data row41 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row41_col4\" class=\"data row41 col4\" >0.700000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row42\" class=\"row_heading level0 row42\" >5170</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row42_col0\" class=\"data row42 col0\" >The Chullachaki (Quechua,\n",
       "\"one-footed\", from\n",
       "\"chulla\" or \"ch'ulla\" =\n",
       "single, odd, unpaired,\n",
       "asymmetric, \"chaki\" =\n",
       "foot; spelling sometimes\n",
       "also used in Spanish) or\n",
       "Chullachaqui\n",
       "(hispanicized spelling),\n",
       "also known as the\n",
       "Shapishico, is a mythical\n",
       "forest creature of the\n",
       "Peruvian and Brazilian\n",
       "Amazonian jungle.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row42_col1\" class=\"data row42 col1\" >The Chullachaki lives in\n",
       "the Brazilian Amazonian\n",
       "jungle.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row42_col2\" class=\"data row42 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row42_col3\" class=\"data row42 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row42_col4\" class=\"data row42 col4\" >0.961000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row43\" class=\"row_heading level0 row43\" >5214</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row43_col0\" class=\"data row43 col0\" >David Scott \"Dave\" Foley\n",
       "(born January 4, 1963) is\n",
       "a Canadian actor, stand-\n",
       "up comedian, director,\n",
       "producer and writer. He\n",
       "is known as a co-founder\n",
       "of the comedy group \"The\n",
       "Kids in the Hall\",\n",
       "responsible for their\n",
       "eponymous sketch show and\n",
       "the feature-length film\n",
       "\"\". He played Dave Nelson\n",
       "in the sitcom\n",
       "\"NewsRadio\", voiced Flik\n",
       "in \"A Bug's Life\" and\n",
       "hosted the game show\n",
       "\"Celebrity Poker\n",
       "Showdown\".</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row43_col1\" class=\"data row43 col1\" >David Foley is a\n",
       "Capricorn.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row43_col2\" class=\"data row43 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row43_col3\" class=\"data row43 col3\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row43_col4\" class=\"data row43 col4\" >0.984000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row44\" class=\"row_heading level0 row44\" >5377</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row44_col0\" class=\"data row44 col0\" >The Alhambra was an\n",
       "entertainment complex in\n",
       "Blackpool, Lancashire,\n",
       "England, from 1899–1903.\n",
       "A financial disaster, the\n",
       "building was bought by\n",
       "the Blackpool Tower\n",
       "Company and reopened in\n",
       "1904 as the Palace. The\n",
       "building was demolished\n",
       "in 1961 to make way for\n",
       "Lewis's department store.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row44_col1\" class=\"data row44 col1\" >The Alhambra was standing\n",
       "for 58 years</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row44_col2\" class=\"data row44 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row44_col3\" class=\"data row44 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row44_col4\" class=\"data row44 col4\" >0.672000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row45\" class=\"row_heading level0 row45\" >5652</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row45_col0\" class=\"data row45 col0\" >The Bavarian Mountain\n",
       "Hound (German =\n",
       "\"Bayerischer\n",
       "Gebirgsschweißhund\") is a\n",
       "breed of dog from\n",
       "Germany. As a scent\n",
       "hound, it has been used\n",
       "in Germany since the\n",
       "early 20th century to\n",
       "trail wounded game. It is\n",
       "a cross between the\n",
       "Bavarian Hound and the\n",
       "Hanover Hound.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row45_col1\" class=\"data row45 col1\" >the bavarian mountain\n",
       "hound was breed in the\n",
       "20th century and is a\n",
       "cross between the\n",
       "bavarian hound and the\n",
       "hanover hound</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row45_col2\" class=\"data row45 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row45_col3\" class=\"data row45 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row45_col4\" class=\"data row45 col4\" >0.892000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row46\" class=\"row_heading level0 row46\" >5669</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row46_col0\" class=\"data row46 col0\" >Shehzad Sheikh or Shahzad\n",
       "Sheikh is a Pakistani\n",
       "film and television actor\n",
       "and model, known for\n",
       "playing the lead role in\n",
       "the 2015 film \"Karachi Se\n",
       "Lahore\". He also starred\n",
       "in the series \"Annie Ki\n",
       "Ayegi Baraat\", \"Mi\n",
       "Raqsam\", and \"Mere\n",
       "Hamrahi\", and a TV film\n",
       "\"Main Kukkoo Aur woh\". He\n",
       "is the son of well-known\n",
       "actor Javed Sheikh.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row46_col1\" class=\"data row46 col1\" >The film Main Kukkoo Aur\n",
       "woh was released in 2015.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row46_col2\" class=\"data row46 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row46_col3\" class=\"data row46 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row46_col4\" class=\"data row46 col4\" >0.881000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row47\" class=\"row_heading level0 row47\" >5753</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row47_col0\" class=\"data row47 col0\" >Vampires appear\n",
       "throughout Stephen King's\n",
       "fictional multiverse.\n",
       "They appear in the novels\n",
       "\"'Salem's Lot\", \"\", \"\",\n",
       "and \"\"; the short stories\n",
       "\"One for the Road\", \"The\n",
       "Night Flier\", \"Popsy\",\n",
       "and \"The Little Sisters\n",
       "of Eluria\"; and are\n",
       "mentioned in a number of\n",
       "other stories. Marvel\n",
       "Comics' \"The Dark Tower:\n",
       "End-World Almanac\"\n",
       "includes a detailed entry\n",
       "on their categorization.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row47_col1\" class=\"data row47 col1\" >Stephen King is not a\n",
       "vampire.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row47_col2\" class=\"data row47 col2\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row47_col3\" class=\"data row47 col3\" >contradiction</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row47_col4\" class=\"data row47 col4\" >0.476000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row48\" class=\"row_heading level0 row48\" >5853</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row48_col0\" class=\"data row48 col0\" >Three Preludes is a\n",
       "ballet made for Mikhail\n",
       "Baryshnikov by Mark\n",
       "Morris to eponymous music\n",
       "by George Gershwin for\n",
       "his own company and\n",
       "presented as a piece\n",
       "d'occasion by the New\n",
       "York City Ballet. The\n",
       "performance took place\n",
       "June 16, 1992, at the New\n",
       "York State Theater,\n",
       "Lincoln Center.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row48_col1\" class=\"data row48 col1\" >Three Preludes has been\n",
       "performed at least once\n",
       "after 1996</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row48_col2\" class=\"data row48 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row48_col3\" class=\"data row48 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row48_col4\" class=\"data row48 col4\" >0.838000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_3777b158_a792_11ed_8264_acbc32902497level0_row49\" class=\"row_heading level0 row49\" >6092</th>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row49_col0\" class=\"data row49 col0\" >Thomas Morrison (born 30\n",
       "June 1983) is an English\n",
       "actor who has performed\n",
       "in theatre, TV and film.\n",
       "He is best known for his\n",
       "appearances in \"On the\n",
       "Shore of the Wide World\"\n",
       "and as Scripps in Cast B\n",
       "and C of Alan Bennett's\n",
       "\"The History Boys\".\n",
       "working alongside Steven\n",
       "Webb and Matt Smith.</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row49_col1\" class=\"data row49 col1\" >Thomas Morrison has\n",
       "performed in theatre, TV,\n",
       "film, and other platforms</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row49_col2\" class=\"data row49 col2\" >neutral</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row49_col3\" class=\"data row49 col3\" >entailment</td>\n",
       "                        <td id=\"T_3777b158_a792_11ed_8264_acbc32902497row49_col4\" class=\"data row49 col4\" >0.892000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fe378920d30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "EASY_THRESH = 0.75\n",
    "MEDIUM_THRESH = 0.5\n",
    "HARD_THRESH = 0.25\n",
    "IMPOSSIBLE_THRESH = 0.0\n",
    "\n",
    "COL_WIDTH = 25\n",
    "\n",
    "_, prediction_dict = load_confidences(BASE_PATH.replace('SEED',str(42)), predictions=True)\n",
    "\n",
    "\n",
    "HARD_THRESH = 0.25\n",
    "def show_examples(source_df, source_map, base_df=None, num=5):\n",
    "    \n",
    "    source_map = source_map[0]\n",
    "    \n",
    "    difficulty_dict = {}\n",
    "    \n",
    "    easy_mask = source_map.mean_conf > EASY_THRESH\n",
    "    medium_mask = (source_map.mean_conf > MEDIUM_THRESH) & (source_map.mean_conf < EASY_THRESH)\n",
    "    hard_mask = (source_map.mean_conf > HARD_THRESH) & (source_map.mean_conf < MEDIUM_THRESH)\n",
    "    impossible_mask = (source_map.mean_conf > IMPOSSIBLE_THRESH) & (source_map.mean_conf < 0.25)\n",
    "    \n",
    "    difficulty_dict['Easy'] = source_map[easy_mask]\n",
    "    difficulty_dict['Medium'] = source_map[medium_mask]\n",
    "    difficulty_dict['Hard'] = source_map[hard_mask]\n",
    "    difficulty_dict['Impossible'] = source_map[impossible_mask]\n",
    "    \n",
    "#     print(source_df[source_df.ID=='dfaed736-3361-4401-a646-3d7f501eaabd'])\n",
    "    \n",
    "#     print('Examples from classes EASY, MEDIUM, HARD, and IMPOSSIBLE for {} \\n'.format(source_map.Name.tolist()[0]))\n",
    "    \n",
    "#     print('\\nConfidence Thresholds:\\nEasy: {}\\nMedium: {}\\nHard: {}\\nImpossible: {}\\n'.format(EASY_THRESH, MEDIUM_THRESH, HARD_THRESH, IMPOSSIBLE_THRESH))\n",
    "    \n",
    "    id2label = {0: \"entailment\",\n",
    "                1: \"contradiction\",\n",
    "                2: \"neutral\"}\n",
    "    \n",
    "    for key in ['Impossible']:#difficulty_dict.keys():\n",
    "        \n",
    "        examples = source_df[source_df.ID.astype('str').isin(difficulty_dict[key].id.astype('string'))].head(num)\n",
    "        examples['Model Prediction'] = examples.ID.apply(lambda x: id2label[np.argmax(prediction_dict[x]['prediction'])])\n",
    "        examples['Model Confidence'] = examples.ID.apply(lambda x: round(np.max(prediction_dict[x]['prediction']),3))\n",
    "        \n",
    "        print(str(key).capitalize() + ' examples:')\n",
    "        examples = examples[['Premise', 'Hypothesis', 'Label', 'Model Prediction', 'Model Confidence']]\n",
    "        examples['Hypothesis'] = examples['Hypothesis'].str.wrap(COL_WIDTH).astype('string')\n",
    "        examples['Premise'] = examples['Premise'].str.wrap(COL_WIDTH).astype('string')\n",
    "        \n",
    "        examples = examples.style.set_properties(**{'text-align': 'left'})\n",
    "        examples = examples.set_table_styles([dict(selector = 'th', props=[('text-align', 'left')])])\n",
    "        display(examples)\n",
    "   \n",
    "\n",
    "show_examples(source_df=snli, source_map=snli_maps, base_df=None, num=50)\n",
    "show_examples(source_df=wanli, source_map=wanli_maps, base_df=None, num=50)\n",
    "show_examples(source_df=anli, source_map=anli_maps, base_df=None, num=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "extensive-basic",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Premise</th>\n",
       "      <th>Hypothesis</th>\n",
       "      <th>Label</th>\n",
       "      <th>ID</th>\n",
       "      <th>Dataset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4512</th>\n",
       "      <td>A piece of paper with a stamp on it is worth less than a piece of paper without a stamp.</td>\n",
       "      <td>A piece of paper without a stamp is worth more than a piece of paper with a stamp.</td>\n",
       "      <td>entailment</td>\n",
       "      <td>15138</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81229</th>\n",
       "      <td>When he is done, he will put the papers in a folder and seal it with a stamp.</td>\n",
       "      <td>The coppersmith is able to seal the folder.</td>\n",
       "      <td>neutral</td>\n",
       "      <td>152617</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                        Premise  \\\n",
       "4512   A piece of paper with a stamp on it is worth less than a piece of paper without a stamp.   \n",
       "81229             When he is done, he will put the papers in a folder and seal it with a stamp.   \n",
       "\n",
       "                                                                               Hypothesis  \\\n",
       "4512   A piece of paper without a stamp is worth more than a piece of paper with a stamp.   \n",
       "81229                                         The coppersmith is able to seal the folder.   \n",
       "\n",
       "            Label      ID Dataset  \n",
       "4512   entailment   15138   WANLI  \n",
       "81229     neutral  152617   WANLI  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wanli[wanli.Premise.str.contains(\"with a stamp\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "reliable-myrtle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.03274274617433548, 0.8178133368492126, 0.14944390952587128]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction_dict['15138']['prediction']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "future-quebec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Premise</th>\n",
       "      <th>Hypothesis</th>\n",
       "      <th>Label</th>\n",
       "      <th>ID</th>\n",
       "      <th>Dataset</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1005</th>\n",
       "      <td>I don't know.</td>\n",
       "      <td>The person said he didn't know.</td>\n",
       "      <td>neutral</td>\n",
       "      <td>356310</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1092</th>\n",
       "      <td>\"Must be right good stock,\" Fenner observed.</td>\n",
       "      <td>\"Must be pretty good stock,\" Fenner said.</td>\n",
       "      <td>entailment</td>\n",
       "      <td>332303</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1786</th>\n",
       "      <td>You can't make an omelet without breaking a few eggs.</td>\n",
       "      <td>We've never broken an egg.</td>\n",
       "      <td>neutral</td>\n",
       "      <td>112626</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3416</th>\n",
       "      <td>'What's the matter?'</td>\n",
       "      <td>What is the matter with you?</td>\n",
       "      <td>neutral</td>\n",
       "      <td>139025</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5062</th>\n",
       "      <td>You can't be serious.</td>\n",
       "      <td>I am very serious.</td>\n",
       "      <td>neutral</td>\n",
       "      <td>244858</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102585</th>\n",
       "      <td>It's the same as the others.</td>\n",
       "      <td>It's not as good as the others.</td>\n",
       "      <td>neutral</td>\n",
       "      <td>68153</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102644</th>\n",
       "      <td>You can do it.</td>\n",
       "      <td>I can do it.</td>\n",
       "      <td>neutral</td>\n",
       "      <td>201902</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102699</th>\n",
       "      <td>I'll be back in a minute.</td>\n",
       "      <td>I said I would be back in a few minutes.</td>\n",
       "      <td>entailment</td>\n",
       "      <td>10765</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102841</th>\n",
       "      <td>What is the point of this?</td>\n",
       "      <td>What is the point of this exercise?</td>\n",
       "      <td>entailment</td>\n",
       "      <td>200900</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103078</th>\n",
       "      <td>Are you sure?</td>\n",
       "      <td>Is it true?</td>\n",
       "      <td>neutral</td>\n",
       "      <td>293327</td>\n",
       "      <td>WANLI</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>850 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      Premise  \\\n",
       "1005                                            I don't know.   \n",
       "1092             \"Must be right good stock,\" Fenner observed.   \n",
       "1786    You can't make an omelet without breaking a few eggs.   \n",
       "3416                                     'What's the matter?'   \n",
       "5062                                    You can't be serious.   \n",
       "...                                                       ...   \n",
       "102585                           It's the same as the others.   \n",
       "102644                                         You can do it.   \n",
       "102699                              I'll be back in a minute.   \n",
       "102841                             What is the point of this?   \n",
       "103078                                          Are you sure?   \n",
       "\n",
       "                                       Hypothesis       Label      ID Dataset  \n",
       "1005              The person said he didn't know.     neutral  356310   WANLI  \n",
       "1092    \"Must be pretty good stock,\" Fenner said.  entailment  332303   WANLI  \n",
       "1786                   We've never broken an egg.     neutral  112626   WANLI  \n",
       "3416                 What is the matter with you?     neutral  139025   WANLI  \n",
       "5062                           I am very serious.     neutral  244858   WANLI  \n",
       "...                                           ...         ...     ...     ...  \n",
       "102585            It's not as good as the others.     neutral   68153   WANLI  \n",
       "102644                               I can do it.     neutral  201902   WANLI  \n",
       "102699   I said I would be back in a few minutes.  entailment   10765   WANLI  \n",
       "102841        What is the point of this exercise?  entailment  200900   WANLI  \n",
       "103078                                Is it true?     neutral  293327   WANLI  \n",
       "\n",
       "[850 rows x 5 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wanli[wanli.Premise.duplicated()]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "immune-bones",
   "metadata": {},
   "source": [
    "# Uncertainty metric:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "essential-homework",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_uncertainty(strategies, base_path, strategy_path):\n",
    "    \n",
    "    results = {key: [] for key in strategies}\n",
    "    \n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        print('Computing UNC for seed: {}'.format(seed))\n",
    "        \n",
    "        confidences = pd.read_pickle(base_path.replace('SEED', str(seed)))\n",
    "        \n",
    "        seed_entropies = {}\n",
    "        \n",
    "        sample_ids = list(confidences.keys())\n",
    "        for sample_id in sample_ids:\n",
    "    \n",
    "            times_correct = []\n",
    "\n",
    "            outputs = confidences[sample_id]\n",
    "            output = outputs[-1] # last epoch\n",
    "            \n",
    "            prediction = output[0]\n",
    "            pred_entropy = entropy(prediction)\n",
    "            seed_entropies[sample_id] = pred_entropy\n",
    "        \n",
    "        def get_entropies(x):\n",
    "            \n",
    "            if x in seed_entropies.keys():\n",
    "                return seed_entropies[x]\n",
    "            else:\n",
    "                return False\n",
    "        \n",
    "        for strategy in strategies:\n",
    "            \n",
    "            strategy_df = load_strategy_acquisitions(strategy=strategy, path=strategy_path.replace('SEED', str(seed)).replace('STRATEGY', str(strategy)))\n",
    "            strategy_df.ID = strategy_df.ID.astype('str')\n",
    "            \n",
    "            strategy_df['entropies'] = strategy_df.ID.apply(lambda x: get_entropies(x))\n",
    "            uncertainty = np.mean(strategy_df.entropies.tolist())\n",
    "            \n",
    "            results[strategy].append(uncertainty)\n",
    "        \n",
    "    return results\n",
    "\n",
    "uncertainty = compute_uncertainty(strategies=['random', 'bald', 'dal', 'mc-max-entropy'], base_path=BASE_PATH, strategy_path=STRATEGY_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "desperate-metabolism",
   "metadata": {},
   "outputs": [],
   "source": [
    "uncertainty = compute_uncertainty(strategies=['random', 'bald', 'dal', 'mc-max-entropy'], base_path=BASE_PATH, strategy_path=STRATEGY_PATH)\n",
    "\n",
    "for strategy in list(uncertainty.keys()):\n",
    "    print('mean unc for {}: {}, {}'.format(strategy, np.mean(uncertainty[strategy]), np.std(uncertainty[strategy])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "stainless-spyware",
   "metadata": {},
   "outputs": [],
   "source": [
    "for strategy in list(uncertainty.keys()):\n",
    "    print('mean unc for {}: {}, {}'.format(strategy, np.mean(uncertainty[strategy]), np.std(uncertainty[strategy])))\n",
    "#     uncertainty[strategy] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "molecular-niger",
   "metadata": {},
   "outputs": [],
   "source": [
    "strats = ['random', 'bald', 'dal', 'mc-max-entropy']\n",
    "\n",
    "CMAP2 = 'viridis'\n",
    "def plot_uncertainty(uncertainty, size=15, dpi=60):\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "\n",
    "    width = 1\n",
    "    alpha = ALPHA\n",
    "#     labels = np.array(range(len(uncertainty.keys())))\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(CMAP2)\n",
    "    \n",
    "    for i, key in enumerate(strats):\n",
    "        ax.bar(key, np.mean(uncertainty[key]), yerr=np.std(uncertainty[key]), width=width, label='{}'.format(key), color=cmap(1-(i/4)), alpha=alpha)\n",
    "    \n",
    "    ax.set_ylabel('Mean predictive entropy')\n",
    "    ax.set_xlabel('Strategy')\n",
    "    ax.set_xticks([])\n",
    "    ax.set_ylim(0.0, 0.6)\n",
    "    ax.set_title('Uncertainty per Strategy')\n",
    "    ax.legend(loc='upper left')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('uncertainty.pdf', format='pdf')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "maritime-scratch",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def plot_input_div(input_div, size=20, dpi=60):\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(size,size))\n",
    "    plt.rcParams['figure.dpi'] = dpi\n",
    "\n",
    "    width = 1\n",
    "    alpha = ALPHA\n",
    "    labels = np.array(range(len(uncertainty.keys())))\n",
    "\n",
    "    cmap = mpl.cm.get_cmap(CMAP2)\n",
    "    \n",
    "    for i, key in enumerate(strats):\n",
    "        ax.bar(key, input_div[key][0], yerr=input_div[key][1], width=width, label='{}'.format(key), color=cmap(1-(i/4)), alpha=alpha)\n",
    "\n",
    "    \n",
    "    ax.set_ylabel('Jaccard similarity')\n",
    "    ax.set_xlabel('Strategy')\n",
    "    ax.set_xticks([])\n",
    "    ax.set_ylim(0.2, 0.4)\n",
    "    ax.set_title('Input diversity per Strategy')\n",
    "    ax.legend(loc='upper left')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('input_div.pdf', format='pdf')\n",
    "    plt.show()\n",
    "\n",
    "# input_div = {'dal':[0.2761, 0.004296],\n",
    "# 'max-entropy': [0.3308, 0.01008],\n",
    "# 'random': [0.2756, 0.00295],\n",
    "# 'bald': [0.2203, 0.01226],\n",
    "# 'mc-max-entropy':[0.3227, 0.00429]}\n",
    "\n",
    "SIZE = 5\n",
    "DPI = 70\n",
    "FONT_SIZE = 15\n",
    "plt.rcParams.update({'font.size': FONT_SIZE})\n",
    "# plot_input_div(input_div, size=SIZE, dpi=DPI)\n",
    "plot_uncertainty(uncertainty, size=SIZE, dpi=DPI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "driven-assist",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
