{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1db4eafe",
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "colors = sns.color_palette(as_cmap=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cbdb4963",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7b900ecf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path\n",
    "\n",
    "from google.auth.transport.requests import Request\n",
    "from google.oauth2.credentials import Credentials\n",
    "from google_auth_oauthlib.flow import InstalledAppFlow\n",
    "from googleapiclient.discovery import build\n",
    "from googleapiclient.errors import HttpError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "784acb26",
   "metadata": {},
   "outputs": [],
   "source": [
    "SAMPLE_SPREADSHEET_ID = \"1Vy4DplZAUlB0KENHUKb4A95aSNM5w7ezm2Vnl3qAG6s\"\n",
    "SCOPES = ['https://www.googleapis.com/auth/spreadsheets.readonly']\n",
    "creds = Credentials.from_authorized_user_file('/n/fs/nlp-mengzhou/space3/hf/auth/token.json', SCOPES)\n",
    "service = build('sheets', 'v4', credentials=creds)\n",
    "sheet = service.spreadsheets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4f9f1d9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results(data_range):\n",
    "    result = sheet.values().get(spreadsheetId=SAMPLE_SPREADSHEET_ID, range=data_range).execute()\n",
    "    data = np.array(result[\"values\"])\n",
    "    data = np.stack([pd.to_numeric(d, errors='coerce') for d in data])\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "052c11c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_range(task, i):\n",
    "    if task == \"mnli\":\n",
    "        last_col = \"T\"\n",
    "    else:\n",
    "        last_col = \"N\"\n",
    "    if i == 0:\n",
    "        return f\"fewshot-{task}!C4:{last_col}12\"\n",
    "    elif i == 1:\n",
    "        return f\"fewshot-{task}!C16:{last_col}24\"\n",
    "    elif i == 2:\n",
    "        return f\"fewshot-{task}!C28:{last_col}36\"\n",
    "    else:\n",
    "        return f\"fewshot-{task}!C40:{last_col}48\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cf77fb57",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_range_v2(task):\n",
    "    if task == \"sst2\":\n",
    "        return f\"raw-fewshot-v2!C3:F15\"\n",
    "    elif task == \"sst5\":\n",
    "        return f\"raw-fewshot-v2!G3:J12\"\n",
    "    elif task == \"mr\":\n",
    "        return f\"raw-fewshot-v2!K3:N15\"\n",
    "    elif task == \"mnli\":\n",
    "        return f\"raw-fewshot-v2!C19:F31\"\n",
    "    elif task == \"snli\":\n",
    "        return f\"raw-fewshot-v2!K19:N31\"\n",
    "    elif task == \"qnli\":\n",
    "        return f\"raw-fewshot-v2!S19:V31\"\n",
    "    elif task == \"boolq\":\n",
    "        return f\"raw-fewshot-v2!C35:F47\"\n",
    "    elif task == \"agnews\":\n",
    "        return f\"raw-fewshot-v2!G35:J47\"\n",
    "    elif task == \"rte\":\n",
    "        return f\"raw-fewshot-v2!O19:R28\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "023b7ed7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_data(task, data):\n",
    "    def get_k(i):\n",
    "        if i < 4:\n",
    "            return 16\n",
    "        elif i < 7:\n",
    "            return 32\n",
    "        elif i < 10:\n",
    "            return 256\n",
    "        else:\n",
    "            return 1024\n",
    "    def get_model(i):\n",
    "        if i in [0, 4, 7, 10]:\n",
    "            return \"bert\"\n",
    "        elif i in [1, 5, 8, 11]:\n",
    "            return \"roberta\"\n",
    "        elif i in [2]:\n",
    "            return \"roberta-large\"\n",
    "        else:\n",
    "            return \"electra\"\n",
    "    def get_type(j):\n",
    "        if j == 0:\n",
    "            return \"standard_FT\"\n",
    "        else:\n",
    "            return \"prompt_FT\"\n",
    "    datas = []\n",
    "    for i, row in enumerate(data):\n",
    "        for j in [0, 2]:\n",
    "            k = get_k(i)\n",
    "            model = get_model(i)\n",
    "            t = get_type(j)\n",
    "            dev = data[i][j]\n",
    "            std = data[i][j+1]\n",
    "            datas.append([task, k, model, t, dev, std])\n",
    "    data = pd.DataFrame(data=datas, columns=[\"task\", \"k\", \"model\", \"training type\", \"dev\", \"std\"])\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ae2c61c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_type(i, num_cols):\n",
    "    index = i // num_cols\n",
    "    \n",
    "    index1 = index % 2\n",
    "    if index1 == 0:\n",
    "        training_type = \"standard_FT\"\n",
    "    elif index1 == 1:\n",
    "        training_type = \"prompt_FT\"\n",
    "    \n",
    "    index2 = (index // 2) % 3\n",
    "    if index2 == 0:\n",
    "        bert = \"bert\"\n",
    "    elif index2 == 1:\n",
    "        bert = \"roberta\"\n",
    "    else:\n",
    "        bert = \"electra\"\n",
    "    return training_type, bert\n",
    "\n",
    "rows = []\n",
    "tasks = [\"sst2\", \"sst5\", \"mr\", \"mnli\", \"qnli\", \"snli\", \"boolq\", \"agnews\", \"rte\"]\n",
    "\n",
    "datas = []\n",
    "for task in tasks:\n",
    "    data = process_data(task, get_results(get_range_v2(task)))\n",
    "    datas.append(data)\n",
    "data = pd.concat(datas)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0d4c95f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = sheet.values().get(spreadsheetId=SAMPLE_SPREADSHEET_ID, range=\"raw-zeroshot!A2:D12\").execute()\n",
    "zero_re = pd.DataFrame(data=result[\"values\"], columns=[\"task\", \"bert\", \"roberta\", \"electra\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b8c29aa1",
   "metadata": {},
   "outputs": [],
   "source": [
    "c = sns.color_palette()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a048d6b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "ks = [16, 32, 256, 1024]\n",
    "task_matching = {\"sst2\": \"SST-2\", \"sst5\": \"SST-5\", \"snli\": \"SNLI\", \"qnli\": \"QNLI\", \"agnews\": \"AGNews\", \"boolq\": \"BoolQ\", \"mr\": \"MR\", \"rte\": \"RTE\", \"mnli\": \"MNLI\"}\n",
    "def pplot(data, task, ax):\n",
    "    task_data = data[data[\"task\"] == task]\n",
    "    for model in [\"roberta\", \"electra\"]:\n",
    "        for training_type in [\"standard_FT\", \"prompt_FT\"]:\n",
    "            raw = task_data[(task_data[\"model\"] == model) & (task_data[\"training type\"] == training_type)]\n",
    "            dev = raw[\"dev\"]\n",
    "            std = raw[\"std\"]\n",
    "            xlabel = ks[:len(dev)]\n",
    "            x = list(range(1, len(dev)+1))\n",
    "            y = dev\n",
    "#             if training_type == \"prompt_FT\":\n",
    "#                 zero_shot_perf = float(zero_re[zero_re[\"task\"] == task][model].values[0])\n",
    "#                 x = [0] + x\n",
    "#                 xlabel = [0] + xlabel\n",
    "#                 y = np.concatenate([[zero_shot_perf], y])\n",
    "            label = \"RoBERTa\" if model.startswith(\"r\") else \"ELECTRA\"\n",
    "            label += \" Standard FT\" if training_type.startswith(\"s\") else \" Prompt FT\"\n",
    "            color = c[0] if model == \"roberta\" else c[1]\n",
    "            g = sns.lineplot(ax=ax, x=x, y=y, marker='o', markersize=10, label=label, legend=False, ci=None, color=color, linestyle=\"--\" if training_type == \"standard_FT\" else \"-\" )\n",
    "            g.tick_params(labelsize=15)\n",
    "#             plt.setp(g.get_legend().get_texts(), fontsize='15')\n",
    "#             if training_type == \"prompt_FT\":\n",
    "#                 ax.fill_between(x[1:], dev-std, dev+std, alpha=0.2, color=color)\n",
    "#             else:\n",
    "            ax.fill_between(x, dev-std, dev+std, alpha=0.2, color=color)\n",
    "            ax.set_title(task_matching[task], fontsize=24)\n",
    "            ax.set_xlabel(\"K\", fontsize=24)\n",
    "            ax.set_ylabel(\"Accuracy (%)\", fontsize=24)\n",
    "        ax.set_xticks([1, 2, 3, 4], labels=ks, fontsize=24)\n",
    "        ax.tick_params(axis='y', labelsize=24 )\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "c2bfee40",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>k</th>\n",
       "      <th>model</th>\n",
       "      <th>training type</th>\n",
       "      <th>dev</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sst2</td>\n",
       "      <td>16</td>\n",
       "      <td>bert</td>\n",
       "      <td>standard_FT</td>\n",
       "      <td>72.77</td>\n",
       "      <td>6.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sst2</td>\n",
       "      <td>16</td>\n",
       "      <td>bert</td>\n",
       "      <td>prompt_FT</td>\n",
       "      <td>84.56</td>\n",
       "      <td>1.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sst2</td>\n",
       "      <td>16</td>\n",
       "      <td>roberta</td>\n",
       "      <td>standard_FT</td>\n",
       "      <td>84.54</td>\n",
       "      <td>2.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sst2</td>\n",
       "      <td>16</td>\n",
       "      <td>roberta</td>\n",
       "      <td>prompt_FT</td>\n",
       "      <td>89.88</td>\n",
       "      <td>0.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sst2</td>\n",
       "      <td>16</td>\n",
       "      <td>roberta-large</td>\n",
       "      <td>standard_FT</td>\n",
       "      <td>85.92</td>\n",
       "      <td>4.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>rte</td>\n",
       "      <td>256</td>\n",
       "      <td>bert</td>\n",
       "      <td>prompt_FT</td>\n",
       "      <td>63.83</td>\n",
       "      <td>2.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>rte</td>\n",
       "      <td>256</td>\n",
       "      <td>roberta</td>\n",
       "      <td>standard_FT</td>\n",
       "      <td>66.06</td>\n",
       "      <td>3.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>rte</td>\n",
       "      <td>256</td>\n",
       "      <td>roberta</td>\n",
       "      <td>prompt_FT</td>\n",
       "      <td>71.48</td>\n",
       "      <td>1.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>rte</td>\n",
       "      <td>256</td>\n",
       "      <td>electra</td>\n",
       "      <td>standard_FT</td>\n",
       "      <td>71.55</td>\n",
       "      <td>2.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>rte</td>\n",
       "      <td>256</td>\n",
       "      <td>electra</td>\n",
       "      <td>prompt_FT</td>\n",
       "      <td>77.04</td>\n",
       "      <td>1.79</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>222 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    task    k          model training type    dev   std\n",
       "0   sst2   16           bert   standard_FT  72.77  6.43\n",
       "1   sst2   16           bert     prompt_FT  84.56  1.01\n",
       "2   sst2   16        roberta   standard_FT  84.54  2.25\n",
       "3   sst2   16        roberta     prompt_FT  89.88  0.63\n",
       "4   sst2   16  roberta-large   standard_FT  85.92  4.14\n",
       "..   ...  ...            ...           ...    ...   ...\n",
       "15   rte  256           bert     prompt_FT  63.83  2.65\n",
       "16   rte  256        roberta   standard_FT  66.06  3.49\n",
       "17   rte  256        roberta     prompt_FT  71.48  1.09\n",
       "18   rte  256        electra   standard_FT  71.55  2.28\n",
       "19   rte  256        electra     prompt_FT  77.04  1.79\n",
       "\n",
       "[222 rows x 6 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d4fe86b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f5fcfd8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 205584 characters]\n",
      "text/plain": [
       "<Figure size 936x648 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.set_style(\"whitegrid\", {'axes.grid' : False})\n",
    "selected_tasks = [\"sst2\", \"sst5\", \"snli\", \"qnli\"]\n",
    "# selected_tasks = [\"boolq\", \"rte\", \"agnews\", \"mr\"]\n",
    "fig, axes = plt.subplots(2, 2, figsize=(13, 9), sharex=True)\n",
    "for i in range(len(selected_tasks)):\n",
    "    row = i // 2\n",
    "    column = i % 2\n",
    "    ax = axes[row][column]\n",
    "    pplot(data, selected_tasks[i], ax)\n",
    "fig.tight_layout(pad=0.8)\n",
    "#legend\n",
    "ax1 = axes[0][0]\n",
    "lines_labels = [ax1.get_legend_handles_labels()]\n",
    "lines, labels = [sum(lol, []) for lol in zip(*lines_labels)]\n",
    "leg = fig.legend(lines, labels,  ncol=2, loc=\"upper center\", fontsize=24, bbox_to_anchor=(0.5, 0.5, 0, 0.65))\n",
    "\n",
    "plt.savefig(\"images/kplot.pdf\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22a4d15a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "b83268f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>k</th>\n",
       "      <th>model</th>\n",
       "      <th>training type</th>\n",
       "      <th>dev</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [task, k, model, training type, dev, std]\n",
       "Index: []"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data[\"task\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "4c0e42e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>k</th>\n",
       "      <th>model</th>\n",
       "      <th>training type</th>\n",
       "      <th>k_dev</th>\n",
       "      <th>dev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sst2</td>\n",
       "      <td>16</td>\n",
       "      <td>roberta</td>\n",
       "      <td>prompt_FT</td>\n",
       "      <td>0.9688</td>\n",
       "      <td>0.8922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>sst2</td>\n",
       "      <td>32</td>\n",
       "      <td>roberta</td>\n",
       "      <td>prompt_FT</td>\n",
       "      <td>0.9219</td>\n",
       "      <td>0.8670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>sst2</td>\n",
       "      <td>256</td>\n",
       "      <td>roberta</td>\n",
       "      <td>prompt_FT</td>\n",
       "      <td>0.9062</td>\n",
       "      <td>0.9106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>sst2</td>\n",
       "      <td>1024</td>\n",
       "      <td>roberta</td>\n",
       "      <td>prompt_FT</td>\n",
       "      <td>0.9116</td>\n",
       "      <td>0.9300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    task     k    model training type   k_dev     dev\n",
       "3   sst2    16  roberta     prompt_FT  0.9688  0.8922\n",
       "9   sst2    32  roberta     prompt_FT  0.9219  0.8670\n",
       "15  sst2   256  roberta     prompt_FT  0.9062  0.9106\n",
       "21  sst2  1024  roberta     prompt_FT  0.9116  0.9300"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "task_data[(task_data[\"model\"] == \"roberta\") & (task_data[\"training type\"] == \"prompt_FT\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "a70f5e2a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sst2': 0,\n",
       " 'sst5': 1,\n",
       " 'snli': 22,\n",
       " 'mnli': 8,\n",
       " 'rte': 12,\n",
       " 'qnli': 7,\n",
       " 'ag_news': 31,\n",
       " 'boolq': 23,\n",
       " 'mr': 4,\n",
       " 'imdb': 2}"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{'sst2': 0, 'sst5': 1, 'snli': 22, 'mnli': 8, 'rte': 12, 'qnli': 7, 'ag_news': 31, 'boolq': 23, 'mr': 4, 'imdb': 2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "1032973c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(np.array([np.nan, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "333b2af8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False,  True])"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "~np.isnan(np.array([np.nan, 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6f504d26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['sst2', 'sst5', 'mr', 'imdb', 'mnli', 'qnli', 'snli', 'rte',\n",
       "       'boolq', 'agnews'], dtype=object)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"task\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e13cea0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "updated",
   "language": "python",
   "name": "updated"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
