{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "248600f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "\n",
    "task = \"storycloze\"\n",
    "all_re = []\n",
    "for pattern_id in [0]:\n",
    "    patter_re = []\n",
    "    for bs in [2, 4, 8]:\n",
    "        for lr in [\"1e-5\", \"2e-5\", \"3e-5\"]:\n",
    "            outdir = f\"/n/fs/nlp-mengzhou/space3/out/pet/{task}/pattern{pattern_id}/shot32/bs{bs}_lr{lr}\"\n",
    "            file = os.path.join(outdir, \"bert-base-uncased-log.txt\")\n",
    "            lines = open(file, \"r\").readlines()\n",
    "            lines = [line for line in lines if \"EVAL\" in line ]\n",
    "            try:\n",
    "                res = [eval(line[line.index(\"EVAL\"):].strip()[6:]) for line in lines]\n",
    "            \n",
    "                collected_re = []\n",
    "                for seed in [3]:\n",
    "                    eval_re = [re[\"fewshot_validation\"] for re in res if re[\"seed\"] == seed]\n",
    "                    fewshot_eval_re = [re[\"validation\"] for re in res if re[\"seed\"] == seed]\n",
    "                    index = np.argmax(eval_re)\n",
    "                    best_re = fewshot_eval_re[index]\n",
    "                    patter_re.append(best_re*100)\n",
    "            except:\n",
    "                print(outdir)\n",
    "    all_re.append(patter_re)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "730a2cd6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55.95 (0.3)\n"
     ]
    }
   ],
   "source": [
    "for row in np.array(all_re):\n",
    "    score = np.sort(row)[-3:]\n",
    "    print(f\"{round(np.mean(score), 2)} ({round(np.std(score), 1)})\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0910f2c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/n/fs/nlp-mengzhou/space3/out/pet/hellaswag/pattern2/shot32/bs2_lr2e-5/roberta-large/p2-i0\n",
      "/n/fs/nlp-mengzhou/space3/out/pet/hellaswag/pattern2/shot32/bs4_lr1e-5/roberta-large/p2-i0\n",
      "/n/fs/nlp-mengzhou/space3/out/pet/hellaswag/pattern2/shot32/bs4_lr2e-5/roberta-large/p2-i0\n",
      "/n/fs/nlp-mengzhou/space3/out/pet/hellaswag/pattern2/shot32/bs4_lr3e-5/roberta-large/p2-i0\n",
      "/n/fs/nlp-mengzhou/space3/out/pet/hellaswag/pattern2/shot32/bs8_lr1e-5/roberta-large/p2-i0\n",
      "/n/fs/nlp-mengzhou/space3/out/pet/hellaswag/pattern2/shot32/bs8_lr2e-5/roberta-large/p2-i0\n",
      "/n/fs/nlp-mengzhou/space3/out/pet/hellaswag/pattern2/shot32/bs8_lr3e-5/roberta-large/p2-i0\n"
     ]
    }
   ],
   "source": [
    "task = \"hellaswag\"\n",
    "all_re = []\n",
    "for pattern_id in [2]:\n",
    "    patter_re = []\n",
    "    for bs in [2, 4, 8]:\n",
    "        for lr in [\"1e-5\", \"2e-5\", \"3e-5\"]:\n",
    "            outdir = f\"/n/fs/nlp-mengzhou/space3/out/pet/{task}/pattern{pattern_id}/shot32/bs{bs}_lr{lr}/roberta-large/p2-i0\"\n",
    "            file = os.path.join(outdir, \"eval-log.txt\")\n",
    "            lines = open(file, \"r\").readlines()\n",
    "            lines = [line for line in lines if \"'acc'\" in line ]\n",
    "            try:\n",
    "                re = eval(lines[-1].strip().split(\" - \")[-1])[\"acc\"]\n",
    "                patter_re.append(re*100)\n",
    "            except:\n",
    "                print(outdir)\n",
    "    all_re.append(patter_re)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a9d73db3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31.25 (0.0)\n"
     ]
    }
   ],
   "source": [
    "for row in np.array(all_re):\n",
    "    score = np.sort(row)[-3:]\n",
    "    print(f\"{round(np.mean(score), 2)} ({round(np.std(score), 1)})\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "eb1d54ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[31.23879705238,\n",
       "  29.396534554869547,\n",
       "  28.599880501892056,\n",
       "  31.228838876717784,\n",
       "  30.551682931686912,\n",
       "  29.376618203545114,\n",
       "  31.288587930691097,\n",
       "  30.90021907986457,\n",
       "  29.61561441943836]]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "55fa3ca7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.3266499161421599"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.std(final_re)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a02ccb1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "def load_jsonl(file):\n",
    "    examples = []\n",
    "    for line in open(file, \"r\"):\n",
    "        example = json.loads(line.strip())\n",
    "        examples.append(example)\n",
    "    return examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "494ecd48",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_record_dev():\n",
    "    fewglue_record_train = load_jsonl(\"/n/fs/nlp-mengzhou/space3/fewglue/FewGLUE/ReCoRD/train.jsonl\")\n",
    "    record_train = load_jsonl(\"/n/fs/nlp-mengzhou/space3/data/downstream/SuperGLUE/ReCoRD/train.jsonl\")\n",
    "    record_train_text = [re[\"passage\"][\"text\"] for re in record_train] \n",
    "    fewglue_record_train_text = [re[\"passage\"][\"text\"] for re in fewglue_record_train] \n",
    "    for text in fewglue_record_train_text:\n",
    "        while text in record_train_text:\n",
    "            index = record_train_text.index(text)\n",
    "            record_train.pop(index)\n",
    "            record_train_text.pop(index)\n",
    "    dev_examples = np.random.permutation(record_train)[:32]\n",
    "    outfile = \"/n/fs/nlp-mengzhou/space3/fewglue/FewGLUE/ReCoRD/fewshot_val.jsonl\"\n",
    "    with open(outfile, \"w\") as f:\n",
    "        for example in dev_examples:\n",
    "            f.write(f\"{json.dumps(example)}\\n\")            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "de6d21f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_copa_dev():\n",
    "    fewglue_record_train = load_jsonl(\"/n/fs/nlp-mengzhou/space3/fewglue/FewGLUE/COPA/train.jsonl\")\n",
    "    record_train = load_jsonl(\"/n/fs/nlp-mengzhou/space3/data/downstream/SuperGLUE/COPA/train.jsonl\")\n",
    "    record_train_text = [re[\"premise\"] for re in record_train] \n",
    "    fewglue_record_train_text = [re[\"premise\"] for re in fewglue_record_train] \n",
    "    for text in fewglue_record_train_text:\n",
    "        while text in record_train_text:\n",
    "            index = record_train_text.index(text)\n",
    "            record_train.pop(index)\n",
    "            record_train_text.pop(index)\n",
    "    dev_examples = np.random.permutation(record_train)[:32]\n",
    "    outfile = \"/n/fs/nlp-mengzhou/space3/fewglue/FewGLUE/COPA/fewshot_val.jsonl\"\n",
    "    with open(outfile, \"w\") as f:\n",
    "        for example in dev_examples:\n",
    "            f.write(f\"{json.dumps(example)}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "54321f01",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_storycloze_train_dev():\n",
    "    train = open(\"/n/fs/nlp-mengzhou/space3/data/downstream/storycloze/spring2016.val.tsv\", \"r\").readlines()\n",
    "    permutated_train_examples = np.random.permutation(train)\n",
    "    train_examples = permutated_train_examples[:32]\n",
    "    dev_examples = permutated_train_examples[32:64]\n",
    "    outfile = \"/n/fs/nlp-mengzhou/space3/data/downstream/storycloze/fewshot_train.jsonl\"\n",
    "    with open(outfile, \"w\") as f:\n",
    "        for example in train_examples:\n",
    "            f.write(f\"{example}\")\n",
    "    outfile = \"/n/fs/nlp-mengzhou/space3/data/downstream/storycloze/fewshot_val.jsonl\"\n",
    "    with open(outfile, \"w\") as f:\n",
    "        for example in dev_examples:\n",
    "            f.write(f\"{example}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "3c6db28c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_piqa_train_dev():\n",
    "    train = load_jsonl(\"/n/fs/nlp-mengzhou/space3/data/downstream/piqa/train.jsonl\")\n",
    "    train_label = open(\"/n/fs/nlp-mengzhou/space3/data/downstream/piqa/train-labels.lst\", \"r\").readlines()\n",
    "    train_label = [int(i) for i in train_label]\n",
    "    index = np.random.permutation(len(train))\n",
    "    train_index = index[:32]\n",
    "    valid_index = index[32:64]\n",
    "    train_examples = [train[i] for i in train_index]\n",
    "    dev_examples = [train[i] for i in valid_index]\n",
    "    train_labels = [train_label[i] for i in train_index]\n",
    "    dev_labels = [train_label[i] for i in valid_index]\n",
    "    outfile = \"/n/fs/nlp-mengzhou/space3/data/downstream/piqa/fewshot_train.jsonl\"\n",
    "    with open(outfile, \"w\") as f:\n",
    "        for example, label in zip(train_examples, train_labels):\n",
    "            example[\"label\"] = label\n",
    "            f.write(f\"{json.dumps(example)}\\n\")\n",
    "    outfile = \"/n/fs/nlp-mengzhou/space3/data/downstream/piqa/fewshot_val.jsonl\"\n",
    "    with open(outfile, \"w\") as f:\n",
    "        for example, label in zip(dev_examples, dev_labels):\n",
    "            example[\"label\"] = label\n",
    "            f.write(f\"{json.dumps(example)}\\n\")\n",
    "    valid = load_jsonl(\"/n/fs/nlp-mengzhou/space3/data/downstream/piqa/valid.jsonl\")\n",
    "    valid_labels = open(\"/n/fs/nlp-mengzhou/space3/data/downstream/piqa/valid-labels.lst\", \"r\").readlines()\n",
    "    valid_labels = [int(i) for i in valid_labels]\n",
    "    outfile = \"/n/fs/nlp-mengzhou/space3/data/downstream/piqa/val.jsonl\"\n",
    "    with open(outfile, \"w\") as f:\n",
    "        for example, label in zip(valid, valid_labels):\n",
    "            example[\"label\"] = label\n",
    "            f.write(f\"{json.dumps(example)}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d9ef205c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_hellaswag_train_dev():\n",
    "    base_dir = \"/n/fs/nlp-mengzhou/space3/data/downstream/hellaswag\"\n",
    "    train = load_jsonl(f\"{base_dir}/hellaswag_train.jsonl\")\n",
    "    index = np.random.permutation(len(train))\n",
    "    train_index = index[:32]\n",
    "    valid_index = index[32:64]\n",
    "    train_examples = [train[i] for i in train_index]\n",
    "    dev_examples = [train[i] for i in valid_index]\n",
    "    outfile = f\"{base_dir}/fewshot/fewshot_train.jsonl\"\n",
    "    with open(outfile, \"w\") as f:\n",
    "        for example in train_examples:\n",
    "            f.write(f\"{json.dumps(example)}\\n\")\n",
    "    outfile = f\"{base_dir}/fewshot/fewshot_val.jsonl\"\n",
    "    with open(outfile, \"w\") as f:\n",
    "        for example in dev_examples:\n",
    "            f.write(f\"{json.dumps(example)}\\n\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c72096ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "build_hellaswag_train_dev()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "62a729c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "build_piqa_train_dev()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b674a940",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7205b350",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "65677"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(record_train_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "ddd398b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/n/fs/nlp-mengzhou/anaconda3/envs/pytorch17/lib/python3.7/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:100.)\n",
      "  return torch._C._cuda_getDeviceCount() > 0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4be51d6dac64b1886e7f02d8767d559",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=482.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoConfig\n",
    "config = AutoConfig.from_pretrained(\"roberta-base\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94744d96",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
