#! /usr/bin/zsh

# Create k-shot splits from training data.
# NOTE: This was done for k = 8, 32 after arxiv submission.

dataset=data/FUGUE/QQP/
# for dataset in data/FUGUE/* ; do
  # Temporarily shuffle and split the training data, so we know there is no
  # train/dev overlap
  shuf $dataset/train.all.jsonl > /tmp/shuffled;
  split -n l/2 /tmp/shuffled /tmp/shuffled.;
  for k in 8 32 ; do
    python scripts/truncate.py -k $k /tmp/shuffled.aa $dataset/train.$k.jsonl
    python scripts/truncate.py -k $k /tmp/shuffled.ab $dataset/dev.$k.jsonl
  done
  rm /tmp/shuffled*
# done
