num_epochs: 1
eval_batches: 20

train_file: ../data/fincausal/processed/train.json
eval_file: ../data/fincausal/processed/dev.json
max_train_samples: null
max_eval_samples: null

# This is FCR's reward model used as the reward for FinCausal training
reward_model: ../evaluation/output/classifier/fcr-entailment-v3-xsmall-combined/
reward_type: entailment
max_reward_seq_length: 400

extraction_model: output/extraction/fincausal/flan-t5-large
max_generation_length: 128

output_dir: output/rl
run_name: fincausal-fcr-flan-t5-large-entailment
log_with: null

# Batch sizes need to be lower for models larger than T5-Base
reward_batch_size: 16
ppo_batch_size: 16
ppo_minibatch_size: 16

# PPO parameters
kl_penalty: kl
adaptive_kl_ctrl: true
init_kl_coef: 0.4
use_reward_scaling: false
use_reward_norm: false

# Greedy decoding
generation_top_k: null
generation_top_p: null
degeneration_penalty: null
generation_do_sample: true
generation_num_beams: 2
