#!/bin/bash
set -euo pipefail

export CUDA_VISIBLE_DEVICES=1

export HF_TOKEN=""   # TODO: DO NOT COMMIT THIS

export DATA_DIR="TODO: replace with your directory"
export MODELS_DIR="/nfsclient/os5800/vlm_project/data/pllum-vlm/models"
export RESULTS_DIR="/nfsclient/os5800/vlm_project/data/pllum-vlm/results"


MODELS=(
    # Baseline models:
    # "llava-hf/llava-v1.6-mistral-7b-hf"
    # "llava-hf/llava-v1.6-vicuna-13b-hf"
    # "llava-hf/llava-onevision-qwen2-7b-si-hf"
    # "google/paligemma2-3b-mix-224"
    "google/paligemma2-10b-mix-224"
    # "mistralai/Pixtral-12B-2409"
    # "Qwen/Qwen2.5-VL-7B-Instruct"

    # Our models:
    # ${MODELS_DIR}/finetuned/pllum-12b-nc-instruct-250715-siglip2-so400m-patch14-384-llava-next-finetuned-2M-lora-layers_all-r128-2025-10-04_12-30-10
    # ${MODELS_DIR}/finetuned/bielik-11b-v2.6-instruct-siglip2-so400m-patch14-384-llava-next-finetuned-2025-12-03_13-38-36
    # ${MODELS_DIR}/finetuned/pllum-12b-nc-instruct-siglip2-so400m-patch14-384-llava-next-finetuned-2025-12-05_01-47-54
    # ${MODELS_DIR}/finetuned/pllum-12b-nc-instruct-250715-siglip2-so400m-patch14-384-llava-next-finetuned-2025-12-16_10-00-08
)


for MODEL_NAME_OR_PATH in "${MODELS[@]}"; do
    MODEL_NAME=$(basename "${MODEL_NAME_OR_PATH}")
    echo "Evaluating model: ${MODEL_NAME_OR_PATH}"

    python -m src.datasets.pllum_vlm_eval.make_evaluations \
        --model_name_or_path="${MODEL_NAME_OR_PATH}" \
        --input_file="${DATA_DIR}/raw/pllum-vlm-eval-1.0/pllum-vlm-eval.json" \
        --images_dir="${DATA_DIR}/static/raw/pllum-vlm-eval-1.0" \
        --output_file="${RESULTS_DIR}/pllum-vlm-eval-1.0/${MODEL_NAME}.json"
done
