export DATA_DIR="TODO: replace with your directory"


# SynthDogPL 500k

python -m src.datasets.synthdog.make_conversations \
    --input_path ${DATA_DIR}/static/raw/synthdog/SynthDoG_pl_500k/train/metadata.jsonl \
    --instructions_path ${DATA_DIR}/interim/synthdog/instructions-pl.json \
    --output_path ${DATA_DIR}/interim/synthdog/synthdog_pl_500k.jsonl \
    --source "synthdog-pl-500k" \
    --seed 42

export SYNTHDOG_DIR="${DATA_DIR}/static/raw/synthdog/SynthDoG_pl_100k"

# Process conversations dataset
python -m src.datasets.synthdog.process_instruct \
    --input_path ${SYNTHDOG_DIR}/train/metadata.jsonl \
    --output_path ${DATA_DIR}/processed/instruct/synthdog_pl_100k.jsonl \
    --random_state 42

# Process static images
rm -rf ${DATA_DIR}/static/pretrain/synthdog

python -m src.datasets.images.shard_images \
    --input_path ${DATA_DIR}/processed/pretrain/synthdog_pl.jsonl \
    --input_images_dir ${SYNTHDOG_DIR}/train \
    --output_images_dir ${DATA_DIR}/static/pretrain/synthdog \
    --images_per_shard 512