DATA_DIR="TODO: replace with your directory"

USER_AGENT="TODO: replace with your user agent string"
WIT_IMAGES_DIR=$DATA_DIR/static/raw/wit

# Crawl WIT images from wikicommons
echo "Crawling WIT images..."
python -m src.datasets.wit.crawl_images \
    --input_file $DATA_DIR/interim/wit/wit_v1.train_filtered.pl.jsonl \
    --images_dir $WIT_IMAGES_DIR \
    --remove_invalid \
    --user_agent $USER_AGENT \
    --workers 16 \

# Resize crawled images to max 2560x2560
echo "Resizing WIT images..."
python -m src.datasets.images.resize_images \
    --input_images_dir $WIT_IMAGES_DIR \
    --output_images_dir $WIT_IMAGES_DIR \
    --max_width_or_height 2560 \
    --workers 16