DATA_DIR="TODO: replace with your directory"


# Extract Polish text corpora
echo "Extracting Polish text corpora..."
python -m src.datasets.wit.make_text_corpora \
    --input_dir $DATA_DIR/raw/wit/train \
    --output_path $DATA_DIR/interim/wit/wit_v1.corpora.pl.txt \
    --lang pl --min_wc 250


# Extract English text corpora
echo "Extracting English text corpora..."
python -m src.datasets.wit.make_text_corpora \
    --input_dir $DATA_DIR/raw/wit/train \
    --output_path $DATA_DIR/interim/wit/wit_v1.corpora.en.txt \
    --lang en --min_wc 250
