root=./
export PYTHONPATH=$root:$PYTHONPATH

TEXT=$root/data-bin/wmt19_ch_en
python $root/fairseq_cli/preprocess.py \
    --source-lang ch --target-lang en \
    --trainpref $TEXT/train.ch-en --validpref $TEXT/valid.ch-en \
    --testpref $TEXT/test.ch-en \
    --srcdict $TEXT/dict.ch.txt --tgtdict $TEXT/dict.en.txt \
    --nwordssrc 47751 --nwordstgt 32111 \
    --destdir $root/data-bin/wmt19_ch_en \
    --workers 20
