root=./
export PYTHONPATH=$root:$PYTHONPATH

TEXT=$root/data-bin/wmt14_en_de
python $root/fairseq_cli/preprocess.py \
    --source-lang en --target-lang de \
    --trainpref $TEXT/train.en-de --validpref $TEXT/valid.en-de \
    --testpref $TEXT/test.en-de \
    --srcdict $TEXT/dict.en.txt --tgtdict $TEXT/dict.de.txt \
    --nwordssrc 32768 --nwordstgt 32768 \
    --destdir $root/data-bin/wmt14_en_de \
    --workers 20
