#!/usr/bin/env bash

pwd=$(dirname $(readlink -f $0))
ccPrefix=/home/tiger/cc100/sample
summDIR=/home/tiger/summ
DATAVER=EnFr_Summ_DeEsZh_Unsup_Noise2_Doc_Batch

TMPDIR=~/tmp_afterTokenize
mkdir $TMPDIR
# echo "Tokenizing summarization data ..."
# bash tokenizeSumm.sh ${summDIR} en en_XX MSPM
# bash tokenizeSumm.sh ${summDIR} fr fr_XX MSPM

TASK_TAG='denoise'
# for split in train dev
# # for split in dev
# do
#     # echo "[${split}] Tokenizing unsupervised data ..."
#     INDIR=${ccPrefix}_${split}_doc
#     OUTDIR=$INDIR/MSPM
#     # mkdir $OUTDIR
    
#     lg=es
#     LG_TAG="es_XX"
#     python3 ../addNoise.py -m noiseV2 -i ${INDIR}/${lg}.doc.spm.sample -o ${TMPDIR}/${lg}.doc.spm.noise -l ${lg}
#     cp ${INDIR}/${lg}.doc.spm.sample ${TMPDIR}/
#     cat ${TMPDIR}/${lg}.doc.spm.noise | sed -e "s/<q>/ <\/s>/g" -e "s/^/<s> /" -e "s/$/ <\/s>/" -e "s/^/[${LG_TAG}] [${TASK_TAG}] /" \
#         > ${OUTDIR}/${lg}.doc.spm.noise
#     cat ${TMPDIR}/${lg}.doc.spm.sample | sed -e "s/<q>/ <\/s>/g" -e "s/^/<s> /" -e "s/$/ <\/s>/" -e "s/^/[${LG_TAG}] [${TASK_TAG}] /" \
#         > ${OUTDIR}/${lg}.doc.spm.sample

#     lg=zh
#     LG_TAG="zh_CN"
#     python3 ../addNoise.py -m noiseV2 -i ${INDIR}/${lg}.doc.spm.sample -o ${TMPDIR}/${lg}.doc.spm.noise -l ${lg}
#     cp ${INDIR}/${lg}.doc.spm.sample ${TMPDIR}/
#     cat ${TMPDIR}/${lg}.doc.spm.noise | sed -e "s/<q>/ <\/s>/g" -e "s/^/<s> /" -e "s/$/ <\/s>/" -e "s/^/[${LG_TAG}] [${TASK_TAG}] /" \
#         > ${OUTDIR}/${lg}.doc.spm.noise
#     cat ${TMPDIR}/${lg}.doc.spm.sample | sed -e "s/<q>/ <\/s>/g" -e "s/^/<s> /" -e "s/$/ <\/s>/" -e "s/^/[${LG_TAG}] [${TASK_TAG}] /" \
#         > ${OUTDIR}/${lg}.doc.spm.sample

#     lg=de
#     LG_TAG="de_DE"
#     python3 ../addNoise.py -m noiseV2 -i ${INDIR}/${lg}.doc.spm.sample -o ${TMPDIR}/${lg}.doc.spm.noise -l ${lg}
#     cp ${INDIR}/${lg}.doc.spm.sample ${TMPDIR}/
#     cat ${TMPDIR}/${lg}.doc.spm.noise | sed -e "s/<q>/ <\/s>/g" -e "s/^/<s> /" -e "s/$/ <\/s>/" -e "s/^/[${LG_TAG}] [${TASK_TAG}] /" \
#         > ${OUTDIR}/${lg}.doc.spm.noise
#     cat ${TMPDIR}/${lg}.doc.spm.sample | sed -e "s/<q>/ <\/s>/g" -e "s/^/<s> /" -e "s/$/ <\/s>/" -e "s/^/[${LG_TAG}] [${TASK_TAG}] /" \
#         > ${OUTDIR}/${lg}.doc.spm.sample
# done

bash concat2.sh $summDIR/MSPM ${ccPrefix}_train_doc/MSPM $DATAVER

echo "Binary data ..."
bash binary.sh $summDIR $DATAVER MSPM

rm -r $TMPDIR
