
job_name=6T4Mm

CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
deepspeed --num_nodes 1 \
    general_distill.py \
        --job_name ${job_name} \
        --deepspeed \
        --deepspeed_config ds_lamb_config.json \
        --student_model_class SkipBert \
        --num_train_epochs 10 \
        --pregenerated_data /tmp/corpus \
        --teacher bert-base-uncased \
        --student_model huawei-noah/TinyBERT_General_4L_312D \
        --output_dir output_${job_name} \
        --num_hidden_layers 10 \
        --num_full_hidden_layers 4 \
        --num_masked_layers_teacher 4