@inproceedings{tarnavskyi-etal-2022-ensembling,
    title = "Ensembling and Knowledge Distilling of Large Sequence Taggers for Grammatical Error Correction",
    author = "Tarnavskyi, Maksym  and
      Chernodub, Artem  and
      Omelianchuk, Kostiantyn",
    editor = "Muresan, Smaranda  and
      Nakov, Preslav  and
      Villavicencio, Aline",
    booktitle = "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2022.acl-long.266/",
    doi = "10.18653/v1/2022.acl-long.266",
    pages = "3842--3852",
    abstract = "In this paper, we investigate improvements to the GEC sequence tagging architecture with a focus on ensembling of recent cutting-edge Transformer-based encoders in Large configurations. We encourage ensembling models by majority votes on span-level edits because this approach is tolerant to the model architecture and vocabulary size. Our best ensemble achieves a new SOTA result with an $F_{0.5}$ score of 76.05 on BEA-2019 (test), even without pre-training on synthetic datasets. In addition, we perform knowledge distillation with a trained ensemble to generate new synthetic training datasets, {\textquotedblleft}Troy-Blogs{\textquotedblright} and {\textquotedblleft}Troy-1BW{\textquotedblright}. Our best single sequence tagging model that is pretrained on the generated Troy- datasets in combination with the publicly available synthetic PIE dataset achieves a near-SOTA result with an $F_{0.5}$ score of 73.21 on BEA-2019 (test). The code, datasets, and trained models are publicly available."
}