@inproceedings{huang-2022-domain,
    title = "Domain Specific Augmentations as Low Cost Teachers for Large Students",
    author = "Huang, Po-Wei",
    editor = "Ghosal, Tirthankar  and
      Blanco-Cuaresma, Sergi  and
      Accomazzi, Alberto  and
      Patton, Robert M.  and
      Grezes, Felix  and
      Allen, Thomas",
    booktitle = "Proceedings of the first Workshop on Information Extraction from Scientific Publications",
    month = nov,
    year = "2022",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-sig-urls/2022.wiesp-1.10/",
    doi = "10.18653/v1/2022.wiesp-1.10",
    pages = "84--90",
    abstract = "Current neural network solutions in scientific document processing employ models pretrained on domain-specific corpi, which are usually limited in model size, as pretraining can be costly and limited by training resources. We introduce a framework that uses data augmentation from such domain-specific pretrained models to transfer domain specific knowledge to larger general pretrained models and improve performance on downstream tasks. Our method improves the performance of Named Entity Recognition in the astrophysical domain by more than 20{\%} compared to domain-specific pretrained models finetuned to the target dataset."
}