@inproceedings{poerner-etal-2020-inexpensive,
    title = "Inexpensive Domain Adaptation of Pretrained Language Models: Case Studies on Biomedical {NER} and Covid-19 {QA}",
    author = {Poerner, Nina  and
      Waltinger, Ulli  and
      Sch{\"u}tze, Hinrich},
    editor = "Cohn, Trevor  and
      He, Yulan  and
      Liu, Yang",
    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2020",
    month = nov,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-sig-urls/2020.findings-emnlp.134/",
    doi = "10.18653/v1/2020.findings-emnlp.134",
    pages = "1482--1490",
    abstract = "Domain adaptation of Pretrained Language Models (PTLMs) is typically achieved by unsupervised pretraining on target-domain text. While successful, this approach is expensive in terms of hardware, runtime and CO 2 emissions. Here, we propose a cheaper alternative: We train Word2Vec on target-domain text and align the resulting word vectors with the wordpiece vectors of a general-domain PTLM. We evaluate on eight English biomedical Named Entity Recognition (NER) tasks and compare against the recently proposed BioBERT model. We cover over 60{\%} of the BioBERT - BERT F1 delta, at 5{\%} of BioBERT{'}s CO 2 footprint and 2{\%} of its cloud compute cost. We also show how to quickly adapt an existing general-domain Question Answering (QA) model to an emerging domain: the Covid-19 pandemic."
}