@inproceedings{moon-okazaki-2020-patchbert,
    title = "{P}atch{BERT}: Just-in-Time, Out-of-Vocabulary Patching",
    author = "Moon, Sangwhan  and
      Okazaki, Naoaki",
    editor = "Webber, Bonnie  and
      Cohn, Trevor  and
      He, Yulan  and
      Liu, Yang",
    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    month = nov,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2020.emnlp-main.631/",
    doi = "10.18653/v1/2020.emnlp-main.631",
    pages = "7846--7852",
    abstract = "Large scale pre-trained language models have shown groundbreaking performance improvements for transfer learning in the domain of natural language processing. In our paper, we study a pre-trained multilingual BERT model and analyze the OOV rate on downstream tasks, how it introduces information loss, and as a side-effect, obstructs the potential of the underlying model. We then propose multiple approaches for mitigation and demonstrate that it improves performance with the same parameter count when combined with fine-tuning."
}