@inproceedings{lee-bloem-2023-comparing,
    title = "Comparing domain-specific and domain-general {BERT} variants for inferred real-world knowledge through rare grammatical features in {S}erbian",
    author = "Lee, Sofia  and
      Bloem, Jelke",
    editor = "Piskorski, Jakub  and
      Marci{\'n}czuk, Micha{\l}  and
      Nakov, Preslav  and
      Ogrodniczuk, Maciej  and
      Pollak, Senja  and
      P{\v{r}}ib{\'a}{\v{n}}, Pavel  and
      Rybak, Piotr  and
      Steinberger, Josef  and
      Yangarber, Roman",
    booktitle = "Proceedings of the 9th Workshop on Slavic Natural Language Processing 2023 (SlavicNLP 2023)",
    month = may,
    year = "2023",
    address = "Dubrovnik, Croatia",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/ingest-emnlp/2023.bsnlp-1.7/",
    doi = "10.18653/v1/2023.bsnlp-1.7",
    pages = "47--60",
    abstract = "Transfer learning is one of the prevailing approaches towards training language-specific BERT models. However, some languages have uncommon features that may prove to be challenging to more domain-general models but not domain-specific models. Comparing the performance of BERTi{\'c}, a Bosnian-Croatian-Montenegrin-Serbian model, and Multilingual BERT on a Named-Entity Recognition (NER) task and Masked Language Modelling (MLM) task based around a rare phenomenon of indeclinable female foreign names in Serbian reveals how the different training approaches impacts their performance. Multilingual BERT is shown to perform better than BERTi{\'c} in the NER task, but BERTi{\'c} greatly exceeds in the MLM task. Thus, there are applications both for domain-general training and domain-specific training depending on the tasks at hand."
}