@inproceedings{mahmoud-nakov-2023-bertastic,
    title = "{BERT}astic at {S}em{E}val-2023 Task 3: Fine-Tuning Pretrained Multilingual Transformers Does Order Matter?",
    author = "Mahmoud, Tarek  and
      Nakov, Preslav",
    editor = {Ojha, Atul Kr.  and
      Do{\u{g}}ru{\"o}z, A. Seza  and
      Da San Martino, Giovanni  and
      Tayyar Madabushi, Harish  and
      Kumar, Ritesh  and
      Sartori, Elisa},
    booktitle = "Proceedings of the 17th International Workshop on Semantic Evaluation (SemEval-2023)",
    month = jul,
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-sig-urls/2023.semeval-1.7/",
    doi = "10.18653/v1/2023.semeval-1.7",
    pages = "58--63",
    abstract = {The naive approach for fine-tuning pretrained deep learning models on downstream tasks involves feeding them mini-batches of randomly sampled data. In this paper, we propose a more elaborate method for fine-tuning Pretrained Multilingual Transformers (PMTs) on multilingual data. Inspired by the success of curriculum learning approaches, we investigate the significance of fine-tuning PMTs on multilingual data in a sequential fashion language by language. Unlike the curriculum learning paradigm where the model is presented with increasingly complex examples, we do not adopt a notion of ``easy'' and ``hard'' samples. Instead, our experiments draw insight from psychological findings on how the human brain processes new information and the persistence of newly learned concepts. We perform our experiments on a challenging news-framing dataset that contains texts in six languages. Our proposed method outperforms the na{\"i}ve approach by achieving improvements of 2.57{\textbackslash}{\%} in terms of F1 score. Even when we supplement the na{\"i}ve approach with recency fine-tuning, we still achieve an improvement of 1.34{\textbackslash}{\%} with a 3.63{\textbackslash}{\%}{\$} convergence speed-up. Moreover, we are the first to observe an interesting pattern in which deep learning models exhibit a human-like primacy-recency effect.}
}