@inproceedings{riess-jorgensen-2025-brage,
    title = "The {BRAGE} Benchmark: {Evaluating} Zero-shot Learning Capabilities of Large Language Models for {Norwegian} Customer Service Dialogues",
    author = "Riess, Mike  and
      J{\o}rgensen, Tollef Emil",
    editor = "Johansson, Richard  and
      Stymne, Sara",
    booktitle = "Proceedings of the Joint 25th Nordic Conference on Computational Linguistics and 11th Baltic Conference on Human Language Technologies (NoDaLiDa/Baltic-HLT 2025)",
    month = mar,
    year = "2025",
    address = "Tallinn, Estonia",
    publisher = "University of Tartu Library",
    url = "https://preview.aclanthology.org/fix-sig-urls/2025.nodalida-1.57/",
    pages = "525--536",
    ISBN = "978-9908-53-109-0",
    abstract = "This study explores the capabilities of open-weight Large Language Models in a zero-shot learning setting, testing their ability to classify the content of customer service dialogues in Norwegian from a single instruction, named the BRAGE benchmark. By comparing results against widely used downstream tasks such as question-answering and named entity recognition, we find that (1) specific instruction models greatly exceed base models on the benchmark, (2) both English and multilingual instruction models outperform the tested Norwegian models of similar sizes, and (3) the difference between base and instruction models is less pronounced than in other generative tasks, suggesting that BRAGE is a challenging benchmark, requiring precise and generalizable instruction-tuning."
}