@inproceedings{rehlinger-2026-fine,
    title = "A Fine-Grained Linguistic Evaluation of Low-Resource {L}uxembourgish{--}{E}nglish {MT}",
    author = "Rehlinger, Nils",
    editor = "Ojha, Atul Kr.  and
      Liu, Chao-hong  and
      Vylomova, Ekaterina  and
      Pirinen, Flammie  and
      Washington, Jonathan  and
      Oco, Nathaniel  and
      Zhao, Xiaobing",
    booktitle = "Proceedings for the Ninth Workshop on Technologies for Machine Translation of Low Resource Languages ({L}o{R}es{MT} 2026)",
    month = mar,
    year = "2026",
    address = "Rabat, Morocco",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/manual-author-scripts/2026.loresmt-1.12/",
    pages = "138--150",
    ISBN = "979-8-89176-366-1",
    abstract = "Machine translation (MT) evaluation is central in guiding researchers on how to improve a model{'}s performance. Current automatic evaluation practices fail to provide reliable insights into the specific translation errors that occur, especially for low-resource languages. This paper introduces the Lux-MT-Test-Suite, enabling a linguistically motivated and fine-grained analysis of Luxembourgish{--}English (LB-EN) MT based on 896 test items covering 12 linguistic categories and 36 linguistic phenomena. We compare a baseline local LLM (Gemma 3), its fine-tuned counterpart (LuxMT), and a proprietary state-of-the-art LLM (GPT-5) to analyse what local LLMs learn through fine-tuning in a low-resource setting and to assess performance differences between local and proprietary systems. The findings identify specific performance gains through fine-tuning, minor degradations, a difference in translation strategies, performance gaps between local and proprietary models, and remaining challenges."
}