@inproceedings{ezquerro-etal-2025-better,
    title = "Better Benchmarking {LLM}s for Zero-Shot Dependency Parsing",
    author = "Ezquerro, Ana  and
      G{\'o}mez-Rodr{\'i}guez, Carlos  and
      Vilares, David",
    editor = "Johansson, Richard  and
      Stymne, Sara",
    booktitle = "Proceedings of the Joint 25th Nordic Conference on Computational Linguistics and 11th Baltic Conference on Human Language Technologies (NoDaLiDa/Baltic-HLT 2025)",
    month = mar,
    year = "2025",
    address = "Tallinn, Estonia",
    publisher = "University of Tartu Library",
    url = "https://preview.aclanthology.org/fix-sig-urls/2025.nodalida-1.13/",
    pages = "121--135",
    ISBN = "978-9908-53-109-0",
    abstract = "While LLMs excel in zero-shot tasks, their performance in linguistic challenges like syntactic parsing has been less scrutinized. This paper studies state-of-the-art open-weight LLMs on the task by comparing them to baselines that do not have access to the input sentence, including baselines that have not been used in this context such as random projective trees or optimal linear arrangements. The results show that most of the tested LLMs cannot outperform the best uninformed baselines, with only the newest and largest versions of LLaMA doing so for most languages, and still achieving rather low performance. Thus, accurate zero-shot syntactic parsing is not forthcoming with open LLMs."
}