@inproceedings{wang-etal-2025-llms-perform,
    title = "{LLM}s can Perform Multi-Dimensional Analytic Writing Assessments: A Case Study of {L}2 Graduate-Level Academic {E}nglish Writing",
    author = "Wang, Zhengxiang  and
      Makarova, Veronika  and
      Li, Zhi  and
      Kodner, Jordan  and
      Rambow, Owen",
    editor = "Che, Wanxiang  and
      Nabende, Joyce  and
      Shutova, Ekaterina  and
      Pilehvar, Mohammad Taher",
    booktitle = "Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = jul,
    year = "2025",
    address = "Vienna, Austria",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/ingestion-acl-25/2025.acl-long.423/",
    pages = "8637--8663",
    ISBN = "979-8-89176-251-0",
    abstract = "The paper explores the performance of LLMs in the context of multi-dimensional analytic writing assessments, i.e. their ability to provide both scores and comments based on multiple assessment criteria. Using a corpus of literature reviews written by L2 graduate students and assessed by human experts against 9 analytic criteria, we prompt several popular LLMs to perform the same task under various conditions. To evaluate the quality of feedback comments, we apply a novel feedback comment quality evaluation framework. This framework is interpretable, cost-efficient, scalable, and reproducible, compared to existing methods that rely on manual judgments. We find that LLMs can generate reasonably good and generally reliable multi-dimensional analytic assessments. We release our corpus and code for reproducibility."
}