@inproceedings{eigler-etal-2026-llm,
    title = "{LLM} as a Meta-Judge: Synthetic Data for {NLP} Evaluation Metric Validation",
    author = "Eigler, Luk{\'a}{\v{s}}  and
      Libovick{\'y}, Jind{\v{r}}ich  and
      Hurych, David",
    editor = "T.Y.S.S., Santosh  and
      Rodriguez, Juan Diego  and
      de Gibert, Ona",
    booktitle = "Proceedings of the 64th Annual Meeting of the {A}ssociation for {C}omputational {L}inguistics ({ACL} 2026)",
    month = jul,
    year = "2026",
    address = "San Diego, California, United States",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/ingest-acl/2026.acl-srw.125/",
    pages = "1417--1435",
    ISBN = "979-8-89176-393-7",
    abstract = "Validating evaluation metrics for NLG typically relies on expensive and time-consuming human annotations, which predominantly exists only for English datasets. We propose LLM as a Meta-Judge, a scalable framework that utilizes LLMs to generate synthetic evaluation datasets via controlled semantic degradation of real data, replacing human judgment. We validate our approach using meta-correlation, measuring the alignment between metric rankings derived from synthetic data and those from standard human benchmarks. Experiments across Machine Translation, Question Answering, and Summarization demonstrate that synthetic validation serves as a reliable proxy for human judgment, achieving meta-correlations exceeding 0.9 in multilingual QA and proves to be a viable alternative where human judgments are unavailable or too expensive to obtain. Our code and data are publicly available at \url{https://github.com/eiglerl/meta-judge}."
}