@inproceedings{roewer-despres-etal-2025-accord,
    title = "{ACCORD}: Closing the Commonsense Measurability Gap",
    author = "Roewer-Despr{\'e}s, Fran{\c{c}}ois  and
      Feng, Jinyue  and
      Zhu, Zining  and
      Rudzicz, Frank",
    editor = "Chiruzzo, Luis  and
      Ritter, Alan  and
      Wang, Lu",
    booktitle = "Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)",
    month = apr,
    year = "2025",
    address = "Albuquerque, New Mexico",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/Ingest-2025-COMPUTEL/2025.naacl-long.193/",
    pages = "3799--3829",
    ISBN = "979-8-89176-189-6",
    abstract = "We present ACCORD, a framework and benchmark suite for disentangling the commonsense grounding and reasoning abilities of large language models (LLMs) through controlled, multi-hop counterfactuals. ACCORD introduces formal elements to commonsense reasoning to explicitly control and quantify reasoning complexity beyond the typical 1 or 2 hops. Uniquely, ACCORD can automatically generate benchmarks of arbitrary reasoning complexity, so it scales with future LLM improvements. Indeed, our experiments on state-of-the-art LLMs show performance degrading to below random chance with only moderate scaling, leaving substantial headroom for improvement. We release a leaderboard of the benchmark suite tested in this work, as well as code for automatically generating more complex benchmarks."
}