@inproceedings{halder-etal-2026-riddlebench,
    title = "{R}iddle{B}ench: A New Generative Reasoning Benchmark for {LLM}s",
    author = "Halder, Deepon  and
      Saji, Alan  and
      Jayakumar, Thanmay  and
      Kunchukuttan, Anoop  and
      Puduppully, Ratish  and
      Dabre, Raj",
    editor = "Demberg, Vera  and
      Inui, Kentaro  and
      Marquez, Llu{\'i}s",
    booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {EACL} 2026",
    month = mar,
    year = "2026",
    address = "Rabat, Morocco",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/ingest-eacl/2026.findings-eacl.228/",
    pages = "4363--4372",
    ISBN = "979-8-89176-386-9",
    abstract = "While Large Language Models (LLMs) show remarkable capabilities, their complex reasoning skills require deeper investigation. We introduce **RiddleBench**, a new benchmark of 1,737 challenging puzzles designed to test reasoning beyond simple pattern matching. Our evaluation of state-of-the-art models reveals significant limitations, including hallucination cascades (uncritically accepting flawed peer reasoning) and poor self-correction due to strong self-confirmation bias. We also find that model performance is fragile, degrading when faced with reordered constraints or irrelevant information. RiddleBench serves as a resource for diagnosing these issues and guiding the development of more robust LLMs."
}