@inproceedings{zou-etal-2025-docbench,
    title = "{D}oc{B}ench: A Benchmark for Evaluating {LLM}-based Document Reading Systems",
    author = "Zou, Anni  and
      Yu, Wenhao  and
      Zhang, Hongming  and
      Ma, Kaixin  and
      Cai, Deng  and
      Zhang, Zhuosheng  and
      Zhao, Hai  and
      Yu, Dong",
    editor = "Shi, Weijia  and
      Yu, Wenhao  and
      Asai, Akari  and
      Jiang, Meng  and
      Durrett, Greg  and
      Hajishirzi, Hannaneh  and
      Zettlemoyer, Luke",
    booktitle = "Proceedings of the 4th International Workshop on Knowledge-Augmented Methods for Natural Language Processing",
    month = may,
    year = "2025",
    address = "Albuquerque, New Mexico, USA",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/moar-dois/2025.knowledgenlp-1.29/",
    doi = "10.18653/v1/2025.knowledgenlp-1.29",
    pages = "359--373",
    ISBN = "979-8-89176-229-9",
    abstract = "Recent advancements in proprietary large language models (LLMs), such as those from OpenAI and Anthropic, have led to the development of document reading systems capable of handling raw files with complex layouts, intricate formatting, lengthy content, and multi-modal information. However, the absence of a standardized benchmark hinders objective evaluation of these systems. To address this gap, we introduce DocBench, a benchmark designed to simulate real-world scenarios, where each raw file consists of a document paired with one or more questions. DocBench uniquely evaluates entire document reading systems and adopts a user-centric approach, allowing users to identify the system best suited to their needs."
}