@inproceedings{yu-etal-2025-long,
    title = "Long-context Language Models Fail in Basic Retrieval Tasks Without Sufficient Reasoning Steps",
    author = "Yu, Yijiong  and
      Qi, Zhixiao  and
      Huang, Yongfeng  and
      Wang, Wei  and
      Weifeng.liu  and
      Chen, Ran  and
      Pei, Ji",
    editor = "Christodoulopoulos, Christos  and
      Chakraborty, Tanmoy  and
      Rose, Carolyn  and
      Peng, Violet",
    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2025",
    month = nov,
    year = "2025",
    address = "Suzhou, China",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/author-page-yu-wang-polytechnic/2025.findings-emnlp.301/",
    doi = "10.18653/v1/2025.findings-emnlp.301",
    pages = "5615--5634",
    ISBN = "979-8-89176-335-7",
    abstract = "Long-context language models (LCLMs), characterized by their extensive context window, are becoming popular. However, despite the fact that they are nearly perfect at standard long-context retrieval tasks, our evaluations demonstrate they fail in some basic cases. Later, we find they can be well addressed with a sufficient number of reasoning steps, guided by specific CoT prompts. This result emphasizes the potential necessity of solving specific long-context tasks using long-CoT methods, while previous long-context benchmarks always ignore the necessity of long reasoning for long-context tasks and treat them as direct QA tasks. Our code and datasets are available at https://github.com/yuyijiong/hard{\_}retrieval{\_}for{\_}llm"
}