@inproceedings{yuan-zhang-2025-domainsum,
    title = "{D}omain{S}um: A Hierarchical Benchmark for Fine-Grained Domain Shift in Abstractive Text Summarization",
    author = "Yuan, Haohan  and
      Zhang, Haopeng",
    editor = "Chiruzzo, Luis  and
      Ritter, Alan  and
      Wang, Lu",
    booktitle = "Findings of the Association for Computational Linguistics: NAACL 2025",
    month = apr,
    year = "2025",
    address = "Albuquerque, New Mexico",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/ingest-emnlp/2025.findings-naacl.118/",
    doi = "10.18653/v1/2025.findings-naacl.118",
    pages = "2219--2231",
    ISBN = "979-8-89176-195-7",
    abstract = "Most research on abstractive summarization focuses on single-domain applications, often neglecting how domain shifts between documents affect performance and the generalization ability of summarization models. To address this issue, we introduce DomainSum, a hierarchical benchmark designed to capture fine-grained domain shifts in abstractive summarization. We categorize these shifts into three levels: genre, style, and topic, and demonstrate through comprehensive benchmark analysis that they follow a hierarchical structure. Furthermore, we evaluate the domain generalization capabilities of commonly used pre-trained language models (PLMs) and large language models (LLMs) in both in-domain and cross-domain settings. Our benchmark and source code are released at https://github.com/hpzhang94/DomainSum."
}