@inproceedings{azad-etal-2025-predicting,
    title = "Predicting The Scholarly Impact of Research Papers Using Retrieval-Augmented {LLM}s",
    author = "Azad, Tamjid  and
      Azher, Ibrahim Al  and
      Choudhury, Sagnik Ray  and
      Alhoori, Hamed",
    editor = "Ghosal, Tirthankar  and
      Mayr, Philipp  and
      Singh, Amanpreet  and
      Naik, Aakanksha  and
      Rehm, Georg  and
      Freitag, Dayne  and
      Li, Dan  and
      Schimmler, Sonja  and
      De Waard, Anita",
    booktitle = "Proceedings of the Fifth Workshop on Scholarly Document Processing (SDP 2025)",
    month = jul,
    year = "2025",
    address = "Vienna, Austria",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/landing_page/2025.sdp-1.11/",
    doi = "10.18653/v1/2025.sdp-1.11",
    pages = "124--131",
    ISBN = "979-8-89176-265-7",
    abstract = "Assessing a research paper{'}s scholarly impact is an important phase in the scientific research process; however, metrics typically take some time after publication to accurately capture the impact. Our study examines how Large Language Models (LLMs) can predict scholarly impact accurately. We utilize Retrieval-Augmented Generation (RAG) to examine the degree to which the LLM performance improves compared to zero-shot prompting. Results show that LLama3-8b with RAG achieved the best overall performance, while Gemma-7b benefited the most from RAG, exhibiting the most significant reduction in Mean Absolute Error (MAE). Our findings suggest that retrieval-augmented LLMs offer a promising approach for early research evaluation. Our code and dataset for this project are publicly available."
}