@article{yu-etal-2025-diverse,
    title = "Diverse {AI} Feedback For Large Language Model Alignment",
    author = "Yu, Tianshu  and
      Lin, Ting-En  and
      Wu, Yuchuan  and
      Yang, Min  and
      Huang, Fei  and
      Li, Yongbin",
    journal = "Transactions of the Association for Computational Linguistics",
    volume = "13",
    year = "2025",
    address = "Cambridge, MA",
    publisher = "MIT Press",
    url = "https://preview.aclanthology.org/corrections-2025-07/2025.tacl-1.19/",
    doi = "10.1162/tacl_a_00746",
    pages = "392--407",
    abstract = "Recent advances in large language models (LLMs) focus on aligning models with human values to minimize harmful content. However, existing methods often rely on a single type of feedback, such as preferences, annotated labels, or critiques, which can lead to overfitting and suboptimal performance. In this paper, we propose Diverse AIFeedback (DAIF), a novel approach that integrates three types of feedback{---}critique, refinement, and preference{---}tailored to tasks of varying uncertainty levels. Through an analysis of information gain, we show that critique feedback is most effective for low-uncertainty tasks, refinement feedback for medium-uncertainty tasks, and preference feedback for high-uncertainty tasks. Training with this diversified feedback reduces overfitting and improves alignment. Experimental results across three tasks{---}question answering, dialog generation, and text summarization{--}demonstrate that DAIF outperforms traditional methods relying on a single feedback type.1"
}