@article{rashkin-etal-2023-measuring,
    title = "Measuring Attribution in Natural Language Generation Models",
    author = "Rashkin, Hannah  and
      Nikolaev, Vitaly  and
      Lamm, Matthew  and
      Aroyo, Lora  and
      Collins, Michael  and
      Das, Dipanjan  and
      Petrov, Slav  and
      Tomar, Gaurav Singh  and
      Turc, Iulia  and
      Reitter, David",
    journal = "Computational Linguistics",
    volume = "49",
    number = "4",
    month = dec,
    year = "2023",
    address = "Cambridge, MA",
    publisher = "MIT Press",
    url = "https://aclanthology.org/2023.cl-4.2",
    doi = "10.1162/coli_a_00486",
    pages = "777--840",
    abstract = "Large neural models have brought a new challenge to natural language generation (NLG): It has become imperative to ensure the safety and reliability of the output of models that generate freely. To this end, we present an evaluation framework, Attributable to Identified Sources (AIS), stipulating that NLG output pertaining to the external world is to be verified against an independent, provided source. We define AIS and a two-stage annotation pipeline for allowing annotators to evaluate model output according to annotation guidelines. We successfully validate this approach on generation datasets spanning three tasks (two conversational QA datasets, a summarization dataset, and a table-to-text dataset). We provide full annotation guidelines in the appendices and publicly release the annotated data at https://github.com/google-research-datasets/AIS.",
}