@inproceedings{thomson-reiter-2020-gold,
    title = "A Gold Standard Methodology for Evaluating Accuracy in Data-To-Text Systems",
    author = "Thomson, Craig  and
      Reiter, Ehud",
    editor = "Davis, Brian  and
      Graham, Yvette  and
      Kelleher, John  and
      Sripada, Yaji",
    booktitle = "Proceedings of the 13th International Conference on Natural Language Generation",
    month = dec,
    year = "2020",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2020.inlg-1.22/",
    doi = "10.18653/v1/2020.inlg-1.22",
    pages = "158--168",
    abstract = "Most Natural Language Generation systems need to produce accurate texts. We propose a methodology for high-quality human evaluation of the accuracy of generated texts, which is intended to serve as a gold-standard for accuracy evaluations of data-to-text systems. We use our methodology to evaluate the accuracy of computer generated basketball summaries. We then show how our gold standard evaluation can be used to validate automated metrics."
}