@article{passonneau-carpenter-2014-benefits,
    title = "The Benefits of a Model of Annotation",
    author = "Passonneau, Rebecca J.  and
      Carpenter, Bob",
    editor = "Lin, Dekang  and
      Collins, Michael  and
      Lee, Lillian",
    journal = "Transactions of the Association for Computational Linguistics",
    volume = "2",
    year = "2014",
    address = "Cambridge, MA",
    publisher = "MIT Press",
    url = "https://preview.aclanthology.org/add-emnlp-2024-awards/Q14-1025/",
    doi = "10.1162/tacl_a_00185",
    pages = "311--326",
    abstract = "Standard agreement measures for interannotator reliability are neither necessary nor sufficient to ensure a high quality corpus. In a case study of word sense annotation, conventional methods for evaluating labels from trained annotators are contrasted with a probabilistic annotation model applied to crowdsourced data. The annotation model provides far more information, including a certainty measure for each gold standard label; the crowdsourced data was collected at less than half the cost of the conventional approach."
}