@proceedings{humeval-2021-human,
    title = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.0/"
}
@inproceedings{clinciu-etal-2021-commonsense,
    title = "It{'}s Commonsense, isn{'}t it? Demystifying Human Evaluations in Commonsense-Enhanced {NLG} Systems",
    author = "Clinciu, Miruna-Adriana  and
      Gkatzia, Dimitra  and
      Mahamood, Saad",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.1/",
    pages = "1--12"
}
@inproceedings{nyberg-etal-2021-estimating,
    title = "Estimating Subjective Crowd-Evaluations as an Additional Objective to Improve Natural Language Generation",
    author = "Nyberg, Jakob  and
      Paetzel, Maike  and
      Manuvinakurike, Ramesh",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.2/",
    pages = "13--24"
}
@inproceedings{zhang-etal-2021-trading,
    title = "Trading Off Diversity and Quality in Natural Language Generation",
    author = "Zhang, Hugh  and
      Duckworth, Daniel  and
      Ippolito, Daphne  and
      Neelakantan, Arvind",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.3/",
    pages = "25--33"
}
@inproceedings{castilho-2021-towards,
    title = "Towards Document-Level Human {MT} Evaluation: On the Issues of Annotator Agreement, Effort and Misevaluation",
    author = "Castilho, Sheila",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.4/",
    pages = "34--45"
}
@inproceedings{sudoh-etal-2021-translation,
    title = "Is This Translation Error Critical?: Classification-Based Human and Automatic Machine Translation Evaluation Focusing on Critical Errors",
    author = "Sudoh, Katsuhito  and
      Takahashi, Kosuke  and
      Nakamura, Satoshi",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.5/",
    pages = "46--55"
}
@inproceedings{moramarco-etal-2021-towards,
    title = "Towards Objectively Evaluating the Quality of Generated Medical Summaries",
    author = "Moramarco, Francesco  and
      Juric, Damir  and
      Savkov, Aleksandar  and
      Reiter, Ehud",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.6/",
    pages = "56--61"
}
@inproceedings{moramarco-etal-2021-preliminary,
    title = "A Preliminary Study on Evaluating Consultation Notes With Post-Editing",
    author = "Moramarco, Francesco  and
      Papadopoulos Korfiatis, Alex  and
      Savkov, Aleksandar  and
      Reiter, Ehud",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.7/",
    pages = "62--68"
}
@inproceedings{hamalainen-alnajjar-2021-great,
    title = "The Great Misalignment Problem in Human Evaluation of {NLP} Methods",
    author = {H{\"a}m{\"a}l{\"a}inen, Mika  and
      Alnajjar, Khalid},
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.8/",
    pages = "69--74"
}
@inproceedings{chierici-habash-2021-view,
    title = "A View From the Crowd: Evaluation Challenges for Time-Offset Interaction Applications",
    author = "Chierici, Alberto  and
      Habash, Nizar",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.9/",
    pages = "75--85"
}
@inproceedings{iskender-etal-2021-reliability,
    title = "Reliability of Human Evaluation for Text Summarization: Lessons Learned and Challenges Ahead",
    author = {Iskender, Neslihan  and
      Polzehl, Tim  and
      M{\"o}ller, Sebastian},
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.10/",
    pages = "86--96"
}
@inproceedings{grundkiewicz-etal-2021-user,
    title = "On User Interfaces for Large-Scale Document-Level Human Evaluation of Machine Translation Outputs",
    author = "Grundkiewicz, Roman  and
      Junczys-Dowmunt, Marcin  and
      Federmann, Christian  and
      Kocmi, Tom",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.11/",
    pages = "97--106"
}
@inproceedings{van-boven-bloem-2021-eliciting,
    title = "Eliciting Explicit Knowledge From Domain Experts in Direct Intrinsic Evaluation of Word Embeddings for Specialized Domains",
    author = "van Boven, Goya  and
      Bloem, Jelke",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.12/",
    pages = "107--113"
}
@inproceedings{kloudova-etal-2021-detecting,
    title = "Detecting Post-Edited References and Their Effect on Human Evaluation",
    author = "Kloudov{\'a}, V{\v{e}}ra  and
      Bojar, Ond{\v{r}}ej  and
      Popel, Martin",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.13/",
    pages = "114--119"
}
@inproceedings{bhatt-etal-2021-case,
    title = "A Case Study of Efficacy and Challenges in Practical Human-in-Loop Evaluation of {NLP} Systems Using Checklist",
    author = "Bhatt, Shaily  and
      Jain, Rahul  and
      Dandapat, Sandipan  and
      Sitaram, Sunayana",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.14/",
    pages = "120--130"
}
@inproceedings{oortwijn-etal-2021-interrater,
    title = "Interrater Disagreement Resolution: A Systematic Procedure to Reach Consensus in Annotation Tasks",
    author = "Oortwijn, Yvette  and
      Ossenkoppele, Thijs  and
      Betti, Arianna",
    editor = "Belz, Anya  and
      Agarwal, Shubham  and
      Graham, Yvette  and
      Reiter, Ehud  and
      Shimorina, Anastasia",
    booktitle = "Proceedings of the Workshop on Human Evaluation of NLP Systems (HumEval)",
    month = apr,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-old-resolve/2021.humeval-1.15/",
    pages = "131--141"
}