@proceedings{humeval-2024-human,
    title = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.0/"
}
@inproceedings{zouhar-bojar-2024-quality,
    title = "Quality and Quantity of Machine Translation References for Automatic Metrics",
    author = "Zouhar, Vil{\'e}m  and
      Bojar, Ond{\v{r}}ej",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.1/",
    pages = "1--11"
}
@inproceedings{rigouts-terryn-de-lhoneux-2024-exploratory,
    title = "Exploratory Study on the Impact of {E}nglish Bias of Generative Large Language Models in {D}utch and {F}rench",
    author = "Rigouts Terryn, Ayla  and
      de Lhoneux, Miryam",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.2/",
    pages = "12--27"
}
@inproceedings{elaraby-etal-2024-adding,
    title = "Adding Argumentation into Human Evaluation of Long Document Abstractive Summarization: A Case Study on Legal Opinions",
    author = "Elaraby, Mohamed  and
      Xu, Huihui  and
      Gray, Morgan  and
      Ashley, Kevin  and
      Litman, Diane",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.3/",
    pages = "28--35"
}
@inproceedings{miletic-miletic-2024-gold,
    title = "A Gold Standard with Silver Linings: Scaling Up Annotation for Distinguishing {B}osnian, {C}roatian, {M}ontenegrin and {S}erbian",
    author = "Mileti{\'c}, Aleksandra  and
      Mileti{\'c}, Filip",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.4/",
    pages = "36--46"
}
@inproceedings{lewis-etal-2024-insights,
    title = "Insights of a Usability Study for {KBQA} Interactive Semantic Parsing: Generation Yields Benefits over Templates but External Validity Remains Challenging",
    author = "Lewis, Ashley  and
      Mo, Lingbo  and
      de Marneffe, Marie-Catherine  and
      Sun, Huan  and
      White, Michael",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.5/",
    pages = "47--62"
}
@inproceedings{antoine-etal-2024-extrinsic,
    title = "Extrinsic evaluation of question generation methods with user journey logs",
    author = "Antoine, Elie  and
      Besnehard, El{\'e}onore  and
      Bechet, Frederic  and
      Damnati, Geraldine  and
      Kergosien, Eric  and
      Laborderie, Arnaud",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.6/",
    pages = "63--70"
}
@inproceedings{carrer-etal-2024-towards,
    title = "Towards Holistic Human Evaluation of Automatic Text Simplification",
    author = {Carrer, Luisa  and
      S{\"a}uberli, Andreas  and
      Kappus, Martin  and
      Ebling, Sarah},
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.7/",
    pages = "71--80"
}
@inproceedings{frummet-elsweiler-2024-decoding,
    title = "Decoding the Metrics Maze: Navigating the Landscape of Conversational Question Answering System Evaluation in Procedural Tasks",
    author = "Frummet, Alexander  and
      Elsweiler, David",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.8/",
    pages = "81--90"
}
@inproceedings{belz-thomson-2024-2024,
    title = "The 2024 {R}epro{NLP} Shared Task on Reproducibility of Evaluations in {NLP}: Overview and Results",
    author = "Belz, Anya  and
      Thomson, Craig",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.9/",
    pages = "91--105"
}
@inproceedings{florescu-etal-2024-upon,
    title = "Once Upon a Replication: It is Humans' Turn to Evaluate {AI}{'}s Understanding of Children{'}s Stories for {QA} Generation",
    author = "Florescu, Andra-Maria  and
      Micluta-Campeanu, Marius  and
      Dinu, Liviu P.",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.10/",
    pages = "106--113"
}
@inproceedings{sasidharan-nair-etal-2024-exploring,
    title = "Exploring Reproducibility of Human-Labelled Data for Code-Mixed Sentiment Analysis",
    author = "Sasidharan Nair, Sachin  and
      Dinkar, Tanvi  and
      Abercrombie, Gavin",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.11/",
    pages = "114--124"
}
@inproceedings{lorandi-belz-2024-reproducing,
    title = "Reproducing the Metric-Based Evaluation of a Set of Controllable Text Generation Techniques",
    author = "Lorandi, Michela  and
      Belz, Anya",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.12/",
    pages = "125--131"
}
@inproceedings{van-miltenburg-etal-2024-reprohum,
    title = "{R}epro{H}um: {\#}0033-03: How Reproducible Are Fluency Ratings of Generated Text? A Reproduction of {A}ugust et al. 2022",
    author = "van Miltenburg, Emiel  and
      Braggaar, Anouck  and
      Braun, Nadine  and
      Goudbeek, Martijn  and
      Krahmer, Emiel  and
      van der Lee, Chris  and
      Pauws, Steffen  and
      Tomas, Fr{\'e}d{\'e}ric",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.13/",
    pages = "132--144"
}
@inproceedings{dinkar-etal-2024-reprohum,
    title = "{R}epro{H}um {\#}0927-03: {DE}xpert Evaluation? Reproducing Human Judgements of the Fluency of Generated Text",
    author = "Dinkar, Tanvi  and
      Abercrombie, Gavin  and
      Rieser, Verena",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.14/",
    pages = "145--152"
}
@inproceedings{gonzalez-corbelle-etal-2024-reprohum,
    title = "{R}epro{H}um {\#}0927-3: Reproducing The Human Evaluation Of The {DE}xperts Controlled Text Generation Method",
    author = "Gonz{\'a}lez Corbelle, Javier  and
      Vivel Couso, Ainhoa  and
      Alonso-Moral, Jose Maria  and
      Bugar{\'i}n-Diz, Alberto",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.15/",
    pages = "153--162"
}
@inproceedings{klubicka-kelleher-2024-reprohum,
    title = "{R}epro{H}um {\#}1018-09: Reproducing Human Evaluations of Redundancy Errors in Data-To-Text Systems",
    author = "Klubi{\v{c}}ka, Filip  and
      Kelleher, John D.",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.16/",
    pages = "163--198"
}
@inproceedings{fresen-etal-2024-reprohum,
    title = "{R}epro{H}um{\#}0043: Human Evaluation Reproducing Language Model as an Annotator: Exploring Dialogue Summarization on {AMI} Dataset",
    author = "Fresen, Vivian  and
      Wu-Urbanek, Mei-Shin  and
      Eger, Steffen",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.17/",
    pages = "199--209"
}
@inproceedings{arvan-parde-2024-reprohum,
    title = "{R}epro{H}um {\#}0712-01: Human Evaluation Reproduction Report for ``Hierarchical Sketch Induction for Paraphrase Generation''",
    author = "Arvan, Mohammad  and
      Parde, Natalie",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.18/",
    pages = "210--220"
}
@inproceedings{watson-gkatzia-2024-reprohum,
    title = "{R}epro{H}um {\#}0712-01: Reproducing Human Evaluation of Meaning Preservation in Paraphrase Generation",
    author = "Watson, Lewis N.  and
      Gkatzia, Dimitra",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.19/",
    pages = "221--228"
}
@inproceedings{lango-etal-2024-reprohum,
    title = "{R}epro{H}um {\#}0043-4: Evaluating Summarization Models: investigating the impact of education and language proficiency on reproducibility",
    author = "Lango, Mateusz  and
      Schmidtova, Patricia  and
      Balloccu, Simone  and
      Dusek, Ondrej",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.20/",
    pages = "229--237"
}
@inproceedings{li-etal-2024-reprohum,
    title = "{R}epro{H}um {\#}0033-3: Comparable Relative Results with Lower Absolute Values in a Reproduction Study",
    author = "Li, Yiru  and
      Lai, Huiyuan  and
      Toral, Antonio  and
      Nissim, Malvina",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.21/",
    pages = "238--249"
}
@inproceedings{mahamood-2024-reprohum,
    title = "{R}epro{H}um {\#}0124-03: Reproducing Human Evaluations of end-to-end approaches for Referring Expression Generation",
    author = "Mahamood, Saad",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.22/",
    pages = "250--254"
}
@inproceedings{loakman-lin-2024-reprohum,
    title = "{R}epro{H}um {\#}0087-01: Human Evaluation Reproduction Report for Generating Fact Checking Explanations",
    author = "Loakman, Tyler  and
      Lin, Chenghua",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.23/",
    pages = "255--260"
}
@inproceedings{mondella-etal-2024-reprohum,
    title = "{R}epro{H}um {\#}0892-01: The painful route to consistent results: A reproduction study of human evaluation in {NLG}",
    author = "Mondella, Irene  and
      Lai, Huiyuan  and
      Nissim, Malvina",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.24/",
    pages = "261--268"
}
@inproceedings{gao-etal-2024-reprohum,
    title = "{R}epro{H}um {\#}0087-01: A Reproduction Study of the Human Evaluation of the Coverage of Fact Checking Explanations",
    author = "Gao, Mingqi  and
      Ruan, Jie  and
      Wan, Xiaojun",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.25/",
    pages = "269--273"
}
@inproceedings{mahlaza-etal-2024-reprohum,
    title = "{R}epro{H}um {\#}0866-04: Another Evaluation of Readers' Reactions to News Headlines",
    author = "Mahlaza, Zola  and
      Raboanary, Toky Hajatiana  and
      Seakgwa, Kyle  and
      Keet, C. Maria",
    editor = "Balloccu, Simone  and
      Belz, Anya  and
      Huidrom, Rudali  and
      Reiter, Ehud  and
      Sedoc, Joao  and
      Thomson, Craig",
    booktitle = "Proceedings of the Fourth Workshop on Human Evaluation of NLP Systems (HumEval) @ LREC-COLING 2024",
    month = may,
    year = "2024",
    address = "Torino, Italia",
    publisher = "ELRA and ICCL",
    url = "https://preview.aclanthology.org/fix-old-resolve/2024.humeval-1.26/",
    pages = "274--280"
}