% Use this file for citations not found in the ACL Anthology (contained in "anthology.bib").

% DISCOGEM

@article{SpoorenDegand+2010+241+266,
    url = {https://doi.org/10.1515/cllt.2010.009},
    title = {Coding coherence relations: Reliability and validity},
    title = {},
    author = {Wilbert Spooren and Liesbeth Degand},
    pages = {241--266},
    volume = {6},
    number = {2},
    journal = {Corpus Linguistics and Linguistic Theory},
    doi = {doi:10.1515/cllt.2010.009},
    year = {2010},
    lastchecked = {2025-09-11}
}

%Webber et al 2019
@book{webber2019pdtb3,
title={{The Penn Discourse Treebank 3.0 annotation manual}},
author={Webber, Bonnie and Prasad, Rashmi and Lee, Alan and Joshi, Aravind},
publisher={Philadelphia, University of Pennsylvania},
year={2019}
}

@incollection{carlson2003building,
  title={Building a discourse-tagged corpus in the framework of rhetorical structure theory},
  author={Carlson, Lynn and Marcu, Daniel and Okurowski, Mary Ellen},
  booktitle={Current and new directions in discourse and dialogue},
  pages={85--112},
  year={2003},
  publisher={Springer}
}

@book{AB2/SUU9CB_2019,
    author = {Prasad, Rashmi and Webber, Bonnie and Lee, Alan and Joshi, Aravind},
    publisher = {Abacus Data Network},
    title = {{Penn Discourse Treebank Version 3.0}},
    year = {2019},
    version = {V1},
    doi = {11272.1/AB2/SUU9CB},
    url = {https://hdl.handle.net/11272.1/AB2/SUU9CB}
}

@inproceedings{hoek-etal-2021-less,
    title = "Is there less annotator agreement when the discourse relation is underspecified?",
    author = "Hoek, Jet  and
      Scholman, Merel C.J.  and
      Sanders, Ted J.M.",
    editor = "De Kuthy, Kordula  and
      Meurers, Detmar",
    booktitle = "Proceedings of the First Workshop on Integrating Perspectives on Discourse Annotation",
    month = oct,
    year = "2021",
    address = {T{\"u}bingen, Germany},
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.discann-1.1/",
    pages = "1--6"
}

@inproceedings{Scholman2017CrowdsourcingDI,
  title={Crowdsourcing discourse interpretations: On the influence of context and the reliability of a connective insertion task},
  author={Merel C. J. Scholman and Vera Demberg},
  booktitle={LAW@ACL},
  year={2017},
  url={https://api.semanticscholar.org/CorpusID:17185562}
}

@inproceedings{yung-demberg-2025-crowdsourcing,title = "On Crowdsourcing Task Design for Discourse Relation Annotation",author = "Yung, Frances and Demberg, Vera",editor = "Roth, Michael and Schlechtweg, Dominik",booktitle = "Proceedings of Context and Meaning: Navigating Disagreements in NLP Annotation",month = jan,year = "2025",address = "Abu Dhabi, UAE",publisher = "International Committee on Computational Linguistics",url = anth # {2025.comedi-1.2/},pages = "12--19"}

@inproceedings{yung-etal-2024-discogem,title = "{D}isco{G}e{M} 2.0: A Parallel Corpus of {E}nglish, {G}erman, {F}rench and {C}zech Implicit Discourse Relations",author = "Yung, Frances and Scholman, Merel and Zikanova, Sarka and Demberg, Vera",editor = "Calzolari, Nicoletta and Kan, Min-Yen and Hoste, Veronique and Lenci, Alessandro and Sakti, Sakriani and Xue, Nianwen",booktitle = "Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)",month = may,year = "2024",address = "Torino, Italia",publisher = "ELRA and ICCL",url = anth # {2024.lrec-main.443/},pages = "4940--4956"}

@inproceedings{yung-etal-2024-prompting,title = "Prompting Implicit Discourse Relation Annotation",author = "Yung, Frances and Ahmad, Mansoor and Scholman, Merel and Demberg, Vera",editor = "Henning, Sophie and Stede, Manfred",booktitle = "Proceedings of the 18th Linguistic Annotation Workshop (LAW-XVIII)",month = mar,year = "2024",address = "St. Julians, Malta",publisher = acl,url = anth # {2024.law-1.15/},pages = "150--165"}

@article{pyatkin-etal-2023-design,title = "Design Choices for Crowdsourcing Implicit Discourse Relations: Revealing the Biases Introduced by Task Design",author = "Pyatkin, Valentina and Yung, Frances and Scholman, Merel C. J. and Tsarfaty, Reut and Dagan, Ido and Demberg, Vera",journal = "Transactions of the Association for Computational Linguistics",volume = "11",year = "2023",address = "Cambridge, MA",publisher = "MIT Press",url = anth # {2023.tacl-1.57/},doi = "10.1162/tacl_a_00586",pages = "1014--1032"}

@inproceedings{yung-etal-2023-investigating,title = "Investigating Explicitation of Discourse Connectives in Translation using Automatic Annotations",author = {Yung, Frances and Scholman, Merel and Lapshinova-Koltunski, Ekaterina and Pollkl{\"a}sener, Christina and Demberg, Vera},editor = "Stoyanchev, Svetlana and Joty, Shafiq and Schlangen, David and Dusek, Ondrej and Kennington, Casey and Alikhani, Malihe",booktitle = "Proceedings of the 24th Annual Meeting of the Special Interest Group on Discourse and Dialogue",month = sep,year = "2023",address = "Prague, Czechia",publisher = acl,url = anth # {2023.sigdial-1.2/},doi = "10.18653/v1/2023.sigdial-1.2",pages = "21--30"}

% DISRPT PAPERS
@inproceedings{long2024multi,
  title={Multi-Label Classification for Implicit Discourse Relation Recognition},
  author={Long, Wanqiu and Siddharth, N and Webber, Bonnie},
  booktitle={Findings of the Association for Computational Linguistics ACL 2024},
  pages={8437--8451},
  year={2024}
}

@inproceedings{braud-etal-2024-disrpt,title = "{DISRPT}: A Multilingual, Multi-domain, Cross-framework Benchmark for Discourse Processing",author = "Braud, Chlo{\'e} and Zeldes, Amir and Rivi{\`e}re, Laura and Liu, Yang Janet and Muller, Philippe and Sileo, Damien and Aoyama, Tatsuya",editor = "Calzolari, Nicoletta and Kan, Min-Yen and Hoste, Veronique and Lenci, Alessandro and Sakti, Sakriani and Xue, Nianwen",booktitle = "Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)",month = may,year = "2024",address = "Torino, Italia",publisher = "ELRA and ICCL",url = anth # {2024.lrec-main.447/},pages = "4990--5005"}

@proceedings{disrpt-2023-shared,title = "Proceedings of the 3rd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2023)",editor = "Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir",month = jul,year = "2023",address = "Toronto, Canada",publisher = "The Association for Computational Linguistics",url = anth # {2023.disrpt-1.0/}}

@inproceedings{braud-etal-2023-disrpt,title = "The {DISRPT} 2023 Shared Task on Elementary Discourse Unit Segmentation, Connective Detection, and Relation Classification",author = "Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir",editor = "Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir",booktitle = "Proceedings of the 3rd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2023)",month = jul,year = "2023",address = "Toronto, Canada",publisher = "The Association for Computational Linguistics",url = anth # {2023.disrpt-1.1/},doi = "10.18653/v1/2023.disrpt-1.1",pages = "1--21"}

@inproceedings{anuranjana-2023-discoflan,title = "{D}isco{F}lan: Instruction Fine-tuning and Refined Text Generation for Discourse Relation Label Classification",author = "Anuranjana, Kaveri",editor = "Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir",booktitle = "Proceedings of the 3rd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2023)",month = jul,year = "2023",address = "Toronto, Canada",publisher = "The Association for Computational Linguistics",url = anth # {2023.disrpt-1.2/},doi = "10.18653/v1/2023.disrpt-1.2",pages = "22--28"}

@inproceedings{metheniti-etal-2023-discut,title = "{D}is{C}ut and {D}isc{R}e{T}: {MELODI} at {DISRPT} 2023",author = "Metheniti, Eleni and Braud, Chlo{\'e} and Muller, Philippe and Rivi{\`e}re, Laura",editor = "Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir",booktitle = "Proceedings of the 3rd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2023)",month = jul,year = "2023",address = "Toronto, Canada",publisher = "The Association for Computational Linguistics",url = anth # {2023.disrpt-1.3/},doi = "10.18653/v1/2023.disrpt-1.3",pages = "29--42"}

@inproceedings{liu-etal-2023-hits,title = "{HITS} at {DISRPT} 2023: Discourse Segmentation, Connective Detection, and Relation Classification",author = "Liu, Wei and Fan, Yi and Strube, Michael",editor = "Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir",booktitle = "Proceedings of the 3rd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2023)",month = jul,year = "2023",address = "Toronto, Canada",publisher = "The Association for Computational Linguistics",url = anth # {2023.disrpt-1.4/},doi = "10.18653/v1/2023.disrpt-1.4",pages = "43--49"}

@proceedings{disrpt-2021-shared,title = "Proceedings of the 2nd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2021)",editor = "Zeldes, Amir and Liu, Yang Janet and Iruskieta, Mikel and Muller, Philippe and Braud, Chlo{\'e} and Badene, Sonia",month = nov,year = "2021",address = "Punta Cana, Dominican Republic",publisher = acl,url = anth # {2021.disrpt-1.0/}}

% DISAGR

@inproceedings{sandri-etal-2023-dont,
    title = "Why Don{'}t You Do It Right? Analysing Annotators' Disagreement in Subjective Tasks",
    author = "Sandri, Marta  and
      Leonardelli, Elisa  and
      Tonelli, Sara  and
      Jezek, Elisabetta",
    editor = "Vlachos, Andreas  and
      Augenstein, Isabelle",
    booktitle = "Proceedings of the 17th Conference of the European Chapter of the Association for Computational Linguistics",
    month = may,
    year = "2023",
    address = "Dubrovnik, Croatia",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.eacl-main.178/",
    doi = "10.18653/v1/2023.eacl-main.178",
    pages = "2428--2441",
    abstract = "Annotators' disagreement in linguistic data has been recently the focus of multiple initiatives aimed at raising awareness on issues related to `majority voting' when aggregating diverging annotations. Disagreement can indeed reflect different aspects of linguistic annotation, from annotators' subjectivity to sloppiness or lack of enough context to interpret a text. In this work we first propose a taxonomy of possible reasons leading to annotators' disagreement in subjective tasks. Then, we manually label part of a Twitter dataset for offensive language detection in English following this taxonomy, identifying how the different categories are distributed. Finally we run a set of experiments aimed at assessing the impact of the different types of disagreement on classification performance. In particular, we investigate how accurately tweets belonging to different categories of disagreement can be classified as offensive or not, and how injecting data with different types of disagreement in the training set affects performance. We also perform offensive language detection as a multi-task framework, using disagreement classification as an auxiliary task."
}

@inproceedings{jiang-etal-2023-ecologically,title = "Ecologically Valid Explanations for Label Variation in {NLI}",author = "Jiang, Nan-Jiang and Tan, Chenhao and de Marneffe, Marie-Catherine",editor = "Bouamor, Houda and Pino, Juan and Bali, Kalika",booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2023",month = dec,year = "2023",address = "Singapore",publisher = acl,url = anth # {2023.findings-emnlp.712/},doi = "10.18653/v1/2023.findings-emnlp.712",pages = "10622--10633"}

@article{jiang-marneffe-2022-investigating,title = "Investigating Reasons for Disagreement in Natural Language Inference",author = "Jiang, Nan-Jiang and de Marneffe, Marie-Catherine",editor = "Roark, Brian and Nenkova, Ani",journal = "Transactions of the Association for Computational Linguistics",volume = "10",year = "2022",address = "Cambridge, MA",publisher = "MIT Press",url = anth # {2022.tacl-1.78/},doi = "10.1162/tacl_a_00523",pages = "1357--1374"}

@inproceedings{weber2024-varierrnli,
    title = "{V}ari{E}rr {NLI}: Separating Annotation Error from Human Label Variation",
    author = "Weber-Genzel, Leon  and
      Peng, Siyao  and
      De Marneffe, Marie-Catherine  and
      Plank, Barbara",
    editor = "Ku, Lun-Wei  and
      Martins, Andre  and
      Srikumar, Vivek",
    booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = aug,
    year = "2024",
    address = "Bangkok, Thailand",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2024.acl-long.123/",
    doi = "10.18653/v1/2024.acl-long.123",
    pages = "2256--2269"
}

@article{liu2019roberta,
  title={Roberta: A robustly optimized bert pretraining approach},
  author={Liu, Yinhan and Ott, Myle and Goyal, Naman and Du, Jingfei and Joshi, Mandar and Chen, Danqi and Levy, Omer and Lewis, Mike and Zettlemoyer, Luke and Stoyanov, Veselin},
  journal={arXiv preprint arXiv:1907.11692},
  year={2019}
}

@inproceedings{fleisig-etal-2024-perspectivist,
    title = "The Perspectivist Paradigm Shift: Assumptions and Challenges of Capturing Human Labels",
    author = "Fleisig, Eve  and
      Blodgett, Su Lin  and
      Klein, Dan  and
      Talat, Zeerak",
    editor = "Duh, Kevin  and
      Gomez, Helena  and
      Bethard, Steven",
    booktitle = "Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)",
    month = jun,
    year = "2024",
    address = "Mexico City, Mexico",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2024.naacl-long.126/",
    doi = "10.18653/v1/2024.naacl-long.126",
    pages = "2279--2292",
    abstract = "Longstanding data labeling practices in machine learning involve collecting and aggregating labels from multiple annotators. But what should we do when annotators disagree? Though annotator disagreement has long been seen as a problem to minimize, new perspectivist approaches challenge this assumption by treating disagreement as a valuable source of information. In this position paper, we examine practices and assumptions surrounding the causes of disagreement{--}some challenged by perspectivist approaches, and some that remain to be addressed{--}as well as practical and normative challenges for work operating under these assumptions. We conclude with recommendations for the data labeling pipeline and avenues for future research engaging with subjectivity and disagreement."
}

@article{prabhakaran2021releasing,
  title={On releasing annotator-level labels and information in datasets},
  author={Prabhakaran, Vinodkumar and Davani, Aida Mostafazadeh and Diaz, Mark},
  journal={arXiv preprint arXiv:2110.05699},
  year={2021}
}

@article{garten2019incorporating,
  title={Incorporating demographic embeddings into language understanding},
  author={Garten, Justin and Kennedy, Brendan and Hoover, Joe and Sagae, Kenji and Dehghani, Morteza},
  journal={Cognitive science},
  volume={43},
  number={1},
  pages={e12701},
  year={2019},
  publisher={Wiley Online Library}
}

@inproceedings{diaz2018addressing,
  title={Addressing age-related bias in sentiment analysis},
  author={D{\'\i}az, Mark and Johnson, Isaac and Lazar, Amanda and Piper, Anne Marie and Gergle, Darren},
  booktitle={Proceedings of the 2018 chi conference on human factors in computing systems},
  pages={1--14},
  year={2018}
}

@inproceedings{basile2020s,
  title={It’s the end of the gold standard as we know it. on the impact of pre-aggregation on the evaluation of highly subjective tasks},
  author={Basile, Valerio},
  booktitle={2020 AIxIA Discussion Papers Workshop, AIxIA 2020 DP},
  volume={2776},
  pages={31--40},
  year={2020},
  organization={CEUR-WS}
}

@article{davani2023hate,
  title={Hate Speech Classifiers Learn Normative Social Stereotypes},
  author={Davani, Aida Mostafazadeh and Atari, Mohammad and Kennedy, Brendan and Dehghani, Morteza},
  journal={Transactions of the Association for Computational Linguistics},
  volume={11},
  pages={300--319},
  year={2023},
  publisher={MIT Press}
}

@inproceedings{sang2022origin,
  title={The Origin and Value of Disagreement Among Data Labelers: A Case Study of Individual Differences in Hate Speech Annotation},
  author={Sang, Yisi and Stanton, Jeffrey},
  booktitle={International Conference on Information},
  pages={425--444},
  year={2022},
  organization={Springer}
}

@article{Cabitza_Campagner_Basile_2023,
    title={Toward a Perspectivist Turn in Ground Truthing for Predictive Computing},
    volume={37},
    url={https://ojs.aaai.org/index.php/AAAI/article/view/25840},
    DOI={10.1609/aaai.v37i6.25840},
    abstractNote={Most current Artificial Intelligence applications are based on supervised Machine Learning (ML), which ultimately grounds on data annotated by small teams of experts or large ensemble of volunteers. The annotation process is often performed in terms of a majority vote, however this has been proved to be often problematic by recent evaluation studies. In this article, we describe and advocate for a different paradigm, which we call perspectivism: this counters the removal of disagreement and, consequently, the assumption of correctness of traditionally aggregated gold-standard datasets, and proposes the adoption of methods that preserve divergence of opinions and integrate multiple perspectives in the ground truthing process of ML development. Drawing on previous works which inspired it, mainly from the crowdsourcing and multi-rater labeling settings, we survey the state-of-the-art and describe the potential of our proposal for not only the more subjective tasks (e.g. those related to human language) but also those tasks commonly understood as objective (e.g. medical decision making). We present the main benefits of adopting a perspectivist stance in ML, as well as possible disadvantages, and various ways in which such a stance can be implemented in practice. Finally, we share a set of recommendations and outline a research agenda to advance the perspectivist stance in ML.},
    number={6},
    journal={Proceedings of the AAAI Conference on Artificial Intelligence},
    author={Cabitza, Federico and Campagner, Andrea and Basile, Valerio},
    year={2023},
    month={Jun.},
    pages={6860-6868}
}

@inproceedings{larimore2021reconsidering,
  title={Reconsidering Annotator Disagreement about Racist Language: Noise or Signal?},
  author={Larimore, Savannah and Kennedy, Ian and Haskett, Breon and Arseniev-Koehler, Alina},
  booktitle={Proceedings of the Ninth International Workshop on Natural Language Processing for Social Media},
  pages={81--90},
  year={2021}
}

@article{pavlick2019inherent,
  title={Inherent disagreements in human textual inferences},
  author={Pavlick, Ellie and Kwiatkowski, Tom},
  journal={Transactions of the Association for Computational Linguistics},
  volume={7},
  pages={677--694},
  year={2019},
  publisher={MIT Press}
}

@article{frenda2024perspectivist,
  title={Perspectivist approaches to natural language processing: a survey},
  author={Frenda, Simona and Abercrombie, Gavin and Basile, Valerio and Pedrani, Alessandro and Panizzon, Raffaella and Cignarella, Alessandra Teresa and Marco, Cristina and Bernardi, Davide},
  journal={Language Resources and Evaluation},
  pages={1--28},
  year={2024},
  publisher={Springer}
}

@inproceedings{plank2014linguistically,
  title={Linguistically debatable or just plain wrong?},
  author={Plank, Barbara and Hovy, Dirk and S{\o}gaard, Anders},
  booktitle={Proceedings of the 52nd Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
  pages={507--511},
  year={2014}
}

@article{sap2021annotators,
  title={Annotators with attitudes: How annotator beliefs and identities bias toxic language detection},
  author={Sap, Maarten and Swayamdipta, Swabha and Vianna, Laura and Zhou, Xuhui and Choi, Yejin and Smith, Noah A},
  journal={arXiv preprint arXiv:2111.07997},
  year={2021}
}

@inproceedings{akhtar2019new,
  title={A new measure of polarization in the annotation of hate speech},
  author={Akhtar, Sohail and Basile, Valerio and Patti, Viviana},
  booktitle={International Conference of the Italian Association for Artificial Intelligence},
  pages={588--603},
  year={2019},
  organization={Springer}
}


% OTHER

@inproceedings{10.1145/3627673.3679584,
author = {Wang, Zhipang and Hong, Yu and Lu, Yuxiang and Zhou, Xiabing and Yao, Jianmin and Zhou, Guodong},
title = {Learning to Differentiate Pairwise-Argument Representations for Implicit Discourse Relation Recognition},
year = {2024},
isbn = {9798400704369},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3627673.3679584},
doi = {10.1145/3627673.3679584},
abstract = {Recognizing implicit discourse relations between texts is challenging due to the absence of explicit connectives. Encoding texts into distinguishable semantic representations is beneficial to connective-free relation determination. To enable encoders to produce clearly distinguishable representations, we propose a joint learning framework. It combines prototypical and adversarial learning as well as hub-migration based redistribution. We experiment on PDTB 2.0, PDTB 3.0 and the CoNLL-2016 shared benchmark dataset. Experimental results show that our methods yield substantial improvements, compared to BERT, RoBERTa and DeBERTa baselines. In the separate comparison experiment where implicit connectives are disable during training, our models outperform the previous work for the four main discourse relation types (Temporality, Comparison, Contingency and Expansion), achieving F1-scores of about 60.75\%, 63.48\%, 75.70\% and 77.09\%. On the other hand, our models obtain comparable performance compared to part of the state-of-the-art models which adopt implicit connectives as observable hints for training. When the pretrained language models are used as the backbones, our methods yield the extra time consumption, which is about 1 hour at worst when training is conducted on Tesla 40GB A100 GPU.},
booktitle = {Proceedings of the 33rd ACM International Conference on Information and Knowledge Management},
pages = {2503–2512},
numpages = {10},
keywords = {adversarial learning, distinguishable representation, hub-migration based redistribution, implicit discourse relation recognition, joint learning framework, prototypical learning},
location = {Boise, ID, USA},
series = {CIKM '24}
}

@inproceedings{deng-etal-2023-annotate,
    title = "You Are What You Annotate: Towards Better Models through Annotator Representations",
    author = "Deng, Naihao  and
      Zhang, Xinliang  and
      Liu, Siyang  and
      Wu, Winston  and
      Wang, Lu  and
      Mihalcea, Rada",
    editor = "Bouamor, Houda  and
      Pino, Juan  and
      Bali, Kalika",
    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2023",
    month = dec,
    year = "2023",
    address = "Singapore",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.findings-emnlp.832",
    doi = "10.18653/v1/2023.findings-emnlp.832",
    pages = "12475--12498",
    abstract = "Annotator disagreement is ubiquitous in natural language processing (NLP) tasks. There are multiple reasons for such disagreements, including the subjectivity of the task, difficult cases, unclear guidelines, and so on. Rather than simply aggregating labels to obtain data annotations, we instead try to directly model the diverse perspectives of the annotators, and explicitly account for annotators{'} idiosyncrasies in the modeling process by creating representations for each annotator (*annotator embeddings*) and also their annotations (*annotation embeddings*). In addition, we propose **TID-8**, **T**he **I**nherent **D**isagreement - **8** dataset, a benchmark that consists of eight existing language understanding datasets that have inherent annotator disagreement. We test our approach on TID-8 and show that our approach helps models learn significantly better from disagreements on six different datasets in TID-8 while increasing model size by fewer than 1{\%} parameters. By capturing the unique tendencies and subjectivity of individual annotators through embeddings, our representations prime AI models to be inclusive of diverse viewpoints.",
}

@inproceedings{hewett-stede-2025-disagreements,
    title = "Disagreements in analyses of rhetorical text structure: A new dataset and first analyses",
    author = "Hewett, Freya  and
      Stede, Manfred",
    editor = "Peng, Siyao  and
      Rehbein, Ines",
    booktitle = "Proceedings of the 19th Linguistic Annotation Workshop (LAW-XIX-2025)",
    month = jul,
    year = "2025",
    address = "Vienna, Austria",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2025.law-1.3/",
    doi = "10.18653/v1/2025.law-1.3",
    pages = "35--47",
    ISBN = "979-8-89176-262-6",
    abstract = "Discourse structure annotation is known to involve a high level of subjectivity, which often results in low inter-annotator agreement. In this paper, we focus on ``legitimate disagreements'', by which we refer to multiple valid annotations for a text or text segment. We provide a new dataset of English and German texts, where each text comes with two parallel analyses (both done by well-trained annotators) in the framework of Rhetorical Structure Theory. Using the RST Tace tool, we build a list of all conflicting annotation decisions and present some statistics for the corpus. Thereafter, we undertake a qualitative analysis of the disagreements and propose a typology of underlying reasons. From this we derive the need to differentiate two kinds of ambiguities in RST annotation: those that result from inherent ``everyday'' linguistic ambiguity, and those that arise from specifications in the theory and/or the annotation schemes."
}

@article{sanders1992toward,
  title={Toward a taxonomy of coherence relations},
  author={Sanders, Ted JM and Spooren, Wilbert PM and Noordman, Leo GM},
  journal={Discourse processes},
  volume={15},
  number={1},
  pages={1--35},
  year={1992},
  publisher={Taylor \& Francis}
}

@incollection{prasad2017penn,
  title={The Penn Discourse Treebank: An annotated corpus of discourse relations},
  author={Prasad, Rashmi and Webber, Bonnie and Joshi, Aravind},
  booktitle={Handbook of linguistic annotation},
  pages={1197--1217},
  year={2017},
  publisher={Springer}
}

@article{ignatev2025annotator,
  title={Annotator disagreement in RST annotation schemes},
  author={Ignatev, Daniil and Paperno, Denis and Poesio, Massimo},
  journal={Society for Computation in Linguistics},
  volume={8},
  number={1},
  year={2025},
  publisher={University of Massachusetts Amherst Libraries}
}

@inproceedings{zikanova2024text,
  title={Text structure and its ambiguities: corpus annotation as a helpful guide},
  author={Zik{\'a}nov{\'a}, S{\'a}rka},
  booktitle={ITAT},
  pages={2--12},
  year={2024}
}

@inproceedings{zikanova2025gold,
  title={Gold Data and Multiple Understanding of Discourse Relations},
  author={Zik{\'a}nov{\'a}, {\v{S}}{\'a}rka and Nedoluzhko, Anna and M{\'\i}rovsk{\`y}, Ji{\v{r}}{\'\i} and Haji{\v{c}}ov{\'a}, Eva},
  booktitle={International Conference on Text, Speech, and Dialogue},
  pages={250--262},
  year={2025},
  organization={Springer}
}

@misc{liu2023whats,
      title={What's Hard in English RST Parsing? Predictive Models for Error Analysis}, 
      author={Yang Janet Liu and Tatsuya Aoyama and Amir Zeldes},
      year={2023},
      eprint={2309.04940},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@article{mann1988rhetorical,
  title={Rhetorical structure theory: Toward a functional theory of text organization},
  author={Mann, William C and Thompson, Sandra A},
  journal={Text-interdisciplinary Journal for the Study of Discourse},
  volume={8},
  number={3},
  pages={243--281},
  year={1988},
  publisher={Walter de Gruyter, Berlin/New York Berlin, New York}
}

@inproceedings{mokhberian-etal-2024-capturing,
    title = "Capturing Perspectives of Crowdsourced Annotators in Subjective Learning Tasks",
    author = "Mokhberian, Negar  and
      Marmarelis, Myrl  and
      Hopp, Frederic  and
      Basile, Valerio  and
      Morstatter, Fred  and
      Lerman, Kristina",
    booktitle = "Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)",
    month = jun,
    year = "2024",
    address = "Mexico City, Mexico",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2024.naacl-long.407/",
    doi = "10.18653/v1/2024.naacl-long.407",
    pages = "7337--7349",
}

@misc{deng2023annotate,
      title={You Are What You Annotate: Towards Better Models through Annotator Representations}, 
      author={Naihao Deng and Xinliang Frederick Zhang and Siyang Liu and Winston Wu and Lu Wang and Rada Mihalcea},
      year={2023},
      eprint={2305.14663},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@inproceedings{harju2023evaluating,
  title={Evaluating Classification Systems Against Soft Labels with Fuzzy Precision and Recall},
  author={Harju, Manu and Mesaros, Annamaria},
  booktitle={Workshop on Detection and Classification of Acoustic Scenes and Events},
  pages={46--50},
  year={2023},
  organization={Tampere University}
}

@inproceedings{LeWiDi2025,
  title={LeWiDi-2025 at NLPerspectives: third Edition of the Learning with Disagreements Shared Task
},
  author={Leonardelli,Elisa and Casola,Silvia and Peng,Siyao and Rizzi,Giulia and Basile,Valerio and Fersini,Elisabetta and Frassinelli,Diego and Jang,Hyewon and Pavlovic,Maja and Plank,Barbara and Poesio,Massimo},
  booktitle={Proceedings of the 4th Workshop on Perspectivist Approaches to NLP (NLPerspectives)},
month = {nov},  
year={2025},
publisher = {Association for Computational Linguistics}
}

@inproceedings{fornaciari2021beyond,
  title={Beyond black \& white: Leveraging annotator disagreement via soft-label multi-task learning},
  author={Fornaciari, Tommaso and Uma, Alexandra and Paun, Silviu and Plank, Barbara and Hovy, Dirk and Poesio, Massimo and others},
  booktitle={Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
  year={2021},
  organization={Association for Computational Linguistics}
}

@inproceedings{rizzi2024soft,
  title={Soft metrics for evaluation with disagreements: an assessment},
  author={Rizzi, Giulia and Leonardelli, Elisa and Poesio, Massimo and Uma, Alexandra and Pavlovic, Maja and Paun, Silviu and Rosso, Paolo and Fersini, Elisabetta},
  booktitle={3rd Workshop on Perspectivist Approaches to NLP, NLPerspectives 2024},
  pages={84--94},
  year={2024},
  organization={European Language Resources Association (ELRA)}
}


@inproceedings{weber-genzel-etal-2024-varierr,
    title = "{V}ari{E}rr {NLI}: Separating Annotation Error from Human Label Variation",
    author = "Weber-Genzel, Leon  and
      Peng, Siyao  and
      De Marneffe, Marie-Catherine  and
      Plank, Barbara",
    editor = "Ku, Lun-Wei  and
      Martins, Andre  and
      Srikumar, Vivek",
    booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = aug,
    year = "2024",
    address = "Bangkok, Thailand",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2024.acl-long.123/",
    doi = "10.18653/v1/2024.acl-long.123",
    pages = "2256--2269",
    abstract = "Human label variation arises when annotators assign different labels to the same item for valid reasons, while annotation errors occur when labels are assigned for invalid reasons. These two issues are prevalent in NLP benchmarks, yet existing research has studied them in isolation. To the best of our knowledge, there exists no prior work that focuses on teasing apart error from signal, especially in cases where signal is beyond black-and-white.To fill this gap, we introduce a systematic methodology and a new dataset, VariErr (variation versus error), focusing on the NLI task in English. We propose a 2-round annotation procedure with annotators explaining each label and subsequently judging the validity of label-explanation pairs.VariErr contains 7,732 validity judgments on 1,933 explanations for 500 re-annotated MNLI items. We assess the effectiveness of various automatic error detection (AED) methods and GPTs in uncovering errors versus human label variation. We find that state-of-the-art AED methods significantly underperform GPTs and humans. While GPT-4 is the best system, it still falls short of human performance. Our methodology is applicable beyond NLI, offering fertile ground for future research on error versus plausible variation, which in turn can yield better and more trustworthy NLP systems."
}