%%% ====================================================================
%%%  BibTeX-file{
%%%     author          = "David Rhead",
%%%     version         = "1.00",
%%%     date            = "17 Feb 1990",
%%%     time            = "17:00 GMT",
%%%     filename        = "test.bib",
%%%     address         = "Cripps Computing Centre,
%%%                        University of Nottingham,
%%%                        University Park,
%%%                        Nottingham,
%%%                        NG7 2RD,
%%%                        United Kingdom",
%%%     telephone       = "+44 602 484848 Ext 2670",
%%%     FAX             = "+44 602 588138",
%%%     checksum        = "05151 839 2908 25082",
%%%     email           = "David_Rhead at uk.ac.nott.vme (JANET)",
%%%     codetable       = "ISO/ASCII",
%%%     keywords        = "bibliography, citation, references",
%%%     supported       = "no",
%%%     docstring       = "This BibTeX database file contains entries
%%%                        designed for testing whether a BibTeX style
%%%                        file lays references out as recommended by
%%%                        certain authorities.  (Note, however, that
%%%                        the BS 1629 examples are from the 1976
%%%                        edition.  The file needs updating to use
%%%                        examples from the 1989 edition instead.)
%%%
%%%                        The checksum field above contains a CRC-16
%%%                        checksum as the first value, followed by the
%%%                        equivalent of the standard UNIX wc (word
%%%                        count) utility output of lines, words, and
%%%                        characters.  This is produced by Robert
%%%                        Solovay's checksum utility.",
%%%  }
%%% ====================================================================
%%
@comment{Some other standard works describing conventions for citations
%%      and bibliographies}
% Use this file for citations not found in the ACL Anthology (contained in "anthology.bib").
@InProceedings{plank-2022-problem,
  title        = {The ``Problem'' of Human Label Variation: On Ground Truth in Data, Modeling and Evaluation},
  author       = {Plank, Barbara},
  year         = 2022,
  month        = dec,
  booktitle    = {Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing},
  publisher    = {Association for Computational Linguistics},
  address      = {Abu Dhabi, United Arab Emirates},
  pages        = {10671--10682},
  doi          = {10.18653/v1/2022.emnlp-main.731},
  URL          = {https://aclanthology.org/2022.emnlp-main.731/},
  editor       = {Goldberg, Yoav and Kozareva, Zornitsa and Zhang, Yue}
}
@InProceedings{ferracane-etal-2021-answer,
  title        = {Did they answer? Subjective acts and intents in conversational discourse},
  author       = {Ferracane, Elisa and Durrett, Greg and Li, Junyi Jessy and Erk, Katrin},
  year         = 2021,
  month        = jun,
  booktitle    = {Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
  publisher    = {Association for Computational Linguistics},
  address      = {Online},
  pages        = {1626--1644},
  doi          = {10.18653/v1/2021.naacl-main.129},
  URL          = {https://aclanthology.org/2021.naacl-main.129/},
  editor       = {Toutanova, Kristina and Rumshisky, Anna and Zettlemoyer, Luke and Hakkani-Tur, Dilek and Beltagy, Iz and Bethard, Steven and Cotterell, Ryan and Chakraborty, Tanmoy and Zhou, Yichao}
}
@InProceedings{nie-etal-2020-learn,
  title        = {What Can We Learn from Collective Human Opinions on Natural Language Inference Data?},
  author       = {Nie, Yixin and Zhou, Xiang and Bansal, Mohit},
  year         = 2020,
  month        = nov,
  booktitle    = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
  publisher    = {Association for Computational Linguistics},
  address      = {Online},
  pages        = {9131--9143},
  doi          = {10.18653/v1/2020.emnlp-main.734},
  URL          = {https://aclanthology.org/2020.emnlp-main.734/},
  editor       = {Webber, Bonnie and Cohn, Trevor and He, Yulan and Liu, Yang}
}
@Article{davani-etal-2022-dealing,
  title        = {Dealing with Disagreements: Looking Beyond the Majority Vote in Subjective Annotations},
  author       = {Davani, Aida Mostafazadeh and D{\'i}az, Mark and Prabhakaran, Vinodkumar},
  year         = 2022,
  journal      = {Transactions of the Association for Computational Linguistics},
  publisher    = {MIT Press},
  address      = {Cambridge, MA},
  volume       = 10,
  pages        = {92--110},
  doi          = {10.1162/tacl_a_00449},
  URL          = {https://aclanthology.org/2022.tacl-1.6/},
  editor       = {Roark, Brian and Nenkova, Ani}
}
@InProceedings{hoek-scholman-2017-evaluating,
  title        = {Evaluating discourse annotation: Some recent insights and new approaches},
  author       = {Hoek, Jet and Scholman, Merel},
  year         = 2017,
  booktitle    = {Proceedings of the 13th Joint {ISO}-{ACL} Workshop on Interoperable Semantic Annotation ({ISA}-13)},
  URL          = {https://aclanthology.org/W17-7401/}
}
@inproceedings{yung2019crowdsourcing,
  title={Crowdsourcing discourse relation annotations by a two-step connective insertion task},
  author={Yung, Frances and Demberg, Vera and Scholman, Merel},
  booktitle={Proceedings of the 13th Linguistic Annotation Workshop},
  pages={16--25},
  year={2019}
}
@InProceedings{das-etal-2017-good,
  title        = {The Good, the Bad, and the Disagreement: Complex ground truth in rhetorical structure analysis},
  author       = {Das, Debopam and Stede, Manfred and Taboada, Maite},
  year         = 2017,
  month        = sep,
  booktitle    = {Proceedings of the 6th Workshop on Recent Advances in {RST} and Related Formalisms},
  publisher    = {Association for Computational Linguistics},
  address      = {Santiago de Compostela, Spain},
  pages        = {11--19},
  doi          = {10.18653/v1/W17-3602},
  ISBN         = {978-1-945626-78-4},
  URL          = {https://aclanthology.org/W17-3602/},
  editor       = {Taboada, M. and da Cunha, I. and Maziero, E.G. and Cardoso, P. and Antonio, J.D. and Iruskieta, M.}
}
@InProceedings{yung-etal-2022-label,
  title        = {Label distributions help implicit discourse relation classification},
  author       = {Yung, Frances and Anuranjana, Kaveri and Scholman, Merel and Demberg, Vera},
  year         = 2022,
  month        = oct,
  booktitle    = {Proceedings of the 3rd Workshop on Computational Approaches to Discourse},
  publisher    = {International Conference on Computational Linguistics},
  address      = {Gyeongju, Republic of Korea and Online},
  pages        = {48--53},
  URL          = {https://aclanthology.org/2022.codi-1.7/},
  editor       = {Braud, Chlo{\'e} and Hardmeier, Christian and Li, Junyi Jessy and Loaiciga, Sharid and Strube, Michael and Zeldes, Amir}
}
@InProceedings{huang-etal-2021-balancing,
  title        = {Balancing Methods for Multi-label Text Classification with Long-Tailed Class Distribution},
  author       = {Huang, Yi and Giledereli, Buse and K{\"o}ksal, Abdullatif and {\"O}zg{\"u}r, Arzucan and Ozkirimli, Elif},
  year         = 2021,
  month        = nov,
  booktitle    = {Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},
  publisher    = {Association for Computational Linguistics},
  address      = {Online and Punta Cana, Dominican Republic},
  pages        = {8153--8161},
  doi          = {10.18653/v1/2021.emnlp-main.643},
  URL          = {https://aclanthology.org/2021.emnlp-main.643/},
  editor       = {Moens, Marie-Francine and Huang, Xuanjing and Specia, Lucia and Yih, Scott Wen-tau}
}

@Article{kennedy2022introducing,
  title        = {Introducing the {Gab Hate Corpus}: Defining and applying hate-based rhetoric to social media posts at scale},
  author       = {Kennedy, Brendan and Atari, Mohammad and Mostafazadeh Davani, Aida and Yeh, Leigh and Omrani, Ali and Kim, Yehsong and Coombs Jr., Kris and Havaldar, Shreya and Portillo-Wightman, Gwenyth and Gonzalez, Elaine and Hoover, Joe and Azatian, Aida and Cardenas, Gabriel and Hussain, Alyzeh and Lara, Austin and Omary, Adam and Park, Christina and Wang, Xin and Wijaya, Clarisa and Zhang, Yong and Meyerowitz, Beth and Dehghani, Morteza},
  year         = 2022,
  month        = feb,
  journal      = {Language Resources and Evaluation},
  publisher    = {Springer Netherlands},
  volume       = 1,
  pages        = {79--108},
  URL          = {https://doi.org/10.1007/s10579-021-09569-x},
  issue        = 16
}

@InProceedings{kanclerz-etal-2022-ground,
  title        = {What If Ground Truth Is Subjective? Personalized Deep Neural Hate Speech Detection},
  author       = {Kanclerz, Kamil and Gruza, Marcin and Karanowski, Konrad and Bielaniewicz, Julita and Milkowski, Piotr and Kocon, Jan and Kazienko, Przemyslaw},
  year         = 2022,
  month        = jun,
  booktitle    = {Proceedings of the 1st Workshop on Perspectivist Approaches to NLP @LREC2022},
  publisher    = {European Language Resources Association},
  address      = {Marseille, France},
  pages        = {37--45},
  URL          = {https://aclanthology.org/2022.nlperspectives-1.6/},
  editor       = {Abercrombie, Gavin and Basile, Valerio and Tonelli, Sara and Rieser, Verena and Uma, Alexandra}
}
@InProceedings{rottger-etal-2022-two,
  title        = {Two Contrasting Data Annotation Paradigms for Subjective {NLP} Tasks},
  author       = {R{\"o}ttger, Paul and Vidgen, Bertie and Hovy, Dirk and Pierrehumbert, Janet},
  year         = 2022,
  month        = jul,
  booktitle    = {Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
  publisher    = {Association for Computational Linguistics},
  address      = {Seattle, United States},
  pages        = {175--190},
  doi          = {10.18653/v1/2022.naacl-main.13},
  URL          = {https://aclanthology.org/2022.naacl-main.13/},
  editor       = {Carpuat, Marine and de Marneffe, Marie-Catherine and Meza Ruiz, Ivan Vladimir}
}
@InProceedings{leonardelli-etal-2021-agreeing,
  title        = {Agreeing to Disagree: Annotating Offensive Language Datasets with Annotators' Disagreement},
  author       = {Leonardelli, Elisa and Menini, Stefano and Palmero Aprosio, Alessio and Guerini, Marco and Tonelli, Sara},
  year         = 2021,
  month        = nov,
  booktitle    = {Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},
  publisher    = {Association for Computational Linguistics},
  address      = {Online and Punta Cana, Dominican Republic},
  pages        = {10528--10539},
  doi          = {10.18653/v1/2021.emnlp-main.822},
  URL          = {https://aclanthology.org/2021.emnlp-main.822/},
  editor       = {Moens, Marie-Francine and Huang, Xuanjing and Specia, Lucia and Yih, Scott Wen-tau}
}
@InProceedings{ngo-etal-2022-studemo,
  title        = {{S}tud{E}mo: A Non-aggregated Review Dataset for Personalized Emotion Recognition},
  author       = {Ngo, Anh and Candri, Agri and Ferdinan, Teddy and Kocon, Jan and Korczynski, Wojciech},
  year         = 2022,
  month        = jun,
  booktitle    = {Proceedings of the 1st Workshop on Perspectivist Approaches to NLP @LREC2022},
  publisher    = {European Language Resources Association},
  address      = {Marseille, France},
  pages        = {46--55},
  URL          = {https://aclanthology.org/2022.nlperspectives-1.7/},
  editor       = {Abercrombie, Gavin and Basile, Valerio and Tonelli, Sara and Rieser, Verena and Uma, Alexandra}
}
@InProceedings{sanders-etal-2022-towards,
  title        = {Towards a Progression-Aware Autonomous Dialogue Agent},
  author       = {Sanders, Abraham and Strzalkowski, Tomek and Si, Mei and Chang, Albert and Dey, Deepanshu and Braasch, Jonas and Wang, Dakuo},
  year         = 2022,
  month        = jul,
  booktitle    = {Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
  publisher    = {Association for Computational Linguistics},
  address      = {Seattle, United States},
  pages        = {1194--1212},
  doi          = {10.18653/v1/2022.naacl-main.87},
  URL          = {https://aclanthology.org/2022.naacl-main.87/},
  editor       = {Carpuat, Marine and de Marneffe, Marie-Catherine and Meza Ruiz, Ivan Vladimir}
}
@Article{artstein-poesio-2008-survey,
  title        = {Survey Article: Inter-Coder Agreement for Computational Linguistics},
  author       = {Artstein, Ron and Poesio, Massimo},
  year         = 2008,
  journal      = {Computational Linguistics},
  volume       = 34,
  number       = 4,
  pages        = {555--596},
  doi          = {10.1162/coli.07-034-R2},
  URL          = {https://aclanthology.org/J08-4004/}
}
% DISCOGEM
@Article{SpoorenDegand+2010+241+266,
  title        = {Coding coherence relations: Reliability and validity},
  author       = {Spooren, Wilbert PM and Degand, Liesbeth},
  year         = 2010,
  journal      = {Corpus Linguistics and Linguistic Theory},
  volume       = 6,
  number       = 2,
  pages        = {241--266},
  doi          = {doi:10.1515/cllt.2010.009},
  URL          = {https://doi.org/10.1515/cllt.2010.009},
  lastchecked  = {2025-09-11}
}

%Webber et al 2019
@Book{webber2019pdtb3,
  title        = {{The {P}enn {D}iscourse {T}reebank 3.0 annotation manual}},
  author       = {Webber, Bonnie and Prasad, Rashmi and Lee, Alan and Joshi, Aravind},
  year         = 2019,
  publisher    = {Philadelphia, University of Pennsylvania}
}
@InCollection{carlson2003building,
  title        = {Building a discourse-tagged corpus in the framework of rhetorical structure theory},
  author       = {Carlson, Lynn and Marcu, Daniel and Okurowski, Mary Ellen},
  year         = 2003,
  booktitle    = {Current and new directions in discourse and dialogue},
  publisher    = {Springer},
  pages        = {85--112}
}
@Article{zwaan1998situation,
  title        = {Situation models in language comprehension and memory},
  author       = {Zwaan, Rolf A and Radvansky, Gabriel A},
  year         = 1998,
  journal      = {Psychological Bulletin},
  publisher    = {American Psychological Association},
  volume       = 123,
  number       = 2,
  pages        = 162
}
@Article{graesser1994constructing,
  title        = {Constructing inferences during narrative text comprehension},
  author       = {Graesser, Arthur C and Singer, Murray and Trabasso, Tom},
  year         = 1994,
  journal      = {Psychological Review},
  publisher    = {American Psychological Association},
  volume       = 101,
  number       = 3,
  pages        = 371
}
@Article{scholman2020individual,
  title        = {Individual differences in expecting coherence relations: Exploring the variability in sensitivity to contextual signals in discourse},
  author       = {Scholman, Merel and Demberg, Vera and Sanders, Ted JM},
  year         = 2020,
  journal      = {Discourse Processes},
  publisher    = {Taylor \& Francis},
  volume       = 57,
  number       = 10,
  pages        = {844--861}
}
@Article{marchal2022effect,
  title        = {The effect of domain knowledge on discourse relation inferences: Relation marking and interpretation strategies},
  author       = {Marchal, Marian and Scholman, Merel and Demberg, Vera},
  year         = 2022,
  journal      = {Dialogue and Discourse},
  publisher    = {The Dialogue \& Discourse Board of Editors},
  volume       = 13,
  number       = 2,
  pages        = {49--78}
}
@InProceedings{hoek-etal-2021-less,
  title        = {Is there less annotator agreement when the discourse relation is underspecified?},
  author       = {Hoek, Jet  and Scholman, Merel  and Sanders, Ted JM},
  year         = 2021,
  month        = oct,
  booktitle    = {Proceedings of the First Workshop on Integrating Perspectives on Discourse Annotation},
  publisher    = {Association for Computational Linguistics},
  address      = {T{\"u}bingen, Germany},
  pages        = {1--6},
  URL          = {https://aclanthology.org/2021.discann-1.1/},
  editor       = {De Kuthy, Kordula  and Meurers, Detmar}
}
@Article{akhtar2021whose,
  title        = {Whose opinions matter? perspective-aware models to identify opinions of hate speech victims in abusive language detection},
  author       = {Akhtar, Sohail and Basile, Valerio and Patti, Viviana},
  year         = 2021,
  journal      = {arXiv preprint arXiv:2106.15896}
}
# no non-archival version

@InProceedings{Scholman2017CrowdsourcingDI,
  title        = {Crowdsourcing discourse interpretations: On the influence of context and the reliability of a connective insertion task},
  author       = {Scholman, Merel  and Demberg, Vera},
  year         = 2017,
  month        = apr,
  booktitle    = {Proceedings of the 11th Linguistic Annotation Workshop},
  publisher    = {Association for Computational Linguistics},
  address      = {Valencia, Spain},
  pages        = {24--33},
  doi          = {10.18653/v1/W17-0803},
  url          = {https://aclanthology.org/W17-0803/},
  editor       = {Schneider, Nathan  and Xue, Nianwen},
  abstract     = {Traditional discourse annotation tasks are considered costly and time-consuming, and the reliability and validity of these tasks is in question. In this paper, we investigate whether crowdsourcing can be used to obtain reliable discourse relation annotations. We also examine the influence of context on the reliability of the data. The results of a crowdsourced connective insertion task showed that the method can be used to obtain reliable annotations: The majority of the inserted connectives converged with the original label. Further, the method is sensitive to the fact that multiple senses can often be inferred for a single relation. Regarding the presence of context, the results show no significant difference in distributions of insertions between conditions overall. However, a by-item comparison revealed several characteristics of segments that determine whether the presence of context makes a difference in annotations. The findings discussed in this paper can be taken as evidence that crowdsourcing can be used as a valuable method to obtain insights into the sense(s) of relations.}
}

@InProceedings{yung-demberg-2025-crowdsourcing,
  title        = {On Crowdsourcing Task Design for Discourse Relation Annotation},
  author       = {Yung, Frances and Demberg, Vera},
  year         = 2025,
  month        = jan,
  booktitle    = {Proceedings of Context and Meaning: Navigating Disagreements in NLP Annotation},
  publisher    = {International Committee on Computational Linguistics},
  address      = {Abu Dhabi, UAE},
  pages        = {12--19},
  URL          = {https://aclanthology.org/2025.comedi-1.2/},
  editor       = {Roth, Michael and Schlechtweg, Dominik}
}
@InProceedings{yung-etal-2024-discogem,
  title        = {{D}isco{G}e{M} 2.0: A Parallel Corpus of {E}nglish, {G}erman, {F}rench and {C}zech Implicit Discourse Relations},
  author       = {Yung, Frances and Scholman, Merel and Zik{\'a}nov{\'a}, {\v{S}}{\'a}rka and Demberg, Vera},
  year         = 2024,
  month        = may,
  booktitle    = {Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)},
  publisher    = {ELRA and ICCL},
  address      = {Torino, Italia},
  pages        = {4940--4956},
  URL          = {https://aclanthology.org/2024.lrec-main.443/},
  editor       = {Calzolari, Nicoletta and Kan, Min-Yen and Hoste, Veronique and Lenci, Alessandro and Sakti, Sakriani and Xue, Nianwen}
}
@InProceedings{yung-etal-2024-prompting,
  title        = {Prompting Implicit Discourse Relation Annotation},
  author       = {Yung, Frances and Ahmad, Mansoor and Scholman, Merel and Demberg, Vera},
  year         = 2024,
  month        = mar,
  booktitle    = {Proceedings of the 18th Linguistic Annotation Workshop (LAW-XVIII)},
  publisher    = {Association for Computational Linguistics},
  address      = {St. Julians, Malta},
  pages        = {150--165},
  URL          = {https://aclanthology.org/2024.law-1.15/},
  editor       = {Henning, Sophie and Stede, Manfred}
}
@InProceedings{scholman2022design,
  title        = {Design choices in crowdsourcing discourse relation annotations: The effect of worker selection and training},
  author       = {Scholman, Merel and Pyatkin, Valentina and Yung, Frances and Dagan, Ido and Tsarfaty, Reut and Demberg, Vera},
  booktitle    = {Proceedings of the thirteenth language resources and evaluation conference},
  pages        = {2148--2156},
  year         = {2022}
}
@Article{pyatkin-etal-2023-design,
  title        = {Design Choices for Crowdsourcing Implicit Discourse Relations: Revealing the Biases Introduced by Task Design},
  author       = {Pyatkin, Valentina and Yung, Frances and Scholman, Merel and Tsarfaty, Reut and Dagan, Ido and Demberg, Vera},
  year         = 2023,
  journal      = {Transactions of the Association for Computational Linguistics},
  publisher    = {MIT Press},
  address      = {Cambridge, MA},
  volume       = 11,
  pages        = {1014--1032},
  doi          = {10.1162/tacl_a_00586},
  URL          = {https://aclanthology.org/2023.tacl-1.57/}
}
@InProceedings{yung-etal-2023-investigating,
  title        = {Investigating Explicitation of Discourse Connectives in Translation using Automatic Annotations},
  author       = {Yung, Frances and Scholman, Merel and Lapshinova-Koltunski, Ekaterina and Pollkl{\"a}sener, Christina and Demberg, Vera},
  year         = 2023,
  month        = sep,
  booktitle    = {Proceedings of the 24th Annual Meeting of the Special Interest Group on Discourse and Dialogue},
  publisher    = {Association for Computational Linguistics},
  address      = {Prague, Czechia},
  pages        = {21--30},
  doi          = {10.18653/v1/2023.sigdial-1.2},
  URL          = {https://aclanthology.org/2023.sigdial-1.2/},
  editor       = {Stoyanchev, Svetlana and Joty, Shafiq and Schlangen, David and Dusek, Ondrej and Kennington, Casey and Alikhani, Malihe}
}
% DISRPT PAPERS
@Article{yu2020gradient,
  title        = {Gradient surgery for multi-task learning},
  author       = {Yu, Tianhe and Kumar, Saurabh and Gupta, Abhishek and Levine, Sergey and Hausman, Karol and Finn, Chelsea},
  year         = 2020,
  journal      = {Advances in neural information processing systems},
  volume       = 33,
  pages        = {5824--5836}
}
@InProceedings{costa2024exploring,
  title        = {Exploring soft-label training for implicit discourse relation recognition},
  author       = {Costa, Nelson Filipe and Kosseim, Leila},
  year         = 2024,
  booktitle    = {Proceedings of the 5th Workshop on Computational Approaches to Discourse (CODI 2024)},
  pages        = {120--126}
}
@InProceedings{long-etal-2024-multi,
  title        = {Multi-Label Classification for Implicit Discourse Relation Recognition},
  author       = {Long, Wanqiu  and Siddharth, N.  and Webber, Bonnie},
  year         = 2024,
  month        = aug,
  booktitle    = {Findings of the Association for Computational Linguistics: ACL 2024},
  publisher    = {Association for Computational Linguistics},
  address      = {Bangkok, Thailand},
  pages        = {8437--8451},
  doi          = {10.18653/v1/2024.findings-acl.500},
  URL          = {https://aclanthology.org/2024.findings-acl.500/},
  editor       = {Ku, Lun-Wei  and Martins, Andre  and Srikumar, Vivek},
  abstract     = {Discourse relations play a pivotal role in establishing coherence within textual content, uniting sentences and clauses into a cohesive narrative. The Penn Discourse Treebank (PDTB) stands as one of the most extensively utilized datasets in this domain. In PDTB-3, the annotators can assign multiple labels to an example, when they believe the simultaneous presence of multiple relations. Prior research in discourse relation recognition has treated these instances as separate examples during training, with a gold-standard prediction matching one of the labels considered correct at test time. However, this approach is inadequate, as it fails to account for the interdependence of labels in real-world contexts and to distinguish between cases where only one sense relation holds and cases where multiple relations hold simultaneously. In our work, we address this challenge by exploring various multi-label classification frameworks to handle implicit discourse relation recognition. We show that the methods for multi-label prediction don{'}t depress performance for single-label prediction. Additionally, we give comprehensive analysis of results and data. Our work contributes to advancing the understanding and application of discourse relations and provide a foundation for the future study.}
}
@InProceedings{braud-etal-2024-disrpt,
  title        = {{DISRPT}: A Multilingual, Multi-domain, Cross-framework Benchmark for Discourse Processing},
  author       = {Braud, Chlo{\'e} and Zeldes, Amir and Rivi{\`e}re, Laura and Liu, Yang Janet and Muller, Philippe and Sileo, Damien and Aoyama, Tatsuya},
  year         = 2024,
  month        = may,
  booktitle    = {Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)},
  publisher    = {ELRA and ICCL},
  address      = {Torino, Italia},
  pages        = {4990--5005},
  URL          = {https://aclanthology.org/2024.lrec-main.447/},
  editor       = {Calzolari, Nicoletta and Kan, Min-Yen and Hoste, Veronique and Lenci, Alessandro and Sakti, Sakriani and Xue, Nianwen}
}
@proceedings{disrpt-2023-shared,
  title        = {Proceedings of the 3rd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2023)},
  year         = 2023,
  month        = jul,
  publisher    = {The Association for Computational Linguistics},
  address      = {Toronto, Canada},
  URL          = {https://aclanthology.org/2023.disrpt-1.0/},
  editor       = {Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir}
}
@InProceedings{braud-etal-2023-disrpt,
  title        = {The {DISRPT} 2023 Shared Task on Elementary Discourse Unit Segmentation, Connective Detection, and Relation Classification},
  author       = {Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir},
  year         = 2023,
  month        = jul,
  booktitle    = {Proceedings of the 3rd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2023)},
  publisher    = {The Association for Computational Linguistics},
  address      = {Toronto, Canada},
  pages        = {1--21},
  doi          = {10.18653/v1/2023.disrpt-1.1},
  URL          = {https://aclanthology.org/2023.disrpt-1.1/},
  editor       = {Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir}
}
@InProceedings{anuranjana-2023-discoflan,
  title        = {{D}isco{F}lan: Instruction Fine-tuning and Refined Text Generation for Discourse Relation Label Classification},
  author       = {Anuranjana, Kaveri},
  year         = 2023,
  month        = jul,
  booktitle    = {Proceedings of the 3rd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2023)},
  publisher    = {The Association for Computational Linguistics},
  address      = {Toronto, Canada},
  pages        = {22--28},
  doi          = {10.18653/v1/2023.disrpt-1.2},
  URL          = {https://aclanthology.org/2023.disrpt-1.2/},
  editor       = {Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir}
}
@InProceedings{metheniti-etal-2023-discut,
  title        = {{D}is{C}ut and {D}isc{R}e{T}: {MELODI} at {DISRPT} 2023},
  author       = {Metheniti, Eleni and Braud, Chlo{\'e} and Muller, Philippe and Rivi{\`e}re, Laura},
  year         = 2023,
  month        = jul,
  booktitle    = {Proceedings of the 3rd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2023)},
  publisher    = {The Association for Computational Linguistics},
  address      = {Toronto, Canada},
  pages        = {29--42},
  doi          = {10.18653/v1/2023.disrpt-1.3},
  URL          = {https://aclanthology.org/2023.disrpt-1.3/},
  editor       = {Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir}
}
@InProceedings{liu-etal-2023-hits,
  title        = {{HITS} at {DISRPT} 2023: Discourse Segmentation, Connective Detection, and Relation Classification},
  author       = {Liu, Wei and Fan, Yi and Strube, Michael},
  year         = 2023,
  month        = jul,
  booktitle    = {Proceedings of the 3rd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2023)},
  publisher    = {The Association for Computational Linguistics},
  address      = {Toronto, Canada},
  pages        = {43--49},
  doi          = {10.18653/v1/2023.disrpt-1.4},
  URL          = {https://aclanthology.org/2023.disrpt-1.4/},
  editor       = {Braud, Chlo{\'e} and Liu, Yang Janet and Metheniti, Eleni and Muller, Philippe and Rivi{\`e}re, Laura and Rutherford, Attapol and Zeldes, Amir}
}
@Proceedings{disrpt-2021-shared,
  title        = {Proceedings of the 2nd Shared Task on Discourse Relation Parsing and Treebanking (DISRPT 2021)},
  year         = 2021,
  month        = nov,
  publisher    = {Association for Computational Linguistics},
  address      = {Punta Cana, Dominican Republic},
  URL          = {https://aclanthology.org/2021.disrpt-1.0/},
  editor       = {Zeldes, Amir and Liu, Yang Janet and Iruskieta, Mikel and Muller, Philippe and Braud, Chlo{\'e} and Badene, Sonia}
}
% DISAGR
@InProceedings{poesio&artstein:ACL-ANNO-05,
  title        = {The Reliability of Anaphoric Annotation, Reconsidered: Taking Ambiguity into Account},
  author       = {Poesio, Massimo and Artstein, Ron},
  year         = 2005,
  month        = {June},
  booktitle    = {Proceedings of the Workshop on Frontiers in Corpus Annotations {II}: Pie in the Sky},
  address      = {Ann Arbor, Michigan},
  pages        = {76--83},
  URL          = {https://aclanthology.org/W05-0311/},
  shortauthor  = {Poesio, M. and Artstein, R.},
  editor       = {Meyers, Adam}
}
@Article{aroyo&welty:AIMagazine15,
  title        = {Truth is A Lie: Crowd Truth and the Seven Myths of Human Annotation},
  author       = {Aroyo, Lora and Welty, Chris},
  year         = 2015,
  journal      = {AI Magazine},
  volume       = 36,
  number       = 1,
  pages        = {15--24}
}
@InProceedings{poesio-et-al:NAACL19,
  title        = {A Crowdsourced Corpus of Multiple Judgments and Disagreement on Anaphoric Interpretation},
  author       = {Poesio, Massimo and Chamberlain, Jon  and Paun, Silviu  and Yu, Juntao  and Uma, Alexandra  and Kruschwitz, Udo},
  year         = 2019,
  booktitle    = {Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},
  publisher    = {Association for Computational Linguistics},
  address      = {Minneapolis},
  pages        = {1778--1789},
  DOI          = {10.18653/v1/N19-1176},
  URL          = {https://aclanthology.org/N19-1176/}
}
@InProceedings{sandri-etal-2023-dont,
  title        = {Why Don{'}t You Do It Right? Analysing Annotators' Disagreement in Subjective Tasks},
  author       = {Sandri, Marta  and Leonardelli, Elisa  and Tonelli, Sara  and Jezek, Elisabetta},
  year         = 2023,
  month        = may,
  booktitle    = {Proceedings of the 17th Conference of the European Chapter of the Association for Computational Linguistics},
  publisher    = {Association for Computational Linguistics},
  address      = {Dubrovnik, Croatia},
  pages        = {2428--2441},
  doi          = {10.18653/v1/2023.eacl-main.178},
  URL          = {https://aclanthology.org/2023.eacl-main.178/},
  editor       = {Vlachos, Andreas  and Augenstein, Isabelle},
  abstract     = {Annotators' disagreement in linguistic data has been recently the focus of multiple initiatives aimed at raising awareness on issues related to `majority voting' when aggregating diverging annotations. Disagreement can indeed reflect different aspects of linguistic annotation, from annotators' subjectivity to sloppiness or lack of enough context to interpret a text. In this work we first propose a taxonomy of possible reasons leading to annotators' disagreement in subjective tasks. Then, we manually label part of a Twitter dataset for offensive language detection in English following this taxonomy, identifying how the different categories are distributed. Finally we run a set of experiments aimed at assessing the impact of the different types of disagreement on classification performance. In particular, we investigate how accurately tweets belonging to different categories of disagreement can be classified as offensive or not, and how injecting data with different types of disagreement in the training set affects performance. We also perform offensive language detection as a multi-task framework, using disagreement classification as an auxiliary task.}
}
@InProceedings{jiang-etal-2023-ecologically,
  title        = {Ecologically Valid Explanations for Label Variation in {NLI}},
  author       = {Jiang, Nan-Jiang and Tan, Chenhao and de Marneffe, Marie-Catherine},
  year         = 2023,
  month        = dec,
  booktitle    = {Findings of the Association for Computational Linguistics: EMNLP 2023},
  publisher    = {Association for Computational Linguistics},
  address      = {Singapore},
  pages        = {10622--10633},
  doi          = {10.18653/v1/2023.findings-emnlp.712},
  URL          = {https://aclanthology.org/2023.findings-emnlp.712/},
  editor       = {Bouamor, Houda and Pino, Juan and Bali, Kalika}
}
@Article{jiang-marneffe-2022-investigating,
  title        = {Investigating Reasons for Disagreement in Natural Language Inference},
  author       = {Jiang, Nan-Jiang and de Marneffe, Marie-Catherine},
  year         = 2022,
  journal      = {Transactions of the Association for Computational Linguistics},
  publisher    = {MIT Press},
  address      = {Cambridge, MA},
  volume       = 10,
  pages        = {1357--1374},
  doi          = {10.1162/tacl_a_00523},
  URL          = {https://aclanthology.org/2022.tacl-1.78/},
  editor       = {Roark, Brian and Nenkova, Ani}
}
@InProceedings{weber2024-varierrnli,
  title        = {{V}ari{E}rr {NLI}: Separating Annotation Error from Human Label Variation},
  author       = {Weber-Genzel, Leon  and Peng, Siyao  and de Marneffe, Marie-Catherine  and Plank, Barbara},
  year         = 2024,
  month        = aug,
  booktitle    = {Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  publisher    = {Association for Computational Linguistics},
  address      = {Bangkok, Thailand},
  pages        = {2256--2269},
  doi          = {10.18653/v1/2024.acl-long.123},
  URL          = {https://aclanthology.org/2024.acl-long.123/},
  editor       = {Ku, Lun-Wei  and Martins, Andre  and Srikumar, Vivek}
}
@Article{liu2019roberta,
  title        = {Roberta: A robustly optimized bert pretraining approach},
  author       = {Liu, Yinhan and Ott, Myle and Goyal, Naman and Du, Jingfei and Joshi, Mandar and Chen, Danqi and Levy, Omer and Lewis, Mike and Zettlemoyer, Luke and Stoyanov, Veselin},
  year         = 2019,
  journal      = {arXiv preprint arXiv:1907.11692}
}
# only archival version
@InProceedings{fleisig-etal-2024-perspectivist,
  title        = {The Perspectivist Paradigm Shift: Assumptions and Challenges of Capturing Human Labels},
  author       = {Fleisig, Eve  and Blodgett, Su Lin  and Klein, Dan  and Talat, Zeerak},
  year         = 2024,
  month        = jun,
  booktitle    = {Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)},
  publisher    = {Association for Computational Linguistics},
  address      = {Mexico City, Mexico},
  pages        = {2279--2292},
  doi          = {10.18653/v1/2024.naacl-long.126},
  URL          = {https://aclanthology.org/2024.naacl-long.126/},
  editor       = {Duh, Kevin  and Gomez, Helena  and Bethard, Steven},
  abstract     = {Longstanding data labeling practices in machine learning involve collecting and aggregating labels from multiple annotators. But what should we do when annotators disagree? Though annotator disagreement has long been seen as a problem to minimize, new perspectivist approaches challenge this assumption by treating disagreement as a valuable source of information. In this position paper, we examine practices and assumptions surrounding the causes of disagreement{--}some challenged by perspectivist approaches, and some that remain to be addressed{--}as well as practical and normative challenges for work operating under these assumptions. We conclude with recommendations for the data labeling pipeline and avenues for future research engaging with subjectivity and disagreement.}
}

@InProceedings{prabhakaran2021releasing,
    title = "On Releasing Annotator-Level Labels and Information in Datasets",
    author = "Prabhakaran, Vinodkumar  and
      Mostafazadeh Davani, Aida  and
      D{\'i}az, Mark",
    editor = "Bonial, Claire  and
      Xue, Nianwen",
    booktitle = "Proceedings of the Joint 15th Linguistic Annotation Workshop (LAW) and 3rd Designing Meaning Representations (DMR) Workshop",
    month = nov,
    year = "2021",
    address = "Punta Cana, Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.law-1.14/",
    doi = "10.18653/v1/2021.law-1.14",
    pages = "133--138",
    abstract = "A common practice in building NLP datasets, especially using crowd-sourced annotations, involves obtaining multiple annotator judgements on the same data instances, which are then flattened to produce a single ``ground truth'' label or score, through majority voting, averaging, or adjudication. While these approaches may be appropriate in certain annotation tasks, such aggregations overlook the socially constructed nature of human perceptions that annotations for relatively more subjective tasks are meant to capture. In particular, systematic disagreements between annotators owing to their socio-cultural backgrounds and/or lived experiences are often obfuscated through such aggregations. In this paper, we empirically demonstrate that label aggregation may introduce representational biases of individual and group perspectives. Based on this finding, we propose a set of recommendations for increased utility and transparency of datasets for downstream use cases."
}

@Article{garten2019incorporating,
  title        = {Incorporating demographic embeddings into language understanding},
  author       = {Garten, Justin and Kennedy, Brendan and Hoover, Joe and Sagae, Kenji and Dehghani, Morteza},
  year         = 2019,
  journal      = {Cognitive science},
  publisher    = {Wiley Online Library},
  volume       = 43,
  number       = 1,
  pages        = {e12701}
}
@InProceedings{diaz2018addressing,
  title        = {Addressing age-related bias in sentiment analysis},
  author       = {D{\'\i}az, Mark and Johnson, Isaac and Lazar, Amanda and Piper, Anne Marie and Gergle, Darren},
  year         = 2018,
  booktitle    = {Proceedings of the 2018 chi conference on human factors in computing systems},
  pages        = {1--14}
}
@InProceedings{basile2020s,
  title        = {It’s the end of the gold standard as we know it. on the impact of pre-aggregation on the evaluation of highly subjective tasks},
  author       = {Basile, Valerio},
  year         = 2020,
  booktitle    = {2020 AIxIA Discussion Papers Workshop, AIxIA 2020 DP},
  volume       = 2776,
  pages        = {31--40},
  organization = {CEUR-WS}
}
@Article{davani2023hate,
  title        = {Hate Speech Classifiers Learn Normative Social Stereotypes},
  author       = {Davani, Aida Mostafazadeh and Atari, Mohammad and Kennedy, Brendan and Dehghani, Morteza},
  year         = 2023,
  journal      = {Transactions of the Association for Computational Linguistics},
  publisher    = {MIT Press},
  address      = {Cambridge, MA},
  volume       = 11,
  pages        = {300--319},
  url = {https://aclanthology.org/2023.tacl-1.18/},
  doi = {10.1162/tacl_a_00550}
}
@InProceedings{sang2022origin,
  title        = {The Origin and Value of Disagreement Among Data Labelers: A Case Study of Individual Differences in Hate Speech Annotation},
  author       = {Sang, Yisi and Stanton, Jeffrey},
  year         = 2022,
  booktitle    = {International Conference on Information},
  pages        = {425--444},
  organization = {Springer}
}
@Article{Cabitza_Campagner_Basile_2023,
  title        = {Toward a Perspectivist Turn in Ground Truthing for Predictive Computing},
  author       = {Cabitza, Federico and Campagner, Andrea and Basile, Valerio},
  year         = 2023,
  month        = {Jun.},
  journal      = {Proceedings of the AAAI Conference on Artificial Intelligence},
  volume       = 37,
  number       = 6,
  pages        = {6860--6868},
  DOI          = {10.1609/aaai.v37i6.25840},
  url          = {https://ojs.aaai.org/index.php/AAAI/article/view/25840},
  abstractNote = {Most current Artificial Intelligence applications are based on supervised Machine Learning (ML), which ultimately grounds on data annotated by small teams of experts or large ensemble of volunteers. The annotation process is often performed in terms of a majority vote, however this has been proved to be often problematic by recent evaluation studies. In this article, we describe and advocate for a different paradigm, which we call perspectivism: this counters the removal of disagreement and, consequently, the assumption of correctness of traditionally aggregated gold-standard datasets, and proposes the adoption of methods that preserve divergence of opinions and integrate multiple perspectives in the ground truthing process of ML development. Drawing on previous works which inspired it, mainly from the crowdsourcing and multi-rater labeling settings, we survey the state-of-the-art and describe the potential of our proposal for not only the more subjective tasks (e.g. those related to human language) but also those tasks commonly understood as objective (e.g. medical decision making). We present the main benefits of adopting a perspectivist stance in ML, as well as possible disadvantages, and various ways in which such a stance can be implemented in practice. Finally, we share a set of recommendations and outline a research agenda to advance the perspectivist stance in ML.}
}
@InProceedings{larimore-etal-2021-reconsidering,
  title        = {Reconsidering Annotator Disagreement about Racist Language: Noise or Signal?},
  author       = {Larimore, Savannah and Kennedy, Ian and Haskett, Breon and Arseniev-Koehler, Alina},
  year         = 2021,
  month        = jun,
  booktitle    = {Proceedings of the Ninth International Workshop on Natural Language Processing for Social Media},
  publisher    = {Association for Computational Linguistics},
  address      = {Online},
  pages        = {81--90},
  doi          = {10.18653/v1/2021.socialnlp-1.7},
  URL          = {https://aclanthology.org/2021.socialnlp-1.7/},
  editor       = {Ku, Lun-Wei and Li, Cheng-Te}
}
@Article{pavlick2019inherent,
  title        = {Inherent disagreements in human textual inferences},
  author       = {Pavlick, Ellie and Kwiatkowski, Tom},
  year         = 2019,
  journal      = {Transactions of the Association for Computational Linguistics},
  publisher    = {MIT Press},
  address      = {Cambridge, MA},
  volume       = 7,
  pages        = {677--694},
  url = {https://aclanthology.org/Q19-1043/},
  doi = {10.1162/tacl_a_00293},
}
@Article{frenda2024perspectivist,
  title        = {Perspectivist approaches to natural language processing: a survey},
  author       = {Frenda, Simona and Abercrombie, Gavin and Basile, Valerio and Pedrani, Alessandro and Panizzon, Raffaella and Cignarella, Alessandra Teresa and Marco, Cristina and Bernardi, Davide},
  year         = 2024,
  journal      = {Language Resources and Evaluation},
  publisher    = {Springer},
  pages        = {1--28}
}

@InProceedings{plank2014linguistically,
    title = "Linguistically debatable or just plain wrong?",
    author = "Plank, Barbara  and
      Hovy, Dirk  and
      S{\o}gaard, Anders",
    editor = "Toutanova, Kristina  and
      Wu, Hua",
    booktitle = "Proceedings of the 52nd Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)",
    month = jun,
    year = "2014",
    address = "Baltimore, Maryland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/P14-2083/",
    doi = "10.3115/v1/P14-2083",
    pages = "507--511"
}


@InProceedings{sap2022annotators,
    title = "Annotators with Attitudes: How Annotator Beliefs And Identities Bias Toxic Language Detection",
    author = "Sap, Maarten  and
      Swayamdipta, Swabha  and
      Vianna, Laura  and
      Zhou, Xuhui  and
      Choi, Yejin  and
      Smith, Noah A.",
    editor = "Carpuat, Marine  and
      de Marneffe, Marie-Catherine  and
      Meza Ruiz, Ivan Vladimir",
    booktitle = "Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",
    month = jul,
    year = "2022",
    address = "Seattle, United States",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.naacl-main.431/",
    doi = "10.18653/v1/2022.naacl-main.431",
    pages = "5884--5906",
    abstract = "The perceived toxicity of language can vary based on someone{'}s identity and beliefs, but this variation is often ignored when collecting toxic language datasets, resulting in dataset and model biases. We seek to understand the *who*, *why*, and *what* behind biases in toxicity annotations. In two online studies with demographically and politically diverse participants, we investigate the effect of annotator identities (*who*) and beliefs (*why*), drawing from social psychology research about hate speech, free speech, racist beliefs, political leaning, and more. We disentangle *what* is annotated as toxic by considering posts with three characteristics: anti-Black language, African American English (AAE) dialect, and vulgarity. Our results show strong associations between annotator identity and beliefs and their ratings of toxicity. Notably, more conservative annotators and those who scored highly on our scale for racist beliefs were less likely to rate anti-Black language as toxic, but more likely to rate AAE as toxic. We additionally present a case study illustrating how a popular toxicity detection system{'}s ratings inherently reflect only specific beliefs and perspectives. Our findings call for contextualizing toxicity labels in social variables, which raises immense implications for toxic language annotation and detection."
}
@InProceedings{akhtar2019new,
  title        = {A new measure of polarization in the annotation of hate speech},
  author       = {Akhtar, Sohail and Basile, Valerio and Patti, Viviana},
  year         = 2019,
  booktitle    = {International Conference of the Italian Association for Artificial Intelligence},
  pages        = {588--603},
  organization = {Springer}
}
% OTHER
@InProceedings{10.1145/3627673.3679584,
  title        = {Learning to Differentiate Pairwise-Argument Representations for Implicit Discourse Relation Recognition},
  author       = {Wang, Zhipang and Hong, Yu and Lu, Yuxiang and Zhou, Xiabing and Yao, Jianmin and Zhou, Guodong},
  year         = 2024,
  booktitle    = {Proceedings of the 33rd ACM International Conference on Information and Knowledge Management},
  location     = {Boise, ID, USA},
  publisher    = {Association for Computing Machinery},
  address      = {New York, NY, USA},
  series       = {CIKM '24},
  pages        = {2503--2512},
  doi          = {10.1145/3627673.3679584},
  isbn         = 9798400704369,
  URL          = {https://doi.org/10.1145/3627673.3679584},
  abstract     = {Recognizing implicit discourse relations between texts is challenging due to the absence of explicit connectives. Encoding texts into distinguishable semantic representations is beneficial to connective-free relation determination. To enable encoders to produce clearly distinguishable representations, we propose a joint learning framework. It combines prototypical and adversarial learning as well as hub-migration based redistribution. We experiment on PDTB 2.0, PDTB 3.0 and the CoNLL-2016 shared benchmark dataset. Experimental results show that our methods yield substantial improvements, compared to BERT, RoBERTa and DeBERTa baselines. In the separate comparison experiment where implicit connectives are disable during training, our models outperform the previous work for the four main discourse relation types (Temporality, Comparison, Contingency and Expansion), achieving F1-scores of about 60.75\%, 63.48\%, 75.70\% and 77.09\%. On the other hand, our models obtain comparable performance compared to part of the state-of-the-art models which adopt implicit connectives as observable hints for training. When the pretrained language models are used as the backbones, our methods yield the extra time consumption, which is about 1 hour at worst when training is conducted on Tesla 40GB A100 GPU.},
  numpages     = 10,
  keywords     = {adversarial learning, distinguishable representation, hub-migration based redistribution, implicit discourse relation recognition, joint learning framework, prototypical learning}
}
@InProceedings{deng-etal-2023-annotate,
  title        = {You Are What You Annotate: Towards Better Models through Annotator Representations},
  author       = {Deng, Naihao  and Zhang, Xinliang  and Liu, Siyang  and Wu, Winston  and Wang, Lu  and Mihalcea, Rada},
  year         = 2023,
  month        = dec,
  booktitle    = {Findings of the Association for Computational Linguistics: EMNLP 2023},
  publisher    = {Association for Computational Linguistics},
  address      = {Singapore},
  pages        = {12475--12498},
  doi          = {10.18653/v1/2023.findings-emnlp.832},
  URL          = {https://aclanthology.org/2023.findings-emnlp.832},
  editor       = {Bouamor, Houda  and Pino, Juan  and Bali, Kalika},
  abstract     = {Annotator disagreement is ubiquitous in natural language processing (NLP) tasks. There are multiple reasons for such disagreements, including the subjectivity of the task, difficult cases, unclear guidelines, and so on. Rather than simply aggregating labels to obtain data annotations, we instead try to directly model the diverse perspectives of the annotators, and explicitly account for annotators{'} idiosyncrasies in the modeling process by creating representations for each annotator (*annotator embeddings*) and also their annotations (*annotation embeddings*). In addition, we propose **TID-8**, **T**he **I**nherent **D**isagreement - **8** dataset, a benchmark that consists of eight existing language understanding datasets that have inherent annotator disagreement. We test our approach on TID-8 and show that our approach helps models learn significantly better from disagreements on six different datasets in TID-8 while increasing model size by fewer than 1{\%} parameters. By capturing the unique tendencies and subjectivity of individual annotators through embeddings, our representations prime AI models to be inclusive of diverse viewpoints.}
}
@Article{sanders1992toward,
  title        = {Toward a taxonomy of coherence relations},
  author       = {Sanders, Ted JM and Spooren, Wilbert PM and Noordman, Leo GM},
  year         = 1992,
  journal      = {Discourse processes},
  publisher    = {Taylor \& Francis},
  volume       = 15,
  number       = 1,
  pages        = {1--35}
}
@InCollection{prasad2017penn,
  title        = {The Penn Discourse Treebank: An annotated corpus of discourse relations},
  author       = {Prasad, Rashmi and Webber, Bonnie and Joshi, Aravind},
  year         = 2017,
  booktitle    = {Handbook of linguistic annotation},
  publisher    = {Springer},
  pages        = {1197--1217}
}
@Article{ignatev2025annotator,
  title        = {Annotator disagreement in RST annotation schemes},
  author       = {Ignatev, Daniil and Paperno, Denis and Poesio, Massimo},
  year         = 2025,
  journal      = {Society for Computation in Linguistics},
  publisher    = {University of Massachusetts Amherst Libraries},
  volume       = 8,
  number       = 1,
  URL          = {https://openpublishing.library.umass.edu/scil/article/id/3137/},
  doi          = {https://doi.org/10.7275/scil.3137}
}
@InProceedings{zikanova2024text,
  title        = {Text structure and its ambiguities: corpus annotation as a helpful guide},
  author       = {Zik{\'a}nov{\'a}, {\v{S}}{\'a}rka},
  year         = 2024,
  booktitle    = {ITAT},
  pages        = {2--12}
}
@InProceedings{liu2023whats,
  title        = {What’s Hard in English RST Parsing? Predictive Models for Error Analysis},
  author       = {Liu, Yang Janet and Aoyama, Tatsuya and Zeldes, Amir},
  year         = 2023,
  booktitle    = {Proceedings of the 24th Annual Meeting of the Special Interest Group on Discourse and Dialogue},
  pages        = {31--42}
}
@Article{mann1988rhetorical,
  title        = {Rhetorical structure theory: Toward a functional theory of text organization},
  author       = {Mann, William C and Thompson, Sandra A},
  year         = 1988,
  journal      = {Text-interdisciplinary Journal for the Study of Discourse},
  publisher    = {Walter de Gruyter, Berlin/New York Berlin, New York},
  volume       = 8,
  number       = 3,
  pages        = {243--281}
}
@InProceedings{mokhberian-etal-2024-capturing,
  title        = {Capturing Perspectives of Crowdsourced Annotators in Subjective Learning Tasks},
  author       = {Mokhberian, Negar  and Marmarelis, Myrl  and Hopp, Frederic  and Basile, Valerio  and Morstatter, Fred  and Lerman, Kristina},
  year         = 2024,
  month        = jun,
  booktitle    = {Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)},
  publisher    = {Association for Computational Linguistics},
  address      = {Mexico City, Mexico},
  pages        = {7337--7349},
  doi          = {10.18653/v1/2024.naacl-long.407},
  URL          = {https://aclanthology.org/2024.naacl-long.407/}
}
@InProceedings{harju2023evaluating,
  title        = {Evaluating Classification Systems Against Soft Labels with Fuzzy Precision and Recall},
  author       = {Harju, Manu and Mesaros, Annamaria},
  year         = 2023,
  booktitle    = {Workshop on Detection and Classification of Acoustic Scenes and Events},
  pages        = {46--50},
  organization = {Tampere University}
}
@InProceedings{LeWiDi2025,
  title        = {LeWiDi-2025 at NLPerspectives: third Edition of the Learning with Disagreements Shared Task},
  author       = {Leonardelli, Elisa and Casola, Silvia and Peng, Siyao and Rizzi, Giulia and Basile, Valerio and Fersini, Elisabetta and Frassinelli, Diego and Jang, Hyewon and Pavlovic, Maja and Plank, Barbara and Poesio, Massimo},
  year         = 2025,
  month        = {nov},
  booktitle    = {Proceedings of the 4th Workshop on Perspectivist Approaches to NLP (NLPerspectives)},
  publisher    = {Association for Computational Linguistics}
}

@InProceedings{fornaciari2021beyond,
    title = "Beyond Black {\&} White: Leveraging Annotator Disagreement via Soft-Label Multi-Task Learning",
    author = "Fornaciari, Tommaso  and
      Uma, Alexandra  and
      Paun, Silviu  and
      Plank, Barbara  and
      Hovy, Dirk  and
      Poesio, Massimo",
    editor = "Toutanova, Kristina  and
      Rumshisky, Anna  and
      Zettlemoyer, Luke  and
      Hakkani-Tur, Dilek  and
      Beltagy, Iz  and
      Bethard, Steven  and
      Cotterell, Ryan  and
      Chakraborty, Tanmoy  and
      Zhou, Yichao",
    booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",
    month = jun,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.naacl-main.204/",
    doi = "10.18653/v1/2021.naacl-main.204",
    pages = "2591--2597",
    abstract = "Supervised learning assumes that a ground truth label exists. However, the reliability of this ground truth depends on human annotators, who often disagree. Prior work has shown that this disagreement can be helpful in training models. We propose a novel method to incorporate this disagreement as information: in addition to the standard error computation, we use soft-labels (i.e., probability distributions over the annotator labels) as an auxiliary task in a multi-task neural network. We measure the divergence between the predictions and the target soft-labels with several loss-functions and evaluate the models on various NLP tasks. We find that the soft-label prediction auxiliary task reduces the penalty for errors on ambiguous entities, and thereby mitigates overfitting. It significantly improves performance across tasks, beyond the standard approach and prior work."
}

@InProceedings{rizzi2024soft,
  title        = {Soft metrics for evaluation with disagreements: an assessment},
  author       = {Rizzi, Giulia and Leonardelli, Elisa and Poesio, Massimo and Uma, Alexandra and Pavlovic, Maja and Paun, Silviu and Rosso, Paolo and Fersini, Elisabetta},
  year         = 2024,
  booktitle    = {3rd Workshop on Perspectivist Approaches to NLP, NLPerspectives 2024},
  pages        = {84--94},
  organization = {European Language Resources Association (ELRA)}
}
@InProceedings{weber-genzel-etal-2024-varierr,
  title        = {{V}ari{E}rr {NLI}: Separating Annotation Error from Human Label Variation},
  author       = {Weber-Genzel, Leon  and Peng, Siyao  and de Marneffe, Marie-Catherine  and Plank, Barbara},
  year         = 2024,
  month        = aug,
  booktitle    = {Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  publisher    = {Association for Computational Linguistics},
  address      = {Bangkok, Thailand},
  pages        = {2256--2269},
  doi          = {10.18653/v1/2024.acl-long.123},
  URL          = {https://aclanthology.org/2024.acl-long.123/},
  editor       = {Ku, Lun-Wei  and Martins, Andre  and Srikumar, Vivek},
  abstract     = {Human label variation arises when annotators assign different labels to the same item for valid reasons, while annotation errors occur when labels are assigned for invalid reasons. These two issues are prevalent in NLP benchmarks, yet existing research has studied them in isolation. To the best of our knowledge, there exists no prior work that focuses on teasing apart error from signal, especially in cases where signal is beyond black-and-white.To fill this gap, we introduce a systematic methodology and a new dataset, VariErr (variation versus error), focusing on the NLI task in English. We propose a 2-round annotation procedure with annotators explaining each label and subsequently judging the validity of label-explanation pairs.VariErr contains 7,732 validity judgments on 1,933 explanations for 500 re-annotated MNLI items. We assess the effectiveness of various automatic error detection (AED) methods and GPTs in uncovering errors versus human label variation. We find that state-of-the-art AED methods significantly underperform GPTs and humans. While GPT-4 is the best system, it still falls short of human performance. Our methodology is applicable beyond NLI, offering fertile ground for future research on error versus plausible variation, which in turn can yield better and more trustworthy NLP systems.}
}
@InProceedings{costa-kosseim-2024-exploring,
  title        = {Exploring Soft-Label Training for Implicit Discourse Relation Recognition},
  author       = {Costa, Nelson Filipe  and Kosseim, Leila},
  year         = 2024,
  month        = mar,
  booktitle    = {Proceedings of the 5th Workshop on Computational Approaches to Discourse (CODI 2024)},
  publisher    = {Association for Computational Linguistics},
  address      = {St. Julians, Malta},
  pages        = {120--126},
  URL          = {https://aclanthology.org/2024.codi-1.11/},
  editor       = {Strube, Michael  and Braud, Chlo{\'e}  and Hardmeier, Christian  and Li, Junyi Jessy  and Loaiciga, Sharid  and Zeldes, Amir  and Li, Chuyuan},
  abstract     = {This paper proposes a classification model for single label implicit discourse relation recognition trained on soft-label distributions. It follows the PDTB 3.0 framework and it was trained and tested on the DiscoGeM corpus, where it achieves an F1-score of 51.38 on third-level sense classification of implicit discourse relations. We argue that training on soft-label distributions allows the model to better discern between more ambiguous discourse relations.}
}
@InProceedings{he2018joint,
  title        = {Joint binary neural network for multi-label learning with applications to emotion classification},
  author       = {He, Huihui and Xia, Rui},
  year         = 2018,
  booktitle    = {CCF International Conference on Natural Language Processing and Chinese Computing},
  pages        = {250--259},
  organization = {Springer}
}
@Article{uma2021learning,
  title        = {Learning from disagreement: A survey},
  author       = {Uma, Alexandra and Fornaciari, Tommaso and Hovy, Dirk and Paun, Silviu and Plank, Barbara and Poesio, Massimo},
  year         = 2021,
  journal      = {Journal of Artificial Intelligence Research},
  volume       = 72,
  pages        = {1385--1470}
}
@Article{xiang2023survey,
  title        = {A survey of implicit discourse relation recognition},
  author       = {Xiang, Wei and Wang, Bang},
  year         = 2023,
  journal      = {ACM Computing Surveys},
  publisher    = {ACM New York, NY},
  volume       = 55,
  number       = 12,
  pages        = {1--34}
}

@InProceedings{wang-etal-2025-using,
  title        = {Using Subtext to Enhance Generative {IDRR}},
  author       = {Wang, Zhipang  and Hong, Yu  and Sun, Weihao  and Zhou, Guodong},
  year         = 2025,
  month        = jul,
  booktitle    = {Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)},
  publisher    = {Association for Computational Linguistics},
  address      = {Vienna, Austria},
  pages        = {464--473},
  doi          = {10.18653/v1/2025.acl-short.35},
  ISBN         = {979-8-89176-252-7},
  URL          = {https://aclanthology.org/2025.acl-short.35/},
  editor       = {Che, Wanxiang  and Nabende, Joyce  and Shutova, Ekaterina  and Pilehvar, Mohammad Taher},
  abstract     = {Implicit Discourse Relation Recognition (abbr., IDRR) is a NLP task of classifying argument pairs into different types of semantic relations. Arguments contain subtexts, some of which are beneficial to the perception of semantic relations. However, subtexts are connotative. The neural IDRR model fails to be aware of them without being given pertinent prompts. In this paper, we leverage LLaMA to generate subtexts for argument pairs, and verify the effectiveness of subtext-based IDRR. We construct an IDRR baseline using the decoder-only backbone LLaMA, and enhance it with subtext-aware relation reasoning. A confidence-diagnosed dual-channel network is used for collaboration between in-subtext and out-of-subtext IDRR. We experiment on PDTB-2.0 and PDTB-3.0 for both the main-level and secondary-level relation taxonomies. The test results show that our approach yields substantial improvements compared to the baseline, and achieves higher $F$1-scores on both benchmarks than the previous decoder-only IDRR models. We make the source codes and data publicly available.}
}
@Article{zeldes2025erst,
  title        = {e{RST}: A Signaled Graph Theory of Discourse Relations and Organization},
  author       = {Zeldes, Amir and Aoyama, Tatsuya and Liu, Yang Janet and Peng, Siyao and Das, Debopam and Gessler, Luke},
  year         = 2025,
  journal      = {Computational Linguistics},
  publisher    = {MIT Press},
  address      = {Cambridge, MA},
  volume       = 51,
  number       = 1,
  pages        = {23--72}
}

@article{zhang2024diverging,
  title={Diverging Preferences: When do Annotators Disagree and do Models Know?},
  author={Zhang, Michael JQ and Wang, Zhilin and Hwang, Jena D and Dong, Yi and Delalleau, Olivier and Choi, Yejin and Choi, Eunsol and Ren, Xiang and Pyatkin, Valentina},
  journal={arXiv preprint arXiv:2410.14632},
  year={2024}
}

@inproceedings{mccoy_right_2019,
	address = {Florence, Italy},
	title = {Right for the {Wrong} {Reasons}: {Diagnosing} {Syntactic} {Heuristics} in {Natural} {Language} {Inference}},
	shorttitle = {Right for the {Wrong} {Reasons}},
	url = {https://aclanthology.org/P19-1334/},
	doi = {10.18653/v1/P19-1334},
	abstract = {A machine learning system can score well on a given test set by relying on heuristics that are effective for frequent example types but break down in more challenging cases. We study this issue within natural language inference (NLI), the task of determining whether one sentence entails another. We hypothesize that statistical NLI models may adopt three fallible syntactic heuristics: the lexical overlap heuristic, the subsequence heuristic, and the constituent heuristic. To determine whether models have adopted these heuristics, we introduce a controlled evaluation set called HANS (Heuristic Analysis for NLI Systems), which contains many examples where the heuristics fail. We find that models trained on MNLI, including BERT, a state-of-the-art model, perform very poorly on HANS, suggesting that they have indeed adopted these heuristics. We conclude that there is substantial room for improvement in NLI systems, and that the HANS dataset can motivate and measure progress in this area.},
	urldate = {2026-03-10},
	booktitle = {Proceedings of the 57th {Annual} {Meeting} of the {Association} for {Computational} {Linguistics}},
	publisher = {Association for Computational Linguistics},
	author = {McCoy, R. Thomas and Pavlick, Ellie and Linzen, Tal},
	editor = {Korhonen, Anna and Traum, David and Màrquez, Lluís},
	month = jul,
	year = {2019},
	pages = {3428--3448},
	file = {Full Text PDF:/home/daniiligantev/Zotero/storage/Q6A99GKB/McCoy et al. - 2019 - Right for the Wrong Reasons Diagnosing Syntactic Heuristics in Natural Language Inference.pdf:application/pdf},
}

@inproceedings{feng_misleading_2019,
	address = {Florence, Italy},
	title = {Misleading {Failures} of {Partial}-input {Baselines}},
	url = {https://aclanthology.org/P19-1554/},
	doi = {10.18653/v1/P19-1554},
	abstract = {Recent work establishes dataset difficulty and removes annotation artifacts via partial-input baselines (e.g., hypothesis-only model for SNLI or question-only model for VQA). A successful partial-input baseline indicates that the dataset is cheatable. But the converse is not necessarily true: failures of partial-input baselines do not mean the dataset is free of artifacts. We first design artificial datasets to illustrate how the trivial patterns that are only visible in the full input can evade any partial-input baseline. Next, we identify such artifacts in the SNLI dataset—a hypothesis-only model augmented with trivial patterns in the premise can solve 15\% of previously-thought “hard” examples. Our work provides a caveat for the use and creation of partial-input baselines for datasets.},
	urldate = {2026-03-10},
	booktitle = {Proceedings of the 57th {Annual} {Meeting} of the {Association} for {Computational} {Linguistics}},
	publisher = {Association for Computational Linguistics},
	author = {Feng, Shi and Wallace, Eric and Boyd-Graber, Jordan},
	editor = {Korhonen, Anna and Traum, David and Màrquez, Lluís},
	month = jul,
	year = {2019},
	pages = {5533--5538},
	file = {Full Text PDF:/home/daniiligantev/Zotero/storage/Q6QFGPCW/Feng et al. - 2019 - Misleading Failures of Partial-input Baselines.pdf:application/pdf},
}

@inproceedings{gururangan_annotation_2018,
	address = {New Orleans, Louisiana},
	title = {Annotation {Artifacts} in {Natural} {Language} {Inference} {Data}},
	url = {https://aclanthology.org/N18-2017/},
	doi = {10.18653/v1/N18-2017},
	abstract = {Large-scale datasets for natural language inference are created by presenting crowd workers with a sentence (premise), and asking them to generate three new sentences (hypotheses) that it entails, contradicts, or is logically neutral with respect to. We show that, in a significant portion of such data, this protocol leaves clues that make it possible to identify the label by looking only at the hypothesis, without observing the premise. Specifically, we show that a simple text categorization model can correctly classify the hypothesis alone in about 67\% of SNLI (Bowman et. al, 2015) and 53\% of MultiNLI (Williams et. al, 2017). Our analysis reveals that specific linguistic phenomena such as negation and vagueness are highly correlated with certain inference classes. Our findings suggest that the success of natural language inference models to date has been overestimated, and that the task remains a hard open problem.},
	urldate = {2026-03-10},
	booktitle = {Proceedings of the 2018 {Conference} of the {North} {American} {Chapter} of the {Association} for {Computational} {Linguistics}: {Human} {Language} {Technologies}, {Volume} 2 ({Short} {Papers})},
	publisher = {Association for Computational Linguistics},
	author = {Gururangan, Suchin and Swayamdipta, Swabha and Levy, Omer and Schwartz, Roy and Bowman, Samuel and Smith, Noah A.},
	editor = {Walker, Marilyn and Ji, Heng and Stent, Amanda},
	month = jun,
	year = {2018},
	pages = {107--112},
	file = {Full Text PDF:/home/daniiligantev/Zotero/storage/96LP3M7W/Gururangan et al. - 2018 - Annotation Artifacts in Natural Language Inference Data.pdf:application/pdf},
}

@misc{anand_dont_2024,
	title = {Don't {Blame} the {Data}, {Blame} the {Model}: {Understanding} {Noise} and {Bias} {When} {Learning} from {Subjective} {Annotations}},
	shorttitle = {Don't {Blame} the {Data}, {Blame} the {Model}},
	url = {http://arxiv.org/abs/2403.04085},
	doi = {10.48550/arXiv.2403.04085},
	abstract = {Researchers have raised awareness about the harms of aggregating labels especially in subjective tasks that naturally contain disagreements among human annotators. In this work we show that models that are only provided aggregated labels show low confidence on high-disagreement data instances. While previous studies consider such instances as mislabeled, we argue that the reason the high-disagreement text instances have been hard-to-learn is that the conventional aggregated models underperform in extracting useful signals from subjective tasks. Inspired by recent studies demonstrating the effectiveness of learning from raw annotations, we investigate classifying using Multiple Ground Truth (Multi-GT) approaches. Our experiments show an improvement of confidence for the high-disagreement instances.},
	urldate = {2026-03-02},
	publisher = {arXiv},
	author = {Anand, Abhishek and Mokhberian, Negar and Kumar, Prathyusha Naresh and Saha, Anweasha and He, Zihao and Rao, Ashwin and Morstatter, Fred and Lerman, Kristina},
	month = mar,
	year = {2024},
	note = {arXiv:2403.04085 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Computers and Society},
	file = {Preprint PDF:/home/daniiligantev/Zotero/storage/LLH7CXG7/Anand et al. - 2024 - Don't Blame the Data, Blame the Model Understanding Noise and Bias When Learning from Subjective An.pdf:application/pdf;Snapshot:/home/daniiligantev/Zotero/storage/7CQQMQ28/2403.html:text/html},
}

@inproceedings{swayamdipta_dataset_2020,
	address = {Online},
	title = {Dataset {Cartography}: {Mapping} and {Diagnosing} {Datasets} with {Training} {Dynamics}},
	shorttitle = {Dataset {Cartography}},
	url = {https://aclanthology.org/2020.emnlp-main.746/},
	doi = {10.18653/v1/2020.emnlp-main.746},
	abstract = {Large datasets have become commonplace in NLP research. However, the increased emphasis on data quantity has made it challenging to assess the quality of data. We introduce Data Maps—a model-based tool to characterize and diagnose datasets. We leverage a largely ignored source of information: the behavior of the model on individual instances during training (training dynamics) for building data maps. This yields two intuitive measures for each example—the model's confidence in the true class, and the variability of this confidence across epochs—obtained in a single run of training. Experiments on four datasets show that these model-dependent measures reveal three distinct regions in the data map, each with pronounced characteristics. First, our data maps show the presence of “ambiguous” regions with respect to the model, which contribute the most towards out-of-distribution generalization. Second, the most populous regions in the data are “easy to learn” for the model, and play an important role in model optimization. Finally, data maps uncover a region with instances that the model finds “hard to learn”; these often correspond to labeling errors. Our results indicate that a shift in focus from quantity to quality of data could lead to robust models and improved out-of-distribution generalization.},
	urldate = {2026-03-10},
	booktitle = {Proceedings of the 2020 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing} ({EMNLP})},
	publisher = {Association for Computational Linguistics},
	author = {Swayamdipta, Swabha and Schwartz, Roy and Lourie, Nicholas and Wang, Yizhong and Hajishirzi, Hannaneh and Smith, Noah A. and Choi, Yejin},
	editor = {Webber, Bonnie and Cohn, Trevor and He, Yulan and Liu, Yang},
	month = nov,
	year = {2020},
	pages = {9275--9293},
	file = {Full Text PDF:/home/daniiligantev/Zotero/storage/JU5NQDPP/Swayamdipta et al. - 2020 - Dataset Cartography Mapping and Diagnosing Datasets with Training Dynamics.pdf:application/pdf},
}

@inproceedings{bowman_large_2015,
	address = {Lisbon, Portugal},
	title = {A large annotated corpus for learning natural language inference},
	url = {https://aclanthology.org/D15-1075/},
	doi = {10.18653/v1/D15-1075},
	urldate = {2026-03-10},
	booktitle = {Proceedings of the 2015 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing}},
	publisher = {Association for Computational Linguistics},
	author = {Bowman, Samuel R. and Angeli, Gabor and Potts, Christopher and Manning, Christopher D.},
	editor = {Màrquez, Lluís and Callison-Burch, Chris and Su, Jian},
	month = sep,
	year = {2015},
	pages = {632--642},
	file = {Full Text PDF:/home/daniiligantev/Zotero/storage/ZTMF9U7Q/Bowman et al. - 2015 - A large annotated corpus for learning natural language inference.pdf:application/pdf},
}

@incollection{zhang2021cartography,
  title={Cartography Active Learning},
  author={Zhang, Mike and Plank, Barbara},
  booktitle={Findings of the Association for Computational Linguistics: EMNLP 2021},
  pages={395--406},
  year={2021},
  publisher={Association for Computational Linguistics}
}

@misc{yung_human_2026,
	title = {Human {Label} {Variation} in {Implicit} {Discourse} {Relation} {Recognition}},
	url = {http://arxiv.org/abs/2602.22723},
	doi = {10.48550/arXiv.2602.22723},
	abstract = {There is growing recognition that many NLP tasks lack a single ground truth, as human judgments reflect diverse perspectives. To capture this variation, models have been developed to predict full annotation distributions rather than majority labels, while perspectivist models aim to reproduce the interpretations of individual annotators. In this work, we compare these approaches on Implicit Discourse Relation Recognition (IDRR), a highly ambiguous task where disagreement often arises from cognitive complexity rather than ideological bias. Our experiments show that existing annotator-specific models perform poorly in IDRR unless ambiguity is reduced, whereas models trained on label distributions yield more stable predictions. Further analysis indicates that frequent cognitively demanding cases drive inconsistency in human interpretation, posing challenges for perspectivist modeling in IDRR.},
	language = {en},
	urldate = {2026-03-10},
	publisher = {arXiv},
	author = {Yung, Frances and Ignatev, Daniil and Scholman, Merel and Demberg, Vera and Poesio, Massimo},
	month = feb,
	year = {2026},
	note = {arXiv:2602.22723 [cs]},
	keywords = {Computer Science - Computation and Language},
	file = {PDF:/home/daniiligantev/Zotero/storage/NEJWW6ZY/Yung et al. - 2026 - Human Label Variation in Implicit Discourse Relation Recognition.pdf:application/pdf},
}

@misc{he_debertav3_2023,
	title = {{DeBERTaV3}: {Improving} {DeBERTa} using {ELECTRA}-{Style} {Pre}-{Training} with {Gradient}-{Disentangled} {Embedding} {Sharing}},
	shorttitle = {{DeBERTaV3}},
	url = {http://arxiv.org/abs/2111.09543},
	doi = {10.48550/arXiv.2111.09543},
	abstract = {This paper presents a new pre-trained language model, DeBERTaV3, which improves the original DeBERTa model by replacing mask language modeling (MLM) with replaced token detection (RTD), a more sample-efficient pre-training task. Our analysis shows that vanilla embedding sharing in ELECTRA hurts training efficiency and model performance. This is because the training losses of the discriminator and the generator pull token embeddings in different directions, creating the "tug-of-war" dynamics. We thus propose a new gradient-disentangled embedding sharing method that avoids the tug-of-war dynamics, improving both training efficiency and the quality of the pre-trained model. We have pre-trained DeBERTaV3 using the same settings as DeBERTa to demonstrate its exceptional performance on a wide range of downstream natural language understanding (NLU) tasks. Taking the GLUE benchmark with eight tasks as an example, the DeBERTaV3 Large model achieves a 91.37\% average score, which is 1.37\% over DeBERTa and 1.91\% over ELECTRA, setting a new state-of-the-art (SOTA) among the models with a similar structure. Furthermore, we have pre-trained a multi-lingual model mDeBERTa and observed a larger improvement over strong baselines compared to English models. For example, the mDeBERTa Base achieves a 79.8\% zero-shot cross-lingual accuracy on XNLI and a 3.6\% improvement over XLM-R Base, creating a new SOTA on this benchmark. We have made our pre-trained models and inference code publicly available at https://github.com/microsoft/DeBERTa.},
	urldate = {2026-03-10},
	publisher = {arXiv},
	author = {He, Pengcheng and Gao, Jianfeng and Chen, Weizhu},
	month = mar,
	year = {2023},
	note = {arXiv:2111.09543 [cs]},
	keywords = {Computer Science - Computation and Language, Computer Science - Machine Learning},
	annote = {Comment: 16 pages, 10 tables, 2 Figures. The DeBERTaV3 model significantly improves performance of the downstream NLU tasks over models with a similar structure, e.g. DeBERTaV3 large achieves 91.37\% average GLUE score which is 1.37\% over DeBERTa large. XSmall has only 22M backbone parameters, but significantly outperforms RoBERTa/XLNet-base. Paper is published as a conference paper at ICLR 2023},
	file = {Preprint PDF:/home/daniiligantev/Zotero/storage/YE67SY9A/He et al. - 2023 - DeBERTaV3 Improving DeBERTa using ELECTRA-Style Pre-Training with Gradient-Disentangled Embedding S.pdf:application/pdf;Snapshot:/home/daniiligantev/Zotero/storage/85HBWNQY/2111.html:text/html},
}
@article{klie_annotation_2023,
	address = {Cambridge, MA},
	title = {Annotation {Error} {Detection}: {Analyzing} the {Past} and {Present} for a {More} {Coherent} {Future}},
	volume = {49},
	shorttitle = {Annotation {Error} {Detection}},
	url = {https://aclanthology.org/2023.cl-1.4/},
	doi = {10.1162/coli_a_00464},
	abstract = {Annotated data is an essential ingredient in natural language processing for training and evaluating machine learning models. It is therefore very desirable for the annotations to be of high quality. Recent work, however, has shown that several popular datasets contain a surprising number of annotation errors or inconsistencies. To alleviate this issue, many methods for annotation error detection have been devised over the years. While researchers show that their approaches work well on their newly introduced datasets, they rarely compare their methods to previous work or on the same datasets. This raises strong concerns on methods' general performance and makes it difficult to assess their strengths and weaknesses. We therefore reimplement 18 methods for detecting potential annotation errors and evaluate them on 9 English datasets for text classification as well as token and span labeling. In addition, we define a uniform evaluation setup including a new formalization of the annotation error detection task, evaluation protocol, and general best practices. To facilitate future research and reproducibility, we release our datasets and implementations in an easy-to-use and open source software package.1},
	number = {1},
	urldate = {2026-03-11},
	journal = {Computational Linguistics},
	publisher = {MIT Press},
	author = {Klie, Jan-Christoph and Webber, Bonnie and Gurevych, Iryna},
	month = mar,
	year = {2023},
	pages = {157--198},
	file = {Full Text PDF:/home/daniiligantev/Zotero/storage/JL7VFZGU/Klie et al. - 2023 - Annotation Error Detection Analyzing the Past and Present for a More Coherent Future.pdf:application/pdf},
}

@inproceedings{rohde_filling_2016,
	address = {Berlin, Germany},
	title = {Filling in the {Blanks} in {Understanding} {Discourse} {Adverbials}: {Consistency}, {Conflict}, and {Context}-{Dependence} in a {Crowdsourced} {Elicitation} {Task}},
	shorttitle = {Filling in the {Blanks} in {Understanding} {Discourse} {Adverbials}},
	url = {https://aclanthology.org/W16-1707/},
	doi = {10.18653/v1/W16-1707},
	urldate = {2026-03-11},
	booktitle = {Proceedings of the 10th {Linguistic} {Annotation} {Workshop} held in conjunction with {ACL} 2016 ({LAW}-{X} 2016)},
	publisher = {Association for Computational Linguistics},
	author = {Rohde, Hannah and Dickinson, Anna and Schneider, Nathan and Clark, Christopher N. L. and Louis, Annie and Webber, Bonnie},
	editor = {Friedrich, Annemarie and Tomanek, Katrin},
	month = aug,
	year = {2016},
	pages = {49--58},
	file = {Full Text PDF:/home/daniiligantev/Zotero/storage/IH44IBIG/Rohde et al. - 2016 - Filling in the Blanks in Understanding Discourse Adverbials Consistency, Conflict, and Context-Depe.pdf:application/pdf},
}

@inproceedings{rohde_discourse_2018,
	address = {Melbourne, Australia},
	title = {Discourse {Coherence}: {Concurrent} {Explicit} and {Implicit} {Relations}},
	shorttitle = {Discourse {Coherence}},
	url = {https://aclanthology.org/P18-1210/},
	doi = {10.18653/v1/P18-1210},
	abstract = {Theories of discourse coherence posit relations between discourse segments as a key feature of coherent text. Our prior work suggests that multiple discourse relations can be simultaneously operative between two segments for reasons not predicted by the literature. Here we test how this joint presence can lead participants to endorse seemingly divergent conjunctions (e.g., BUT and SO) to express the link they see between two segments. These apparent divergences are not symptomatic of participant naivety or bias, but arise reliably from the concurrent availability of multiple relations between segments – some available through explicit signals and some via inference. We believe that these new results can both inform future progress in theoretical work on discourse coherence and lead to higher levels of performance in discourse parsing.},
	urldate = {2026-03-11},
	booktitle = {Proceedings of the 56th {Annual} {Meeting} of the {Association} for {Computational} {Linguistics} ({Volume} 1: {Long} {Papers})},
	publisher = {Association for Computational Linguistics},
	author = {Rohde, Hannah and Johnson, Alexander and Schneider, Nathan and Webber, Bonnie},
	editor = {Gurevych, Iryna and Miyao, Yusuke},
	month = jul,
	year = {2018},
	pages = {2257--2267},
	file = {Full Text PDF:/home/daniiligantev/Zotero/storage/VIB57PQZ/Rohde et al. - 2018 - Discourse Coherence Concurrent Explicit and Implicit Relations.pdf:application/pdf},
}

@inproceedings{rohde_recovering_2015,
	address = {Lisbon, Portugal},
	title = {Recovering discourse relations: {Varying} influence of discourse adverbials},
	shorttitle = {Recovering discourse relations},
	url = {https://aclanthology.org/W15-2703/},
	doi = {10.18653/v1/W15-2703},
	urldate = {2026-03-11},
	booktitle = {Proceedings of the {First} {Workshop} on {Linking} {Computational} {Models} of {Lexical}, {Sentential} and {Discourse}-level {Semantics}},
	publisher = {Association for Computational Linguistics},
	author = {Rohde, Hannah and Dickinson, Anna and Clark, Chris and Louis, Annie and Webber, Bonnie},
	editor = {Roth, Michael and Louis, Annie and Webber, Bonnie and Baldwin, Tim},
	month = sep,
	year = {2015},
	pages = {22--31},
	file = {Full Text PDF:/home/daniiligantev/Zotero/storage/DL4UMK9T/Rohde et al. - 2015 - Recovering discourse relations Varying influence of discourse adverbials.pdf:application/pdf},
}

@inproceedings{rohde_exploring_2017,
	title = {Exploring {Substitutability} through {Discourse} {Adverbials} and {Multiple} {Judgments}},
	url = {https://aclanthology.org/W17-6814/},
	urldate = {2026-03-11},
	booktitle = {Proceedings of the 12th {International} {Conference} on {Computational} {Semantics} ({IWCS}) — {Long} papers},
	author = {Rohde, Hannah and Dickinson, Anna and Schneider, Nathan and Louis, Annie and Webber, Bonnie},
	editor = {Gardent, Claire and Retoré, Christian},
	year = {2017},
	file = {Full Text PDF:/home/daniiligantev/Zotero/storage/FMFI8973/Rohde et al. - 2017 - Exploring Substitutability through Discourse Adverbials and Multiple Judgments.pdf:application/pdf},
}



@inproceedings{polakova2013introducing,
  title        = {Introducing the prague discourse treebank 1.0},
  author       = {Pol{\'a}kov{\'a}, Lucie and M{\'\i}rovsk{\`y}, Ji{\v{r}}{\'\i} and Nedoluzhko, Anna and J{\'\i}nov{\'a}, Pavl{\'\i}na and Zik{\'a}nov{\'a}, {\v{S}}{\'a}rka and Hajicov{\'a}, Eva},
  year         = 2013,
  editor = "Mitkov, Ruslan  and
      Park, Jong C.",
  booktitle    = {Proceedings of the Sixth International Joint Conference on Natural Language Processing},
  pages        = {91--99},
  publisher = "Asian Federation of Natural Language Processing",
  url = "https://aclanthology.org/I13-1011/"
}

@inproceedings{AB2/SUU9CB_2019,
    author = {Prasad, Rashmi and Webber, Bonnie and Lee, Alan and Joshi, Aravind},
    publisher = {Abacus Data Network},
    title = {{Penn Discourse Treebank Version 3.0}},
    year = {2019},
    version = {V1},
    doi = {11272.1/AB2/SUU9CB},
    pid = {https://hdl.handle.net/11272.1/AB2/SUU9CB}
}

@inproceedings{scholman-etal-2022-discogem,
    title = "{D}isco{G}e{M}: A Crowdsourced Corpus of Genre-Mixed Implicit Discourse Relations",
    author = "Scholman, Merel  and
      Dong, Tianai  and
      Yung, Frances  and
      Demberg, Vera",
    editor = "Calzolari, Nicoletta  and
      B{\'e}chet, Fr{\'e}d{\'e}ric  and
      Blache, Philippe  and
      Choukri, Khalid  and
      Cieri, Christopher  and
      Declerck, Thierry  and
      Goggi, Sara  and
      Isahara, Hitoshi  and
      Maegaard, Bente  and
      Mariani, Joseph  and
      Mazo, H{\'e}l{\`e}ne  and
      Odijk, Jan  and
      Piperidis, Stelios",
    booktitle = "Proceedings of the Thirteenth Language Resources and Evaluation Conference",
    month = jun,
    year = "2022",
    address = "Marseille, France",
    publisher = "European Language Resources Association",
    pid = "https://aclanthology.org/2022.lrec-1.351/",
    pages = "3281--3290",
}

@inproceedings{zeldes-etal-2025-erst,
    title = "e{RST}: A Signaled Graph Theory of Discourse Relations and Organization",
    author = "Zeldes, Amir  and
      Aoyama, Tatsuya  and
      Liu, Yang Janet  and
      Peng, Siyao  and
      Das, Debopam  and
      Gessler, Luke",
    journal = "Computational Linguistics",
    volume = "51",
    number = "1",
    month = mar,
    year = "2025",
    address = "Cambridge, MA",
    publisher = "MIT Press",
    pid = "https://aclanthology.org/2025.cl-1.3/",
    doi = "10.1162/coli_a_00538",
    pages = "23--72",
}

@inproceedings{hewett-stede-2025-disagreements,
    title = "Disagreements in analyses of rhetorical text structure: A new dataset and first analyses",
    author = "Hewett, Freya  and
      Stede, Manfred",
    editor = "Peng, Siyao  and
      Rehbein, Ines",
    booktitle = "Proceedings of the 19th Linguistic Annotation Workshop (LAW-XIX-2025)",
    month = jul,
    year = "2025",
    address = "Vienna, Austria",
    publisher = "Association for Computational Linguistics",
    pid = "https://aclanthology.org/2025.law-1.3/",
    doi = "10.18653/v1/2025.law-1.3",
    pages = "35--47",
    ISBN = "979-8-89176-262-6",
}

@inproceedings{zikanova2025gold,
  title={Gold Data and Multiple Understanding of Discourse Relations},
  author={Zik{\'a}nov{\'a}, {\v{S}}{\'a}rka and Nedoluzhko, Anna and M{\'\i}rovsk{\`y}, Ji{\v{r}}{\'\i} and Haji{\v{c}}ov{\'a}, Eva},
  booktitle={International Conference on Text, Speech, and Dialogue},
  pages={250--262},
  year={2025},
  Publisher={Springer},
  pid = "https://link.springer.com/chapter/10.1007/978-3-032-02551-7\_22"
}

@inproceedings{Speecon,
    author = "{Speecon Consortium}",
    title = "Catalan Speecon database",
    publisher = "Speecon Project, distributed via ELRA: ELRA-Id S0327",
    year = "2011",
    series = "Speecon resources",
    edition = "1.0",
	islrn = "935-211-147-357-5",
	organization="SpeeCon"
}

@misc{xu_beyond_2026,
	title = {Beyond {Consensus}: {Perspectivist} {Modeling} and {Evaluation} of {Annotator} {Disagreement} in {NLP}},
	shorttitle = {Beyond {Consensus}},
	url = {http://arxiv.org/abs/2601.09065},
	doi = {10.48550/arXiv.2601.09065},
	abstract = {Annotator disagreement is widespread in NLP, particularly for subjective and ambiguous tasks such as toxicity detection and stance analysis. While early approaches treated disagreement as noise to be removed, recent work increasingly models it as a meaningful signal reflecting variation in interpretation and perspective. This survey provides a unified view of disagreement-aware NLP methods. We first present a domain-agnostic taxonomy of the sources of disagreement spanning data, task, and annotator factors. We then synthesize modeling approaches using a common framework defined by prediction targets and pooling structure, highlighting a shift from consensus learning toward explicitly modeling disagreement, and toward capturing structured relationships among annotators. We review evaluation metrics for both predictive performance and annotator behavior, and noting that most fairness evaluations remain descriptive rather than normative. We conclude by identifying open challenges and future directions, including integrating multiple sources of variation, developing disagreement-aware interpretability frameworks, and grappling with the practical tradeoffs of perspectivist modeling.},
	urldate = {2026-03-03},
	publisher = {arXiv},
	author = {Xu, Yinuo and Jurgens, David},
	month = jan,
	year = {2026},
	note = {arXiv:2601.09065 [cs]},
	keywords = {Computer Science - Computation and Language},
}

@article{scikit-learn,
  title={Scikit-learn: Machine Learning in {P}ython},
  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
          and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
          and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
          Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
  journal={Journal of Machine Learning Research},
  volume={12},
  pages={2825--2830},
  year={2011}
}

@inproceedings{chistova2025bridging,
  title={Bridging Discourse Treebanks with a Unified Rhetorical Structure Parser},
  author={Chistova, Elena},
  booktitle={Proceedings of the 6th Workshop on Computational Approaches to Discourse, Context and Document-Level Inferences (CODI 2025)},
  pages={197--208},
  year={2025}
}

@inproceedings{costa-kosseim-2025-multi,
    title = "Multi-Lingual Implicit Discourse Relation Recognition with Multi-Label Hierarchical Learning",
    author = "Costa, Nelson Filipe  and
      Kosseim, Leila",
    editor = "B{\'e}chet, Fr{\'e}d{\'e}ric  and
      Lef{\`e}vre, Fabrice  and
      Asher, Nicholas  and
      Kim, Seokhwan  and
      Merlin, Teva",
    booktitle = "Proceedings of the 26th Annual Meeting of the Special Interest Group on Discourse and Dialogue",
    month = aug,
    year = "2025",
    address = "Avignon, France",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2025.sigdial-1.4/",
    pages = "48--61"
}