\subsection{The CODI-CRAC 2021 Corpus}
\label{sec:data}

\todo{This section to be greatly shortened if not completely removed (MP)}

% One of the objectives of the {\CODICRAC} Shared Task was to annotate new data for studying anaphora in dialogue.
The only existing dataset covering the full range of phenomena and with some coverage of dialogue, the {\ARRAU} data used for the {\CRAC} 2018 Shared Task, was used as training material and as one of the development sets.
In addition, new data from four dialogue corpora--{\AMI}, {\LIGHT}, {\PERSUASION} and {\SWITCHBOARD}--were annotated for development and testing using the same annotation scheme used in {\ARRAU}.
The dataset was annotated using the  {\MMAX}2 tool \CITE{mueller&strube:06}. 
% The dataset was annotated using the same {\MMAX}2 tool \CITE{mueller&strube:06} -- indeed, almost exactly the same  {\MMAX} style -- and by the same  annotators who annotated and checked  {\ARRAU} Release 3,  which is currently being prepared for release. 
After annotation, the documents were converted into the {\CONLL}-{\UA} `exploded' format. % used by the scorer.
%described by a document on the Universal Anaphora site.\footnote{\url{https://github.com/UniversalAnaphora/UniversalAnaphora/blob/main/UA\_CONLL\_U\_Plus\_proposal\_v1.0.md}}
All the publically distributable data ({\AMI}, {\LIGHT} and {\PERSUASION}) are available from the Codalab shared task site and  the Universal Anaphora site.
For more details see \cite{khosla-et-al:CODICRAC21}.
%on the datasets and the annotation process.


% \paragraph{ARRAU} 
% The {\ARRAU} corpus\footnote{\url{http://www.arrauproject.org}} \cite{poesio&artstein:LREC08,uryupina-et-al:NLEJ} covers a variety of genres.
% In addition to news, 
% covered by the subset of the Penn Treebank \CITE{marcus-et-al:CL93} 
% that was annotated in  the \ACRO{rst} treebank
% \CITE{carlson(lynn)-et-al:03},
% it also includes dialogue data--all the task-oriented dialogues
% in the {\TRAINS-93}  
% corpus\footnote{\url{http://www.ldc.upenn.edu/Catalog/catalogEntry.jsp?catalogId=LDC95S25}}--
% spoken narratives--the Pear Stories that provided some of the early evidence on salience and anaphoric reference   
% \CITE{chafe:80}--and scholarly text in the humanities in the {\GNOME} corpus \CITE{poesio:ACL-discanno04}.

% All noun phrases are annotated, including singletons and non-referring noun phrases.
% The range of annotated anaphoric relations includes split antecedent anaphora, bridging reference, and discourse deixis.

% \paragraph{Switchboard}
% {\SWITCHBOARD}\footnote{\url{https://catalog.ldc.upenn.edu/LDC97S62}}
% \cite{godfrey-et-al:ICASSP92} 
% is one of the best known dialogue corpora.
% It consists of 1,155 five-minute spontaneous telephone conversations between two participants not previously acquainted with each other. 
% In these conversations, callers question receivers on provided topics, such as child care, recycling, and news media. 
% 440 speakers participate in these 1,155 conversations, producing 221,616 utterances.
% It was annotated for dialogue acts by \CITEA{stolcke-et-al:CL97}\footnote{This version is available from \url{https://convokit.cornell.edu/documentation/switchboard.html}} and for information status by \CITEA{nissim-et-al:LREC04}.


% \paragraph{AMI} The {\AMI} corpus\footnote{\url{https://groups.inf.ed.ac.uk/ami/corpus/}} \cite{carletta:06} 
% is a collection of 100 hours of meeting recordings between several participants. 
%           The recordings include signals from close-talking and far-field microphones, individual and room-view video cameras, and output from a slide projector and an electronic whiteboard.
% Several types of annotation were carried out, including dialogue acts, topics, summaries, named entities, and focus of attention.

% \paragraph{Light} 
% Amazon, Facebook, Google, and other \ACRO{ai} companies have all created dialogue corpora in recent years to support their research on conversational agents. 
% {\LIGHT}  \cite{urbanek-et-al:arxiv2019:LIGHT} is one of the many recently created corpora available on the \CODE{Parl.ai} platform.\footnote{\url{https://parl.ai/projects/light/}} 
% {\LIGHT} is a large-scale fantasy text adventure game research platform for training agents that can both talk and act, interacting either with other models or with humans. 
% The {\LIGHT} corpus was entirely created through crowdsourcing at different levels.
% In the first round, workers created a number of settings (the King's palace, the dark forest, etc); then in a second round workers created fitting characters for each scenario, providing information about their background history, their personality, etc. 
% Finally, in a third round, workers created dialogues between these characters. 

% \paragraph{Persuasion} The Persuasion for Good corpus\footnote{\url{https://convokit.cornell.edu/documentation/persuasionforgood.html}} \cite{wang(xuewei)-et-al:ACL19} 
% is a collection of online conversations generated by Amazon Mechanical Turk workers, where one participant (the persuader) tries to convince the other (the persuadee) to donate to a charity. 
% 1017 conversations were collected in total, along with demographic data and responses to psychological surveys from users. 
% Several speaker-level annotations were marked, including, e.g., demographics, the big five personality traits, etc.


% \subsection{ARRAU: Corpus and Annotation Scheme}

% \paragraph{Genres} The {\ARRAU} corpus\footnote{\url{http://www.arrauproject.org}} \cite{poesio&artstein:LREC08,uryupina-et-al:NLEJ}
% was designed to cover a variety of genres. 
% It includes a substantial amount of news text in a  
% sub-corpus called \CODE{RST}, consisting of the entire subset of the Penn Treebank \CITE{marcus-et-al:CL93}
% that was annotated in  the \ACRO{rst} treebank
% \CITE{carlson(lynn)-et-al:03}.
% In addition to the news data, {\ARRAU} includes three more
% sub-corpora.
% The \CODE{TRAINS} sub-corpus includes all the task-oriented dialogues
% in the {\TRAINS-93}  
% corpus\footnote{\url{http://www.ldc.upenn.edu/Catalog/catalogEntry.jsp?catalogId=LDC95S25}}  
% as well as the pilot dialogues in the so-called {\TRAINS-91} corpus. 
% The \CODE{PEAR} sub-corpus consists of 
% the complete collection of spoken narratives in the Pear Stories that provided some of the early evidence on salience and anaphoric reference   
% \CITE{chafe:80},
% and the \CODE{GNOME} sub-corpus covers documents from the medical and art history genres covered by the {\GNOME} corpus \CITE{poesio:ACL-discanno04}.
% The same coding scheme was used for all sub-corpora, 
% but separate guidelines were written for the textual  and the
% spoken dialogue sub-corpora.
% \CODE{RST}, \CODE{TRAINS}-93 and \CODE{PEAR} were used for the {\CRAC} 2018 shared task.
% For this year's shared task, \CODE{TRAINS}-91 was used for development in the trial phase, whereas all other datasets were used for training.\footnote{The original intention had been to use the soon-to-be-released {\ARRAU} 3, but as the work on this version  was still under way by the time the training data had to be released, {\ARRAU} 2 was used instead--i.e., the exact same version used for the {\CRAC} 2018 shared task.}

% \paragraph{Annotation scheme} The original annotation scheme used for Release 1 \cite{poesio&artstein:LREC08} is distributed with the dataset and is also available  from the {\ARRAU} corpus page.
% For the second release \cite{uryupina-et-al:NLEJ}, the guidelines for bridging were extended and genericity was also annotated using the {\GNOME} guidelines, but a complete new manual was not produced. However, a fairly extensive description can be found in \CITEA{uryupina-et-al:NLEJ}. 
% Following the {\CRAC} 2018 shared task, the guidelines for bridging, semantic category and genericity were further revised as part of the work on {\ARRAU} Release 3, which is almost ready, and a full revision of the annotation manual was also started.
% These guidelines were used in the current annotation.
% A brief summary follows.

% \paragraph{Markable definition}
% Many, especially among the older, anaphorically annotated  corpora impose syntactic, semantic or discourse-based restrictions on
% markables.
% For instance, in {\ONTONOTES} neither expletives nor singletons
% are annotated (for a discussion of the state of the art in anaphoric
% annotation, see \CITEA{poesio-et-al:ana-book-corpora}).
% By contrast,
% in {\ARRAU} \textit{all} {\NP}s are considered as markables, including 
% %when they are
% non-referring expressions (e.g., %because either 
% expletives such as \LINGEX{it}
% or  predicative {\NP}s such as
% \LINGEX{a busy place}) in
% \SREF{ex:non-referring},
% %\LINGEX{[It] seems to be [a busy place]}, 
% %and when they 
% and expressions do not corefer with any other markable and thus form a
% singleton coreference chain.
% Moreover, in {\ARRAU} non-referring markables are manually sub-classified into
% expletives, predicative, and quantifiers. % (see below).
% In addition, all generic references are marked, including
% premodifiers when the  entity referred to
% is mentioned again, e.g., in the case of the proper name \LINGEX{US} in
% \SREF{ex:ARRAU-premodifiers:a}, 
% and premodifiers that refer to a  kind, like
% \LINGEX{exchange-rate} in \SREF{ex:ARRAU-premodifiers:b}. 

% \begin{exe}
% \ENEW{ex:non-referring} 
%       [It] seems to be [a busy place]
% \ENEW{ex:ARRAU-premodifiers:a}
% \ldots The Treasury Department said that the [US]_1 trade deficit may worsen next year after two years of significant improvement\ldots
% The statement was the [US]_1's government first acknowledgment of
% what other groups, such as the International Monetary Fund, have been
% predicting for months. 

% \ENEW{ex:ARRAU-premodifiers:b}
% The Treasury report, which is required annually by a provision
% of the 1988 trade act, again took South Korea to task for its
% [exchange-rate]_1 policies. ``We believe there have continued to be
% indications of [exchange-rate]_1 manipulation \ldots
% \end{exe}

% \paragraph{Types of anaphoric relations marked}

% The {\ARRAU} guidelines support annotation of different types of anaphoric relations. 
% All referring markables are marked as either \CODE{discourse new} or
% \CODE{discourse old}. 
% Discourse new mentions introduce new entities and thus are not marked as being coreferent with an entity already introduced  (\NEWTERM{antecedent}).
% For discourse-old mentions, an antecedent can be identified, either of type
% \CODE{phrase} (if the antecedent was introduced using a nominal
% markable) or  
% \CODE{segment} (not introduced by a nominal markable,
% for \NEWTERM{discourse deixis}).\footnote{Identity
%   anaphora also includes split antecedent plural anaphoric reference.}   
% In addition, referring {\NP}s can be marked as \NEWTERM{related} to a
% previously mentioned discourse entity %in order
% to identify them as
% examples of  associative (\NEWTERM{bridging}) anaphora.

% \paragraph{Bridging references}
% Annotating --- indeed, even identifying --- bridging references in a reliable
% way is difficult, which is one
% of the reasons why so few large-scale corpora for anaphora include
% this type of annotation \CITE{poesio-et-al:ana-book-corpora,kobayashi-ng-2020-bridging}. 
% The {\ARRAU} guidelines for bridging anaphora are based on 
% experiments that were started by Vieira and Poesio
% \CITE{poesio&vieira:CL} 
% and continued in the {\GNOME} project \CITE{poesio:ACL-discanno04}.
% The {\ARRAU} Release 1 guidelines followed the {\GNOME} guidelines,
% but with an extension and a simplification.  
% Annotators were asked to mark a markable as \CODE{related} to a particular antecedent if it stood to that antecedent in one of the relations identified in 
% {\GNOME} (indeed, the same examples were used), 
% and in addition, if they stood in two additional relations (but
% without testing the reliability of this annotation): 
% \begin{itemize}
% \item \CODE{other}, for \LINGEX{other} {\NP}s, broadly following the
%   guidelines in \CITEA{modjeska:thesis}; 
% \item an \CODE{undersp-rel} relation for `obvious cases of bridging
%   that didn't fit any other category'.  
% \end{itemize}
% % Some statistics about bridging references in {\ARRAU} Release 2 are
% % shown in Table \ref{t:bridging}. 

% \paragraph{Discourse deixis}
% Discourse deixis in its full form 
% is a very complex form of reference,
% both to annotate \cite{kolhatkar-et-al:CL18} 
% and to resolve.
% Very few anaphoric annotation projects have attempted to annotate
% discourse deixis in its entirety
% \CITE{kolhatkar-et-al:CL18}.
% More typical is a partial annotation, as in
% \CITE{byron&allen:98,navarretta:00}, 
% %\todo{citation not added to bib file}
% who annotated pronominal reference to abstract objects; 
% in \ACRO{OntoNotes}, where event anaphora was marked
% \CITE{pradhan-et-al:ICSC07}; 
% and in the work of \CITEA{kolhatkar&hirst:EMNLP14}, which focused on
% so-called shell nouns.
% In {\ARRAU},
% A coder  specifying that a referring expression is discourse-old
%   is asked  whether its antecedent was introduced using a
%   \CODE{phrase} (markable)  or a \CODE{segment} (discourse segment).   
% Coders who choose \CODE{segment} %as the type of antecedent
%   have to
%   mark a sequence of \textit{predefined} clauses.
% % Statistics about discourse deixis in {\ARRAU} Release 2 are shown in
% % Table \ref{t:ddeix}. 
% % A total of 1633 cases of discourse deixis were marked.

% % \begin{table*}
% % %\scriptsize
% % %\small
% %  \begin{center}
% % \begin{tabular}{|r|r|r|r|r|}
% % \hline
% % RST  & TRAINS & GNOME & PEAR & TOTAL\\
% % \hline\hline
% % 631  &  862   &  73   &  67  & 1633\\
% % \hline
% % \end{tabular}
% % \caption{Distribution of discourse deixis in the subdomains of {\ARRAU}.}
% % %, Release 2.}
% % \label{t:ddeix}
% %  \end{center}
% % %\normalsize
% % \end{table*}

% \subsection{New Data}

% The annotated corpus prepared for the {\CODICRAC} shared task consists of conversations from four well-known conversational datasets:
% the {\AMI} corpus \cite{carletta:06},
% the {\LIGHT} corpus \cite{urbanek-et-al:arxiv2019:LIGHT},
% the {\PERSUASION} corpus \cite{wang(xuewei)-et-al:ACL19} and 
% {\SWITCHBOARD} \cite{godfrey-et-al:ICASSP92}. 
% For each of these datasets, documents for about 15K tokens were annotated for development, and about the same number of tokens were annotated for testing, according to the {\ARRAU} annotation scheme.

% % \todo{INCOMPLETE: If there is time, add examples of the dialogues and more discussion}

% \paragraph{Switchboard}
% {\SWITCHBOARD}\footnote{\url{https://catalog.ldc.upenn.edu/LDC97S62}}
% \cite{godfrey-et-al:ICASSP92} 
% is one of the best known dialogue corpora.
% It consists of 1,155 five-minute spontaneous telephone conversations between two participants not previously acquainted with each other. 
% In these conversations, callers question receivers on provided topics, such as child care, recycling, and news media. 
% 440 speakers participate in these 1,155 conversations, producing 221,616 utterances.
% It was annotated for dialogue acts by \CITEA{stolcke-et-al:CL97}\footnote{This version is available from \url{https://convokit.cornell.edu/documentation/switchboard.html}} and for information status by \CITEA{nissim-et-al:LREC04}.


% \paragraph{AMI} The {\AMI} corpus\footnote{\url{https://groups.inf.ed.ac.uk/ami/corpus/}} \cite{carletta:06} 
% is a collection of 100 hours of meeting recordings between several participants. 
%           The recordings include signals from close-talking and far-field microphones, individual and room-view video cameras, and output from a slide projector and an electronic whiteboard.
% Several types of annotation were carried out, including dialogue acts, topics, summaries, named entities, and focus of attention.

% \paragraph{Light} 
% Amazon, Facebook, Google, and other \ACRO{ai} companies have all created dialogue corpora in recent years to support their research on conversational agents. 
% {\LIGHT}  \cite{urbanek-et-al:arxiv2019:LIGHT} is one of the many recently created corpora available on the \CODE{Parl.ai} platform.\footnote{\url{https://parl.ai/projects/light/}} 
% {\LIGHT} is a large-scale fantasy text adventure game research platform for training agents that can both talk and act, interacting either with other models or with humans. 
% The {\LIGHT} corpus was entirely created through crowdsourcing at different levels.
% In the first round, workers created a number of settings (the King's palace, the dark forest, etc); then in a second round workers created fitting characters for each scenario, providing information about their background history, their personality, etc. 
% Finally, in a third round, workers created dialogues between these characters. 

% \paragraph{Persuasion} The Persuasion for Good corpus\footnote{\url{https://convokit.cornell.edu/documentation/persuasionforgood.html}} \cite{wang(xuewei)-et-al:ACL19} 
% is a collection of online conversations generated by Amazon Mechanical Turk workers, where one participant (the persuader) tries to convince the other (the persuadee) to donate to a charity. 
% 1017 conversations were collected in total, along with demographic data and responses to psychological surveys from users. 
% Several speaker-level annotations were marked, including, e.g., demographics, the big five personality traits, etc.

% \subsection{Annotation}

% The dataset was annotated using the same {\MMAX}2 tool \CITE{mueller&strube:06} -- indeed, almost exactly the same  {\MMAX} style -- and by the same two annotators from the {\DALI} team at Queen Mary University and University of Essex,  Dr. Maris Camilleri and Dr. Paloma Carretero Garcia, who annotated and checked  {\ARRAU} Release 3,  which is currently being prepared for release. 
% However, due to time constraints, each document was only annotated by a single annotator, with spot checks carried out by the other annotator and Massimo Poesio (in {\ARRAU} 3 each document was looked at by both annotators, and most documents were also independently checked by Massimo Poesio). 

% To prepare the data for the shared task, mentions were automatically extracted using the mention detector from \CITEA{yu-et-al:LREC20-mention}, and the output converted into {\MMAX} \ACRO{xml} format.

% \subsection{The Corpus}

% Some basic statistics about the {\CODICRAC} dataset are provided in Table \ref{tab:codicrac-stats}.
% For each dataset, the Table reports number of documents, size in tokens, number of markables, and how many of these are Discourse Old (Identity Coreference) anaphors (DO), 
% bridging references, 
% and discourse deixis. 
% With a total of 147,725 tokens and 41,807 markables, the {\CODICRAC} dataset is to our knowledge the largest dataset annotated for anaphoric interpretation in dialogue, almost twice the size of {\ARRAU}'s \CODE{TRAINS} sub-corpus in tokens and more than twice its size in markables.

% \begin{table*}
% %\scriptsize
% %\small
%  \begin{center}
% \begin{tabular}{ll|r|r|r|r|r|r}
% \toprule
%  \multicolumn{1}{l}{} & & \textbf{Docs}  & \textbf{Tokens} & \textbf{Markables} & \textbf{DO} & \textbf{Bridging} & \textbf{Disc. Deix} \\
% \midrule
% {\AMI}& dev  & 7 & 33741 & 8935 & 4400 & 850 & 230\\% 7 &  33741 & 8941 & 4765 & 832 & 230\\
%      & test  & 3 & 18260 & 4879 & 2300 & 633 & 118\\%3 &  18620 & 4891 & 2448 & 639 & 118\\
% \midrule
% {\LIGHT}& dev& 20 & 11495 & 3877 & 2120 & 381 & 62\\%20&  11495 & 3882 & 2220 & 384 & 69 \\
%      & test  & 21 & 11824 & 3931 & 2174 & 415 & 80\\%21&  11824 & 3935 & 2264 & 417 & 80 \\
% \midrule
% {\PERSUASION}&dev&22 & 9757 & 2929 & 1191 & 242 & 94\\%22&9757 & 2829 & 1295 & 242 & 94 \\
%      & test  & 28 & 12629 & 3839 & 1605 & 288 & 123\\%28&  12629 & 3840 & 1744 & 290 & 123\\
% \midrule
% {\SWITCHBOARD}&dev&11 & 14992 & 4025 & 1664 & 602 & 127\\%11&14992&4026 & 1816 & 608 & 127\\
%      & test  & 22 & 35027 & 9392 & 3992 & 1190 & 263\\%22&  35027 & 9394 & 4278 & 1197& 263\\
% \midrule
% \textbf{Totals}&      &134 & 147,725 & 41,807 & 19,446 & 4601 & 1097\\%&134&148,085 &41,738&20,830& 4609& 1104 \\
% \bottomrule
% \end{tabular}
% \caption{Statistics about the {\CODICRAC} corpus (new datasets only)}
% \label{tab:codicrac-stats}
%  \end{center}
% %\normalsize
% \end{table*}
% %\todo{Sopan, Juntao - check figures? (These are from MMAX)}

% %\todo{Can you explain the motivation behind the conversion?}
% After annotation, the documents were converted into the {\CONLL}-{\UA} `Extended' format used by the scorer, described by a document on the Universal Anaphora site.\footnote{\url{https://github.com/UniversalAnaphora/UniversalAnaphora/blob/main/UA\_CONLL\_U\_Plus\_proposal\_v1.0.md}}

% {\AMI}, {\LIGHT} and {\PERSUASION} are freely available from the Shared Task Codalab site.
% {\ARRAU} and {\SWITCHBOARD} are distributed by  \ACRO{ldc}.\footnote{{\ARRAU} is also freely available to any group that purchased the Penn Treebank and {\TRAINS}-93 corpora from \ACRO{ldc}.}