% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage[]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{enumitem}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

\usepackage{caption}
\usepackage{array}
\newcolumntype{C}[1]{>{\centering\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}
\newcolumntype{L}[1]{>{\raggedright\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}
\newcolumntype{R}[1]{>{\raggedleft\let\newline\\\arraybackslash\hspace{0pt}}m{#1}}

\usepackage{colortbl} 
\usepackage{soul}
\usepackage{booktabs}
% \usepackage[table,xcdraw]{xcolor}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Feature-augmented model for multilingual discourse relation classification}

% Multilingual discourse relation model 

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}


 \author{$^1$Eleni Metheniti \and $^{1,2,3}$Chloé Braud \and $^{1,3}$Philippe Muller \\
       $^1$UT3 - IRIT ; $^2$CNRS ; $^3$ANITI \\ \texttt{firstname.lastname@irit.fr}}

% \author{Eleni Metheniti \\
%   % Affiliation / Address line 1 \\
%   % Affiliation / Address line 2 \\
%   % Affiliation / Address line 3 \\
%      \\\And
%   Chloé Braud \\
%   % Institut de Recherche en Informatique de Toulouse (IRIT)  \\
%   % Paul Sabatier University \\
%   % Toulouse, France \\
%   % \texttt{firstname.lastname@irit.fr} 
%   \\\And
%   Philippe Muller \\
%   % Affiliation / Address line 1 \\
%   % Affiliation / Address line 2 \\
%   % Affiliation / Address line 3 \\

%  \\
%   }

\begin{document}
\maketitle
\begin{abstract}

Discourse relation classification within a multilingual, cross-framework setting is a challenging task, 
and the best-performing systems so far have relied on monolingual and mono-framework approaches.
In this paper, we introduce transformer-based multilingual models, trained jointly over all datasets---thus covering different languages and discourse frameworks. 
We demonstrate their ability to outperform single-corpus models and to overcome (to some extent) the disparity among corpora, by relying on linguistic features and generic information about the nature of the datasets. 
We also compare the performance of different multilingual pretrained models, as well as the encoding of the relation direction, a key component for the task.
Our results on the $16$ datasets of the DISRPT 2021 benchmark show improvements in accuracy in (almost) all datasets compared to the monolingual models, with at best $65.91$\% in average accuracy, thus corresponding to a $4$\% improvement over the state-of-the-art. 

% Discourse relation classification with a multilingual approach is a challenging and under-represented task. 
% We propose an approach based
% on transformer %-based 
% multilingual models, enhanced with features: 
% %it has been shown that some linguistic features can be helpful for the task, but not within a multilingual setting ; moreover, we investigate the addition of information targeting the 
% We use the datasets from the DISRPT 2021 Shared Task on Relation Classification across Formalisms, and the features created by \citep{gessler2021discodisco}, alongside methods and suggestions from other Shared Task participating teams. 
% Our results show that a multilingual, feature-augmented approach achieves higher accuracy in (almost) all datasets of the Shared Task, compared to the monolingual models developed by \citep{gessler2021discodisco}. 
% Even when features are not informative for all datasets, their presence improves performance. 
% Additional corpus-specific tokens also help in the joint multilingual setting. Finally, relation direction is most helpful as an additional annotation to the input sequence.  

\end{abstract}

\section{Introduction}

Discourse relation classification is the process of identifying the semantic-pragmatic relations between clauses or sentences, forming 
%two events, or sentences or phrases, inside 
the discourse structure of a document. 
It is considered a crucial step in building knowledge graphs \citep{zhang2020aser} and NLP downstream tasks requiring textual coherence and additional context, for example, text generation \citep{bosselut-etal-2018-discourse} or summarization \cite{xu-etal-2020-discourse}, text categorization \citep{liu-etal-2021-exploring-discourse}, and question answering \citep{jansen-etal-2014-discourse}. 

These relations, also called rhetorical relations, may be considered \textit{explicit}, when the connection is denoted by the presence of distinct words called \textit{connectives}, or \textit{implicit}, i.e. relations expressed without a discourse connective. For example, the \textit{concession} relation between the two arguments is expressed with the connective
\textit{however}
%\textit{\underline{however}} 
in the first example below, while in the second example, the relation \textit{manner} is implicit.
Most previous studies focused on implicit discourse relation classification, which is considered a harder task than the prediction of explicit relations. However, our setting requires that the system identifies both explicit and implicit relations simultaneously, a configuration that is more realistic and includes corpora with and without annotations of explicit markers. 



\begin{enumerate}[leftmargin=*]
    \item  $[$It's best to wash adults' overalls alone, especially men's.$]$ $[${\textit{However}, it is okay to wash just a few items with them, like blue jeans.}$]$ (GUM\_whow\_overalls) \\
\textbf{Label: }\textsc{concession} 
    \item $[$The ad would have run during the World Series tomorrow,$]$ $[$replacing the debut commercial of Shearson's new ad campaign, ``Leadership by Example.''$]$ (wsj\_2201) \\
\textbf{Label: }\textsc{Expansion.Manner} 
\end{enumerate}


% \begin{enumerate}[leftmargin=*]
%     \item  $[${\footnotesize \textit{Arg1}} \textit{It's best to wash adults' overalls alone, especially men's.}$]$ $[${\footnotesize \textit{Arg2}} \textit{\underline{However}, it is okay to wash just a few items with them, like blue jeans.}$]$ (GUM\_whow\_overalls) \\
% \textbf{Label: }\textsc{concession} (1<2)
%     \item $[${\footnotesize \textit{Arg1}} \textit{The ad would have run during the World Series tomorrow,}$]$ $[${\footnotesize \textit{Arg2}} replacing the debut commercial of Shearson's new ad campaign, ``Leadership by Example.''$]$ (wsj\_2201) \\
% \textbf{Label: }\textsc{Expansion.Manner} (1>2)
% \end{enumerate}


% For example, the \textit{concession} %\textsc{concession} 
% relation between the two arguments is expressed with the connective \textit{\underline{however}}: \\

% \noindent [{\footnotesize Arg1} \textit{It's best to wash adults' overalls alone, especially men's.}] [{\footnotesize Arg2} \textit{\underline{However}, it is okay to wash just a few items with them, like blue jeans.}] (GUM\_whow\_overalls) \\
% \textbf{Label: }\textsc{concession} (1<2) \\

% In the implicit relation \textit{manner}, there is no defining word for the connection of the two units: \\

% \noindent [{\footnotesize Arg1} \textit{The ad would have run during the World Series tomorrow,}] [{\footnotesize Arg2} replacing the debut commercial of Shearson's new ad campaign, ``Leadership by Example."] (wsj\_2201) \\
% \textbf{Label: }\textsc{Expansion.Manner} (1>2) \\


Varied typologies of discourse relations have been presented in the literature and applied to annotate several corpora in different languages. In this paper, we are presenting an approach to address multilingual, multi-framework discourse relation classification. We use as a take-off point the DISRPT Shared Task on \textit{Discourse Relation Classification across Formalisms} and its datasets covering
various languages and frameworks \cite{zeldes2021disrpt}, and compare our results to the 
current state-of-the-art system on the DISRPT data, 
%most successful system of the last two editions, 
which is composed of monolingual models, DisCoDisCo \citep{gessler2021discodisco}.


Our multilingual approach is based on joint training across all available corpora, covering varied languages and discourse frameworks. We conduct experiments over $16$ corpora, covering $11$ languages and $3$ discourse frameworks. We jointly train a classifier with all the datasets of the Shared Task, and we compare different transformer-based multilingual pretrained models. 
We extend the feature-based approach proposed by \citet{gessler2021discodisco} and \citet{gessler-etal-2022-midas}, to investigate its effect within a multilingual, cross-domain setting. Each DisCoDisCo monolingual model used different features, hence we evaluate which features are more informative in our joint setting.
We also enhance our models with features targeted to our multilingual, cross-framework setting.
% Finally, we test the impact of two different encodings of the direction of the relation.%: some relations are directed, meaning that the two arguments have specific roles within the definition of the relation (e.g. \textit{cause}(e1,e2) is not the same as \textit{cause}(e2,e1)).
Moreover, we test the effect of relation direction to the classification process. 
We examine two methods of expressing the direction of the relation between two units, either by annotating it with new tokens \citep{gessler2021discodisco} or by switching the position to unify it across relations \citep{metheniti-etal-2023-discut}.
%New bit: talk about the environmental impat
We adhere to the use of pretrained models of base size and fine-tune them for the discourse relation classification task. This ensures reproducibility, and shorter training times and computational power required.

Overall, we observe that XLM-RoBERTa models perform better than BERT models and that, contrary to the monolingual models presented in \cite{gessler2021discodisco},
for the multilingual, cross-framework settings, using all available features is the most beneficial for all models.
% Moreover, switching the argument position is less beneficial than encoding the direction with tokens, meaning that the pretrained language model is not very sensitive to the argument order.
For the encoding of the relation position, we observe that encoding with additional tokens is more beneficial %to 
than
switching the argument position, and both approaches are better than none.
Finally, we report state-of-the-art performance on discourse relation classification with a maximum of $65.91$\% in average accuracy over all the datasets, thus outperforming previous results by about $4$\%. The code for fine-tuning the classifiers can be found on GitLab\footnote{\href{https://gitlab.irit.fr/melodi/andiamo/discret_feat}{gitlab.irit.fr/melodi/andiamo/discret\_feat}}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Previous Work}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% ------ EXPLICIT -----
Most of the existing literature on discourse relation classification has focused on  {\it implicit} relations, since explicit ones are considered easier to predict, with already accuracy above $90$\% with simple models and features \citep{pitler-nenkova-2009-using}. However, it has been shown that the task can be more difficult for different domains or languages associated with small datasets \cite{xue-etal-2016-conll,scholman2021comparison,johannsen-sogaard-2013-disambiguating}.
%since this latter case is much easier with accuracy scores above 95\% based on a marker inventory and a few linguistic features \citep{pitler-nenkova-2009-using}. 
%\hl{TODO: More things to add here?}

% Explicit discourse relation classification is a straightforward task since the presence of discourse connectives denotes the relation present between discourse units. 

% ------ IMPLICIT ------
%However, there is a significant performance difference between explicit and implicit relation classification ($85.8\%$ vs. $57.6\%$) \citep{liu-li-2016-recognizing}. 
Approaches for \textbf{implicit relation classification} have either made use of linguistic features \citep{lin-etal-2009-recognizing} or the least ambiguous connectives as implicit connectives \citep{qin-etal-2017-adversarial}, or even explicit connectives \citep{shi-etal-2017-using, kurfali-ostling-2021-lets}. 
More recently, several approaches have been proposed relying on transformer-based architectures and pre-trained language models, demonstrating their effectiveness for domain transfer \cite{shi-demberg-2019-learning},  or for learning effective representation of sentences for the task \cite{nie-etal-2019-dissent,sileo-etal-2019-mining}, with also attempts relying on additional pre-training of language models \citep{kishimoto-etal-2020-adapting}.
% With advances in neural networks and transformer-based architectures, complex architectures have been developed for learning implicit relations \citep{lei2017swim, guo-etal-2018-implicit}, with experiments also including additional pre-training of language models \citep{kishimoto-etal-2020-adapting}.

%and implicit relation classification has also been incorporated in multi-task learning framework as another training objective \citep{kishimoto-etal-2020-adapting}.



% ----- DISRPT ---------
The \textbf{DISRPT Shared Tasks} were created to motivate research on challenging discourse analysis tasks, within a multilingual, cross-framework setting, by providing unified file formats for multiple discourse datasets.
There have been two editions including the task on Discourse Relation Classification \cite{zeldes2021disrpt,disrpt-2023-shared}: since not all datasets have annotations distinguishing between explicit and implicit relations, the focus is on predicting simultaneously all types of relations. This makes for a more realistic scenario, where the nature of the relation is not assumed to be known, and it corresponds to the task performed by a discourse parser. 
%, with multilingual and multi-framework approaches, and providing a unified text format for multiple discourse annotation frameworks. There have been two editions including the task on Discourse Relation Classification.

In DISRPT 2021 \citep{zeldes2021disrpt}, there were two submitted systems, for 16 datasets and 11 languages. \textbf{DisCoDisCo} \citep{gessler2021discodisco} is a system based on monolingual and corpus-specific classifiers based on pretrained BERT language models. The inputs were enriched with handcrafted features and direction annotations (described in detail in Section~\ref{sec:methodology-disco}). It was the most successful system, with an average $61.82\%$ accuracy. Meanwhile, \textbf{DiscRel} \citep{varachkina-pannach-2021-unified} aimed for a hierarchical and multilingual approach. They used sentence-level embeddings made with SentenceBERT \citep{reimers-gurevych-2019-sentence} and stacked random forest classifiers, to predict coarse-grained relations first and then fine-grained ones. They achieved $54.23\%$ averaged accuracy. 


In DISRPT 2023 \citep{braud-etal-2023-disrpt}, three systems were submitted. %competed. 
Some datasets were updated from 2021, a new framework was added (DEP, \citealp{yang-li-2018-scidtb}), and 10 new datasets and 2 new languages were added, for a total of 26 datasets and 13 languages. 
%However, none of the 2023 submitted systems surpassed DisCoDisCo's performance on the common datasets of 2021 and 2023. 
\textbf{HITS} \citep{liu-etal-2023-hits} was the system with the best performance for 2023. It employed a combination of framework-based, multilingual, and monolingual classifiers, based on large pretrained language models.  To enhance performance, they also employed bootstrap aggregating techniques and adversarial training.  The average accuracy score was $62.36\%$ overall. 
When the score is calculated by including only the corpora available in 2021, the average accuracy is $58.18$\% \cite{disrpt-2023-shared}, thus a lower score than DisCoDisCo.\footnote{Note that 
the comparison is inequitable, %especially 
because
%since
there have been changes in some corpora, e.g. English GUM.}
In \textbf{DiscReT}, we \citep{metheniti-etal-2023-discut} created multilingual classifiers trained jointly on all languages and corpora. We used pretrained multilingual BERT language models \citep{devlin-etal-2019-bert} and adapters \citep{houlsby2019parameter}. We also incorporated modifications on the label distribution to reduce the total number of labels across all corpora (see Section~\ref{sec:methodology-discret}); however, there were problems with fully reverting the labels for the evaluation process. The average accuracy was $54.44\%$. 
\textbf{DiscoFlan} \citep{anuranjana-2023-discoflan} used the Flan-T5 generative language model \citep{chung2022scaling} and trained monolingual models. The prompts queried the model for the relation between the two units. They post-process the model's output to match the labels of each corpus label set (see Section~\ref{sec:methodology-discoflan}). Accuracy was 31.2\% on average. 
%However, none of the 2023 submitted systems surpassed DisCoDisCo's performance on the common datasets of 2021 and 2023. 

% %==========
% \hl{not sure we should keep this or we'd need more details to show relevance: methods used for instance; in any case it's better at the end of the section to no cut the discourse part (PM)}
% % ---- Multilingual work ------
% Research on relation classification in a \textbf{multilingual} setting has been limited. One of the few multilingual sources that include relation annotations is the XNLI Dataset \citep{conneau-etal-2018-xnli}, which was created for Natural Language Inference tasks. It has been used for data augmentation and downstream tasks.


% \clearpage

\section{Methodology}

\subsection{Dataset}

For the multilingual, cross-framework motivation of our experiments, we use the datasets created for the DISRPT Shared Task \citep{zeldes2021disrpt} for Task 3: \textit{Discourse Relation Classification across Formalisms}.\footnote{The datasets and their statistics can be found in \href{https://github.com/disrpt/sharedtask2021}{github.com/disrpt/sharedtask2021}.}
We are using the datasets of the 2021 edition so that our results can be directly compared to the results of \citet{gessler2021discodisco}. These datasets are made of $16$ corpora, in $11$ languages, annotated in one of the following theoretical frameworks:
PDTB (Penn Discourse Treebank \citealp{prasad-etal-2004-annotation}), RST (Rhetorical Structure Theory, \citealp{mann:rhetorical:1988}) and SDRT (Segmented Discourse Representation Theory, \citealp{asher:logics:2003}).
%annotated for segmentation and relations. 
In all datasets, despite the different frameworks, discourse relations are annotated between pairs of segments that are primarily clauses or at most sentences.
% Even within the same framework, label sets differ among corpora.


\subsection{DisCoDisCo augmentation methodology}\label{sec:methodology-disco}

\input{latex/tables/full-features}

\citet{gessler2021discodisco} was the winning system of the DISRPT 2021, and compared to the results of the 2023 models on the common corpora, it is still the most successful system on the relation classification task.
The submitted system is composed of multiple models; each model is a classifier fine-tuning a monolingual pretrained BERT model trained on one dataset. They use the same monolingual pretrained model for datasets of the same language but train each dataset separately. They apply two methods of feature augmentation: hand-crafted features in addition to the input sequence, and annotation of the relation direction between the two units. 

\paragraph{DisCoDisCo features}
Regarding the additional features of the input sequence, \citeauthor{gessler2021discodisco} insert manually created features as a dense embedding before the encoder. The feature vector is added between the \texttt{[CLS]} token and the input sequence tokens, and it includes sequence-level information with categorical and numerical features. Categorical features are embedded whereas numerical features are log-scaled or binned and embedded, and the feature layer is padded for the leftover dimensions. 

\noindent The authors create a total of 28 features for each input sequence. These features were created by exploiting existing annotations (e.g. \textsc{Genre} from the GUM corpus, \textsc{Speaker} identities from STAC corpora), by calculating them (e.g. \textsc{Length}, \textsc{Distance}), with the help of the syntactic parses from the DISRPT 2021 Tasks 1-2 datasets, or with external %Python 
libraries (e.g. SpaCy \citep{spacy} to eliminate stop-words for the \textsc{Lexical Overlap} features). 
The full list of these features can be found in Table~\ref{tab:disco-features-full}, which includes information from \citet{gessler2021discodisco} and the system's source code. While they generate all features for all inputs and corpora, in their submitted system for the DISRPT 2021 Shared Task, for the discourse relation classification task, they only use a few of these features for each corpus-specific model. 
These decisions seemed to be geared toward optimizing performance rather than being based on language, framework, or human insights;
%These decisions were geared toward optimizing performance and were not taken based on language, framework, or human insights; 
for example, only using the features of one of the units. 
For our experiments, we are testing both the use of all features and the use of only the
``common''
%`common' 
features that were used for at least one dataset. 

\paragraph{Unit direction annotation}
Discourse relations are annotated between pairs of text segments.
Some relations can be directed, meaning that the order of the arguments of the relation is meaningful.
This feature depends on the way relations are encoded, we could have different labels with the arguments following the order of the text (e.g. \textit{cause vs result}), or one unique label where 
the first argument has always the same role 
%(e.g. cause) 
compared to the second 
regarding the semantics of the relation.
%(e.g. result).
All existing studies focusing on discourse relation identification consider this information as given: they present to the learning model the arguments in the order given by the annotation, thus first, then second argument of the relation.
It is not the case when one performs full discourse parsing: the parser knows that two segments are attached, but not in which order, and the segments are presented in the order of the text.
In order to better understand this important aspect of the task, we investigate different encodings of this information within a transformer architecture.


%The DISRPT datasets include segment units in pairs per relation, where the relation direction may be cause-to-result (\texttt{1>2}) or result-to-cause (\texttt{2>1}). 
In the DISRPT datasets, the pairs of segments are presented in the linear order of the text, but an additional column indicates the order of the arguments for the annotated relation.
\citeauthor{gessler2021discodisco} introduced two pseudo-tokens (not as BERT special tokens) in order to encode %annotate 
the direction between the two units:
\begin{itemize}[leftmargin=*, topsep=0pt,itemsep=0pt]
    \item If the direction of the relation follows the linear order of the text, a case annotated as (1>2) in DISRPT data, 
    %\item For cause-to-result relations (1>2), 
    the $\}$ token is added after the \texttt{[CLS]} token and before Unit 1 and the $>$ token before Unit 2.

    \item If the direction of the relation is reversed, a case annotated as (1<2) in DISRPT data, 
    %\item For result-to-cause relations (1<2), 
    the $<$ token is added after Unit 1 and the $\{$ token after Unit 2.
\end{itemize}




\subsection{Proposed additional augmentation}

\paragraph{Corpus-specific features}
Previous approaches to training multilingual, cross-framework classifiers with all corpora and languages reported results lower than monolingual systems. We assumed that one issue was the lack of guidance of the model, where it was hard for the model to make correlations between datasets.
%Since we are training multilingual classifiers with all corpora and languages jointly, we 
In order to tackle this issue, we add at the start of each sequence some additional tokens that characterize the dataset and should help the model to link samples from the same language or framework.
%of the language, corpus name, and framework of each input. 
We add as additional tokens, after the \texttt{[CLS]} token and before the input sequence tokens, the following tokens: 
%the corpus language name, the name of the corpus, and the annotation framework.

\begin{itemize}[leftmargin=*]%, topsep=0pt,itemsep=0pt]
    \item \underline
    {Language}: the language of the corpus in English (e.g. English, French, German, etc.);\footnote{Preliminary experiments with the language token in the corpus' original language (e.g. English, Français, Deutsch, etc.) showed the same performance as with the language token in English since the models we are using contain multilingual embeddings.
    }
    \item \underline{Corpus}:  the name of the dataset in the DISRPT 2021 data (e.g. \texttt{deu.rst.pcc}, \texttt{eng.rst.rstdt}, \texttt{fra.sdrt.annodis},  etc);
    \item \underline{Framework}: the framework name (e.g. rst, pdtb, sdrt, dep).
\end{itemize}

\paragraph{Feature embedding as tokens}

Instead of creating a dense embedding as \citeauthor{gessler2021discodisco} did, we are adding the additional features in the input sequence as tokens. Each feature value (numerical, categorical, and Boolean) is added to the vocabulary, in order not to be split into subwords by the tokenizer. Only the value of the feature is added, not its key, to not create an excessive amount of new tokens (e.g. all numbers encoded separately for each numerical feature). For example, the new token \texttt{0.1} does not refer to a number in the text but may refer to the feature \textbf{u1\_position} or \textbf{length\_ratio}, depending on its order in the input sequence.
This extends the size of the vocabulary and, therefore, extends the size of the token embedding matrix of the model to match the embedding matrix of the tokenizer. This technique stays close to the process of concatenating the feature vector with the token vector while assuring reproducibility with the HuggingFace models 
\citep{wolf-etal-2020-transformers}. 


\paragraph{Unit direction unification} % paraphrase this
In addition to implementing the relation direction annotation of the DisCoDisCo system (i.e. additional tokens), 
we are also testing the effectiveness of unifying the direction by switching unit positions. In \citet{metheniti-etal-2023-discut}, we proposed to reorder the two units in the input sequence, to follow the order of the arguments of the relation, instead of the linear order of the text as encoded in DISRPT files.
If the arguments are in the same order as in the text (1>2), then the input is unchanged, but if they are in reverse order (1<2), %then they are passed in reverse order in the model.  
the units have their position switched in the input sequence of the model.
%to follow the cause-to-result order for all inputs and relations. Inputs annotated with a cause-to-result (\texttt{1>2}) relation remain unchanged, while result-to-cause relations (\texttt{2>1}) are passed in the model with Unit 2 preceding Unit 1. 

\paragraph{Label merging} \label{sec:methodology-discret}
The joint training set of the $16$ corpora of the DISRPT 2021 Shared Task contains $126$ labels, making for a complex learning problem. These labels come from three different annotation frameworks, and sometimes overlap; for example, the labels \textit{Expansion.Correction} in \texttt{tur.pdtb.tdb} (Turkish, PDTB) and \textit{correction} in \texttt{eng.sdrt.stac} (English, SDRT) point to the same relation. Suggestions for unified label sets are limited to specific frameworks or do not cover all relations present in corpora \citep{benamara-taboada-2015-mapping, braud-etal-2017-cross, varachkina-pannach-2021-unified}.
We adapt the label harmonization that we proposed for the DISRPT 2023 Shared Task datasets, which implements minimal substitutions to less-frequent labels, and lower-casing \citep{metheniti-etal-2023-discut}. The number of our labels was reduced from $126$ originally to $102$ labels.


\paragraph{Label Filtering} \label{sec:methodology-discoflan}
Multilingual, multi-framework classification models provide a probability distribution of every label included in the training set, regardless of the target language and framework. We took inspiration from the strategy of \citet{anuranjana-2023-discoflan} who addressed the problem of generating annotations that may not match the labels of the training set by filtering the output of their generative model so that it converts them to existing labels. 
We are also post-processing our classification model label outputs, and we keep in the predictions only labels coming from the target corpus' framework. 
Thus a label that is present in the combined training corpus but not in the target framework label set will not be returned, even if it were assigned a higher probability by the model.


\input{latex/tables/average-accuracies-all-models-all-directions}
%cool ;)

\subsection{Classification models} 

We fine-tune multilingual classifiers built on pretrained multilingual transformer-based models. Fine-tuning is performed with all training sets of all languages and datasets jointly, while evaluation is performed on the evaluation and test sets of each dataset individually.
We used PyTorch \citep{NEURIPS20199015} and the HuggingFace libraries to build our classifiers, with the models: \texttt{bert-base-multilingual-cased},\footnote{\href{https://huggingface.co/bert-base-multilingual-cased}{huggingface.co/bert-base-multilingual-cased}} \texttt{distilbert-base-multilingual-cased},\footnote{\href{https://huggingface.co/distilbert-base-multilingual-cased}{huggingface.co/distilbert-base-multilingual-cased}} 
and \texttt{xlm-roberta-base}.\footnote{\href{https://huggingface.co/xlm-roberta-base}{huggingface.co/xlm-roberta-base}}
Each classification model is trained for $10$ epochs, keeping the best result out of the $10$ epochs, based on the development set. 
% new bit as well
The fine-tuning process for these models, per epoch, was around $1$ hour for DistilmBERT, $2$ hours for mBERT, and $2$ hours $10$ minutes for XLM-RoBERTa, on a GPU cluster with $4$ Nvidia Geforce GTX 1080TI graphics cards.


Multilingual BERT (mBERT) \citep{devlin-etal-2019-bert} is a pretrained model based on BERT. It has been trained on Wikipedia data of the top $104$ languages, with masked language modeling (MLM) and next-sentence prediction objectives. The base and cased version of the model contains $12$ layers, $12$ heads, and $177$M parameters. We selected it, in order to compare it with the DisCoDisCo models that were built on monolingual BERT-base architectures.
DistilmBERT \citep{sanh:hierarchical:2019} is a multilingual distilled version of mBERT with the same training set and objectives. The base and cased model has $6$ layers, $12$ heads, and $134$M parameters. As a lighter version of BERT, it would be interesting to compare a BERT-based model with fewer parameters. 
XLM-RoBERTa \citep{conneau-etal-2020-unsupervised} is a multilingual pretrained model based on RoBERTa. It is pretrained on $2.5$TB of filtered CommonCrawl data in $100$ languages. The base version of the model has $12$ layers and $279$M parameters. RoBERTa models have outperformed BERT in several datasets in the Shared Task \citep{liu-etal-2023-hits}, therefore we decided to include them in our experiments.


\section{Results}

In Table~\ref{tab:results-avg-all-models} we present the average accuracy for all the multilingual classification models we trained, with different pretrained models, with different combinations of features, and with different handling of relation annotation. 
We report the results of DisCoDisCo \cite{gessler2021discodisco} in the second column, and the results obtained by our system in the others.
The second row indicates how the direction of the relation is encoded, based on unit direction annotation (``Add. tokens'') as in \citeauthor{gessler2021discodisco} or by unit direction modification (``Switching units'') as in 
\citeauthor{metheniti-etal-2023-discut}
%DiscReT.
In the Appendix, the results for individual test sets can be found: in Table~\ref{tab:disco-results} for models trained with features and 
direction annotation based on additional tokens,
%DisCoDisCo direction annotation, 
in Table~\ref{tab:discret-results} for models trained with features and 
%DiscReT 
direction unification based on switching units, 
and in Table~\ref{tab:no-features-results} for models trained without features, with different direction handling (including no encoding of the direction at all).

Overall, our models outperform the state-of-the-art system DisCoDisCo in several settings, when linguistic features (i.e. ``Common/All DisCoDisCo features'') and/or dataset information (``LCF'', Language, Corpus, Framework) are used, with at best $65.91$\% in average accuracy, against $61.82$\% for DisCoDisCo. 
This demonstrates that single multilingual, cross-framework models are able to leverage correlations between the different datasets, and thus take advantage of a larger amount of data if fed with additional information.

For the models most similar to DisCoDisCo, i.e. using mBERT with annotations of direction (``Add. Tokens''), our results are very close to theirs: $60.41$\% \textit{vs} $59.54$\% (``No features'') and $61.82$\% \textit{vs} $62.56$\% (``Common features''). XLM-RoBERTa models performed better than the mBERT-based models and also surpassed the lightweight DistilmBERT models. They were the ones that steadily surpassed the DisCoDisCo system,
with $62.09$\% and $64.86$\% respectively for the same configurations.
Moreover, 
%However, 
the mBERT models also performed better than the DisCoDisCo baseline, when they were provided with the LCF tokens, either when limited to ``Common features'' ($63.46$\%), or when using ``All features'' ($63.67$\%). 


% Using XLM-RoBERTa models performed better than the mBERT-based models and the lightweight DistilmBERT models. They were the ones that steadily surpassed the DisCoDisCo system. 
% However, the mBERT models also performed better than the DisCoDisCo baseline, when they were provided with all DisCoDisCo features and with the LCF tokens. 

% Judging the different sets of features that we used, overall, features are beneficial for multilingual classification. 
Observing the different sets of features that we used, the addition of any features improves the accuracy of multilingual classification, and the best configuration was, in most cases, the features used by \citet{gessler2021discodisco}, with the addition of corpus-specific features.
When we used 
additional tokens to encode the direction of the relation,
%the DisCoDisCo direction annotations, 
the most beneficial set of features for all the models was the ``common'' features, i.e. only the features used by at least one model in the DisCoDisCo 2021 system. 
We notice that the model with the highest accuracy of all is the XLM-RoBERTa model,
using this encoding of the direction and
%with 
the common DisCoDisCo and
%features, the 
LCF features.
%, and the DisCoDisCo annotation of direction . 
However, using all features in this setting leads to very similar results (-$0.4\%$).
%However, the same model, with all DisCoDisCo and LCF features, was only $0.4\%$ less accurate. 
%On the other hand, when the direction is encoded by switching the units ("DiscReT" direction unification), the situation is reversed: results are better when using all the features rather than only the common ones. 
% This is probably linked to the fact that this encoding of the direction seems less efficient, and thus the additional features are more helpful. 
% Adding also the LCF features in that case is mostly beneficial with BERT, while, with XLM-RoBERTa, results are on-par, meaning that the model manages to make connection between datasets by itself. 
%On the other hand, 
When the direction is encoded by switching the units, the situation is reversed: results are better when using all the features rather than only the common ones. 
%However, for the DiscReT direction unification, using all the DisCoDisCo features brought better performance than the ``common'' features (but did not outperform direction annotation in the same feature setting). 
The addition of the LCF tokens, alongside the DisCoDisCo features, showed an increase in accuracy as well. For the XLM-RoBERTa models, the presence of all features was also most beneficial, but not necessarily the presence of the LCF features. 

Looking at individual datasets, our models outperformed the DisCoDisCo 2021 system in all but one dataset, the Basque \texttt{eus.rst.ert} (by $0.15\%$, Table \ref{tab:disco-results}). 
For some datasets, the improvement was 
significant 
(with XLM-RoBERTa models), for example up to $15.72\%$ for \texttt{spa.rst.sctb} (Spanish) and over $8\%$ for \texttt{fas.rst.prstc} (Farsi) and \texttt{zho.rst.sctb} (Chinese). The mBERT models trailed not far behind the XLM-RoBERTa ones, however, there was an instance where an mBERT model was more successful, mBERT with all DisCoDisCo and LCF features for \texttt{fra.sdrt.annodis} (French). Also, models with all features were more successful for the French, Portuguese, and Spanish datasets. 
%Detailed Tables are given in Appendix \ref{app}

Comparing the performance between the
two ways of handling the direction of the relation,
%DiscReT direction unification and the DisCoDisCo direction annotation, 
the direction annotation
based on additional tokens
was the better option for most datasets when the DisCoDisCo features were 
used.
%present. %(around $+2$\% of improvement if we compare the two best models for each configuration)
However, for the Dutch \texttt{nld.rst.nldt} and Portuguese \texttt{por.rst.cstn} datasets, the performance was identical with either setting. 
Observing the effect of the direction handling without the addition of features, we note that, while the 
method based on additional tokens
%DisCoDisCo direction annotation 
performed better overall, there were instances where 
switching the arguments
%the DiscReT direction unification 
was better (English \texttt{eng.rst.rstdt, eng.sdrt.stac, Spanish spa.rst.rststb}), and one dataset for which no change was marginally better (Portuguese \texttt{por.rst.cstn}), 
see Tables \ref{tab:disco-results}, \ref{tab:discret-results} and \ref{tab:no-features-results} in Appendix.



\section{Discussion}

Our multilingual approach outperformed the monolingual approach of DisCoDisCo \citep{gessler2021discodisco}
in all but one dataset, the Basque one. Our initial assumption was that the use of the multilingual setting would be beneficial since the use of more data is favorable to the models and the instances of less frequent labels would be higher. 
Indeed, for the very small datasets \texttt{spa.rst.sctb} (Spanish, $326$ train sentences) and \texttt{zho.rst.sctb} (Chinese, $361$ train sentences), the improvement was elevated with all models. 
For the largest datasets (\texttt{eng.pdtb.pdtb}, English, $44.5$K train sentences; \texttt{rus.rst.rrt}, Russian, $19$K train sentences; \texttt{tur.pdtb.tdb}, Turkish, $25$K train sentences) there was also an improvement of $3-4\%$. In the case of \texttt{eus.rst.ert} (Basque) with $1.6$K train sentences, we observed the label distribution; it has 25 unique labels and similar distributions to \texttt{spa.rst.rststb}, \texttt{spa.rst.sctb}, and \texttt{zho.rst.sctb}. We observe the classification report results \citep{scikit-learn} for the most successful model in Table~\ref{tab:class-report-eus}. In the 2021 edition of the data, there were a few labels in this dataset with misspellings (\textit{motibation} instead of \textit{motivation}), which were corrected in the 2023 edition. These labels were not changed by the DiscReT mappings and were not corrected in order to stay true to the 2021 data. Even with these errors, however, this is not the smallest, most complex, or relation-rich dataset. Therefore, the failure of the Basque dataset may be related to the language's typological difference from the rest, as it benefits less from the multilingual pretrained language models.

% Please add the following required packages to your document preamble:
% \usepackage{booktabs}
\begin{table}[t]
\footnotesize
\setlength\tabcolsep{2.5pt}
\centering
%\begin{tabular}{@{}lcccc@{}}
\begin{tabular}{@{}lrrrr@{}}
\toprule
 & \textbf{precision} & \textbf{recall} & \textbf{f1-score} & \textbf{support} \\ \midrule
\textbf{motibation} & 0 & 0 & 0 & 2 \\
\textbf{summary} & 0 & 0 & 0 & 3 \\
\textbf{concession} & 0.58 & 0.65 & 0.61 & 17 \\
\textbf{purpose} & 0.87 & 0.80 & 0.83 & 50 \\
\textbf{joint} & 0 & 0 & 0 & 1 \\
\textbf{causation} & 0.51 & 0.51 & 0.51 & 37 \\
\textbf{interpretation} & 0.50 & 0.08 & 0.13 & 13 \\
\textbf{circumstance} & 0.76 & 0.67 & 0.71 & 48 \\
\textbf{expansion.conjunction} & 0.28 & 0.48 & 0.35 & 25 \\
\textbf{unconditional} & 0.50 & 0.25 & 0.33 & 4 \\
\textbf{evaluation} & 0.30 & 0.56 & 0.39 & 16 \\
\textbf{anthitesis} & 0 & 0 & 0 & 5 \\
\textbf{unless} & 0.59 & 0.62 & 0.60 & 21 \\
\textbf{solution-hood} & 0 & 0 & 0 & 8 \\
\textbf{result} & 0.50 & 0.53 & 0.51 & 34 \\
\textbf{background} & 0.52 & 0.76 & 0.62 & 29 \\
\textbf{means} & 0.69 & 0.68 & 0.68 & 37 \\
\textbf{conditional} & 1.00 & 0.33 & 0.50 & 9 \\
\textbf{preparation} & 0.90 & 0.85 & 0.87 & 73 \\
\textbf{elaboration} & 0.66 & 0.69 & 0.67 & 140 \\
\textbf{list} & 0.63 & 0.44 & 0.52 & 54 \\
\textbf{evidence} & 0.40 & 0.25 & 0.31 & 8 \\
\textbf{sequence} & 0.37 & 0.48 & 0.42 & 23 \\
\textbf{justify} & 0.42 & 0.62 & 0.50 & 8 \\
\textbf{restatement} & 0.60 & 0.23 & 0.33 & 13 \\ \midrule
\textbf{accuracy} &  &  & 0.60 & 678 \\
\textbf{macro avg} & 0.41 & 0.37 & 0.37 & 678 \\
\textbf{weighted avg} & 0.62 & 0.60 & 0.60 & 678 \\ \bottomrule
\end{tabular}
\caption{Classification report for the \texttt{eus.rst.ert} (Basque) test set, with the XLM-RoBERTa model with Common and LCF features (epoch 8).}
\label{tab:class-report-eus}
\end{table}


Comparing the use of different models, we observe that the XLM-RoBERTa base models are more successful, probably because of their larger number of parameters. For the original DisCoDisCo model, the use of BERT-based models was obligatory for most languages, as at the time there were fewer options available, especially for less common languages. The mBERT base models were not far less successful than the XLM-RoBERTa, with the help of features. The DistilmBERT models are far too optimized and lightweight, missing parameters that were, as is shown, necessary for the classification process. 

Overall, the addition of features, even as simple as additional tokens in the input sequence, improved classification accuracy significantly. In the monolingual setting, it was possible to test different feature sets to configure which was the best, but for the multilingual setting, selecting features is not straightforward, as different corpora contain different annotations (e.g. the GUM and STAC corpora are the only ones with the \textsc{Speaker} information). The small differences in accuracy between using all features and only the ones used for the DisCoDisCo 2021 system are produced because, in a multilingual setting with all the datasets used jointly, some features that are informative for some corpora will not be for others, if the annotation does not exist. The addition of the language, framework, and corpus name was also beneficial, in order to annotate the presence of corpus-specific features, even if the information of language is not directly accessible to the model. 

Finally, regarding the relation direction, human intuition is different than the way models process input. 
%The DiscReT 
The proposal to unify all relation directions 
by switching the arguments \cite{metheniti-etal-2023-discut}
%as cause-to-result, in theory, 
sounds beneficial
in theory, especially when the same relation can be initiated in either unit.  
However, transformer-based models are not necessarily sensitive to word order; even though positional information is injected in them, some research suggests that they are not sensitive to permutations \citep{pham-etal-2021-order, gupta2021bert}. However, other research supports that not all permutations are processed equally \citep{sinha-etal-2021-unnatural} and that the models learn structural information \citep{wang-chen-2020-position, papadimitriou-etal-2022-classifying}. 
It is, therefore, understandable that the presence of additional tokens noting the direction 
as in \citep{gessler2021discodisco}
%(as DisCoDisCo implemented) 
may communicate more information about the relation direction to the models, than switching unit positions.
%as DiscReT suggested. 
However, there was also a smaller improvement with the unification of the direction; this points to the models either being capable of constructing a rudimentary structure of the two arguments or the models not being completely insensible to word order.

\section{Conclusion}

In this paper, we reprised the DISRPT 2021 Shared Task on Relation Classification across Formalisms and revisited the most successful model of the last two editions, DisCoDisCo \cite{gessler2021discodisco}. 
We adapted DisCoDisCo methodologies to multilingual relation classification models, with the addition of techniques and suggestions from other participating teams of the 2023 edition \citep{metheniti-etal-2023-discut, anuranjana-2023-discoflan}.

We found that XLM-RoBERTa models outperform BERT models, in the multilingual setting, especially with the presence of DisCoDisCo's hand-crafted features. The most successful model was trained only with the features used by DisCoDisCo models, as opposed to all features created in the preprocessing stage---but this success was only marginal to the use of all features, and was not true for all architectures. The addition of corpus-specific tokens (language, corpus name, framework) was also beneficial in the multilingual setting. Finally, annotating the relation direction 
with additional tokens
was more successful than unifying the position of the two arguments, due to the make of transformer-based models.
It should be noted that this information proved crucial and that further studies are needed on this aspect, in particular on the possibility of predicting direction and on the heterogeneity of existing corpora with regard to its encoding. 
 

As a future direction, we are considering using our approach on the updated DISRPT 2023 benchmark, which includes modified corpora, additional corpora in more languages, and some small validation datasets that allow for testing out-of-domain performance. 



%\newpage 
\section*{Acknowledgements}

This work is supported by the AnDiaMO project (ANR-21-CE23-0020).
Our work has benefited from the AI Interdisciplinary Institute ANITI. ANITI is funded by the French ``Investing for the Future – PIA3'' program under the Grant agreement n°ANR-19-PI3A-0004. 
%This work was partially supported by the ANR (ANR-19-PI3A-0004) through the AI Interdisciplinary Institute, ANITI, as a part of France’s “Investing for the Future — PIA3" program. 
This work is also partially supported by the SLANT project (ANR-19-CE23-0025).
Chloé Braud and Philippe Muller are part of the program DesCartes and are also supported by the National Research Foundation, Prime Minister’s Office, Singapore under its Campus for Research Excellence and Technological Enterprise (CREATE) program.
The work was also supported by the ANR grant SUMM-RE (ANR-20-CE23-0017). 


% Entries for the entire Anthology, followed by custom entries
\bibliography{anthology,custom}

\clearpage
\appendix
\onecolumn

\section{Appendix: Full classification results}\label{sec:appendix}

% discodisco-direction results
\vspace{20pt}
\input{latex/tables/disco-results}
\vspace{20pt}
% discret-direction  results
\input{latex/tables/discret-results}
% \vspace{-25pt}
% no features 
\input{latex/tables/no-features-results}

\end{document}
