% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}
\usepackage{array}
\newcolumntype{P}[1]{>{\centering\arraybackslash}p{#1}}

% Remove the "review" option to generate the final version.
%\usepackage[review]{ACL2023}
\usepackage{ACL2023}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{graphicx}
\makeatletter
\newcommand*{\rom}[1]{\expandafter\@slowromancap\romannumeral #1@}
\makeatother

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

% Replace "TEAM-NAME" with your official team name.
% You can change the part that says "Your Paper Title"
\title{ silp\_nlp at SemEval-2024 Task 1: Cross-lingual Knowledge Transfer for Mono-lingual Learning  }
% NOTE:
% Give your paper a descriptive title, possibly mentioning your methodology/approach.

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Sumit Singh,  Pankaj Kumar Goyal\and  Uma Shanker Tiwary \\Indian Institute of Information Technology, Allahabad 
   \\ {\tt \{sumitrsch, pankajgoyal02003\}@gmail.com}  \\   {\tt ust@iiita.ac.in} \\}
% \author{Sumit singh \\   Indian Institute of Information Technology, Allahabad \\  
%   \texttt{sumitrsch@gmail.com} \\\And
%   Uma Shanker Tiwary \\ Indian Institute of Information Technology, Allahabad \\
%   \texttt{ust@iiita.ac.in} \\}

\begin{document}
\maketitle
\begin{abstract}
Our team, silp\_nlp, participated in all three tracks of SemEval2024 Task 1: Semantic Textual Relatedness (STR). We created systems for a total of 29 subtasks across all tracks: nine subtasks for track A, 10 subtasks for track B, and ten subtasks for track C. To make the most of our knowledge across all subtasks, we used transformer-based pre-trained models, which are known for their strong cross-lingual transferability. For track A, we trained our model in two stages. In the first stage, we focused on multi-lingual learning from all tracks. In the second stage, we fine-tuned the model for individual tracks. For track B, we used a unigram and bigram representation with suport vector regression (SVR) and eXtreme Gradient Boosting (XGBoost) regression. For track C, we again utilized cross-lingual transferability without the use of targeted subtask data. Our work highlights the fact that knowledge gained from all subtasks can be transferred to an individual subtask if the base language model has strong cross-lingual characteristics. Our system ranked first in the Indonesian subtask of Track B (C7) and in the top three for four other subtasks.
\end{abstract}


\section{Introduction}
\label{intro}
 
The importance of semantic relatedness in language has been long recognized. Applications include sentence representation, question answering, and text summarization \cite{abdalla-etal-2023-makes}.
Sentences can be related through either paraphrasal or entailment relations, or through broader commonalities such as shared topics, viewpoints, temporal origins, and logical connections.

This shared task \cite{ousidhoum-etal-2024-semeval} aims to expand the scope of significant research in natural language processing (NLP) by incorporating 14 languages. The focus of the research is on semantic similarity and has predominantly been conducted in English. The languages included in the task are Afrikaans, Algerian Arabic, Amharic, English, Hausa, Hindi, Indonesian, Kinyarwanda, Marathi, Moroccan Arabic, Modern Standard Arabic, Punjabi, Spanish, and Telugu.

The task requires predicting the degree of semantic textual relatedness (STR) between pairs of sentences in multiple languages. The task is to rank these sentence pairs based on their level of relatedness, with scores ranging from 0 (completely unrelated) to 1 (maximally related). This task is divided into the following three tracks.

\textbf{Track A: Supervised }Challenge was developing a system trained on labelled datasets provided for the 11 subtasks. Publicly available related datasets could be utilized, but no additional dataset could be used in this work.

\textbf{Track B: Unsupervised}
The challenge was developing a system without using labelled datasets to measure semantic relatedness or similarity with the text units longer than two words only. We took advantage of unigram and bigram features with SVM regression and XGBoost.

\textbf{Track C: Cross-Lingual}
The challenge was to develop a system without labelled datasets in the target language and with the use of labelled dataset(s) in at least one other language (subtask). All datasets of track A other than the targeted subtask are utilized for similarity prediction in this work.

Our system utilizes cross-lingual learning by implementing multi-stage training methods similar to those used in \cite{wang-etal-2022-damo}, \cite{he-etal-2022-infrrd}, and \cite{singh-tiwary-2023-silp} for track A. In the first stage, we selected pre-trained cross-lingual language models that cover the languages used in our task and fine-tuned them on a combined dataset of all subtasks in track A for five epochs. This created a model checkpoint that had knowledge of multiple languages relevant to our task. In the second stage, we fine-tuned the model checkpoint generated in the first stage for each track individually. For track C, we created a dataset by combining all the data from track A, except for the targeted subtask, and fine-tuned it with the language model in a supervised manner. We used unigram and bigram bag-of-words representation with SVM-based regression (SVR) and XGBoost for each subtask in track B.

Our team achieved the best model for the Telugu and Marathi subtasks of track A by using the MuRIL large model \cite{https://doi.org/10.48550/arxiv.2103.10730}. In the first stage, we fine-tuned the model for all three tracks (English, Telugu, and Marathi) and then fine-tuned the model checkpoints for Telugu and Marathi. 

For all track A languages, we fine-tuned XLM-R for subtasks in the 1st stage, and we fine-tuned the checkpoint generated in stage one for all mono-lingual tracks in the 2nd stage.

% For English, we achieved the best result using the Roberta-large in one-stage fine-tuning.

For Track B, only monogram or bigram representation is allowed for supervised training. We obtained comparable results using both unigram and bigram representations in combination with SVR and XGBoost.

In Track C, we used all training data from Track A except for the current subtask data since the use of the same language data was not allowed in Track C. We adopted a cross-lingual transfer approach, where MuRIL gave the best result for the Hindi subtask, while XLM-R predicted the best results for the other subtasks.

The results for each subtask are presented in Table \ref{tab:trackA_results}, \ref{tab:trackB_results} and \ref{tab:trackC_results}, along with the baseline results. In addition to being multilingual, the key challenges of the task were the presence of many low-resource languages that lack proper pre-trained models for learning and the limited availability of training examples for some subtasks of Track A (see Table \ref{tab:Satasets}). To address these challenges, we utilized language models (LMs) with cross-lingual transferability, along with a two-stage training strategy.
Our code can be found here\footnote{\url{https://github.com/singhsumit1/Semeval-Semantic_textual-relatedness.git}}.





\section{Related Work}
\begin{figure*}
\centering
 \includegraphics[width=\linewidth,height=225pt]{images/multistage.drawio.png}
  \caption{General Architecture of two-stage training.}
  \label{fig:mainmodel}
\end{figure*}
The Semantic Textual Similarity (STR) task 2015 \cite{agirre-etal-2015-semeval} had three subtasks. The findings showed that the UMBC PairingWords system achieved the best score by semantically differentiating distributionally similar terms \cite{han-etal-2015-samsung}. In the subsequent STR task \cite{cer-etal-2017-semeval}, there are seven tasks that concentrate on multilingual and cross-lingual pairs. Additionally, one sub-track will delve into MT quality estimation data. The team ECNU \cite{tian-etal-2017-ecnu} achieved the highest score using ensembles of well-performing feature-engineered models with deep learning methods. These models used random forest (RF), gradient boosting (GB), and XGBoost (XGB) regression methods. However, statistical and machine learning models were not the best, as transformer-based models gained attention after \cite{devlin2019bert}. These models are pre-trained on large amounts of data and fine-tuned for various downstream tasks. Researchers have created the sentence transformer \cite{reimers-gurevych-2019-sentence} architecture for finding similar sentences. It is useful when given multiple sentences corresponding to a sentence, and we need to find the most similar one. However, fine-tuning the sentence transformer with the downstream task requires proper alignment between the dataset on which the sentence transformer is pre-trained and the dataset of the downstream task. In this task, there are multiple subtasks associated with multiple languages. Therefore, motivated by the performance of cross-lingual transformer-based models, we have used transformer-based language models that have strong cross-lingual transferability. It has been seen that cross-lingual transferability has advantages in various NLP tasks \cite{singh-tiwary-2023-silp, wang-etal-2022-damo}.

\section{Data}
\begin{table*}[ht]
\footnotesize
\centering
\begin{tabular}{|l|l|l|l|l|l|l|l|l|l|l|l|l|l|l|}
\hline
Data  & afr & amh   & arb & arq   & ary   & eng   & esp   & hau   & hin   & ind & kin   & mar   & pan & tel   \\ \hline
Train & -   & 992   & -   & 1,262 & 925   & 5,500 & 1,562 & 1,763 & -     & -   & 778   & 1,155 & -   & 1,146 \\ \hline
Dev   & 375 & 95    & 32  & 92    & 70    & 250   & 140   & 212   & 288   & 144 & 102   & 293   & 242 & 130   \\ \hline
Test  & 375 & 171   & 595 & 584   & 427   & 2,500 & 600   & 603   & 968   & 360 & 222   & 298   & 634 & 297   \\ \hline
Total & 750 & 1,258 & 627 & 1,938 & 1,422 & 8,250 & 2,299 & 2,578 & 1,256 & 504 & 1,102 & 1,746 & 876 & 1,573 \\ \hline
\end{tabular}
\caption {The datasets varied in the number of instances within their training, development, and test sets. Languages such as afr, arb, hin, ind, and pan lacked training data and were exclusively utilized in unsupervised and cross-lingual contexts.} 
\label{tab:Satasets}
\end{table*}



This shared task provided fourteen sets of monolingual data \cite{ousidhoum2024semrel2024}. There were nine languages for track A, each with training, validation, and testing data. For tracks B and C, only validation and testing data were provided for the Afr, Arb, Hin, Ind, and Pan languages. The training, validation, and testing data distribution for all languages is tabulated in Table \ref{tab:Satasets}. The English language had over 5,500 training examples, while other languages had comparatively fewer data provided.

\section{Methodology}



Our system has chosen robust cross-lingual transfer models such as XLM-R \cite{conneau2020unsupervised}, which is pre-trained on over 100 languages, and MuRIL \cite{https://doi.org/10.48550/arxiv.2103.10730}, which is pre-trained on all Indic and English languages, for track A and C. We have followed a two-stage training approach for track A. The detailed methodology for each track is explained in the following subsections.

For tracks A and C, we combined both sentences of an example data with a special token of LM (</sep> for XLM-R and Roberta, and [SEP] for MuRIL) in the preprocessing stage. Model tokenizer tokenizes the combined sentence into tokens and generates token IDs and attention masks.
% For the English subtask, we used Roberta-large (Roberta) for feature extraction because it's specific to the English language and has been pre-trained on a good amount of data, performing well for the downstream task of the English language. 
For other subtasks, we utilized XLM-R and MuRIL with the two-stage training approach.
\subsection{Methodology for track A}
\subsubsection{Two-stage training} In the initial stage of our project, we conducted multi-lingual training by utilizing the training data of all subtasks of track A in a selected LM that supports them. As shown in Fig. \ref{fig:mainmodel}, our model used the annotations of all subtasks of track A in this stage. We performed experiments using various hyperparameters across five epochs, selecting the best multilingual checkpoint based on the average validation data loss. In the second stage, we fine-tuned the multilingual checkpoint from the first stage and utilized it as an initial model for fine-tuning each monolingual subtask in track A. We trained each mono-lingual track with different hyper-parameters in the second stage and selected the final model based on the validation data loss of the corresponding subtask.
\\
\subsubsection{Model Architecture}
Figure \ref{fig:mainmodel} shows that the model tokenizer first tokenizes the input sentence. To improve GPU utilization, the tokenizers are set to a length of 92. Next, the language model generates word embeddings for each token. The embedding of the first token (which is <s> for the XLM-R and [SEP] for MuRIL) is passed through a linear layer, which projects it into logits, a vector of size one that represents the predicted similarity. Finally, we apply the Mean Square Error (MSE) loss function to calculate the difference between the prediction and the ground truth.
%<> and CRF <> which gives the log-likelihood of the given sequence of tags.
\begin{table*}[ht]
\footnotesize

\centering
\begin{tabular}{|l|l|l|l|l|l|l|l|l|l|}
\hline
Language             & A1 arq & A2 amh & A3 eng & A4 hau & A5 kin & A6 mar & A7 ary & A8 esp & A9 tel \\ \hline
XLM-R (one-stage)    & 0.49   & 0.49   & 0.78   & 0.68   & 0.23   & 0.86   & 0.63   & 0.58   & 0.78   \\ \hline
MuRIL (one-stage)    & -      & -      & 0.77   & -      & -      & 0.856   & -      & -      & 0.838   \\ \hline
XLM-R (two-stage)    & 0.59   & 0.84   & 0.84   & 0.72   & 0.49   & 0861     & 0.81   & 0.66   & 0.789     \\ \hline
MuRIL (two-stage)    & -      & -      & -      & -      & -      & 0.862   & -      & -      & 0.842   \\ \hline
Baseline\_Score      & 0.6    & 0.85   & 0.83   & 0.69   & 0.72   & 0.88   & 0.77   & 0.7    & 0.82   \\ \hline
Rank                 & 7      & 8      & 6      & 5      & 15     & 13     & 10     & 17     & 6      \\ \hline
\end{tabular}
\caption {Results of all subtasks of track A has been tabulated for both the settings one-stage and two-sage with both the LMs XLM-R and MuRIL.} \label{tab:trackA_results}
\end{table*}
\subsection{Methodology for track B} For track B, the sentences were converted into unigram and bigram representation and Support Vector Regression\footnote{\href{https://scikit-learn.org/stable/modules/generated/sklearn.svm.SVR.html}{Support Vector Regression}} (SVR) \cite{fu-etal-2016-iscas, liu-etal-2017-itnlp}.
\subsubsection{Unigrams /Bigrams embeddings} Both sentences in the examples are combined and transformed into a vector. To create the vector, each sentence is indexed based on the presence of unigrams/bigrams, and the corresponding index value is filled with the count of unigrams/bigrams. The resulting vector is then fed into the SVR model along with the label values for training.
\subsection{Methodology for track C} The methodology followed in track C is similar to the first stage of track A, with the difference that the combined dataset includes all subtasks except for the targeted subtask. For instance, to build a system for the English (eng) subtask, all data from the subtasks of track A, except for the eng subtask, is collected. The model architecture is also similar to that of track A but with only one stage involved.


\section{Experimental setup} 
\subsection{Track A and Track C}We achieved our best score using the MuRIL setup for the Telugu and Marathi subtasks, while the XLM-R setup worked best for the other track. During the training process, we experimented with different learning rates (5e-6, 2e-5, 5e-5, 8e-5, and 1e-4) and batch sizes (16, 32, and 64) in both stages. We selected the best model based on validation loss after five epochs of training. 

For track A and track C, we used the setups outlined in Fig. \ref{fig:mainmodel}. We implemented our task using the xxxTokenClassification class defined in \cite{wolf2020huggingfaces} for regression, where xxx refers to the selected model. We set the number of labels to one. The other hyperparameters for achieving the best results with both language models are listed in Table \ref{tab:hyperparameter}.

\subsection{Track B}
For subtasks which are training data available in track, we have generated monogram and bigram embedding and performed supervised learning with support vector regression (SVR) and gradient Boosting regression (XGBoost) 
 with the Scikitlearn\footnote{\href{https://scikit-learn.org/stable/}{scikit-learn.org/stable}} library.

 \textbf{Evaluation metrics} Results are Pearson correlation coefficient, which shows the similarity between two sentences.

 
%While implementation with CRF layer used PyTorch library.  



% Please add the following required packages to your document preamble:
% \usepackage[table,xcdraw]{xcolor}
% Beamer presentation requires \usepackage{colortbl} instead of \usepackage[table,xcdraw]{xcolor}
% Please add the following required packages to your document preamble:
% \usepackage[table,xcdraw]{xcolor}
% Beamer presentation requires \usepackage{colortbl} instead of \usepackage[table,xcdraw]{xcolor}




\begin{table*}[ht]
\footnotesize

\centering
\begin{tabular}{|l|l|l|l|l|l|l|l|l|l|l|}
\hline
Language        & B1 afr & B2 arq & B3 amh & B4 eng & B5 hau & B6 hin & B7 ind & B8 kin & B9 arb & B10 ary \\ \hline
Unigram SVR     & -      & 0.4    & 0.31   & 0.39   & 0.41   & -      & -      & 0.47   & -      & 0.68    \\ \hline
Bigram SVR      & -      & 0.4    & 0.64   & 0.32   & 0.39   & -      & -      & 0.35   & -      & 0.55    \\ \hline
Unigram XGBoost & -      & 0.3    & 0.4    & 0.28   & 0.35   & -      & -      & 0.38   & -      & 0.72    \\ \hline
Bigram XGBoost  & -      & 0.31   & 0.4    & 0.33   & 0.33   & -      & -      & 0.37   & -      & 0.72    \\ \hline
Dice Loss       & 0.73   & 0.44      & 0.69     & 0.74      & 0.42      & 0.57   & 0.53   & 0.36      & 0.31   & 0.6       \\ \hline
Baseline\_Score & 0.74   & 0.43   & 0.72   & 0.68   & 0.16   & 0.93   & 0.68   & 0.74   & 0.56   & 0.27    \\ \hline
Rank            & 5      & 4      & 5      & 10     & 3      & 6      & 1      & 6      & 6      & 3       \\ \hline
\end{tabular}
\caption {All the results for subtasks of Track B have been displayed. For subtasks B1, B7, B8, and B9, labelled data was not provided, so only the Pearson scores predicted by the Dice loss are shown. For the other subtasks, the Pearson scores are displayed for unigram SVR, bigram SVR, unigram XGBoost, bigram XGBoost, and dice loss. } \label{tab:trackB_results}
\end{table*}




\begin{table*}[ht]
\footnotesize
\begin{tabular}{|c|c|c|c|c|c|c|c|c|c|c|}
\hline
Language                                                        & C1 afr & C2 arq & C3 amh & C4 eng & C5 hau & C6 hin & C7 ind & C9 arb & C10 ary & C12 esp \\ \hline
\begin{tabular}[c]{@{}c@{}}Cross-lingual\\ (XLM-R)\end{tabular} & 0.7468 & 0.3867 & 0.8048 & 0.7372 & 0.6428 & 0.7476 & 0.4716 & 0.4267 & 0.6732  & 0.5691  \\ \hline
\begin{tabular}[c]{@{}c@{}}Cross-lingual\\ (MuRIL)\end{tabular} & -      & -      & -      & -      & -      & 0.8008 & -      & -      & -       & -       \\ \hline
Baseline\_Score                                                 & 0.79   & 0.46   & 0.84   & 0.8    & 0.64   & 0.76   & 0.47   & 0.61   & 0.4     & 0.62    \\ \hline
Rank                                                            & 7      & 6      & 5      & 6      & 4      & 5      & 5      & 6      & 3       & 9       \\ \hline
\end{tabular}
\caption {Table shows the results of all subtasks of track C. MuRIL LM support Hindi (C6) subtask of the track C therefore only Pearson score of C6 given for the MuRIL LM.} \label{tab:trackC_results}
\end{table*}


\section{Results and Analysis}
\label{sec:results}
The table below shows the Pearson score with official rank for Track A, Track B and Track C, along with the baseline score. Please refer to Table \ref{tab:trackA_results}, \ref{tab:trackB_results} and \ref{tab:trackC_results} for more details. Our system performed exceptionally well in the Indonesian (ind) subtask of track B (B7), achieving 1st rank with a Pearson score of 0.53\%. We secured 3rd rank in the three subtasks: B15, B10 and C10.


\textbf{Track A: }  Two-stage MuRIL setup achieved the best scores for Telugu and Marathi subtasks, while two-stage XLM-R setup achieved the best score for all other subtasks.


\textbf{Comparison between one-stage and two-stage methods with XLM-R LM}. A comparison has been illustrated in Fig. \ref{fig:coparison}. Based on the average performance of all subtasks in track A, it can be inferred that the two-stage strategy outperforms the one-stage strategy. The average score for all subtasks using the two-stage strategy was 0.73, while the average score for all subtasks using the one-stage strategy was 0.61.


\textbf{Comparison between MuRIL and XLM-R for the Telugu and Marathi}  Table \ref{tab:trackA_results} shows that for Telugu and Marathi, MuRIL performed better than XLM-R. Two-stage MuRIL produces a 0.05 higher score for Telugu subtask than two-stage XLM-R. For Marathi, the Two-stage MuRIL produces slightly more than the Two-stage XLM-R. 





\textbf{track B} Pearson score of track B tabulated in Table \ref{tab:trackB_results}. It is clear from Table \ref{tab:trackB_results} that SVR performed better than XGBoost. The performance of SVR with unigram and bigram is not straightforward. Results showed that for B4, B5, B8, and B10, bigram embeddings perform better than unigram embeddings. However for the B3 unigram performed better. 


\textbf{track C} Pearson score of the track C also showed in Table \ref{tab:trackC_results}.
 Table \ref{tab:trackC_results} shows that for Hindi (C6), MuRIL performed better than XLM-R. All the other subtasks of this track are only predicted by the XLM-R in cross-lingual settings.





% \textbf{ Our model vs baseline } As shown in Table \ref{} for subtask ... our model performed better than baseline. 

\section{Conclusion}


In this paper, we utilized multi-lingual track knowledge for the STR shared task to enhance the performance of monolingual models. Our team achieved first rank in the B7 subtask and third rank in the B5, B10, and C10 subtasks. We demonstrate that two-stage fine-tuning can help the monolingual models learn from the training data of all languages, leading to better performance. The results of track C illustrate the effectiveness of cross-lingual learning in a zero-shot scenario.
 








% \section*{Acknowledgements}
% Add any acknowledgements by uncommenting this section.


% Entries for the entire Anthology, followed by custom entries
\bibliography{anthology,custom}
\bibliographystyle{acl_natbib}

%% uncomment the below section to add an appendix.
% \appendix
% \section{Example Appendix}
% \label{sec:appendix}

% This is a section in the appendix.
% \appendix
\section{Appendix}\label{sec:appendix}

\subsection{Details of Hyper-parameters} Table \ref{tab:hyperparameter} shows the details of Hyper-parameters of best models for MuRIL and XLM-R setups with two-stage setup for Track A.


\begin{table*}[h]
\begin{center}
\begin{tabular} {| P {6 cm} | P {4 cm} | P {4 cm} |}
    \hline
    \textbf{Hyper parameters} & \textbf{ MuRIL setup}  & \textbf{ XLM-R setup} \\ \hline
Baseline language model for first stage & google/MuRIL-large-cased & XLM-Roberta-large\\ \hline    
Loss function & MSE & MSE \\ \hline
Hidden size for language model & 1024 & 1024 \\ \hline

Learning rate for language models & 5e-05 & 5e-05\\ \hline

First-stage training epochs & 5 & 5 \\ \hline
Second-stage training epochs & 5 & 5 \\ \hline
Batch size & 64 & 64\\ \hline  
Dropout rate & 0.1 & 0.1\\ \hline
% $\alpha$  for the second stage training & 5 (Softmax and Span), 100 for CRF  \\ \hline

Optimizer & AdamW & AdamW \\ \hline


% \\
\end{tabular}

\caption {Hyper-parameters for MuRIL and XLM-R setups with two-stage setup for Track A.} \label{tab:hyperparameter}
\end{center}
\end{table*}

\subsection{Comparison of Results of Track A} A comparison of subtasks of track A with one-stage XLM-R and two-stage XLM-R are shown in Fig. \ref{fig:coparison}. For all the subtasks two-stage architecture performed better than one-stage architecture.
\begin{figure*}
\centering
 \includegraphics[width=.75\linewidth,height=220pt]{images/comparison_plot.png}
  \caption{A comparison of subtasks of track A with one-stage XLM-R and two-stage XLM-R.}
  \label{fig:coparison}
\end{figure*}

\end{document}
