% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}
% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{float}
\usepackage{graphicx}
\usepackage{dblfloatfix}
\usepackage{amsmath}


% \usepackage{fontspec}
% \setmain\setmainfont{Times New Roman}
% \setsansfont{Arial}
% \setmonofont[Color={0019D4}]{Courier New}
% \usepackage{polyglossia}

% \setmainlanguage{english}
% \setotherlanguage{telegu}

% \usepackage{fontspec}
% \setmainfont{Times New Roman}
% %%% For language switching -- like babel, but for xelatex
% \usepackage{polyglossia}

% \setmainlanguage{english}
% \setmainfont{TimesNewRomanPSMT}
% \setsansfont{Arial}
% \setmonofont[Color={0019D4}]{Courier New}
% \setotherlanguages{telugu} %% or other languages
% \newfontfamily\telugufont[Script=Telugu]{Noto Sans Telugu}



% \documentclass{article}
\usepackage{geometry}
\usepackage{booktabs}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% \usepackage{telugu}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{SemanticCUETSync at SemEval-2024 Task 1: Finetuning Sentence Transformer to Find Semantic Textual Relatedness}


\author{Md. Sajjad Hossain,\ Ashraful Islam Paran,\ Symom Hossain Shohan,\ Jawad Hossain\\
 and {\bf Mohammed Moshiul Hoque} 
 \\
Department of Computer Science and Engineering \\
Chittagong University of Engineering and Technology \\
\texttt{\{u1904031,u1904029,u1904048,u1704039\}@student.cuet.ac.bd} \\
\texttt{moshiul\_240@cuet.ac.bd} \\
}



% if the names do not fit well on one line use
%        Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\


% \author{First Author \\
%   Affiliation / Address line 1 \\
%  Affiliation / Address line 2 \\
% Affiliation / Address line 3 \\
% \texttt{email@domain} \\\And
% Second Author \\
% Affiliation / Address line 1 \\
% Affiliation / Address line 2 \\
% Affiliation / Address line 3 \\
% \texttt{email@domain} \\}

\begin{document}
\maketitle
\begin{abstract}
Semantic textual relatedness is crucial to Natural Language Processing (NLP). Methodologies often exhibit superior performance in high-resource languages such as English compared to low-resource ones like Marathi, Telugu, and Spanish. This study leverages various machine learning (ML) approaches, including Support Vector Regression (SVR) and Random Forest, deep learning (DL) techniques such as Siamese Neural Networks, and transformer-based models such as MiniLM-L6-v2, Marathi-sbert, Telugu-sentence-bert-nli, and Roberta-bne-sentiment-analysis-es, to assess semantic relatedness across English, Marathi, Telugu, and Spanish. The developed transformer-based methods notably outperformed other models in determining semantic textual relatedness across these languages, achieving a Spearman correlation coefficient of 0.822 (for English), 0.870 (for Marathi), 0.820 (for Telugu), and 0.677 (for Spanish). These results led to our work attaining rankings of 22\textsuperscript{th} (for English), 11\textsuperscript{th} (for Marathi), 11\textsuperscript{th} (for Telegu) and 14\textsuperscript{th} (for Spanish), respectively.



\end{abstract}


\section{Introduction}
Semantic textual relatedness measures the conceptual and contextual similarity of two sentences. It specifies how alike the two sentences are in terms of meaning. Determining semantic textual relatedness is crucial for various language-processing tasks, including contemporary technology, search engines, chatbots, virtual assistants, plagiarism detection, paraphrasing, question answering, text generation, and other related applications. It is possible to determine how comparable two natural language sentences are based on the quantity and quality of matched elements in each sentence. These matches offer essential insights into the relationship between and degree of semantic similarity between the two sentences and the likelihood of successful word matching in semantically equivalent text pairs. Another critical aspect of semantic relatedness is understanding the context of sentences. Singnificant research has been done in this area, specifically in SemEval competition from 2012 to 2017 \cite{agirre-etal-2012-semeval, agirre-etal-2013-sem, agirre-etal-2014-semeval, agirre-etal-2015-semeval, agirre-etal-2016-semeval, cer-etal-2017-semeval}. Most proposed methodologies perform better in high-resource languages like English and Spanish but could be better in other low-resource languages like Telegu, Marathi, and Bengali. In certain studies, translating from a low-resource language to English is conducted \cite{wu-etal-2017-bit} prior to semantic analysis, contributing to suboptimal performance. Other factors include the failure of specific methods to capture the meaning of phrases and idioms within sentences effectively \cite{spiewak-etal-2017-opi}, as well as a tendency for some approaches to focus excessively on individual word meanings \cite{spiewak-etal-2017-opi}. This work addresses these shortcomings by proposing a transformer-based model, which gives us a good result in finding semantic textual relatedness.
The main contributions of this work are:
\begin{itemize}
    \item Investigated several ML, DL, and transformer-based models to find the semantic relatedness in various texts of four languages.
    \item Explored various pre-trained transformer-based methods with necessary tuning to identify semantic textual relatedness in English, Spanish, Telegu, and Marathi.
\end{itemize}
The code will be publicly available at \url{https://github.com/ashrafulparan2/SemanticCUETSync-at-SemEval-2024-Task-1}.


\section{Related Work}
Numerous studies have been accomplished on semantic relatedness in high-resource languages. \citealp{article} presented the process for calculating semantic similarity and proposed a feature-based metric for building semantic vectors. Their knowledge feature-based method found similarity measure of 0.82. \citealp{abdalla-etal-2023-makes} proposed a dataset to explore questions on what makes sentences semantically related. \citealp{reimers-2019-sentence-bert} suggested using a sentence transformer model, which creates a 768-dimensional dense vector space from sentences and paragraphs. The relatedness of two sentences can be assessed using this model. Their approach achieved the best score of 0.8492 with SRoBERTa-STSb-base model. Three transformer-based clinical semantic textual similarity models intended to detect semantic relatedness in medical data were presented by \citealp{info:doi/10.2196/19735}.
\citealp{chen-etal-2022-generate} proposed a semi-supervised sentence embedding technique called GenSE, which effectively uses large-scale unlabeled data. It achieved promising results on several STS datasets with an average correlation score of 0.8519. Meanwhile, \citealp{gatto2023text} evaluated sentence similarity among texts using large language models (LLM). According to their research, ChatGPT and other models are proficient in recognizing textual similarity within particular areas. In addition to the transformer models, \citealp{verma2020semantic} introduced a deep learning model, specifically a Recurrent Neural Network (RNN). This model utilized document embedding vectors to infer the meaning of small paragraphs comprising one, two, or three sentences. 

Significant challenges arise when representing sentences with low-resource languages (LRLs), such as Telugu, Marathi, and Bengali. Furthermore, limited datasets make the process of textual similarity detection more challenging in LRLs. \citealp{joshi2022l3cubemahasbert} focused on two low-resource Indian languages (Hindi and Marathi), and their proposed model was evaluated on real text classification datasets to show embeddings obtained from synthetic data, which will be an effective training strategy for low-resource languages.
They achieved the highest score of 0.83 utilizing the MahaBERT and LaBSE models. A cross-lingual model for finding similarity between sentences was proposed by \citealp{deode2023l3cubeindicsbert}. Their system obtained an accuracy of 0.82 for finding semantic relatedness in low-resource languages like Hindi, Marathi, Kannada, Telugu, Malayalam, Tamil, Gujarati, Odia, Bengali, and Punjabi. \citealp{Tang2018ImprovingMS} proposed a multilingual framework for finding semantic textual similarity in low-resource languages utilizing rich annotation data from a high-resource language. Their shared sentence encoder approach archived score of 0.825 for Spanish language.  
 
 
\section{Dataset and Task Description}
The dataset is developed by Ousidhoum et al. \cite{ousidhoum2024semrel2024} to evaluate the semantic textual similarity to perform the shard task at SemEval-2024. It includes data for Telugu, Marathi, English, and Spanish, which assess semantic relatedness between sentences. There are labeled and unlabeled data in the dataset. Labeled data is used for Track-A; each row has two sentences and a score that ranges from 0 to 1, representing the semantic relatedness of the sentences. Moreover, the dev, test, and train categories are applied to every language dataset. Table~\ref{dataset_stat} shows the distribution of dev, test, and train sets for the English, Marathi, Telugu, and Spanish datasets, respectively.
\begin{table}[h!]
\centering
\begin{tabular}{l|c|c|c}
\hline
\textbf{Language} & \textbf{Dev set} & \textbf{Test set } & \textbf{Train set}\\
\hline
English & 250 & 2600 & 11000 \\
Marathi & 294 & 299 & 2400 \\
Telegu & 130 & 297 & 2340 \\
Spanish & 140 & 600 & 1562 \\ \hline
Total & 814 & 3796 & 17302\\
\hline
\end{tabular}
\caption{Dataset statistics for Track-A}
\label{dataset_stat}
\end{table}

%\begin{figure}[H]
   % \centering
   % \includegraphics[width=1\linewidth]%{image/dataset.png}
    %\caption{Dataset}
    %\label{fig:enter}
%\end{figure}
The task for Track-A \cite{ousidhoum-etal-2024-semeval} calculates the semantic relationship and provides a score (degree of semantic relatedness) between 0 to 1. Figure \ref{fig:example} illustrates few examples of the task of sentence relatedness with the score.

% \begin{table}[h!]
%   \centering
%   \begin{tabular}{p{0.4\linewidth}p{0.3\linewidth}c}
%     \toprule
%     \textbf{Sentence \#1} &  \textbf{Sentence \#2} & \textbf{Score} \\ 
%     \midrule
%     If that happens, just pull the plug. & I’ve been searching the entire abbey for you. & 1.0 \\\hline
%    La niÃ±a estÃ¡ regando las plantas & Un patinador estÃ¡ haciendo trucos en un half pipe & 0.28 \\\hline
%    Una niÃ±a estÃ¡ intentando esconderse de sus padres &
%    Una muchacha joven tratando de esconderse de su mamÃ¡. & 0.63 \\
%     \bottomrule
%   \end{tabular}
%   \caption{Track-A task sample with Similarity score for English, \textcolor{red}{xxx, and xxx Mention the other Two languages}}
%   \label{tab:example}
% \end{table}

\begin{figure}[h!]
    \centering
    \includegraphics[width=1\linewidth]{image/example.png}
    \caption{Track-A task sample with Similarity score for English, Spanish, Telugu and Marathi}
    \label{fig:example}
\end{figure}



\section{System Overview}
Various textual features will be extracted to employ ML and DL models. Several transformer models are exploited for the task, including MiniLM-L6-v2, Marathi-SBERT, Telugu-Sentence-BERT-NLI, and Roberta-BNE-Sentiment-Analysis-ES, to assess semantic relatedness. Figure \ref{methodology} illustrates the schematic structure of the proposed approach.
\begin{figure}[h!]
    \centering    \includegraphics[width=1\linewidth]{image/overviewReview2.drawio.png}
    \caption{Schematic process of finding semantic textual relatedness} 
        \label{methodology}
\end{figure}


\textbf{Textual Feature Extraction:} Feature extraction is necessary for ML and DL models to learn from text. We used TF-IDF \cite{Tokunaga1994TextCB} to extract the features to apply different ML algorithms. Word2Vec \cite{mikolov2013distributed} and FastText \cite{grave-etal-2018-learning} embeddings were used to extract features for DL models. 


\textbf{ML-based Approaches:} This work employed traditional ML approaches, including SVR and RF. Following dataset preprocessing, we trained the model using the SVR for the English language. Similarly, we utilized the RF, configuring "n-estimators = 100" during training. A BaggingRegressor model is employed with the RF as the base estimator, with "n-estimators = 10" set for the base estimator.  

\textbf{DL-based Approaches:} This work explored a Siamese Neural Network (SNN) architecture implemented in PyTorch to perform the task. The SNN model was applied exclusively to the English dataset. The text is transformed into vectors and subsequently passed through LSTM layers. The similarity between the two texts was determined using cosine embedding loss, and optimization was carried out using the `Adam' optimizer. We set learning rate = 10, hidden-dim = 128, embedding-dim = 128 and max-length = 1000 and ran it for 10 epochs. Table \ref{ml-hyper} shows the several ML parameters and DL hyperparameters used for the models.

\begin{table}[h!]
\centering
\begin{tabular}{l|l|l}
\hline
\textbf{Classifier} & \textbf{Parameters} & \textbf{Value}\\
\hline
SVR & kernel & rbf \\
 & degree & 3 \\
 & gamma & scale \\
 & C & 1.0 \\
 \hline
RF & n-estimator & 100 \\
\hline
Bagging+RF & n-estimator & 10 \\
\hline
SNN & epoch & 10 \\
 & lr & 9e-5 \\
 & hidden-dim & 128 \\
  & embedding-dim & 128 \\
   & max-words & 10000 \\
    & max-length & 1000 \\
 \hline
\end{tabular}
\caption{Several ML parameters and DL hyperparameters of the employed models}
\label{ml-hyper}
\end{table}


\textbf{Transformer-based Models:} This work developed the task solutions in four languages: English, Marathi, Telugu, and Spanish. Thus, four different kinds of transformers were explored, including all-MiniLM-L6-v2 \cite{wang2020minilm}, telugu-sentence-bert-nli \cite{deode2023l3cube}, marathi-sentence-similarity-sbert \cite{joshi2022l3cubemahasbert} and dccuchile-bert-base-spanish-wwm-uncased \cite{CaneteCFP2020}. MiniLM-L6-v2 is a sentence transformer that maps sentences and paragraphs to a 384-dimensional dense vector space and can be used for tasks like clustering or semantic search. Similarly, telugu-sentence-bert-nli is a TeluguBERT model trained on a large dataset. MahaSBERT(marathi-sentence-similarity-sbert) is also fine-tuned on the STS dataset. Before applying these models, we have used other transformer models such as msmarco-distilbert-cos-v5, all-mpnet-base-v2, and all-MiniLM-L12-v2 for English. For evaluating Marathi, we also used pre-trained models like stsb-xlm-r-multilingual , marathi-roberta, and marathi-sentence-bert-nli. Similarly, we have fine-tuned transformer models like LaBSE and telugu-sentence-similarity-sbert for Tamil. Finally, for Spanish, we also used roberta-bne-sentiment-analysis-es and stsb-xlm-r-multilingual. We called all these models from
\verb|Huggingface|\footnote{https://huggingface.co/sentence-transformers} sentence transformers library. All models were trained on the task datasets. MahaSBERT and TeluguBERT usually perform well for low-resource languages. Dataloader was used from a torch to prepare the data before passing it to the model. We followed a similar approach to train the corresponding transformer model for all languages. Table \ref{tab:Hyper Parameter Tuning} demonstrates the hyperparameters used to train transformer-based models. 

\begin{table}[h!]
\begin{tabular}{l|c|c|c}
\hline
\textbf{Models}  & \textbf{LR} & \textbf{WD} & \textbf{WS} \\
\hline
all-Minilm-L6-v2  & 9e-5 & 9e-2 & 750 \\

all-Minilm-L12-v2  & 9e-5 & 9e-2 & 750 \\

marathi--sbert  & 9e-5 & 5e-2 & 500 \\

telugu--bert-nli & 9.5e-5 & 5e-5 & 750 \\

bert-base-spanish  & 9e-5 & 9.5e-7 & 700 \\
\hline
\end{tabular}
\caption{Tuned hyperparameter for the transformer-based models, where LR, WD, and WS denotes learning rate, weight decay and warmup steps, respectively}
\label{tab:Hyper Parameter Tuning}
\end{table}


\section{Experiments}
During the development, this study utilized Python 3 (3.10.12) and Python-based packages from \verb|PyTorch|\footnote{https://pytorch.org/} framework to implement sentence transformers (MiniLM-L6, MarathiSBERT, TeluguSBERT, SpanishSBERT). To implement the models, 29GB of RAM, 16GB of VRAM, and 73.1GB of storage space were used. We utilized NVIDIA Tesla P100 GPU from  \verb|Kaggle|\footnote{https://www.kaggle.com/}. We used pandas (2.1.4) and numpy (1.24.3) to analyze and prepare the data. The ML models were developed with the scikit-learn (1.2.2) packages, and the DL models were trained
with Keras (2.13.1) and TensorFlow (2.13.0). The PyTorch (2.0.0) packages, transformers (4.36.2), and sentence transformers (2,6,1) were used to implement transformer models.


The superiority of the models is determined based on the Spearman rank correlation coefficient ($\rho$) \cite{sennrich-etal-2016-improving}, which measures how well the system predicted rankings of test instances. This work also measures the Kendall correlation ($\tau$) and Pearson correlation ($R$).

\section{Results and Analysis}
Table \ref{tab:Result} exhibits the evaluation results of ML, DL, and transformer-based models for four languages: English, Marathi, Telegu, and Spanish.  
\begin{table}[h!]
\centering
\begin{tabular}{l|p{0.2\linewidth}|c|c|c}
\hline
\textbf{Language} & \textbf{Models} & \textbf{$\rho$} & \textbf{$\tau$} & \textbf{$R$} \\
\hline
& SVR & 0.161 & 0.105 & 0.181 \\
& RF & 0.177 & 0.115 & 0.153 \\
& Bagging + RF & 0.178 & 0.114 & 0.156 \\
& SNN & 0.418 & 0.284 & 0.473 \\

English & MiniLM-L12 & 0.815 & 0.614 & 0.821 \\
& Mpnet-v2 & 0.821 & 0.620 & 0.832 \\
& Distilbert & 0.821 & 0.619 & 0.829 \\

& \textbf{MiniLM-L6} & \textbf{0.822} & 0.620 & 0.832 \\
\hline
 & Stsb-xlm & 0.764 & 0.566 & 0.779 \\
Marathi & Marathi-Roberta & 0.810 & 0.619 & 0.810 \\
& Marathi-BERT-nli & 0.866 & 0.684 & 0.866 \\

& \textbf{Marathi-SBERT} & \textbf{0.870} & 0.688 & 0.875 \\
\hline
& Telugu-SBERT & 0.761 & 0.567 & 0.795 \\
Telegu & LaBSE & 0.804 & 0.608 & 0.814 \\
& \textbf{Telugu-BERT-nli} & \textbf{0.820} & 0.617 & 0.827 \\
\hline
 & Roberta-bne & 0.659 & 0.479 & 0.713 \\
Spanish & Stsb-xlm & 0.655 & 0.473 & 0.707 \\
& \textbf{Spanish-BERT} & \textbf{0.677} & 0.503 & 0.719 \\
\hline
\end{tabular}
\caption{Performance of the employed models on the test set}
\label{tab:Result}
 \end{table}

The results demonstrate that the ML models perform poorly. DL models are slightly better, but they need to be better. Transformer-based models demonstrated exceptional performance across all languages. For the English language, Mpnet-v2, Distilbert, and MiniLM-L12 scored 0.821, 0.821, and 0.815, respectively. The top-performing model for English was MiniLM-L6, with a maximum score of 0.822. For the Marathi language, Stsb-xlm, Marathi-BERT-nli and MarathiRoberta received scores of 0.764, 0.866, and 0.810, respectively. The MarathiSBERT model performed best, with a Spearman correlation score of 0.870. For the Telugu language, Telugu-SBERT and LaBSE had scores of 0.761 and 0.804, respectively. The best model for this language is Telugu-BERT-nli, which has a Spearman correlation rank of 0.804. For Spanish, Roberta-bne and Stsb-xlm have scores of 0.659 and 0.677, respectively, but Spanish-BERT outperforms both by 0.677. Figure~\ref{performance} illustrates the summary of the best-performed models in four languages of the task.

\begin{figure}[h!]
    \centering    \includegraphics[width=1\linewidth]{image/barChart.png}
    \caption{Performance summary of the best model in each task language} 
    \label{performance}
\end{figure}

Transformer-based outperformed other models by a wide margin for four languages in the task. One explanation is its ability to capture information's bidirectional context. In addition, it produces contextual word representations that make polysemy understandable and enables capturing minute variations in meaning depending on context. Since these transformer-based model has been pre-trained on various linguistic tasks using a sizable dataset, fine-tuning these models improves results.  

\subsection{Error Analysis}
Transformer models can recognize sentence patterns more apparently with better textual features. Thus, these models outperformed ML and DL techniques. Figure \ref{fig:error_comparison} depicts some example scores (actual and predicted) regarding two sentences in task languages.

\begin{figure}[h!]
    \centering
    \includegraphics[width=1\linewidth]{image/error5.png}    \caption{Sample prediction with Similarity scores: Actual (AS) and Predicted (PS)} 
    \label{fig:error_comparison}
\end{figure}

 \textcolor{black}{Based on annotations using Best-Worst Scaling, actual scores were computed by deducting the number of times a phrase pair was selected as the least related from the fraction of times it was selected as the most related.} The predicted and actual scores are incredibly close in samples 1, 2, and 4. However, the predicted score is more significant for sample 3. This may happen when a sentence is shorter than the longer sentence and contains similar terms. In this case, the model has a more challenging time figuring out the Spearman correlation between these two uneven-length sentences.

\section{Conclusion}
\label{sec:Conclusion}
This study explores the efficacy of various machine learning (ML), deep learning (DL), and transformer-based models for analyzing semantic relatedness within texts across four languages: English, Spanish, Telugu, and Marathi. Experimental assessments reveal subpar performance of both ML and DL models across all languages. However, transformer-based models exhibit superior capabilities in discerning semantic relatedness within the given task. Specifically, the MiniLM-L6 model excels for English, MarathiSBERT for Marathi, TeluguSBERT for Telugu, and SpanishSBERT for Spanish, achieving peak $\rho$ scores of 0.822, 0.870, 0.820, and 0.677, respectively. The study suggests that augmenting training data could enhance the performance of current models. Additionally, leveraging advanced techniques such as Large Language Models (LLM) and Generative Pre-trained Transformers (GPT) holds promise for further improvement.

% \textcolor{red}{add one more sentences; mention include of LLM and GPT models for future improvement}.

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only


\bibliography{custom}



\end{document}
