% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
% \pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
\usepackage{amsmath} 



% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

% \documentclass[11pt]{article}

% Remove the "review" option to generate the final version.

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{multirow}
\usepackage{graphicx}
\usepackage{float}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.

% This is also not strictly necessary and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Fired\_from\_NLP at SemEval-2024 Task 1: Towards Developing Semantic Textual Relatedness Predictor - A Transformer-based Approach}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Anik Mahmud Shanto, Md. Sajid Alam Chowdhury, Mostak Mahmud Chowdhury, \\ \bf Udoy Das, Hasan Murad\\
Department of Computer Science and Engineering\\
Chittagong University of Engineering and Technology, Bangladesh\\
\texttt u1904\{049, 064, 055\}@student.cuet.ac.bd, u1804109@student.cuet.ac.bd, \\ \texttt hasanmurad@cuet.ac.bd}

% \author{First Author \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\\And
%   Second Author \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\}

\begin{document}
\maketitle
\begin{abstract}
Predicting semantic textual relatedness (STR) is one of the most challenging tasks in the field of natural language processing. Semantic relatedness prediction has real-life practical applications while developing search engines and modern text generation systems. A shared task on semantic textual relatedness has been organized by SemEval 2024, where the organizer has proposed a dataset on semantic textual relatedness in the English language under Shared Task 1 (Track A3). In this work, we have developed models to predict semantic textual relatedness between pairs of English sentences by training and evaluating various transformer-based model architectures, deep learning, and machine learning methods using the shared dataset. Moreover, we have utilized existing semantic textual relatedness datasets such as the stsb multilingual benchmark dataset, the SemEval 2014 Task 1 dataset,  and the SemEval 2015 Task 2 dataset. Our findings show that in the SemEval 2024 Shared Task 1 (Track A3), the fine-tuned-STS-BERT model performed the best, scoring 0.8103 on the test set and placing 25\textsuperscript{th} out of all participants.
 
% The document itself conforms to its specifications and is therefore an example of what your manuscript should look like.
% These instructions should be used both for papers submitted for review and for final versions of accepted papers.
\end{abstract}

\section{Introduction}
%motivation
Nowadays, there have been notable advancements in understanding and measuring pairwise semantic relatedness between texts within the domain of natural language processing. Predicting semantic relatedness plays a significant role in improving search engines, question-answering systems, text summarization tools, and machine translation.

However, previous works in natural language processing have mainly dealt with semantic similarity, a smaller aspect of relatedness, mainly due to the limited availability of relatedness datasets.
Besides, dealing with ambiguous words or phrases that have multiple meanings can make semantic relatedness difficult. Understanding cultural context in language has been complex, and existing models have struggled to capture these variations. As language evolves, models struggle to adapt quickly to new linguistic patterns and expressions. To bridge these gaps, we need improved models that understand not just words but also context, cultural differences, and how language changes over time.

%need cites of several related works (3 lines)
Semantic relatedness models have been developed using various transformer-based, deep learning, and machine learning techniques. Traditional machine learning methods \cite{buscaldi-etal-2015-sopa} have relied on predefined rules and features and offered moderate results. These approaches have often struggled with complex semantic relationships. Deep learning-based \cite{wang2018deep} approaches have surpassed traditional machine learning models in capturing complex relationships, particularly in tasks requiring a deep understanding of context. However, transformer-based approaches \cite{devlin2018bert} have outperformed others when it comes to capturing semantic relationships, particularly in understanding context, managing long-range dependencies, and handling contextual embeddings.

SemEval has arranged a shared task named SemEval 2024 Task 1: Semantic Textual Relatedness (STR) \cite{semrel2024task}, introducing a novel dataset called Shared Task 1 (Track A3) \cite{semrel2024dataset} for determining the level of pairwise semantic relatedness between sentences based on the similarity score that ranges from 0.0 to 1.0.

%objective
The primary goal of this task is to build a robust and accurate model to predict the semantic relatedness between pairs of English sentences.

To accomplish this goal, we have used a variety of models, incorporating machine learning models (Linear Regression, Random Forest, XGBoost), models of deep learning (LSTM, BiLSTM), and pre-trained models based on transformer (RoBERTa, bert-base-uncased). We have named our approach of using the bert-base-uncased model as STS-BERT.

By training and assessing every model, we have carried out a comparison analysis on the Semeval 2024 Task 1 (Track A3) dataset \cite{semrel2024dataset}, STSB multilingual dataset \cite{huggingface:dataset:stsb_multi_mt}, SemEval 2014 Task 2 dataset \cite{marelli-etal-2014-semeval} and dataset provided for Task 1 in SemEval 2015 \cite{agirre-etal-2015-semeval} and have finally come to a conclusion that the STS-BERT model has demonstrated better performance compared to others boasting an impressive Spearman correlation coefficient of 0.81033 on the test dataset. 

% The BERT-STS model utilizes BERT-based embeddings to represent the input sentence pairs. It then computes the cosine similarity between the embeddings and employs a Mean Squared Error loss for training. The training process involves optimizing the model parameters by minimizing Mean Squared Error loss with AdamW optimizer and linear learning rate scheduling. The final result is a fine-tuned BERT model capable of predicting semantic similarity scores for given sentence pairs.

Key contributions of our research work are listed below -
\begin{itemize}
    \item We have developed a fine-tuned-STS-BERT model that significantly helps in accurately predicting semantic textual relatedness across diverse sentence pairs. 
    \item We have evaluated the model's performance through various tests conducted using the dataset and subsequently performed an in-depth evaluation of the outcomes.
\end{itemize}

The GitHub repository that follows has the implementation details available - \url{https://github.com/Fired-from-NLP/SemEval-2024-task-1-track-A-eng}. 



\section{Related Works}

The associated works on semantic textual similarity can be generally categorized into three parts, approaches focused on machine learning, deep learning, and attention-based mechanism (transformer).

Among machine learning models, the Support Vector Regression model has been applied for calculating the semantic relationship between two short sentences \cite{sultan-etal-2013-dls}. In this system, three distinct measures, namely overlap in word n-gram, overlap in character n-gram, and semantic overlap, have been used for predicting similarity. In \cite{buscaldi-etal-2015-sopa}, a Random forest-based approach has been utilized to find the semantic sentence similarity.  The approach has relied on various similarity measures such as WordNet-based conceptual similarity, IC-based similarity, syntactic dependencies, and information retrieval-based similarity.

Traditional deep learning methods have depended on single or multiple granularity representations for detecting similarity. Apart from that, a different architecture that has focused on multiple positional sentence representations has been proposed \cite{wang2018deep}. It has used Bi-LSTM for generating representations that enable the model to capture better context understanding. Another architecture has introduced a Siamese adaptation of LSTM \cite{Mueller_Thyagarajan_2016}. Using a fixed-sized vector and a simple Manhattan metric, the model transforms sentence representation that represents semantic relationships. Another paper has described an architecture that has been built using deep learning paradigms \cite{zhao-etal-2015-ecnu}. This architecture has been trained using a combination of features like features based on a string, features based on a corpus, and features based on syntactic similarity, as well as newer matrices derived from distributed word embedding.

Transformer-based approaches have surpassed both machine learning and deep learning models in calculating semantic sentence relationships. Unlabeled text can be used to pre-train deep bidirectional representations using BERT \cite{devlin2018bert}. BERT can be fine-tuned to do various NLP-related tasks like semantic analysis. A replication of BERT called RoBERTa \cite{Liu2019RoBERTaAR}, has focused on hyperparameters and training data size to improve model performance. 


In this shared task, we have used BERT-based pre-trained models as they have been proven to be superior to other models available.

\section{Dataset}
We have employed the dataset made available as part of Shared Task 1 (Track A3) of the SemEval 2024: Semantic Textual Relatedness (STR) which contains 5500 samples in the training dataset and 250 samples in the dev dataset. Besides, the stsb multilingual benchmark dataset \cite{huggingface:dataset:stsb_multi_mt}, the SemEval 2014 Task 1 dataset \cite{marelli-etal-2014-semeval} and the Semeval 2015 Task 2 dataset \cite{agirre-etal-2015-semeval} have been used.
%explain  Semantic Textual Relatedness (STR) and the brmson STS dataset

\begin{table}[h]
\centering
\begin{tabular}{|c|c|c|c|}
\hline
\multirow{2}{*}{Task} & \multicolumn{3}{|c|}{Sentence Pairs} \\ \cline{2-4}
 & Train & Validation & Test \\ \hline
SemEval 2014 & 4500 & 500 & 4928 \\ \hline
SemEval 2015 & 2997 & 750 & 6729 \\ \hline
stsb-multi-mt & 5749 & 1500 & 1379 \\ \hline
\end{tabular}
\caption{Data sizes for external datasets}
\label{tab:pards}
\end{table}

Table \ref{tab:pards} shows the distribution of samples that we have used from external datasets. These datasets have been merged to get a total of 32508 samples and then divided further into two sets: train and validation comprising 25676 and 6732 samples respectively. We have replaced the similarity score of duplicate sentence pairs with the average value to avoid labeling biases among different datasets. For the test dataset, The dataset made available as part of Shared Task 1 (Track A3), which consists of 2600 samples, has been used.
\begin{figure}[ht]
    \centering
    \includegraphics[width=7.6cm]{latex/ds1.png}
    \caption{Word distribution of sentence1}
    \label{fig:dataset1}
\end{figure}
\begin{figure}[ht]
    \centering
    \includegraphics[width=7.6cm]{latex/ds2.png}
    \caption{Word distribution of sentence2}
    \label{fig:dataset2}
\end{figure}
These datasets contain a pair of sentences in each row, which we have split into two separate sentences namely sentence1 and sentence2. Figure \ref{fig:dataset1} and Figure \ref{fig:dataset2} show that sentence1 contains an average of 6-12 words, while sentence2 contains 3-12 words.

\section{System Overview}
In this section, we have outlined our methodology to develop models for determining sentence relatedness. First, we have used various extraction strategies to extract characteristics and then utilized a variety of machine learning and deep learning algorithms. Moreover, we have employed different transformer models to develop the system. Figure \ref{fig:methodology} provides a summary of our working methods. 

\subsection{Machine Learning-based Approaches} 
For determining sentence relatedness, we have applied traditional Machine learning-based methods such as Linear Regression and Random Forest. Moreover, To increase the performance, we have employed an ensemble classifier called XGBoost. Here, we have tokenized the dataset using NLTKTokenizer, and then have we used Word2Vec to extract features. We also have used FastText for feature extraction as it not only captures semantic meaning like Word2Vec but also encodes subword information, allowing it to handle out-of-vocabulary words and morphologically rich languages more effectively. We have set the number of decision trees or boosting rounds to n\_estimators for the ensemble approach at 100.

\begin{figure}[]
    \centering
    \includegraphics[width=7cm, height=7cm]{latex/fig1.png}
    \caption{An outline of our approach}
    \label{fig:methodology}
\end{figure}


\subsection{Deep Learning-based Approaches} 
Deep learning-based models have been utilized for determining sentence relatedness. We have implemented both models based on LSTM and Bi-LSTM. Two LSTM layers with various numbers of LSTM cells have been applied to the LSTM model. Each of the two directional layers has 50 or 100 LSTM cells in it. We have employed two Bi-LSTM layers, each with 100 and 50 Bi-LSTM cells, in the Bi-LSTM model.
%All three models have been trained for 10 epochs with a batch size of 32, using mean squared error as the loss function.


\subsection{Transformer-based Approaches} 
Methods based on transformers are now widely employed in many different contexts. We have employed STS-BERT \cite{devlin2018bert} and RoBERTa to tackle this task. As the sentences can be diverse, having a single representation and better understanding of the sentences is very important. For this reason, we have used the feature vector of the pooling layer as shown in Figure \ref{fig:bert-model}. 
\begin{figure}[]
    \centering
    \includegraphics[width=5cm, height=7.5cm]{latex/fig2.png}
    \caption{STS-BERT: Transformer-based model architecture for predicting semantic textual relatedness }
    \label{fig:bert-model}
\end{figure}

In our approach, we have first split the pair of sentences in the dataset into two. We have used two bert-based-uncased \cite{devlin2018bert} for two sentences. We have obtained the feature vector from the pooling layer of these two Bert models. After obtaining the pooled embeddings, we feed them to the cosine similarity for performing the relatedness task and compare them with the ground truth relatedness score. Then, we compute the loss using MSE (Mean Squared Error) based on the predicted relatedness and the actual relatedness. After the loss is calculated to improve the performance and minimize the loss, we have updated the model parameters using gradient descent.

%Lastly, we use AdamW optimizer for effectiveness in transformers.  We utilize the adaptive learning rate mechanism and weight decay regularization to prevent the model from overfitting. To ensure improved convergence and stability during convergence, we have employed the learning rate scheduler as well. This allows the model to navigate the loss landscape more effectively.

\section{Experimental Setup}
This section gives a summary of our experimental setup while training and evaluating our model architectures for semantic textual relatedness. 

%introductory line
\subsection{Environment Setting}
% The simulation has been run on a personal computer equipped with an Intel Core i7-9700 CPU (3.00 GHz) and an NVIDIA GeForce GTX 2060 GPU. Furthermore, to guarantee sufficient processing power, a Kaggle Notebook configured with a P100 GPU was employed.

The simulation was executed on a personal computer featuring an Intel Core i7-9700 CPU clocked at 3.00 GHz and an NVIDIA GeForce GTX 2060 GPU. Additionally, to ensure ample processing capability, a Kaggle Notebook equipped with a P100 GPU was utilized.
\subsection{Data Preparation}
Besides the dataset provided in this competition, we have used three external datasets. We have used the stsb multilingual benchmark dataset \cite{huggingface:dataset:stsb_multi_mt}, the SemEval-2014 Task 1 dataset \cite{marelli-etal-2014-semeval}, and the SemEval-2015 Task 2 dataset \cite{agirre-etal-2015-semeval}. We have combined all three datasets. The similarity score of external datasets ranges from 0.0 to 5.0. However, the provided dataset for this competition holds the relatedness between sentences ranging from 0.0 to 1.0. We multiplied the relatedness score of the dataset offered in the competition by 5.0 to match the similarity score in the combined dataset. We have replaced the similarity score of duplicate sentence pairs with the average value. Then, we have split the combined dataset into the training dataset and the validation dataset. The final size of the training dataset is 25676, whereas the overall size of the validation dataset is 6732. We have used the test dataset provided in the competition. The test set contains 2600 samples. % test set

% In the external datasets we have used, the similarity score varies from 0.0 to 5.0. However, the dataset used for the competition had similarity scores ranging from 0.0 to 1.0. We have multiplied the similarity score by 5.0 in order to match the datasets.

\subsection{Parameter Settings}
Table \ref{tab:par} shows the parameter settings used in LSTM. BiLSTM, and RoBERTa models.

\begin{table}[ht]
\centering
\begin{tabular}{|l|l|l|l|l|}
\hline
\textbf{Model} & \textbf{lr} & \textbf{optim} & \textbf{bs} & \textbf{epoch}\\
\hline
%HBERT & \(2e^{-5}\) & adafactor & 16 & 0.01 & 0.1\\
%updated this for explaning parameters for both berts
% Linear Regression & \(2e^{-5}\) & Adam & - & - \\
% Random Forest & \(2e^{-5}\) & Adam & - & - \\
% XGBoost & \(1e^{-6}\) & Adam & - & - \\
LSTM & \(1e^{-6}\) & Adam & 32 & 10 \\
BiLSTM & \(1e^{-6}\) & Adam & 32 & 10 \\
RoBERTa & \(1e^{-6}\) & Adam & 32 & 12 \\
\hline
\end{tabular}
\caption{Parameter configurations for various models}
\label{tab:par}
\end{table}

In Table \ref{tab:par}, learning rate, optimizer, batch size, and number of epochs are represented by the variables lr, optim, bs, and epoch, in that order.\\
%explain table

Table \ref{tab:param} summarizes the parameter settings used in our proposed STS-BERT model.
\begin{table}[h]
    \centering
    \begin{tabular}{|l|l|}
    \hline
    \textbf{Parameter} & \textbf{Value} \\
    \hline
    Learning Rate & $1 \times 10^{-6}$ \\
    Optimizer & AdamW \\
    Batch Size & 8 \\
    Number of Epochs & 12 \\
    Loss Function & Mean Squared Error (MSE) \\
    Pooling & Mean Pooling \\
    \hline
    \end{tabular}
    \caption{Model parameter settings.}
    \label{tab:param}
\end{table}

% \end{itemize}
\subsection{Evaluation Metrics}
The instruction of Shared Task 1 of SemEval 2024  has been to use the Spearman correlation to evaluate the performance of our model using the test dataset. The mathematical representation of the Spearman correlation is provided in equation \ref{eq:Spearman}.  Besides, we have used Cosine similarity in our model to predict similarity between sentences. Equation \ref{eq:Spearman} presents the mathematical representation for Cosime similarity.
\begin{equation}
\rho = 1 - \frac{6\sum d_i^2}{n(n^2 - 1)}
\label{eq:Spearman}
\end{equation}
In this context, $\rho$ denotes the Spearman correlation coefficient.
$d_i$ represents the difference between the ranks of corresponding observations in the two variables, while $n$ indicates the total number of observations.
\begin{equation}
\text{cos\_sim}(A, B) = \frac{{\sum_{i=1}^{n} A_i \cdot B_i}}{{\sqrt{\sum_{i=1}^{n} A_i^2} \cdot \sqrt{\sum_{i=1}^{n} B_i^2}}}
\label{eq:cos-sim}
\end{equation}
Where, $ \text{cos\_sim}(A, B) $ is the cosine similarity between vectors $ A $ and $ B $. $ A_i $ and $ B_i $ denote the components of vectors of $ A $ and $ B $ respectively. $ n $ indicates the dimensionality of the vectors.
\section{Experimental Results}
In this section, we have showcased the experimental findings obtained during the training and evaluation stages of the proposed model for semantic textual relatedness prediction. 

Table \ref{tab:parameters} presents a comparative analysis of different types of models, evaluating their performance using the Spearman correlation coefficient on the test dataset.
\begin{table}[H]
\centering
\begin{tabular}{|c|c|c|c|}
\hline
\textbf{Category} & \textbf{Model} & \textbf{Embedding} & \textbf{Score} \\ \hline
\multirow{6}{*}{ML} & Linear & word2vec & 0.0507 \\ \cline{3-4}
& Regression & fasttext & 0.0507 \\ \cline{2-4}
& Random & word2vec & 0.1298 \\ \cline{3-4}
& Forest & fasttext & 0.1198 \\ \cline{2-4}
& \multirow{2}{*}{XGBoost} & word2vec & 0.3178 \\ \cline{3-4}
&  & fasttext & 0.2072 \\ \hline
\multirow{4}{*}{DL} & \multirow{2}{*}{LSTM} & word2vec & 0.445 \\ \cline{3-4}
& & fasttext & 0.420 \\ \cline{2-4}
& \multirow{2}{*}{BiLSTM} & word2vec & 0.4990 \\ \cline{3-4}
& & fasttext & 0.429 %\\ \cline{2-4}
% & \multirow{2}{*}{GRU} & word2vec & 0.3178 \\
% & & fasttext & 0.2072 
\\ \hline
\multirow{2}{*}{BERT} & RoBERTa & - & 0.749 \\ \cline{2-4}
& STS-BERT & - & \textbf{0.810} \\ \hline
\end{tabular}
\caption{Results of different models on the test dataset}
\label{tab:parameters}
\end{table}

Among the machine learning models, we have found that the XGBoost model with word2vec embedding has achieved the highest score of 0.3178. In the deep learning category, we have seen better performance as both LSTM and BiLSTM models have higher scores than the machine learning models. The BiLSTM model achieved a score of 0.499, slightly outperforming the LSTM model, which obtained a score of 0.445.

In some cases, Fasttext word embedding has obtained the best results compared to word2vec \cite{Meden2022}. Therefore, we have also tested the performance of the model using Fasttext embedding.
However, the transformer-based models have clearly outperformed other models based on machine learning and deep learning. For instance, the RoBERTa model achieved a score of 0.749, while our proposed STS-BERT model demonstrated exceptional performance with an impressive score of 0.810.

%training results (loss curses)
% \begin{figure}[ht]
%     \centering
%     \includegraphics[width=7.5cm]{latex/combined_plot.png}
%     \caption{Loss curve of the sts-bert-uncased model}
%     \label{fig:losscurve}
% \end{figure}
% %comparative study with table
% %Error Analysis

% % \begin{itemize}
% %     \item The model's performance is tracked over multiple epochs, and the results are stored in a DataFrame.
% % \end{itemize}

% Figure \ref{fig:losscurve} shows the loss vs epoch curve of the STS-BERT model during the training phase. 

\section{Error Analysis}
% udoy
% Our proposed system has achieved the Spearman score of 0.8103, prone to scores from 0.805 to 0.811. In the development phase external datasets, SemEval 2014 Task 1 \cite{marelli-etal-2014-semeval}, SemEval 2015 Task 2 \cite{agirre-etal-2015-semeval} and multilingual benchmark dataset \cite{huggingface:dataset:stsb_multi_mt} along with the competition dataset have been utilized. Hence the training set becomes more diverse and our model fails to learn about the relatedness between the sentences. The similarity scores of the external datasets ranged between 0.0 to 5.0. To make all the scores similar we have multiplied the scores of the competition dataset by 5.0 and normalized the whole training set by dividing all the scores by 5.0. Due to multiple conversions of the range of scores, precision loss has occurred. Our model has been unable to understand the context of two sentences and has focused on some specific words like name, place, etc. Sentence transformation has been another key reason for the poor performance of the model. When the second sentence is the transformation of the first sentence, our model cannot detect it. As a result, the overall performance of our proposed system has degraded.

 In the development phase external datasets, SemEval 2014 Task 1 \cite{marelli-etal-2014-semeval}, SemEval 2015 Task 2 \cite{agirre-etal-2015-semeval} and multilingual benchmark dataset \cite{huggingface:dataset:stsb_multi_mt} along with the competition dataset have been utilized. Hence the training set becomes more diverse and our model fails to learn about the relatedness between the sentences. The similarity scores of the external datasets ranged between 0.0 to 5.0. To make all the scores similar we have multiplied the scores of the competition dataset by 5.0 and normalized the whole training set by dividing all the scores by 5.0. Due to multiple conversions of the range of scores, precision loss has occurred. Sentence transformation has been another key reason for the poor performance of the model. When the second sentence is the transformation of the first sentence, our model can not detect it. For example, if the first sentence is in simple form and the second sentence is in the complex form of the first sentence, the model shows poor performance in that case. As a result, the overall performance of our proposed system has degraded.

\section{Conclusion}
In this research, we have conducted a comparative performance analysis, assessing a range of machine learning, deep learning, and transformer-based models to predict the semantic textual relatedness between pairs of English sentences. We have utilized the Task 1 (Track A3) dataset provided in the shared task, along with additional external datasets, for training various models. Our results indicate that the STS-BERT model has outperformed all other models, achieving an impressive score of 0.810. However, after analyzing errors, we have discovered that the slight score decrease is due to the integration of large external STS datasets with varying output ranges. To address this in future work, we plan to implement alternative strategies. Moreover, we will work on Task 1 (Track B and C) to have more comprehensive findings.

% it lacks discussion on the future scope of the research.

\section{Ethical Considerations}
To advance semantic text relatedness, we commit to emphasizing privacy through informed consent, reducing biases, as well as transparent modeling. Our ethical position prioritizes responsibility, accessibility, and privacy to build a positive and open technology environment.


\bibliography{custom}

% \appendix

% \section{Example Appendix}
% \label{sec:appendix}

% This is an appendix.

\end{document}
