% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

\usepackage{adjustbox}
\usepackage{tikz}
\usetikzlibrary{shapes,arrows,positioning}


% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\usepackage{graphicx}

\title{All-MPNet at SemEval-2024 Task 1: Application of MPNet for Evaluating Semantic Textual Relatedness}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Marco Siino \\
Department of Electrical, Electronic \\and
Computer Engineering \\
  University of Catania \\
  Italy \\
  \texttt{marco.siino@unipa.it} }

\begin{document}
\maketitle
\begin{abstract}
In this study, we tackle the task of automatically discerning the level of semantic relatedness between pairs of sentences. Specifically, Task 1 at SemEval-2024 involves predicting the Semantic Textual Relatedness (STR) of sentence pairs. Participants are tasked with ranking sentence pairs based on their proximity in meaning, quantified by their degree of semantic relatedness, across 14 different languages. To each sentence pair is assigned a manually determined relatedness score ranging from 0 (indicating complete lack of relation) to 1 (denoting maximum relatedness). In our submitted approach on the official test set, focusing on Task 1 (a supervised task in English and Spanish), we achieve a Spearman rank correlation coefficient of 0.808 for the English language and of 0.611 for the Spanish language.

\end{abstract}

\section{Introduction}


The notion of semantic relatedness between language units has been foundational in understanding meaning. Automatic determination of relatedness has wide-ranging applications, including assessing sentence representation methods, question answering, and summarization \citep{guarino1997semantic}.

Two sentences are deemed semantically similar when they exhibit a relationship of entailment or paraphrases. In contrast, relatedness encompasses a broader concept, encompassing all commonalities between two sentences: whether they pertain to the same topic, convey the same viewpoint, stem from the same temporal context, one elaborates on the other, and so forth \cite{hadj2020survey}. 

Historically, much of NLP research has focused on semantic similarity, predominantly in English. However, in the shared Task 1 hosted at SemEval 2024 \citep{ousidhoum2024semeval,ousidhoum2024semrel2024}, the organizers extend their coverage to include the following languages: Afrikaans, Algerian Arabic, Amharic, English, Hausa, Hindi, Indonesian, Kinyarwanda, Marathi, Moroccan Arabic, Modern Standard Arabic, Punjabi, Spanish, and Telugu. 


With the advancement of machine and deep learning architectures in recent years, there has been a surge of interest in NLP. Numerous efforts have been dedicated to creating algorithms capable of automatically identifying and categorizing text information available on the internet. In the literature, several strategies have already been proposed. In the last fifteen years, some of the most successful strategies have been based on SVM \cite{colas2006comparison,croce2022svm}, on Convolutional Neural Network (CNN) \cite{kim2014convolutional,siino2021detection}, on Graph Neural Network (GNN) \cite{lomonaco2022courage}, on ensemble models \cite{miri2022ensemble,siino2022t100} and, recently, on Transformers \cite{vaswani2017attention, siino2022mcrock}.

The increasing adoption of Transformer-based architectures in academic research has also been bolstered by various methodologies showcased at SemEval 2024. These methodologies tackle diverse tasks and yield noteworthy findings. For instance, at the Task 2 \citep{jullien-etal-2024-semeval}, where to address the challenge of identifying the inference relation between a plain language statement and Clinical Trial Reports is used T5 \cite{siino2024t5medical}; Task 4 \citep{dimitrov2024semeval} where is employed a Mistral 7B model to detect persuasion techniques in memes \cite{siino2024mcrock}; and Task 8 \citep{wang2024semeval}, that utilizes a DistilBERT model to identify machine-generated text \cite{siino2024badrock}.

For our model development, we devised a two-stage architecture. In the first stage, we utilized a Sentence Transformer specifically trained in a multilingual domain. Subsequently, we computed the cosine similarity of the generated embeddings to predict the relatedness between the analyzed sentences.

The remainder of this paper is structured as follows: Section \ref{sec:background} provides background information on Task 1 hosted at SemEval-2024. Section \ref{sec:system_overview} presents an explanation of the submitted approach. We detail the experimental setup required to reproduce our work in Section \ref{sec:experimental_setup}. The results of the formal assignment and pertinent discussions are presented in Section \ref{sec:results}. Finally, we conclude with our findings and suggestions for future research in Section \ref{sec:conclusion}.


We make all the code publicly available and reusable on GitHub\footnote{\raggedright\url{https://github.com/marco-siino/SemEval2024/tree/main/Task\%201}}.

\section{Background}
\label{sec:background}


Data for Semantic Textual Relatedness (STR) Shared Task 1\footnote{\url{https://semantic-textual-relatedness.github.io/}} includes sentence pairs labeled with scores representing the degree of semantic textual relatedness between them, ranging from 0 (completely unrelated) to 1 (maximally related). These scores have been determined through manual annotation using a comparative annotation approach to mitigate biases commonly associated with traditional rating scale methods. This annotation process ensures a high reliability of the relatedness rankings.

The task involves predicting the STR of sentence pairs in 14 different languages. Participating teams were asked to submit systems for one, two, or all of the following tracks:

\begin{itemize}
    \item Track A: Supervised — Systems trained using labeled training datasets provided. Additional publicly available datasets can be used, but teams must report the additional data and its impact on results.
  
    \item Track B: Unsupervised — Systems developed without using labeled datasets related to semantic relatedness or similarity between text units longer than two words in any language. Use of unigram or bigram relatedness datasets is permitted.

    \item Track C: Cross-lingual — Systems developed without using labeled semantic similarity or relatedness datasets in the target language, but with the use of labeled dataset(s) from at least one other language. Using labeled data from another track is mandatory for submissions to this track.
\end{itemize}



\section{System Overview}
\label{sec:system_overview}


\begin{figure*}[!ht]
    \centering
    \resizebox{\linewidth}{!}{%
\begin{tikzpicture}[node distance = 2cm, auto]
    % Define styles
    \tikzstyle{block} = [rectangle, draw, text width=3cm, text centered, rounded corners, minimum height=1cm]
    \tikzstyle{line} = [draw, -latex']
    
    % Place nodes
    \node [block] (dataset) {dataset};
    \node [block, right=2cm of dataset] (allmpnet) {All-MPNet};
    \node [block, right=2cm of allmpnet, text width=2cm, align=center] (cos_sim) {cos\_sim};
    
    % Draw arrows
    \draw [line] (dataset) ++(2.2,0.2) -- +(1,0) node[midway,above] {"The book"};
    \draw [line] (dataset) ++(2.2,-0.2) -- +(1,0) node[midway,below] {"A cat"};
     \draw [line] (allmpnet) ++(2,0.2) -- +(1.5,0) node[midway,above] {embed\_1};
    \draw [line] (allmpnet) ++(2,-0.2) -- +(1.5,0) node[midway,below] {embed\_2};
    \draw [line] (cos_sim)  ++(1.3,-0.) -- +(1,0) node[midway,above] {0.19}; %++(2.2,0);
\end{tikzpicture}
}
 \caption{Block diagram of our proposed approach. Given a sample from the dataset, the first sentence is encoded with an All-MPNet embedding for generating an embedding vector as output. Also, the second sentence in the sample is encoded in the same way. Then the two embedding vectors are compared using cosine similarity to produce the final prediction on the semantic similarity of the two sentences.}
    \label{fig:blockdiagram}
\end{figure*}

The illustration of the proposed approach is provided in the Figure \ref{fig:blockdiagram}. Upon selecting a sample (i.e., a pair of sentences) from the dataset, the initial step involves encoding the first sentence using the All-MPNet embedding, thereby generating an embedding vector. Subsequently, employing an identical procedure, the second sentence from the sample is also encoded. The resulting embedding vectors are then subjected to a cosine similarity computation, facilitating the derivation of the semantic similarity prediction between two sentences.



To develop our model, we thought of a two-stage architecture. In the first stage, we used a \textit{Sentence Transformer}. This is a Python framework for cutting-edge sentence, text, and image embeddings. The initial work is described in \cite{reimers-2019-sentence-bert}. More than 100 languages have sentences and text embeddings that can be computed using this method. Sentences with a similar meaning can subsequently be found by comparing these embeddings, for example, using cosine-similarity. Semantic search, paraphrase mining, and semantic textual similarity can all benefit from these embeddings. The framework offers a huge selection of pre-trained models suited for different tasks and is built on PyTorch and Transformers. Moreover, fine-tuning models is also feasible.


The model used as Sentence Transformer is \textit{all-mpnet-base-v2}, and it is available on HuggingFace\footnote{\url{https://huggingface.co/sentence-transformers/all-mpnet-base-v2}}. The model is based on MPNet \cite{song2020mpnet}. MPNet introduces a novel pre-training approach that combines the strengths of BERT and XLNet while addressing their respective limitations. Unlike BERT's masked language modeling (MLM), MPNet utilizes permuted language modeling (PLM) to capture dependencies among predicted tokens more effectively. Additionally, MPNet incorporates auxiliary position information as input, allowing the model to process full sentences and mitigate position discrepancy issues present in XLNet. Pre-training of MPNet is conducted on a large-scale dataset exceeding 160 GB  of text corpora, followed by fine-tuning on various downstream tasks such as GLUE \citep{wang2018glue} and SQuAD \citep{rajpurkar2016squad}. Experimental findings demonstrate that MPNet significantly outperforms both MLM and PLM, achieving superior results across these tasks compared to previous state-of-the-art pre-trained models like BERT, XLNet, and RoBERTa, all under the same model configuration.


The model was used to map all the words present in the text to a word embedding space. Following the embeddings, the cosine distance between two sentences was calculated. The cosine similarity between the two embedding vectors is calculated as shown in the Equation \ref{eq:cosine_distance}.

\begin{equation}\label{eq:cosine_distance}
\cos (\theta ) =   \frac {A \cdot B} {\left\| A\right\| _{2}\left\| B\right\| _{2}} 
\end{equation}

The value provided as cosine similarity was then provided as the requested prediction for the two sentences considered.
Our code is available online together with the predictions generated and sent in relation to the test set.

The recent study by \cite{siino2024text} highlights that preprocessing for text classification tasks lacks significant impact when employing Transformers. Specifically, the study finds that the optimal preprocessing strategies do not substantially differ from performing no preprocessing at all, particularly in the case of Transformers. Consequently, in order to maintain our system's efficiency, speed, and computational lightness, we have opted to not conduct any preprocessing on the text. This decision aligns with the findings of the study and underscores the effectiveness of Transformers in handling raw text data without the need for extensive preprocessing steps.


\section{Experimental Setup}
\label{sec:experimental_setup}


We implemented our model on Google Colab\footnote{\url{https://colab.research.google.com/}}. The library we used was Sentence Transformer. The library requires Python\footnote{\url{https://www.python.org/}} (>= 3.8) and PyTorch\footnote{\url{https://pytorch.org/}} (>=1.11.0). The dataset provided for all the phases are available on the official competition page.   On the basis of our preliminary experiments, we found beneficial to set the threshold value upon the cosine similarity equal to 0.5. We did not perform any additional fine-tuning on the MPNet embeddings. To run the experiment, a T4 GPU from Google has been used. After the generation of the predictions, we exported the results on the JSON format required by the organizers. As already mentioned, all of our code is available on GitHub.


\section{Results}
\label{sec:results}

The official evaluation metric for this task is the Spearman rank correlation coefficient, which evaluates how closely the rankings predicted by the system align with human judgments. The evaluation script for this shared task is available on the GitHub page, providing a standardized method for assessing the performance of participating systems. The formula to compute the Spearman correlation coefficient is provided in the Equation \ref{eq:spearman}.



\begin{equation}
\rho = 1- {\frac {6 \sum d_i^2}{n(n^2 - 1)}}
\label{eq:spearman}
\end{equation}

Where \textit{d} represents the pairwise distances of the ranks of the variables $x_i$ and $y_i$, while \textit{n} is the number of samples.


In Table \ref{tab:mpnet_results}, are reported the results obtained on the two languages we considered for our participation at the task. Considered the very low effort required to run the proposed approach and to generate the predictions, the score of 0.611 and 0.808 appears to be an interesting baseline, while still exhibiting room for improvements. It is worth noticing that the approach is a Zero-Shot one with no prior knowledge on the specific task.

\begin{center}
\begin{table}
\begin{center}
\begin{tabular}{ cccc } 
    \hline
  LANGUAGE & Score \\ 
  \hline
  English & 0.808 \\ 
  \hline
   Spanish & 0.611 \\ 
  \hline
\end{tabular}
\end{center}
\caption{\label{tab:mpnet_results}The suggested method's performance on the test set. Our results are related to our participation in the Track A, for the English and for the Spanish languages only.}
\end{table}
\end{center}

In the Table \ref{tab:en_competition_results} and \ref{tab:es_competition_results}, the results obtained by the first three teams and by the last one, as showed on the official CodaLab page, are reported. Furthermore, we reported the baselines for the two languages. Compared to the best performing models, our simple approach exhibits some room for improvements. However, it is worth notice that it required no further pre-training and the computational cost to address the task is manageable with the free online resources offered by Google Colab.


\begin{center}
\begin{table}
\begin{center}
\begin{tabular}{ cccc } 
    \hline
  TEAM NAME & Score \\ 
  \hline
  PALI (1) & 0.859 \\ 
  \hline
  UAlberta (2) & 0.853 \\ 
   \hline
  Tübingen-CL (3) & 0.850 \\
  \hline
   SemRel-SemEval Baseline (*) & 0.830 \\ 
  \hline
  YNUNLP2023 (36) & 0.557 \\ 
  \hline
\end{tabular}
\end{center}
\caption{\label{tab:en_competition_results}Comparing performance on the test set for the English language. In the table are shown the results obtained by the first three users and by the last one. In parentheses is reported the position in the official ranking.}
\end{table}
\end{center}


\begin{center}
\begin{table}
\begin{center}
\begin{tabular}{ cccc } 
    \hline
  TEAM NAME & Score \\ 
  \hline
  AAdaM (1) & 0.740 \\ 
  \hline
  GIL-IIMAS UNAM (2) & 0.731 \\ 
   \hline
  PALI (3) & 0.724 \\
  \hline
  SemRel-SemEval Baseline (*) & 0.7 \\
  \hline
 YNUNLP2023 (25) & 0.404 \\ 
  \hline
\end{tabular}
\end{center}
\caption{\label{tab:es_competition_results}Comparing performance on the test set for the Spanish language. In the table are shown the results obtained by the first three users and by the last one. Furthermore, the baseline is also provided. In parentheses is reported the position in the official ranking.}
\end{table}
\end{center}

Even if our approach is simple and straightforward, here we want to qualitative analyze some results obtained with our approach to better motivate some classification mistakes. With regard to the English language, the first relevant misclassification sample is related to the sample pair: “This book is very compelling. -- best book so far in the series!”. Our system predicts a cosine similarity equal to 0.47 while the actual similarity is 0.73. In this case, in fact, it is very hard to assess if “being very compelling” is so semantically similar to the concept of “being the best so far in a series”. Furthermore, looking at the sample: “A woman in a black coat eats dinner while her dog looks on. -- A little boy is standing on the street while a man in overalls is working on a stone wall.”, the prediction using our approach is equal to 0.0 (no semantic similarity) while the actual target for the provided test set is 0.29. Another interesting case — i.e., our approach predicts a high similarity of 0.92 while the actual target is 0.74 — is given by the sample: “My favorite by far is definitely Chris and I think he will win!! -- My favorite and the selection for winner is Chris.”. From a semantic perspective, however, both the sentences provide the same two concepts (i.e., Chris is my favorite, I think he will win). Given these and several others differences between our predictions and the actual target similarity in the provided test set, some concerns on the labelling process and on the correctness of the provided target similarity values can be raised.




\section{Conclusion}
\label{sec:conclusion}


This paper introduces the utilization of an All-MPNet model embedding to tackle Task 1 at SemEval-2024. In our submission, we opted for a straightforward Zero-Shot learning approach, leveraging pre-trained Transformers that are already tailored to a multilingual-domain. Following this approach, we utilized the contextual embeddings generated by the Sentence Transformer, and we employed cosine distance to measure the similarity between pairs of sentences, thus quantifying the STR between them. Despite the effectiveness of our method, there remains room for improvement, as indicated by the final ranking. Potential alternative approaches could involve leveraging the zero-shot capabilities of models such as GPT and T5, expanding the training data size by incorporating additional datasets, or exploring different methods of integrating ontology-based domain knowledge into our approach.
 Furthermore, given the interesting results recently provided on a plethora of tasks, also few-shot learning \cite{wang2023few, maia2024transformers, siino2023profiling, meng2024hierarchical} or data augmentation strategies \cite{muftie2023indobert, siino2024backtranslate, tapia2020data, siino2023xlnet} could be employed to improve the performance.
Compared to the best performing models, our simple approach exhibits some room for improvements. However, our qualitative analysis raised some concerns on the labels provided for the test set. Then, we are not able to correctly assess the actual performance of our proposed approach. Eventually, it is worth notice that thanks to our approach no further pre-training is required and the computational cost to address the task is manageable with the free online resources offered by Google Colab.



\section*{Acknowledgments}
We would like to thank anonymous reviewers for their comments and suggestions that have helped to improve the presentation of the paper.

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

%\appendix

%\section{Example Appendix}
%\label{sec:appendix}
%This is an appendix.

\end{document}
