% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}
\usepackage{booktabs}
\usepackage{adjustbox}

% Remove the "review" option to generate the final version.
\usepackage{ACL2023}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{hyperref}
% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}


% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Pinealai at SemEval-2024 Task 1: Exploring Semantic Relatedness Prediction using Syntactic, TF-IDF, and Distance-Based Features.}
% Semantic Textual Relatedness Prediction: An Experimental Exploration for Explainablity}
% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

%\author{First Author \\
%  Affiliation / Address line 1 \\
%  Affiliation / Address line 2 \\
%  Affiliation / Address line 3 \\
%  \texttt{email@domain} \\\And
%  Second Author \\
%  Affiliation / Address line 1 \\
%  Affiliation / Address line 2 \\
%  Affiliation / Address line 3 \\
%  \texttt{email@domain} \\}

% \author{Anvi Alex Eponon \and Luis Israel Ramos Perez \and Ildar Batyrshin\and Grigori Sidorov \and Olga Kolesnikova \and Hiram \\
%         Instituto Polit\'ecnico Nacional,\\Centro de Investigacio\'on en Computaci\'on, \\ Mexico City, Mexico\\\texttt{\{lramos2020, aeponon2023\}@cic.ipn.mx}}
\author{
  \textbf{Anvi Alex Eponon}$^{1}$,
  \textbf{Luis Ramos}$^{1}$, \\
  \textbf{Ildar Batyrshin}$^{1}$,
  \textbf{Grigori Sidorov}$^{1}$,
  \textbf{Olga Kolesnikova}$^{1}$,
  \textbf{Hiram Calvo}$^{1}$ \\
  \vspace{0.5em}
  $^{1}$Instituto Politécnico Nacional (IPN), Centro de Investigación en Computación (CIC),\\ Mexico City, Mexico \\\texttt{\{aeponon2023, lramos2020, ibatyr1, sidorov, kolesnikova, hcalvo\}@cic.ipn.mx}
}

% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\

\begin{document}
\maketitle
%\begin{abstract}
%NAACL 2023.
%\end{abstract}


\begin{abstract}

The central aim of this experiment is to establish a system proficient in predicting semantic relatedness between pairs of English texts. Additionally, the study seeks to delve into diverse features capable of enhancing the ability of models to identify semantic relatedness within given sentences.  Several strategies have been used that combine TF-IDF, syntactic features, and similarity measures to train machine learning models to predict semantic relatedness between pairs of sentences. The results obtained were above the baseline with an approximate Spearman score of 0.84. 
\end{abstract}

\section{Introduction}
The prediction of semantic relatedness between texts is a crucial task with applications in various natural language processing domains. In this study, our focus is on creating a system capable of predicting semantic relatedness across languages while investigating the features that contribute to this prediction. The development of such a system is not only beneficial for understanding semantic relationships within texts but also holds promise for enhancing deep learning models in tasks such as assessing sentence representation methods, question answering, and text summarization \cite{abdalla-etal-2023-makes}.

Despite the advancements in word representation techniques, especially using embeddings, the complexity of human languages presents persistent challenges in accurately capturing semantic relatedness. Our experiment primarily concentrates on the English language, acknowledging its significance as a widely used language in various applications. The inherent difficulty in identifying and quantifying the shared elements between two texts necessitates a thoughtful exploration of diverse features and methodologies.

In the subsequent sections, we describe the dataset used for this experiment, outline the methodology employed for predicting semantic relatedness, discuss the results obtained, and conclude with insights into the implications and potential future directions of this research. The research will delve into exploring features that enhance the prediction of semantic textual relatedness by developing several strategies concerning feature extractions and model training.



\section{Literature Review}
%Semantic Textual Relatedness
The complexity of machine-based human language modeling involves a nuanced understanding of various linguistic aspects, notably Pragmatics and Semantics \cite{abdalla2021makes, miller-1995}. This research specifically emphasizes semantic modeling, with a focus on semantic relatedness, as opposed to the more commonly studied word similarity \cite{islam-2012, atoum-2016, yum-2021}.

Traditionally, approaches like Bag of Words have been explored \cite{islam-2012, feng-f8}, but they often fall short in achieving high performance for semantic relatedness tasks. WordNets models, while prioritized, face limitations in language coverage and comprehensive embedding of semantic relationships \cite{jordan-b05}.

A notable contribution by \cite{gomaa2019multi} introduced a model utilizing multiple similarity features, including cosine similarity and Jaccard. Their multi-layer architecture demonstrated that employing various similarity features collectively yields significantly better results than applying each measure in isolation. However, the approach did not add or consider syntactic features for the enhancement of the semantic prediction on textual data.

Recent advancements in deep learning models exhibit superior semantic similarity and relatedness performance. However, there remains a scarcity of research focusing on the distinctive features between Semantic Textual Similarity (STS) and Semantic Textual Relatedness (STR), and how models can better capture the nuances of semantic relatedness between words and sentences \cite{peter-kolb-09}. 

\section{Task Description}
The primary objective of this experiment is to create a system that can predict the semantic relatedness between pairs of texts across various languages but also explore the different features that could help models identify semantic relatedness between given sentences. Although the current experiment is focused on English, the development of such a system holds the potential to enhance deep learning models for various tasks, including assessing sentence representation methods, question answering, and text summarization \cite{abdalla-etal-2023-makes}. 

\section{Data Description}

SemEval 2024 Track 1 utilized data provided by organizers, featuring sentence pairs in training, development, and test sets. Each instance is annotated with a score indicating semantic textual relatedness, which ranges from 0 (unrelated) to 1 (related). Table \ref{tab:data_info} presents statistics about the dataset, while Figure \ref{fig:scores_comparison} illustrates the distribution of scores, including the counts for scores of 1.0, 0.0, >0.80 and <0.50, in the training and development set.
%...
\begin{table}[htbp]
    \centering
    \caption{Dataset Information}
    \begin{adjustbox}{width=\columnwidth}
    \begin{tabular}{@{}llll@{}}
    \toprule
    \textbf{Dataset} & \textbf{Total Pairs} & \textbf{Pairs with Score 1.0} & \textbf{Pairs with Score 0.0} \\ \midrule
    Training & 5500 & 25 & 5 \\
    Development & 250 & 7 & 123 \\
    Test & 2600 & - & - \\ \bottomrule
    \label{tab:data_info}
    \end{tabular}
    \end{adjustbox}
\end{table}

\begin{figure}[h!]
  \centering
  \begin{subfigure}[b]{0.8\linewidth} 
    \centering
    \includegraphics[width=\linewidth]{train_score.png}
    \caption{Number of strict 1.0 and 0.0 in the Train set.} 
    \label{fig:train_scores}
  \end{subfigure}
  \hspace{0.05\linewidth}
  \begin{subfigure}[b]{0.8\linewidth}
    \centering
    \includegraphics[width=\linewidth]{dev_score.png}
    \caption{Number of scores greater than 0.80 and lesser than 0.5 in the Dev set.}
    \label{fig:dev_scores}
  \end{subfigure}
  \caption{Train and dev scores comparison.}
  \label{fig:scores_comparison} 
\end{figure}

%...
Table \ref{tab:data_samples} displays sample instances, with additional details available in \cite{abdalla2021makes, ousidhoum-etal-2024-semeval, ousidhoum2024semrel2024}.

\begin{table}
\centering
\small
\begin{tabular}{lc}
\hline
\textbf{Instance} & \textbf{Score}\\
\hline
It that happens, just pull the plug.&1.0\\
if that ever happens, just pull the plug.&\\
&\\
The two little girls jump on the bed.&0.5\\ 
A little girl is jumping down a sandy hill.&\\ 
&\\
you're taking a sweater in a shop.&0.03\\ 
to taking the life of Conor Greenleaf.&\\\hline
\end{tabular}
\caption{Sample instances of data set}
\label{tab:data_samples}
\end{table}

\section{Methodology}
In this study, our objective is to predict semantic textual relatedness between two texts. We made two key assumptions: firstly, we refrained from preprocessing the corpus to preserve sentence structure, essential for information retrieval and semantic identification \cite{hirst-1987}. 

Secondly, we intentionally excluded Large Language Models (LLMs) from experiments, anticipating challenges in interpreting specific features contributing to semantic identification due to their contextual abilities and complexity \cite{turton-2020}. To extract diverse features, we employed various distance measures, including Jaccard distance, Cosine similarity, Levenshtein distance, and Word Mover’s Distance (WMD) \cite{boubacar2014valuing, kusner2015word, su2008plagiarism}. These measures compute the similarity between text pairs based on common words, term vectors, and the minimum distance embedded words need to travel between documents. Feature extraction utilized the bag-of-words (BoW) technique, specifically Term-Frequency Inverse Document Frequency (TF-IDF) \cite{hakim2014automated}, and hidden vectors from a pre-trained SentenceBert model to compute cosine similarity \cite{reimers-2019-sentence-bert}. 

Additionally, syntactic features were extracted, parsing each sentence pair to identify words with the same dependency role. This approach resulted in three features: probability of exact word matching, probability of unique words, and probability of related words. The rationale was to explore the impact of words with common dependency roles on predicting semantic relatedness and assess the effect of their absence on the English corpus.

The reason behind using traditional models such as Gradient Boost is that this type of model can easily handle non-linear relationships in texts, which is crucial while capturing semantic relatedness but also can deal with imbalanced datasets \cite{natekin2013gradient}.

A diagram of our methodology as well as the source code is freely available on GitHub. The link can be found in the Appendix section.

\section{Results}

The performance metrics of the different models submitted for semantic relatedness are presented. The evaluation metrics include solely the Spearman score. Even though the models submitted were Fasttext and Naive Bayes, during the training phase Linear Regression, XGradient Boost, Random Forest, and an ensemble of two traditional models were trained.

\subsection{Train phase}
During the training phase, several models were tested using different strategies, and in Table \ref{tab:train_metrics} we display the performance for each model. The main strategy that gave the results presented in this document has been explained earlier in the methodology section.

\begin{table}[htbp]
  \centering

  \begin{tabular}{lccc}
    \hline
    \textbf{Model} & \textbf{Spearman}  \\
    \hline
    Linear Reg. & 0.8512 \\
    \textbf{Gradient Boost} & \textbf{0.8527} \\
    XGB & 0.8467 \\
    Random Forest & 0.8481 \\
    Ensemble Model & 0.8521 \\
    \hline
  \end{tabular}
  \caption{Training Model Performances}
  \label{tab:train_metrics}
\end{table}


\subsection{Development phase}
At this stage of the experiment, the same models were tested on the development dataset which comprised of few number of samples, precisely 250 different instances. In Table \ref{tab:dev_metrics} we display the performance over the development set. 

\begin{table}[htbp]
  \centering

  \begin{tabular}{lccc}
    \hline
    \textbf{Model} & \textbf{Spearman}  \\
    \hline
    Linear Reg. & 0.8512 \\
    \textbf{Gradient Boost} & \textbf{0.8527} \\
    XGB & 0.8467 \\
    Random Forest & 0.8481 \\
    Ensemble Model & 0.8521 \\
    \hline
  \end{tabular}
  \caption{Development Model Performances}
  \label{tab:dev_metrics}
\end{table}

\subsection{Test phase}
At the final stage of the experiments, the Gradient Boost model was chosen for the final tests on the testing dataset which comprised 5000 different instances. In Table \ref{tab:test_metrics}, we display the results of the Gradient Boost models, compared to the baseline and the highest performance model in the same task.

\begin{table}[htbp]
  \centering
  \begin{tabular}{lccc}
    \hline
    \textbf{Team} & \textbf{Spearman} & \textbf{Rank}\\
    \hline
    PALI & 0.8595 & 1\\
    \textbf{Pinealai} & \textbf{0.8371} & 10\\
    SemRel-Baseline & 0.8300 & * \\
    \hline
  \end{tabular}
  \caption{Final Model Performance Metrics}
  \label{tab:test_metrics}
\end{table}

\section{Discussions}
In the methodology section, our primary objective is to identify key features that enhance the capability of models in discerning semantic relatedness within textual data. We pursued two distinct approaches. First, we trained various models by employing Tfidf, Jaccard, or extracting specific syntactic features from the texts. This process does not take into account the internal structures of the texts which could give more insights about their meaning.

When exclusively utilizing syntactic features for model training, we achieved a maximum Spearman score of 0.32. Training models solely that used TF-IDF features to compute a cosine similarity and used the metric to predict yielded a separate score of 0.533. Incorporating features extracted from Sbert on top of the syntactic features resulted in a notable score increase of 2, reaching approximately 0.70. Finally, combining all these strategies during the training phase produced a score of 0.85, with a corresponding score of 0.83 during the testing phase.

\section{Conclusion}
In conclusion, the study aimed to predict semantic relatedness in English, exploring diverse features and strategies. Limitations included the absence of word sense disambiguation algorithms, the exclusion of Transformer models for explainability, and the decision not to merge training and development sets. Despite these constraints, the models exhibited competitive performance, particularly the Gradient Boost model, which achieved a Spearman score of 0.8371. However, the experiment conducted can not help us derive a conclusion that the model can make a strict difference between semantic relatedness (STR) and semantic textual similarity (STS) in texts. The methodology highlighted the impact of syntactic features, TF-IDF representations, and SentenceBert embeddings. Moving forward, addressing limitations, incorporating advanced algorithms, and leveraging diverse datasets but also developing approaches that help models distinguish between STR and STS will contribute to a deeper understanding of semantic relations in textual data and further improvements in predictive capabilities.

\section*{Limitations}

The study of semantic relatedness is a vast and tedious endeavor. The research conducted was very limited in many aspects. Firstly, the absence of algorithms specifically targeting direct word sense disambiguation represents a notable limitation. The incorporation of such algorithms could have potentially enhanced the models' effectiveness in this particular task. Also, the research did not explore the preprocessing techniques that could positively impact the semantic relatedness prediction.

Secondly, our study was confined to the training and testing of traditional machine learning models, excluding the exploration of Transformer models. While Transformers might have yielded superior results, their reduced explainability deterred their inclusion in our investigation.


Lastly, the decision not to merge the training and development sets for a final model training phase or add more datasets related to semantics relatedness or even semantic similarity represents another constraint. By solely transitioning to the testing phase with the models having learned solely from the training set given by the organizers, we may have missed opportunities for improvement. Combining both sets or augmenting the datasets through specific techniques in the final training phase could have potentially elevated the models' predictive capabilities, resulting in a more accurate score.

\section*{Ethics Statement}
We affirm our commitment to ethical research practices and compliance with ACL guidelines in conducting and presenting our study. No ethical concerns or conflicts of interest arose during this research.

\section*{Acknowledgements}
The work was done with partial support from the Mexican Government through the grant A1-S-47854 of CONACYT, Mexico, grants 20241816, 20241819, and 20240951 of the Secretaría de Investigación y Posgrado of the Instituto Politécnico Nacional, Mexico. The authors thank the CONACYT for the computing resources brought to them through the Plataforma de Aprendizaje Profundo para Tecnologías del Lenguaje of the Laboratorio de Supercómputo of the INAOE, Mexico, and acknowledge the support of Microsoft through the Microsoft Latin America PhD Award.

% Entries for the entire Anthology, followed by custom entries
\bibliography{anthology,custom}
\bibliographystyle{acl_natbib}

\section*{Appendix}
The diagram of our method and the source code can be found on GitHub at this URL: 
\href{https://github.com/Anvi98/semEval2024_code}{semEval2024 code}
\end{document}
