% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.


\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage[]{ACL2023}

% Standard package includes
\usepackage[export]{adjustbox}
\usepackage[utf8x]{inputenc}
\usepackage{graphicx}
\usepackage{times}
\usepackage{latexsym}
\usepackage{longtable}
\usepackage{tabularx}
\usepackage{array,multirow,graphicx}
\usepackage{tabularray}
\usepackage{floatpag}
\usepackage{float}
% \usepackage{adjustbox} % for adjusting table width
\usepackage{booktabs} % for better-looking tables


% \usepackage{showframe} % to visualize writing area and margins ...
% \usepackage{multicol}
\usepackage{hhline}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
%\usepackage[utf8]{inputenc}
\usepackage[utf8x]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

\input{tables}
% If the title and author information does not fit in the area allocated, uncomment the following
%
\setlength\titlebox{6cm}
%
% and set <dim> to something 5cm or larger.

\title{HausaNLP at SemEval-2024 Task 1: Textual Relatedness Analysis for Semantic Representation of Sentences}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}
% Lukman Aliyu, Nur Bala Rabiu, Mahmoud Said Ahmad, Idi Mohammed, Aliyu Rabiu Shuaibu, Alamin Musa, Auwal Shehu Ali and Zedong Nie

\author{Saheed Abdullahi Salahudeen$^{1,2}$, {\bf Falalu Ibrahim Lawan$^{2}$}, Aliyu Yusuf$^{3}$,\\ 
\bf Amina Abubakar Imam$^{4}$, Lukman Aliyu, Nur Bala Rabiu$^{5}$, Mahmoud Said Ahmad, \\
\bf Idi Mohammed$^{6}$, Aliyu Rabiu Shuaibu$^{7}$, Alamin Musa,\\
{\bf Auwal Shehu Ali$^{8}$, Zedong Nie$^{1}$} \\[2mm]
  $^{1}$Shenzhen Institute of Advanced Technology, CAS, $^{2}$Kaduna State University, \\ $^{3}$Universiti Teknologi PETRONAS, $^{4}$University of Abuja, $^{5}$Khalifa Isyaka Rabiu University,\\ $^{6}$AUST, Abuja $^{7}$Nile University,  $^{8}$Bayero University Kano,
  $^\forall$HausaNLP. \\[2mm]
  \texttt{Contact: zd.nie@siat.ac.cn} \\}

\begin{document}
\maketitle
\begin{abstract}
 Semantic Text Relatedness (STR), a measure of meaning similarity between text elements, has become a key focus in the field of Natural Language Processing (NLP). We describe SemEval-2024 task 1 on Semantic Textual Relatedness featuring three tracks: supervised learning, unsupervised learning and cross-lingual learning across African and Asian languages including Afrikaans, Algerian Arabic, Amharic, Hausa, Hindi, Indonesian, Kinyarwanda, Marathi, Moroccan Arabic, Modern Standard Arabic, Punjabi, Spanish, and Telugu. Our goal is to analyse the semantic representation of sentences textual relatedness trained on mBert, all-MiniLM-L6-v2 and Bert-Based-uncased. The effectiveness of these models is evaluated using the Spearman Correlation metric, which assesses the strength of the relationship between paired data. The finding reveals the viability of transformer models in multilingual STR tasks.

\end{abstract}

\section{Introduction}
The rapid increase in digital information has presented a critical challenge for researchers. The web hosts around 50 million pages of text, which is beyond the capacity of human interpretation alone. To interpret this extensive text data effectively, it is essential to comprehend the meanings of various words \cite{bib1}. Semantic Text Relatedness (STR) is a semantic analysis of the relationship between two pieces of text based on their meanings. STR of two language units has long been considered fundamental to understanding meaning \cite{bib4,bib3},  It’s a metric used to measure the similarity in meaning between two terms or documents. It is a subset of computational linguistics and one of the fundamental concepts of  Natural Language Processing (NLP). STR can be measured using datasets designed by experts, which are made up of word pairs that are known to be related. It can be used in identifying a paraphrase or duplicate, as well as search engines to give users relevant and personalized results.\\ \\
When two sentences have a paraphrase or entailment relation, they are considered to be semantically similar and When evaluating the semantic relatedness between them, humans typically focus on identifying shared meanings. In the case of the sentence pairs below, most English speakers would agree that the sentences in the first pair are more closely related in meaning than those in the second pair, whether they are from the same topic, express the same view or originate from the same time period etc.\cite{bib0}.\\
\\
\textbf{Pair 1: a.} \textit{There was a lemon tree next to the house.}\\
  \textbf{b.} \textit{The boy enjoyed reading under the lemon tree.}\\
\\
\textbf{Pair 2: a.} \textit{There was a lemon tree next to the house.}\\
  \textbf{b.}  \textit{The boy was an excellent football player.}\\  
\\
Previous NLP research has mainly dealt with semantic relatedness primarily in English language. However, in this task, we address a variety of languages, including Afrikaans, Algerian Arabic, Amharic, Hausa, Hindi, Indonesian, Kinyarwanda, Marathi, Moroccan Arabic, Modern Standard Arabic, Punjabi, Spanish, and Telugu. The task featured the following tracks: Track A which is a supervised learning, track B is an unsupervised learning and Track C is a cross-lingual learning.\\


\section{Related Works}

Sentences are considered semantically related when they share commonalities in meaning, such as paraphrasal or entailment relations. A study by \cite{bib0} developed a Semantic Textual Relatedness dataset (STR-2022) to manually annotate English sentence pairs and explore the factors that contribute to the semantic relatedness of sentences. The dataset has been used to study the degree of semantic relatedness and the reliability of human intuition in determining the relatedness of sentence pairs while
\cite{bib2} assessed the methods for semantic relatedness between words based on knowledge sources. These methods exploit features from both structural and statistical approaches, emphasizing on semantic representation, measures of semantic similarity, and knowledge-based text mining.\\
\cite{bib3} proposed Explicit Semantic Analysis (ESA), a recently introduced approach that signifies the meaning of texts by computing the semantic relevance of natural language texts. This approach assumes the need for substantial amounts of common sense and domain-specific knowledge, utilizing machine learning techniques to explicitly depict the meaning of any text. This is achieved by creating a weighted vector based on concepts from Wikipedia. ESA undergoes continuous development, ensuring a consistent expansion of its breadth and depth over time.

\section{Task Description}
STR Shared Task 1 \cite{bib12} consists of predicting the semantic relatedness of sentence pairs. Sentence pairs will be rank based on their closeness in meaning in 14 different languages. All sentence pairs will have manually determined relatedness scores between 0 (completely unrelated) and 1 (maximally related). Participants are provided with a gold label scores with a comparative annotation approach that led to a high reliability of the final relatedness rankings. The shared task consists of three tracks: supervised learning, unsupervised learning and cross-lingual learning. In this paper, we concentrate on all the three tracks.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.90\linewidth]{str_hausanlp_framework.jpg}
   
    \caption{\textbf{Task Overview}
    }
    \label{fig:sam_examples}
    
\end{figure}

\subsection{Track A: Supervised}

This track relies on labelled input and output training data. We used the labeled training datasets for 9 languages provided for the shared task which include: Algerian Arabic, Amharic, English, Hausa, Kinyarwanda, Marathi,Moroccan Arabic, Spanish and Telgu.

\subsection{Track B: Unsupervised}

Unsupervised learning analyzes and cluster unlabeled datasets, it is typically used when the goal is to identify patterns and relationships in data. We make this analysis using 12 languages: Afrikaans, Algerian Arabic, Amharic, English, Hausa, Hindi,Indonesian, Kinyarwanda, Modern Standard Arabic, Moroccan Arabic, Punjabi and Spanish.

\subsection{Track C: Cross-lingual}

Cross-lingual learning involves transferring models from one language to another, typically to improve performance. For this track we make use of 12 languages: Afrikaans, Algerian Arabic, Amharic, English, Hausa, Hindi,Indonesian, Kinyarwanda, Modern Standard Arabic, Moroccan Arabic, Punjabi and Spanish.

\section{Experiment and Evaluation}

This section describes the system overview which comprises the dataset description, model description and evaluation metric.


\subsection{Dataset Description}

The dataset consists of an instance of a sentence pair of both the training, development and test sets. Each instance is annotated with a gold label score that represents the degree of semantic text relatedness between two sentences \cite{bib11}. The gold label scores are determine by manual annotation and range from 0 (not related at all) to 1 (very related at all). A comparative annotation approach is used to avoid biases of the traditional rating scales and can result to a high reliability of final relatedness rankings. The dataset used in this shared task are from the following languages: Afrikaans, Algerian Arabic, Amharic, English, Hausa, Hindi, Indonesian, Kinyarwanda, Marathi, Morrocan Arabic, Modern Standard Arabic, Punjabi, Spanish, and Telugu.



% \inserttrainingdata
%\begin{figure}[h]
%\includegraphics[width=0.5\textwidth, inner]{Example_Sentences.pdf}
%\caption{Example Sentences}
%\end{figure}

% \clearpage
% \begin{figure*}[ht!]
%     \centering
%     \includegraphics[clip, trim=0.6cm 16cm 2.6cm 8.1cm, width=0.8\textwidth]{Example_Sentences.pdf}
%      \captionof{table}[]{Example of sentence instaces and its corresponding gold label score for each language..}
%     \label{fig:Examples}
% \end{figure*}

\subsection{Models Description}

We experiment with multiple pre-trained models before deciding to go with the selected models based on the tracks. However, due to time constraint and resources, we reported for the competitive models across various languages based on the task specification. 

\subsubsection{mBERT}

We used mBERT in a supervised aapproach, mBert is a multilingual derivative of BERT and trained on a diverse set of 104 languages. The pre-training process for mBERT involves masked language modeling (MLM) and the next-sentence prediction task \cite{bib6}. To tailor the model for our specific task, we fine-tune the mBERT-base-cased model, which boasts 172 million parameters. A 70-30 train-test split is executed with a learning rate of 1e-5 on Adam optimizer. 


\subsubsection{all-MiniLM-L6-v2}

The all-MiniLM-L6-v2 model was used in an unsupervised approach in this task, it is a lightweight transformer-based model for semantic similarity comparison with optimized model size and faster inference \cite{bib7}. It has 66 Million Parameters compressed in a Student-Mimicking-Teacher network relationship. Using self attention distribution, we utilized the Teacher’s last layer to guide the training of the student distillation in an unsupervised manner and generated effective and flexible results for the 12 languages used.

\subsubsection{BERT-BASED-UNCASED}

The Bert-Based-Uncased model was used in a cross-lingual approach in this task. It is a pre-trained autoencoding language model trained on vast English Wikipedia and BookCorpus with a sequence length of 512. The model is based on the architecture presented in \cite{bib8}. As the track description, some of the languages were initially trained on different language before applying task on new language. Bert-Based-Uncased use WordPiece tokenizer, it has 110 parameters 12-layer, 768-hidden, 12- attention heads. 





% \inserttrainingresults
\inserttrainingsamples

\subsection{Spearman Correlation}
The Spearman Correlation is a non parametric and normality for monotonic relationship between variables \cite{bib9}. It measures the strength of relationship between paired data. It is similar to Pearson’s Product Moment Correlation Coefficient \cite{bib10}, or Pearson’s r. It indicates magnitude and direction of the association between two variables that are on interval or ratio scale. For this task, we used Spearman Correlation to measure the similarity between two sentences.



\section{Results and Discussion}

This section presents the results of the Shared task Tracks.
Table ~\ref{tab:the_results} displays the Spearman correlation scores for the evaluation of 14 low-resource languages for semantic relatedness.
The SemEval-2024 task on STR provided an opportunity to explore the effectiveness of transformer models. The models capture semantic relatedness across multiple languages. In This section, the analyses and interpretations of the results obtained from the given tasks are Task A (supervised learning), Task B (unsupervised learning) and Task C (cross-lingual).

In supervised learning track A, the multilingual BERT (mBERT) model was used. The model demonstrated different levels of performance across the languages. Notably, mBERT exhibited strong correlation scores in languages such as English with 0.76, Marathi with 0.81, and Telugu with 0.79 correlation. This indicates the model's ability to generalize well across linguistic contexts in semantic relatedness tasks. These findings suggest that mBERT can effectively capture semantic relatedness, even in low-resource languages, highlighting its robustness and cross-lingual generalization capabilities. However, challenges were observed in languages with complex morphological structures, underscoring the need for further research to address such linguistic nuances.

Conversely, the unsupervised learning track B featured the All-MiniLM-L6-v2 model, which achieved promising results in certain languages, particularly English with 0.82, Spanish with 0.60, and Moroccan Arabic with 0.5 Spearman correlation value. Despite its effectiveness, the model faced difficulties in languages such as Punjabi and Amharic, where semantic relatedness was harder to capture without labelled data. These challenges emphasize the importance of developing techniques to improve unsupervised learning models' performance, especially in low-resource language settings.

Similarly, track C (cross-lingual) which were entirely trained with BERT-BASED-UNCASED performed promisingly despite training and predicting on different language pairs. The Spearman correlation for Spanish achieved 0.81 and was trained on English, while Hausa achieved the lowest with 0.63 despite being trained on Kinyarwanda training dataset. This performance especially in Semantic Textual Relationship shows that cross-lingual hold a prospective future for generalization of NLP tasks.

However, the findings highlight the effectiveness of transformer models, in capturing semantic relatedness across diverse languages. The choice of evaluation metrics, such as Spearman correlation, proved instrumental in assessing the models' performance and understanding their ability to capture the ordinal relationship between predicted and true semantic relatedness scores. Furthermore, the results contribute valuable insights into advancing the understanding and application of semantic textual relatedness in multilingual NLP tasks, paving the way for future research in this domain.


\section{Conclusion and Future Works}

The study on Semantic Text Relatedness (STR) across multiple languages has demonstrated the effectiveness of transformer models in capturing semantic relatedness. The multilingual BERT (mBERT) model showed strong correlation scores in languages such as English, Marathi, and Telugu, indicating its ability to generalize well across linguistic contexts. The All-MiniLM-L6-v2 model achieved promising results in English, Spanish, and Moroccan Arabic, while facing challenges in languages like Punjabi and Amharic. The cross-lingual track, using BERT-BASED-UNCASED, also performed well, especially in Spanish, trained on English data. These findings underscore the potential of transformer models in NLP tasks and the importance of appropriate evaluation metrics like Spearman Correlation. The study contributes valuable insights into advancing semantic textual relatedness in multilingual NLP, highlighting areas for future research and development.

Future work should focus on exploring advanced transformer Large Language Models (LLMs) like GPT-3 and T5 to improve performance across diverse languages, including low-resource ones. Expanding language coverage, incorporating contextual and cultural information, and fine-tuning with language-specific data will enhance model accuracy. Cross-lingual transfer learning techniques can be investigated to adapt high-resource language models to low-resource settings. Hybrid approaches combining different learning methods may offer improved results, while new evaluation metrics could better capture semantic nuances. Additionally, exploring multimodal STR and applying research findings to real-world applications will increase the practical impact of STR systems.


\section*{Acknowledgements}
We would like to acknowledge HausaNLP Management for their tireless support.

% Entries for the entire Anthology, followed by custom entries
\bibliography{anthology,references}
\bibliographystyle{acl_natbib}

\appendix

\section{Appendix}
\label{sec:appendix}

%Appendix goes here.
% \inserttrainingdata
\begin{figure}[h]
\includegraphics[width=0.50\textwidth, inner]{Example_Sentences.pdf}
\caption{Example Sentences}
\end{figure}

\end{document}
