% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}
\usepackage{graphicx}
% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{NLU-STR at SemEval-2024 Task 1: Generative-based Augmentation and Encoder-based Scoring for Semantic Textual Relatedness}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

% \author{First Author \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\\And
%   Second Author \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\}
\author{Sanad Malaysha, Mustafa Jarrar, Mohammed Khalilia \\
   Birzeit University, Palestine \\
    \texttt{\{smalaysha, mjarrar, mkhalilia\}@birzeit.edu} \\
 }
 
\begin{document}
\maketitle
\begin{abstract}
Semantic textual relatedness is a broader concept of semantic similarity. It measures the extent to which two chunks of text convey similar meaning or topics, or share related concepts or contexts. This notion of relatedness can be applied in various applications, such as document clustering and summarizing. SemRel-2024, a shared task in SemEval-2024, aims at reducing the gap in the semantic relatedness task by providing datasets for fourteen languages and dialects including Arabic. This paper reports on our participation in Track A (Algerian and Moroccan dialects) and Track B (Modern Standard Arabic). A BERT-based model is augmented and fine-tuned for regression scoring in supervised track (A), while BERT-based cosine similarity is employed for unsupervised track (B). Our system ranked 1\textsuperscript{st} in SemRel-2024 for MSA with a Spearman correlation score of 0.49. We ranked 5\textsuperscript{th} for Moroccan and 12\textsuperscript{th} for Algerian with scores of 0.83 and 0.53, respectively.
\end{abstract}

\section{Introduction}
\label{sec:intro}
%The common task in the literature is the semantic similarity but semantic relatedness is less studied because of complexity and dearth of datasets. While the first task only checks the degree to which two linguistic units (i.e, words, phrases, sentences, etc.) represent similar meanings, the latter, semantic textual relatedness (STR), studies and evaluates the similarity from multiple dimensions, spanning topical similarity, conceptual overlap, contextual coherence, pragmatic connection,  themes, scopes, ideas, stylistic conditions, ontological relations, entailment, as well as semantic similarity itself.
The literature commonly examines semantic similarity, where the focus is on whether two linguistic units (words, phrases, sentences, etc.) share similar meanings \citep{BentivogliBMMBZ16}. However, semantic textual relatedness (STR) is less explored due to its complexity and the scarcity of datasets \citep{AbdallaVM23,DH21}. While the former task checks for the presence of similar meaning or paraphrase, STR takes a more comprehensive approach, evaluating relatedness across multiple dimensions, spanning topical similarity, conceptual overlap, contextual coherence, pragmatic connection, themes, scopes, ideas, stylistic conditions, ontological relations, entailment, temporal relation, as well as semantic similarity itself \citep{miller1991contextual, halliday2014cohesion,J21, J11}. For example, consider the two sentences (\textit{The Earth orbits the sun at a speed of \textasciitilde110,000 km/h.}) and (\textit{Earth rotates at \textasciitilde1670 km/h around its axis.}). They hold semantic relatedness through the shared topic of Earth's speeds. In contrast, both sentences are not semantically similar as they possess distinct meanings. This illustrates the broader range of STR as described by \citet{AbdallaVM23}, which ranges from highly relevant sentences, expressing the same idea with different wording, to entirely unrelated sentences, discussing unrelated topics.
%(sanad): talk about symmetric and asymmetric STR if more space left. 

Semantic relatedness has proven to be useful in evaluating sentence representations generated by language models \citep{AsaadiMK19}, in addition to question answering \citep{TsatsaronisVV14}, machine translation \citep{MiX24}, plagiarism detection \citep{SabirMP19}, word-sense disambiguation \citep{arabglossbert, MalayshaJK23}, among others. Exploring the relatedness and similar tasks in languages other than English is hindered by the lack of data \citep{JMHK23,HJ21}. The SemRel-2024 shared task \citep{semrel2024dataset} provided datasets in fourteen languages and offered three tracks. In the supervised track (A), training and testing are performed on the same language. In the unsupervised track (B), the use of labeled data for training is prohibited; and in the cross-lingual track (C), testing is conducted on a different language than the one used for training.

This paper presents our contribution to track A and track B. 
In track A, we fine-tuned BERT models using the Algerian and Moroccan sentence pairs to produce similarity scores. To enrich the data, we augmented the SemRel-2024 dataset \citep{semrel2024dataset} by generating additional sentence pairs from Google Gemini \footnote{https://gemini.google.com/}, a generative model, using a predefined prompt template. These generated pairs imitated the style and meaning of the existing pairs, and we assigned them scores corresponding to the originals. We used the same datasets provided by the Shared Task in addition to a \textasciitilde760 augmented Moroccan pairs to fine-tune BERT models, AraBERTv2 \citep{antoun2020arabert} and ArBERTv2 \citep{Abdul-MageedEN20}, which resulted in a performance enhancement of 0.05 points. 
%In track A, we approached the task as regression scoring. We fine-tuned two BERT-based models:  ArBERTv2 \citep{Abdul-MageedEN20} for Moroccan and AraBERTv2 \citep{antoun2020arabert} for Algerian. Additionally, we augmented the Moroccan dataset using Google Gemini \footnote{https://gemini.google.com/}, formerly known as Bard, and collected around 760 additional scored pairs, resulting in a performance enhancement of 0.05 points. 
In track B, as training on labeled data is not allowed, we used cosine similarity using average pooling embedding \citep{ZhaoZHCY22} on top of each model. %In the results analysis (Section \ref{sec:result}), we will explain why we used each model for each dialect in each track, as we found that one model might be more effective than the other for specific language (dialect). Due to resources and time limitations in participating in the shared task, the augmentation was limited to a subset of the Moroccan data only. Nonetheless,
Our approaches achieved Spearman scores \citep{TsatsaronisVV14} of 0.49 for MSA (ranked first), 0.83 for Moroccan (ranked fifth), and 0.53 for Algerian (ranked twelfth).

\section{Related Work}
\label{sec:related-work}

%The STR shown to be effective task serving many NLP applications, along with evaluating LLMs. Nonetheless, to decide the degree of relatedness it found to be challenging task in computational semantics because it covers broad range of commonalities other than meaning between sentences such as view, topic, and period, which requires deeper understanding than semantic similarity. For example, reading these two sentences (\textit{He heard the waves crashing gently}) and (\textit{Making him feel calm and peaceful}). They are easy for human to understand that are very related and describing same view (beach scene). However, machines require advanced lexical and statistical techniques to achieve it. STR techniques come mainly from four approaches including lexical similarity, Semantic similarity, deep learning, and LLMs.

Semantic textual relatedness (STR) has proven to be a valuable task in numerous NLP applications, including the evaluation of LLMs \citep{AsaadiMK19, NaseemRKP21}. Determining the degree of relatedness in STR, however, remains a challenging task in computational semantics. That is because STR encompasses a broader range of commonalities beyond just meaning, including shared viewpoint, topic, and period, demanding a deeper understanding than semantic similarity alone \citep{AsaadiMK19, AbdallaVM23}. For example, consider reading these two sentences (\textit{He heard the waves crashing gently}) and (\textit{Making him feel calm and peaceful}). While humans easily recognize their strong relatedness and shared description of the same view (a beach scene), machines require advanced lexical and statistical methods to achieve the same level of understanding. STR techniques mainly come from four approaches: lexical similarity \citep{ChenSY18a, jarrardb, ADJ19}, semantic similarity \citep{hasan2020knowledge,GJJB23}, deep learning \citep{zhang2019multi}, and LLMs \cite{LiYC21}.

%Recently, \citet{AbdallaVM23} created their own STR dataset (STR-2022) using fine-grained scores, such that score values range between 0 (least related and 1 (completely related). Their dataset consists of 5,500 scored English sentence pairs. They formulated the task as supervised regression, where they fine-tuned two language models, BERT-base \citep{kenton2019bert} and RoBERTa-base \citep{roberta}, and applied average-pooling on top of the final embedding layer. Testing the models on their SRT-2022 dataset showed average Spearman correlation of 82\% using BERT-base and 83\% using RoBERTa-base. On the other side, in their unsupervised experimentation, using  Word2Vec \citep{word2vec} has achieved 60\% correlation score, which outperformed both BERT-base (58\%) and RoBERTa (48\%) with 2\% and 12\% differences, respectively.
Recently, \citet{AbdallaVM23} introduced their STR-2022 dataset, which uses fine-grained scores ranging from 0 (least related) to 1 (completely related). Their dataset consists of 5,500 scored English sentence pairs. They framed the task as supervised regression, where they fine-tuned two language models, BERT-base \citep{kenton2019bert} and RoBERTa-base \citep{roberta}, and applied average pooling on top of the final embedding layer. Their testing of these models on the STR-2022 dataset yielded an average Spearman correlation of 0.82 for BERT-base and 0.83 for RoBERTa-base. On the other hand, their unsupervised experiments using Word2Vec \citep{word2vec} achieved a correlation score of 0.60, outperforming both BERT-base (0.58) and RoBERTa-base (0.48) by 0.02 and 0.12 points, respectively.

%\citet{AsaadiMK19} created bi-gram semantic relatedness dataset (BiRD) for the examination of the semantic composition. They followed fine-grained bi-gram pairs scoring (0-1) using best–worst scaling (BWS) \citep{KiritchenkoM17} annotation technique instead of rating scales (1-5) to void inconsistency and bias. The dataset contains 3,345 scored English term pairs. They utilized three models to generate words representations:  GloVe \citep{glove}, FastText \citep{fasttext}, and  word–context co-occurrence matrix \citep{TurneyNAC11}. In order to calculate relatedness scores among pairs, they employed cosine similarity between the generated addition-pooled vectors. The best performing model was FastText with 60\% Pearson correlation.
\citet{AsaadiMK19} created the Bi-gram Semantic Relatedness Dataset (BiRD) for examining semantic composition. To avoid inconsistencies and biases from traditional 1-5 rating scales, they employed fine-grained scoring of bi-gram pairs (0-1) using the best-worst scaling (BWS) annotation technique \citep{KiritchenkoM17}. The dataset consists of 3,345 scored English term pairs. They utilised three models to generate word representations: GloVe \citep{glove}, FastText \citep{fasttext}, and a word-context co-occurrence matrix \citep{TurneyNAC11}. To calculate relatedness scores between pairs, they employed cosine similarity between the generated addition-pooled vectors. The FastText model achieved the highest performance with a Pearson correlation of 0.60.

%The semantic relatedness was studied between noun-pairs using contextual similarity by \citep{miller1991contextual}. They tried to understand distinctions between nouns in contextual discourse, and how the similarity can be wider than just the meaning. However, the task evolved reaching the up-to-date dataset presented by the SemRel-2024 shared task. Their dataset annotation scores is on the level of sentence-pairs. They shared baseline results for fourteen languages and dialects using Spearman correlation score. As our focus on Arabic so we pick its results to show. For example, their baseline is is 42\% for MSA in track B using mBERT, 60\% for Algerian and 77\% for Moroccan in track A using Label Agnostic BERT Sentence embeddings (LaBSE) \citep{FengYCA022}. 

The semantic relatedness between noun-pairs was studied using contextual similarity by \citet{miller1991contextual}. They attempted to understand distinctions between nouns in contextual discourse and how the similarity can be broader than just the meaning. Additional ideas could rely on extracting named entities \citep{LJKOA23,JKG22} to measure the relatedness \cite{GhoshDKZA23}. However, the task evolved, leading to the creation of the up-to-date dataset presented by the SemRel-2024 shared task \citep{semrel2024task}. Their dataset annotation scores are at the level of sentence pairs. They shared baseline results for fourteen languages and dialects using Spearman correlation score. Since our focus is on Arabic, we have chosen its results to show. For example, their baseline is 0.42 for MSA in track B using multi-lingual BERT (mBERT) \citep{kenton2019bert}, 0.60 for Algerian and 0.77 for Moroccan in track A using Label Agnostic BERT Sentence embeddings (LaBSE) \citep{FengYCA022}. Specifically, their Algerian Arabic dataset offers 1,261 training and 583 test instances, Moroccan Arabic dataset includes 924 training and 425 test instances, and MSA Arabic dataset has 595 instances for testing.

Many efforts have been made to understand Arabic dialects, such as dialect identification, intent detection, and morphological annotations \cite{EJHZ22, ANMFTM23, JZHNW23, JHRAZ17, JBKEG23}, but none studied STR between dialects.


%Our participation in the SemRel-2024 shared task is building on top of the techniques presented by \citet{AbdallaVM23, semrel2024task}, we fine-tune Arabic BERT-based models on regression scoring. We augmented SemRel-2024 dataset by adding more pairs using generative models, where we employed the generative model with predefined prompt template to generate similar sentences to the sentences in the pairs, creating additional pairs with scores corresponding to the original pairs. Their Arabic Algerian dataset consists of 1261 train and 583 test instances, Arabic Moroccan dataset consists of 924 train and 425 test instances, and Arabic MSA consists of 595 instances for testing purposes.

\begin{figure}[ht!]
    \centering
    \includegraphics[scale=0.40]{supervised.jpg}
    \caption{BERT-based Supervised Architecture (A).}
    \label{fig:supervised-arch}
\end{figure}

\section{System Overview}
\label{sec:system-overview}
%The sentence-pairs annotations provided in the SemRel-2024 shared task have fine-grained scores, covering the interval [0,1]. That requires utilizing techniques which can capture continues values, i.e., regression problem \citep{malaysha2022classification}. Since the shared task has three tracks (supervised, unsupervised, and cross-lingual), we have to chose methods fits their objectives, specifically supervised and supervised that we targeted them. As we participate only in the Arabic dialects, we have to utilize suitable language models, therefore we mainly used two BERT-based Arabic models, namely ArBERTv2 \citep{Abdul-MageedEN20} and AraBERTv2 \cite{antoun2020arabert}, which usually achieve high performance \citep{salma, JarrarAKTEHO23}. In the following subsections, we discuss the techniques, datasets, and data augmentation.

%The SemRel-2024 shared task provides sentence-pair annotations with fine-grained scores on a scale of 0 to 1. This requires utilizing techniques capable of capturing continuous values, i.e., regression problem. Since the shared task has three tracks (supervised, unsupervised, and cross-lingual), we have to choose methods that fit their objectives, specifically the supervised and unsupervised tracks that we targeted. 
%As our participation in the SemRel-2024 shared task is focused on tracks A and B, 
%solely on Arabic dialects, we chose appropriate language models. We primarily used two BERT-based Arabic models, ArBERTv2 \citep{Abdul-MageedEN20} and AraBERTv2 \cite{antoun2020arabert}, known for their strong performance \citep{salma, JarrarAKTEHO23}. In 
This section presents the techniques, datasets, and the augmentation we employed in tracks A and B.

\subsection{Supervised Track (A)}
\label{sub:supervised}
%As the datasets provides continues values so we tackled the task as regression problem. We fine-tuned the selected models, with a regressor output layer represented by one neuron to predict the fine-grained score. The loss function used to optimize the learning is Mean Square Error (MSE). because we target Algerian and Moroccan dialects in this track, initial fine-tuning went under different setups to understand each model fits to which language better. We found AraBERTv2 \footnote{https://github.com/aub-mind/arabert} model fits Algerian dialect and ArBERTv2 \footnote{https://huggingface.co/UBC-NLP/ARBERTv2} model fits Moroccan better, MSA would be discussed in next subsection. We pre-processed the data using the technique presented in \citep{antoun2020arabert} before fine-tuning to have standardized data format. Each input sentence-pair is concatenated using the special tokens per this format: [CLS]Sentence1[SEP]Sentence2[SEP]. In subsection \ref{sub:data-augmentation} we discuss the dataset and augmentation. Figure \ref{fig:supervised-arch} depicts the method architecture for supervised track (A).

Since the datasets use continuous scoring values, we tackled STR as a regression problem. We fine-tuned BERT with Mean Squared Error (MSE) objective. The model uses a regressor output layer, represented by a single neuron to predict the scores of the sentence-pairs. The data was pre-processed using the technique presented in \citep{antoun2020arabert} to achieve standardized word forms. Before supplying the sentence pairs to the model, each was concatenated using the special tokens of the model input in this format:\texttt{[CLS]Sentence1[SEP]Sentence2[SEP]}. Figure \ref{fig:supervised-arch} depicts our method architecture for the supervised track (A). Since we focused on the Algerian and Moroccan dialects in this track, we investigated various model parameters including learning rates, number of epochs, and pre-trained models to understand which model is better suited for each dialect. We found that both models, AraBERTv2 \footnote{https://github.com/aub-mind/arabert} and ArBERTv2 \footnote{https://huggingface.co/UBC-NLP/ARBERTv2}, best fits the Moroccan dialect more than Algerian. Nonetheless, we used same models for the Algerian dataset.
%sanad : add MSE equation

\begin{figure}[ht!]
    \centering
    \includegraphics[scale=0.40]{unsupervised.jpg}
    \caption{BERT-based Unsupervised Architecture (B).}
    \label{fig:unsupervised-arch}
\end{figure}

\begin{figure*}[ht!]
    \centering
    \includegraphics[scale=0.650]{augments.jpg}
    \caption{Example of the augmented sentence-pairs.}
    \label{fig:augments}
\end{figure*}

\subsection{Unsupervised Track (B)}
\label{sub:unsupervised}
%The MSA dialect is not covered in track A but only in B and C. We focused on track B (unsupervised learning), where the training (or fine-tuning) is permitted. We employed cosine similarity as unsupervised technique to calculate the sentence-pairs scores. We made initial experimentations using the same aformentioned models, ArBERTv2 and AraBERTv2 for generating sentence representations. Various pooling options (CLS, average, max, and min) are applied on the final embedding layer in each (freezed) model, and found AraBERTv2 with average-pooling is better suited for MSA in this track. Same data pre-processing in track A applied here. Figure \ref{fig:unsupervised-arch} illustrated our architecture for the unsupervised track using cosine similarity. 
The STR using MSA is covered in track  B (unsupervised learning), where training (or fine-tuning) on labeled data is not permitted. We employed cosine similarity \citep{cosinesimilarity} as an unsupervised technique to calculate the sentence-pair scores. Figure \ref{fig:unsupervised-arch} illustrates our architecture. We conducted initial experiments using the same aforementioned models, ArBERTv2 and AraBERTv2, for generating sentence representations. Various pooling options (CLS, average, max, and min) \citep{ZhaoZHCY22} were applied on the final embedding layer in each (frozen) model, and found that AraBERTv2 with average-pooling is better suited for MSA in this track. The same data pre-processing used in track A is applied in B.
%sanad: add cosine similarity equation

\subsection{Datasets}
\label{sub:data-augmentation}
%The datasets provided by the SemRel-2024 shared task cover fourteen languages and dialects, and classified into three tracks as already stated. We worked on the three Arabic dialects supplied in the datasets. Table \ref{table:datasets} presents their data splits, including train, development, and testing. MSA needs no training data as it included in Tracks B \& C. However, for the other two dialects, we employed BERT-based models, they require as large as possible training data \citep{bevilacqua2021recent}. To augment the data there are many possible methods, such as back-translation \cite{lin2021context} and generative models \citep{SaidiJKS22}. The back-translation technique was tested by \citep{MalayshaJK23} and shown minor improvement in the performance. The high quality of generative models recently got available such as ChatGPT \footnote{https://chat.openai.com/} and Google Gemini \footnote{https://gemini.google.com/} encouraged us to employ them in the automatic augmentation. We employed in-context learning \citep{incontextlearning} by prompting both models with the request depicted in Figure \ref{fig:prompt}.

The datasets provided by the SemRel-2024 shared task cover fourteen languages and dialects. In the paper, we used three Arabic datasets (Algerian, Moroccan, and MSA). Table \ref{table:datasets} presents their data splits, including train, development, and testing. MSA has no labeled train data as it is included in Track B. However, for the other two dialects, we employed BERT-based models, that requires large train data \citep{bevilacqua2021recent}. 

\begin{table}[ht!]
\centering
\begin{tabular}{llll}
\hline
               & \textbf{MSA} & \textbf{Algerian} & \textbf{Moroccan} \\ \hline
\textbf{Train} &              &                   &                   \\
\hspace{5pt}Original       & --           & 1,261             & 924               \\
\hspace{5pt}Augments       & --           & --                & 757               \\
\hspace{5pt}Total          & --           & 1,261             & 1,681             \\ \hline
\textbf{Dev.}   & 32           & 97                & 70                \\ \hline
\textbf{Test}  & 595          & 583               & 425               \\ \hline
\end{tabular}
\caption{\label{table:datasets}
The original and augmented datasets splits.
}
\end{table}

Different methods can be used for data augmentation, such as back-translation \cite{lin2021context} and generative models \citep{SaidiJKS22}. The back-translation technique was tested by \citep{MalayshaJK23} and showed minor improvement in performance. The availability of high-quality generative models, such as ChatGPT \footnote{https://chat.openai.com/} and Google Gemini, encouraged us to employ them in automatic augmentation. We employed in-context learning \citep{incontextlearning} by prompting both models with the request depicted in Figure \ref{fig:prompt}.

%The initial tests using  twenty prompts of Moroccan and Algerian sentences showed that both models are weak in the Algerian comprehension, ChatGPT is weak in the Moroccan dialect also, and Gemini proved high understanding of Moroccan dialect, therefore we decided to employ Gemini in augmenting the augmentation of the Moroccan train split. From each pair, we took each sentence and prompted it using the template in Figure \ref{fig:prompt} and paired the augmented (new) sentence from the model with the other sentence in the same pair using the same score of the pair, as illustrated in Figure \ref{fig:augments}. By manually reviewing all the model replies we found cases are not valid (wrong content), and accordingly we made filters to exclude the not applicable data per the following rules:
% \begin{itemize}
%     \item The model admits in the reply he is just language model and can not full fill the request.
%     \item The case when the reply is went far from the original meaning.
%     \item When the model reject to augment because the requested sentence contains Information that break the model policy, i.e., talking about famous figures or faithful discussions.
% \end{itemize}
%Finally, after filtering the not valid augmentations, we reach to 757 accepted sentences which created additional 757 pairs in the Morrocan train, to reach the total of 1,681 instances.

\begin{figure}[ht!]
    \centering
    \includegraphics[scale=0.6]{prompts.jpg}
    \caption{The prompt template employed for Gemini.}
    \label{fig:prompt}
\end{figure}

The initial manual reviews and tests for twenty prompts of Moroccan and Algerian sentences showed that both models are weak in Algerian comprehension. ChatGPT is also weak in the Moroccan, while Gemini demonstrated a high understanding of the Moroccan. Therefore, we decided to employ Gemini to augment the Moroccan train split. From every sentence-pair, we took each sentence and prompted it using the template in Figure \ref{fig:prompt}. We mapped the augmented (new) sentence from the model with the other sentence in the same pair using the same score of the pair, as illustrated in Figure \ref{fig:augments}. By manually reviewing all the model replies, we found cases that were not valid (wrong content), and accordingly, we defined filters to exclude the not applicable data per the following rules:
\begin{itemize}
    \item The model admits in the reply that it is just a language model and cannot fulfill the request. The model reply in such case has common format to rely on for the filter comparison.
    \item The case when the reply goes far from the original meaning. This option is achieved by manually reviewing the paraphrased contents.
    \item When the model rejects augmentation because the requested sentence contains information that breaks the model policy, i.e., talking about public figures or sensitive discussions. Similar to first rule, it has common reply format to automatically compare with.
\end{itemize}
Finally, after filtering the invalid augmentations, we reached 757 accepted sentences which we added to the Moroccan training set (See Table \ref{table:datasets}), reaching a total of 1,681 instances.

\begin{table*}[ht!]
\centering
\begin{tabular}{llll|l}
                   \textit{Development Phase} & \multicolumn{3}{c|}{\textbf{Track A}} & \textbf{Track B} \\ \hline
                   & Algerian            & Moroccan  & Augmented Moroccan      & MSA              \\ \cline{2-5}        \hline
\textbf{ArBERTv2}  & 0.55           & 0.82 & 0.88$\uparrow$ & 0.42             \\ \hline
\textbf{AraBERTv2} & 0.69  & 0.84 & 0.79$\downarrow$          & 0.58    \\ \hline
\end{tabular}
\caption{\label{table:dev-results}
Our results on the development phase (i.e., on development split).
}
\end{table*}

\begin{table*}[ht!]
\centering
\begin{tabular}{llll|l}
                   \textit{TEST Phase} & \multicolumn{3}{c|}{\textbf{Track A}} & \textbf{Track B} \\ \hline
                   & Algerian            & Moroccan  & Augmented Moroccan      & MSA              \\ \cline{2-5} 
\textbf{Baseline} \scriptsize\citep{semrel2024dataset}   & 0.60           & 0.77 & 0.77          & 0.42             \\ \hline
\textbf{ArBERTv2}  & 0.42$\downarrow$           & 0.78$\uparrow$ & \textbf{0.83}$\uparrow$ & 0.34$\downarrow$             \\ \hline
\textbf{AraBERTv2} & \textbf{0.53}$\downarrow$  & 0.79$\uparrow$ & 0.77$\uparrow$          & \textbf{0.49}$\uparrow$    \\ \hline
\end{tabular}
\caption{\label{table:results}
The evaluation results on the test data. Our official ranked scores are in bold.
}
\end{table*}

\section{Experimental Setup}
\label{sec:experimental-setup}
%Our experiments included fine-tuning two language models for Algerian and Moroccan, where we used these checkpoints: maubmindlab/bert-base-arabertv02 and UBC-NLP/ARBERTv2. We employed the training data supplied by the shared task, in addition to our augmentation when applies. The development data is excluded  from either training or test, as well as the testing is done only on the test split of the shared task dataset (See Table \ref{table:datasets}). The data-pairs was concatenated using the special tokens ([CLS] and [SEP]), depicted in Figure \ref{fig:supervised-arch}, and digested by the models. The fine-tuning was done as a regression task using one neuron in the out layer, optimized by using MSE as loss function, and used R-squared \citep{miles2005r} to measure the improvement. The major hyper-parameters in the fine-tuning are: 10 epochs training, 4 epochs early stopping, 16 for batch size, learning rate=$2e^{-5}$, 1,412 warm-up steps, and 42 seed.
Our experiments fine-tuned two language models for Algerian and Moroccan, where we used the following pre-trained models: maubmindlab/bert-base-arabertv02 \citep{antoun2020arabert} and UBC-NLP/ARBERTv2 \citep{Abdul-MageedEN20}. We employed the training data provided by the shared task, in addition to the data generated by our augmentation technique, when applied. The development data is excluded from either training or testing in the official evaluation phase, and testing is done on the shared task test set (See Table \ref{table:datasets}). The data pairs were concatenated using special tokens (\texttt{[CLS]} and \texttt{[SEP]}), as depicted in Figure \ref{fig:supervised-arch}, and digested by the models. The fine-tuning was done as a regression task using one neuron in the output layer, optimized using MSE as the loss function, and we used R-squared \citep{miles2005r} to measure the improvement. The final hyper-parameters in the fine-tuning process were: 10 epochs for training, 4 epochs for early stopping, a batch size of 16, 512 is the maximum sequence length, a learning rate of $2e^{-5}$, 50 evaluation steps, a seed of 42, and train ($\pm$ augmented data) split.

%In the experiments of the unsupervised track for MSA, no fine-tuning is needed, therefore requires neither train data nor augmentation. We employed average-pooling on the embedding of the final layer in each model, then calculated the cosine similarity between the embedding of the sentences in each pair to estimate the fine-grained scores for the data of test split provided by the shared task. The shared task uses the Spearman correlation score for evaluating the submitted test results against the ground-truth, as will be discussed in next section (Results: Section \ref{sec:result}).
In the experiments of B track for the MSA, no supervised fine-tuning is needed. Therefore, we neither used labeled data nor augmentation. We employed average-pooling on the embeddings of the sentence tokens from the final layer in each model. Then, we calculated the cosine similarity between the average embeddings of the sentences in each pair. This was done to estimate the fine-grained scores for the test (or development) data provided by the shared task. The shared task considers Spearman correlation score to evaluate the submitted predictions against their ground truth.%, which we will discuss in the next section.

\section{Results}
\label{sec:result}
%Our approaches has achieved competitive ranks in the SemRel-2024 shared task. The results of the tracks we participated in are shown in Table \ref{table:results}. We ranked first in Track B for MSA, with Spearman correlation score 0.49 using AraBERTv2 model, outperforming the baseline by 0.07 points. ArBERTv2 did not perform well in the MSA track B. I contrast, ArBERTv2 achieved high score in track A for Moroccan dialect when fine-tuned on both train split and augmentation data. Where it outperformed the baseline by 0.06 points, achieved rank 5 in the submitted systems. Nonetheless, Neither of the models was able to outperform the baseline of the Algerian dialect in track A, where our rank is 12. Probably, if we were able to augment the Algerian data as well to perform better, similar to the improvement achieved in the Moroccan. However, it worth to note that AraBertv2 outperformed both baseline and ArBERTv2 on the original train of the Moroccan dataset, but its performance degraded once included the augmentation in the fine-tuning, on the contrary to what happened with ArBERTv2 model.That might be because of the nature of the data utilized in pre-training phase.  
Our approaches have achieved competitive ranks in the SemRel-2024 shared task. The official results of the tracks we participated in, as well as the baselines that were introduced by \citet{semrel2024dataset}, are shown in Table \ref{table:results}. Additionally, our results on the development data are presented in Table \ref{table:dev-results}. In the test evaluation, we ranked first in Track B for the MSA, with a Spearman correlation score of 0.49 using the AraBERTv2 model, outperforming the baseline by 0.07 points. However, ArBERTv2 did not perform well in Track B for MSA on both test and development splits. In contrast, ArBERTv2 achieved a high score in Track A for the Moroccan dialect when fine-tuned on both the train split and augmentation data, outperforming the baseline by 0.06 points on test split, ranking 5th among the submitted systems. Nonetheless, neither of the models, ArBERTv2 or AraBERTv2, surpassed the baseline for the Algerian dialect in Track A, where our rank is 12. Similarly, both models achieved low performance on the Algerian development split. It is possible that if we were able to augment the Algerian data as well, it could have performed better, similar to the improvement achieved in the Moroccan dataset. It is worth noting that AraBERTv2 outperformed both the baseline and ArBERTv2 on the original training data of the Moroccan dataset. However, its performance degraded on both test and development splits once the augmentation was included in the fine-tuning, unlike what happened with the ArBERTv2 model, on both splits. This could be due to the nature of the data utilized in the pre-training phase of the model. Due to the anisotropy problem \citep{BaggettoF22} inherent in BERT-based pre-trained models, we noted that computing cosine similarity directly between sentence representations is insufficient for discerning relatedness.

\section{Conclusion}
\label{sec:conclusion}
%In this paper, we introduced our system as a participation in the SemRel-2024 shared task. This shared task provides dataset for Semantic Textual Relatedness in 14 languages. We targeted three Arabic dialects covered by the datasets, including MSA, Algerian, and Moroccan. Our approaches employed supervised and unsupervised techniques using commonly known language models, namely ArBERT and AraBERT. We augmented the train data using Generative models which enhanced the models performance. Our system ranked first (MSA), fifth (Moroccan), and twelfth (Algerian) across the different tracks. We plan to augment additional data of Moroccan and Algerian using other models than what we used in this work. We will use the augmentations to experiment them in both Arabic mono-dialect and cross-dialect fine-tuning.
We presented our contributions to the SemRel-2024 shared task. We targeted three Arabic dialects covered by the shared task datasets, including MSA, Algerian, and Moroccan. Our approaches employed supervised and unsupervised techniques using commonly known language models, namely ArBERT and AraBERT. We augmented the training data using generative models, which enhanced the models' performance. Our system ranked first (MSA), fifth (Moroccan), and twelfth (Algerian) across the different tracks. We plan to augment additional data of Moroccan and Algerian using other models than what we used in this work. We will use the augmentations to experiment with both Arabic mono-dialect and cross-dialect fine-tuning.

% \section{Limitations}
% \label{sec:limitations}

% \section{Ethical Considerations}
% \label{sec:ethics}


% \section*{Acknowledgements}
% \label{sec:ack}


% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom,MyReferences}

\end{document}


% for the introduction section:
%a natural approach to get at the meaning of a unit is to determine how close it is to other units
%existing approaches of sentence representation capture semantic relatedness
%Semantic relatedness is central to textual coherence and narrative structure
%The ability to assess semantic relatedness is central to the use and understanding of language
%Work in semantic representation explores how best to represent the meanings of words, phrases, and sentences.
%A common approach to evaluate different methods of representing words via vectors is through their ability to rank pairs of words by closeness in meaning
%Estimating the semantic relatedness of two text fragments – such as words, sentences, or entire documents – is important for many natural language processing or information retrieval applications
%Exploring and quantifying semantic relatedness is central to representing language. It holds significant implications across various NLP tasks, including offering insights into the capabilities and performance of Large Language Models (LLMs).
%Exploring and quantifying semantic relatedness is central to representing language. It holds significant implications across various NLP tasks, including offering insights into the capabilities and performance of Large Language Models (LLMs).



% for the related-work section:

%rank the term pairs in BiRD by relatedness scores
% We test three vector space models to obtain


%Bigram Relatedness Dataset (BiRD) is computed by taking the cosine


% 5-fold cross-validation (CV)
% report the average of the Spearman

% abdallah: humans generally look for commonalities in meaning: whether they are on the same topic, express the same view, originate from the same time period, one elaborates on (or follows from) the other, etc. paraphrasal or entailment relation

% big bird: semantically related concepts may not have many properties in common, but there exists some relationship between them which lends them the property of being semantically close

%sick: the assess- ment of semantic relatedness involves a deeper understanding of the text than the evaluation of semantic similarity, one which requires common sense and domain-specifc knowledge.

% Approaches
% Lexical similarity: measures based on word overlap or co-occurrence
% Semantic similarity: measures utilizing distributional semantics or word embeddings
% deep learning models: trained specifically for textual similarity tasks
% LLMs: fine-tuned specifically for textual similarity tasks