% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\usepackage{booktabs}
\usepackage{graphicx}

\title{SATLab at SemEval-2024 Task 1: A Fully Instance-Specific Approach for Semantic Textual Relatedness Prediction}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Yves Bestgen \\
  Statistical Analysis of Text Laboratory (SATLab) \\
  Universit\'e catholique de Louvain \\
  Place Cardinal Mercier, 10 1348 Louvain-la-Neuve, Belgium \\
  \texttt{yves.bestgen@uclouvain.be} \\}

\begin{document}
\maketitle
\begin{abstract}
This paper presents the SATLab participation in SemEval 2024 Task 1 on Semantic Textual Relatedness. The proposed system predicts semantic relatedness by means of the Euclidean distance between the character ngram frequencies in the two sentences to evaluate. It employs no external resources, nor information from other instances present in the material. The system performs well, coming first in five of the twelve languages. However, there is little difference between the best systems.
\end{abstract}

\section{Introduction}

Semantic similarity between words, phrases and texts has long attracted the attention of NLP researchers. It is obviously a useful source of information in tasks such as information retrieval, text summarization, question answering or machine translation \citep{agirre-etal-2012-semeval}. It has been the subject of several shared tasks within SemEval since 2012 \citep{agirre-etal-2012-semeval,marelli-etal-2014-semeval,cer-etal-2017-semeval}. More recently, interest has also focused on Semantic Textual Relatedness (STR), which is supposed to be a more general concept. As \citet{abdalla-etal-2023-makes} point out, two sentences must be paraphrases or present an entailment relation to be semantically similar, whereas to be related, it is sufficient that they deal with similar themes or express similar points of view on a given issue. Work on STR is less advanced due to the lack of annotated datasets on this dimension \citep{abdalla-etal-2023-makes}. It should be noted, however, that the human annotators who evaluated semantic textual similarity for the SILK dataset \citep{marelli-etal-2014-semeval} clearly evaluated relatedness, since they considered pairs of sentences that contradict each other as semantically very similar (96\% similarity), such as in SILK Instance 466:
\newline \textit{- A man is performing a trick on a green bicycle.}
\newline \textit{- There is no man performing a trick on a green bicycle.}

The SILK dataset contains many other examples of this kind of judgement. This observation suggests that the term "relatedness" is more appropriate to describe this field of research, at least when dealing with the intuition of native speakers. It also suggests that techniques that are effective in automatically estimating semantic similarity should also be effective in estimating relatedness. These are mainly state-of-the-art deep learning algorithms \citep{cer-etal-2017-semeval}.

In this context, \citet{semrel2024task} have proposed the SemEval 2024 Task 1, which has a number of specific features compared with previous work. Firstly, the task focuses on relatedness, and is based on material consisting of sentence pairs that have been annotated on this dimension by native speakers. Secondly, the task is highly multilingual, covering more than ten languages, some of which are very poorly resourced. Finally, it includes three subtasks: supervised, unsupervised and crosslingual. In the supervised subtask, the systems were to be trained using training datasets provided by the task organizers. In the unsupervised subtask, no datasets labeled according to semantic relatedness or semantic similarity could be used. In the crosslingual subtask, the system had to be trained on a language other than the target language. 

\section{The Proposed Approach}
Due to its highly multilingual nature (twelve languages), the unsupervised subtask seemed a priori to be particularly interesting for the development of a generic approach, as language-independent as possible. This would be the case of a system that estimates the semantic relatedness of a pair of sentences without recourse to any resources external to the material and even without taking into account the other instances present in the material. A system takes other instances into account when, for example, it weights an instance features according to their frequency in the complete material, using the classic TF-IDF. A system is completely independent of other instances when the processing of one instance is not affected in any way by the other instances it has to predict. The system proposed by the SATLab fulfills this requirement by using the Euclidean distance between the two sentences, calculated on the basis of the frequency of the ngrams of characters that make them up. If such a system proves successful to predict semantic relatedness, it could become a potential candidate for the analysis of any language.

Admittedly, such a system is more akin to a baseline than a state-of-the-art system. However, it should also be noted that systems based on character ngrams have for many years been considered particularly effective for NLP tasks such as language identification, error correction, information retrieval and even for hate speech and offensive content identification \citep{Damashek1995,BYFIRE21}. Character ngrams have the advantage of not requiring material to be tokenized, which can be problematic in some Asian languages, and of being able to extract morphological information at very low cost \citep{peng-etal-2003-language-task}.

This paper presents SATLab's participation in SemEval 2024 Task 1 with this fully instance-specific system. The following section introduces the task and describes the proposed system. The results obtained are then reported.

\section{The Unsupervised Task}
Subtask 1B of SemEval 2023 \citep{semrel2024task} asked participating teams to estimate the semantic relatedness between pairs of sentences in twelve languages: five Afro-Asiatic (Algerian Arabic [arq], Amharic [amh], Hausa [hau], Modern Standard Arabic [arb], Moroccan Arabic [ary]), five Indo-European (Afrikaans [afr], English [eng], Hindi [hin], Punjabi [pan] and Spanish [spa]), one Austronesian (Indonesian [ind]) and one from the Niger-Congo family (Kinyarwanda [kin]). The material, collected by \citet{semrel2024dataset}, was selected from various resources such as semantic similarity datasets, news articles and Wikipedia texts. After this material had been carefully checked, it was submitted to native speakers whose task was to assess the semantic relatedness between pairs of sentences using the Best-Worst Scaling procedure. \citet{semrel2024dataset} reported high to near-perfect inter-rater reliabilities (split-half correlations: Min = 0.64, Max = 0.96).

In this Task 1B, the systems had to be unsupervised, since no dataset including evaluations of semantic relatedness between sentence pairs or texts could be employed. It should be noted, however, that the organizers provided participants with development data similar to that provided later for the testing phase, and that a team's predictions for these data could be evaluated by submitting them to CodaLab. The few tests I carried out showed that performance varied greatly depending on the language. It therefore didn't seem advisable to rely on this development material to make general decisions about the system to be developed. In the testing phase, only one prediction for each language could be submitted, and the performance measure was Spearman's rank correlation coefficient.

\section{The SATLab System}
A single system was used for all twelve languages. It is adapted from the one developed for the authorship identification of source code \citep{BYFIRE20}. This system takes as input each pair of utterances and outputs a distance between them without any other information, either from the rest of the material or external to it. Each pair of utterances is therefore processed in a way that is completely independent of the other pairs present in the material. 

The only pre-processing is the lower-casing of all texts as included in SAS. I have to admit that it's not obvious to me what impact this has on languages as unknown to me as Kinyarwanda or Amharic. No tokenization or lemmatization has been applied. The system uses character ngrams made up of 1 to 5 characters. All characters are taken into account, including spaces, punctuation marks, symbols, characters from other writing systems, etc. The ngrams at the beginning and end of each statement are distinguished from the others. All ngrams in a statement are retained, so there is no frequency threshold. The frequency of each feature is weighted by a logarithmic function using the formula: 
$1 + \log(Freq)$. Finally, the features of each statement are weighted by the L2 norm (thus instance-wise). Most of these system components have be taken from the one developed for a difficult language identification problem \citep{bestgen-2021-optimizing}.

The Euclidean distance between the sets of ngrams of each utterance in a pair is used to estimate the semantic dissimilarity between these utterances. Before submission, these distances are transformed into similarity by ranking them from largest to smallest. No information is lost through such ranking, since the organizers have chosen a rank correlation as the efficiency criterion.

\begin{figure*}
 \begin{center}
  \includegraphics[width=15cm]{graphFull}
 \end{center}
 \caption{Performances of all systems for the twelve languages}
\end{figure*}

\section{Analysis and Results}

\subsection{Official Results}
Twelve teams took part in the test phase of Task 1B, but only five proposed solutions for all twelve languages. One team proposed a solution for all languages except Spanish. The organizers provided a baseline based on the number of shared words between the two sentences of a pair (SemRel Lexical Overlap Baseline, see \citet{semrel2024task} for details).

Figure 1 shows the performance of all the systems for the twelve languages, highlighting the baseline and the system proposed by the SATLab. Marks not connected by a line are from systems that did not submit a solution for all languages. I don't know whether the systems proposed by the other teams are identical for all twelve languages, as is the case for the baseline and the SATLab.

This figure merits several comments. Firstly, when we analyze the overall results, we observe that the profiles of the teams\footnote{In this discussion of results, the baseline is considered a "team".} who submitted for all languages are similar. This observation is confirmed by an analysis of the Pearson correlations between these profiles. The lowest correlation is 0.54, only two are below 0.63 and half of them are above 0.73. These profiles highlight strong variations in performance according to language. While almost all the teams performed well to very well for Afrikaans (afr), Amharic (amh), English (eng) and Spanish (spa), they performed poorly for Punjabi (pan), with the SATLab system even achieving a negative correlation. It therefore appears that the material for some languages is considerably more complicated than for others. A detailed analysis of the differences between these materials would therefore be very useful. 

Figure 1 also shows that the SATLab's performance is as good as or better than that of other teams in the vast majority of languages, but there is little difference between the best teams. This second observation would certainly be confirmed if confidence intervals, obtained by bootstrapping \citep{bestgen-2022-please}, were presented, but their calculation requires access to the predictions of all systems. In any case, when performances are so close, it is essential to take into account other factors such as computational complexity, which will be possible when reading the system descriptions of the other teams.

Finally, Figure 1 also shows that the organizers' baseline is superior to all other systems for two languages: Hindi (hin) and Moroccan Arabic [ary]. Clearly, this is an underperformance by all participants.
\begin{table*}
\small
\centering
\label{tab:par2}
\begin{tabular}{lrrrrrrrrrrr}
\toprule
Expe & \multicolumn{1}{c}{afr} & \multicolumn{1}{c}{amh} & \multicolumn{1}{c}{arb} & \multicolumn{1}{c}{arq} & \multicolumn{1}{c}{ary} & \multicolumn{1}{c}{eng} & \multicolumn{1}{c}{hau} & \multicolumn{1}{c}{hin} & \multicolumn{1}{c}{ind} & \multicolumn{1}{c}{kin} & \multicolumn{1}{c}{pan} \\
\midrule
Submitted & 0.761 & 0.764 & 0.487 & 0.521 & 0.599 & 0.774 & 0.513 & 0.649 & 0.491 & 0.458 & -0.215 \\ 
\midrule
No Lowercase & 0.005 & 0.000 & 0.000 & 0.000 & 0.000 & -0.02 & -0.028 & 0.000 & 0.005 & 0.005 & 0.000 \\ 
%Log(1+freq) & 0.000 & 0.003 & -0.006 & 0.000 & -0.011 & 0.000 & -0.003 & 0.001 & 0.000 & 0.000 & 0.005 \\ 
4-grams & -0.003 & -0.001 & -0.016 & 0.005 & -0.012 & -0.002 & -0.015 & -0.007 & 0.007 & 0.012 & 0.018 \\ 
6-grams & 0.001 & 0.000 & 0.004 & -0.009 & -0.002 & -0.001 & 0.003 & 0.000 & -0.012 & -0.004 & -0.009 \\ 
TF-IDF & 0.021 & 0.001 & 0.061 & 0.052 & 0.024 & 0.024 & 0.057 & 0.046 & -0.052 & 0.069 & 0.002 \\ 
BM25 & 0.011 & -0.008 & 0.043 & -0.085 & 0.005 & 0.014 & 0.026 & -0.091 & -0.077 & 0.083 & 0.032 \\ 
No L2 & -0.144 & -0.247 & -0.421 & -0.574 & 0.211 & -0.245 & -0.059 & -0.432 & 0.003 & -0.157 & -0.135 \\ 
Cosinus & -0.008 & 0.013 & -0.013 & -0.022 & -0.003 & -0.005 & -0.012 & -0.020 & 0.001 & -0.035 & 0.005 \\ 
Dice & -0.001 & 0.003 & 0.032 & 0.036 & 0.022 & 0.012 & 0.029 & 0.002 & 0.005 & -0.021 & 0.003 \\ 
\midrule
Best & 0.782 & 0.765 & 0.548 & 0.573 & 0.623 & 0.798 & 0.570 & 0.695 & 0.439 & 0.527 & -0.213 \\
\bottomrule
\end{tabular}
\caption{Analysis of the impact of the system components}
\end{table*} 
\subsection{System Component Analysis}

To assess the contribution of each component to the system overall performance, all of them were modified, one at a time, and the system was re-evaluated using the gold standard provided by the task organizers for eleven languages. The results are shown in Table 1 using the difference between each modified system and the official SATLab system, whose performance is shown in the first row. 

The only pre-processing of the material carried out, the lower casing, brings benefits in only two languages. Presumably, it doesn't affect the many languages that don't use Latin characters. Using ngrams whose maximum length is one character shorter or one character longer has very little impact. On the other hand, feature weighting by TF-IDF is beneficial in ten out of eleven languages. Not using L2 normalization profoundly alters performance. While it brings significant benefit in one language, the impact is negative in nine languages, and can reach -0.574. As far as distance is concerned, Dice is more efficient than the Euclidean distance, but the gain is significantly lower than that obtained by applying the Euclidean distance to the weights transformed by TF-IDF.

The last line gives the correlations obtained by the system when TF-IDF is used instead of the logarithmic weighting. The gains over the official SATLab submission are sufficiently large to conclude that a fully instance-specific approach is significantly less effective at predicting STR than an approach that takes into account the other instances of the test material (which TF-IDF does, as explained in the introduction). There is no point in comparing these correlations with those of the other participants, since they would certainly have submitted a different system if they had been able to optimize it as just done. 

\section{Conclusion}

This paper presents the SATLab participation in SemEval 2024 Task 1: Semantic Textual Relatedness (STR). The proposed system predicts semantic relatedness by means of the Euclidean distance between two sentences, calculated on the basis of the frequency of the ngrams of characters that make them up. It employs no resources external to the material and extracts no information from other instances present in the material. The system performs well, coming first in five of the twelve languages. However, there is little difference between the best systems. What's more, the baseline proposed by the organizers was better than all the systems proposed by the participants in two languages.

Analysis of the system's components shows that the decision to develop a fully instance-specific approach was clearly the wrong one. Simply taking into account the frequencies of features in the material as a whole, as the TF-IDF weighting system does, provides a significant benefit, as \citet{Damashek1995} has already pointed out when character ngrams are used in other NLP tasks. 

The performance of all teams varies considerably according to language. It would be very interesting to carry out further research to try and understand the origin of these fluctuations. Otherwise, this type of unsupervised approach cannot be recommended, since negative correlations are observed for one of the languages. It is possible that this is linked to the way in which the material has been designed, which varies greatly depending on the language for obvious reasons of unavailability of certain resources \citep{semrel2024dataset}.

\section{Ethical Considerations}
The ethical issues raised by this research are identical to those described by the researchers who collected the data \citep{semrel2024dataset} and by the researchers who organized this task \citep{semrel2024task}.

\section*{Acknowledgements}
The author wishes to thank the organizers of this shared task for putting together this valuable event and for their availability throughout the task. He is a Research Associate of the Fonds de la Recherche Scientifique - FNRS (F\'ed\'eration Wallonie Bruxelles de Belgique).

\bibliography{anthology,custom}

\end{document}
