% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
% \usepackage[review]{acl}

\usepackage[preprint]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage {graphicx}
% \usepackage{url}


% This is also not strictly necessary and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{HW-TSC 2024 Submission for the SemEval-2024 Task 1: Semantic Textual Relatedness (STR)}

\author{Mengyao Piao, Chang Su, Yuang Li, Xiaosong Qiao, Xiaofeng Zhao, \\
        \textbf{Yinglu Li, Min Zhang, Hao Yang, Dandan Tu} \\
        Huawei Translation Services Center, China\\
        \{piaomengyao1, suchang8, liyuang3, qiaoxiaosong, zhaoxiaofeng14,\\ liyinglu, zhangmin186, yanghao30, tudandan\}@huawei.com}

\begin{document}
\maketitle
\begin{abstract}
The degree of semantic relatedness of two units of language has long been considered fundamental to understanding meaning. In this paper, we present the system of Huawei Translation Services Center (HW-TSC) for Task 1 of SemEval 2024, which aims to automatically measure the semantic relatedness of sentence pairs in African and Asian languages. The task dataset for this task covers about 14 different languages, These languages originate from five distinct language families and are predominantly spoken in Africa and Asia. For this shared task, we describe our proposed solutions, including ideas and the implementation steps of the task, as well as the outcomes of each experiment on the development dataset. To enhance the performance, we leverage these experimental outcomes and construct an ensemble one. Our results demonstrate that our system achieves impressive performance on test datasets in unsupervised track B and ranked first place for the Punjabi language pair ~\footnote{\url{https://docs.google.com/spreadsheets/d/1KGN26MYVlfEOqooq-bzD6EBNnpl-YT5XrY9COKESS-g/edit?usp=sharing}}.
\end{abstract}

\section{Introduction}
The semantic relatedness of two units of language is the degree to which they are close in terms of their meaning~\cite{abdalla2021makes}. The linguistic units can be words, phrases, sentences, etc. Though our intuition of semantic relatedness is dependent on many factors such as the context of assessment, age, and socioeconomic status~\cite{harispe2015semantic}, it is argued
that a consensus can usually be reached for many pairs~\cite{harispe2015semantic}. In the SemEval 2024 shared task 1~\cite{semrel2024task}, there are three sub-tracks --- Track A: Supervised, Track B: Unsupervised, and Track C: Cross-lingual and each track involves several language pairs. Our team --- Huawei Translation Services Center (HW-TSC) --- participated in the Track B: Unsupervised one which covers most African and Asian language pairs and has to be developed without the use of any labeled data for semantic relatedness. In this paper, we describe HW-TSC's system for unsupervised semantic relatedness tasks, which leverages multiple pre-trained multilingual language models to capture the semantic relatedness of different language pairs. The main features of our system are as follows:
% 方法总结

\begin{itemize}
    \item \textbf{N-gram Chars Method}: We employ the tokenizers of two base models for this method. The first one is XLM-RoBERTa~\cite{conneau2019unsupervised}, a large unsupervised cross-lingual model that extends Facebook’s RoBERTa model with more languages and data. The second one is Multilingual-BERT~\cite{devlin2018bert}, a transformers model that is pre-trained on a large multilingual corpus using self-supervised objectives. To measure the similarity between two sentences, we use their n-gram dictionaries as features and compute a similarity score based on them. 
    \item \textbf{BERTScore Method}: This method adopts a metric, named BERTScore~\cite{bert-score}. It is a metric to assess the quality of the generated text. BERTScore is mainly based on the idea of computing a score from the cosine similarity of the token-level representations obtained from the BERT model for the generated and reference texts.
    \item \textbf{Pretrained Large Language Model Method}:  We use XGLM~\cite{lin2021few}, a large-scale auto-regressive language model, as the backbone of this method. XGLM is a pre-trained language model that can handle multiple languages and domains. By leveraging the powerful large language model, we can efficiently obtain the token logits and perform calculations with them.
    \item \textbf{Translate to English and N-gram Chars Method}: This method needs us to process data with a translation system first, which converts the data from various languages into English. After the translation, we follow the same procedure as the N-gram Chars Method, which uses the n-gram character dictionaries of the generated and the reference texts to compute a similarity score.
    \item \textbf{Dataset}: We utilize the original development and test dataset from SemRel2024~\cite{semrel2024dataset}, a novel collection of semantic relatedness datasets annotated by native speakers for 14 languages: Afrikaans, Algerian Arabic, Amharic, English, Hausa, Hindi, Indonesian, Kinyarwanda, Marathi, Moroccan Arabic, Modern Standard Arabic, Punjabi, Spanish, and Telugu.
\end{itemize}

In this paper, we analyze the characteristics of the shared task and describe our solutions, which include the ideas and implementation processes. We use Sentence-BERT~\cite{reimers2019sentence} as our baseline for the experiment. We conduct various experiments with the base model, large language models, etc. Our model achieves the best performance for the Punjabi language pairs in the unsupervised track B. The results are encouraging for semantic relatedness, although there is still scope for improvement.

\section{Related Work}
Our track is unsupervised, meaning that the systems submitted by participants do not rely on any labeled data for measuring semantic relatedness or similarity between text units longer than two words in any language. Consequently, any pre-trained language models that are further fine-tuned with text similarity data, using methods such as instruct-tuning, classification, or a similarity objective, are disqualified from our track. For our baseline score, we used Sentence-BERT~\cite{reimers2019sentence} (SBERT), a variant of the pre-trained BERT network that employs siamese and triplet architectures to generate sentence embeddings that are semantically meaningful and comparable by cosine similarity. SBERT has been fine-tuned on natural language inference (NLI) data, resulting in sentence embeddings that surpass other state-of-the-art methods. Hence, we selected SBERT as our baseline model and obtained our baseline score.

We introduce BERTScore~\cite{bert-score} secondary, which is an automatic evaluation metric for text generation. Analogously to common metrics, BERTScore computes a similarity score for each token in the candidate sentence with each token in the reference sentence. What is more, different from other matches, it computes token similarity using contextual embeddings. BERTScore correlates better with human judgments and
provides stronger model selection performance than existing metrics.

\section{Method}
\subsection{N-gram Chars Method}
% Ngram 词匹配
The n-gram method~\cite{kondrak2005n} is a statistical method used in natural language processing (NLP) to analyze the co-occurrence of words in a given text. It involves breaking down the text into sequences of words, where each sequence contains a fixed number of words, referred to as n-grams. The most common types of n-grams are bi-grams (2-grams), tri-grams (3-grams), and quadri-grams (4-grams), but n-grams can have any length. The primary purpose of using n-gram models is to capture the statistical dependencies between words in a language. By analyzing these dependencies, n-gram models can be used for various NLP tasks, such as language modeling, text generation, machine translation, and information retrieval. In this task, we first realized this way, for tokenizing, we tried pre-trained model XLM-Roberta (XLMR) and Multilingual-BERT (MBERT) because it is a multilingual task. At last, we calculate the similarity score with two sentences' n-gram dictionary shown as algorithm\ref{alg:ngram}.

\begin{algorithm}[!htbp]
\caption{N-gram Chars Score Method}
% \BlankLine
\begin{algorithmic}[1]
\REQUIRE Word sequences of the two sentences $Sq_a$, $Sq_b$; there length $Len_a \gets len(Sq_a)$, $Len_b \gets len(Sq_b)$; N-gram window width $N$
\ENSURE $0 < N <min(Len_a, Len_b)$

\STATE $Dict_{\{a,b\}} \gets \{\}$ 
\FOR{ $i\leftarrow 0$  \textbf{ to }  $Len_{\{a,b\}} - N$}
    \STATE $W_{\{a,b\}i} \gets Sq_{\{a,b\}}[i:i+N] $
    \IF{$W_{\{a,b\}i}$ \textbf{not in} $Dict_{\{a,b\}}$}
        \STATE $Dict_{\{a,b}\}[W_{\{a,b\}i}]=1$
    \ELSE
        \STATE $ state \gets Dict_{\{a,b\}}[W_{\{a,b\}i}]+1 $
        \STATE $ Dict_{\{a,b\}}[W_{\{a,b\}i}] \gets state $
    \ENDIF
\ENDFOR

\STATE $same \gets 0$ 
\FORALL{$key$  \textbf{ from }  $Dict_a $}
    \IF{$key$  \textbf{ is in }  $Dict_b$}
        \STATE $count \gets min(Dict_a[key], Dict_b[key])$
        \STATE $same \gets same + count$ 
    \ENDIF
\ENDFOR

\STATE $score \gets \frac{2 \times same}{Len_a + Len_b - 2N + 2}$ 
\RETURN $score$

\end{algorithmic}
\label{alg:ngram}
\end{algorithm}

\subsection{BERTScore Method}
% BERTScore 进行得分预测，
BERTScore is a metric for evaluating the quality of text generation, particularly for tasks like machine translation, summarization, and text completion. BERTScore leverages the pre-trained BERT model (Bidirectional Encoder Representations from Transformers) to measure the semantic and syntactic alignment between the generated text and its reference or target text. The core idea behind BERTScore is to compute a score based on the cosine similarity of token-level representations from the BERT model for the generated text and the reference text. Additionally, BERTScore does not require training or tuning and is based on a publicly available pre-trained model. This makes it a useful and practical tool for evaluating the quality of generated text in various natural language processing tasks. Therefore, we calculate the score with the leverage of BERTScore.

% , shown as figure\ref{fig:bertscore}.

% \begin{figure*}[!ht]
% \centering
%   \includegraphics[width=\linewidth]{latex/pics/bertscore.png}
%   \caption{Illustration of the computation of bertscore recall metric.}
%   \label{fig:bertscore}\cite{bert-score}
% \end{figure*}

\subsection{Pretrained Large Language Model Method}
% 使用大模型进行token的logits 预测
Different from the method above, we take advantage of the pre-trained large language model to obtain the logits of the token and compute the score. XGLM is an open-source general language model pre-training framework\footnote{\url{https://huggingface.co/docs/transformers/main/en/model_doc/xglm}}. The model architecture is general and can be easily extended, supporting various model scales and task-specific architectures. XGLM uses a Transformer-based architecture, after pre-training XGLM learns language structure and grammatical rules, and can generate high-quality natural language text. All in all, XGLM is a flexible and powerful general language model pre-training framework, that supports only Chinese and English. Therefore, we first use the model on the English task to get the logits of the token. Then try to calculate the sum, mean, and half of the logits in proper order.

\subsection{Translate to English and N-gram Chars Method}
% 小语种翻译成英语使用ngram 词匹配
Though XLMR and MBERT can support multiple languages, if we look closely at the training data we can see that most of it is in English. Our track mainly faced 14 different African and Asian languages, in order to satisfy our track more, we took advantage of our team to process the data with a translation system to make the data from African and Asian languages into English. And then get the logits of the token as well as the last one.

\section{Experiments Results}
%% 实验结果 
In the beginning, we applied the following three methods to the English development dataset: N-gram Chars Method with XLMR and MBERT, BERTScore Method, and Pre-trained Large Language Model Method with XGLM to calculate the sum, mean, and half of the logits. See Table~\ref{tab: eng} Different methods on English development dataset. We can see Ngram-XLMR, Ngram-MBERT, and BERTScore got really impressive performance than every method on XGLM, though XGLM is a large language model and can generate high-quality natural language text almost all the methods with XGLM are below 0.5 in the score.

\begin{table}[!hbtp]
\centering
\begin{tabular}{c c}
\hline
\textbf{Method-Model} & \textbf{Score}\\
\hline
\verb|Ngram-XLMR|  & {\ 0.651} \\
\verb|Ngram-MBERT| & {\ 0.604} \\
\verb|BERTScore|   & {\ 0.650} \\ 
\verb|sum-XGLM|    & {\ 0.091} \\ 
\verb|mean-XGLM|   & {\ 0.314} \\
\verb|half-XGLM|   & {\ 0.211} \\
\hline
\end{tabular}
\caption{Different method on English development dataset}
\label{tab: eng}
\end{table}

Afterwards, we use these methods on the Afrikaans development dataset. What's more, we add Translate to English and N-gram Chars Based method. See Table~\ref{tab:afr} Different methods on Afrikaans development dataset. we can conclude that Ngram-XLMR and BERTScore still perform better than other methods. What is more, the Translate to English and N-gram Chars Based method did not bring us too many surprises. The table shows that the methods that translate to English are almost all below the methods that did not.

\begin{table}[!hbtp]
\centering
\begin{tabular}{c c}
\hline
\textbf{Method-Model} & \textbf{Score}\\
\hline
\verb|Ngram-XLMR| & {\ 0.475} \\
\verb|Ngram-MBERT| & {\ -0.170} \\
\verb|BERTScore| & {\ 0.102} \\ 
\verb|eng-Ngram-XLMR| & {\ -0.171} \\ 
\verb|eng-Ngram-MBERT| & {\ 0.014} \\
\verb|eng-BERTScore| & {\ 0.102}  \\
\hline
\end{tabular}
\caption{Different method on Afrikaans development dataset}
\label{tab:afr}
\end{table}

From the experiments above, we can see N-gram Chars Based with XLMR and MBERT, BERTScore Based can always get better performance in English and Afrikaans. Will they get a better performance in the other 12 languages? See Table~\ref{table:mqm} this shows three methods and a Baseline on all language development datasets. To compare with the result from the three methods and baseline, we can see Ngram-XLMR and BERTScore always get better scores in all languages.

\begin{table*}[!hbtp]
\centering
\begin{tabular}{c c c c | c}
\hline
\textbf{Language} & \textbf{Ngram-XLMR} & \textbf{BERTScore}& \textbf{Ngram-MBERT}& \textbf{SBERT(Baseline)} \\
\hline
eng & 0.651 & 0.650&  0.604  &0.758 \\
afr & 0.475 & 0.102&  -0.170 &0.639 \\
amh & 0.630 & 0.085&  0.630  &0.650 \\
arb & 0.214 & 0.226&  0.214  &0.402\\
arq & 0.487 & 0.420&  0.408  &0.296\\
ary & 0.565 & 0.524&  0.462  &0.460\\
hau & 0.325 & 0.240&  0.325  &0.382\\
hin & 0.585 & 0.684&  0.581  &0.613\\
ind & 0.490 & 0.468&  0.501  &0.445\\
kin & 0.115 & 0.010&  0.130  &0.323\\
pan & 0.099 & 0.175&  0.099  &0.173\\
\hline
\end{tabular}
\caption{Top three methods and Baseline on all languages development dataset}
\label{table:mqm}
\end{table*}

At last, we make other experiments to ensemble the results of Ngram-XLMR and BERTScore methods to find out if this way can bring us better performance. We make Ngram-XLMR with ratio A, and BERTScore method with ratio (1-A). See Table~\ref{tab: ensemble eng} Different ensemble ratio with Ngram-XLMR and BERTScore on English development dataset. See Table~\ref{tab: ensemble pan} Different ensemble ratio with Ngram-XLMR and BERTScore on Punjabi development dataset. We can see that the ensemble way may or may not improve the performance. 

\begin{table}[!hbtp]
\centering
\begin{tabular}{c c}
\hline
\textbf{Ngram-XLMR ratio} & \textbf{Score}\\
\hline
\verb|0|   & {\ 0.650} \\
\verb|0.1| & {\ 0.689} \\
\verb|0.2| & {\ 0.690} \\
\verb|0.3| & {\ 0.689} \\ 
\verb|0.4| & {\ 0.685} \\ 
\verb|0.5| & {\ 0.680} \\
\verb|0.6| & {\ 0.676} \\
\verb|0.7| & {\ 0.674} \\
\verb|0.8| & {\ 0.673} \\
\verb|0.9| & {\ 0.672} \\
\verb|1|   & {\ 0.651} \\
\hline
\end{tabular}
\caption{Different ensemble ratio with Ngram-XLMR and BERTScore on English development dataset}
\label{tab: ensemble eng}
\end{table}

\begin{table}[!hbtp]
\centering
\begin{tabular}{c c}
\hline
\textbf{Ngram-XLMR ratio} & \textbf{Score}\\
\hline
\verb|0|   & {\ 0.175} \\
\verb|0.1| & {\ 0.126} \\
\verb|0.2| & {\ 0.106} \\
\verb|0.3| & {\ 0.093} \\ 
\verb|0.4| & {\ 0.088} \\ 
\verb|0.5| & {\ 0.084} \\
\verb|0.6| & {\ 0.082} \\
\verb|0.7| & {\ 0.080} \\
\verb|0.8| & {\ 0.080} \\
\verb|0.9| & {\ 0.080} \\
\verb|1|   & {\ 0.099} \\
\hline
\end{tabular}
\caption{Different ensemble ratio with Ngram-XLMR and BERTScore on Punjabi development dataset}
\label{tab: ensemble pan}
\end{table}

\section{Conclusion}
% 写结论
This paper describes HW-TSC’s unsupervised system for Semantic Textual Relatednes shared task held in SemEval 2024 Task 1 and
also presents the design, the data, and the results. The participants of the shared task were provided with a collection of unsupervised 
datasets in multiple languages. The shared task is challenging, partly due to the unsupervised development data, and can not use models that have fine-tuned with text similarity data whether through instruct-tuning (e.g., BLOOMZ~\cite{muennighoff2022crosslingual}), classification, or a similarity objective (like SBERT). Our system uses three base models with the dataset and carries out comprehensive experiments with different pre-trained models and methods. Finally, our system achieved the 1st best performance in the Punjabi language. For some of the problems reflected in this task, there is still a lot of research space. In the future, we will investigate the transfer method to transfer the knowledge of one language to multiple languages to improve efficiency and we plan to leverage other multiple languages model's skills.

\bibliography{custom}

\end{document}
