% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
%\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Essential packages
\usepackage{fontspec}
\usepackage{polyglossia}
\setmainlanguage{english}
\setotherlanguage{amharic}

% Set a font that supports Amharic script
\newfontfamily{\amharicfont}[Script=Ethiopic, Scale=1]{Abyssinica SIL}

% Awei 这里进行修改
% Remove the "review" option to generate the final version.
%\usepackage[review]{ACL2023}
\usepackage{ACL2023}


% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% Multilingual Xlatex + two packages
%\usepackage{fontspec}
%\usepackage{polyglossia}

% Graphicx
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{placeins}
\usepackage{subcaption}

% Set Color
\usepackage{xcolor}
\definecolor{blue}{RGB}{47,85,151}   
\definecolor{brown}{RGB}{197,90,17}
\definecolor{green}{RGB}{112,173,71}
\definecolor{red}{RGB}{255,0,0}
\definecolor{purple}{RGB}{112,48,160}

% Set Table
\usepackage{booktabs}
\usepackage{tabularx}
\usepackage{adjustbox}



% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{YNU-HPCC at SemEval-2024 Task 1: Self-Instruction Learning with Black-box Optimization for Semantic Textual Relatedness}

\author{
  Weijie Li, Jin Wang and Xuejie Zhang\\
  School of Information Science and Engineering \\
  Yunnan University \\
  Kunming, China \\
  \texttt{liweijie01@stu.ynu.edu.cn, \{wangjin, xjzhang\}@ynu.edu.cn}
}

\begin{document}
\maketitle
\begin{abstract}
This paper introduces a system designed for SemEval-2024 Task 1 that focuses on assessing Semantic Textual Relatedness (STR) between sentence pairs, including its multilingual version. STR, which evaluates the coherence of sentences, is distinct from Semantic Textual Similarity (STS). However, Large Language Models (LLMs) such as ERNIE-Bot-turbo, typically trained on STS data, often struggle to differentiate between the two concepts. To address this, we developed a self-instruction method that enhances their performance distinguishing STR, particularly in cases with high STS but low STR. Beginning with a task description, the system generates new task instructions refined through human feedback. It then iteratively enhances these instructions by comparing them to the original and evaluating the differences. Utilizing the Large Language Models' (LLMs) natural language comprehension abilities, the system aims to produce progressively optimized instructions based on the resulting scores. Through our optimized instructions, ERNIE-Bot-turbo exceeds the performance of conventional models in Track A,achieving a score enhancement of 4 to 7\% on multilingual development datasets.    
\end{abstract}

\section{Introduction}
SemEval-2024 Task 1\cite{ousidhoum2024semrel2024} addresses the challenge of Semantic Textual Relatedness (STR), which goes beyond paraphrasing and entailment of Semantic Textual Similarity (STS)\cite{agirre-etal-2012-semeval,agirre-etal-2016-semeval,cer-etal-2017-semeval,xu-etal-2015-semeval} by considering topics and logical connections between sentence pairs. This task is particularly complex due to the nuanced context required for STR, a feature not fully captured by existing models trained predominantly on STS data. This gap can lead to black-box Large Language Models (LLMs) misinterpretations like ERNIE-Bot-turbo\footnote{https://yiyan.baidu.com/}.

Our study introduces a self-instruction method to enhance the distinction between STS and STR in LLMs\cite{chen2023instructzero,zhang-etal-2023-auto,hou-etal-2022-metaprompting,wei2021finetuned}. In our approach, back translation\cite{sennrich-etal-2016-improving} converts low-resource language sentence pairs into English as inputs for LLMs. With a task description as the starting point, the black-box LLMs generate a new task instruction, which will be refined based on human feedback. The system iteratively refines the enhanced instruction by assessing it against the original and using the resulting score to produce increasingly optimized instructions. Our method improves how LLMs deal with tricky cases of similar but unrelated texts. Using our optimized instructions, ERNIE-Bot-turbo outperforms standard models and boosts scores by 4 to 7\% on multilingual development datasets in Track A. The ranking of each Tack A's test dataset is as follows: English (36), Amharic (11), Algerian Arabic (24), Telugu (24), Spanish (24), Moroccan Arabic (24), Marathi (25), Kinyarwanda (20), and Hausa (20). The remainder of this paper is organized as follows. \hyperref[sec:2]{Section 2} describes the model and method used in our system, \hyperref[sec:target]{Section 3} discusses the results of the experiments, and finally, conclusions are drawn in \hyperref[sec:4]{Section 4}.


\section{Methodology}
\label{sec:2}

\begin{figure*}[!t]
\label{fig:self-instruction-method}
\centering
\includegraphics[width=0.95\textwidth]{img/The framework.pdf}
\caption{The framework of self-instruction method}
\end{figure*}

\hyperref[fig:self-instruction-method]{Figure 1} illustrates the overall framework of our self-instruction method. We employ back translation for datasets encompassing multiple languages to render sentence pairs into English as the input for LLMs. With a task description as the starting point, the black-box LLMs generates a new task instruction which will be refined based on human feedback. The enhanced instruction is subsequently assessed against the original, generating a score that informs the optimization cycle. The system progressively refines the instructions in response to this score, resulting in progressively more optimized directives. The ensuing section will delve into a detailed analysis of this iterative optimization process.

\subsection{Design of task instruction}
Using sentences from the Amharic dataset as examples of hard sample with high Semantic Textual Similarity (STS) yet subtle Textual Relatedness (STR) – for instance, What made him so certain? What contributed to his happiness? (original Amharic: "\textamharic{ይህን ያህል እርግጠኛ እንዲሆን ያደረገው ምንድን ነው? ደስተኛ እንዲሆን አስተዋጽኦ ያደረገው ምንድን ነው?}"; goal label: 0.39) – we underscore the significance of three components: instruction, the Chain of Thought (CoT)\cite{wei2022chain}, and easily confused examples\cite{zhang-etal-2022-active,li-qiu-2023-finding}. Human generated instructions aid LLMs in grasping the primary task but may not adequately explicate the concept of semantic textual relatedness (\hyperref[fig:task_instruction_design]{Figure 2.a}) (Pred Score: 0.83). The CoT process facilitates LLMs in logical reasoning and analyzing sentence pairs, yet it encounters obstacles with complex samples prone to creating illusions (\hyperref[fig:task_instruction_design]{Figure 2.b}) (Pred Score: 0.77). Easily confused examples are practical in dissecting hard samples but can skew the assessment of standard samples (\hyperref[fig:task_instruction_design]{Figure 2.c}) (Pred Score: 0.67). Consequently, merging these approaches could provide more practical guidance for LLMs in discerning the relatedness of sentence pairs (\hyperref[fig:task_instruction_design]{Figure 2.d}) (Prediction Score: 0.35). Detailed findings from the ablation study are discussed in  \hyperref[sec:target]{Section 3}.

\begin{figure*}[!t]
\label{fig:task_instruction_design}
\centering
\includegraphics[width=\textwidth]{img/task instruction design.pdf} % Replace with the actual path to your image
\caption{The process of task instruction design. (a)\textcolor{blue}{instruction}(b)\textcolor{blue}{instruction} + \textcolor{brown}{chain of thought}(c)\textcolor{blue}{instruction} + \textcolor{green}{easily confused examples}(d)\textcolor{blue}{instruction} + \textcolor{brown}{chain of thought} + \textcolor{green}{easily confused examples}}
\end{figure*}

\subsection{Two fundamental components to generate the task instruction}

\textbf{Making use of natural language task description}. LLMs excel in understanding natural language and simplifying the definition of optimization tasks. Capitalizing on this, we employ LLMs to convert the initial task description into detailed task instruction, guiding the LLMs to perform tasks such as STR analysis effectively, as indicated in \hyperref[fig:original-instruction-optimization]{Figure 3.a}.
\\ \textbf{Refining task instruction through human feedback and evaluating their performance.} While Large Language Models (LLMs) can generate task instructions from description, these instructions often fall short of being optimal and thus require human refinement and critical feedback. For instance, LLMs may overlook the significance of high and low relatedness (\hyperref[fig:original-instruction-optimization]{Figure 3.b}). Subsequently, the improved instructions are evaluated, and their scores and instructions are integrated into the original framework, streamlining the subsequent optimization process (\hyperref[fig:original-instruction-optimization]{Figure 3.c}).

\begin{figure}[!t]
\label{fig:original-instruction-optimization}
\centering
\includegraphics[width=\linewidth]{img/Framework1.pdf}
\caption{The process of original instruction optimization:(a)task instruction(b)task instruction optimization (c)overall process}
\end{figure}

\subsection{LLMs as the black-box optimizer}

\begin{figure}[!t]
\label{fig:instruction-optimization}
\centering
\includegraphics[width=\linewidth]{img/Framework2.pdf}
\caption{The process of utilizing LLM as the black-box optimizer}
\end{figure}

After obtaining the original instruction-score (\hyperref[fig:instruction-optimization]{Figure 4.a}), we utilize LLMs as the black-box optimizer to update and optimize the instructions iteratively. In each optimization step, the optimizer LLM generates candidate optimal instructions by analyzing the strengths of high-scoring and the weaknesses of low-scoring instructions (\hyperref[fig:instruction-optimization]{Figure 4.b}). Subsequently, the new instruction is evaluated and added to the instruction-score list for sorting. From the instruction-score list, the top five high-scoring and the bottom five low-scoring instructions are selected and added to the instruction optimization template. The optimization process continues until the LLMs cannot propose new solutions with better optimization scores or the maximum number of optimization steps is reached.

\section{Experimental Result}
\label{sec:target}

\begin{figure*}[!t]
\label{fig:dataset}
\centering
\includegraphics[width=\textwidth]{img/datasets.pdf}
\caption{The composition of the Track A's training(a), test(b), and development(c) dataset}
\end{figure*}

\textbf{Datasets.} The STR task dataset comprises datasets in 14 distinct languages, including 9 languages specifically for Track A. Each language dataset contains pairs of sentences, where each pair in the training, development, and test sets is assigned a gold score. This score reflects the degree of STR between the two sentences, ranging from 0 to 1, as determined by manual annotation. \hyperref[fig:dataset]{Figure 5} below presents the composition of the training, test, and development sets for Track A.\\
\textbf{Evaluation Metrics.} The STR in Track A is evaluated using the spearman rank correlation coefficient\cite{sedgwick2014spearman}, which measures how well the system predicted rankings of test instances align with human judgment. The metric will be calculated as follows:

\begin{equation}
    \rho = 1 - \frac{6 \sum d_{i}^{2}}{n(n^{2}-1)}
\end{equation}
where $d_{i}$ represents the difference between the ranks of the $i$-th pair of sentences,$n$ is the number of pairs of sentences,$\rho$ is the spearman rank correlation.

\subsection{Implementation Details}

\begin{table}[!t]
\caption{The evaluation scores of representative models from the four model methods on the training set.}
\label{tab:score-model}
\centering
\resizebox{\linewidth}{!}{
\begin{tabular}{@{}cccc@{}}
\toprule
\multicolumn{2}{c}{BERT} & \multicolumn{2}{c}{Dual Sentence Encoding} \\ \midrule
Model & Score & Model & Score \\ \midrule
bert-base-uncased & 0.673 & all-mpnet-base-v2 & 0.787 \\
bert-large-uncased & 0.609 & \textbf{all-MiniLM-L6-v2} & \textbf{0.824} \\
distilbert-base-uncased & 0.673 & all-MiniLM-L12-v2 & 0.816 \\
deberta-base & 0.668 & all-distilroberta-v1 & 0.802 \\
\textbf{deberta-large} & \textbf{0.678} & sentence-t5-base & 0.805 \\
deberta-large-mnli & 0.659 & sentence-t5-large & 0.81 \\
deberta-xlarge-mnli & 0.651 & sentence-t5-xl & 0.805 \\
distilroberta-base & 0.618 & moco-sentencebertV2.0 & 0.797 \\
roberta-base & 0.635 &  &  \\
roberta-large & 0.44 &  &  \\
roberta-large-mnli & 0.439 &  &  \\
\midrule
\multicolumn{2}{c}{Contrastive Learning} & \multicolumn{2}{c}{LLM} \\
\midrule
Model & Score & Model & Score \\
\midrule
sup-SimCSE-VietNamese-phobert-base & 0.64 & t5-base & 0.705 \\
sup-simcse-roberta-large & 0.743 & t5-large & 0.702 \\
sup-simcse-roberta-base & 0.744 & flan-t5-base & 0.665 \\
\textbf{sup-simcse-bert-base-uncased} & \textbf{0.8} & flan-t5-large & 0.679 \\
unsup-simcse-roberta-large & 0.769 & ERNIE-Bot-turbo(w/o opt) & 0.782 \\
diffcse-bert-base-uncased-sts & 0.783 & \textbf{ERNIE-Bot-turbo(w/ opt)} & \textbf{0.883} \\
diffcse-bert-base-uncased-trans & 0.761 &  &  \\
diffcse-roberta-base-sts & 0.774 &  &  \\
diffcse-roberta-base-trans & 0.78 &  &  \\
esimcse-bert-base-uncased & 0.778 &  &  \\
\textbf{esimcse-bert-large-uncased} & \textbf{0.798} &  &  \\
\textbf{esimcse-roberta-base} & \textbf{0.792} &  &  \\
esimcse-roberta-large & 0.764 &  &  \\
pcl-bert-base-uncased & 0.776 &  &  \\
\textbf{pcl-bert-large-uncased} & \textbf{0.799} &  &  \\
pcl-roberta-base & 0.766 &  &  \\
pcl-roberta-large & 0.755 &  &  \\
\bottomrule
\end{tabular}}
\end{table}


Our approach, addressing the scarcity of low-resource languages, uses back translation to convert their sentence pairs into English for (LLMs) inputs. This experiment prioritizes scoring on English datasets to select the most effective score model. We assess four baseline methods: BERT\cite{devlin-etal-2019-bert,sanh2019distilbert,he2020deberta,delobelle-etal-2020-robbert,raffel2020exploring,chung2022scaling}, dual sentence encoding\cite{reimers-gurevych-2019-sentence,ni-etal-2022-sentence}, contrastive learning\cite{gao-etal-2021-simcse,song2020mpnet,wang2020minilm,chuang-etal-2022-diffcse,wu-etal-2022-esimcse,wu2022pcl} , and LLMs. These models were evaluated using the training set, with results presented in \hyperref[tab:score-model]{Table 1}. Considering our experimental objective of analyzing hard samples and scoring sentence pair STR, ERNIE-Bot-turbo was chosen as the scoring model. The LLMs utilized as the optimizer and scorer are: (a)optimizer LLM: gpt-3.5-turbo and (b)scorer LLM: ERNIE-Bot-turbo.

\subsection{Desigin of task instruction}

At the experiment's outset, we performed adaptation tests on the English training dataset using four variations of instruction templates: (1) instruction only, (2) instruction with chain-of-thought, (3) instruction with easily confused examples, and (4) instruction with both chain-of-thought and easily confused examples. The experimental results in \hyperref[fig:ablation-study]{Figure 6} suggest that combining instruction, chain-of-thought, and easily confused examples significantly aids LLMs in semantic textual relatedness analysis.

\begin{figure}[!t]
\centering
\includegraphics[width=\linewidth]{img/ablation-study.pdf}
\caption{The ablation tests for four variations of instruction templates: 1.\textcolor{red}{instruction} 2. \textcolor{green}{instruction + chain of thought} 3.\textcolor{blue}{instruction + easily confused examples} 4.\textcolor{brown}{instruction + chain of thought + easily confused examples}}
\label{fig:ablation-study}
\end{figure}

\subsection{Prompt optimization}

The score LLM operates at a temperature of 0, ensuring deterministic decoding, whereas the optimizing LLM uses a temperature of 0.95 promoting creativity in instruction generation. \hyperref[fig:a]{Figure 7.a} illustrates the accuracy fluctuations during the model's evaluation on the English training dataset. \hyperref[fig:b]{Figure 7.b} presents the scores for Track A's development in three scenarios: without optimization, optimized (val-score: 0.8360) and further optimized (val-score: 0.8839). \hyperref[fig:challenge sample]{Figure 8} delves into the impact of these three optimization scenarios on hard samples. Consequently, our methodology effectively reduces the hallucinations of LLMs in STS and STR tasks. This leads to a more comprehensive analysis of hard samples and consistently improves performance evaluations on the training dataset through an iterative process.

\begin{figure}[!t]
    \centering
    \begin{subfigure}[b]{\linewidth}
        \centering
        \includegraphics[width=\linewidth]{img/opt-a.pdf}
        \label{fig:a}
    \end{subfigure}
    \begin{subfigure}[b]{\linewidth}
        \centering
        \includegraphics[width=\linewidth]{img/opt-b.pdf}
        \label{fig:b}
    \end{subfigure}
    \caption{(a) shows changes in accuracy during evaluation on the English training dataset. (b) shows the development scores for Track A in scenarios: without optimization, optimized (val-score: 0.8360), and further optimized (val-score: 0.8839).}
    \label{fig:opt}
\end{figure}

\begin{figure*}[!t]
\centering
\includegraphics[width=\linewidth]{img/challenge sample.pdf}
\caption{It demonstrates how the scoring model assesses the impact of these three optimization scenarios on hard samples.(red: \textcolor{red}{score}, brown: \textcolor{brown}{chain-of-thought optimization},blue: \textcolor{blue}{example analysis optimization}, purple: \textcolor{purple}{instruction optimization})}
\label{fig:challenge sample}
\end{figure*}

\subsection{Result and Discussion}

\begin{figure}[!t]
\label{fig:dev}
\centering
\includegraphics[width=\linewidth]{img/dev.pdf}
\caption{The performance of “no optimization” and “optimization” in development dataset.}

\end{figure}

\textbf{Results.} Our final evaluation compared the 'no optimization' approach to 'optimization' across Track A's nine language development datasets using back translation, as shown in \hyperref[fig:opt]{Figure 9}.The outcomes indicate that optimized instructions significantly enhanced performance by 4 to 7\% over the non-optimized approach.The ranking of each test dataset are as follows: English (36), Amharic (11), Algerian Arabic (24), Telugu (24), Spanish (24), Moroccan Arabic (24), Marathi (25), Kinyarwanda (20), and Hausa (20).\\
\textbf{Discussion.} The experimental results suggest the following:
\begin{itemize}
    \item Our self-instruction method effectively reduces confusion between STS and STR in Large Language Models (LLMs), thereby improving accuracy and enhancing the LLMs's capability to analyze standard samples, particularly in examining hard sample.
    \item However, the experimental outcomes are somewhat modest due to the coarse granularity of the STR task and the considerable overlap between semantic textual similarity and relatedness.
    \item The back translation method encounters notable challenges when utilized with low-resource languages such as Arabic. This is primarily due to significant language biases between low-resource and high-resource languages like English within the semantic space, directly influencing the scoring model's judgment.
    \item The limitation of the score model is still an obstacle to performance. ERNIE-Bot-turbo (score model), trained on Chinese and English datasets corpus, demonstrates weaker proficiency in evaluating English sentence pairs.
\end{itemize}

\section{Conclusion}
\label{sec:4}

In this paper, we developed a self-instruction method that enhances LLMs' ability to distinguish between Semantic Textual Similarity (STS) and Semantic Textual Relatedness (STR), particularly in hard samples(High STS but low STR).Through this method, ERNIE-Bot-turbo (score LLM) not only surpasses the performance of conventional models, achieving a score enhancement of 4 to 7 \% on multilingual development datasets, but also effectively reduces confusion between STS and STR in Large Language Models (LLMs). Additionally, it achieved a commendable ranking in the final test evaluation. Our work demonstrates that optimized instructions, chain of thought, and easily confused examples enable LLMs to mitigate errors even in few-shot samples. Future research will aim to refine LLMs' capacity to grasp the overall semantic meaning of sentences further.


\section*{Acknowledgement}

The authors would like to thank the anonymous reviewers for their constructive comments.This work was supported by the National Natural Science Foundation of China (NSFC) under Grant Nos.61966038 and 62266051.

% Entries for the entire Anthology, followed by custom entries
\bibliographystyle{acl_natbib}
\bibliography{anthology,custom}

\end{document}
