% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
\newcommand{\setParDis}{\setlength {\parskip} {0.2cm} }
\newcommand{\setParDef}{\setlength {\parskip} {0pt} }
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage{ACL2023}
\usepackage{graphicx}
% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{amsmath}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets
\usepackage{booktabs}
% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
\usepackage{amsmath}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{YNU-HPCC at SemEval-2024 Task 9: Using Pre-trained Language Models with LoRA for Multiple-choice Answering Tasks}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Jie Wang, Jin Wang, and Xuejie Zhang \\
School of Information Science and Engineering \\
Yunnan University \\
Kunming, China \\
wangjie\_qpqj@stu.ynu.edu.cn, \{wangjin, xjzhang\}@ynu.edu.cn}

\begin{document}

\maketitle
\begin{abstract}
This study describes the model built in Task 9: brainteaser in the SemEval-2024 competition, which is a multiple-choice task. As active participants in Task 9, our system strategically employs the decoding-enhanced BERT (DeBERTa) architecture enriched with disentangled attention mechanisms. Additionally, we fine-tuned our model using low-rank adaptation (LoRA) to optimize its performance further. Moreover, we integrate focal loss into our framework to address label imbalance issues. The systematic integration of these techniques has resulted in outstanding performance metrics. Upon evaluation using the provided test dataset, our system showcases commendable results, with a remarkable accuracy score of 0.9 for subtask 1, positioning us fifth among all participants. Similarly, for subtask 2, our system exhibits a substantial accuracy rate of 0.781, securing a commendable seventh-place ranking. The code for this paper is published at: \url{https://github.com/123yunnandaxue/Semveal-2024_task9}.
\end{abstract}

\section{Introduction}
The human reasoning process includes two types of thinking: vertical and horizontal. Vertical thinking is a sequential analysis process based on rationality, logic, and rules. Horizontal thinking is a divergent and creative process. The success of language models has inspired the natural language model (NLP) community to focus on tasks that require implicit and complex reasoning. Although this type of vertical thinking task is widespread, horizontal thinking puzzles have received little attention \cite{jiang-ilievski-ma:2024:SemEval2024}. Task 9 in the SemEval-2024 competition: brainteaser is a multiple-choice task that tests the model's ability to demonstrate horizontal thinking and challenge default common sense associations. The task consists of two subtasks, sentence and word puzzles \cite{jiang-etal-2023-brainteaser}.
\begin{itemize}
    \item Subtask 1: Sentence-type brain teaser where the puzzle defying commonsense is centered on sentence snippets.
    \item Subtask 2: Word-type brain teaser where the answer violates the default meaning of the word and focuses on the letter composition of the target question.
\end{itemize}
\paragraph{}
In recent years, machine learning models have garnered significant attention. Traditionally, these models have employed a two-step process involving the extraction of hand-crafted features from documents followed by classification using algorithms like Na\"ive Bayes \cite{zhang2004optimality}, SVM \cite{cortes1995support}, HMM \cite{trabelsi2012hmm}, or random forests \cite{ren2015global}. However, this approach presents limitations, such as the need for meticulous feature engineering and reliance on domain knowledge for feature design. To address these shortcomings, neural approaches have emerged. Early attempts, such as latent semantic analysis (LSA) \cite{dumais1988using} and neural language models, initially underperformed compared to classical models but paved the way for developing more powerful embedding models. Significant advancements were made with the introduction of word2vec \cite{mikolov2013efficient}, ELMo \cite{peters1802deep}, RoBERTa \cite{liu2019roberta}, GPT \cite{radford2019language}, BERT \cite{devlin2018bert}, and subsequent models like GPT-3 (Brown et al., 2020) and GShard \cite{lepikhin2020gshard}, which boast increasingly more significant parameters and training datasets \cite{minaee2021deep}.
\paragraph{}
This paper proposes a deep learning system for Task 9 in SemEval-2024, titled brainteaser. We use the decoding-enhanced BERT (DeBERTa) \citep{he2020deberta} model with disentangled attention as the base model and use LoRA \cite{hu2021lora} to fine-tune the model. Focal loss was used to address the issue of label imbalance. The back-translation method is used to enhance the original dataset, and the processed dataset is used to train the model. The experimental results of this paper were ultimately presented in Task 9 of the SemEval-2024 competition. On the original dataset, the accuracy of Task 1 was 0.9, ranking fifth; The accuracy of Subtask 2 was 0.78, ranking seventh. The rest of this paper is organized as follows. In Section 2, we provided a detailed description of the proposed system and model. The experiment and results are discussed in Section 3. Finally, Section 4 presents the conclusion.

\section{DeBERTa}
Transformer \cite{vaswani2017attention} has become the most effective neural network architecture for neural language models. Unlike recurrent neural networks (RNNs) \cite{zaremba2014recurrent} that process text sequentially, transformers apply self-attention functionality to parallelly calculate the attention weight of each word in the input text. Therefore, compared to RNNs, they can perform large-scale model training in parallel. In this paper, the DeBERTa model we use is a new transformer neural language model that improves the Bert model using two novel techniques: a disentangled attention mechanism and an enhanced mask decoder. Figure~\ref{fig:Deberta} shows the structure of the system.
\begin{figure}[!t]
\centering
\includegraphics[width=0.7\linewidth]{deberta.png}
\caption{The overall architecture of the proposed method.}
\label{fig:Deberta}
\end{figure}
\subsection{Tokenizer}
Given a training data $\mathcal{D}=\{{{X}^{(m)}},{{y}^{(m)}}\}_{m=1}^{M}$, ${{X}^{(m)}}$ is the input text, ${{y}^{(m)}}$ is the corresponding ground-true label, tokenizer is applied to transform ${{X}^{(m)}}$ as,
\begin{equation}
{X}=\{[\text{CLS}],{{x}_{1}},{{x}_{2}},...,{{x}_{n}},[\text{SEP}]\}	
\end{equation}

\noindent where ${x}_{i}$ is the token in the text, [CLS] represents the classified characters, and [SEP] represents the terminating characters.
\subsection{Encoder}
DeBERTa's encoder is mainly composed of multi-layer transformer encoders, and each transformer encoder is composed of multiple sub-layers. The following are the main components of the encoder of the DeBERTa model.
\setParDef

\setParDis
\noindent\textbf{Token embeddings.}{} Each token in the input text is first converted into the corresponding word embedding vector. First, we use an embedding layer to map each token ${{x}_{i}}$ to its corresponding word embedding vector. The embedding matrix $E$ is with dimension $V\times d$, where $V$ is the size of the vocabulary and $d$ is the dimension of the word embedding. Then, the embedding vector corresponding to the $i$-th token ${{x}_{i}}$ can be expressed as ${{e}_{i}}=E[{{x}_{i}}]$.
\setParDef

\setParDis
\noindent\textbf{Positional encoding.}\hspace{0.5em}Positional encoding represents the absolute position of each word in the input sequence. Suppose we have a position encoding matrix $P$ with dimensions $N\times d$, where $d$ is the dimension of the word embedding. Then, the position encoding vector corresponding to the i-th position ${{p}_{i}}$ can be expressed as ${{p}_{i}}=P[i]$. After adding positional encoding, the new word embedding sequence we get is $E(X)+[{{p}_{1}},{{p}_{2}},...,{{p}_{N}}]$.
\setParDef

\setParDis
\noindent\textbf{Relative positional encoding.}\hspace{0.5em}The relative position encoding matrix is a learnable parameter matrix with dimensions$L\times 2D$, where $L$ is the maximum sequence length and $D$ is the word embedding dimension. In DeBERTa, the calculation process of relative position encoding is as follows: 
\paragraph{}
For each pair of words $(i,j)$, we calculate its relative position relationship vector ${{r}_{ij}}$.
\begin{equation}
r_{ij} = {PE}_{(i-j)}
\end{equation}
	
\noindent where ${PE}_{(i-j)}$ represents the encoding vector at position $(i-j)$ in the relative position encoding matrix. Finally, the input text sequence X that needs to be sent to transformer encoder layers can be obtained by adding the word embedding vector, position encoding, and relative position encoding.
\begin{equation}
 {{x}_{i}}=E[{{x}_{i}}]+{{p}_{i}}+{{r}_{ij}}	   
\end{equation}
\begin{equation}
{X}=\{{{x}_{1}},{{x}_{2}},...,{{x}_{n}}\}  
\end{equation}
\setParDef

\setParDis
\noindent\textbf{Transformer encoder layers.} Each transformer encoder layer contains the following sub-layers: 
\begin{enumerate}
    \item Multi-head self-attention. This sub-layer allows the model to focus on different parts of the input sequence simultaneously to capture global information.
    \item Feed-forward neural network. This sub-layer contains a feed-forward neural network for non-linear transformation and feature extraction of the context vector at each position.
\end{enumerate}

\paragraph{}
When the input text sequence passes through the encoder of the DeBERTa model, it can be expressed as:
\begin{equation}
{H}=\operatorname{Encoder}(X)   
\end{equation}
\noindent where $H$ is the encoded context representation, $X$ is the input text sequence, and $\text{Encoder()}$ is the Encoder part of the DeBERTa model.

\subsection{Output Layer}
In the DeBERTa model, the output layer is usually used to predict downstream tasks, such as text classification, named entity recognition, etc. 
\setParDef

\setParDis
\noindent\textbf{Linear transformation.} First, map the output of the transformer encoder to the output space, usually through a linear transformation (fully connected layer). Assuming we have a weight matrix $W$ and a bias vector $b$, the calculation of the linear transformation can be expressed as:
\begin{equation}
Z = H \cdot W + b 
\end{equation}
\noindent where  ${Z}$ is the output after linear transformation.
\setParDef

\setParDis
\noindent\textbf{Activation function.} The softmax function is a commonly used activation function in the output layer in multi-classification problems. The formulation of the softmax function is as follows:

\begin{equation}
{softmax}(x_i) = \frac{e^{x_i}}{\sum_{j=1}^{N} e^{x_j}}
\end{equation}

\noindent where the ${{x}_{i}}$ is the $i$-th element in the input vector $Z$, and $N$ is the length of the input vector. Finally, the category label is obtained by the following formula.
\begin{equation}
\hat{y}=argmax(softmax(H \cdot W+b))
\end{equation}
\setParDef

\setParDis
\noindent\textbf{Focal loss.}\hspace{0.5em}Focal loss is a dynamically scaled cross-entropy loss. A dynamic scaling factor can dynamically reduce the weight of easily distinguishable samples during training, thereby quickly focusing the center of gravity on those difficult-to-distinguish samples. The formula for focal loss is as follows.
\begin{equation}
FL({{p}_{t}})=-{{\alpha }_{t}}{{(1-{{p}_{t}})}^{\gamma }}\log ({{p}_{t}}) 
\end{equation}
where the ${{\alpha }_{t}}$ is a trainable parameter, the $\gamma $ is a hyperparameter, and the ${{p}_{t}}$ represents the probability of the category of t obtained by softmax function.
\subsection{LoRA}
The low-rank adapter (LoRA) significantly reduces the number of trainable parameters for downstream tasks by freezing the weights of pre-trained models and injecting trainable rank decomposition matrices into each layer of the transformer architecture. Research has shown that the model quality and fine-tuning of LoRA on RoBERTa, DeBERTa, GPT-2 \cite{radford2019language}, and GPT-3 \cite{brown2020language} are equivalent or better.
\paragraph{}
LoRA injects trainable low-rank matrices into transformer layers to approximate the parameter update. For a pre-trained weight matrix $W\in {{\mathcal{R}}^{n\times d}}$, LoRA decomposes the update with a low-rank factorization,
\begin{equation}
W+\Delta W=W+{{W}^{{down}}}{{W}^{up}}
\end{equation}
\noindent where ${{W}^{{down}}}$ and ${{W}^{{up}}}$ are both trainable parameters. Specifically, LoRA applied such an update to the query and value projection matrix in the multi-head attention. For a specific input ${{H}_{{l-1}}}$ to the linear projection in multi-head attention, LoRA can be defined as,
\begin{equation}
{{{H}}_{l}}={{{H}}_{l}}+\gamma \cdot {{{H}}_{l-1}}{{W}^{{down}}}{{W}^{{up}}}
\end{equation}
\noindent where $\gamma$ was used to scale the contribution of LoRA.
\section{Experimental Results}
\setParDef

\setParDis
\noindent\textbf{Datasets.}\hspace{0.5em}The training set for subtask 1 (507 data) and subtask 2 (396 data) are processed using back-translation to enhance the model's efficiency. The dataset is translated into Chinese, Russian, Arabic, French, German, Spanish, Portuguese, Italian, and Japanese and re-translated into English. The translated results are added to the dataset to obtain the enhanced dataset: the training set for subtask 1 (5070 data) and the training set for subtask 2 (3960 data).
\setParDef

\setParDis
\noindent\textbf{Evaluation Metrics.} In this paper, the task will be evaluated based on the following two accuracy indicators.
\begin{itemize}
    \item Example-based accuracy: treat each problem (primitive/adversarial) as a separate instance.
    \item Based on group accuracy: each problem and its related adversarial instances form a group.
\end{itemize}
\setParDef

\setParDis
\noindent\textbf{Implementation Details.} In this paper, we use the back-translation method for data augmentation to improve the model's efficiency. After obtaining more data, use focal loss to address the issue of category imbalance in the data. LoRA is used to reduce the trainable parameters of downstream tasks to save computational costs.\hspace{0.5em}This is the DeBERTa-V2-xxlarge model with 48 layers and a 1536 hidden size. The total parameters are 1.5B, and it is trained with 160GB of raw data.
\setParDef

\setParDis
\noindent\textbf{Comparative\hspace{0.5em}Results.}\hspace{0.5em}In\hspace{0.5em}addition\hspace{0.5em}to\hspace{0.5em}using LoRA, this paper also attempted methods such as AdaLoRA \cite{zhang2023adaptive}, Prompt-Tuning \cite{lester2021power}, R-Drop \cite{wu2021r}, using cross-entropy and focal loss as losses, with accuracy as the evaluation metric. The results are shown in Table~\ref{tab:compressive_results}.

\begin{table}[!t]
    \centering
    \small
    \begin{tabular}{@{}llcc@{}}
        \toprule
        Method          & Loss           & Subtask 1 & Subtask 2 \\
        \midrule
        LoRA            & CE             & 0.93      & 0.51      \\
                        & Focal          & \textbf{0.83}      & \textbf{0.67 }     \\
        AdaLoRA         & CE             & 0.37      & 0.37      \\
                        & Focal          & 0.51      & 0.32      \\
        Prompt-Tuning   & CE             & 0.17      & 0.22      \\
                        & Focal          & 0.17      & 0.27      \\
        R-Drop          & CE             & \textbf{0.96}      & \textbf{0.80}      \\
                        & Focal          & 0.34      & 0.40      \\
        \bottomrule
    \end{tabular}
    \caption{Accuracy of each strategy in dev data. Results in bold are the best performance.}
    \label{tab:compressive_results}
\end{table}
\paragraph{}
Figure~\ref{fig:example1} depicts validation set accuracy for subtask 1 across various methods. Models employing LoRA and R-Drop exhibit higher accuracy with cross-entropy loss. Transitioning to focal loss saw a 0.1 drop for LoRA, whereas R-Drop experienced a significant decrease. AdaLoRA's accuracy increased by 0.14 with focal loss adoption, though performance remains subpar. 
Prompt-Tuning's accuracy remains stagnant regardless of the loss function, indicating poor performance.

\begin{figure}[!t]
    \centering
    \includegraphics[width=0.7\linewidth]{图片1.png}
    \caption{Accuracy of each strategy in dev data (Subtask 1).}
    \label{fig:example1}
\end{figure}

\paragraph{}
Figure~\ref{fig:example2} shows the accuracy of various methods on the validation set for subtask 2. From the figure, we can see that the accuracy of the model using LoRA increased by 0.16 after using focal loss. After using focal loss, AdaLoRA's accuracy dropped by 0.05. Moreover, no matter which method the model uses, its performance on subtask 2 is worse than on subtask 1.
\begin{figure}[!t]
    \centering
    \includegraphics[width=0.7\linewidth]{图片2.png}
    \caption{Accuracy of each strategy in dev data (Subtask 2).}
    \label{fig:example2}
\end{figure}
\begin{table}[!t]
    \centering
    \small % 减小字体大小
    \begin{tabular}{@{}lcc@{}}
        \toprule
        Dataset & Subtask 1 & Subtask 2 \\
        \midrule
        Back-translation method dataset & 1.0 & 1.0 \\
        Original dataset & 0.96 & 0.98 \\
        \bottomrule
    \end{tabular}
    \caption{Accuracy of LoRA in dev data.}
     \label{tab:LoRA_results}
\end{table}
\begin{table*}[!t] % 使用table*环境
    \centering
    \begin{tabular}{@{}lcc@{}}
        \toprule
        Dataset & Subtask 1 & Subtask 2 \\
        \midrule
        Original dataset & 0.900 (5) & 0.781 (7) \\
        Semantic reconstruction & 0.825 (8) & 0.719 (9) \\
        Recontextualization & 0.800 (7) & 0.812 (6) \\
        Original dataset + Semantic reconstruction & 0.825 (8) & 0.719 (9) \\
        Original dataset + Recontextualization & 0.725 (8) & 0.625 (10) \\
        Original dataset + Semantic reconstruction + Recontextualization & 0.842 (12) & 0.771 (13) \\
        \bottomrule
    \end{tabular}
    \caption{Result in the Test}
    \label{tab:final_results}
\end{table*}
\paragraph{}
Finally, we found that when using cross-entropy, R-Drop achieved the best results, with LoRA ranking second. However, after using focal loss, the accuracy of R-Drop decreased significantly. Based on the results of cross-entropy and focal loss, using LoRA yields the best result. Therefore, LoRA was chosen for model fine-tuning, and then the data augmentation dataset was used to train the model. The obtained model was retrained using the original dataset, and the results on dev are shown in Table~\ref{tab:LoRA_results}.
\paragraph{}
Table~\ref{tab:final_results} shows the competition results on the Test, with rankings displayed in parentheses. To ensure that the task evaluates reasoning ability rather than memory ability, adversarial versions of the original data are constructed in two ways. 
\begin{itemize}
    \item Semantic reconstruction: rephrasing the original question without changing the correct answer and interfering factors.
    \item Context reconstruction: maintains the original reasoning path but changes the question and answer to describe the new contextual context.
\end{itemize}
\paragraph{}
As shown in Table~\ref{tab:final_results}, our model achieved good results on both subtask 1 and subtask 2 on the original dataset.\hspace{0.5em}In subtask 1, the accuracy reached 0.9, ranking fifth, and in subtask 2, the accuracy reached 0.781, ranking seventh. Except for the final dataset, the accuracy of our model ranks in the top ten. Moreover, our model performs better on subtask 1 except for the context reconstruction dataset. It is well proven that our system has demonstrated competitive performance.
\section{Conclusions}
This paper describes a deep learning model for a multiple-choice task (Task 9: brainteaser in the SemEval-2024 competition), using DeBERTa as the base model and achieving good results, ranking fifth in accuracy in subtask 1 and ranking seventh in accuracy in subtask 2. However, there is still considerable room for improvement in the model. Therefore, we will try more methods to improve the model's efficiency in the future.
\section*{Acknowledgements}
The authors would like to thank the anonymous reviewers for their constructive comments. This work was supported by the National Natural Science Foundation of China (NSFC) under Grant Nos. 61966038 and 62266051.
% Entries for the entire Anthology, followed by custom entries

\bibliography{reference}
\bibliographystyle{acl_natbib}

\end{document}