% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}


\usepackage{graphicx}
% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.
\usepackage[
    backend=biber,
    style=numeric,
    sorting=none
]{biblatex}
\addbibresource{ref.bib}

\title{JN666 at SemEval-2024 Task 7: NumEval: Numeral-Aware Language Understanding and Generation}

% Author information can be set in various styles:
% For several authors from the same institution:
\author{Xinyi Liu, Xintong Liu \and Hengyang Lu \\
        School of Artificial Intelligence and Computer Science, Jiangnan University, Wuxi, China\\1130174701@qq.com\\liuxintong@stu.jiangnan.edu.cn\\luhengyang@jiangnan.edu.cn}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

% \author{Xinyi Liu
%   , Xintong Liu
%   , Hengyang Lu}


\begin{document}
\maketitle
\begin{abstract}
This paper is submitted for SemEval-2027 task 7: Enhancing the Model’s Understanding and Generation of Numerical Values. The dataset for this task is NQuAD \cite{10.1145/3459637.3482155}, which requires us to select the most suitable option number from four numerical options to fill in the blank in a news article based on the context. Based on the BertForMultipleChoice model, we proposed two new models, MC BERT and SSC BERT, and improved the model’s numerical understanding ability by pre-training the model on numerical comparison tasks. Ultimately, our best-performing model achieved an accuracy rate of 79.40\%, which is 9.45\% higher than the accuracy rate of NEMo \cite{10.1145/3459637.3482155}.
\end{abstract}

\section{Introduction}
In the field of Natural Language Processing (NLP), the understanding and analysis of textual data have always been the main focus. However, the numerical information in these textual data is often overlooked. Numerals play a significant role in our daily life and work, providing rich information such as dates, times, quantities, proportions, and money. Although numerals may not occupy a large proportion in the text, their existence is crucial for understanding the meaning of the text.

To better evaluate the numerical understanding ability of models, \cite{10.1145/3459637.3482155} proposed the Numeral-related Question Answering Dataset (NQuAD) \cite{10.1145/3459637.3482155}, which is specifically designed to evaluate and enhance the model's ability in Reading Comprehension of the Numerals in Text. In our work, our goal is to improve the performance of the BERT model on the NQuAD \cite{10.1145/3459637.3482155}. For this purpose, we propose a new method that can effectively handle numerical information. Our experimental results show that our method has achieved significant performance improvement on the NQuAD \cite{10.1145/3459637.3482155}. This proves the effectiveness of our method in handling numerical information and also shows the potential of our method in enhancing the numerical understanding ability of the model.

The rest of this paper is organized as follows. Section 2 introduces the related work. Section 3 introduces the dataset and tasks. Section 4 describes our method in detail. Section 5 reports our experimental results and analysis. Finally, we conclude our work.


\section{Related Work}

\begin{figure*}
    \centering
    \center{\includegraphics[scale=0.4]{images/BertForMultipleChoice.png}}
    \caption{\textbf{Architecture of BertForMultipleChoice model}}
    \label{fig:bfmc}
\end{figure*}

In recent years, pre-training tasks have become increasingly prevalent in the field of Natural Language Processing (NLP). The design goal of pre-training tasks is to enhance the model’s understanding of natural language through learning from a large amount of unlabeled data, thereby improving the model’s performance on specific tasks in the subsequent fine-tuning stage. In this trend, \cite{chen-etal-2023-improving} shows that by pre-training on numerical comparison tasks, the model’s understanding of numerals can be significantly improved. The model, after pre-training, also shows a significant performance improvement on other numeral-related tasks.

Multiple choice \cite{Liu2019Neural} format represents a category within machine reading comprehension tasks. In this context, 
\cite{Jin2020Mmm:} proposed a Multi-stage Multi-task learning framework (MMM) aimed at enhancing the performance of multiple choice tasks, \cite{Zhang2020DCMN+:}introduced the Dual Co-Matching Network (DCMN+), a model that emulates human problem-solving strategies, and \cite{ma-du-2023-poe} presents a two-step strategy for enhancing the performance of Large Language Models (LLMs) on multiple choice tasks.

Recent research has increasingly recognized the significance of numerical data within text. The NQuAD \cite{10.1145/3459637.3482155} dataset has been instrumental in examining the relationship between numerical values in news headlines and the corresponding figures within the articles. Similarly, the FNXL \cite{sharma-etal-2023-financial} dataset has brought attention to the numerical data contained within the periodic financial reports submitted by publicly listed companies.

The BertForMultipleChoice model is a variant of the BERT \cite{devlin-etal-2019-bert}, specifically designed for multiple-choice tasks. Figure \ref{fig:bfmc} is the architecture of BertForMultipleChoice, it adds a multiple-choice classification head on top of the BERT model, mainly for handling multiple-choice tasks. The input to the BertForMultipleChoice model includes a question and multiple potential answers (options), and the goal of the model is to select the most reasonable answer. This model has shown excellent performance in tasks that require choosing one answer from multiple options, such as reading comprehension and sentiment analysis.


\section{Dataset and Tasks}


Figure \ref{fig:NQuAD} shows an example in NQuAD \cite{10.1145/3459637.3482155} dataset, including a news article, a question stem and four answer options. Our task is predicting the correct the option.

NQuAD \cite{10.1145/3459637.3482155} collects news articles from the data vendor, MoneyDJ1, and get the news articles within the period from June 22, 2013 to June 20, 2018. A total of 75,448 Chinese news articles are collected. A total of 43,787 news articles are selected, and 46.97\% of the headlines contain more than one numeral. The average number of numerals in the headline and in the content are 1.65 and 29.48, respectively. Each numeral in each headline is used to form a question, thus NQuAD \cite{10.1145/3459637.3482155} dataset finally obtain 71,998 questions and separate 80\% of the instances as the training set and the rest of the instances form a test set.


\section{Methods}

\begin{figure}
    \centering
    \fcolorbox{black}{white!10}{\parbox{.9\linewidth}{
    \textbf{News Article:}\\
    Major banks take the lead in self-discipline. The five major banks’ newly imposed mortgage interest rates climbed to \textbf{1.986}\% in May. ... Also approaching \textbf{2}\% integer alert ... Up to \textbf{2.5}\% ... Also increased by \textbf{0.04} percentage points from the previous month ... Prevent the housing market bubble from fully starting. \\

    \textbf{Question Stem}: Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly\_\_\_\%. \\
    
    \textbf{Answer Options}: (A) 0.04 (B) 1.986 (C) 2 (D) 2.5\\
    
    \textbf{Answer}: (C)
    }
    }
    \includegraphics{}
    \caption{\textbf{An example question in NQuAD.}}
    \label{fig:NQuAD}
\end{figure}

\begin{figure*}
    \centering
    \center{\includegraphics[scale=0.4]{images/mc.png}}
    \caption{\textbf{Architecture of the proposed model, MC BERT}}
    \label{fig:mc}
\end{figure*}

\begin{figure*}
    \centering
    \center{\includegraphics[scale=0.4]{images/ssc.png}}
    \caption{\textbf{Architecture of the proposed model, SSC BERT}}
    \label{fig:ssc}
\end{figure*}

\begin{figure*}
    \centering
    \fcolorbox{black}{white!10}{\parbox{.9\linewidth}{
    \textbf{MC BERT Input:}\\
    \texe{[[CLS]}Also increased by \textbf{0.04} percentage points from the previous month;he five major banks' newly-imposed mortgage interest rates climbed to \textbf{1.986}\% in May.;Also approaching \textbf{2}\% integer alert;Up to \textbf{2.5}[SEP]Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly \textbf{0.04}\%.[SEP],\\
    \texe{[CLS]}Also increased by \textbf{0.04} percentage points from the previous month;he five major banks' newly-imposed mortgage interest rates climbed to \textbf{1.986}\% in May.;Also approaching \textbf{2}\% integer alert;Up to \textbf{2.5}[SEP]Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly \textbf{1.986}\%.[SEP],\\
    \texe{[CLS]}Also increased by \textbf{0.04} percentage points from the previous month;he five major banks' newly-imposed mortgage interest rates climbed to \textbf{1.986}\% in May.;Also approaching \textbf{2}\% integer alert;Up to \textbf{2.5}[SEP]Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly \textbf{2}\%.[SEP],\\
    \texe{[CLS]}Also increased by \textbf{0.04} percentage points from the previous month;he five major banks' newly-imposed mortgage interest rates climbed to \textbf{1.986}\% in May.;Also approaching \textbf{2}\% integer alert;Up to \textbf{2.5}[SEP]Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly \textbf{2.5}\%.[SEP]]
    \\

    \textbf{SSC BERT Input:}\\
    \texe{[[CLS]}Also increased by \textbf{0.04} percentage points from the previous month[SEP]Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly \textbf{0.04}\%.[SEP],\\
    \texe{[CLS]}he five major banks' newly-imposed mortgage interest rates climbed to \textbf{1.986}\% in May.[SEP]Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly \textbf{1.986}\%.[SEP],\\
    \texe{[CLS]}Also approaching \textbf{2}\% integer alert[SEP]Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly \textbf{2}\%.[SEP],\\
    \texe{[CLS]}Up to \textbf{2.5}[SEP]Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly \textbf{2.5}\%.[SEP]]\\
    }
    }
    \includegraphics{}
    \caption{\textbf{Examples of MC BERT Input and SSC BERT Input.}}
    \label{fig:inputs}
\end{figure*}

\subsection{Multiple Choice}
Multiple Choice \cite{Liu2019Neural} format represents a category within machine reading comprehension tasks. Our present endeavor aligns with this format. The BertForMultipleChoice model, an adaptation of the BERT \cite{devlin-etal-2019-bert} framework, is expressly engineered to process tasks involving this format. So we choose the BertForMultipleChoice model as our baseline model.

Resent research \cite{Zhang2020DCMN+:} suggests that existing multi-choice MRC models learn the passage representation with all the sentences in one-shot, which is inefficient and counter-intuitive. Their research indicates that the model should be extremely beneficial if it focuses on a few key evidence sentences. At the same time, the “sentences\_containing\_the\_numeral\_in\_answer\_options” in the NQuAD \cite{10.1145/3459637.3482155} dataset are four sentences that contain each of the options. So, our strategy is using this sentences as the context of inputs. In the BertForMultipleChoice model, we connect this sentences as the context, and the Question Stem and Answer Options are used as the question and Choices. After that, we find that humans often employ the method of substituting potential solutions into the given problem in the context of problem-solving. Be inspired by this, we fill in the blanks of the question with the options. As shown in Figure \ref{fig:mc}, we named this new model MC BERT. Furthermore, when humans solve problems, they will finally compare the question and options with key sentences one by one, and then choose the most matching option. So we further propose an improvement strategy, that is, changing the context to the sentence corresponding to each option to increase differentiation. As shown in Figure \ref{fig:ssc}, we named this improved model SSC BERT. Here are examples of input:
\begin{itemize}
    \item 
    Context:Also increased by \textbf{0.04} percentage points from the previous month,he five major banks' newly-imposed mortgage interest rates climbed to \textbf{1.986}\% in May.,Also approaching \textbf{2}\% integer alert,Up to \textbf{2.5}\%
    \item 
    $S_A$: Also increased by \textbf{0.04} percentage points from the previous month
    \item 
    $S_B$: he five major banks' newly-imposed mortgage interest rates climbed to \textbf{1.986}\% in May.
    \item 
    $S_C$: Also approaching \textbf{2}\% integer alert
    \item 
    $S_D$: Up to \textbf{2.5}\%
    \item 
    Question: Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly \_\_\_\%. 
    \item
    $QC_A/QC_B/QC_C/QC_D$: Driven by self-discipline, the five major banks’ new mortgage interest rates are approaching nearly \textbf{0.04/1.986/2/2.5}\%.    
\end{itemize}
Figure \ref{fig:inputs} are final input examples of MC BERT and SSC BERT.


\subsection{Pre-training}
Chapter 2 mentioned that pre-training models on simple numerical comparison tasks can enhance the model’s numerical understanding. Therefore, we pre-train ‘bert-base-chinese’ on numerical comparison tasks according to the method mentioned in the \cite{chen-etal-2023-improving}. The pre-training model is then trained according to the three frameworks mentioned above. We finally obtain six models: BertForMultipleChoice, MC BERT, SSC BERT, Pre-BertForMultipleChoice, Pre-MC BERT and Pre-SSC BERT.

\subsection{Implementation Details}
\textbf{Trained on the ComNum dataset \cite{chen-etal-2023-improving}.} Given that NQuAD \cite{10.1145/3459637.3482155} is a Chinese dataset, we first replaced the English in the ComNum dataset \cite{chen-etal-2023-improving} with the corresponding Chinese. The pre-trained model we used is bert-base-chinese, the maximum sequence length is 32, batch size is 32. The loss function employed is the cross-entropy loss, with the AdamW optimizer. The learning rate is set at 5e-6, and epsilon is 1e-8.

\textbf{Trained on the NQuAD dataset \cite{10.1145/3459637.3482155}.} We employed the BertForMultipleChoice from the transformers library to train the model and divided the training set into a training set and a validation set at a ratio of 4:1. The maximum sequence length is 128, batch size is 32. The loss function employed is the cross-entropy loss, with the AdamW optimizer. The learning rate is set at 5e-6 (If the model used was trained on the ComNum dataset \cite{chen-etal-2023-improving}, the learning rate is set at 5e-5), and epsilon is 1e-8. After applying softmax function to the model’s output, we selected the index with the highest probability as the output.

\section{Result}
As shown in Table \ref{tab:result}, the accuracy of our six models all exceed NEMo \cite{10.1145/3459637.3482155}, which indicates that the BertForMultipleChoice model framework can effectively handle and solve the specific requirements and challenges of this task. Among the three models obtained by directly training with bert-base-chinese, the accuracy of MC BERT is higher than that of BertForMultipleChoice, indicating that directly filling in the blanks of the question with options is effective. Furthermore, the accuracy of SSC BERT surpasses that of MC BERT, suggesting that comparing the question and options with key sentences individually can also enhance the model’s performance. Meanwhile, The accuracy of the three models trained using pre-trained models are higher than that of the models obtained by directly training with bert-base-chinese, which further confirms the conclusion that in some simple numerical related tasks, pre-trained models can enhance the model’s numerical understanding ability \cite{chen-etal-2023-improving}.
\begin{table}
    \centering
    \begin{tabular}{cc}
        \hline
         Model&Accuracy \\
        \hline
        BERT Embedding Similarity &57.30\% \\
         Vanilla BERT&66.41\% \\
         BERT-BiGRU&67.15\% \\
         BERT-CNN&63.92\% \\
         NEMo&69.95\% \\
         BertForMultipleChoice&74.48\% \\
         MC BERT(ours)&76.83\% \\
         SSC BERT(ours)&78.21\%\\
         Pre-BertForMultipleChoice&74.91\% \\
         Pre-MC BERT(ours)&77.16\% \\
         Pre-SSC BERT(ours)&\textbf{79.40}\% \\
        \hline
    \end{tabular}
    \caption{Experimental results.}
    \label{tab:result}
\end{table}

\section{Conclusion}
In this work, we select the BertForMultiple model as the baseline model and propose two new models based on it: MC BERT and SSC BERT. The accuracy of these models all exceeded the results of NEMo \cite{10.1145/3459637.3482155}, which confirmed the applicability of the BertForMultiple model framework for our task, the results also indicate that the method of inserting options into the blanks of the question and individually comparing them with key sentences is effective. We also refer to the method in reference \cite{chen-etal-2023-improving}, pre-trained the bert-base-chinese on numerical comparison task, and use the pre-trained model for subsequent training. The experimental results show that this method can improve the model’s numerical understanding ability, thereby making the model perform better on numerical related tasks. Finally, we found that the Pre-SSC BERT model had the highest accuracy, and its accuracy was 9.45\% higher than the NEMo model \cite{10.1145/3459637.3482155}, which further proved the effectiveness of our method.In summary, our research proposes a new model training and optimization strategy, which has been proven to be effective and superior in experiments.

However, in analyzing the cases where our model made incorrect judgments, we identify some shortcomings. We notice that humans can easily understand the equivalence between different descriptions of the same thing, but the model cannot. For example, our model cannot understand that ‘\%’ and ‘Cheng’(‘into’)are equivalent, ‘EPS’ and ‘Earning Per Share’ are equivalent, and it cannot understand that ‘January’ is ‘Q1’. To address this issue, we tried to add some specific examples to the dataset of numerical comparison tasks, such as ‘10\% is equal to 1 Cheng’ and ‘EPS 100 is equal to Earning Per Share 100’, hoping that the model could understand the equivalence between two different descriptions of things through this method. However, the final result did not meet our expectations, indicating that our model still needs improvement in handling these types of problems. We will continue to explore this issue in future research, with the aim of improving the model’s understanding ability and accuracy.

\section{ACKNOWLEDGMENTS}
This research was funded by the China Postdoctoral Science Foundation (Grant No. 2022M711360), in part by the Laboratory for Advanced Computing and  Intelligence Engineering.

\printbibliography
% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
%\bibliography{custom}

%\appendix

%\section{Example Appendix}
%\label{sec:appendix}

%This is an appendix.

\end{document}
https://www.overleaf.com/project/65c071e2dbc4c3c9eac2c15c