% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

\usepackage{graphicx}
\graphicspath{{./images/}}

\usepackage{stfloats}
\usepackage{tabularray}
\UseTblrLibrary{booktabs}

\usepackage[]{mdframed}

\newcommand{\framedtext}[1]{%
\par%
\noindent\fbox{%
    \parbox{\dimexpr\linewidth-2\fboxsep-2\fboxrule}{#1}%
}%
}

\usepackage{lipsum}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Team NP\_PROBLEM at SemEval-2024 Task 7: Numerical Reasoning in Headline Generation with Preference Optimization}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Pawan Kumar Rajpoot $^*$  \\
  SCB DataX, Thailand \\
  pawankumar.rajpoot@data-x.ai \\
 \\\And
  Nut Chukamphaeng $^*$  \\
  SCB DataX, Thailand \\
  nut.chukamphaeng@data-x.ai \\
 \\}

\begin{document}
\maketitle
\begin{abstract}
While large language models (LLMs) exhibit impressive linguistic abilities, their numerical reasoning skills within real-world contexts remain under-explored. This paper describes our participation in a headline-generation challenge by Numeval at Semeval 2024, which focused on numerical reasoning. Our system achieved an overall top numerical accuracy of 73.49\% on the task. We explore the system's design choices contributing to this result and analyze common error patterns. Our findings highlight the potential and ongoing challenges of integrating numerical reasoning within large language model-based headline generation. 
\end{abstract}
\def\thefootnote{*}\footnotetext{Equal Contribution}\def\thefootnote{\arabic{footnote}}
\section{Introduction}

The capacity to understand and manipulate numerical information within natural language text is essential for various NLP applications. Tasks such as news summarization, report generation, and the creation of data-driven narratives increasingly rely on the accurate interpretation and generation of numerical expressions. SemEval 2024 Task 7 \cite{numeval-report} addresses these challenges through two intriguing subtasks: numerical headline generation and numerical headline number fill-in-the-blanks.

Generating numerical headlines necessitates models capable of synthesizing a succinct and attention-grabbing title that accurately reflects a news article's core numerical quantities and trends. Conversely, the fill-in-the-blanks subtask tests the model's ability to comprehend numerical relationships and infer the missing value to complete a provided headline. These tasks present a complex intersection of numerical reasoning and natural language generation/understanding.

Existing text generation and numerical understanding work often leverage sequence-to-sequence architectures and specialized pre-trained language models. However, SemEval 2024 Task 7's emphasis on numerical reasoning within headlines creates a distinct demand for techniques capable of accurately grounding representations of numbers and quantities within the linguistic context.
This paper describes our approach to SemEval 2024 Task 7. We worked on both tasks separately and created two separate models. We used techniques such as parameter-efficient fine-tuning of large language models and then doing Direct Preference Optimization on top to align models better.

The remainder of this paper proceeds as follows. Section 3 reviews related work in numerical reasoning and headline generation. Section 4 details our models and methodology. Section 4 presents our experimental evaluation of the SemEval 2024 Task 7 dataset and includes a thorough analysis of our results. Finally, Section 5 summarizes our findings and outlines potential future research directions.

\section{Background and Related Work}
Headline generation within NLP has a rich history, evolving from early extractive techniques towards modern abstractive generation methods.
Initial extractive approaches primarily focused on selecting the most salient sentences from the source document to compose the headline \cite{dorr-etal-2003-hedge} \cite{DBLP:journals/corr/abs-1109-2128, mihalcea-tarau-2004-textrank}. These methods offered interpretability but lacked the fluency and novelty often desired in generated headlines.
The advent of deep learning and sequence-to-sequence models enabled abstractive headline generation, empowering models to synthesize new phrases and expressions \cite{DBLP:journals/corr/RushCW15, nallapati-etal-2016-abstractive}. Attention mechanisms \cite{DBLP:journals/corr/BahdanauCB14} proved pivotal in aligning source text and headline generation. 
Recent advancements in Generative AI have led to significant improvements in this field with state-of-the-art results \cite{DBLP:journals/corr/abs-1912-08777}.
\textit{GSum} \cite{DBLP:journals/corr/abs-2010-08014}, for example, initially performs extractive summarization and then incorporates the extractive summaries into the input for abstractive summarization. \textit{SEASON} \cite{wang-etal-2022-salience} adopts a dual approach, learning to predict the informativeness of each sentence and using this predicted information to guide abstractive summarization
Notably, most of these works focus on the selection of words and the structure of sentences.

\section{Numeval}
Numeval is part of Semeval 2024; the task we focused on and worked on requires models to generate concise and informative headlines that accurately reflect the core numerical information in news articles. Systems must demonstrate an understanding of how numbers convey meaning and should prioritize the most relevant numerical aspects for inclusion in the headline.

Subtask 1: Numerical Reasoning  -  models are required to compute the correct number to fill the blank in a news headline.

Subtask 2:  Abstractive Headline Generation - models must construct a headline based on the provided news; this headline should incorporate the numerical reasoning within.
\begin{figure}[h]
    \centering
    \includegraphics[width=7cm]{images/task_example.png}
    \caption{Numeval Task 3-1 examples.}
    \label{fig:bart_mistral_loss}
\end{figure}
The organizers released a novel dataset designed to facilitate research on numeral-aware headline generation.
The NumHG \cite{huang2023numhg} dataset addresses the issue of inaccurate numeral generation in headline creation. It provides over 27,000 news articles with detailed annotations designed to facilitate the development of models that accurately understand and summarize numerical information.
For subtask 1, each data point has an answer operator added, which signifies how the numerical answer is obtained, which includes Copy (direct retrieval), Trans, Span, Round, Paraphrase, Add, Subtract, Multiply, and Divide.
\begin{figure}[h]
    \centering
    \includegraphics[width=7cm]{images/taks_example2.png}
    \caption{Numeval SubTask 2 examples.}
    \label{fig:bart_mistral_loss}
\end{figure}
Meanwhile, subtask 2 requires the model to generate complete headlines from given news content. 
\section{Methodology}

\subsection{subtask 1}
\begin{figure}[h]
    \centering
    \includegraphics[width=7cm]{images/subtsk2.png}
    \caption{Overall Stage 1 train subtask 1}
    \label{fig:pipeline_st1}
\end{figure}

For this subtask, we start by passing the passage and question in the blank statement with an aligned answer to GPT 4 and ask it to generate a rationale for this given answer.
The below prompt is used in this process.
\begin{verbatim}
PASSAGE: ARTICLE-HERE
QUESTION: FILL_IN_THE_BLANKS-HERE
WHY ANSWER TO THIS IS ANSWER-HERE ? 
EXPLAIN\nRESULT: 
\end{verbatim}
Once we get the reasoning from this module, we re-structure the training data in the following manner.
\begin{verbatim}
PASSAGE: ARTICLE-HERE
QUESTION: FILL_IN_THE_BLANKS-HERE
WHY ANSWER TO THIS IS ANSWER-HERE ? 
REASON: GPT_RATIONALE_HERE 
\end{verbatim}
We use this to train our main model for subtask 1.
\subsection{subtask 2}

1) \textbf{Numerical Reasoning}: The model must demonstrate fluency in numerical calculations.

\noindent2) \textbf{Headline Matching}: Generated headlines must stylistically align with the data.

We worked on an end-to-end solution leveraging a Large Language Model (LLM) to address these challenges. First, we fine-tune the LLM to enhance its mathematical reasoning capabilities. This approach targets accurately interpreting numerical data and producing suitable headlines.

\begin{figure}[h]
    \centering
    \includegraphics[ height=4cm,width=7cm]{workflow-task2-v2}
    \caption{Overall workflow for subtask 2}
    \label{fig:workflow-task2}
\end{figure}

\subsection{Fine-tuning}

For both tasks separately, we selected Mistral-7B \cite{jiang2023mistral} as our trained LLM based on its strong performance on diverse benchmarks, including those focused on numerical reasoning benchmarks like GSM8K, which suggests a solid foundation for further fine-tuning on our specific numerical headline generation task.

As illustrated in Figure \ref{fig:bart_mistral_loss}, Mistral-7B achieve a lower fine-tuning loss than BART \cite{DBLP:journals/corr/abs-1910-13461}, a state-of-the-art text summarization model. This reinforces its suitability for our task.

For efficient fine-tuning, we employ Parameter Efficient Fine Tuning (PEFT). Due to memory constraints, we employed 4-bit QLoRA\cite{dettmers2023qlora} quantization (cite reference) with a rank of 128 and an alpha of 256. This quantization technique was applied specifically to the self-attention Query, Key, and Value matrices along with the Linear layers of the model.  To optimize the process, we used gradient accumulation (steps=2), a paged 32-bit Adamw optimizer, a cosine learning rate schedule (LR=2e-5), a decay rate of 0.01, and a short 5-step warmup period. The entire fine-tuning process was facilitated using the {axolotl} library. This technique reduces the model's memory footprint while minimizing performance degradation. This is particularly advantageous when working with LLMs.

\paragraph{Prompt template}

\begin{mdframed}
Given the news article, please write an appropriate headline \\
\{news content\} \\
\\
Headline:
\end{mdframed}

\subsection{Direct Preference Optimization (DPO)}
For both subtasks, we further aligned our fine-tuned models to learn better using the dev set.
We did not use the dev data split in the first train stage.
While aligning, we used dev data to first run through the model. We realigned the fine-tuned model with incorrect outcome results, that is, the dev results where the predicted number in the generated headline was incorrect or rejected. We still use the rationale (for subtask 1) for DPO, while DPO training for the second subtask only contains the predicted headline(wrong/rejected) and the correct/choosen headline.
One example of the DPO train data for subtask 1 is below.

\begin{mdframed}
PASSAGE: \\
Stocks made gains today, extending a winning streak into its fourth day, MarketWatch reports. Merck rose 12.5\% on announcement of its merger with Schering-Plough, while General Motors built on recent gains with a 22.9\% jump. The Dow closed up 53.92 at 7,223.98., \\QUESTION:\\
"Dow Up \_\_\_\_, Gains 9\% for Week \\
\textbf{Choosen}:54
REASON:
rounding off to nearest integer\\
\textbf{Rejected}: 
53.92
REASON:copy from text
\end{mdframed}
To align the generated fill-in-the-blanks/headlines style with the target dataset, we utilize the Direct Preference Optimization (DPO) alignment technique \cite{rafailov2023direct}. Direct Preference Optimization (DPO) is a novel approach for aligning large language models (LLMs) with human preferences. Unlike traditional methods that rely on reward models and reinforcement learning, DPO leverages human feedback through preferred and dispreferred outputs to directly train the LLM. This simplifies the training process and avoids the complexities of reward model design. This helps us fine-tune the model beyond numerical correctness to produce stylistically suitable headlines.

\begin{table*}[htbp!]
  \centering
  \resizebox{\linewidth}{!}{
    \begin{talltblr}[
caption = {Automatic evaluation results subtask 2.},
  label = {table:xxx},
                    ]{colspec = {Q[, mode=text]|Q[c, mode=text]|Q[c, mode=text]|Q[c, mode=text]|Q[c, mode=text]|Q[c, mode=text]|Q[c, mode=text]|Q[c, mode=text]|Q[c, mode=text]|Q[c, mode=text]|Q[c, mode=text]},
                      row{1,2}  = {c, mode=text},
                      }
    \toprule
& 
    \SetCell[c=3]{c}   Num Acc. 
        &   &  &  \SetCell[c=3]{c}   ROUGE 
                &   &  &  \SetCell[c=3]{c} BERTScore
                        &  &  &  \SetCell[r=2]{c} MoverScore \\
    \cmidrule[r]{2-11}
 & Overall     & Copy    & Reasoning       & 1    & 2    & L    & P  & R & F1 & \\
    \midrule
ClusterCore & 38.233 & 51.571 & 13.942 & 33.467 & 11.837 & 28.927 & 31.876 & 42.232 & 37.026 & 56.405 \\
Noot Noot & 38.393 & 57.481 & 3.6331 & 31.47 & 11.139 & 27.284 & 25.389 & 43.977 & 34.539 & 55.559 \\
Infrrd.ai & 65.840 & 68.354 & 61.263 & 46.789 & 22.36 & 42.095 & 51.005 & 47.260 & 49.134 & 59.731 \\ 
hinoki & 62.347 & 66.284 & 55.177 & 43.072 & 19.719 & 38.999 & 47.223 & 43.444 & 45.342 & 58.711 \\ 
Challenges & 72.956 & \textbf{82.170} & 56.176 & 31.220 & 12.235 & 26.859 & 19.530 & 47.559 & 33.132 & 55.362 \\
NCL\_NLP & 62.122 & 65.536 & 55.904 & 43.506 & 19.388 & 38.878 & 46.402 & 45.039 & 45.734 & 58.861 \\
YNU-HPCC & 69.044 & 73.018 & 61.807 & \textbf{48.852} & \textbf{24.681} & \textbf{44.175} & \textbf{51.553} & \textbf{50.095} & \textbf{50.381} & \textbf{60.551} \\
NoNameTeam & 55.715 & 57.681 & 52.134 & 40.646 & 17.261 & 35.745 & 44.256 & 40.387 & 42.324 & 57.736 \\
\hline
\textbf{np\_problem (ours)} & \textbf{73.487} & 76.908 & \textbf{67.257} & 39.816 & 17.577 & 34.339 & 27.800 & 48.557 & 37.816 & 57.024 \\
      \bottomrule
    \end{talltblr}
    }
\end{table*}

\begin{figure}[h]
    \centering
    \includegraphics[width=7cm]{bart_vs_mistral_loss_v2}
    \caption{Comparing fine-tuning losses between BART and Mistral-7B}
    \label{fig:bart_mistral_loss}
\end{figure}

\section{Results}
For subtask 1, we evaluate our model on two fill-in-the-blank types. One accuracy on Copy, where the answer is directly copied from the article. Second is Reasoning, which includes different reasoning techniques such as addition, subtraction, multiplication, and paraphrasing.

\end{center}
\begin{center}
\captionof{table}{Accuracy on tasks type subtask 1}
\begin{tabular}{|c|c| } 
\hline
\textbf{Copy} & \textbf{Reasoning} \\
\hline
0.922 & 0.784 \\ 
\hline
\end{tabular}
\end{center}

For comparative semeval two-stage evaluations, we scored 0.89 in the open and 0.86 in the hidden stage, respectively.

\newpage
\begin{center}
\captionof{table}{Comparitive num accuracy on subtask 1}
\begin{tabular}{ |p{3cm}||p{1.2cm}|p{1.2cm}| }
\hline
\textbf{Team} & \textbf{Open-Score}  & \textbf{Hidden-Score}  \\
\hline
CTYUN-AI & \textbf{0.95} & \textbf{0.95} \\ 
zhen qian & 0.94 & 0.94 \\ 
YNU-HPCC & 0.93 & 0.94 \\ 
NP-Problem(ours) & 0.89 & 0.86 \\ 
\hline
\end{tabular}
\end{center}
For subtask 2 we performed best in numerical accuracy overall and reasoning scores, we out-shined in reasoning accuracy as difference between 1st and 2nd rank was around 7 points. We open-source our final models on Huggingface \footnote{\url{https://huggingface.co/lingjoor/numeval-task7-1}, \url{https://huggingface.co/lingjoor/numeval-task7-2}}.

\section{Limitations}
Since our method is based on 7B LLM, our performance is capped by this model's ability to draw rationale for the numerical reasoning. This limitation is in line with the hardware resource as well; We used an RTX 4090 24GB GPU-based machine for our work, which can load and fine-tune the models with upto 7-10 B parameters as well.
\section*{Conclusion}
We present a modular solution to the numeral problem with an alignment module to increase the model's ability to understand numerical reasoning across both tasks.
We thank the organizing committee of SemEval-2024, along with the task-setting team of Numeval, for allowing us to work on this problem.


% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

% \appendix

% \section{Example Appendix}
% \label{sec:appendix}

% This is an appendix.

\end{document}
