% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
\usepackage{multirow}
\usepackage{times}
\usepackage{latexsym}

\usepackage{float}
% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{hyperref}
\usepackage{times}
\usepackage{tabularx}
\usepackage{latexsym}
\usepackage{graphicx}
\renewcommand{\UrlFont}{\ttfamily\small}
\usepackage{booktabs, adjustbox}
\usepackage{amsmath}
\usepackage{calligra}
\usepackage{multirow}
\usepackage{hyperref}
\usepackage{makecell}
\pagestyle{empty}

% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{ClusterCore at SemEval-2024 Task 7: Few Shot Prompting With Large Language Models for Numeral-Aware Headline Generation}

\author{Monika Singh, Sujit Kumar, Tanveen   and Sanasam Ranbir Singh \\ Indian Institute of Technology Guwahati \\\texttt{\{s.monika, sujitkumar, t.tanveen , ranbir\}@iitg.ac.in}}











% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

% \author{First Author \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\\And
%   Second Author \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\}

\begin{document}
\maketitle
\begin{abstract}
The generation of headlines, a crucial aspect of abstractive summarization, aims to compress an entire article into a concise, single line of text despite the effectiveness of modern encoder-decoder models for text generation and summarization tasks. The encoder-decoder model commonly faces challenges in accurately generating numerical content within headlines. This study empirically explored LLMs for numeral-aware headline generation and proposed few-shot prompting with LLMs for numeral-aware headline generations. Experiments conducted on the \textit{NumHG} dataset and NumEval-2024 test set suggest that fine-tuning LLMs on \textit{NumHG} dataset enhances the performance of LLMs for numeral aware headline generation. Furthermore, few-shot prompting with LLMs surpassed the performance of fine-tuned LLMs for numeral-aware headline generation.  



\end{abstract}

\section{Introduction}
News articles are one of the most important sources of information in everyday life. News headlines are vital in selecting which news seems relevant to read. As delineated in studies~\cite{wei2017learning,gabielkov2016social},  headlines play a significant role in making news viral on social media and influencing readers' opinions ~\cite{tannenbaum1953effect}. Inaccurate, incongruent or misinformation headlines also lead to the spread of misinformation and disinformation over digital platforms~\cite{chesney2017incongruent,kumar2022detecting,kumar2023gated}. Consequently, generating an accurate headline for a news body is essential. Therefore, ensuring the accuracy of headlines is essential for maintaining the credibility and usefulness of news publications. The task of headline generation, which is a form of text summarization, aims to condense a lengthy source text into a concise summary. This summary, typically presented as a headline, encapsulates the main points of the original text, providing readers with a quick overview of the content~\cite{huang2023numhg}.\par 

% Headline generation, a challenging task that requires models to intricately navigate the variation of language and context to construct compelling and accurate headlines. As news consumption continues to surge, the demand for efficient and effective methods to distill information into concise and engaging headlines has never been greater. this research addresses the specific demand for headline generation models that not only distill information accurately, but also enhance user engagement and streamline the information retrieval process in our rapidly evolving digital landscape.
In earlier studies on headline generation, various sequence-to-sequence and encoder-decoder methods have been employed to extract relevant headlines from news articles~\cite{nallapati-etal-2016-abstractive,chen-etal-2020-exclusive,paulus2018deep,song2019abstractive}. However, encoder-decoder methods faced challenges in processing large sequences of text. To address these limitations, recent studies~\cite{radford2018improving,devlin2018bert,lewis2019bart,liu2019roberta,raffel2020exploring} have proposed transformer-based models for headline generation by summarizing news articles. While transformer-based models have indeed showcased enhanced capabilities in handling longer text sequences and have exhibited promising outcomes in headline generation tasks; it is noteworthy that their performance in numeral-aware headline generation tasks need to be consistently superior. Despite their overall advancements, transformer-based models may face challenges in accurately incorporating and representing numeric information within generated headlines. Motivated by such observations, the study~\cite{huang2023numhg} proposed  numeral aware headline generation datasets. \\
This paper introduces our proposed approach and provides a comprehensive analysis of the task of \textit{Numeral-Aware Headline Generation} (Task 3 (2)). Our proposed methodology leverages Few-shot prompting with LLMs, which involves applying few-shot learning techniques to large language models (LLMs) for numeral-aware headline generation tasks. We conduct our experiments using the NumHG dataset~\cite{huang2023numhg} and the test set provided by the organizer of NumEval Task-3(2). Our experimental results suggest that few-shot prompting-based methods with LLMs are efficient for numeral-aware headline generation.




 


 \begin{figure*}[t]
    \centering
    \includegraphics[width=5in]{latex/images/semevaldig2-1.pdf}
      \caption{Working diagram of the proposed method.} 
\label{fig:propl}
\end{figure*}

\section{Related Work}
Headline generation, a type of text summarization, condenses lengthy source text into a brief summary, usually presented as a headline. This summary captures the main points of the original text, offering readers a quick overview~\cite{huang2023numhg}. Summarization involves extractive and abstractive methods: Extractive selects key sentences, while abstractive generates novel summaries. In prior research investigating headline generation, a range of sequence-to-sequence and encoder-decoder approaches were employed to derive relevant headlines from news articles~\cite{nallapati-etal-2016-abstractive,chen-etal-2020-exclusive,paulus2018deep,song2019abstractive}. However, these approaches encountered challenges, particularly in processing lengthy text sequences. The limitations of encoder-decoder methods in handling large sequences of text hindered their effectiveness in accurately summarizing news articles. To address these shortcomings and enhance the capability of headline generation models, recent research has focused on developing transformer-based architectures~\cite{radford2018improving,devlin2018bert,lewis2019bart,liu2019roberta,raffel2020exploring}. Similarly, Large Language Models LLMs  such as GPT~\cite{radford2019language}, BART~\cite{lewis2020bart}, T5~\cite{raffel2020exploring} and LLaMA~\cite{touvron2023llama} have also  shown    promising state-of-the-art models  performance for text generation and summarization task. 




% Recent studies also indicate that few-shot prompting with LLMs enhances the performance of text summarization tasks~\cite{van2023enhancing,schick2021few,luo2022few}.\par

Most studies above emphasize word selection and sentence structure, overlooking the significance of accurate numeric values in news headlines. Addressing this gap in the literature, a study ~\cite{huang2023numhg}  introduced numeral-aware headline generation datasets to facilitate the development of numeral-aware headline generation methods. Considering the superior performance of Large Language Models (LLMs) in text generation and summarization tasks~\cite{basyal2023text}, this study conducts an empirical study of LLMs for numeral-aware headline generation. Additionally, an error analysis is performed on the responses of LLMs for numeral aware headline generation. Subsequently, we propose Few-shot prompting with Large Language Models (LLMs) for numeral-aware headline generation.     



% \\
% This paper introduces our proposed approach and provides a comprehensive analysis of the task of \textit{Numeral-Aware Headline Generation} (Task 3 (2)). Our proposed methodology leverages Few-shot prompting with LLMs, which involves applying few-shot learning techniques to large language models (LLMs) for numeral-aware headline generation tasks. We conduct our experiments using the NumHG dataset~\cite{huang2023numhg} and the test set provided by the organizer of NumEval Task-3(2). Our experimental results suggest that few-shot prompting-based methods with LLMs are efficient for numeral-aware headline generation.

\begin{figure*}[t]
    \centering
    \includegraphics[width=6in]{latex/images/USNews.pdf}
    \caption{Presents an example comparison of a headline generated by a fine-tuned $T5$ model and a headline generated by a $T5$ model with three shot prompt} 
\label{fig:exampl}
\end{figure*}
\section{Proposed Method}
As mentioned above, the paper aims to study
the effect of two important aspects of numeral aware headline generations. First, we study the effectiveness of large language models (LLMs) for numeral-aware headline generations. Second, we propose a few prompting-based methods for numeral-aware headline generations.  
\subsection{Large Language Models (LLMs):} 
Considering the effectiveness of LLMs in text summarization~\cite{basyal2023text} and headline generations task~\cite{ding2023harnessing}. We fine-tune RoBERTa\footnote{\url{https://huggingface.co/google/roberta2roberta_L-24_gigaword}}~\cite{rothe2020leveraging}, {\it Generative Pre-trained Transformer} (GPT-2)\footnote{\url{https://huggingface.co/MU-NLPC/CzeGPT-2_headline_generator}}~\cite{radford2019language}, {\it Bidirectional and Auto-Regressive Transformers}~\textbf{BART}\footnote{\url{facebook/bart-large-cnn · Hugging Face}}~\cite{lewis2020bart} and {\it \textbf{T}ext-\textbf{T}o-\textbf{T}ext~\textbf{T}ransfer \textbf{T}ransformer}~\textbf{T5}\footnote{\url{https://huggingface.co/Michau/t5-base-en-generate-headline}}~\cite{raffel2020exploring} for numeral aware headline generations.










% \\
%  \\  
% \textcolor{magenta}{Why only these dour models} \\
% \textcolor{magenta}{What are the specific benefits in fine-tuning for numeral aware headline generations}

\subsection{Few Shot Prompting:}
 In-context learning denotes a methodology whereby language models acquire proficiency in tasks by utilizing a limited number of examples provided as demonstrations~\cite{dong2022survey}. The utilization of shot prompting guides the model's output. This approach encompasses three distinct strategies: zero-shot, one-shot, and few-shot prompting. Zero-shot prompting, also called direct prompting, entails assigning a task to the model without providing specific examples, relying solely on the knowledge the model has gained through its training. In contrast, one-shot and few-shot prompting involve presenting examples or 'shots' to the model during runtime, which are references for the expected response's structure or context~\cite{reynolds2021prompt}. The model then utilizes these examples to perform the task. Because these examples are presented in natural language, they offer an accessible method for interacting with language models and facilitate the integration of human knowledge into these models through demonstrations and templates. As evidenced by the findings of several recent studies~\cite{van2023enhancing,schick2021few,luo2022few}, the integration of few-shot learning techniques coupled with prompt instructions has demonstrated a noteworthy enhancement in the quality of text generated or summarized by large language models (LLMs). These observations underscore the potential effectiveness of leveraging few-shot learning methodologies alongside prompt guidance to augment the capabilities of LLMs in generating text of higher quality and relevance. Motivated by such observations regarding few-shot learning with quick text generation and summarization instructions, this study proposes few-shot and prompt engineering-based methods for numeral-aware headline generations.  Figure~\ref{fig:propl} presents the working diagram of our few shot prompting with LLMs-based numeral aware headline generation method. There are three key components of  our proposed method, namely: 
\begin{enumerate}
    \item \textbf{Few Shot:} We explore three distinct strategies of few-shot prompting: zero-shot, one-shot, and few-shot prompting. These strategies encompass varying degrees of example provision to guide the model's output, allowing for a comprehensive analysis of their respective efficacy in facilitating model performance across different tasks. We have used three examples for methods in our study, which will be considered three-shot prompting.
    \item \textbf{Base Prompt:} Here, we provide instruction to a model which guides the model in numeral-aware headline generations. Below is one example of prompt instruction we provided to LLMs for generating numeral-aware headlines. \newline
    \textbf{ Prompt (P1) :} \texttt{Generate a short headline for a given news article. The headline should be concise and small but represent the content of the news body. The headline may contain a number that could be obtained by performing simple arithmetic operations like addition, subtraction, division, and multiplication or obtained by copying the same valid number from the news article if required to summarize the article.}

    \item \textbf{Large Language models (LLMs):}  This study considers three prominent large language models: GPT~\cite{radford2018improving}, T5~\cite{raffel2020exploring}, and LLaMA~\footnote{\href{https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML}{LLaMA}}~\cite{touvron2023llama}. These models generate headlines that accurately represent given news bodies, utilizing input consisting of the news body itself, prompt instructions, and a few-shot example.        
\end{enumerate}   
\section{Experimental Results and Discussions}



\subsection{Dataset}
We consider the NumHG dataset curated by study~\cite{huang2023numhg} for training models and the test set provided by {\it NumEval} organizers for evaluating models. Table~\ref{tab:datatset} presents the characteristics of experimental datasets. 





\subsection{Experimental Setups} This study incorporates several performance evaluation metrics to assess the effectiveness of models, namely \textit{Recall-Oriented Understudy for Gisting Evaluation} (ROUGE)\footnote{\url{https://huggingface.co/spaces/evaluate-metric/rouge}}~\cite{lin-2004-rouge}, BERTScore\footnote{\url{https://huggingface.co/spaces/evaluate-metric/bertscore}}~\cite{bert-score}, MoverScore\cite{zhao2019moverscore} and Num Acc.~\cite{huang2023numhg} as performance evaluation metrics to evaluate the performance of models. These metrics provide comprehensive insights into various aspects of model performance, including linguistic quality, content overlap, semantic similarity, and numeral accuracy, respectively. Table~\ref{tab:hyper_values} presents the details of experimental hyperparameters. To replicate the findings in this work, visit GitHub \url{https://github.com/MONIKASINGH16999/ClusterCore_SemEval2024Task7} to access our code repository.
\begin{table*}[]
\centering
\caption{presents the performance of the models over test datasets}
\label{tab:main-res}
\adjustbox{max width=\textwidth}{%
\centering
\begin{tabular}{|l|l|l|l|l|lll|lll|c|}
\hline
\textbf{} &
  \textbf{Model} &\multicolumn{3}{l|}{\textbf{Num Acc.}}&
  \multicolumn{3}{c|}{\textbf{ROUGE}} &
  \multicolumn{3}{c|}{\textbf{BERTScore}} &
  \textbf{MoverScore} \\ \hline
   & &\textbf{Overall} &\textbf{Copy}& \textbf{Reasoning}
   &\multicolumn{1}{l|}{\textbf{1}} &
  \multicolumn{1}{l|}{\textbf{2}} & \textbf{L}
   &
  \multicolumn{1}{l|}{\textbf{P}} &
  \multicolumn{1}{l|}{\textbf{R}} &\textbf{F1.}
   & 
  
   \\ \cline{2-12} 
\multirow{4}{*}{\textbf{Pretrained}} &
  \textbf{RoBERTa} &20.761 &31.943&9.579&
  \multicolumn{1}{l|}{18.558} &
  \multicolumn{1}{l|}{10.325} &
 17.394 &
\multicolumn{1}{l|} {83.611}&
  \multicolumn{1}{l|}{84.728}&
  84.158 & 53.258 
   \\ \cline{2-12} 
 &
  \textbf{GPT} &24.028&34.529&11.527&\multicolumn{1}{l|}{18.596} &
  \multicolumn{1}{l|}{12.356} &16.879
   &
  \multicolumn{1}{l|}{81.192} &
  \multicolumn{1}{l|}{76.925} &79.058
   & 54.217
  
   \\ \cline{2-12} 
 &
  \textbf{BART} &24.137&35.529&12.746&
  \multicolumn{1}{l|}{15.7} &
  \multicolumn{1}{l|}{11.72} &14.846
   &
  \multicolumn{1}{l|}{84.264} &
  \multicolumn{1}{l|}{84.382} &84.323
   &55.321
   \\ \cline{2-12} 
 &
  \textbf{T5} &23.988&35.995&11.982&
 \multicolumn{1}{l|}{\textbf{19.023}} &
  \multicolumn{1}{l|}{9.365} &\textbf{17.152}
   &
  \multicolumn{1}{l|}{\textbf{85.985}} &
  \multicolumn{1}{l|}{\textbf{85.355}} &\textbf{85.638}
   &\textbf{57.298}
   \\ \hline
\multirow{4}{*}{\textbf{Finetuned}} &
  \textbf{RoBERTa} &21.726&32.594&10.859&
  \multicolumn{1}{l|}{18.558} &
  \multicolumn{1}{l|}{10.325} &
 17.394 &
\multicolumn{1}{l|} {85.5}&
  \multicolumn{1}{l|}{86.355}&
  85.907 & 54.258 
   \\ \cline{2-12} 
 &
  \textbf{GPT} &23.265&34.952&11.578&
  \multicolumn{1}{l|}{31.896} &
  \multicolumn{1}{l|}{14.256} &29.854
   &
  \multicolumn{1}{l|}{86.935} &
  \multicolumn{1}{l|}{81.325} &84.13

   &55.941
   \\ \cline{2-12} 
 &
  \textbf{BART} &25.623&35.621&13.291&
  \multicolumn{1}{l|}{32.64} &
  \multicolumn{1}{l|}{13.587} &30.466
   &
  \multicolumn{1}{l|}{86.435} &
  \multicolumn{1}{l|}{88.324} &87.377
   & 57.689
   \\ \cline{2-12} 
 &
  \textbf{T5} &36.985&37.514&12.852&
  \multicolumn{1}{l|}{\textbf{34.352}} &
  \multicolumn{1}{l|}{\textbf{13.876}} &\textbf{32.365}
   &
  \multicolumn{1}{l|}{\textbf{87.383}} &
  \multicolumn{1}{l|}{\textbf{89.682}} &\textbf{88.532}
   &\textbf{59.982}
  
   \\ \hline
\multirow{3}{*}{\textbf{Shot Prompting}} &
  \textbf{GPT} &37.259&37.594&12.589&
  \multicolumn{1}{l|}{31.746} &
  \multicolumn{1}{l|}{12.653} &29.356
   &
  \multicolumn{1}{l|}{87.659} &
  \multicolumn{1}{l|}{86.926} &87.292
   &54.989
   \\ \cline{2-12} 
 &
  \textbf{T5} &37.569&37.295&12.958&
  \multicolumn{1}{l|}{30.245} &
  \multicolumn{1}{l|}{10.941} &29.596
   &
  \multicolumn{1}{l|}{89.111} &
  \multicolumn{1}{l|}{86.922} &87.988
   &58.364
   \\ \cline{2-12} 
 &
  \textbf{LLaMA} &\textcolor{magenta}{\textbf{38.233}}&\textcolor{magenta}{\textbf{38.233}}& \textcolor{magenta}{\textbf{13.942}}&
  \multicolumn{1}{l|}{\textcolor{magenta}{\textbf{37.985}}} &
  \multicolumn{1}{l|}{\textcolor{magenta}{\textbf{14.854}}} &\textcolor{magenta}{\textbf{33.592}}
   &
  \multicolumn{1}{l|}{\textcolor{magenta}{\textbf{90.359}}} &
  \multicolumn{1}{l|}{\textcolor{magenta}{\textbf{89.856}}} &\textcolor{magenta}{\textbf{90.107}}
   &\textcolor{magenta}{\textbf{59.983}}
   \\ \hline
\end{tabular}}
\end{table*}

\begin{table}[]
\centering
\caption{Presents the human evaluation of headlines generated by our proposed system (few shot prompting with LLMs) submitted to NumEval-224. The organizer of NumEval-2024 does this human evaluation of generated headlines.
 }
\label{tab:huma-eval}\adjustbox{max width=\columnwidth}{%
\centering
\begin{tabular}{|l|c|c|}
\hline
\textbf{Submission}              & \textbf{Num Acc. (50 Headlines)} & \textbf{Recommendation (100 News)} \\ \hline
\textbf{ClusterCore}             & \textbf{1.60}                    & \textcolor{magenta}{\textbf{31}}                        \\ \hline
Noot Noot  & 1.68 & 11 \\ \hline
Infrrd.ai & \textbf{1.81}                    & \textbf{22}                        \\ \hline
np_problem & 1.57 & 14 \\ \hline
hinoki     & 1.67 & 16 \\ \hline
Challenges & 1.70 & 10 \\ \hline
NCL_NLP    & 1.73 & 16 \\ \hline
YNU-HPCC   & 1.69 & 15 \\ \hline
NoNameTeam & 1.59 & 12 \\ \hline
\end{tabular}}
\end{table}
\begin{table}[t]
\centering
\caption{Details of Experimental Setups and Hyperparameters}


\label{tab:hyper_values}
\adjustbox{max width=\columnwidth}
\centering
\begin{tabular}{|l|l|}
\hline
\textbf{Hyperparameters}       & \textbf{Value}            \\ \midrule
Batch Size            & 16             \\ \midrule
Learning Rate         & 0.01              \\ \midrule
% % Loss Function         & Cross Entropy               \\ \midrule
% Cell  State Dimension of LSTM &100 \\ \midrule
% Hidden State Dimension of LSTM &100 \\ \midrule
% Number of Layer in MLP &2\\ \midrule
Maximum \#word in news body  &250 \\ \midrule
Maximum \#word in headline  &15 \\ 

\bottomrule
\end{tabular}
\end{table}
\begin{table}[]
\centering

\caption{present the characteristics of experimental datasets. Here, \#sample indicates the number of news headlines and body pairs in the dataset. Similarly, \#head and \#Word indicate the average number of words in the headline and news body. Whereas \#sent indicates the average number of sentences in the news body and \#num indicates the average number of numeric figures in the news body.   }
\label{tab:datatset}
\adjustbox{max width=\columnwidth}{%
\centering
\begin{tabular}{|l|l|l|l|l|l|}
\hline
\textbf{}      & \textbf{\#sample}  & \textbf{\#head} & \textbf{\#sent} & \textbf{\#Word} & \textbf{\#num} \\ \hline
\textbf{Train} &21157                & 7.769               &9.851                &179.116 &9.884               \\ \hline
\textbf{Dev}   & 2367 & 7.723              &9.719               &178.396                &9.595               \\ \hline
\textbf{Test}  & 5227 & 8.082             & 10.427             &190.006                &10.186 \\ \hline
\end{tabular}}
\end{table}
\subsection{Results and discussion}
Table~\ref{tab:main-res} illustrates the performance metrics of large language models (LLMs) across various configurations, including \textit{Pretrained}, \textit{Fine-tuned}, and \textit{Shot Prompting}, evaluated on a designated test dataset. This evaluation aims to provide insights into the efficacy and adaptability of LLMs in different settings for numeral-aware headline generation. Initially, we examine the response of LLMs in both the \textit{Pretrained} and \textit{Fine-tuned} setups for numeral-aware headline generation. From Table~\ref{tab:main-res}, it is evident that the $T5$ model consistently outperforms the $RoBERTa$, $GPT$, and $BART$ models across the test dataset in both the \textit{Pretrained} and \textit{Fine-tuned} setups. From such observations, we can claim that the $T5$ model is more suitable for the headline generation tasks compared to $RoBERTa$, $GPT$, and $BART$.  Referring to Table~\ref{tab:main-res}, it becomes apparent that fine-tuning these models over the training set enhances their performance and headline generation capability. Subsequently, we curate a subset of the dataset consisting of fifty news headlines and corresponding news bodies. This subset is formed by selecting pairs from the validation dataset where the presence of numeral figures in the headline is deemed particularly significant in accurately representing the content of the news body. Upon manual inspection of the news headlines generated by fine-tuned $RoBERTa$, $GPT$, $BART$, and $T5$ models over the subset of the dataset comprising fifty samples, our observations suggest that while the generated headlines are contextually similar to the ground truth headlines and effectively represent the content of the news body, the accuracy in representing numeral figures is notably average. From these observations, we can conclude that fine-tuned $RoBERTa$, $GPT$, $BART$, and $T5$ models exhibit high efficiency in headline generation but display slightly lower efficiency in numeral-aware headline generation. One possible reason behind this discrepancy could be the requirement for complex mathematical reasoning capabilities in numeral-aware headline generation tasks. To enhance the performance of models in numeral-aware headline generation tasks, this study employs shot prompting methods. Shot prompting methods offer several advantages, primarily providing prompts to models that serve as instructions, guiding them on what specific task needs to be performed and how to approach it. Additionally, shot prompting methods supply a few examples to the models, aiding them in inference and comprehension for the underlying task. This approach enables the models to better understand the task and generate more accurate and contextually relevant headlines containing numeral figures. We consider $GPT$, $T5$ and $LLaMA$ in few shot prompt settings. From Table~\ref{tab:main-res} it is apparent that $LLaMA$ the model outperforms $GPT$ and  $T5$ with  few shot prompting. Similarly, it is also evident that $LLaMA$ a model with few shot prompting outperforms $RoBERTa$, $GPT$, $BART$, and $T5$ models in \textit{Pretrained} and \textit{Fine-tuned} setups. Our manual inspection of the news headlines generated by the $GPT$, $T5$, and $LLaMA$ models utilizing few-shot prompting over a subset of the dataset containing fifty samples suggests that the implementation of few-shot prompting enhances the efficiency of numeral-aware headline generation by the models. Based on the findings presented in Table~\ref{tab:main-res}, it's clear that few-shot prompting using the $LLaMA$ model outperforms both few-shot prompting with $T5$ and $GPT$. As a result, we chose to submit headlines generated by the few-shot prompting with the $LLaMA$ model as our final system for evaluation at NumEval-2024. We could have fine-tuned the $LLaMA$ model for better results, but we have only used the pre-trained $LLaMA$ model due to resource constraints.  \par

Table~\ref{tab:huma-eval} presents the human evaluation of headlines generated by our proposed system (few-shot prompting with $LLaMA$), which were submitted to NumEval-2024. The organizers of NumEval-2024 conducted this human evaluation of the generated headlines. It is apparent from Table~\ref{tab:huma-eval} that our proposed system (few-shot prompting with $LLaMA$) achieved the top rank in recommending 100 news.

\section{Error Analysis} This study also conducts an error analysis to examine the strengths and weaknesses of large language models (LLMs) across different setups for numeral-aware headline generation. Through this analysis, we aim to identify patterns of errors and limitations inherent in the models, providing valuable insights into areas for improvement and optimization in future model development and training methodologies. We selected fifty news headline-body pairs, where numeral figures in the headline are crucial for accurately representing the news content. Our examination of the generated headlines by the models revealed the following insights: (i) $RoBERTa$  the model  generates a headline, which is representative of the news body, however in some instances is failed to consider the numeric value for headline generation. Consequently, $RoBERTa$ is deemed unsuitable for numeral-aware headline generation. However, fine-tuning the $RoBERTa$ model enhances generated headline quality, which is also evident by the performance comparison between its \textit{Pretrained} and \textit{Fine-tuned} setups. (ii) The $BART$ models, whether in the \textit{Pretrained} or \textit{Fine-tuned} setups, demonstrate proficiency in generating efficient headlines that include valid numeric values. However, it is noteworthy that the inclusion of valid numeric values in headlines is more prevalent in the fine-tuned models compared to those without fine-tuning. (iii) The $T5$ models, in both the \textit{Pretrained} and \textit{Fine-tuned} setups, consistently produced headlines with more efficient and valid numerical values compared to $RoBERTa$, $BART$, and $GPT$. This indicates that $T5$ models are particularly more effective in numeral aware headline generations. (iv) The $LLaMA$ model stands out for its ability to generate accurate and efficient headlines containing valid numerical values when compared to $RoBERTa$, $BART$, $GPT$, and $T5$. This suggests that the $LLaMA$ model excels in incorporating precise numeric information into its generated headlines, surpassing other models in this aspect. Figure~\ref{fig:exampl} presents an example' comparison between headline generated by fine-tuned $T5$ model and headline generated by $T5$ with three shot prompt. From Figure~\ref{fig:exampl}, it is apparent that the headline generated by $T5$ with three three-shot prompts better represents the news body compared to the headline generated by the fine-tuned T5 model. This further validates our claim that a few-shot prompt helps the LLMs generate headlines. 

\section{Conclusion and Future work}
This study conducted an empirical research on LLMs for numeral-aware headline generation and proposed a few shots prompting with LLMs for numeral-aware headline generation. We conduct our experiments on \textit{NumHG } and test set data provided by the organizer of NumEval-2024. Our experimental results suggest that finetuning LLMs over \textit{NumHG} dataset improves the performance of numeral-aware headline generation. Further, few shot prompting with LLMs outperform fine-tuned LLMs for numeral-aware headline generation. This study identifies prompt tuning using LLMs for numeral-aware headline generation. 



















\bibliography{custom}
% \bibliography{anthology,custom}
\bibliographystyle{acl_natbib}
% \appendix

% \section{Example Appendix}
% \label{sec:appendix}

% This is an appendix.

\end{document}
