 % This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage[]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{listings}% http://ctan.org/pkg/listings
\lstset{
  basicstyle=\ttfamily,
  mathescape
}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage[subtle]{savetrees}
\usepackage{microtype}
\makeatletter
\MT@redefine@patch{eqnum}{%
  \MT@patch@patch\tagform@{(}{\leftprotrusion{(}}%
  \MT@patch@patch\tagform@{)}{\rightprotrusion{)}}%
  \MT@ifdefined@n@TF{eqref }
    {\MT@exp@cs\MT@patch@patch{eqref }}{\MT@patch@patch\eqref}
       {\tagform@}{\@nameuse{MT@patch@saved@\string\tagform@}}%
}{}
\makeatother

\usepackage{url}
\usepackage{times}
\usepackage{latexsym}
\renewcommand{\UrlFont}{\ttfamily\small}
\usepackage{multicol}
\usepackage{multirow}
\usepackage{graphicx}
\usepackage{verbatim}
\usepackage{makecell}
\usepackage{nicefrac, xfrac}
\usepackage{braket}
\usepackage{hhline}
\usepackage{amsmath,bm,amssymb}
\usepackage[linesnumbered,ruled]{algorithm2e}
\usepackage{xcolor}
\definecolor{Green}{rgb}{0.0, 0.5, 0.0}
\definecolor{Amethyst}{rgb}{0.6, 0.4, 0.8}
% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{GPT-4 \texttt{DenseSumm}: Building Entity-Dense Summaries \\ by Iteratively Fusing New Entities into a Fixed Token Budget}

\author{
  Griffin Adams$^{\spadesuit,\clubsuit}$ \\ griffin.adams@columbia.edu \And
  Alexander R. Fabbri$^{\diamondsuit}$ \\ afabbri@salesforce.com \\ \AND Faisal Ladhak $^{\spadesuit}$ \\ faisal@cs.columbia.edu \\\And No\'emie Elhadad$^{\spadesuit,\clubsuit}$ \\ noemie.elhadad@columbia.edu \AND
 Salesforce Research$^{\diamondsuit}$ \quad  Columbia University: Computer Science$^{\spadesuit}$, Biomedical Informatics$^{\clubsuit}$
}

\begin{document}
\maketitle
\begin{abstract}

Selecting the ``right'' amount of information to include in a summary is a difficult task, even for humans. Further, the ``right'' level of detail depends on the specific text being summarized. To generate summaries with \textit{just enough} content, we propose \texttt{DenseSumm}--an iterative method for incrementally fusing salient entities into a partial summary. \texttt{DenseSumm} involves generating an initial \emph{tl;dr}-style summary. Then, $1-3$ critically missing entities are identified and fused into the summary. We prompt GPT-4 with \texttt{IncrSumm} for a fixed number of turns and then, separately, select the candidate which best negotiates conciseness and informativeness. We distill this expensive two-step lengthen-and-select process into a smaller model (LLAMA2-7B), which directly generates the optimal candidate. We analyze the distilled straight-shot summaries and find that they exhibit less of a lead bias while containing a higher density of salient Atomic Content Units (ACUs) than a LLAMA2 variant distilled from length-controlled GPT-4 outputs. We open-source the weights for our variable-length LLAMA2 summarization model.

\end{abstract}

\section{Introduction}

Automatic summarization has come along way in the past few years, largely due to strong zero-shot capabilities from Large Language Models (LLMs).

Yet, summaries produced by LLMs can tend to be overly verbose unless otherwise prompted to stick to a target length (Goyal). While length-prompting can address the verbosity issue, there is no universally agreed upon length for a summary. It is highly subjective and likely dependent on the specific source document being summarized. For shorter documents, a shorter summary might be preferred, and vice versa.

In this paper, we explore whether or not LLMs are capable of identifying the right \emph{amount} of information to include in the summary of a \emph{specific} document. This task requires striking a balance between conciseness and informativeness: capturing the main ideas of the source text in as few words as possible.

To negotiate this trade-off, we use a closed-source LLM (GPT-4) to iteratively lengthen a short summary and then re-prompt it  to select the most preferable. By iteratively fusing new content into existing summaries, we can isolate length because each successive summary differs only in the number of details added.

We distill the knowledge from this complex two-step procedure into a simpler open-source model (LLAMA 2), which is able to generate the optimal length summary in a straight-shot fashion. [[SUMMARY of RESULTS]]

Our primary contributions are: X Y Z.

\section{Methods} \label{sec:methods}

\begin{figure*}[t]
\centering
\includegraphics[width=\linewidth]{images/GENERATING_S2L.pdf}
\caption{\texttt{IncrSumm} is used to create a rollout of 4 progressively longer summaries (\textbf{Step 1})). In \textbf{Step 2}, we use separately prompt GPT-4 to select a stopping point which best balances conciseness and informativeness. We then distill this two step process of roll-out-then-truncate into a smaller open-source model (LLAMA2 7B) with instruction tuning. }
\label{fig:generating-s2l}
\end{figure*}

\subsection{Generating Instruction Data} \label{sec:gpt4-s2l}

As shown in Figure \ref{fig:generating-s2l}, we create instruction data with GPT-4 \citep{gpt4} in a two-step process.

\paragraph{Fixed Rollout.} Given an article, we prompt GPT-4 to generate a short \texttt{tl;dr} style summary:

\begin{verbatim}
{{Article }}

Briefly summarize the above Article
in 1 sentence (20 words or less).  
\end{verbatim}

Then, for a fixed number of turns, we recursively lengthen it. Specifically, we prompt GPT-4 to identify 1-2 key concepts in the Article, which are critically \textit{missing} from the existing summary, and fuse it into the existing summary. Our iterative approach can be loosely viewed as the abstractive counterpart to Maximal Marginal Relevance (MMR), in which . The prompt template is:

\begin{verbatim}
{{Article}}

{{Summary}}

Complete the following 2 steps.

Step 1. Identify 1-2 critical, non-
redundant Concepts from the Article
which are missing from the Summary.

Step 2. Fuse the Missing Concepts
into the Summary while minimally
increasing its length.

{{Additional Guidelines}}
\end{verbatim}

\noindent Please see Appendix \ref{sec:prompts} for the \{\{Additional Guidelines\}\}.

\paragraph{Selecting a Stopping Point.} We instruct GPT-4 to perform $k=3$ sequential lengthening operations and asking it to identify the optimal stopping point in the fixed roll-out. This amounts to deciding on a dynamic stopping point for \texttt{IncrSum}. We attempt to verbalize the question "which summary has just the right amount of detail?" into the following prompt:

\begin{verbatim}
{{Article}}

{{Shuffled Summaries}}

Select the summary which best
balances conciseness and informa-
tiveness.

- There is a tradeoff: concise
summaries tend to be shorter, while
informativeness can sometimes favor
longer summaries.

- The best summary should convey
the main ideas of the Article in
the fewest words.
\end{verbatim}

\subsection{Instruction Tuning}

We fine-tune a model on four tasks, which are constructed based on data generated from \S \ref{sec:gpt4-s2l}. Let $S_{1} = tldr$ represent the initial GPT-4 generated summary at timestep $t=1$. The summary at $t=k$ can be represented as $Incr(S_{k-1})$, where $Incr$ represents the process of identifying and fusing missing content into partial summary $S_{k-1}$. Given a fixed $K$ from \S \ref{sec:gpt4-s2l}, we define $k^* \in [1 ... K] $ as the timestep of the optimal-length summary. This produces a dynamic rollout of summaries: $S_1 \rightarrow ... \rightarrow S_{k^*}$, which is different for each example in the instruction training set\footnote{If $k^* = 1$, then there is only summary in the chain}.

\paragraph{tl;dr.} For \texttt{[TLDR]} task, we learn to generate the initial \textbf{tl;dr} summary from GPT-4 conditioned on the input article. Using the above notation the task: the prompt and completion are \texttt{[TLDR] Doc $\rightarrow S_1$}.

\begin{lstlisting}
Input([TLDR]; Doc) $ \rightarrow \ S_1$
\end{lstlisting}

\paragraph{Stopping Point.} We treat the decision of when a summary is \texttt{[DONE]} as a generative binary classification. The input is the article and a partial summary and the desired output is "Y/N". Each example generates $K$ \texttt{[DONE]} training instances:

\begin{lstlisting}
Input([DONE]; Doc; $S_{k \in [1..k^*]}) \ \rightarrow \ N$
Input([DONE]; Doc; $S_{k \in [k^*...K]}) \ \rightarrow \ Y$
\end{lstlisting}

\paragraph{Add.} The \texttt{[ADD]} task incorporates identification and fusion of missing critical content. $k^* - 1$ \texttt{[Add]} training instances are created from each summary rollout of length $K$. For adjacent pairs of summaries:

\begin{lstlisting}
Input([ADD]; Doc; $S_{t}$) $\rightarrow$ COT $+\ S_{t+1}$
\end{lstlisting}

\noindent $\forall \ \ t \ \ \in [\ 1..k^*\ ]$.

\paragraph{Straight.} The \texttt{[STRAIGHT]} instruction generates $S_{k^*}$ \textit{it}, without any intermediate steps:

\begin{lstlisting}
Input([STRAIGHT]; Doc) $ \rightarrow \ S_{k^*}$
\end{lstlisting}

This operation represents a distillation of the two steps (fixed roll-out and select) from \S \ref{sec:gpt4-s2l}.

\section{Experimental Setup.}

\paragraph{Instruction Data.} We select 1,000 news articles (randomly sampled from the CNN-Dailymail train set) for generating instruction tuning data. We use LLAMA-2 (7B) for instruction tuning on the GPT-4 generated data. We use a PEFT library (Lit-GPT) to fine-tune LLAMA2 with Adapter V2 [Citations] and Flash Attention. We fix the maximum context window to $2,048$ tokens. For 10 epochs, we fine-tune the adapters on a single A6000 48GB GPU with a learning rate of $3e-3$ (linear warmup for $100$ steps before decaying), weight decay of $0.02$, and an effective batch size of $128$ ($128$ gradient accumulation steps).

\paragraph{Test Data.} We select 100 news articles at random from the CNN/DailyMail test set, as well as the Xsum and NYTimes test sets, for a total of 300 articles.

\paragraph{Comparison Method.} The premise behind \texttt{IncrSumm} is that it produces summaries which contain more salient content per word than vanilla approaches.  To test this hypothesis, we need to create a baseline of informativeness versus length. To do so, we distill another LLAMA-2 model from GPT-4 with a different set of prompts. In particular, we use a similar prompt from Goyal et al [CITE]

\begin{verbatim}
{{ARTICLE}}

Summarize the above Article in
{{K}} sentence(s).
\end{verbatim}

\noindent We vary $k$ from $2$ to $4$ and re-use the GPT-4 generated $tl;dr$ from \S \ref{sec:gpt4-s2l} as $k=1$. This produces 4 variable-length summaries for each of the $1,000$ articles in the instruction dataset. This matches the total number of summaries used for training \texttt{IncrSumm} ($4000$).

\paragraph{Inference.} For both baseline and \texttt{IncrSumm}, we perform greedy decoding with a temperature of $0.1$ and a maximum length of $368$ tokens. For \texttt{IncrSumm}, we use the \texttt{[STRAIGHT]} instruction and for baselines we control the number of sentences from $k=1$ to $k=4$.

\section{Analysis}

\paragraph{Varying Length.} Plot overall length variance as a function of source article length.

\paragraph{Density of Salient Content.} Reference-Free ACU...

\paragraph{Other GPT-4 Metrics?} See how they did.

\paragraph{Improvement over Zero-Shot.} TODO.

\paragraph{Negative Result.} TODO.

\section{Conclusion.}

We open-source LLAMA-2 \texttt{IncrSumm}, a model which learns to generate dynamic length summaries of \textit{just the right length} without any hand-crafted prompts. We distill the knowledge from a more expensive procedure (lengthen-then-truncate with GPT-4) into a simpler LLAMA-2 model with just 1,000 CNN/Dailymail articles. We demonstrate that \texttt{IncrSumm} summaries are xxx.

% Entries for the entire Anthology, followed by custom entries
\bibliography{anthology,custom}
\bibliographystyle{acl_natbib}

\appendix

\section{Prompts} \label{sec:prompts}

Below, we show the prompt for the Additional Guidelines for Fusing and Adding content.

\begin{verbatim}
Strictly adhere to the following
Guidelines:

- Missing Concepts must be
non-redundant, concise,
and critically important.

- The new Summary must contain ALL
the information from the previous
Summary PLUS Missing Concepts.

- Missing Concepts can be incorp-
orated into the beginning, middle,
or end of the previous Summary. 

- Break up the Summary into short
and concise sentences.
\end{verbatim}


\end{document}
