% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage[]{ACL2023}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}
\usepackage{graphicx} 

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}


\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{multirow}
\usepackage{amssymb}
\usepackage{pifont}


% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Generating Extractive and Abstractive Summaries in Parallel from Scientific Articles Incorporating Citing Statements}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Sudipta Singha Roy \\
  The University of Western Ontario \\
  London, ON, Canada \\
  \texttt{ssinghar@uwo.ca} \\\And
  Robert E. Mercer \\
  The University of Western Ontario \\
  London, ON, Canada \\
  \texttt{mercer@csd.uwo.ca} \\}

\begin{document}
\maketitle
\begin{abstract}
%Most summarization research done for scientific articles focuses solely on the content of those particular documents, ignoring the observations from the research community which are contained in citing papers. In order to incorporate the citation contexts, we have developed a standalone model for summarizing any scientific document based on the information in the source document and the articles that cite it. A novel aspect of this model is that it produces both an abstractive and an extractive summary in parallel. The reason for generating the two summaries in parallel is that the creation of each summary enhances its counterpart. The extractive summarizer, the first part of the model, is a combination of the heterogeneous graph-based neural network and the graph attention network, and the second part of the model is a BigBird decoder-based abstractive summarizer. These two modules send additional control signals to each other via the loss function, so that high-quality abstractive and extractive summaries are generated simultaneously.
Summarization of scientific articles often overlooks insights from citing papers, focusing solely on the document's content. To incorporate citation contexts, we develop a model to summarize a scientific document using the information in the source and citing documents. It concurrently generates abstractive and extractive summaries, each enhancing the other. The extractive summarizer utilizes a blend of heterogeneous graph-based neural networks and graph attention networks, while the abstractive summarizer employs an autoregressive decoder. These modules exchange control signals through the loss function, ensuring the creation of high-quality summaries in both styles.
\end{abstract}

\section{Introduction}

%Text summarization is the process of shortening a document automatically while retaining a concise narration of the information. Most of the text summarization models built using neural networks \cite{nallapati2016abstractive,zhong2019searching} focus on short documents like the CNN/Daily Mail dataset \cite{cnnDailyMail}. 
%Compared to short text summarization, neural text summarization systems have not been extensively applied for longer documents like scientific research publications. While summarizing scientific documents, the most common approach is to consider only the abstract, introduction, and conclusion sections from the target articles \cite{yasunaga2019ScisummNet}.

Text summarization automates condensing documents while preserving key information. Most neural summarization models, like those by \citet{nallapati2016abstractive,zhong2019searching}, are designed for shorter texts, e.g., the CNN/Daily Mail dataset \cite{cnnDailyMail}. However, applying these models to longer documents, such as scientific research papers, remains limited. In scientific document summarization, it is common to focus solely on abstracts, introductions, and conclusions, as demonstrated in \citet{yasunaga2019ScisummNet}'s work.

%Scientific publications are not only often longer in length but also comprise more sophisticated concepts and domain-specific technical terminologies. Furthermore, they are organized by part and include references to some other works (citations) to both make the presented topics more understandable to readers with proper background knowledge as well as fit in the page limits set by the publications. These make the summarization of scientific articles a more challenging task than summarizing other documents. Furthermore, there remain some hidden impacts of any particular article at the time of publication which may come out later by other researchers. While a paper's abstract is a great way to get a sense of the paper's contents from the authors' perspective, it may not do justice to the paper's actual influence on the domain in the future for the research community, and this influence may evolve over time \cite{yasunaga2019ScisummNet}. 
Summarizing scientific publications presents unique challenges due to their length, complex concepts, technical jargon, structured organization, and 
%references 
citations. These complexities make it a more daunting task compared to summarizing other types of documents. Additionally, the long-term impact of a scientific article may not be fully evident when it is first published, as its significance can evolve over time. While an abstract provides an initial overview from the authors' perspective, it may not capture the full extent of the paper's influence on the research community and its evolving impact \cite{yasunaga2019ScisummNet}.
%%%%%%%%%%%%%%%%%%%% added %%%%%%%%%%%%%%%%%%%%%%%%%%
As an example, we can consider the abstract from %Bergsma and Lin
%the paper ``Bootstrapping path-based pronoun resolution''
\citet{bergsma2006bootstrapping}:
\begin{quote}
    We present an approach to pronoun resolution based on syntactic paths. \ldots %Through a
%simple bootstrapping procedure, 
we learn the likelihood of coreference between a
pronoun and a candidate noun based on the path in the parse tree between the two
entities. \ldots %This path information enables us to handle previously challenging
%resolution instances, and also robustly addresses traditional syntactic coreference
%constraints. 
Highly coreferent paths also allow mining of precise probabilistic
gender/number information. We combine statistical knowledge with well known
features in a Support Vector Machine pronoun resolution classifier. Significant
gains in performance are observed on several datasets.
\end{quote}
This abstract gives insight into the methods the authors used. But the citations emphasize the corpus it presents. For example:
\begin{quote}
    We use the approach of Bergsma and Lin (2006), both because it achieves state-ofthe-art gender classification performance, and because a database of the obtained
noun genders is available online. \cite{bergsma2005automatic}
\end{quote}
\begin{quote}
    For the gender task that we study in our experiments, we acquire class instances
by filtering the dataset of nouns and their genders created by Bergsma and Lin
(2006). \cite{bergsma2013using}
\end{quote}
Jaidka et al.\ \cite{jaidka2016overview,jaidka2019cl} have identified this missing aspect in scientific document summarization and addressed it by introducing a shared task. This task aims to create summaries that take into account not only the information in the body of the documents but also the research community's overview of the documents over time.
The work described here continues in this direction.

With the advancement of neural networks, there have been a few prominent research works in recent years for generating extractive \cite{yasunaga2019ScisummNet} and abstractive \cite{dimsumm,zhang2019hibert} summaries from scientific documents \cite{cohan-etal-2018-discourse,zhang2022improving}. Extractive summarization recognizes key sentences from the source document as the summary but lack the flow of information, whereas the abstractive summarization technique generates new phrases using language models while preserving the semantics of the input document but may miss some important aspects of the text.
%A shortcoming of the existing models is that they are capable of generating either extractive or abstractive summaries, but not both. 
This is a motivation for designing a model to generate both summaries in parallel and help the counterpart to achieve a performance boost with additional guidance.

A key step in extracting brief synopsis sentences from a manuscript is to map the cross-sentence correlations. A lot of recent prominent works \cite{nallapati2017summarunner} have tried to do so using recurrent neural networks (RNNs). However, because of using RNNs, these models fail to capture long-distance sentence-level dependencies. Another approach to preserve sentence-level dependencies from long documents is using graph-based neural networks. \iffalse Initial works \cite{erkan2004lexrank,mihalcea2004textrank} use cosine similarity between sentences to build a sentence-level correlation graph.\fi 
A few recent works (e.g., \cite{cohan2018scientific,yasunaga2017graph}) have utilized discourse information in the article along with inter-sentence correlations for constructing graphs and summarizing document. Another approach is to construct a sentence-level fully connected graph. Zhong et al.\ \cite{zhong2019searching} and Liu et al.\ \cite{liu2019text} used transformer \cite{vaswani2017attention} encoders to determine how sentences interact with each other. Wang et al.\ \cite{heterogeneous} introduced an heterogeneous graph neural network for extractive summarization which used additional semantic units (words) as intermediate nodes to construct relationships between sentences. 

Abstractive summarizers focus heavily on form, with the goal of producing a generalized summary, which tends to necessitate complex language-generating models. These models are typically based on sequence-to-sequence (seq2seq) \iffalse \cite{seq2seq} \fi architectures, in which a source document is seen as one sequence whereas its summary as another. The majority of previous research on neural abstractive summarization depended on large-scale, high-quality datasets of supervised document-summarization pairings \cite{see2017gettothepoint}. Recently, state-of-the-art solutions on abstractive summarization are built upon the transformer \cite{vaswani2017attention} and BERT \cite{devlin2018bert} models. These attention-based abstractive models are being used in different fields like clinical note summarization \cite{attention-based-clinical}, scientific document summarization \cite{zhang2022improving}, and lay-abstract generation \cite{dimsumm}. 

In this paper, addressing the above-mentioned issues, we have built a standalone summarization model which can generate both extractive and abstractive summaries from scientific documents incorporating the citation network. Analyzing the citation network, citing statements from the citing articles are accumulated with the original text document to incorporate the research community's observation on that particular cited manuscript. These summaries are the abstracts of the original papers with additional information reflecting the research community's view. After that, we run the \iffalse BigBird \cite{bigBird} and \fi LongFormer \cite{beltagy2020longformer}  encoder to generate sentence and word representations and train extractive and abstractive summarizers together. For the extractive summarizer, an heterogeneous graph neural network \cite{heterogeneous} is used as it has the ability to preserve sentence-level dependencies utilizing additional semantic units as intermediate nodes in the graph representation. Abstractive summaries are generated by the autoregressive decoder. The loss function is defined in such a way that both summarizers can achieve better ROUGE and METEOR scores. Furthermore, we have developed a corpus containing 10K research articles along with their corresponding citation statements and is a subset of the Semantic Scholar Network (SSN) corpus.
%\footnote{Data and code will be made available once the work is published.} 
The citation statements are collected utilizing the citation graph used in the SSN corpus. In short, the contributions of this work are:
%
\begin{itemize}
    \item We have built a stand-alone summarizer model which can produce both extractive and abstractive summaries and each counterpart helps the other to generate better summaries. 
    \item The summarizer model can work with long scientific text articles
    \item This model considers research communities' observations while generating the summaries
    \item We have proposed a new corpus containing 10K research articles along with the corresponding citing statements to incorporate the research communities' view.
\end{itemize}
%
%Data and code will be made available once the work is published.

\section{Related Work}
%The goal of text summarizing is to provide a synopsis of a text document that reflects the core essence of the original content. In recent years, the NLP research community has provided a number of neural summarization models. When transformer and BERT-based models are used, both the extractive and abstractive summarization models gain a great performance boost. A few notable works have been done on summarizing long scientific documents, as well. However, most of them consider the introduction, abstract and conclusion sections of the articles instead of considering either the whole body of text \cite{dimsumm} or the citation statements made on that article \cite{an2021enhancing}. This section gives a brief description of the prominent works in these fields.
Text summarization aims to distill a document's essence efficiently. Recent NLP research has yielded effective neural summarization models, particularly those using transformer and BERT-based architectures. %While some 
Work summarizing lengthy scientific documents often focuses on specific sections %like introductions, abstracts, and conclusions, 
rather than the entire text \cite{dimsumm} or citation statements \cite{an2021enhancing}. %This section provides a brief overview of these contributions.

\subsection{Extractive Text Summarization}
Extractive text summarization models classify sentences in a document using labels that indicate whether or not a sentence ought to be included in the summary. Originally, these models were designed based on the encoder-decoder architecture using RNNs \cite{nallapati2017summarunner}. Since transformer and BERT-based models provide a more enriched sentence encoding, they have become the foundation for the majority of extractive summarizer models in recent years.  \citet{liu2019text} fine-tuned BERT with stacked layers of transformer to obtain the sentence vectors and then used a sigmoid classifier for identifying the sentences that would be included in the summary. \citet{zhang2019hibert} fine-tuned an hierarchical transformer (HIBERT) for the extractive summarization task. Another prominent approach for extractive summarization is using graph representations which can preserve sentence-level correlations. \iffalse Earlier works like TextRank \cite{mihalcea2004textrank} and LexRank \cite{erkan2004lexrank} built graphs based on the inter-sentence similarity in the document and then the summary-candidate sentences are ranked by PageRank \cite{page1999pagerank}. \fi
Later, the graph convolutional network (GCN) \cite{gcn} has been espoused for building different inter-sentence correlation graphs \cite{yasunaga2017graph} for this task. \citet{heterogeneous} built an heterogeneous graph neural network for extractive summarization (HeterSumGraph) which takes into account additional semantic units at the word level for building the sentence-level correlation graph.

\subsection{Abstractive Text Summarization}
Abstractive text summarization models, unlike the extractive summarizers which work like classifiers, are intended to generate summaries comprising new sentences which may or may not be present in the body of the document. These models are mostly based on the encoder-decoder architecture of the sequence-to-sequence models and language models like BART \cite{bart}, BigBird \cite{bigBird}, and T5 \cite{raffel2020exploring}. \citet{aksenov2020abstractive} applied BERT-windowing to overcome the length limitation of the BERT model and summarize long documents. \citet{gidiotis2020divide} trained the summarizer model to generate separate abstractive summaries for small parts of the document.  \citet{pilault2020extractive} combined both the extractive and abstractive summarization using a transformer language model and built an hybrid summarizer model. \citet{dimsumm} fine-tuned pre-trained BERT as the abstractive summarizer for generating a lay summary from the document.

\subsection{Scientific Article Summarization}
Existing scientific article summarizers, in most cases, are extractive models designed on the idea of sentence selection \cite{cohan2018scientific}. \citet{cohan-etal-2018-discourse} developed the first abstractive summarizer for long scientific articles using an hierarchical encoder and discourse-aware attentive decoder. \citet{mishra2022scientific} applied citation contextualization to extract unique relevant sentences from the document and final summaries are generated using a multi-objective clustering approach. \citet{gupta2022biomedical} applied BERT and graph-based approaches for biomedical document summarization. \citet{li2020cist} fine tuned T5 for generating summaries from long scientific documents and implemented an extractive summarizer using GCN. \citet{yasunaga2019ScisummNet} built a corpus (ScisummNet) that includes a citation network for scientific document summarization and extracted the summary-candidate sentences using a GCN. \citet{an2021enhancing} introduced a large corpus (SSN) with 141K research papers connected with a citation graph. They also proposed a graph-based summarization model (CGSUM) for extractive document summarization. This model can draw information from both the source and the citing texts. 

\section{Methodology}

%This section first defines the problem formulation of scientific document summarization using a citation graph and then the proposed model is explained. The benchmark datasets used for the experiments are also discussed here along with the pre-processing procedures. For summarizing scientific articles we have experimented with two benchmark corpora. The performances of these models are explained in Section \ref{result}.
This section defines the problem of scientific document summarization using a citation graph. Then, the two benchmark datasets used for the scientific article summarization experiments are discussed along with the pre-processing procedures. Finally, the proposed deep learning model is explained. %The performance of this model is provided in Section \ref{result}.

%\subsection{Problem Formulation: Scientific Document Summarization Incorporating Citation Graph}
\subsection{Problem Formulation: Summarization Using Citation Graph}
%The majority of existing approaches for document summarizing conceptualise it as a sequence-to-sequence task. This can be formally defined as: given a document $d$ comprising a sequence of $n$ words ($w_i$), the document can be represented as $d = (w_1, w_2, ... , w_n)$. The objective of the summarization model is to generate another sequence of words $S = (s_1, s_2, ... , s_m)$ where $m<<n$ and $S$ preserves the gist of the document $d$. This $S$ is generated by modeling the conditional probability distribution $P(s_1, s_2, ... , s_n|w_1, w_2, ... , w_n)$ \cite{an2021enhancing}.

%Scientific articles, in addition, contain unique traits, such as citation linkages between them that maintain close relationships between their contents. Scientific studies may also have effects that were not anticipated when they were first published. Also, as research moves forward and changes, an article's importance may vary over time. In such cases, the summaries should ideally take into account both the authors' highlighted points as well as the opinions of the scientific community (citations) \cite{yasunaga2019ScisummNet}. So in this work, when summarizing a scientific article, we take into account the citation statements from articles that cite it. 
Scientific articles possess distinctive attributes, including citation linkages, that establish profound connections between their contents. These studies may also yield unforeseen impacts and evolve in importance as research progresses. In such cases, ideal summaries should encompass both the authors' key points and the perspectives of the scientific community, as reflected in citations \cite{yasunaga2019ScisummNet}.
To serve this intent we have utilized two resources: the citation graph provided in the Semantic Scholar Network (SSN) corpus \cite{an2021enhancing}, and the ScisummNet/CL-SciSumm-2020 (CL-SciSumm-2020) corpus \cite{chandrasekaran2020overview,yasunaga2019ScisummNet} which supplies documents and their corresponding citing statements. % utilizing the same concept. 

\subsection{Description of the Datasets}

As this work is focused on generating 
%both abstractive and extractive 
summaries from scientific articles that incorporate the research community's views, we have considered two benchmark datasets: ScisummNet/CL-SciSumm \cite{chandrasekaran2020overview,yasunaga2019ScisummNet}, and Semantic Scholar Network (SSN) \cite{an2021enhancing} for the experiments done here. To the best of our knowledge, these are the only datasets for the summarization task that also provide citation information. The ScisummNet corpus consists of abstracts of the 1000 most cited research articles from the ACL Anthology Network \cite{radev2013acl} along with 15 citing statements per article. The gold standard summaries for these 1000 documents are manually summarized by domain experts. The CL-SciSumm-2020 corpus \cite{chandrasekaran2020overview} extends the ScisummNet corpus with 40 extra documents and human-generated summaries thereby providing 1040 documents, citation sentences, and summaries. For testing, we have used the test set comprising 200 scientific articles from the CL-SciSumm-2020 corpus. The other benchmark dataset used for this task is the SSN corpus. It includes 140,799 research articles culled from the Semantic Scholar Open Research Corpus (S20RC) \cite{lo2020s2orc} together with a large citation graph. This citation graph has each article as a node %(140,799 nodes in total) 
and 660,908 edges indicating the citations. This corpus covers research articles from three domains: physics, mathematics and computer science. 

%The main goal of this work is to build a deep learning model which can generate summaries of long scientific documents which incorporate observations by other researchers citing the document. The ScisummNet/CL-SciSumm dataset \cite{chandrasekaran2020overview} provides the citation statements, however, the SSN corpus lacks them. The SSN corpus was originally made with documents and documents that they referred to. Since we are interested in citing statements, it has been necessary to modify this dataset to include the citing statements from the citing papers. We have used the citation graph to obtain the citing papers, and from them we have manually obtained the statements referring to the cited article. With nearly 141K articles in the SSN corpus, obtaining citation statements from all of them is time-consuming, so, we have randomly chosen 10K papers to summarize with body lengths ranging from 1000 to 3500 when the background/related work sections are removed as our corpus. The reason for choosing this word length is to make the document along with its citing statements directly feedable to the BigBird model (described in Section \ref{result}), which can work with a maximum of 4096 tokens at a time. The dataset is split into 8000/1000/1000 articles for training/validating/testing purposes. 
The primary objective of this study is to develop a deep learning model capable of generating summaries for lengthy scientific documents while incorporating insights from other researchers citing the document. While the ScisummNet/CL-SciSumm dataset provides citation statements, the SSN corpus lacks this information. Originally, the SSN corpus consisted of documents and their references, but for our purpose of including citing statements, modifications were necessary. We leveraged the citation graph to identify citing papers and manually extracted the statements referring to the cited articles. Given the substantial size of the SSN corpus, containing nearly 141K articles, we randomly selected 10K papers for summarization. These papers have body lengths ranging from 1000 to 3500 words (with background/related work sections removed), aligning with the capacity of the LongFormer model (as described in Section \ref{result}), which can handle a maximum of 4096 tokens at a time. The dataset was divided into training (8000), validation (1000), and testing (1000) articles to facilitate model development and evaluation.

%Since the intentions of the citations made on a paper can be positive, neutral, or negative, after accumulating the citing statements from the citing articles, we have categorized them into these three classes. For each paper we have selected a maximum of 20 citation statements from each class. We have found that often only a few negative citations can be found. So, for those articles that have few negative citing sentences more neutral and positive citation statements are chosen. To categorize the citation statements, we have used a method described in (referencing this would give remove our anonymity; in the final paper we will provide the reference and a short description of the method).
Citations %made on a paper 
can convey positive, neutral, or negative intentions. To capture this diversity, we systematically categorized citing statements into these three classes after gathering them from citing articles. In cases where a paper had limited negative citations, we balanced the selection by including more neutral and positive citation statements. To classify these citation statements, we have employed RoBERTa trained on \citet{athar2014sentiment} following the approach used by \citet{kundu2023citation}.

%The summaries in the SSN corpus are the abstracts of the corresponding research papers. So, these summaries reflect only the authors' views and not the observations of the research community. To provide summaries that combine the author's and the community's information, we have fed the abstracts and corresponding citation statements to T5 \cite{raffel2020exploring} fine-tuned on the ScisummNet/CL-SciSumm-2020 corpus to generate five summaries per document. After generating the five summaries per document, these five summaries are fed to a pre-trained RoBERTa architecture to get five vector representations and against the reference summary the most similar one (based on cosine similarity) is chosen as our T5-Generated Summary.
In the SSN corpus, the summaries are limited to the authors' perspectives as they consist of the paper abstracts. To create more comprehensive summaries, we employed a two-step approach. First, we used a fine-tuned T5 model \cite{raffel2020exploring}, trained on the CL-SciSumm-2020 corpus, to generate five summaries per document by inputting both the abstracts and corresponding citation statements. Then, we have employed a pre-trained RoBERTa architecture to obtain five vector representations for these summaries. The most similar summary to the reference summary, determined by cosine similarity, was selected as our T5-Generated Summary.

\begin{figure}[t]
    \centering
    %\includegraphics[scale=.2]{ijcai2023.drawio.png}
    \includegraphics[scale=.115]{7.png}
    \caption{System architecture of the proposed model}\label{model}
    \label{fig:senPair}
\end{figure}

\subsection{Model Overview}
The investigated summarization model has two units: an extractive and an abstractive summarizer. The overall architecture of the model is portrayed in Figure \ref{model}. This section discusses the architecture and working principle of these two units. 


%~\subsection*{Extractive Summarizer}
While designing the extractive summarizer, we have considered two issues: how the sentences are connected to each other and how semantic units like words affect the sentence level correlations. To fulfill these purposes, we have utilized two different graph-based neural networks: an heterogeneous graph neural network (HeterSumGraph) \cite{heterogeneous} and a graph attention network (GAT) \cite{GAT}. 

For any graph $G = \{V,E\}$, $V$ denotes the nodes and $E$, the edges between them. HeterSumGraph defines $V = V_w \cup V_s$, $V_w$ is the set of unique words and $V_s$ is the set of sentences in the document. For a document with $n$ unique words and $m$ sentences, $E$ is the edge weight matrix, where $e_{i,j}$ represents word $i$ in sentence $j$, $(i \in \{1:n\}, j \in \{1: m\})$ \cite{heterogeneous}. The nodes that represent the sentences are initialized with LongFormer [CLS] tokens. Because LogFormer generates a contextualized word embedding for each occurrence of the word in the document, all of the word embeddings for a word are %added up and then 
averaged to initialize that particular word-representing node in the graph. The edges between the words and sentences are initialized with the corresponding TF-IDF values.

After the graph $G$ is constructed, a graph attention network (GAT) is used to update the node feature values. Considering $h_i \in \mathbb{R}^{d_h}$ where $i \in \{1:(n+m)\}$ as the hidden states of the word and sentence nodes, the GAT layer is designed as:

\begin{gather}
    \mathcal{T}_{i,j} = \textit{LeakyReLU}(\omega_a[\omega_q h_i; \omega_k h_j; e_{i,j}])  \\
    \alpha_{i,j} = \frac{\exp(\mathcal{T}_{i,j})}{\sum_{l \in \mathcal{N}_i} \exp(\mathcal{T}_{i,l})} \\
    u_i = \sigma (\sum_{j \in \mathcal{N}_i} \alpha_{i,j} \omega_v h_j )
\end{gather}
where $\omega_a$, $\omega_q$, $\omega_k$ and $\omega_v$ are learnable weight matrices. $\mathcal{N}_i$ denotes the list of the neighbor nodes. The attention value between $h_i$ and $h_j$ is denoted by $\alpha_{i,j}$. The GAT with multi-head attention (considering $\mathcal{K}$ attention heads) is designed as:
\begin{gather}
    u_i = ||_{k=1}^\mathcal{K} \sigma (\sum_{j \in \mathcal{N}_i} \alpha_{i,j}^k \omega^k h_i )   \label{multihead} 
\end{gather}
To prevent the gradient from vanishing,
%after a number of epochs, 
HeterSumGraph incorporates a residual connection and the final hidden state representation becomes: \iffalse $h_i = u_i + h_i$.\fi
%\iffalse
\begin{gather}
    h_i = u_i + h_i
\end{gather}
%\fi
Through the aforementioned GAT and position-wise feed-forward network (FFN) layer comprising two linear transformations \cite{heterogeneous}, the sentence nodes are updated with their adjacent word nodes:
\begin{gather}
    \mathcal{U}_{w \rightarrow s}^1 = \textit{GAT}(\mathcal{H}_s^0, \mathcal{H}_w^0, \mathcal{H}_w^0) \label{equ:gat} \\
    \mathcal{H}_s^1 = \textit{FFN}(\mathcal{U}_{w \rightarrow s}^1  + \mathcal{H}_s^0)
\end{gather}
where $\mathcal{U}_{w \rightarrow s}^1 \in \mathbb{R}^{n \times d_h}$ $\mathcal{H}_w^1 = \mathcal{H}_w^0 = V_w$, and $\mathcal{H}_s^0 = V_s$. In Eq.\ \ref{equ:gat}, $\mathcal{H}_s^0$ is employed as the attention query and for both the attention key and value $\mathcal{H}_w^0$ is used. Then, the revised sentence nodes are used to generate new representations for the individual word nodes and continue to refine the revised sentence nodes in an iterative fashion. At each iteration, sentence-to-word and word-to-sentence updates continue to be processed. The process can be depicted as follows for the t-th iteration: 
\begin{gather}
    \mathcal{U}_{s \rightarrow w}^{t+1} = \textit{GAT}(\mathcal{H}_w^t, \mathcal{H}_s^t, \mathcal{H}_s^t) \\
    \mathcal{H}_w^{t+1} = \textit{FFN}(\mathcal{U}_{s \rightarrow w}^{t+1}  + \mathcal{H}_w^t)     \\
    \mathcal{U}_{w \rightarrow s}^{t+1} = \textit{GAT}(\mathcal{H}_s^t, \mathcal{H}_w^{t+1}, \mathcal{H}_w^{t+1}) \\
    \mathcal{H}_s^{t+1} = \textit{FFN}(\mathcal{U}_{w \rightarrow s}^{t+1}  + \mathcal{H}_s^t)     
\end{gather}
Once the model training is done, the sentence nodes' representations are used as the sentence vector representations.

For direct sentence-level interactions, we have also used a graph attention neural network (GAT). Here, for the graph $G = \{V,E\}$, $V = V_s$ where $V_s$ is the set of all the sentences in the document. The edge weight matrix $E$ preserves the semantic similarity values between sentences. The nodes are initialized in the same manner as the sentence nodes in HeterSumGraph. For initializing the edges between nodes, at first we have acquired the vector representations of the sentences using pre-trained LongFormer and then computed the cosine similarity between the sentences. The edges are initialized with the corresponding similarity values between sentences. However, as scientific documents come with many sentences, working with a fully connected graph is not computationally cost effective. To reduce the burden of computational overhead, we have dropped the edge connections between nodes whose cosine similarity values are below a certain cut-off value. Throughout the conducted experiments, we have found that if we set the cut-off value below 0.3, the performance of the summarizer model remains the same. 

Considering node features $h = \{ h_1, h_n, ... , h_m \}$ as the input, GAT applies a self attention on the nodes and computes the attention coefficients as follows:
\begin{gather}
    \mathcal{T}_{i,j} = a(\omega h_i, \omega h_j)
\end{gather}
where $a$ is a single-layer feed forward neural network with the \textit{LeakyReLU} activation function, and $\omega$ is a learnable parameter. This attention coefficient shows node j's importance on node i and it is computed only for the corresponding one-hop neighbour nodes ($j \in \mathcal{N}_i$). This attention coefficient value is normalized to compute the attention values as follows:
\begin{gather}
    \alpha_{i,j} = \frac{\exp(\mathcal{T}_{i,j})}{\sum_{l \in \mathcal{N}_i} \exp(\mathcal{T}_{i,l})}
\end{gather}
The multi-head attention is computed in the same way it has been done for HeterSumGraph (Eq.\ \ref{multihead}).

Once the sentence representations from both the HeterSumGraph and GAT are computed, they are concatenated and fed to the feed-forward neural network layer. This is a two-layer position-wise feed-forward layer \cite{heterogeneous} for labeling the sentences with 1 or 0;  1 indicates that particular sentence is included in the extractive summary.

The abstractive summary is generated by the LongFormer decoder. To train the summarizer units in parallel, the training mechanism in \citet{dimsumm} is used. The overall loss $L$ of the model is:
\begin{gather}
    L = L_\textit{ext} + L_\textit{abs}
\end{gather}
where $L_\textit{ext}$ and $L_\textit{abs}$ represent the cross-entropy losses of the extractive and abstractive summarizers, respectively.

\iffalse
\begin{table*}
\centering
\caption{Results on modified SSN corpus. The results for both considering the abstracts and the generated ones incorporating citation statements as the reference summaries are portrayed here.}\label{result:ssn}
\begin{tabular}{lllclcccclcc} 
\hline
\multirow{2}{*}{\textbf{Models}} & \multicolumn{1}{c}{} &  & \multicolumn{4}{c}{\textbf{On Abstracts as Summaries }} &  & \multicolumn{4}{c}{\textbf{ On Generated Summaries }}  \\ 
\cline{4-7}\cline{9-12}
                                 & \multicolumn{1}{c}{} &  & \textbf{R-1}   &  & \textbf{R-2}   & \textbf{R-L}       &  & \textbf{R-1}   &  & \textbf{R-2}   & \textbf{R-L}      \\ 
\hline
\textbf{Extractive}              &                      &  &                &  &                &                    &  &                &  &                &                   \\
TextRank                         &                      &  & 35.49          &  & 9.44           & 31.68              &  & 35.52          &  & 9.23           & 31.65             \\
BERTSumExt                       &                      &  & 42.92          &  & 14.19          & 39.01              &  & 43.11          &  & 14.21          & 39.12             \\
HeterSumGraph                    &                      &  & 44.27          &  & 14.52          & 39.73              &  & 44.30          &  & 14.53          & 39.74             \\ 
\hline
\textbf{Abstractive}             &                      &  &                &  &                &                    &  &                &  &                &                   \\
PTGen+Cov                        &                      &  & 41.66          &  & 13.08          & 36.95              &  & 41.60          &  & 13.10          & 36.72             \\
BERTSumAbs                       &                      &  & 42.06          &  & 14.52          & 38.17              &  & 42.04          &  & 14.56          & 38.17             \\
BERT+CopyTransformer             &                      &  & 42.43          &  & 15.01          & 39.03              &  & 42.44          &  & 15.05          & 39.04             \\ 
\hline
\textbf{Proposed Model}          &                      &  &                &  &                &                    &  &                &  &                &                   \\
Extractive Summarizer            &                      &  & \textbf{45.08} &  & 15.09          & \textbf{40.13}     &  & \textbf{45.05} &  & \textbf{15.14} & \textbf{40.04}    \\
Abstractive Summarizer           &                      &  & 44.72          &  & \textbf{15.12} & 39.24              &  & 44.23          &  & 15.10          & 39.22             \\
\hline
\end{tabular}
\end{table*}
\fi

\begin{table*}
\centering
\caption{Results on the modified SSN corpus. The results consider both the abstracts and the T5-generated summaries incorporating citation statements as the reference summaries. The best results are boldfaced.}\label{result:ssn}
\resizebox{\textwidth}{!}{%
\begin{tabular}{lllcccclcccc} 
\hline
\multirow{2}{*}{\textbf{Models}} &  &  & \multicolumn{4}{c}{\textbf{On Abstracts as Summaries }} &  & \multicolumn{4}{c}{\textbf{ On T5-Generated Summaries }} \\ 
\cline{4-7}\cline{9-12}
     &  &  & \textbf{R-1} & \textbf{R-2} & \textbf{R-L} & \textbf{METEOR} &  & \textbf{R-1} & \textbf{R-2} & \textbf{R-L} & \textbf{METEOR} \\ 
\hline
\textbf{Extractive}   \\
%TextRank &  &  & 35.49 & 9.44 & 31.68 & 24.20 &  & 35.52 & 9.23 & 31.65 & 24.19 \\
BERTSumExt  &  &  & 42.92 & 14.19 & 39.01 & 33.09 &  & 43.11 & 14.21 & 39.12 & 33.07 \\
HeterSumGraph &  &  & 44.27 & 14.52 & 39.73 & 33.18 &  & 44.30 & 14.53 & 39.74 & 33.18 \\
GRETEL &  &  & \textbf{45.22} & \textbf{15.19} & \textbf{40.23} & \textbf{36.87} &  & \textbf{45.23} & 15.19 & \textbf{40.24} & \textbf{36.88} \\
Proposed Model (Extractive) &  &  & 45.19 & 15.18 & 40.21 & 36.83 &  & 45.19 & \textbf{15.21} & 40.23 & 36.85 \\
\hline
\textbf{Abstractive}   \\
PTGen+Cov &  &  & 41.66 & 13.08 & 36.95 & 32.44 &  & 41.60 & 13.10 & 36.72 & 32.40 \\
BERTSumAbs &  &  & 42.06 & 14.52 & 38.17 & 32.49 &  & 42.04 & 14.56 & 38.17 & 32.49 \\
BERT+CopyTransformer &  &  & 42.43 & 15.01 & 39.03 & 32.88 &  & 42.44 & 15.05 & 39.04 & 32.91 \\ 
Proposed Model (Abstractive) &  &  & \textbf{44.82} & \textbf{15.19} & \textbf{39.31} & \textbf{36.50} &  & \textbf{44.83} & \textbf{15.19} & \textbf{39.30} & \textbf{36.51} \\
\hline

\end{tabular}%
}
\end{table*}

\section{Experimental Results and Analysis}\label{result}
This section gives a brief description of the model parameters used in the experiments as well as the results achieved on CL-SciSumm-2020 and the customized SSN datasets.

\subsection{Model Parameters and Training Details}
We have trained our model on a 48GB NVIDIA RTX A6000 GPU. The batch size has been set to 1 as the length of input documents plus the citation statements is large. Since all the experiments are done on a small batch-size, we have followed the training procedure of  \citet{sefid2022scibertsum} and accumulated gradients for 10 steps and updated the parameters. The NOAM scheduler has been utilized to adjust the learning rate and gradients are clipped so that exploding gradients during training can be prevented. The model has been trained for 20,000 epochs. The extractive summarizer is initialized with the LongFormer embeddings. Following that, the LongFormer encoder-decoder architecture for the abstractive summarizer and the extractive summarizer units' forward passes are trained separately. Once both of the forward passes are done for each iteration and the individual losses are calculated, the model's overall loss is calculated. If either of the two unit's validation loss continues to go down for 5 epochs, the parameter settings for that particular unit are saved and that unit's training is postponed for the next 10 epochs. The number of attention-heads for multi-head attention has been set to 8. The stop words and punctuation have been filtered out when pre-processing the word nodes in the graph. Following \citet{heterogeneous}, 10\% of the words in the vocabulary having low TF-IDF values have been further filtered out. The word and sentence nodes have been initialized with 768-dimensional vectors. And the sentence representations from both the HeterSumGraph and GAT are 512-dimensional vectors. So, the final sentence vectors after the concatenation step are 1024-dimensional vectors. The Feed Forward Network hidden layer size is 512. 

\subsection{Performance Analysis of the Model}
We have performed experiments on two datasets: modified SSN and CL-SciSumm-2020. The results achieved by our models are reported as overlapping unigrams, bigrams, and the longest common sequence between the generated summaries and the reference summaries by means of R-1, R-2, and R-L metrics; and semantic compatibility between the reference and generated summaries by means of METEOR metric, respectively, for the modified SSN corpus. R-1, and R-2 show the informativeness, and R-L shows the fluency of the generated summary. The metrics used for analyzing the model performance on CL-SciSumm-2020 are R-2 and R-SU4, which indicate the proportion of bigram overlap and unigram plus skipgram of 4 tokens overlap, respectively, between the reference and generated summaries. The performance here is also analyzed with the METEOR metric. As the Bi-directional encoder and autoregressive decoder we have also experimented with BigBird. However, the better performance was found with LongFormer. That is why in the final model, we have used LongFormer in all the cases for initial encoding and generating abstractive summaries.

\begin{table*}
\centering
\caption{Model performance analysis on two CL-SciSumm-2020 summary categories. All values are F-1 scores.} \label{result:ScisummNet}
\begin{tabular}{llccclccc} 
\hline
\multirow{2}{*}{\textbf{Models}} & & \multicolumn{3}{c}{\textbf{Abstracts as Summaries}} &  & \multicolumn{3}{c}{\textbf{Human-created Summaries}}  \\ 
\cline{3-5}\cline{7-9}
     &  & \textbf{R-2} & \textbf{R-SU4} & \textbf{METEOR} &  & \textbf{R-2} & \textbf{R-SU4} & \textbf{METEOR} \\ 
\hline
Jaccard-focused GCN &  & 0.19931 & 0.09956 & - &  & 0.2042 & 0.14162 & - \\
Clustering  &  & 0.1959 & 0.0962 & - &  & 0.1749 & 0.1169 & - \\
MMR2 &  & 0.15067 & 0.07851 & - &  & 0.15073 & 0.10237 & - \\
LSTM+BabelNet &  & 0.329 & 0.172 & - &  & 0.241 & 0.171 & - \\ 
\hline
\textbf{Proposed Model} \\
Extractive Summarizer &  & 0.43 & 0.266 & 31.12 &  & 0.42 & 0.249 & 30.18 \\
Abstractive Summarizer &  & 0.43 & 0.250 & 30.98 &  & 0.41 & 0.234 & 30.06  \\
\hline
\end{tabular}
\end{table*}

\subsection{Results: Modified SSN Corpus} 
To compare the performance of our model with the existing extractive models, we train and test the following extractive summarizer models on our modified corpus: (1) BERTSumEXT \cite{liu2019text}: a BERT-based model; (2) HeterSumGraph \cite{heterogeneous}: a heterogeneous graph-based approach that considers the cross-sentence correlations using additional semantic units; and (3) GRETEL: fuses semantic information from the document context and gold summary using a hierarchical transformer encoder and graph contrastive learning. For the abstractive summarization baseline, we have experimented with: (1) PTGen+Cov \cite{see2017gettothepoint}: based on a hybrid pointer generator network to copy words from the source text, (2) BERTSumAbs \cite{liu2019text}: a BERT-based model; and (3) BERT+CopyTransformer \cite{aksenov2020abstractive}: applies BERT-windowing for processing data longer than the BERT window. 

The performance of the existing models and our proposed models are shown in Table \ref{result:ssn}. As reference summaries, we have considered both the paper abstracts as well as the summaries we have generated from the abstracts plus the citing statements using T5.  

Although BERTSumExt and BERTSumAbs perform very well with short documents, their performance metrics are not at that level when summarizing scientific documents. The main reason for this is their limitation to working with a maximum 512 input tokens, but scientific documents are much longer. For this, they have applied the greedy algorithm introduced by \citet{nallapati2016abstractive}. 
%However, while working with the whole document as well, the abstractive as well as the extractive summarizers in our model perform better.
%GRETEL, among the other extractive summatizers experimented with here, performed the best with the scientific documents. 
HeterSumGraph considers direct relationships between words and sentences on texts with a 50-sentence maximum, whereas our proposed model considers direct cross-sentence correlations, as well, and can deal with longer text spans (up to 3500 words). These additional features, together with LongFormer's enriched word and sentence features, gives our model a performance boost, but our model requires more computational time and resources. Our model performs better by a good margin compared to the other models apart from GRETEL. Our extractive summarizer shows slightly lower performance compared to GRETEL which is a more complex model. Still, because of the parallel training approach, our model has achieved comparable results.  Our abstractive summarizer model outperforms the other experimental abstractive summarizers by large margins: PTGen+Cov by 2.36, BertSumAbs by 1.14, and BERT+CopyTransformer by 0.28 R-L scores. The METEOR scores achieved by our model are 36.83 and 36.50 for extractive and abstractive summaries, respectively, when tested over the T5-generated summaries. In the experiment with the abstracts as summaries, the METEOR scores are 36.51 and 36.85 for the abstractive and extractive summaries, respectively. Looking at the METEOR scores achieved by the other models (see Table \ref{result:ssn}), it is clearly visible that both the extractive and abstractive summarizer units of our model have outperformed them by at least 3. This observation indicates that the summaries generated by our proposed model are more semantically similar to the reference summaries. To see the importance of the individual units, please check the ablation study in the appendix. 

\subsection{Results: CL-SciSumm-2020 Corpus}



\iffalse
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{table*}
\centering
\caption{Ablation Study} \label{ablation}
\begin{tabular}{llccclccc} 
\hline
\multirow{2}{*}{\textbf{Models}} & & \multicolumn{3}{c}{\textbf{Abstract}} &  & \multicolumn{3}{c}{\textbf{Human}}  \\ 
\cline{3-5}\cline{7-9}
     &  & \textbf{R-2} & \textbf{R-SU4} & \textbf{METEOR} &  & \textbf{R-2} & \textbf{R-SU4} & \textbf{METEOR} \\ 
\hline
Test 1  \\
Test 2   \\
Test 3  \\
Test 4  \\
Test 5 \\
\hline
\end{tabular}
\end{table*}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\fi
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% ablation study %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\iffalse
\begin{table*}
\centering
\caption{Ablation Study}\label{result:ablation}
\begin{tabular}{lllcccclcccc} 
\hline
\multirow{2}{*}{\textbf{Discarded Unit}} &  &  & \multicolumn{4}{c}{\textbf{On Abstracts as Summaries }} &  & \multicolumn{4}{c}{\textbf{ On T5-Generated Summaries }} \\ 
\cline{4-7}\cline{9-12}
     &  &  & \textbf{R-1} & \textbf{R-2} & \textbf{R-L} & \textbf{METEOR} &  & \textbf{R-1} & \textbf{R-2} & \textbf{R-L} & \textbf{METEOR} \\ 
\hline

GAT &  &  & 35.49 & 9.44 & 31.68 & xxx &  & 35.52 & 9.23 & 31.65 & yyy \\
HeterSumGraph  &  &  & 42.92 & 14.19 & 39.01 & xxx &  & 43.11 & 14.21 & 39.12 & yyy \\
Extractive Summarizer &  &  & 44.27 & 14.52 & 39.73 & xxx &  & 44.30 & 14.53 & 39.74 & yyy \\
Abstractive Summarizer &  &  & \textbf{45.08} & \textbf{15.09} & \textbf{40.13} & \textbf{36.76} &  & \textbf{45.05} & \textbf{15.14} & \textbf{40.04} & \textbf{36.76} \\
\hline
\end{tabular}
\end{table*}
\fi

\iffalse
\begin{table*}
\centering
\caption{Ablation Study: Rows labeled with $\dagger$ indicates the extractive summaries and rows labeled with $\ding{42}$ indicates abstractive summaries.}\label{result:ablation}
\begin{tabular}{lllcccc} 
\hline
\multirow{2}{*}{\textbf{Discarded Unit}} &  &  & \multicolumn{4}{c}{\textbf{ On T5-Generated Summaries }}  \\ 
\cline{4-7}
     &  &  & \textbf{R-1} & \textbf{R-2} & \textbf{R-L} & \textbf{METEOR} \\ 
\hline

GAT$\dagger$ &  &  & 44.86 & 14.9 & 39.96 & 36.52 \\
HeterSumGraph$\dagger$  &  &  & 44.78 & 14.81 & 39.84. & 36.49 \\
Extractive Summarizer$\ding{42}$ &  &  & 43.01 & 15.02 & 38.99 & 35.92  \\
Abstractive Summarizer$\dagger$ &  &  & 44.91 & 14.95 & 39.96 & 36.50 \\
\hline
\end{tabular}
\end{table*}
\fi
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% ablation study ends %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
For analyzing our proposed model's performances on CL-SciSumm-2020 Corpus, we have used R-2 and R-SU4 F-1 scores (as the other comparable models are reported with these metrics) %using the performance evaluation script provided by CL-SciSumm. 
We have experimented to generate abstract and human summaries. As benchmarks, we have selected the research works submitted to CL-SciSumm-2019/2020: (1) Jaccard-focused GCN \cite{umapathy2020citeqa}: an extractive summarizer utilizing cross-sentence graph and graph attention networks, (2) Clustering \cite{mishra2020iitp}: based on different clustering algorithms followed by sentence-scoring functions, (3) MMR2 \cite{reddy2020iiitbh}: based on the maximal marginal relevance technique, and (4) LSTM+BabelNet \cite{chiruzzo2019lastus}: BabelNet vectors were used to train the LSTM. The CL-SciSumm task provides a performance metric evaluation script which is used to calculate the R-2 and R-SU4 values for the model-generated summaries against the test set. 

Results on CL-SciSumm-2020 are reported in Table \ref{result:ScisummNet}. Looking at the results, it is clear that our model outperforms the other existing extractive models on every measure. The R-2 and R-SU4 achieved for both of our model-generated extractive and abstractive summaries are very high compared to the other existing extractive models. And this is the case for both the original abstracts and the human-created summaries as reference summaries.
For the human-created reference summaries, our extractive and abstractive summarizers have achieved 0.078 and 0.063 R-SU4 F-1 score gains, respectively, compared to the LSTM+BabelNet model, which comes with the best result among the other considered models. While considering the abstracts of the papers as reference summaries, these gains are 0.094 and 0.078, respectively. For the abstractive summaries, the METEOR score achieved by our model is 30.18 whereas for the extractive summaries, it has achieved a 30.06 METEOR score on the human-generated summaries. Over the abstracts of the papers, these scores are 31.12 and 30.98, respectively. 

\iffalse
\subsection{Ablation Study}
To portray a better grasp of each component's contribution in our suggested model, we have experimented with different units of our model separately and the results are reported in Table \ref{result:ablation}. All of these experiments are performed on the T5-generated corpus which combines the abstract of the paper along with the citation statements.

In our first experiment, we have discarded the GAT unit which works with cross-sentence relationships and kept only the HeterSumGraph for extractive summary generation. This time the performances of the model are lower than the reported results in Tables \ref{result:ssn} (R-1: 44.86, R-2: 14.91, R-L: 39.96, and METEOR: 36.52) for our generated extractive summaries. Still, these results are higher compared to the original HeterSumGraph model. It shows, using the LongFormer encoder in the beginning and using the collective loss function for both the abstractive and extractive summarizer units play a significant role in the performance boost. And it also indicates that taking direct cross-sentence correlations into consideration provides some additional features to enrich the model which helps the model's performance to improve.

In the second experiment, we have discarded the HeterSumGraph unit and used only GAT in the extractive summarization unit. This time the performance metrics for extractive summaries are R-1: 44.78, R-2: 14.81, R-L: 39.84,and METEOR: 36.4. These values are comparably lower than we gained in the last experiment. The reason behind this incident is, though no direct cross-sentence relationships are present, HeterSumGraph, by 2-hop distance, considers the correlations between sentences. 

The third experiment discards the extractive summarizer unit. The LongFormer abstractive summarizer unit achieves very poor R-1: 43.01, R-2: 15.02, R-L: 38.99, and METEOR: 35.92 scores compared to the proposed model.
This poor performance demonstrates the importance of the information that the extractive summarizer provides the abstractive summarizer through the combined loss function.

Finally, we have discarded the abstract summarizer unit and used the combination of HeterSumGraph and GAT for extractive summary generation. During this experiment, the achieved R-1, R-2 R-L, and METEOR scores are 44.91, 14.95, 39.96, and 36.50, respectively, which are more than the cases for the three above-mentioned ablation experiments. It indicates the significance of training the abstractive summarization unit in parallel as well as using the cross-sentence and semantic unit-sentence correlations at the same time. 
\fi
\section{Conclusion and Future Work}
In this paper, we have introduced a summarizer model considering two intentions: first, summarize scientific documents incorporating citation contexts, and second, build a summarizer model which can generate both extractive and abstractive summaries by means of parallel training so that both counterparts can gain a performance boost. For this, we have utilized both the sentence-sentence and sentence-word correlations. Furthermore, we have constructed a corpus comprising 10K scientific articles with their corresponding citation statements for the summarization task. The experimental results show that our model performs well compared to other well-known methods.
Though this work considers the research community's observations (citing statements), it doesn't consider the background information (references presented in the target article). In our future work, we are planning to use both sides of the citation graph (references as the background knowledge and the citing statements as the research community's views) while summarizing a scientific article.

\section*{Acknowledgements}
We would like to thank all reviewers for their comments. This research is partially funded by The Natural Sciences and Engineering Research Council of Canada (NSERC) through a Discovery Grant to R. E. Mercer.
\section*{Limitations}
Our experiments are limited to summarize long scientific texts only. We have not conducted any experiments with short target texts, consequently we are not sure how well the model may perform while summarizing short texts. We are also unsure how well this model may perform for extreme summary generation like TLDR \cite{cachola2020tldr}. Moreover, we have trained both the extractive and abstractive summarizer units for a large number of epochs. Though to prevent any unit from being over-fitted we have checked the curve of validation loss after every 5 epochs. This is very computationally expensive and demands a longer period of time for model training. Furthermore, no tests have been performed to see how the abstractive summarizer unit suffers from hallucination. 

\section*{Ethics Statement}
We do not see any ethics issue here in this paper.



\iffalse
\section{Conclusion}
We have shown that an iterative method such as the preconditioned
conjugate-gradients method may be successfully parallelized by using
highly efficient parallel implementations of the linear algebra
operations involved. We have used the same approach to parallelize
other iterative methods with similar degrees of efficiency (see
\cite{kn:daCunha92a} and \cite {kn:daCunha92b}).
\ack We would like to thank the referees for their comments, which
helped improve this paper considerably
\fi

% Entries for the entire Anthology, followed by custom entries
\bibliography{custom}
\bibliographystyle{acl_natbib}

\newpage\mbox{}
\newpage

\appendix
%\onecolumn
\section{Appendix}
\label{sec:appendix}

\subsection{Ablation Study}
\begin{table}[h]

\centering
\caption{Ablation Study: Rows labeled with $\dagger$ indicate the extractive summaries and rows labeled with $\ding{42}$ indicate abstractive summaries.}\label{result:ablation}
\resizebox{\columnwidth}{!}{%
\begin{tabular}{lllcccc} 
\hline
\multirow{2}{*}{\textbf{Discarded Unit}} &  &  & \multicolumn{4}{c}{\textbf{ On T5-Generated Summaries }}  \\ 
\cline{4-7}
     &  &  & \textbf{R-1} & \textbf{R-2} & \textbf{R-L} & \textbf{METEOR} \\ 
\hline

GAT$\dagger$ &  &  & 44.86 & 14.9 & 39.96 & 36.52 \\
HeterSumGraph$\dagger$  &  &  & 44.78 & 14.81 & 39.84. & 36.49 \\
Extractive Summarizer$\ding{42}$ &  &  & 43.01 & 15.02 & 38.99 & 35.92  \\
Abstractive Summarizer$\dagger$ &  &  & 44.91 & 14.95 & 39.96 & 36.50 \\
\hline
\end{tabular}%
}
\end{table}

To portray a better grasp of each component's contribution in our suggested model, we have experimented with different units of our model separately and the results are reported in Table \ref{result:ablation}. All of these experiments are performed on the T5-generated corpus which combines the abstract of the paper along with the citation statements.

In our first experiment, we have discarded the GAT unit which works with cross-sentence relationships and kept only the HeterSumGraph for extractive summary generation. This time the performances of the model are lower than the reported results in Tables \ref{result:ssn} (R-1: 44.86, R-2: 14.91, R-L: 39.96, and METEOR: 36.52) for our generated extractive summaries. Still, these results are higher compared to the original HeterSumGraph model. It shows, using the LongFormer encoder in the beginning and using the collective loss function for both the abstractive and extractive summarizer units play a significant role in the performance boost. And it also indicates that taking direct cross-sentence correlations into consideration provides some additional features to enrich the model which helps the model's performance to improve.

In the second experiment, we have discarded the HeterSumGraph unit and used only GAT in the extractive summarization unit. This time the performance metrics for extractive summaries are R-1: 44.78, R-2: 14.81, R-L: 39.84,and METEOR: 36.4. These values are comparably lower than we gained in the last experiment. The reason behind this incident is, though no direct cross-sentence relationships are present, HeterSumGraph, by 2-hop distance, considers the correlations between sentences. 

The third experiment discards the extractive summarizer unit. The LongFormer abstractive summarizer unit achieves very poor R-1: 43.01, R-2: 15.02, R-L: 38.99, and METEOR: 35.92 scores compared to the proposed model.
This poor performance demonstrates the importance of the information that the extractive summarizer provides the abstractive summarizer through the combined loss function.

Finally, we have discarded the abstract summarizer unit and used the combination of HeterSumGraph and GAT for extractive summary generation. During this experiment, the achieved R-1, R-2 R-L, and METEOR scores are 44.91, 14.95, 39.96, and 36.50, respectively, which are more than the cases for the three above-mentioned ablation experiments. It indicates the significance of training the abstractive summarization unit in parallel as well as using the cross-sentence and semantic unit-sentence correlations at the same time. 

\subsection{Validity Check of the Proposed Corpus}
%To validate the quality of the proposed corpus, we have performed an analysis on a statistically representative sample of the corpus (95\% confidence, and 3\% error margin) with three human annotators' assistance. From the pool of 10,000 summarization samples, 400 were chosen randomly and annotated by three annotators for this statistical analysis.
%Each annotator assessed whether the T5-generated summaries capture the same information as the abstract plus the citing statements.
%Annotator one said that 374 samples did, annotator two, 368 and annotator three, 371. Annotators one and two agreed that 368 samples compare correctly and 16 do not giving a Cohen's $\kappa=0.89$. Between annotators two and three the agreement is with 396 samples (368 are correctly summarized and 28 are not) with $\kappa=0.93$. Between annotators one and three the agreement is found for 398 summaries (370 correctly summarized and 27 not) with $\kappa=0.94$.

To ascertain the corpus's quality, a rigorous analysis was conducted on a statistically significant subset of the dataset, with a confidence level of 95\% and a margin of error of 3\%, aided by three human annotators. Within the vast pool of 10,000 summarization samples, a random selection of 400 was subject to annotation for this statistical inquiry.

Each annotator was tasked with evaluating whether the summaries generated by the T5 model effectively encapsulated the same information as the combination of the abstract and the citing statements. The first annotator affirmed that 374 samples achieved this concurrence, the second annotator concurred with 368, and the third annotator with 371.

When comparing the assessments of the first and second annotators, it was determined they agreed that 368 samples were appropriately summarized, while 16 were not, resulting in a substantial Cohen's $\kappa$ of 0.89. In the comparison between the second and third annotators, a significant concurrence emerged for 396 samples, where 368 were accurately summarized, and 28 were not, yielding $\kappa$ value of 0.93. Similarly, when examining the assessments of the first and third annotators, agreement was established for 398 summaries, with 370 being correctly summarized and 27 not, resulting in $\kappa$ of 0.94.
\end{document}
