% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{graphicx} 
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{RGAT at SemEval-2024 Task 2: Biomedical Natural Language Inference using Graph Attention Network}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Abir Chakraborty \\
  Microsoft\\
 \texttt{Abir.Chakraborty@microsoft.com} 
\\}

\begin{document}
\maketitle
\begin{abstract}
In this work, we (team RGAT) describe our approaches for the SemEval 2024 Task 2: Safe Biomedical Natural Language Inference for Clinical Trials (NLI4CT). The objective of this task is multi-evidence natural language inference based on different sections of clinical trial reports. We have explored various approaches, (a) dependency tree of the input query as additional features in a Graph Attention Network (GAT) along with the token and parts-of-speech features, (b) sequence-to-sequence approach using various models and synthetic data and finally, (c) in-context learning using large language models (LLMs) like GPT-4. Amongst these three approaches the best result is obtained from the LLM with 0.76 F1-score (the highest being 0.78), 0.86 in faithfulness and 0.74 in consistence.    
\end{abstract}

\section{Introduction}
Clinical trials are advanced treatments and tests to evaluate new ways of treating life-threatening diseases where interventions include new drugs, cells and other biological products, advanced surgical or radiological procedures and devices. As the trial progresses the observations are documented systematically in a Clinical Trial report that includes the subject selection criteria ('Eligibility'), treatments ('Interventions') and results at group level including adverse effects. These reports constitute a rich source of past endeavours to learn from and help in formulating new treatment plans. However, the sheer volume of CT reports\footnote{As of Jan 17, 2024, ClinicalTrials.gov lists 480,795 CT studies} makes it impossible to conduct extensive manual evaluation. Thus, it is necessary to have an automated pipeline that can enquire a CT report for specific hypothesis and provides high accuracy and reliability at the same time.  

Natural language inference or NLI~\citep{devlin-etal-2019-bert} is one of the standard NLP tasks where a hypothesis is qualified as true (entailment) or false (contradiction) or even undetermined (neutral) given a premise. This task is adopted for reasoning over CT reports by \citet{jullien-etal-2023-semeval} where two new tasks are created based on NLI4CT dataset, (1) NLI over CT reports and (2) extracting the evidence/mention from CT reports to support the inference label. The Semeval 2024 Task 2 NLI4CT is also based on the same NLI4CT dataset (identical for training) with modifications in the test split (more details in the Data section). The inferencing is challenging as it requires multi-hop reasoning, i.e., dependency and aggregation are required over different pieces of the document. 

Other than the complexity associated with multi-hop reasoning, the domain and the associated word-distribution also creates significant challenge due to the presence of aliases, acronyms and biomedical terminologies \cite{Biobert_Lee_2019, Shickel_2018, jin-etal-2019-deep}. This results in significant drop in model performance as is evident in the NLI results last year \citep{jullien-etal-2023-semeval} where it was found that majority of the submitted solutions failed to outperform the baseline solution with a significant margin. The challenge is also evident in the overall performance of models on general NLI datasets (e.g., Stanford NLI or SNLI) where the best model results in 93.1\% F1-score~\citep{wang2021entailment}. 

When it comes to different modeling approaches, many of the top-performing models for the SNLI dataset are ensemble in nature. While initial individual models are based on RNN, most of the latest ones are based on the Transformer architecture and pretrained language models like RoBERTa or T5. Similar trend can also be seen in \citet{jullien-etal-2023-semeval} where the best model is an ensemble and both DeBERTa and Flan-T5 made their way to the top. Interestingly, LLMs like GPT3.5 could not make a significant boost in the performance. 

In our approach, we explored three different modeling paradigms, namely, (1) custom Graph Attention Network (GAT) based discriminative model with novel features based on the dependency tree of the input query, (2) generative models based on T5 and Flan-T5 but enriched with synthetic data used for both pre-training and fine-tuning, and (3) LLM like GPT-4 applied with and without few-shot examples. It is not surprising that the best performance was obtained by GPT-4 stressing on the importance of generic knowledge (that is embedded in these LLMs) rather than fine-tuning, especially when the dataset is not large enough.

The organization of the paper is as follows. In the next section we provide a detailed literature survey on the techniques employed for NLI. Next, we present the details of the proposed approaches. Subsequently, the model predictions and comparisons with other baseline methods are discussed. Finally, conclusions are drawn and scope for future works is outlined.

\section{Related Work}
The existing body of work for the general NLI is quite rich where they are based on the Stanford NLI (SNLI) dataset (550k examples but restricted to a single text genre) \cite{bowman-etal-2015-large} and three other NLI datasets present in GLUE~\cite{wang-etal-2018-glue}, namely, MNLI, QNLI and WNLI. The MNLI (Multi-Genre Natural Language Inference Corpus) dataset \citep{williams-etal-2018-broad} is a crowd-sourced NLI dataset gathered from different sources, e.g., government reports (and covers different genres, e.g., fiction, travel). Given a premise-hypothesis pair of sentences, the task is to predict one of the three classes, namely, whether the premise sentence entails the hypothesis (entailment), contradicts the hypothesis (contradiction), or neither (neutral). The QNLI is modified from Stanford Question Answer Dataset \cite{rajpurkar2016squad} where the task is to determine whether the context sentence contains the answer to the question.
Similarly, the WNLI dataset is created from the Winograd Schema Challenge \cite{10.5555/3031843.3031909} where a coreference resolution problem is converted into an entailment problem involving a pronoun and its referent.
Another large NLI dataset is multi-genre NLI (MNLI) that has 433k examples covering multiple genres and supporting cross-genre evaluation. Some of the best performances are obtained by RoBERTa \cite{liu2019roberta}, XLNet \cite{yang2020xlnet}, Multi-Task Deep Neural Network (MT-DNN) \cite{liu2019improving} and generative pre-training (GPT) approach \cite{radford2018improving}. 

There are few NLI datasets in the biomedical domain, namely, MedNLI \cite{romanov-shivade-2018-lessons} and BioNLI \cite{bastan-etal-2022-bionli}. MedNLI has 14k example pairs created by clinicians on 4,683 premises with three categories, entailment, contradiction and neutral. BioNLI, on the other hand, goes beyond sentence-level inference and includes large context as premises that requires handling complex texts as well as domain knowledge. \citeauthor{bastan-etal-2022-bionli} also includes negative examples as adversarial hypothesis using nine strategies which is a speciality of this dataset. 

There are three biomedical domain specific models that are typically used on these datasets. Starting with the available weights of BERT (pretrained on general domain corpora), BioBERT~\cite{Lee_2019} is trained on PubMed abstracts and PMC full-text articles and shown to outperform BERT on NER, relation extraction and Q\&A, all in the biomedical domain. PubMedBERT~\cite{Gu_2021} is a BERT model created from scratch (rather than starting with general domain corpora) on large biomedical domain dataset like PubMed and achieved impressive performance for tasks like NER and Q\&A. BioLinkBERT~\cite{yasunaga-etal-2022-linkbert} further exploited links between PubMed documents to create a richer context that is used to build a language model (LM). This model has obtained SOTA performance on biomedical datasets such as BLURB \cite{Gu_2021} and BioASQ~\cite{Nentidis_2020}. Another model that achieved SOTA performance on MedNLI is SciFive \cite{phan2021scifive} which is based on T5 paradigm. 

There are not many studies on the application of Graph Neural Network for NLI. Inspired by KIM~\cite{chen-etal-2018-neural-natural} where external knowledge is infused for NLI task, \citet{song_kganet_2020} developed a joint training model where Graph Attention Network (GAT) is used to represent the sub-graph associated with entities that are involved in the hypothesis. Another closely related GAT application is from \citet{math9161949} applied for fact verification on Wikipedia articles. Typical applications of GAT in the NLP domain are for question answering, semantic parsing, information extraction and Named Entity Recognition \cite{wu2022graph, chakraborty-2023-rgat}.

\section{Task Description \& Data}
The dataset for Multi-evidence NLI for Clinical Trial (NLI4CT) is based on a collection of breast-cancer CT reports\footnote{extracted from  https://clinicaltrials.gov/ct2/home} containing statements, explanations and labels annotated by domain expert annotators~\cite{jullien-etal-2024-semeval}. Each CT report has four sections: (a) Eligibility criteria (a set of conditions for patients to be included in the trial cohort), (b) Intervention (information regarding the details of treatments administered), (c) Results (what is the outcome of these treatments) and (d) Adverse events (if anything was observed during the period of the trial). The annotated statements (hypothesis) are claims extracted from one of the four sections (with an average length of 19.5 tokens) and may even compare more than one report. Each statement is qualified as either 'Contradiction' or 'Entailment'. 

There are 1700 examples in the training set and 200 in the development/validation set with exactly 50:50 split of the two classes. The test set has 5500 examples with unknown label distribution. A typical example looks like the following:
\begin{enumerate}
\item \textbf{Hypothesis}: 'All the primary trial participants \color{blue}{do not receive}  \color{black} any oral capecitabine, oral lapatinib ditosylate or cixutumumab IV, in contrast all the secondary trial subjects receive these.'

\item \textbf{Primary context}: 'Patients with early stage, ER positive primary breast cancer undergo FLT PET scan at baseline and 1-6 weeks after the start of standard endocrine treatment. The surgery follows 1-7 days after the second FLT PET scan.'

\item \textbf{Secondary context}: \color{blue}'Patients receive oral capecitabine twice daily on days 1-14 and oral lapatinib ditosylate once daily on days 1-21. Courses repeat every 21 days in the absence of disease progression or unacceptable toxicity'\color{black}

\item \textbf{Label}: 'Contradiction'
\end{enumerate}
where the secondary context provides the justification of the label.

\begin{figure}
\centering
\includegraphics[width=3in]{custom_model.png}
\caption{The architecture of the custom model using GAT and Multi-head attention (MHA).}
\label{fig:custom}
\end{figure}

\section{Methodology}
We have explored three different modeling strategies for the prediction of the inference label. They are (1) custom discriminative model with GAT applied to create features from the dependency tree of the hypothesis statement, (2) sequence-to-sequence generative models based on T5 and Flan-T5 but enriched with synthetic data used in both pre-training and fine-tuning and (3) LLM based solution with and without Few-shot examples.

\subsection{Discriminative Model}
The architecture of our custom discriminative model is shown in Fig~\ref{fig:custom}. We use the tokens of both the hypothesis and the premises to generate a representation using either a standard BERT or RoBERTa model (referred as $h_{tokens}$ for the hypothesis and $h_{pre}$ for the premise. Following the RGAT approach of \citet{wang-etal-2020-relational} (originally meant for aspect polarity detection) we utilize the dependency structure of the input hypothesis ($X_{dep}^h$) that captures the grammatical relations by connecting the words with the corresponding dependency type. However, we do not reorient the dependency tree since there is no aspect word in our application. Using GAT based processing of the hypothesis dependency tree we generate additional features $h_{dep}$. Details of the GAT based processing are provided in Appendix~\ref{appendix:gat}. We concatenate both the features of the hypothesis ($h_{dep}$ and $h_{tokens}$) and pass through a linear layer to create the final hypothesis feature, $h_{hyp}$. For the premise, there is only the token based feature, $h_{pre}$, which is used as a key and value in a standard multi-head attention (MHA) with $h_{hyp}$ as the query vector. This process is repeated multiple times (maximum 3) with the output of the previous MHA layer. Finally, we take the first vector of the MHA output (corresponding to [CLS]) and pass it through a linear layer to generate the logits. The model is trained for binary cross-entropy loss.

\subsection{Generative Model}
In the 2023 SemEval challenge \cite{jullien-etal-2023-semeval}, it was found that generative models outperformed discriminative models on the entailment task. We also explore different T5 models (small and base T5 and base SciFive) for the current entailment task with the exception that we have also generated synthetic data for pre-training as well as fine-tuning.  

\begin{table*}[h]
\centering
\begin{tabular}{rcc|cc}
\hline
Model Type & Base Model & Model Parameters & Dev-F1 & Test-F1 \\
\hline
Cross-attention & BERT-base & 110 M & 0.64 \\
Combined pooler & BERT-base & 110 M & 0.65 \\
Cross-attention + GAT & BERT-base & 110 M & {\bf 0.67} & 0.49 \\
Cross-attention + GAT & BERT-large & 351 M & {\bf 0.67} & 0.50 \\
\hline
\end{tabular}
\caption{Performance of the custom discriminative model on the validation and test dataset}
\label{tab:result1}
\end{table*}

\subsubsection{Generation of Synthetic Data}
For generating synthetic data for T5 pre-training we follow (1) the standard T5 random span masking\footnote{https://github.com/google-research/text-to-text-transfer-transformer} for both the hypothesis and premise sentences and (2) ask GPT-4 to identify spans and mask them subsequently. The first approach works better for the quality of the data and we use this approach for generating the final pre-training data. We have used noise density = 0.4 and average noise span length of 2 and generate 73,457 pre-training examples.

For generating additional fine-tuning data, we use GPT-4 (with temperature = 0.7) with three additional tasks, namely, (a) Question answering on the premise text, and (b) additional inference data from the same set of premises and (c) create a contradictory hypothesis from the original hypothesis. For the first task, examples look like
\begin{enumerate}
    \item \color{blue}Question: \color{black}'How many weeks after the start of standard endocrine treatment is the second FLT PET scan conducted?', \color{blue}Answer: \color{black} '1-6 weeks'
    \item \color{blue}Question: \color{black}'On which days is oral capecitabine given in Arm A?', \color{blue}Answer: \color{black} 'days 1-14'
\end{enumerate}
Additional NLI examples are
\begin{enumerate}
    \item \color{blue}Hypothesis:\color{black} No adverse events were reported in the clinical trial., \color{blue}Label: \color{black} Entailment
    \item \color{blue}Hypothesis: \color{black}The clinical trial report had 765 adverse events in one section and 88 in another section., \color{blue}Label: \color{black} Contradiction
\end{enumerate}
In this process we generate 11k Q\&A pairs and 45k NLI pairs and 1700 contradictory NLI examples from the original 1700 training examples.

\subsection{Large Language Model}
It was also observed in 2023 SemEval challenge \cite{jullien-etal-2023-semeval} that increase in model size also improves the performance. We further validate this hypothesis by applying GPT-4 to the NLI task with and without few-shot examples. 

\subsection{Implementation Details}
For the discriminative model we use the bi-affine parser \cite{dozat-manning-2016} from AllenNLP for dependency parsing. For all experiments, the embedding dimension for the dependency relation is same as the hidden dimension of the BERT/RoBERTa model. We use 3 MHA layers with 8 heads and 2 GAT layers with 6 heads and all the dropouts are fixed at 0.3. The model has a total of 110 million parameters for BERT-base and 351 million parameters for BERT-large. The last hidden state of the pre-trained BERT\footnote{https://github.com/huggingface/transformers} is used for the initial token representations which is subsequently fine-tuned. All models are trained for 50 epochs using Adam optimizer \cite{kingma-adam-2014} (with the default parameters), a learning rate of $5\times10^{-5}$ and a batch size of 8. 

We have pretrained both small and base T5 models for subsequent NLI task. Pretraining is done for 20 epochs with a batch size of 16 and learning rate of $5\times10^{-5}$ with Adam optimizer. From the 73,457 span masked examples, we use 66111 for training and 7346 for validation that is used to keep track of the validation loss and saving the model.

\begin{table*}
\centering
\begin{tabular}{rcc|cc}
\hline
Model Type & Model & Additional Data & Dev-F1 & Test-F1 \\
\hline
random initial weight & small T5 (60.5 M) & None & 0.55 \\
random initial weight & small T5 (60.5 M) & synthetic NLI data-I & 0.51 \\
random initial weight & small T5 (60.5 M) & synthetic NLI data-II & 0.53 \\
pretrained with CTR data & small Flan-T5 (76 M) & None & 0.58 \\
\hline
pretrained & base T5 (223 M) & None & 0.64 \\
pretrained & base T5 (223 M) & Synthetic Q\&A data & 0.43 \\
pretrained & base T5 (223 M) & Synthetic NLI-I data & 0.55 \\
pretrained & base T5 (223 M) & Synthetic NLI-II data & 0.54 \\
pretrained & Flan-T5 base (247 M) & None & 0.66 & 0.608 \\
pretrained & Flan-T5 base (247 M) & Synthetic NLI-I & - & 0.535 \\
\hline
 - & GPT-4 ({\tt0613}) & Zero-shot & -  & $\mathbf{0.761}$ \\
\hline
\end{tabular}
\caption{Performance of different generative models including GPT-4.}
\label{tab:result2}
\end{table*}

\section{Results}
In this section, first we describe the performance of the custom discriminative model followed by the performance of the fine-tuned T5 model and finally the results from GPT-4. Although we compute precision, recall and F1-score for all our experiments we report only F1-score here. It is to be noted that we did not evaluate our model on the test dataset for all our experiments and submitted test results only for the best validation performance. Thus, for most of our experiments we report only the validation F1-score and also mention the test F1-score wherever available.  Table~\ref{tab:result1} summarizes the results from the custom discriminative model. There are four flavors of this model, one with BERT-large and three with BERT-base. Within BERT-base, we have one with cross-attention, one without ('combined-pooler' that only concatenates the two BERT outputs) and the third one with cross-attention and GAT. It can be seen that the presence of GAT improves the validation F1 score over the other variants. However, the performance does not improve with the larger BERT model. Surprisingly, the corresponding test F1-score shows significant degradation implying substantial difference in the test data distribution (tokens, nature of problem or label) from that of the validation dataset. The small number of validation dataset also contributes to this mismatch. 

Table~\ref{tab:result2} captures the details of different experiments with generative models like, T5, Flan-T5 and GPT-4. The size of the generative model (small vs. base) has strong contribution to the performance as confirmed earlier~\cite{jullien-etal-2023-semeval}. However, the addition of synthetic data does not improve (rather degrade) the F1-score which is evident for both the small and base version of T5. This challenges the traditional belief of improvement due to multi-task learning and indicates potential conflicts in the synthetic data due to either a mismatch in the nature of the problem (e.g., Q\&A) or accuracy of the synthetic data (since they are not manually verified). The best result is obtained by a base Flan-T5 model trained without any synthetic dataset that results in a test F1-score of 0.61. Finally, using GPT-4 (version {\tt 0613}, maximum context length of 8192) without any Few-shot examples results in the best test F1-score of 0.76.

\section{Conclusion}
In this work we have explored both discriminative and generative models for NLI applied to CT reports. While our custom discriminative model outperforms generative models like T5-base and Flan-T5-base the same is not true when evaluated on the test dataset indicating the limitation of the small validation dataset and significant change in data distribution. Since the training dataset is small (1700) we also explore enriching the same with synthetic data created by LLMs like GPT-4 for additional task (e.g., Q\&A) and the same NLI task. However, the addition of these synthetic data substantially degrades the performance rather than improving pointing to a deeper analysis of the role of synthetic data for NLI task. The only exception is in the pretraining synthetic data created for small Flan-T5 model that boosted the final performance. The best result is obtained by GPT-4 without using Few-shot examples and we suspect both the addition of examples and modification of the prompt can further improve the performance.   

% To load the style file in the review version:
% \begin{quote}
% \begin{verbatim}
% \usepackage[review]{acl}
% \end{verbatim}
% \end{quote}
% For the final version, omit the \verb|review| option:
% \begin{quote}
% \begin{verbatim}
% \usepackage{acl}
% \end{verbatim}
% \end{quote}

% To use Times Roman, put the following in the preamble:
% \begin{quote}
% \begin{verbatim}
% \usepackage{times}
% \end{verbatim}
% \end{quote}
% (Alternatives like txfonts or newtx are also acceptable.)

% Please see the \LaTeX{} source of this document for comments on other packages that may be useful.

% Set the title and author using \verb|\title| and \verb|\author|. Within the author list, format multiple authors using \verb|\and| and \verb|\And| and \verb|\AND|; please see the \LaTeX{} source for examples.

% By default, the box containing the title and author names is set to the minimum of 5 cm. If you need more space, include the following in the preamble:
% \begin{quote}
% \begin{verbatim}
% \setlength\titlebox{<dim>}
% \end{verbatim}
% \end{quote}
% where \verb|<dim>| is replaced with a length. Do not set this length smaller than 5 cm.

% \section{Document Body}

% \subsection{Footnotes}

% Footnotes are inserted with the \verb|\footnote| command.\footnote{This is a footnote.}

% \subsection{Tables and figures}

% See Table~\ref{tab:accents} for an example of a table and its caption.
% \textbf{Do not override the default caption sizes.}

% \begin{table}
% \centering
% \begin{tabular}{lc}
% \hline
% \textbf{Command} & \textbf{Output}\\
% \hline
% \verb|{\"a}| & {\"a} \\
% \verb|{\^e}| & {\^e} \\
% \verb|{\`i}| & {\`i} \\ 
% \verb|{\.I}| & {\.I} \\ 
% \verb|{\o}| & {\o} \\
% \verb|{\'u}| & {\'u}  \\ 
% \verb|{\aa}| & {\aa}  \\\hline
% \end{tabular}
% \begin{tabular}{lc}
% \hline
% \textbf{Command} & \textbf{Output}\\
% \hline
% \verb|{\c c}| & {\c c} \\ 
% \verb|{\u g}| & {\u g} \\ 
% \verb|{\l}| & {\l} \\ 
% \verb|{\~n}| & {\~n} \\ 
% \verb|{\H o}| & {\H o} \\ 
% \verb|{\v r}| & {\v r} \\ 
% \verb|{\ss}| & {\ss} \\
% \hline
% \end{tabular}
% \caption{Example commands for accented characters, to be used in, \emph{e.g.}, Bib\TeX{} entries.}
% \label{tab:accents}
% \end{table}

% \subsection{Hyperlinks}

% Users of older versions of \LaTeX{} may encounter the following error during compilation: 
% \begin{quote}
% \tt\verb|\pdfendlink| ended up in different nesting level than \verb|\pdfstartlink|.
% \end{quote}
% This happens when pdf\LaTeX{} is used and a citation splits across a page boundary. The best way to fix this is to upgrade \LaTeX{} to 2018-12-01 or later.

% \subsection{Citations}

% \begin{table*}
% \centering
% \begin{tabular}{lll}
% \hline
% \textbf{Output} & \textbf{natbib command} & \textbf{ACL only command}\\
% \hline
% \citep{Gusfield:97} & \verb|\citep| &  \\
% \citealp{Gusfield:97} & \verb|\citealp| & \\
% \citet{Gusfield:97} & \verb|\citet| &  \\
%   \citeyearpar{Gusfield:97} & \verb|\citeyearpar| &  \\
%   \citeposs{Gusfield:97}	&	& \verb|\citeposs|\\
% \hline
% \end{tabular}
% \caption{\label{citation-guide}
% Citation commands supported by the style file.
% The style is based on the natbib package and supports all natbib citation commands.
% It also supports commands defined in previous ACL style files for compatibility.
% }
% \end{table*}

% Table~\ref{citation-guide} shows the syntax supported by the style files.
% We encourage you to use the natbib styles.
% You can use the command \verb|\citet| (cite in text) to get ``author (year)'' citations, like this citation to a paper by \citet{Gusfield:97}.
% You can use the command \verb|\citep| (cite in parentheses) to get ``(author, year)'' citations \citep{Gusfield:97}.
% You can use the command \verb|\citealp| (alternative cite without parentheses) to get ``author, year'' citations, which is useful for using citations within parentheses (e.g. \citealp{Gusfield:97}).

% A possessive citation can be made with the command \verb|\citeposs|.
% This is not a standard natbib command, so it is generally not compatible
% with other style files.

% \subsection{References}

% \nocite{Ando2005,andrew2007scalable,rasooli-tetrault-2015}

% The \LaTeX{} and Bib\TeX{} style files provided roughly follow the American Psychological Association format.
% If your own bib file is named \texttt{custom.bib}, then placing the following before any appendices in your \LaTeX{} file will generate the references section for you:
% \begin{quote}
% \begin{verbatim}
% \bibliography{custom}
% \end{verbatim}
% \end{quote}

% You can obtain the complete ACL Anthology as a Bib\TeX{} file from \url{https://aclweb.org/anthology/anthology.bib.gz}.
% To include both the Anthology and your own .bib file, use the following instead of the above.
% \begin{quote}
% \begin{verbatim}
% \bibliography{anthology,custom}
% \end{verbatim}
% \end{quote}

% Please see Section~\ref{sec:bibtex} for information on preparing Bib\TeX{} files.

%\subsection{Appendices}

% Use \verb|\appendix| before any appendix section to switch the section numbering over to letters. See Appendix~\ref{sec:appendix} for an example.

% \section{Bib\TeX{} Files}
% \label{sec:bibtex}

% Unicode cannot be used in Bib\TeX{} entries, and some ways of typing special characters can disrupt Bib\TeX's alphabetization. The recommended way of typing special characters is shown in Table~\ref{tab:accents}.

% Please ensure that Bib\TeX{} records contain DOIs or URLs when possible, and for all the ACL materials that you reference.
% Use the \verb|doi| field for DOIs and the \verb|url| field for URLs.
% If a Bib\TeX{} entry has a URL or DOI field, the paper title in the references section will appear as a hyperlink to the paper, using the hyperref \LaTeX{} package.

% \section*{Acknowledgements}
% This document has been adapted
% by Steven Bethard, Ryan Cotterell and Rui Yan
% from the instructions for earlier ACL and NAACL proceedings, including those for 

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

\appendix
\section{Graph Attention Network}
\label{appendix:gat}
The dependency tree can be represented by a graph structure where each node is a word and the edges between them are represented by the dependency relation, e.g., nominal subject, adverbial modifier, etc. Following \citet{wang-etal-2020-relational}, given a neighborhood of a node $\mathcal{N}_i$, the node embeddings can be iteratively updated using multi-head attention (with $K$ attentional heads) as 
\begin{equation}
    h^{l+1}_{{att}_i} = concat_{k=1}^K \sum_{j \in \mathcal{N}_i} \alpha_{ij}^{lk} W_k^l h_j^l, 
\end{equation}
\begin{equation}
    \alpha_{ij}^{lk} = attention(i, j), 
\end{equation}
where $h^{l+1}_{{att}_i}$ is the attention head of node-$i$ at layer $l+1$ and $\alpha_{ij}^{lk}$ is the normalized attention coefficient computed by the $k$-th attention at layer $l$ and $W^l_k$ is an input transformation matrix.

In addition to the attention head of word-$i$ a relational head is also computed for this node as 
\begin{equation}
    h^{l+1}_{{rel}_i} = concat_{m=1}^M \sum_{j \in \mathcal{N}_i} \beta_{ij}^{lm} W_m^l h_j^l, 
\end{equation}
\begin{equation}
    g_{ij}^{lm} = \sigma (relu \left( r_{ij}W_{m1} + b_{m1}\right)W_{m2} + b_{m2})
\end{equation}
\begin{equation}
    \beta_{ij}^{lm} = exp(g_{ij}^{lm})/\sum_{j \in \mathcal{N}_i}exp(g_{ij}^{lm})
\end{equation}
where $r_{ij}$ denotes the relation embedding between node-$i$ and $j$ and $M$ is the number of relational heads. The final representation of each word (node) is a concatenation of the attention and relational embeddings:
\begin{equation}
    x_i^{l+1} = concat(h^{l+1}_{{att}_i}, h^{l+1}_{{rel}_i})
\end{equation}
\begin{equation}
    h^{l+1}_i = relu \left( W_{l+1}x_i^{l+1} + b_{l+1} \right)
\end{equation}

\end{document}
