% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage[review]{ACL2023}
\usepackage{graphicx}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

\usepackage{pgfplots}
\pgfplotsset{compat=1.17}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{TüDuo at SemEval-2024 Task 2: Flan-T5 and Data Augmentation for Biomedical NLI}

% Author information can be set in various styles:
% For several authors from the same institution:
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

% \author{Veronika Smilga \\\And
%   Hazem Alabiad}

\begin{document}

\author{Veronika Smilga \and Hazem Alabiad \\
        University of Tübingen, Germany \\
        \texttt{first.last@student.uni-tuebingen.de}}

\nolinenumbers
{\makeatletter\acl@finalcopytrue
  \maketitle
}
\begin{abstract}
This paper explores using data augmentation with smaller language models under 3 billion parameters for the SemEval-2024 Task 2 on Biomedical Natural Language Inference for Clinical Trials. We fine-tune models from the Flan-T5 family with and without using augmented data automatically generated by GPT-3.5-Turbo and find that data augmentation through techniques like synonym replacement, syntactic changes, adding random facts, and meaning reversion improves model faithfulness (ability to change predictions for semantically different inputs) and consistency (ability to give same predictions for semantic preserving changes). However, data augmentation tends to decrease performance on the original dataset distribution, as measured by F1 score. Our best system is the Flan-T5 XL model fine-tuned on the original training data combined with over 6,000 augmented examples. The system ranks in the top 10 for all three metrics\footnote{Our code is available at \url{https://github.com/smilni/semeval2024_safe_biomedical_nli}}. 
\end{abstract}

\section{Introduction}

% These instructions are for authors submitting papers to ACL 2023 using \LaTeX. They are not self-contained. All authors must follow the general instructions for *ACL proceedings,\footnote{\url{http://acl-org.github.io/ACLPUB/formatting.html}} as well as guidelines set forth in the ACL 2023 call for papers.\footnote{\url{https://2023.aclweb.org/calls/main_conference/}} This document contains additional instructions for the \LaTeX{} style files.
% The templates include the \LaTeX{} source of this document (\texttt{acl2023.tex}),
% the \LaTeX{} style file used to format it (\texttt{acl2023.sty}),
% an ACL bibliography style (\texttt{acl\_natbib.bst}),
% an example bibliography (\texttt{custom.bib}),
% and the bibliography for the ACL Anthology (\texttt{anthology.bib}).
% I don't really like the n of cited models; check medpalm link

In the recent years, the rapid and triumphant advance of Large Language Models (LLMs) has affected virtually every area of NLP, biomedical NLP included. We aim to prove that Biomedical NLP can still benefit from smaller models of no more than three billion parameters. First, as the saying goes, "You must not use a steam hammer to crack a nut, if a nutcracker would do". In other words, while LLMs' performance is unmatched in complex applications, smaller models may be perfectly sufficient for simpler tasks, such as text classification or natural language inference. Second, being pre-trained on extremely large corpora of unlabelled data, modern LLMs have been shown to exhibit dataset-related bias~\citep{acerbi2023large}. In fields with a high error cost, pre-training and fine-tuning models on smaller, carefully curated, high-quality datasets is safer and more predictable than using black-box giant LLMs in a zero-shot or few-shot setting. Finally, as of now, best-performing state-of-the-art LLMs are either largely unavailable to the end-user due to computational constraints (for open-source models) or cost-inefficient (for proprietary models with access via API). 

The NLI4CT-2024 Shared Task~\citep{jullien-etal-2024-semeval} consists in building a system for natural language inference (NLI) based on a collection of breast cancer Clinical Trial Reports (CTRs) in English. The task's main challenge is the complex and heterogeneous nature of the data. For each datapoint, the premise comes from one of the four sections of a CTR -- Intervention, Eligibility, Results, or Adverse Events. Naturally, the sections are different from each other in terms of the mean length, the proportion of numerical data present, and the level of world knowledge required for drawing conclusions. Compared to the previous year's iteration of the task~\citep{jullien2023semeval}, this year's challenge calls for a system robust to alterations in the data. Apart from F1 measure, two new metrics are used to evaluate the model performance: \textbf{faithfulness}, "measuring the ability of a model to correctly change its predictions when exposed to a semantic-altering intervention", and \textbf{consistency}, "measuring the ability of a system to predict the same label for original statements and contrast statements for semantic preserving interventions". 

According to the last year participants' reports, various augmentation techniques have not led to significant performance improvement in terms of F1 and the top-3 best-performing systems did not use data augmentation at all~\citep{jullien2023semeval}. However, given the new metrics that are used in this year's evaluation, it seems reasonable to continue exploring the effect that various kinds of augmentation have on F1, faithfulness, and consistency at the same time. In this paper, we fine-tune models of Flan-T5 family with and without the use of augmented data automatically generated using GPT-3.5-Turbo. We find that using various kinds of additional data leads to an increase in model's faithfulness and consistency, but a decrease in F1. Our best system is Flan-T5 XL, fine-tuned on 1900 original train and development instances and 6650 automatically generated ones. The system ranks 7th for consistency, 9th for faithfulness, and 10th for F1.

\section{Related work}

\paragraph{Language Models and Biomedical NLP} There has been a surge of LLMs fine-tuned on biomedical data, from relatively small -- 7 billion parameter ChatDoctor~\citep{yunxiang2023chatdoctor}, MedAlpaca~\citep{han2023medalpaca}, PMC-LLAMA~\citep{wu2023pmc}; 6 billion parameter DoctorGLM~\citep{xiong2023doctorglm} and OphGLM~\citep{gao2023ophglm} -- to extremely large ones -- 540B Med-PaLM~\citep{singhal2023large}, 175B Codex-Med~\citep{lievin2022can}, 80B Med-Flamingo~\citep{moor2023med} -- which were reported to break state-of-the-art results on a number of biomedical NLP tasks. Moreover, without any fine-tuning on biomedical data, GPT-4 was reported to have passed every step of the US-medical licensing exam~\citep{nori2023capabilities}. However, researchers argue that smaller language models, such as T5 Base and T5 Large, still outperform gigantic all-purpose models when fine-tuned for a specific task~\citep{lehman2023we}.

\paragraph{Model Robustness in NLI tasks} Many NLI models suffer from bias related to superficial correlations between input text features and labels in the training dataset, which leads to a drop in performance on datasets where these correlations do not hold~\citep{rajaee2022looking}. Among those are hypothesis only bias, where models rely mostly on the hypothesis without taking premise and premise-hypothesis relations into account~\citep{poliak2018hypothesis}, and word-overlap bias, where models rely on the presence of shared words or phrases in premise and hypothesis~\citep{mccoy2019right}. Various techniques may be used to mitigate this bias, such as adversarial training~\citep{stacey2020avoiding} and data augmentation with predicate-argument structures~\citep{moosavi2020improving} and syntactic transformations~\citep{min2020syntactic}. 

\section{Experimental setup}

\begin{figure*}[htb!]
    \centering
    \begin{tikzpicture}
        \begin{axis}[
            ybar,
            bar width=0.36cm,
            enlarge x limits=0.34,
            legend style={at={(0.5,-0.15)},
              anchor=north,legend columns=-1},
            ylabel={Macro-F1 Score},
            symbolic x coords={Zero-shot, Two-shot, Fine-tuned},
            xtick=data,
            nodes near coords={\tiny\pgfmathprintnumber\pgfplotspointmeta},
            nodes near coords align={vertical},
            scaled y ticks=false,
            ]
            \addplot[fill=magenta!20]  coordinates {(Zero-shot,0.02) (Two-shot,0.296) (Fine-tuned,0.643)};
            \addplot[fill=blue!20]  coordinates {(Zero-shot,0.405) (Two-shot,0.415) (Fine-tuned,0.669)};
            \addplot[fill=cyan!20]  coordinates {(Zero-shot,0.519) (Two-shot,0.547) (Fine-tuned,0.727)};
            \addplot[fill=green!20]  coordinates {(Zero-shot,0.612) (Two-shot,0.637) (Fine-tuned,0.799)};
            \legend{Small,Base,Large,XL}
        \end{axis}
    \end{tikzpicture}
    \caption{Flan-T5 model family performance, calculated on development dataset}
    \label{fig:flan-t5-performance}
\end{figure*}

% \begin{table*}[htb!]
% \centering
% \begin{tabular}{p{0.42\linewidth}|p{0.17\linewidth}|p{0.17\linewidth}|p{0.17\linewidth}}
%                          & Precision & Recall   & F1 score  \\ \hline
% Flan-T5 Small zero-shot  & 0.010  & 1.000 & 0.019  \\ \hline
% Flan-T5 Small two-shot   & 0.571  & 0.200 & 0.296  \\ \hline
% Flan-T5 Small fine-tuned & 0.529  & 0.820 & \textbf{0.643}  \\  \hline \hline
% Flan-T5 Base zero-shot   & 0.625  & 0.300 & 0.405  \\ \hline
% Flan-T5 Base two-shot    & 0.593  & 0.320 & 0.415  \\ \hline
% Flan-T5 Base fine-tuned  & 0.548  & 0.860 & \textbf{0.669}  \\  \hline \hline
% Flan-T5 Large zero-shot  & 0.597  & 0.460 & 0.519  \\ \hline
% Flan-T5 Large two-shot   & 0.620  & 0.490 & 0.547  \\ \hline
% Flan-T5 Large fine-tuned & 0.641  & 0.840 & \textbf{0.727}  \\ \hline
% \hline
% Flan-T5 XL zero-shot  & 0.631  & 0.595 & 0.612  \\ \hline
% Flan-T5 XL two-shot   & 0.645  & 0.620 & 0.632  \\ \hline
% Flan-T5 XL fine-tuned & 0.754  & 0.850 & \textbf{0.799}  \\ 

% \end{tabular}
% \caption{Flan-T5 model family performance, calculated on development dataset}
% \label{tab:flan-t5}
% \end{table*}

In our experiments, we aim to test whether language models of relatively small size, under three billion parameters, can achieve decent performance on a task with a simple objective -- as the model chooses between only two options, entailment and contradiction -- and complex data -- as dealing with Clinical Test Reports requires complex reasoning and understanding of numerical data. As a starting point for the experiments, we have chosen Flan-T5.

\subsection{Selecting the model}
\label{sec:flan-t5-model}

Flan-T5~\citep{chung2022scaling} is an updated checkpoint of T5~\citep{raffel2020exploring}, instruction-fine-tuned on a number of new NLP tasks, which outperforms baseline T5 models of the corresponding size on a number of benchmarks. It also features improved instruction-following capabilities and generalizes well on new tasks, not present in the training data. On the previous year's iteration of NLI4CTR, the system that featured fine-tuned Flan-T5-XXL~\citep{kanakarajan2023saama} without any biomedical pre-training data augmentation showed an impressive performance, ranking second.

First, we evaluate the model's performance in three scenarios -- zero-shot, few-shot and after fine-tuning. Due to computational constraints we limit our experiments to language models of under three billion parameters, so we test only Flan-T5 Small (80M parameters), Flan-T5 Base (250M parameters), Flan-T5 Large (780M parameters), and Flan-T5 XL (3B parameters). 

When testing the models in a zero-shot setting, we and use one of the NLI prompt templates provided by Flan-T5 developers~\footnote{\protect\url{https://github.com/google-research/FLAN/blob/main/flan/v2/flan_templates_branched.py}}. In cases where two CTRs are given, we concatenate them using newline character as a separator. For a few-shot setting, we use the same prompt template as on the previous step, but enhance it with two hand-picked short CTR-hypothesis pairs from the training set -- one with entailment and the other with contradiction relation. Refer to Appendix~\ref{sec:appendix-1} for the prompts used for querying Flan-T5 in a zero-shot and two-shot setting.

Finally, we carry out fine-tuning with the use of HuggingFace Transformers library on the entire train set. The same set of hyperparameters is used for all models: \texttt{auto\_find\_batch\_size = True}, \texttt{learning\_rate = 1e-3}, \texttt{optimizer = adamw\_torch}. Flan-T5 Small and Flan-T5 Base are fine-tuned for 5 epochs in full precision. Due to computational constraints, Flan-T5 Large and Flan-T5 XL, however, were fine-tuned for 3 epochs using int8 precision and HuggingFace implementation of Low-Rank Adaptors (LoRA) algorithm~\citep{hu2021lora}.

\begin{table*}[!htb]
\centering
\begin{tabular}{p{0.25\linewidth}p{0.7\linewidth}}
Original hypothesis        & Heart-related adverse events were recorded in both the primary trial and the secondary trial. \texttt{[entailment]}                                                                                          \\
Synonym-based alteration & \textit{Cardiovascular} adverse events were \textit{documented} in both the primary \textit{study} and the secondary \textit{study}. \texttt{[entailment]}                                                                                           \\
Syntactic alteration     & Both the primary trial and the secondary trial recorded adverse events related to the heart. \texttt{[entailment]}                                                                                             \\
Random fact addition     & \textit{Lymphadenopathy is the enlargement of lymph nodes due to infection, inflammation, or cancer.} Heart-related adverse events were recorded in both the primary trial and the secondary trial. \texttt{[entailment]} \\
Semantic-altering change & Heart-related adverse events were \textit{not} recorded in either the primary trial or the secondary trial. \texttt{[contradiction]}                              
\end{tabular}
\caption{Examples for each kind of alterations}
\label{tab:alterations-ex}
\end{table*}

Figure~\ref{fig:flan-t5-performance} summarizes the results obtained after evaluating the model on the development set in three different settings: zero-shot, few-shot, and after fine-tuning. There is a clear correlation between the model's performance and its size and between the model's performance and the number of train examples provided to it as well. In all cases, providing two examples from the training data to the model in a few-shot setting improves the performance of the model slightly, while fine-tuning it on all given training data results in a substantial performance boost. The best-performing model so far is fine-tuned Flan-T5 XL.

\subsection{Data augmentation}
\label{sec:data-aug}

We assume that the training data should be augmented in two key ways to create a \textbf{faithful} and \textbf{consistent} system. First, we should add paraphrased versions of the original datapoints, with semantic meaning and label preserved. It will ensure that the system is \textbf{consistent}, i.e. produces the same output for semantically equivalent inputs. Second, we should include semantically altered versions of the original datapoints, with semantic meaning changed and a reverted label assigned. It will ensure that the system is \textbf{faithful}, i.e. change its output when encountering an input semantically different to the one seen before. The presence of these three types of datapoints -- original, paraphrased in a semantically preserving way, and paraphrased in a semantically altering way -- is expected to improve the model's performance. We assume that these examples will teach the model to consistently handle the semantics of the sentence, mitigating the impact of superficial features like word overlap between a premise and a hypothesis on the model's performance.

We apply four types of alterations to hypotheses:
\begin{enumerate}
\item Synonym-based semantic-preserving changes, where certain words within a sentence are substituted with their synonymous counterparts. 
\item Syntactic semantic-preserving changes, where the syntactic structure of the sentence is changed while the semantic meaning remains the same. 
\item Random fact addition semantic-preserving changes, where a true random fact is appended to the hypothesis without affecting its truth value. 
\item Semantic-altering changes, where a sentence contradictory to the original hypothesis is formulated. 
\end{enumerate}
Semantic-preserving changes 1), 2), and 3) are applied to all hypotheses, while semantic-altering change 4) is only applied to hypotheses that were initially labeled as entailment, changing the label to contradiction. The reason for this decision is that reverting a hypothesis that follows from some text produces a hypothesis that contradicts this text, but not vice versa. You may find examples for each kind of alterations in Table~\ref{tab:alterations-ex}.

We access GPT-3.5-Turbo via OpenAI API to generate new hypotheses for each CTR-hypothesis pair, using a distinct hand-crafted prompt for each kind of alteration. You may find the text of each prompt in Appendix~\ref{sec:appendix-2}. Four new hypotheses are generated for each "entailment" CTR-hypothesis pair, with both semantic-preserving and semantic-altering changes applied, and three new hypotheses are generated for each "contradiction" CTR-hypothesis pair, with only semantic-preserving changes applied. In all cases, CTR text itself remains unaltered, and only hypothesis is affected. 

As a result, we obtain 3400 new entries for 850 original train CTR-hypothesis pairs labelled as entailment and 2550 new entries for 850 original train CTR-hypothesis pairs labelled as contradiction. The process of generating 5950 data points, thus increasing our dataset by 4.5 times, cost \$0.86 and took 1.5 hours to complete.
\begin{table*}[htb!]
\centering
\begin{tabular}{p{0.4\linewidth}|p{0.15\linewidth}|p{0.15\linewidth}|p{0.15\linewidth}}
\multicolumn{1}{l|}{}                                     & F1              & Faithfulness    & Consistency     \\ \hline
Original train data only                         &  0.779    & 0.780    & 0.667          \\ \hline
Original train data + synonym-based alterations     & \textbf{0.780} & 0.715         & 0.681          \\ \hline
Original train data + syntax-based alterations      & 0.764           & 0.748          & 0.698    \\ \hline
Original train data + random facts addition & 0.748          & 0.736          & \textbf{0.725} \\ \hline
Original train data + reverted meaning alterations                    & 0.735          & \textbf{0.854} & 0.686          \\
\end{tabular}
\caption{Flan-T5 XL performance when trained on different kinds of augmented data, calculated on test dataset}
\label{tab:sep-alterations}
\end{table*}

\begin{table*}[htb!]
\centering
\begin{tabular}{p{0.4\linewidth}|p{0.15\linewidth}|p{0.15\linewidth}|p{0.15\linewidth}}
\multicolumn{1}{l|}{}                                     & F1              & Faithfulness    & Consistency     \\ \hline
Original train data only                         & \textbf{0.779}    & 0.780    & 0.667          \\ \hline
Original train data + all augmented train data   & 0.745 & \textbf{0.851}         & 0.748          \\ \hline
Original train and dev data + all augmented train and dev data      & 0.760           & 0.841          & \textbf{0.752}    \\ 
\end{tabular}
\caption{Flan-T5 XL performance when trained on all kinds of augmented data, calculated on test dataset}
\label{tab:all-alterations}
\end{table*}

\section{Results}

\subsection{Individual Augmentation Analysis}

We fine-tune Flan-T5 XL model on augmented data using the same set of hyperparameters as in Section~\ref{sec:flan-t5-model}. First, we fine-tune the model separately on each type of augmented data (combined with the original data) to estimate how augmentation of each kind affects the performance. The results are presented in Table~\ref{tab:sep-alterations}. 

Interestingly, only one kind of augmentation, the synonym-based one, had a positive effect on the model's performance on the original dataset, while the others led to a decrease in F1. All kinds of augmentations resulted in a model with higher consistency, i.e. a model better at producing the same output for hypotheses with the same meaning. The alteration that consisted in adding random true facts to hypotheses led to the highest increase in consistency. However, only semantic-altering change resulted in a more faithful model, i.e. a model better at changing its prediction when encountering a similar but semantically different hypothesis. All semantic-preserving changes led to a decrease in the model's faithfulness. Overall, our data augmentation techniques have proven to be efficient in improving the model's robustness. However, they have simultaneously resulted in a worse performance on the original data.

\subsection{Final Model Selection}

The next step was to try out different combinations of augmented data to reach the optimal performance in terms of the largest increase in both faithfulness and consistency and the smallest decrease in terms of F1. As the goal of the competition was to create a faithful and consistent system, we prioritized these metrics over F1 when choosing the model for the final submission. Thus, we chose the model trained on the entire set of augmented data that demonstrates higher faithfulness and consistency but lower F1. For the final submission, we additionally enriched the dataset with 200 more entries from development data and 700 new augmented entries created using techniques described in Section~\ref{sec:data-aug}. The results obtained after fine-tuning the model on the entire augmented dataset are presented in Table~\ref{tab:all-alterations}. 

\subsection{Other approaches}

Numerical inference is a known challenge for large language models. We assumed that the model's performance might vary across different CTR sections, with a decrease in performance for sections that contain most numbers. To check this assumption, we calculated the final model's F1 for each section separately. Calculations were performed on the development dataset as we had no access to test dataset labels during the development and evaluation stages. The results are presented in Table~\ref{tab:sep-f1}.

\begin{table}[!h]
\centering
\begin{tabular}{l|l}
               & F1      \\ \hline
Adverse Events & 0.711   \\
Eligibility    & 0.821   \\
Intervention   & 0.861   \\
Results        & 0.759   \\ \hline
All sections   & 0.783  
\end{tabular}
\caption{Final model's performance on each CTR section, calculated on development dataset}
\label{tab:sep-f1}
\end{table}

Adverse events, the section that, according to our observations, most often contained numbers in premise as well as hypothesis and required numerical inference to determine the relation between them, had the lowest F1 of all. 

We attempted to develop a separate model, Flan-T5 XL with the same hyperparameter set as in Section~\ref{sec:flan-t5-model}, to tackle CTR-hypothesis pairs of this kind. The model was first pre-fine-tuned on EQUATE dataset~\citep{ravichander2019equate} for 3 epochs in an attempt to enhance its numerical inference capabilities. Then it was further fine-tuned on the original and augmented CTR-hypothesis pairs of Adverse Events category for 3 epochs as well. We then used the original model to produce predictions for Eligibility, Intervention, and Results sections and the new model to produce predictions for Adverse Events section. However, on test data, this approach resulted in a decrease in performance with an F1 of 0.756 (-0.004), faithfulness of 0.781 (-0.06) and consistency of 0.722 (-0.031). We suppose that the decrease in performance is explained by the fact that the second model, trained on \textasciitilde{}1/4 of all data (only one section out of four), simply did not encounter enough data to develop robustness comparable to that of the final model trained on the entire dataset.

\section{Conclusion}

 In this paper, we explore the impact of data augmentation on model performance and robustness. Specifically, we focus on leveraging advanced language models like GPT-3.5-Turbo to expand the training set for fine-tuning smaller models such as Flan-T5 XL. Our experiments involve various prompts to generate new CTR-hypothesis pairs. Enriching the training set with new examples that underwent semantic-preserving changes, such as synonym replacement, change in word or clause order, and random true fact addition, improves the model's consistency. Adding augmented examples that underwent semantic-altering changes, such as meaning reversion, improves the model's faithfulness as well as consistency. However, all kinds of augmentation except for synonym replacement lead to a decrease in model performance in terms of F1 on the original unaltered dataset. The model selected for the final submission is Flan-T5 XL fine-tuned on augmented development and training set. It features higher robustness but lower base performance than Flan-T5 XL fine-tuned on original data only, with faithfulness of 0.841 (+0.061), consistency of 0.752 (+0.085), and F1 of 0.76 (-0.019). 

% As the next steps, we are planning to 1) conduct hyperparameter tuning to select the parameters that yield the best result, including the number of epochs (possibly, with the use of the early stopping technique to ensure the best performance while also avoiding overfitting); 2) explore various data augmentation techniques and the influence they have on the models' performance; 3) try to equip the models with a better understanding of the biological data, either by fine-tuning on a suitable biological corpora, e.g. MedNLI~\citep{romanov2018lessons}, or by providing relevant pieces of information during inference via queries to a biomedical database; 4) if time and computational constraints permit, fine-tune and evaluate other models to compare their performance with Flan-T5. Of particular interest to us are Mistral-7B~\citep{jiang2023mistral}, a large language model which demonstrates state-of-the-art performance on a number of NLP tasks; GatorTron~\citep{yang2022gatortron}, a large language model trained from scratch on biomedical data and demonstrating impressive (over 90\% accuracy) performance on MedNLI; and SciFive~\citep{phan2021scifive}, a T5-based model fine-tuned on biomedical data and NLI tasks and demonstrating 87\% accuracy on MedNLI.

% \subsection{Hyperlinks}

% Users of older versions of \LaTeX{} may encounter the following error during compilation: 
% \begin{quote}
% \tt\verb|\pdfendlink| ended up in different nesting level than \verb|\pdfstartlink|.
% \end{quote}
% This happens when pdf\LaTeX{} is used and a citation splits across a page boundary. The best way to fix this is to upgrade \LaTeX{} to 2018-12-01 or later.

% \subsection{Citations}



% Table~\ref{citation-guide} shows the syntax supported by the style files.
% We encourage you to use the natbib styles.
% You can use the command \verb|\citet| (cite in text) to get ``author (year)'' citations, like this citation to a paper by \citet{Gusfield:97}.
% You can use the command \verb|\citep| (cite in parentheses) to get ``(author, year)'' citations~\citep{Gusfield:97}.
% You can use the command \verb|\citealp| (alternative cite without parentheses) to get ``author, year'' citations, which is useful for using citations within parentheses (e.g. \citealp{Gusfield:97}).


\nocite{Ando2005,augenstein-etal-2016-stance,andrew2007scalable,rasooli-tetrault-2015,goodman-etal-2016-noise,harper-2014-learning}

% The \LaTeX{} and Bib\TeX{} style files provided roughly follow the American Psychological Association format.
% If your own bib file is named \texttt{custom.bib}, then placing the following before any appendices in your \LaTeX{} file will generate the references section for you:
% \begin{quote}
% \begin{verbatim}
% \bibliographystyle{acl_natbib}
% \bibliography{custom}
% \end{verbatim}
% \end{quote}
% You can obtain the complete ACL Anthology as a Bib\TeX{} file from \url{https://aclweb.org/anthology/anthology.bib.gz}.
% To include both the Anthology and your own .bib file, use the following instead of the above.
% \begin{quote}
% \begin{verbatim}
% \bibliographystyle{acl_natbib}
% \bibliography{anthology,custom}
% \end{verbatim}
% \end{quote}
% Please see Section~\ref{sec:bibtex} for information on preparing Bib\TeX{} files.

% \subsection{Appendices}

% Use \verb|\appendix| before any appendix section to switch the section numbering over to letters. See Appendix~\ref{sec:appendix} for an example.

% \section{Bib\TeX{} Files}
% \label{sec:bibtex}

% Unicode cannot be used in Bib\TeX{} entries, and some ways of typing special characters can disrupt Bib\TeX's alphabetization. The recommended way of typing special characters is shown in Table~\ref{tab:accents}.

% Please ensure that Bib\TeX{} records contain DOIs or URLs when possible, and for all the ACL materials that you reference.
% Use the \verb|doi| field for DOIs and the \verb|url| field for URLs.
% If a Bib\TeX{} entry has a URL or DOI field, the paper title in the references section will appear as a hyperlink to the paper, using the hyperref \LaTeX{} package.

% \section*{Limitations}
% ACL 2023 requires all submissions to have a section titled ``Limitations'', for discussing the limitations of the paper as a complement to the discussion of strengths in the main text. This section should occur after the conclusion, but before the references. It will not count towards the page limit.
% The discussion of limitations is mandatory. Papers without a limitation section will be desk-rejected without review.

% While we are open to different types of limitations, just mentioning that a set of results have been shown for English only probably does not reflect what we expect. 
% Mentioning that the method works mostly for languages with limited morphology, like English, is a much better alternative.
% In addition, limitations such as low scalability to long text, the requirement of large GPU resources, or other things that inspire crucial further investigation are welcome.

% \section*{Ethics Statement}
% Scientific work published at ACL 2023 must comply with the ACL Ethics Policy.\footnote{\url{https://www.aclweb.org/portal/content/acl-code-ethics}} We encourage all authors to include an explicit ethics statement on the broader impact of the work, or other ethical considerations after the conclusion but before the references. The ethics statement will not count toward the page limit (8 pages for long, 4 pages for short papers).

\section*{Acknowledgements}
We would like to thank Dr. Çağrı Çöltekin for guiding, supporting and inspiring us on our way.
% This document has been adapted by Jordan Boyd-Graber, Naoaki Okazaki, Anna Rogers from the style files used for earlier ACL, EMNLP and NAACL proceedings, including those for
% EACL 2023 by Isabelle Augenstein and Andreas Vlachos,
% EMNLP 2022 by Yue Zhang, Ryan Cotterell and Lea Frermann,
% ACL 2020 by Steven Bethard, Ryan Cotterell and Rui Yan,
% ACL 2019 by Douwe Kiela and Ivan Vuli\'{c},
% NAACL 2019 by Stephanie Lukin and Alla Roskovskaya, 
% ACL 2018 by Shay Cohen, Kevin Gimpel, and Wei Lu, 
% NAACL 2018 by Margaret Mitchell and Stephanie Lukin,
% Bib\TeX{} suggestions for (NA)ACL 2017/2018 from Jason Eisner,
% ACL 2017 by Dan Gildea and Min-Yen Kan, NAACL 2017 by Margaret Mitchell, 
% ACL 2012 by Maggie Li and Michael White, 
% ACL 2010 by Jing-Shin Chang and Philipp Koehn, 
% ACL 2008 by Johanna D. Moore, Simone Teufel, James Allan, and Sadaoki Furui, 
% ACL 2005 by Hwee Tou Ng and Kemal Oflazer, 
% ACL 2002 by Eugene Charniak and Dekang Lin, 
% and earlier ACL and EACL formats written by several people, including
% John Chen, Henry S. Thompson and Donald Walker.
% Additional elements were taken from the formatting instructions of the \emph{International Joint Conference on Artificial Intelligence} and the \emph{Conference on Computer Vision and Pattern Recognition}.

% Entries for the entire Anthology, followed by custom entries
\bibliography{anthology,custom}
\bibliographystyle{acl_natbib}

\clearpage
\appendix

\section{Prompts used in querying Flan-T5}
\label{sec:appendix-1}
\begin{figure}[!htb]
  \centering
  \includegraphics[width=\linewidth]{img/prompt_zero_shot.png}
  \caption{Zero-shot prompt used in querying Flan-T5.}
  \label{fig:prompt-zero-shot}
\end{figure}

\begin{figure}[!htb]
  \centering
  \includegraphics[width=\linewidth]{img/prompt_2_shot.png}
  \caption{Two-shot prompt used in querying Flan-T5.}
  \label{fig:prompt-few-shot}
\end{figure}

\vfill\eject

\section{Prompts used to obtain augmented data from GPT-3.5-Turbo}
\label{sec:appendix-2}

\begin{figure}[!htb]
  \centering
  \includegraphics[width=\linewidth]{img/synonym.drawio.png}
  \caption{Prompt used to generate a synonym-based paraphrased version of hypothesis.}
\end{figure}

\begin{figure}[!htb]
  \centering
  \includegraphics[width=\linewidth]{img/syntax.drawio.png}
  \caption{Prompt used to generate a syntax-based paraphrased version of hypothesis.}
\end{figure}

\begin{figure}[!htb]
  \centering
  \includegraphics[width=\linewidth]{img/random.drawio.png}
  \caption{Prompt used to generate a random biomedical fact to then append to hypothesis.}
\end{figure}

\begin{figure}[!htb]
  \centering
  \includegraphics[width=\linewidth]{img/revert.drawio.png}
  \caption{Prompt used to generate sentence with meaning contradicting that of hypothesis.}
\end{figure}

\end{document}
