% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{adjustbox}
\usepackage{graphicx}
\usepackage{tikz}
\usepackage{listings}
\usepackage{xcolor}
\usepackage{float}
\usepackage{multirow}
\lstset{language=Python, 
        morekeywords={gpt_model, prompt, input_raw_text},   % Adding gpt_model as a keyword
        keywordstyle=\color{orange}, % Coloring keywords (including gpt_model) in orange
        basicstyle=\ttfamily,
        captionpos=b,
        breaklines=true,
        showstringspaces=false}
\lstset{escapeinside={<@}{@>}}

\usepackage{booktabs}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{PreClinIE: An Annotated Corpus for Information Extraction in Preclinical Studies}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Simona E. Doneva$^{1*}$, Hanna Hubarava$^{1}$, Pia Haervelid$^{1}$, \\ {\bf Wolfgang E. Zürrer
 $^{1}$},  {\bf Julia Bugajska$^{1}$}, {\bf Bernhard Hild$^{1}$}, {\bf David Brüschweiler$^{1}$}, \\ {\bf Tilia R. Ellendorff$^{1}$}, {\bf Gerold Schneider$^{1}$}, {\bf Benjamin V. Ineichen$^{1,2}$}\\
  $^1$ University of Zurich, Zurich, Switzerland \\
  $^2$ University of Bern, Bern, Switzerland \\
  \texttt{*simona.doneva@uzh.ch}
}


\begin{document}
\maketitle
\begin{abstract}
%Why did you do it?
Animal research, sometimes referred to as preclinical research, plays a vital role in bridging the gap between basic science and clinical applications. However, the rapid increase in publications and the complexity of reported findings make it increasingly difficult for researchers to extract and assess relevant information. While automation through natural language processing (NLP) holds great potential for addressing this challenge, progress is hindered by the absence of high-quality, comprehensive annotated resources specific to preclinical studies. 
%What did you do?
To fill this gap, we introduce PreClinIE, a fully open manually annotated dataset. The corpus consists of abstracts and methods sections from 725 publications, annotated for study rigor indicators (e.g., random allocation) and other study characteristics (e.g., species).
%What did you find?
We describe the data collection and annotation process, outlining the challenges of working with preclinical literature. 
%Why are these findings important?
By providing this resource, we aim to accelerate the development of NLP tools that enhance literature mining in preclinical research. % more easily, efficiently, and comprehensively.
\end{abstract}

\section{Introduction}
Developing new therapies from animal models to human treatments, known as bench-to-bedside translation, has a low success rate: Only 1 in 20 therapies advances to human use \cite{ineichen2024analysis}. This contrasts with the extensive use of animals in research, estimated at over 50 million per year globally \cite{taylor2019estimate}. The factors that determine successful translation remain poorly understood \cite{seyhan2019lost}.

A systematic assessment of information on experimental design, model and drug selection, as well as animal usage can provide insights into how animal research informs human health. The full-text, and especially the methods sections of scientific articles contain concrete, verifiable details about these aspects, which are often omitted or misrepresented in article abstracts \cite{li2017scoping}. These factual descriptions form the foundation of a study and are critical for evaluating its design, rigor, and to enable reproducibility \cite{menke2020rigor}.

However, the volume of preclinical animal studies is overwhelming, with hundreds of thousands published annually \cite{ineichen2023data}. While large-scale analysis methods exist, they primarily focus on human data or only on abstract level data \cite{chapman2011overcoming, doneva2024discoverartificial}. Animal studies, with their highly heterogeneous experimental approaches and less standardized reporting, remain largely unaddressed. 

There is a critical need for computational methods to extract and integrate these data at scale, since a more detailed understanding of the drug development process could not only improve experimental animal welfare but also enhance the efficiency of human therapies. As a first step towards that goal, our study aims to create a large, manually annotated corpus of animal study publications, including abstracts and method sections.  We share all resources on GitHub\footnote{\url{https://github.com/Ineichen-Group/Preclinical_IE_Dataset}}.

%Here we bridge this gap by introducing a new gold standard annotated dataset for clinical trial registry data in the domain of neurology/psychiatry. The corpus comprises 1093 clinical trial summaries from ClinicalTrials.gov, one of the largest international clinical trial registries \citep{zarin2019ten}. It has been annotated by two to three annotators for key trial characteristics, i.e., condition (e.g., Alzheimer's disease), therapeutic intervention (e.g., aspirin), and control arms (e.g., placebo). 

 %After publication, it will be linked to Zenodo\footnote{\url{https://zenodo.org/}} and shared via the HuggingFace Dataset API, in compliance with the FAIR principles \citep{wilkinson2016fair}.

\section{Related Work}

NLP methods have been commonly applied in the preclinical domain for abstract classification tasks. For example, a recently published dataset aims to help with the identification of animal studies and alternative experimental models \citep{neves2023automatic}. Another application is the automated selection of relevant published articles for specific literature review questions, as well as the assessment of risk of bias items (e.g., random allocation) \citep{bannach2019machine, wang2022risk}.

Information extraction from preclinical literature is an emerging, but less developed, area of research. \textit{STEED}, for instance, is an R-based text mining tool that uses regular expressions to automatically extract key experimental details, such as animal species, disease models, and randomization from neuroscience in vivo studies. It has been developed on 45 full-text articles and validated on 275 articles \citep{zurrer2024steed}.  Another approach, \textit{Menagerie}, combines rule-based, dictionary-based, and machine learning techniques to extract six predefined animal study characteristics \citep{zeiss2019menagerie}. This work is based on a manually curated dataset of 504 PubMed abstracts, annotated with classes such as species or animal model at the abstract level, and with gene names at the token level. Another related work targets information extraction based on the established framework of Population/Problem, Intervention, Comparator and Outcome (PICO) \citep{wang2022pico}. For this, 400 abstracts of preclinical studies have been annotated for each PICO-related mention, and the task was solved as sentence classification, followed by entity recognition. Another study proposed combining a regex-based method with a generative LLM to extract interventions from preclinical animal studies on Alzheimer’s disease \citep{pu2024intervention}.

Despite recent advances, existing corpora remain limited in scope - typically focusing on narrow disease domains, containing small datasets (around 500 documents), and offering only abstract-level annotations. For example, \textit{Menagerie} was validated solely for Parkinson’s disease. In contrast, our corpus is, to our knowledge, the most comprehensive resource of its kind: it includes 725 documents from the general neuroscience domain, with manual annotations on both the abstract and the methods section, a critical source of experimental detail. This results in 1,450 annotated sections. Importantly, we used three annotation levels (document, sentence, and token), aiming to match the typical granularity of information relevant to researchers. This structure also should reflect the nature of the content: some elements, like conclusions, require sentence- or document-level annotation, whereas others, like individual drugs, can be annotated at the token level. 

\begin{table*}[h]
    \centering
    \small
    \resizebox{\linewidth}{!}{
    \begin{tabular}{p{6cm} p{6cm} p{3cm}} 
        \toprule
        \textbf{Parameter} & \textbf{Label (frequency)} & \textbf{Krippendorff's Alpha (95\% CI)} \\
        \midrule
        \multicolumn{3}{l}{\textbf{Document-level Annotation}} \\
        \midrule
        Animal species (A, M) & 
        Rat (806), Mouse (531), Other (28), Rabbit (28), Monkey (20), Dog (15), Pig (10), Cat (6), Guinea Pig (6) & 
        0.97 (0.95, 1.00) \\
        Control (A, M) & 
        Control-present (1135) & 
        0.51 (0.29, 0.67) \\
        Readout (A, M) & 
        Physiology (400), Behaviour (938), Histology (921), Other (896), Imaging (92) & 
        0.48 (0.39, 0.55) \\
        Animal sex (A, M) & 
        Not reported (717), Male (524), Female (129), Both sexes (63) & 
        1.00 (1.00, 1.00) \\
        \midrule
        \multicolumn{3}{l}{\textbf{Sentence-Level Annotation (Highlight)}} \\
        \midrule
        Study conclusions (A) & 
        Positive (645), Neutral (22), Negative (18), Mixed (15) & 
        0.76 (0.74, 0.78) \\
        Animal disease model (A) & 
        Model (649) & 
        0.62 (0.60, 0.64) \\
        Weight (M) & 
        Weight (514) & 
        0.73 (0.71, 0.75) \\
        Age (M) & 
        Age (476) & 
        0.75 (0.73, 0.78) \\
        Random allocation (A, M) & 
        Randomization (464) & 
        0.60 (0.56, 0.64) \\
        Blinded outcome assessment (A, M) & 
        Blinding (389) & 
        0.97 (0.95, 0.98) \\
        Animal welfare statement (A, M) & 
        Welfare (700) & 
        0.96 (0.95, 0.97) \\
        Animal Research: Reporting of In Vivo Experiments Guidelines (A, M) & 
        ARRIVE (15) & 
        --- \\
        %PREPARE guidelines (A, M) & 
        %PREPARE (0) & 
        %--- \\
        Sample size calculation (A, M) & 
        Power (22) & 
        --- \\
        \bottomrule
    \end{tabular}
   }
    
    \caption{Overview of document-level and sentence-level annotation categories. The ``Label (frequency)'' column lists the available labels for each category along with their frequency in the final complete annotated dataset. For document-level annotations, the frequency represents the number of documents (abstracts or methods) assigned to each label. For sentence-level annotations, it indicates the number of unique sentences associated with each label. The last column provides the Krippendorff's Alpha inter-annotator-agreement score for that label on the subset of the corpus annotated by all annotators (15 articles). The rows with a missing score correspond to the labels not sufficiently represented in the subset. Abbreviations: A, abstract; M, methods.}
    \label{tab:document_sentence_annotation}
\end{table*}

\begin{table*}[h]
    \centering
    \small
    \resizebox{\linewidth}{!}{
    \begin{tabular}{lccccc}
        \toprule
        \textbf{Entity Type} & \textbf{Entity \#} & \textbf{Unique \#} & \textbf{Avg Char Count} & \textbf{Examples} & \textbf{Krippendorff's Alpha (95\% CI)}\\
        \midrule
        Therapy-drug (A, M) & 10348 & 2437 & 17.7 ± 18.1 & beta-lactam antibiotic, ZM241385 & 0.73 (0.70, 0.75)\\
        Therapy-other (A, M) & 5216 & 1728 & 20.3 ± 15.8 & auditory habilitation, treadmill training & 0.59 (0.57, 0.61)\\
        Disease (A) & 3790 & 958 & 19.8 ± 11.0 & minimal seizures, chronic paraplegia & 0.79 (0.76, 0.81)\\
        Strain (A, M) & 1196 & 159 & 10.5 ± 18.7 & Sprague Dawley, Fisher 344 & 0.84 (0.79, 0.88)\\
        Animals-number (A, M) & 342 & 144 & 5.5 ± 4.8 & Eighty-five, 128 & 0.78 (0.50, 0.93)\\
        \bottomrule
    \end{tabular}}
    \caption{Overview of token-level annotations with total entity counts, unique instances counts, average character number, and annotation examples. The last column provides the Krippendorff's Alpha inter-annotator-agreement score as measured on the subset of the corpus annotated by all annotators (15 articles). Abbreviations: A, abstract; M, methods.}
    \label{tab:entity_stats_ner}
\end{table*}

\section{The Corpus}
\subsection{Data Collection}
\label{aact_trials_selection_sampling}

A search string for PubMed and EMBASE was designed to identify animal studies on therapeutic interventions\footnote{Search date: from database inception to October 09, 2023. Full search string available here: \href{https://github.com/Ineichen-Group/Preclinical_IE_Dataset/tree/main/data/search_strings}{dataset search strings}.}. From the retrieved references, 4,000 records were randomly selected for screening by two independent reviewers based on inclusion criteria: primary studies involving drug testing in animals. 

We used the automatic fetch function of the reference management tool EndNote to retrieve PDFs, resorting to manual retrieval when necessary. We used IBM Deepsearch to convert PDFs into text\footnote{\href{https://www.ibm.com/docs/en/rpa/23.0?topic=SSTHBP_23.0.x/using_ibm_rpa/ocr/using/using_pdf_text_extractor.htm}{IBM RPA PDF Extractor}}, followed by a regular expression-based algorithm to classify paper sections such as methods and results. We included the methods sections because they typically provide more detailed descriptions of the employed methodology compared to abstracts. 

\subsection{Data Annotation}
\label{data_annotation_subsection}
\subsubsection{Annotation Guidelines}
\label{annot_guidelines_subsubsection}
We define three levels of annotation. At the \textbf{document} level, one or more labels are assigned to the entire document. At the \textbf{sentence} level, we highlight the sentence where the relevant information appears (\textbf{Table \ref{tab:document_sentence_annotation}}). Finally, at the \textbf{token} level, individual words are annotated as named entities (\textbf{Table \ref{tab:entity_stats_ner}}).  We refined the annotation guidelines iteratively to ensure maximum clarity and optimize inter-rater agreement. The final guidelines can be accessed at \href{https://github.com/Ineichen-Group/Preclinical_IE_Dataset/blob/main/Annotation_guidelines_V5.pdf}{Annotation Guidelines (v5)}, with a shortened version in \textbf{Appendix \ref{sec:appendixF}}. Notably, spans and documents can have more than one label. For example, weight and age of animals are often reported in the same sentence, and a study can involve both mice and rats in its experiments.


 \subsubsection{Annotation Process}
\label{annot_process_subsubsection}
From the 4000 random references, we excluded two due to missing metadata, leaving 3,998 references. Of these, 1,018 met the inclusion criteria during the initial screening.

The annotation was conducted by five senior medical students, starting with two pilot rounds of 20 and then 50 articles annotated by all annotators to familiarize themselves with the task and to refine the guidelines. In the final annotation round, 817 articles were distributed among them, with each annotator receiving 179–181 articles with title, abstract and method sections. Of these, 20 articles were assigned to multiple annotators to calculate inter-annotator agreement (IAA). The annotators were allowed to exclude articles from annotation if they did not fit the inclusion criteria\footnote{For example, some initially included papers were related to a diagnostic procedure rather than an intervention.}. 

The annotators used a custom recipe developed for the browser-based tool Prodigy to perform the manual annotation \cite{prodigy_montani_honnibal}. An annotation task example is shown in \textbf{Supplementary Figure \ref{fig:prodigy}}.  

To compile the final dataset and perform an error analysis, the 20 multiple-assigned articles were reviewed, with conflicts adjudicated through discussion. The final dataset consists of 725 unique articles, and corresponds to 1450 abstract and method sections.   %Further details about the resulting corpus can be found in section \textbf{\ref{subsection_corpus_overview}}.

\subsubsection{Inter-Annotator Agreement}
\label{IAA}
Five of the 20 common documents were excluded by one or more annotators as not meeting the inclusion criteria. To ensure the IAA score is measured among all annotators, we removed the excluded articles from the agreement calculations. This left us with 15 unique articles, each with an abstract and method section (30 annotated documents).

We report the IAA among the five annotators using Cohen's Kappa for pairwise agreement calculation and Krippendorff's Alpha for the calculation of agreement among all annotators \cite{cohen1960coefficient, hughes2021krippendorffsalpha}. 

\subsection{Results}
%The Krippendorff's Alpha IAA scores are presented in the last column of \textbf{Table \ref{tab:annotation}} and \textbf{Table \ref{tab:entity_stats_ner}}, and visualized in \textbf{Supplementary Figure \ref{fig:combined_alpha}}. The pair-wise IAA using Cohen's Kappa is presented in \textbf{Supplementary Figure \ref{fig:all_categories_kappa}, \ref{fig:kappa_and_top_15_ner_entities}}.

\subsubsection{Corpus Overview}
\label{subsection_corpus_overview}
Our final annotated corpus consists of the abstracts and methods sections from 725 published neuroscience articles, primarily dated between 2010 and 2020. The most frequently represented journals include \textit{European Journal of Pharmacology} and \textit{PLoS ONE} (\textbf{Supplementary Figure \ref{fig:metadata}}).

Based on the \textbf{document level} annotations, the corpus predominantly comprises studies involving mice and rats (\textbf{Table \ref{tab:document_sentence_annotation}}). Additionally, there is a marked bias toward male animals and reporting of animal sex information in the methods section. Furthermore, there were often multiple selected options for the experimental readouts (\textbf{Supplementary Figure \ref{top_10_categorical_entities}}).

At the \textbf{sentence level}, the annotations reveal that the majority of conclusion statements within the corpus present positive findings. However, adherence to reporting best-practice appears limited, with relatively little to no mention of ARRIVE and PREPARE guidelines \cite{percie2020arrive, smith2018prepare}. Furthermore, explicit reporting of sample size calculations is sparse (\textbf{Table \ref{tab:document_sentence_annotation}}).

At the \textbf{token level}, therapy-related annotations are the most prevalent, as these terms were annotated in both the abstract and methods sections (\textbf{Table \ref{tab:entity_stats_ner}}). Disease and strain entities exhibit high lexical variability, with high ratio of unique textual representations across the corpus. Additionally, many abbreviations are annotated, such as \textit{AD} for Alzheimer's disease (\textbf{Supplementary Figure \ref{fig:kappa_and_top_15_ner_entities}}).

\subsubsection{Analysis of Annotation Disagreements \label{sec:analysis_disagreements}}
We observed several patterns of discrepancies in the multiply-annotated documents selected for the calculation of IAA:

\begin{itemize}
    \item \textbf{Text level annotations:} ``Readout'' and ``Control'' were the most challenging document-level classification tasks (\textbf{Table \ref{tab:document_sentence_annotation}}). The language describing the readouts varied greatly across papers and was often not explicit. Some annotators selected ``other'' in these cases, while others attempted to infer a more specific readout type. Furthermore, some annotators selected ``histology'' in cases when there was clearly no mention of this readout, suggesting a misunderstanding of the concept. Similarly, the presence of control intervention was rarely specified, even though the text sometimes contained a comparison verb (e.g., ``improved''). In such instances, where the presence of control is implicit, some annotators marked the presence of control, while others did not, leading to a lower agreement score. The variability in pair-wise IAA is evident from \textbf{Supplementary Figure \ref{fig:all_categories_kappa}}.

    \item \textbf{Sentence level annotations:} At the sentence level, annotators often agreed on the study’s overall conclusion but struggled to identify the exact concluding sentence, sometimes confusing it with a summary of findings. Variation in punctuation usage—especially around colons and semicolons—also caused inconsistencies in the selection of annotation spans, resulting in partial agreement. Annotation of ``randomization'' was meant to refer only to the allocation of animals into experimental groups, but one annotator highlighted other contexts as well. 
    % TODO: idea to actually consider the conclusion as abstract classification taks instead of sentence-level, how good is the agreement?
    
    \item \textbf{Token level annotations:} At the token level all entities except ``therapy-other'' achieved a satisfactory level of agreement (\textbf{Table \ref{tab:entity_stats_ner}}). We identified three main discrepancies. First, some annotators occasionally missed entities, either due to human error in reading longer texts or a misunderstanding of the guidelines. For instance gene mentions were sometimes annotated as therapy (Nurr1, Fox2) when, according to the guidelines, they should not be annotated as such. Second, label disagreements arose when different annotators assigned different labels to the same entity. For instance, one annotator consistently labeled antibody therapies as ``therapy-other'', while it should have been ``therapy-drug''. Finally, span disagreements occurred when some annotators included a preceding modifier as per the guidelines instructions (e.g., “morphine-induced”), while another did not. Such discrepancies introduce noise in the dataset and reduce the agreement score. The variability in pair-wise IAA is shown in \textbf{Supplementary Figure \ref{fig:kappa_and_top_15_ner_entities}}.

\end{itemize}

\section{Discussion and Conclusion}
% remind of the question/ aim of the study stated in the introduction
We introduced PreClinIE, an openly available corpus for extracting study rigor indicators and experimental details from published articles describing animal research. 

% discuss annotation process challenges and IAA findings -> acknolwedge difficulty for some parameters
Our annotation process uncovered key challenges. Particularly there was a low agreement in control/comparator annotations, aligning with findings from the related PICO study \citep{wang2022pico}. At the same time, this study had stronger performance for readout extraction, suggesting that token-level annotation may be more suitable for this task. Additionally, the high disagreement in identifying the exact conclusion sentence suggests that study conclusions might be best evaluated using the full document rather than isolated sentences. Furthermore, we observed that crucial study details often appear exclusively in the methods section, emphasizing the importance of section-aware extraction.

In designing the annotation scheme, we made several pragmatic choices to balance granularity, feasibility, and consistency. For example, while our approach captures individual parameters such as sex, strain, and treatment, it does not explicitly encode relationships between these entities. As a result, reconstructing complex experimental groupings may be challenging in studies involving multiple animal subgroups. Nevertheless, this design simplifies annotation and aligns with our primary goal of extracting key methodological features at scale. Future work could explore incorporating relational annotations to capture richer experimental structures. Additionally, it may be possible to use simpler heuristic rules, for instance, pairing species and model terms that occur within a pre-specified window in the text, to make those links more explicit \cite{zeiss2019menagerie}.

Similarly, we chose to restrict some annotations to single sentences. This constraint reduces cognitive load for annotators. Although it may result in missed information that spans multiple sentences, such as animal welfare statements or methodological clarifications, we find that capturing key information once in the text is sufficient for many downstream applications. Future extensions of the annotation scheme could explore cross-sentence linking or section-level annotation to support more nuanced analysis.

% broader conclusions -> implications for future studies + applications
Beyond annotation challenges, our findings highlight a male bias in animal use, a majority of positive conclusion statements indicative of reporting bias, and insufficient reporting of sample size calculations. 
% final conclusion -> synthesizes what has come before and proposes directions for further studies

These patterns warrant further evaluation, as they suggest systemic issues in study design and reporting that could impact the reliability and reproducibility of preclinical research findings \cite{beery2011sex, button2013power}. As future work, we plan to provide a baseline experiment to illustrate how the dataset can support computational information extraction from preclinical literature. The corpus enables a range of NLP tasks, such as named entity recognition and sentence classification, and can serve as a benchmark for model development in this domain. We hope these efforts will inspire further research in NLP models development and evaluation, ultimately contributing to more transparent and reliable scientific practices.

\section*{Limitations}

\textbf{Data Scope.} 
Our developed dataset includes publications focusing mainly on research in neuroscience. This may influence the generalizability of our findings to other areas of research.

\textbf{Annotation Setup.}
Only a small portion of the dataset was multiply-annotated. We conducted two annotation pilots to harmonize understanding among annotators. However, more multiply-annotated documents and additional training sessions would likely have further improved annotation quality.

\textbf{Possible need for enrichment of the data.}
Another challenge is the under-representation of certain classes in our dataset. As an example of the imbalance on the sentence-level annotations, the number of positive study conclusions (625) dwarfs the negative (18), neutral (21) and mixed (15) conclusions. Among document-level annotations, ``animal species'' class shows that the majority of animals used for experiments are mice and rats, with only a handful of other species found in the dataset (see Table \textbf{\ref{tab:document_sentence_annotation}}). Although likely reflecting the natural distribution of the conclusions among publications, this imbalance may limit the model performance for those categories. Potential remedies include merging our dataset with related ones, applying targeted data collection strategies to expand coverage and improve class balance, or augmenting the dataset with synthetic data.

\section*{Acknowledgments}
We thank Vera Lara Bernhard for her support in implementing the confidence interval calculations for the Krippendorff’s alpha algorithm.

\bibliography{custom}

\appendix
\section{Corpus Details and Statistics}
\label{sec:appendixA_corpus_stats}

\subsection{Corpus Overview}

\textbf{Figure ~\ref{fig:metadata}} shows the time range and journals represented in the corpus. 

\begin{figure*}[!ht]
\begin{center}
\includegraphics[scale=0.42]{images/corpus_metadata.pdf} % Adjust the scale value as needed
\caption{Distribution of articles in the corpus by (left) publication year and (right) journal.}
\label{fig:metadata}
\end{center}
\end{figure*}

\subsection{Annotations Overview}
\textbf{Figure ~\ref{top_10_categorical_entities}} outlines the top 10 annotations across the different document-level categories and their distribution by abstract and methods. Rodent models, particularly rats and mice, dominate the dataset, with other species such as rabbits, guinea pigs, dogs, and monkeys appearing infrequently. Experimental outcomes are diverse, with histology and behavior among the most common readouts, often annotated together, indicating a tendency to explore multiple endpoints, as well as possible annotation challenge. The majority of studies include a control group, though fewer are explicitly mentioned in abstracts. For animal sex, a male bias is evident, as well as lack of reporting of animal sex in the abstract.


\textbf{Figure~\ref{fig:kappa_and_top_15_ner_entities} }focuses on entity-level annotations in the corpus. Therapy-related drug entities are the most frequently annotated, with levodopa (109 instances) and L-DOPA (86 instances) leading the list, followed by commonly studied compounds such as morphine, MK-801, and cannabidiol (CBD). Beyond pharmacological interventions, other therapy entities include treatments like exercise, acupuncture, and curcumin. Among disease entities, Alzheimer’s disease (96 instances) and stroke (103 instances) are well-represented, while neurodegenerative and neurological conditions such as Parkinson’s disease (88 instances), epilepsy (48 instances), and spinal cord injury (81 instances) also feature prominently. Regarding animal strains, Sprague-Dawley (239 instances) and Wistar (207 instances) are the most frequently reported. However, annotations for animal numbers show substantial variability. The frequent presence of abbreviations (e.g., AD for Alzheimer’s disease and SD for Sprague-Dawley) suggests that entity disambiguation is critical for accurate text interpretation. 

\begin{figure*}[!ht]
\begin{center}
\includegraphics[scale=0.32]{images/prodigy_screenshot.png} % Adjust the scale value as needed
\caption{Annotation example shown in the annotation tool Prodigy.}
\label{fig:prodigy}
\end{center}
\end{figure*}

\begin{figure*}[!ht]
\begin{center}
\includegraphics[scale=0.34]{images/abstract_classification_combined.pdf} % Adjust the scale value as needed
\caption{Top 10 most frequent document-level annotations for (A) Species, (B) Outcomes, (C) Control, and (D) Sex categories.}
\label{top_10_categorical_entities}
\end{center}
\end{figure*}

\begin{figure*}[!ht]
\begin{center}
\includegraphics[scale=0.34]{images/all_categories_iaa_kappa.pdf} % Adjust the scale value as needed
\caption{Cohen's Kappa scores for inter-annotator pairs for the categorical annotations in the overlapping articles.}
\label{fig:all_categories_kappa}
\end{center}
\end{figure*}

\begin{figure*}[!ht]
\begin{center}
\includegraphics[scale=0.34]{images/ner_iaa_matrix_kappa.pdf} % Adjust the scale value as needed
\caption{Cohen's Kappa scores for inter-annotator pairs for the NER annotations in the overlapping articles (left column). Top 15 most frequent NER entity text spans in the full dataset (right column).}
\label{fig:kappa_and_top_15_ner_entities}
\end{center}
\end{figure*}

\subsection{Inter-Annotator Agreement Scores}

We report IAA for document-level (\textbf{Figure \ref{fig:combined_alpha}}, left), as well as sentence-level and entity-type annotations (\textbf{Figure \ref{fig:combined_alpha}}, right). For the latter, we compute and report Krippendorff's Alpha on the level of tokens (words). This allows to capture partial agreement, when annotators agree on the label but disagree on its span, i.e. where exactly it starts and ends in the text.
\begin{figure*}[t]
  \centering
  \includegraphics[width=0.48\linewidth]{images/categorical.png} \hfill
  \includegraphics[width=0.48\linewidth]{images/token_conclusion_combined.png}
  \caption{Krippendorff's alpha scores for different annotation levels. 
  (Left) Krippendorff's alpha for document-level annotations. 
  (Right) Krippendorff's alpha for token- and sentence-level annotations. Alpha score is computed on a per-token basis.}
  \label{fig:combined_alpha}
\end{figure*}

\section{Annotation Guidelines}
\label{sec:appendixF}

See full document here \href{https://github.com/Ineichen-Group/Preclinical_IE_Dataset/blob/main/Annotation_guidelines_V5.pdf}{Annotation Guidelines (v5)}.

\subsection{Inclusion Criteria for Papers}

Before starting the annotation, ensure the paper meets the following eligibility criteria:

\begin{enumerate}
    \item \textbf{Experimental study in animals} (excluding humans).
    \item The study tests an \textbf{intervention} with the goal of improving animal health. The intervention should be externally applied (e.g., gene knock-out does not qualify).
    \begin{itemize}
        \item Apply criterium generously; include studies where the exact purpose of a drug treatment is not explicitly stated (e.g., testing different substances in animals without claiming a therapeutic benefit).
        \item Exclude studies assessing the effect of endogenous substances (e.g., endogenously excreted miRNA-107).
    \end{itemize}
    \item The study models a \textbf{neurological or psychiatric disease}.
    \begin{itemize}
        \item Apply criterium generously, including studies assessing pain in osteoporosis or mentioning neurological complications in systemic diseases such as cryptococcosis.
    \end{itemize}
\end{enumerate}

If any of these criteria are not met, exclude the study (no annotation required). If pertinent, exclude at the \textbf{abstract level} to ensure all related text sections (abstract, methods, and results) are omitted.

\subsection{General Rules on Annotation}

\subsubsection{Token Annotation}
\begin{enumerate}
    \item \textbf{Consider Context:} Identify Population (P), Intervention (I), Control (C), and Outcomes (O).
    \item \textbf{Annotate Only Relevant Information:} 
    \begin{itemize}
        \item Example: If a study uses male mice but suggests repeating experiments in rats, only mice should be annotated.
        \item Example: If isoflurane is used for anaesthesia but not as a treatment, do not annotate it.
    \end{itemize}
\end{enumerate}

\subsubsection{Annotation Tasks}
\begin{itemize}
    \item \textbf{Text Annotation:} Entire text is classified based on predefined labels.
    \item \textbf{Sentence Annotation:} Entire sentence (including punctuation like colons and question marks) is annotated.
    \item \textbf{Token Annotation:} Specific words or phrases are annotated.
\end{itemize}

\subsubsection{Additional Annotation Rules}
\begin{enumerate}
    \item Sentence annotations should \textbf{exclude references at the end of sentences}.
    \item Include incorrect spelling/grammar if relevant.
    \item Avoid mixing terms with and without brackets in annotations (e.g., annotate \textit{oral appliance} and \textit{OA} separately).
    \item Do not annotate tokens where annotation would require inclusion of punctuation due to interface limitations.
    \item Overlapping annotation between different tags is allowed.
    \item Annotate each parameter once per section (i.e., once in abstract and once in methods); conclusions should only be annotated in the abstract.
    \item Be careful in selecting the correct label, as incorrect annotation affects inter-rater agreement.
    \item Do not annotate punctuation at the end of a sentence.
    \item Ignore manuscript parts mistakenly included in the annotation interface (e.g., misplaced discussion sections).
\end{enumerate}

\subsection{Experimental Parameters}

\subsubsection{Animal species}
\textbf{Type:} Population \\
\textbf{Task:} Text \\
\textbf{Definition:} The animal species used to test the intervention of interest. \\ 
\textbf{Examples:} Rats, mice, monkeys, rabbits, etc. \\ 
\textbf{Location in Paper:} Abstract, Methods \\  
\textbf{Comments:}  
\begin{itemize}
    \item Most studies use rats or mice, while monkeys, pigs, cats, dogs, and rabbits are rarer.
    \item A study could use more than one species.
\end{itemize}


\subsubsection{Animal strain}
\textbf{Type:} Population \\
\textbf{Task:} Token \\
\textbf{Definition:} The animal strain further defining the animal species. A strain is a genetic variant, a subtype, or a culture within a biological species.  \\
\textbf{Examples:} BALB/cJ (mouse), C57BL/6J (mouse), DBA/2J (mouse), Lewis (rat), Sprague-Dawley (rat). \\ 
\textbf{Location in Paper:} Abstract, Methods  \\
\textbf{Comments:}
\begin{itemize}
    \item A study could use more than one strain.
    \item Only annotate the strain (not the species).
    \item Be careful to separate strain from transgenic identification.
\end{itemize}


\subsubsection{Animal sex}
\textbf{Type:} Population \\
\textbf{Task:} Text \\
\textbf{Definition:} The animal sex further defining the animal species.  \\
\textbf{Examples:} Male, female, both sexes.\\  
\textbf{Location in Paper:} Abstract, Methods\\
\textbf{Comments:}
\begin{itemize}
    \item A study could use either male, female, or both sexes.
    \item Some studies do not report the sex used.
    \item Only label the sex used to test the drug of interest.
\end{itemize}
  

\subsubsection{Diseases mentioned}
\textbf{Type:} Population \\
\textbf{Task:} Token \\
\textbf{Definition:} Diseases of interest related to the used animal model(s).\\  
\textbf{Examples:} Multiple sclerosis, stroke, spinal cord injury, etc.\\  
\textbf{Location in Paper:} Title, Abstract \\
\textbf{Comments:}
\begin{itemize}
    \item Annotate only diseases relevant to the study.
    \item Do not annotate disease models like MCAO or EAE.
    \item Include abbreviations (e.g., MS).
    \item Annotate more and less specific mentions (e.g., Alzheimer's disease and dementia).
\end{itemize}
 

\subsubsection{Number of animals used in total}
\textbf{Type:} Population \\
\textbf{Task:} Token \\
\textbf{Definition:} The total number of animals used in the study.\\  
\textbf{Examples:} "A total of 968 animals (618 mice and 350 rats) were used." \\ 
\textbf{Location in Paper:} Abstract, Methods\\ 
\textbf{Comments:}
\begin{itemize}
    \item Only annotate the exact total number.
    \item The number might be reported more than once.
\end{itemize}
 

\subsubsection{Therapy}
\textbf{Type:} Intervention \\
\textbf{Task:} Token \\
\textbf{Definition:} The therapeutic intervention tested.\\ \textbf{Examples:} Electroacupuncture, melatonin therapy. \\ 
\textbf{Location in Paper:} Title, Abstract, Methods\\
\textbf{Comments:}
\begin{itemize}
    \item Two labels: 
    \begin{itemize}
        \item Drug (e.g., a small molecule, siRNA).
        \item Non-drug (e.g., exercise, herbal extracts).
    \end{itemize}
    \item Control treatments should be annotated.
    \item Do not annotate dosing or application information.
\end{itemize}
  

\subsubsection{Control mentioned}
\textbf{Type:} Control \\
\textbf{Task:} Text \\
\textbf{Examples:} "Group 1 (control) received saline."\\  
\textbf{Location in Paper:} Abstract, Methods\\
\textbf{Definition:} Whether the control group/treatment was mentioned.\\  
\textbf{Levels:}
\begin{itemize}
    \item Control yes
    \item Control not reported
\end{itemize}
  

\subsubsection{Readouts}
\textbf{Type:} Outcome \\
\textbf{Task:} Text \\
\textbf{Definition:} The readouts used to assess intervention efficacy. \\ 
\textbf{Examples:} "We used Nissl staining and MRI to assess stroke volume." \\ 
\textbf{Location in Paper:} Abstract, Methods \\
\textbf{Levels:}
\begin{itemize}
    \item Behavior (e.g., rotarod, seizure).
    \item Imaging (e.g., MRI, PET).
    \item Histology (e.g., Nissl staining, H\&E).
    \item Physiology (e.g., blood pressure, EEG).
    \item Other (e.g., PCR, Western blot).
\end{itemize}
 

\subsubsection{Study conclusion}
\textbf{Type:} Outcome \\
\textbf{Task:} Sentence \\
\textbf{Definition:} The main finding of the study, i.e., the overall effect of the intervention.\\  
\textbf{Examples:} "Our findings suggest a potential therapeutic role for Galantamine in attenuating hyperoxia-induced brain injury." \\ 
\textbf{Location in Paper:} Abstract \\
\textbf{Levels:}
\begin{itemize}
    \item Positive
    \item Negative
    \item Neutral
    \item Mixed
\end{itemize}
 

\subsubsection{Animal disease model}
\textbf{Type:} Population \\
\textbf{Task:} Sentence \\
\textbf{Definition:} The animal model mimicking a neurological or psychiatric condition.\\  
\textbf{Examples:} "EAE was induced by immunizing female Lewis rats with MOG55-66." \\ 
\textbf{Location in Paper:} Abstract \\ 

\subsubsection{Animal age}
\textbf{Type:} Population \\
\textbf{Task:} Sentence \\
\textbf{Definition:} The age of animals used.  \\
\textbf{Examples:} "12-week-old female C57BL/6 mice were used." \\ 
\textbf{Location in Paper:} Methods \\ 

\subsubsection{Animal weight}
\textbf{Type:} Population \\
\textbf{Task:} Sentence \\
\textbf{Definition:} The weight of animals used.\\  
\textbf{Examples:} "Male Albino Swiss (20–25 g) mice were used." \\ 
\textbf{Location in Paper:} Methods  

\subsection{Parameters Related to Study Quality}

\subsubsection{Randomization}
\textbf{Type:} Study Quality \\
\textbf{Task:} Sentence \\
\textbf{Definition:} Whether the experimental setup used randomization of animals.\\  
\textbf{Examples:} "We randomly divided the experimental rats into five groups with six animals per group as follows: …" \\ 
\textbf{Location in Paper:} Abstract, Methods  \\
\textbf{Comments:}
\begin{itemize}
    \item Only applies to circumstances describing the randomization of animals into (treatment) groups.
    \item Does NOT apply to other instances of randomization (e.g., “we analyzed 5 random fields of view”).
    \item In most cases, only one sentence describes randomization, but more than one could be annotated if different species are described separately.
    \item Together with blinding, it is one of the most critical study quality items.
    \item Annotate the entire sentence.
    \item If unsure, be generous with annotation.
\end{itemize}

\subsubsection{Blinding}
\textbf{Type:} Study Quality \\
\textbf{Task:} Sentence \\
\textbf{Definition:} Whether the experimental setup used blinding of experimenters.\\  
\textbf{Examples:} 
\begin{itemize}
    \item "Experimenters were blinded to the treatment group."
    \item "Researchers were unaware of the treatment of the animals."
    \item "All behavioral measurements were made by an observer unaware of the treatment."
\end{itemize}
\textbf{Location in Paper:} Abstract, Methods  \\
\textbf{Comments:}
\begin{itemize}
    \item Blinding can occur at any step: during treatment, analysis, or both.
    \item Typically, only one sentence describes blinding, but multiple sentences may exist for different species/experiments.
    \item Together with randomization, it is a crucial study quality item.
    \item Annotate the entire sentence.
    \item If unsure, be generous with annotation.
\end{itemize}

\subsubsection{Animal Welfare Statement}
\textbf{Type:} Study Quality \\
\textbf{Task:} Sentence \\
\textbf{Definition:} Whether the animal study complies with local, regional, national, or international animal welfare guidelines.\\  
\textbf{Examples:} 
\begin{itemize}
    \item "On October 29, 2019, the institutional Ethics Committee at NODCAR and Faculty of Pharmacy, Cairo University, approved all animal procedures."
    \item "All mice were maintained under specific pathogen-free conditions and used for experimentation according to protocols approved by the Swiss Federal Veterinary Office."
\end{itemize}
\textbf{Location in Paper:} Abstract, Methods  \\
\textbf{Comments:}
\begin{itemize}
    \item Usually, only one sentence describes animal welfare, but multiple sentences may exist for different species/experiments.
    \item Statements should mention compliance with guidelines/regulations or approval by an ethics committee.
    \item Commonly reported.
\end{itemize}

\subsubsection{ARRIVE Guidelines}
\textbf{Type:} Study Quality \\
\textbf{Task:} Sentence \\
\textbf{Definition:} Whether the study follows the ARRIVE guidelines, which provide standards for reporting methodological details in animal experiments.\\  
\textbf{Examples:}
\begin{itemize}
    \item "In the current study, we handled the animals consistently in accordance with the ARRIVE guidelines."
    \item "All studies involving animals are reported in accordance with the ARRIVE guidelines for reporting experiments involving animals."
\end{itemize}
\textbf{Location in Paper:} Abstract, Methods  \\
\textbf{Comments:}
\begin{itemize}
    \item Can be identified by searching for "ARRIVE" (always in uppercase).
    \item Rarely reported.
\end{itemize}

\subsubsection{PREPARE Guidelines}
\textbf{Type:} Study Quality \\
\textbf{Task:} Sentence \\
\textbf{Definition:} Whether the study follows the PREPARE guidelines.\\  
\textbf{Examples:} Search for "PREPARE" in the document. \\ 
\textbf{Location in Paper:} Abstract, Methods  \\
\textbf{Comments:}
\begin{itemize}
    \item Can be identified by searching for "PREPARE" (always in uppercase).
    \item Very rarely reported.
\end{itemize}

\subsubsection{Sample Size Calculation}
\textbf{Type:} Study Quality \\
\textbf{Task:} Sentence \\
\textbf{Definition:} Whether the study conducted a prior sample size calculation to determine how many animals were required for the experiments.\\  
\textbf{Examples:} 
\begin{itemize}
    \item "We conducted an a priori sample size calculation."
    \item "The power was calculated based on prior estimates."
\end{itemize}
\textbf{Location in Paper:} Abstract, Methods  \\
\textbf{Comments:}
\begin{itemize}
    \item Very rarely reported.
\end{itemize}
% \begin{figure*}
% % [!ht]
% \begin{center}
% \includegraphics[scale=0.5]{images/span_with_percentage.png} % Adjust the scale value as needed
% \caption{Krippendorff's alpha for token- and sentence-level annotations. Alpha score is computed on the span basis and captures exact matches of labels, as well as their start and end in the text.}
% \label{fig:fig:ner_alpha_percentage}
% \end{center}
% \end{figure*}

% \begin{figure*}
% % [!ht]
% \begin{center}
% \includegraphics[scale=0.5]{images/token_combined-sublabels_with_percentage.png} % Adjust the scale value as needed
% \caption{Krippendorff's alpha and percentage agreement for token- and sentence-level annotations. Alpha score is computed on the per-token basis. Subcategories of "therapy" and "outcome" are combined.}
% \label{fig:fig:ner_alpha_percentage}
% \end{center}
% \end{figure*}




\end{document}
