% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
% \usepackage[review]{EACL2023}
\usepackage[]{EACL2023}
\usepackage{graphicx}
\usepackage{float}
% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
\urlstyle{same}


% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Part-of-Speech Tagging of 16th-Century Latin with GPT}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Elina Stüssi,  Phillip Benjamin Ströbel \\
  Department of Computational Linguistics \\ University of Zurich \\
  \texttt{\{elina.stuessi,phillip.stroebel\}@uzh.ch}}

\begin{document}
\maketitle
\begin{abstract}
Part-of-speech tagging is foundational to natural language processing, transcending mere linguistic functions. However, taggers optimized for Classical Latin struggle when faced with diverse linguistic eras shaped by the language's evolution. Exploring 16th-century Latin from the correspondence and assessing five Latin treebanks, we focused on carefully evaluating tagger accuracy and refining Large Language Models for improved performance in this nuanced linguistic context. Our discoveries unveiled the competitive accuracies of different versions of \textit{GPT}, particularly after fine-tuning. Notably, our best fine-tuned model soared to an average accuracy of 88.99\% over the treebank data, underscoring the remarkable adaptability and learning capabilities when fine-tuned to the specific intricacies of Latin texts. %GPT-4 showcased commendable accuracy, standing at 80.1\%, closely trailed by GPT-3.5-Turbo, achieving 75.18\%, while LatinCy exhibited a notable accuracy of 83.22\%. Specifically focused on the 16th-century dataset, LatinCy demonstrated a proficiency of 79.8\%. Remarkably, GPT-3.5 and GPT-4 displayed robust performances of 80.2\% and 83.5\%, respectively. On this dataset, our best fine-tuned model reached an accuracy of 85.5\%. 
Next to emphasising GPT's part-of-speech tagging capabilities, our second aim is to strengthen taggers' adaptability across different periods. We establish solid groundwork for using Large Language Models in specific natural language processing tasks where part-of-speech tagging is often employed as a pre-processing step. This work significantly advances the use of modern language models in interpreting historical language, bridging the gap between past linguistic epochs and modern computational linguistics.
% main results

\end{abstract}

\section{Introduction}
% + Motivation
Understanding parts-of-speech (POS) is fundamental in linguistic analysis \citep{Jurafsky_Martin_2019}. Automatic POS tagging offers vital clues for parsing and language analysis. Despite Latin's extensive dataset in the Universal Dependencies treebanks,\footnote{There are five in total: Latin-ITTB, Latin-Perseus, Latin-PROIEL, Latin-LLCT, Latin-UDante, all of which we introduce in Section \ref{sec:datasets}. Also, see \url{https://universaldependencies.org/la}.} many historical texts lack syntactic analysis \citep{Nehrdich_Hellwig_2022}.

Latin's enduring relevance in domains like the Catholic Church and classical studies persists despite its decline post-1800 \citep{leonhardt2013latin}. However, this enduring relevance is intertwined with its extensive historical significance, contributing to the language's vast evolutionary timespan.

The evolution of the Latin language spans significant changes over time, notably evident in alterations to case endings and lexical transformations, particularly during the transition from Old Latin to Classical Latin \citep{allen1989vox}. These alterations extend beyond mere word changes, reshaping meanings and structures and resulting in diverse linguistic variations. The historical evolution of the language continued until the Early Modern Latin period of the 16th century, thereby posing challenges for POS tagging systems that have been mostly trained on Classical Latin \citep{Schmid_2019}.

Despite its historical significance, Latin remains classified as a low-resource language due to the scarcity of digitized texts and annotations \citep{Hedderich_et_al_2021}.\footnote{Although there are large text collections like the \textit{Corpus Corporum}, see \url{https://mlat.uzh.ch}.} The absence of speakers poses difficulties in creating a gold standard,\footnote{i.~e., a manually compiled and verified annotated version of a text (in our case, the annotation would concern POS tags only).} a process notably more labor-intensive and error-prone than that for modern languages. Nonetheless, Latin benefits from a wealth of linguistic expertise derived from its extensive historical legacy, offering substantial aid in overcoming these obstacles \citep{mcgillivray2013methods}.

\begin{figure*}
    \centering
    \includegraphics[width=0.95\linewidth]{example.png}
    \caption{Demonstration of a sentence tagged with Gold Standard (GS), LatinCy (LC), RDRPOSTagger (RDR), and GPT-4.}
    \label{fig:example}
\end{figure*}

The nuances in 16th-century epistolary Latin pose challenges for POS taggers, especially. Tagging a sentence from the correspondence of Swiss reformer Heinrich Bullinger (1504--1575) by various systems\footnote{We will introduce the different taggers in Section \ref{sec:taggers}.} highlights discrepancies, as illustrated in Figure \ref{fig:example}. \textit{RDRPOSTagger} misclassified punctuation and the name ``Erasmus'' as verbs. \textit{LatinCy} tagged ``adscribere'' as a noun while \textit{RDRPOSTagger} and \textit{GPT-4} identified it correctly as a verb. \textit{GPT-4}'s similarity to the gold standard underscores the potential of Large Language Models (LLMs) to enhance accuracy in language processing tasks.

Our project's core revolves around tagging 16th-century Latin data with multiple taggers, showcasing the disparities and revealing the potential of LLMs to increase accuracy in the POS tagging process. Motivated by the need to enhance linguistic analysis for historians and linguists, our work addresses the challenges in POS tagging within this historical context. Moreover, our efforts in refining POS tagging algorithms preserve cultural heritage and drive advancements in natural language processing (NLP), extending their impact across machine learning and AI beyond linguistic analysis.

% ---
% + Contributions
% 1. das und das
% 2. das und säb
% 3. no öppis anders
Our contributions encompass a detailed investigation into how fine-tuning influences POS tagging accuracy and the customization of models to distinct datasets for improved precision. We underscore the role of fine-tuning and prompting in notably enhancing performance, particularly when tailoring models to domain-specific data. By conducting extensive comparative analyses between fine-tuned and pre-trained models, we reveal each approach's distinct strengths and limitations, emphasizing the nature of domain-specific training for achieving superior accuracy in POS tagging. These evaluations offer insights important for future research, underscoring the need for tailored models and their potential applications in NLP tasks.

\section{Recent Work}
Exploration of Latin within the field of NLP has remained limited despite the existence of various methodologies designed to enhance its processing efficiency. The inaugural \textit{Workshop on Language Technologies for Historical and Ancient Languages} (LT4HALA) held in 2020 represented a step forward in developing language technologies tailored for historically documented languages, including Latin \citep{Sprungoli_Passarotti_2020}.

As a part of LT4HALA, the \textit{EvaLatin} initiative focused specifically on Latin and investigated lemmatization and POS tagging, scrutinizing their performance across diverse temporal contexts \citep{Sprungoli_Passarotti_2020}. \textit{EvaLatin} encompassed works such as \textit{LSTMVoter} \citep{Stoeckel_et_al_2020} and the \textit{UDPipe2}-based system \citep{straka2020UDPipe}, showcasing advancements in techniques customized for historical Latin texts. Additionally, the \textit{LiLa} project\footnote{See \url{https://lila-erc.eu/}.} significantly fortified the lexical foundation for Latin, fostering a symbiotic relationship between textual and lexical resources \citep{Passarotti_et_al, Pellegrini_et_al}.

However, despite efforts like Chu\footnote{See \url{https://bit.ly/3vUyqNu}.} highlighting the strengths of GPT models for POS tagging, there remains a conspicuous gap in research specifically exploring LLMs as POS taggers. %These influential studies have shaped the methodologies adopted in this work, guiding the assessment of language models and POS taggers in historical Latin texts.

\section{Data and Methodology}

\subsection{Datasets}
\label{sec:datasets}
This study focused on leveraging LLMs, particularly different flavours of GPT, for the POS tagging of historical texts from different periods. Utilizing UPOS tags\footnote{See \url{https://universaldependencies.org/u/pos}.} for consistency, our experiments encompassed samples from our own \textit{Bullinger Digital} corpus \cite{BullingerDigital2020-1} and five treebanks: ITTB \citep{Cecchini_Passarotti_Marongiu_Zeman_2018, passarotti_dell’orletta}, LLCT \citep{korkiakangas_2021}, UDante \citep{cecchini_sprugnoli_moretti_passarotti_2021}, PROIEL \citep{dag_t._t._haug_and_marius_l._jøhndal}, and Perseus \citep{bamman_crane_2006}. Table \ref{tab:datasetstats} provides an overview of the data used.

\begin{table}
    \centering
    \resizebox{\columnwidth}{!}{
    \begin{tabular}{lrrr} 
        \hline
         \textbf{Dataset}&  \textbf{time}& \textbf{\# of sentences}& \textbf{\# of token-tag pairs}\\ 
         \hline
         Bullinger& c.~16 & 200&  3664\\ 
         ITTB&  c.~13 & 24,876&  420,672\\ 
         LLCT&  c.~8 -- c.~10 & 8,173&  218,223\\
         PROIEL& c.~1 BCE -- c.~4 & 11,851& 110,774\\
         UDante&  c.~13 -- c.~14 & 1,157&  38,086\\ 
         Perseus&  c.~1 BCE -- c.~4 & 4,236&  68,283\\
         \hline
         \textbf{Total}& & \textbf{50,493}&  \textbf{859,702}\\
         \hline
    \end{tabular}
    }
    \caption{Overview of different datasets (c. $=$ century).}
    \label{tab:datasetstats}
\end{table}

The Bullinger corpus, derived from Heinrich Bullinger's 16th-century correspondence, offers insights into early modern societal aspects and the reformation process in Switzerland and Europe. This study uses a sample from a corpus comprising approximately 220k sentences, with the selected subset comprising 200 sentences. The sample intentionally includes only the Latin sections from digitized letters structured into XML format, excluding the Early New High German sentences.

The five treebanks served as training material and fine-tuning data for POS tagging models. The ITTB dataset offers morphosyntactic disambiguation and sentence-level syntactic annotation for Latin. The training and test sets provided in CoNLL-U format include 24,876 tagged sentences, totalling 420,627 token-tag pairs.

Similarly, the Universal Dependencies (UD) version of the Late Latin Charter treebank (LLCT) sheds light on 521 Early Medieval Latin records (charters) from 774 CE to 897 CE. These charters present a non-standard Latin variety, focusing on legal documentary genres, and pose linguistic challenges due to their formulaic nature.\footnote{See \url{https://github.com/UniversalDependencies/UD_Latin-LLCT}.} The dataset utilized in this work encompasses 8,173 tagged sentences, totalling 218,223 token-tag pairs.

Additionally, UDante, a project annotating Dante Alighieri's Latin works, includes 1,157 tagged sentences amounting to 38,086 token-tag pairs, focusing on 14th-century literary Medieval Latin.\footnote{See \url{https://github.com/UniversalDependencies/UD_Latin-UDante}.}

Moreover, the Pragmatic Resources in Old Indo-European Languages (PROIEL) project comprises 11,851 tagged sentences totalling 110,774 token-tag pairs, involving annotated texts such as the New Testament and Latin works like Cicero's ``Epistulae ad Atticum'' \citep{Eckhoff_Majer_Welo_Haug_2009}.

Lastly, the Perseus dataset (version 2.1) features semi-automatically annotated texts like Cicero’s ``In Catilinam'', Ovid’s ``Metamorphoses,'' and Augustus’ ``Res Gestae.'' Perseus did not originally use UPOS tags, so we mapped the Perseus tags\footnote{See \url{https://github.com/PerseusDL/treebank_data/blob/master/v2.1/Latin/TAGSET.txt}.} to UPOS tags. Notably, not all UPOS tags had direct equivalents in the Perseus tags. For instance, Perseus employed only ``c'' to represent conjunctions, lacking the differentiation between subordinating conjunctions (SCONJ) and coordinating conjunctions (CCONJ) as observed in UPOS tags. We omitted the files that included Caesar’s ``Commentarii de Bello Gallico'' and Jerome’s ``Vulgata,'' as these texts were already included in the PROIEL dataset. The shortened Perseus dataset utilized in this project comprises 4,236 tagged sentences, totalling 68,283 token-tag pairs.

\subsection{POS Tagging Models}
\label{sec:taggers}
We evaluated the performance of various POS tagging models on 16th-century epistolary Latin sourced from the Bullinger letters (i.~e., the Bullinger sample mentioned in Section \ref{sec:datasets} and Table \ref{tab:datasetstats}). The comparative analysis involved \textit{LatinCy}, \textit{CLTK}, \textit{UDPipe}, \textit{RDRPOSTagger}, and \textit{TreeTagger}, alongside the examination of GPT-3.5-Turbo and GPT-4. The taggers compared in this study encompassed a spectrum of approaches, such as Single Classification Ripple-Down Rule (SCRDR) trees, statistical methods, and other distinct methodologies.

\textit{LatinCy}, a \textit{spaCy}-based Latin NLP toolkit introduced in 2023, employs \textit{spaCy}'s \citep{montani_explosionspacy_2023} POS tagger,\footnote{See \url{https://spacy.io/api/tagger}.} backed by statistical models based on neural networks trained on the OntoNotes 5 corpus \citep{weischedel2013ontonotes}. With three core models, including ``la\_core\_web\_lg'' utilizing sub-word vectors \citep{burns2023latincy}, \textit{LatinCy} comprehends extensive vocabularies beyond its training data \citep{Acs_Kadar_Kornai_2021}. Training incorporates diverse sources like Latin Universal Dependencies treebanks \citep{Celano_2019}, Wikipedia or a pre-processed version of the cc100-latin corpus \citep{ströbel}. Notably, the ``la\_core\_web\_lg'' model we used for our research achieves an impressive 97.41\% accuracy for POS tagging on the respective test data \citep{burns2023latincy}.

The \textit{Classical Language Toolkit} (CLTK),\footnote{See \url{http://cltk.org/}.} established in 2014, caters to ancient languages like Latin and Greek, among others. CLTK's architecture supports various pre-modern languages, providing functionalities for POS tagging, tokenization, and lemmatization \citep{johnson-etal-2021-classical}. Utilizing \textit{Stanza} \citep{qi2020stanza} with bidirectional Long Sort-Term Memory networks \citep{hochreiter1997long}, it attains average accuracies of 68\% for unigrams and 82\% employing a 1, 2, 3-gram back-off tagger on the Perseus test data.\footnote{For further details, see \url{http://cltk.org/blog/2015/08/02/updated-accuracies-pos-taggers.html}.}

\textit{UDPipe2} operates as a multifaceted pipeline, incorporating a neural network with a single joint model for tasks like POS tagging and dependency parsing. It leverages CoNLL-U format data and pre-trained word embeddings, where, e.~g., the ``Latin-ITTB'' model achieved a high accuracy of 98.28\% on the ITTB test data \citep{Straka_2018}. Its flexibility spans over 50 languages, including non-Indo-European ones like Arabic and Irish \citep{UDPipe}.

\textit{RDRPOSTagger} employs SCRDR trees for POS tagging across approximately 80 languages, with three available models showcasing varying accuracies for Latin \citep{nguyen_nguyen_pham_pham_2014}. For our research, we employed the ``UD\_Latin-ITTB'' model. Its conditional rule structure allows controlled interactions between rules, proving adaptable to languages like Latin. We used the ``UD\_Latin-ITTB'' model that yielded an accuracy of 96.85\% on the ITTB test set.

\textit{TreeTagger}, developed through the University of Stuttgart’s textual corpora project, adeptly annotates POS and lemma information in numerous languages like German, English, Chinese, Russian, Greek, and Latin. Its adaptability to new languages hinges on a lexicon and tagged training corpora, underscoring its versatility \citep{Schmid_1994}. There are parameter files for numerous languages available; for Latin, we used the parameter file by Gabriele Brandolini. Functionally resembling traditional \textit{n}-gram taggers, 
\textit{TreeTagger} estimates transition probabilities using a binary decision tree and achieves accuracies ranging from around 95.8\% to 96\% on the Penn-Treebank for bigram and trigram versions \citep{schmid1995improvements}. However, its output does not directly use UPOS tags, necessitating a post-process tag mapping for compatibility.

Our study assessed two LLMs developed by OpenAI: GPT-3.5-Turbo and GPT-4. GPT-3.5, launched in 2022 and based on GPT-3 \citep{gpt3}, boasts 175 billion parameters and excels in tasks like translation, text completion, and question-answering \citep{openai2023gpt4}. Its architecture, excluding the encoder attention part, relies on an unmodified transformer decoder \citep{Gupta_2023}. GPT-4, despite improvements, shares limitations like occasional unreliability and context window constraints. While GPT-4 shows superior task performance and visual data processing, only GPT-3.5-Turbo currently supports fine-tuning with custom data.\footnote{As of 07.11.2023, when we conducted our experiments.}

The various models underwent a thorough assessment, revealing their capabilities and limitations in dealing with historical Latin texts. We applied each model mentioned above to the (test) datasets mentioned in Section \ref{sec:datasets} and compared the obtained accuracies. Incorporating conventional POS taggers and contemporary (fine-tuned) LLMs allowed for a direct comparison between traditional and LLM approaches.

\section{Experiments and Results}

\subsection{Gold Standard}
The initial phase involved the curation of a condensed Bullinger corpus comprising 200 sample sentences extracted from the Bullinger letters. Manual verification ensured a representative compilation spanning various editions, authors, and temporal contexts, subsequently stored in text files for further processing.

After the data curation process, the text was tokenized using the \textit{spaCy} tokenizer with the ``la\_core\_web\_lg'' model. We removed punctuation, including internal parentheses within words.\footnote{E.~g., the transcription could contain ``vestr[um]'' (EN \textit{your}), where the editors added the ``um'' in parentheses. We removed the parentheses to obtain ``vestrum''.} The subsequent application of the UDPipe tagger allowed for assigning reference tags to individual tokens, forming the foundation for creating an accurate gold standard dataset.

Multiple annotators, including a Latin expert, were involved in the verification and correction process of reference UPOS tags assigned by UDPipe. Any discrepancies between the assigned tags and the ideal classifications were addressed through manual verification. The collaboration with the Latin expert played a significant role in establishing a strong gold standard, incorporating mutually agreed-upon tagging principles. We employed Cohen's Kappa to assess inter-annotator agreement (IAA) on the corrected tag versions of the Bullinger corpus, yielding a noteworthy IAA of 0.97.
\begin{figure*}
    \centering
    \includegraphics[width=0.95\linewidth]{prompt.png}
    \caption{Prompt employed for POS tagging using the GPT API.}
    \label{fig:prompt}
\end{figure*}

\subsection{POS Tagging}
\label{sec:postagging}
Our study employed diverse POS tagging models, as detailed in Section \ref{sec:taggers}, each utilizing distinct tagging methods. As test sets, we used the tokenized and manually tagged sample of the Bullinger corpus on the one hand and the tokenized test sets of the treebanks introduced in Section \ref{sec:datasets} on the other hand. %To use the taggers, different methodologies had to be used. \textit{LatinCy} and \textit{CLTK} are available as Python packages, while \textit{TreeTagger} is a command line tool and \textit{RDRPOSTagger} and \textit{UDPipe} rely on R.
We conducted manual post-processing on the taggers' output to ensure consistent token placement, aiming for uniformity in the models' outputs. This manual review became necessary because the taggers occasionally performed additional tokenization on certain words. Specifically, in the case of GPT models, they sometimes added extra text to the response, requiring careful verification to ensure uniform and comparable outputs.

Our experimentation also involved GPT-3.5-Turbo and GPT-4, which required specific prompts for accurate tokenization. We exclusively used system and user prompts, keeping all other parameters (like, e.~g., temperature) unaltered. After encountering tokenization issues with the prompts used for the Bullinger test set, we refined the input by incorporating both the original sentence and a tokenized version aligned with our manually created gold standard. Figure \ref{fig:prompt} displays the used prompt. The variable ``tokens'' refers to the list of tokens and with the variable ``sentence'' we entered the sentence that should be tagged. Furthermore, we explored a token-only approach for the Bullinger corpus, submitting only tokens without a reference sentence in the prompt. The utilized prompt for this approach is displayed in Figure \ref{fig:tok-only}. Since we had used the already tokenized version of the corpora, we explicitly instructed the models in the user prompt not to perform additional tokenization. In contrast to the sentence-included approach, we focused solely on obtaining tags for individual tokens. Consequently, we requested only the tag for each token inserted into the prompt, replacing the ``token'' variable.

\begin{figure*}
    \centering
    \includegraphics[width=0.95\linewidth]{tok-only.png}
    \caption{Prompt employed for the token-only POS tagging approach using the GPT API.}
    \label{fig:tok-only}
\end{figure*}


For the application of GPT models on treebank data lacking sentence boundaries, such as PROIEL, we developed an alternative strategy. Instead of providing complete sentences, we utilized sets of 65 tokens in the prompts, bypassing the requirement for punctuation to delineate sentences. This adaptation enabled the effective use of the GPT models without explicit sentence boundaries. The package size of 65 tokens was selected randomly, aiming to encompass the majority of sentences almost in their entirety. Even when complete inclusion was not possible, the chosen number ensured the presence of contextual information from the sentence, facilitating the disambiguation of words.



\subsection{Fine-Tuning of GPT-3.5-Turbo}
For the fine-tuning of GPT-3.5-Turbo, we obtained the training and test data from treebanks specified in Section \ref{sec:datasets} in the form of samples with an 80/20 split. When a pre-defined test set was absent, as was the case for PROIEL and Perseus, we randomly selected the test set. % We maintained the ratios of the different treebanks through stratified sampling. 
To explore how the model performance varies when provided with differently sized training sets, we crafted subsets for fine-tuning in sizes ranging from 50 to 10,000 sentences. We used stratified sampling to obtain the required examples from each training set to represent the different treebank sizes adequately. These subsets used for fine-tuning were formed by concatenating the chosen samples from the training sets from each resource listed in Section \ref{sec:datasets}. E.~g., ``train5000'' specifies a model fine-tuned on 5,000 sentences sampled from the different treebanks, taking the size of each treebank into account. We then prepared the data as required by the OpenAI API guidelines.\footnote{See \url{https://platform.openai.com/docs/guides/fine-tuning}.}

\subsection{POS Tagging Results}
The pre-trained models' output required some manual post-processing, albeit not to the same extent as the GPT models' outputs. Significant discrepancies were noted within the GPT-generated content, encompassing repetitions, omissions of passages, instances of unexpected text occurrences (as illustrated in Figure \ref{fig:gpt_output}), and irregularities such as unspecified tags and incorrect formats. These findings underscore limitations within the model's performance, notably occurring more frequently with models trained using larger datasets, such as train5000 and train10000.\footnote{The best-performing model on the Bullinger sample (train100) can be accessed with the model name \texttt{ft:gpt-3.5-turbo-0613:cl-uzh:train-100:8GWMiGKN}, the overall best model (train1000) is available under the name  \texttt{ft:gpt-3.5-turbo-0613:cl-uzh:train-1000:8HUHOHgt}.}

\begin{figure}
    \centering
    \includegraphics[width=0.6\linewidth]{gpt_output.png}
    \caption{Example of GPT API output displaying unforeseen textual anomalies}
    \label{fig:gpt_output}
\end{figure}

After the manual clean-up of the output of all taggers on the Bullinger test set, involving the correction of tokens to adhere to the gold standard tokenization, especially in cases where taggers had further tokenized the input tokens, \textit{LatinCy} demonstrated the highest accuracy (79.8\%) among pre-trained models, slightly surpassing CLTK by 2.7\%. RDRPOSTagger displayed the lowest accuracy (64.5\%), indicating limitations in processing 16th-century data. Table \ref{table-taggers} shows an overview of the results.

The token-only approach on the Bullinger corpus yielded the highest accuracy with the fine-tuned model train2000, reaching 78.2\%. Remarkably, despite the absence of a reference sentence and the model being provided with only an individual token, the accuracy was nearly as high as for LatinCy. In contrast, the baseline model GPT-3.5-Turbo achieved only 62.2\% accuracy in this approach, emphasizing the significant improvement brought about by fine-tuning. The non-fine-tuned GPT-4 achieved an accuracy of 74.3\%, showcasing a clear performance difference between GPT-3.5-Turbo and GPT-4. The lowest accuracy, at 58.8\%, was observed with the train500 model.

In the sentence-included approach, the best result was obtained using the train100 model, reaching an accuracy of 85.5\% on the Bullinger corpus, surpassing the traditional tagging models by a significant margin. In this approach, the differences between the fine-tuned and the baseline models GPT-3.5-Turbo and GPT-4 were less pronounced. GPT-3.5-Turbo achieved an accuracy of 80.2\%, and GPT-4 reached an accuracy of 83.5\%. The model with the lowest accuracy in this scenario was train500, with an accuracy of 77.2\%.

% The evaluation aimed to assess the performance of various POS taggers on the Bullinger corpus and treebank test corpora. Evaluation of GPT models using token-only versus sentence-included approaches revealed notable differences. The sentence-inclusive method consistently achieved higher accuracy across all models. The token-only method reached an average accuracy of 69.35\% for GPT-3.5-Turbo, GPT-4 and the fine-tuned models, while the sentence-inclusive method achieved 80.5\% using these models, indicating an over 11\% difference. The model train500 showed the most significant difference between the two methods with 18.4\%, while train2000 had only a difference of 0.1\%.

% The tagging process was time-consuming, particularly for GPT-4, hence each model was tested only once. While pre-trained models were faster, other GPT models occasionally necessitated several hours, contingent upon the overall global demand for GPT resources. Optimal efficiency was observed during off-peak hours for model testing, contrasting with markedly prolonged processing durations experienced during evening hours (UTC+1), reflective of heightened usage.

On the test data from the treebanks, \textit{LatinCy} emerged with the highest average accuracy of the pre-trained models (83.22\%), indicating effective performance as a POS tagger. The fine-tuned train1000 model exhibited the most effective performance (88.99\%), whereas RDRPOSTagger had the lowest (72.36\%). Notably, performance across various test sets varied, with ITTB showing the highest accuracy (84.95\%) and PROIEL the lowest (74.73\%). %For detailed results, please refer to Table \ref{table-taggers}.

\begin{table*}[t]
\centering
\begin{tabular}{ll||lllll|l}
\hline
\textbf{Tagger} & \textbf{Bullinger} & \textbf{ITTB} & \textbf{LLCT} & \textbf{UDante} & \textbf{Perseus} & \textbf{PROIEL} & \textbf{Avg TB} \\
\hline
LatinCy & \textbf{79.8} & 87.93 & 92.01 & 80.94 & 72.38 & 82.84 &83.22\\
CLTK & 77.1 & 88.45 & 79.32 & 77.36 & 72.44 & 80.63 & 79.64\\
UDPipe & 72.8 & 71.59 & 71.45 & 70.51 & 69.2 & 84.49 & 73.45\\
RDRPOSTagger & 64.5 & 82.67 & 67.41 & 71.72 & 64.79 & 75.22 & 72.36\\
TreeTagger & 74.3 & 70.18 & 70.25 & 69.86 & 82.51 & 89.39 & 76.44\\
\hline
GPT-3.5-Turbo & 62.2/80.2 & 74.82 & 78.82 & 74.33 & 68.81 & 79.22 & 75.2\\
GPT-4 & 74.3/83.5 & 79.73 & 84.89 & 77.62 & 73.9 & 84.38 & 80.1\\
\hline
train50 & 70.3/84.8 & 89.59 & 89.2 & 83.55 & 73.2 & 79.37 &82.98\\
train100 & 69.4/\textbf{85.5} & 89.73 & 91.03 & 85.26 & 74.19 & 82.19 & 84.48\\
train200 & 68.2/80.0 & 91.57 & 90.93 & \textbf{85.8} & 73.46 & 83.5 &85.05\\
train500 & 58.8/77.2 &93.2& 93.85 &82.71& 72.09& 86.72&85.71\\
train1000 & 65.8/82.5 & \textbf{94.88} & \textbf{94.5} & 84.94 & 81.39 & 89.25 & \textbf{88.99}\\
train2000 & \textbf{78.2}/78.3 & 87.95 & 87.43 & 81.31 & \textbf{84.31} & 87.83 & 85.77\\
train5000&71.9/76.6 & 88.11& 84.85 & 74.62 &81.99& \textbf{90.0}&83.91\\
train10000 & 74.4/76.4 & 83.84 & 85.39 & 75.36 & 76.34 & 86.01 & 81.39\\
\hline
\end{tabular}
\caption{\label{table-taggers}
Tagger performance across different datasets. Bold numbers indicate the highest accuracy within each test set's column. The taggers starting with ``train'' are our fine-tuned GPT-3-5-Turbo models. For the GPT models, two numbers for the Bullinger data indicate the token-only and sentence-included approach (see Section \ref{sec:postagging}). The average calculated over the test sets of the five treebanks is displayed in column Avg TB.}

\end{table*}

\subsection{Tag Distribution}
The taggers exhibit variations in their outputs. Some taggers allocate certain tags more frequently than others, owing to their dissimilar training data and learning algorithms. Moreover, not all taggers have been exposed to datasets encompassing all UPOS tags. In Figure \ref{fig:bullinger_dist}, depicting tag distribution across four taggers and the gold standard on the Bullinger corpus data, common POS tags like NOUN and VERB are consistently assigned across all versions. However, notable differences emerge in the frequency of tags such as adverbs (ADV), determiners (DET) and pronouns (PRON). DET, for instance, appears significantly more in the gold standard compared to \textit{LatinCy} or the basic GPT models. Only the fine-tuned model train100, boasting the highest accuracy on the Bullinger corpus, mirrors a similar frequency in assigning this tag. Conversely, \textit{LatinCy} frequently uses ADV but assigns the tag for adpositions (ADP) less frequently than other taggers. The gold standard employs PRON less frequently, while GPT-4 allocates this tag almost twice as often as our gold standard did.

\begin{figure}
    \centering
    \includegraphics[width=0.9\linewidth]{bullinger.png}
    \caption{POS tag distribution in the Bullinger corpus.}
    \label{fig:bullinger_dist}
\end{figure}

In analyzing the LLCT treebank data, a distinct disparity emerges in tag distribution when juxtaposed with the Bullinger corpus. Figure \ref{fig:llct_dist} illustrates the tag distribution of the LLCT data. Our comparison involves the tagging outputs of \textit{LatinCy}, GPT-3.5-Turbo, GPT-4, the fine-tuned model \textit{train1000}, and the gold standard. Notably, a greater convergence among the models is evident, signifying more consistent tag assignments. Once again, NOUN and VERB exhibit striking similarities across these models, as do auxiliary verbs (AUX). However, a significant deviation surfaces with the other tag (X). \textit{LatinCy} and the gold standard exclude its usage, while GPT-3.5-Turbo predominantly assigns this tag, potentially indicating problems encountered during tagging processes.
\begin{figure}
    \centering
    \includegraphics[width=0.9\linewidth]{llct.png}
    \caption{POS tag distribution in the LLCT test set.}
    \label{fig:llct_dist}
\end{figure}
A comparable pattern emerges when examining the distribution of coordinating conjunctions (CCONJ) and DET across Figures \ref{fig:bullinger_dist} and \ref{fig:llct_dist}. While CCONJ displays similarity across both test sets, the gold standard and fine-tuned models demonstrate a higher frequency in assigning this tag. Similarly, the DET tag exhibits a parallel pattern, with the fine-tuned models and the gold standard assigning this tag noticeably more frequently than the other three models.


% To assess differences in tag assignment for the Bullinger corpus, confusion matrices were created. Figure \ref{fig:latincy_cf} illustrates the comparison between the gold standard tags in the Bullinger corpus and those assigned by LatinCy. In Figure \ref{fig:gpt4_cf}, the tags assigned by GPT-4 using the sentence-included approach are displayed. These matrices present taggers' predictions along the x-axis and gold standard tags along the y-axis, providing insights into their performance.

% LatinCy exhibits a nearly diagonal line, indicating generally accurate predictions. However, it correctly predicts the SCONJ tag only 38\% of the time, incorrectly predicting ADV 47\% of the time and PRON 13\% of the time. In contrast, GPT-4 displays a more uniform and darker diagonal line, suggesting higher accuracy in tag assignments. GPT-4 demonstrates less dispersion in tag assignments compared to LatinCy. Notably, GPT-4 encounters difficulties in assigning the PART tag, mislabeling it as ADV (26\%) and PRON (0.02\%). 


\section{Discussion}
Evaluating part-of-speech tagging models within the context of 16th-century Latin texts provides valuable insights into language processing methodologies, particularly within historical frameworks. This comprehensive assessment reveals several noteworthy observations.

\subsection{Fine-Tuning and Model Performance}
One key finding is the significant impact of fine-tuning LLMs, such as GPT-3.5-Turbo, on POS tagging accuracy. Despite the superior performance of GPT-4 and GPT-3.5-Turbo, even without fine-tuning, the fine-tuned GPT-3.5-Turbo models outperformed conventional pre-trained taggers, especially on specific test sets. Especially the dominance over LatinCy, the most recent tagger that operates with transformer pipelines, is striking. This underscores the potential for domain-specific fine-tuning to enhance LLM efficacy in linguistically nuanced domains.

The evaluation aimed to assess the performance of various POS taggers on the Bullinger corpus and treebank test corpora. Evaluation of GPT models using token-only versus sentence-included approaches revealed notable differences. The sentence-inclusive method consistently achieved higher accuracy across all models, with a difference of over 11 percentage points compared to the token-only method. The model train500 showed the most significant difference between the two methods with 18.4 percentage points, while train2000 had only a difference of 0.1 percentage points.

The study also emphasizes the importance of prompting strategies when employing LLMs like GPT-4 and GPT-3.5-Turbo for POS tagging. Variations in accuracy between token-only and sentence-included approaches underscore the necessity of prompt engineering to improve contextual comprehension and enhance tagging performance.

\begin{figure}
    \centering
    \includegraphics[width=\linewidth]{LatinCy_confusion_matrix.png}
    \caption{Confusion matrix for LatinCy on the Bullinger sample.}
    \label{fig:latincy_cf}
\end{figure}

\subsection{Tag Assignment Challenges and Contextual Cues}
Inconsistencies in tag assignments, especially for determiners and coordinating conjunctions, highlight the critical need for standardized definitions and categorizations. The study underscores the role of contextual cues in accurate POS tag assignments, particularly for ambiguous word classes like modal verbs and participles.

Confusion matrices were created to assess differences in tag assignment for the Bullinger sample. Figure \ref{fig:latincy_cf} illustrates the comparison between the gold standard tags in the Bullinger sample and those assigned by LatinCy, while Figure \ref{fig:gpt4_cf} displays the tags assigned by GPT-4 using the sentence-included approach. These matrices present taggers' predictions along the x-axis and gold standard tags along the y-axis, providing insights into their performance.

LatinCy exhibits a nearly diagonal line, indicating generally accurate predictions. However, it correctly predicts the SCONJ tag only 38\% of the time, incorrectly predicting ADV 47\% of the time and PRON 13\% of the time. In contrast, GPT-4 displays a more uniform and darker diagonal line, suggesting higher accuracy in tag assignments. GPT-4 demonstrates less dispersion in tag assignments than LatinCy, encountering difficulties in assigning the PART tag, mislabeling it as ADV (26\%) and PRON (0.02\%). 

\begin{figure}
    \centering
    \includegraphics[width=\linewidth]{GPT4_confusion_matrix.png}
    \caption{Confusion matrix for GPT-4 on the Bullinger sample.}
    \label{fig:gpt4_cf}
\end{figure}
\subsection{Comparative Analyses and Challenges in Applying Taggers}
Comparative analyses, akin to prior studies such as \citet{Chu2023}, underscore the significance of prompting in part-of-speech tagging, emphasizing both similarities and disparities in the performance among various GPT models. Additionally, this investigation unveils challenges encountered when applying taggers to the Bullinger corpus, revealing notable differences in tagging standards and word usage.

The tagging process was time-consuming, particularly for GPT-4, hence each model was tested only once. While pre-trained models were faster, other GPT models occasionally necessitated several hours, contingent upon the overall global demand for GPT resources. Optimal efficiency was observed during off-peak hours for model testing, contrasting with markedly prolonged processing durations experienced during evening hours (UTC+1), reflecting heightened usage.

\subsection{Implications and Future Directions}
The study's findings underline the potential and constraints of traditional part-of-speech taggers and LLMs in historical Latin text analysis. The research serves as a pivotal impetus for future studies, prompting advancements in tagging precision and adaptability within historical and resource-limited language contexts. Further exploration of refined contextual models, standardized categorizations, and improved efficiency in deploying LLMs for extensive historical language analyses is encouraged.

\subsection{Considerations for Resource Scalability}
Despite the competitive performance, especially post-fine-tuning, concerns arise regarding the pragmatic utilization of LLMs in historical text analysis due to resource scalability challenges. The time-intensive nature of tagging procedures, particularly with models like GPT-4, raises considerations for their efficiency and scalability in large-scale historical language studies.

\section{Conclusion}
% The evaluation of part-of-speech tagging models within the context of 16th-century Latin texts reveals insights into language processing methodologies, especially within historical frameworks. This comprehensive assessment offers several notable observations.

% This study demonstrates the influence of fine-tuning Large Language Models on part-of-speech tagging accuracy. Although GPT-4 and GPT-3.5-Turbo displayed superior performance even without fine-tuning, the fine-tuned GPT-3.5-Turbo models notably outperformed conventional pre-trained taggers (especially on the test sets they have been trained on). This outcome underscores the substantial potential for domain-specific fine-tuning to enhance Large Language Model efficacy in linguistically nuanced domains. Conversely, conventional pre-trained part-of-speech taggers such as \textit{LatinCy}, \textit{CLTK}, and \textit{RDRPOSTagger} exhibited marked discrepancies in accuracy when confronted with 16th-century Latin compared to their test data. These discrepancies illuminate the challenges inherent in applying standard pre-trained taggers directly to historical language datasets without customizations tailored to the linguistic nuances of the era.

% Furthermore, this study emphasizes the role of prompt strategies when employing Large Language Models like GPT-4 and GPT-3.5-Turbo for POS tagging. Variations in accuracy between token-only and sentence-included approaches underscore the necessity of prompt engineering to improve contextual comprehension and enhance tagging performance.

% Inconsistencies in assigning tags displayed in Figures \ref{fig:bullinger_dist} and \ref{fig:llct_dist}, especially for determiners (DET) and coordinating conjunctions (CCONJ), highlight the critical need for standardized definitions and categorizations. Moreover, the study underscores the role of contextual cues in accurate part-of-speech tag assignments, particularly concerning ambiguous word classes like modal verbs and participles, highlighting the need for refined contextual models or unified approaches in their classification methodologies. When definitions diverge, resulting in different categorizations for the same words, it emphasizes the requirement for precise classification of word types. Modal verbs, for example, can be classified as verbs or auxiliaries, depending on their definition. This introduces ambiguity. Similarly, participles functioning as verbal adjectives present distinct challenges owing to their combined traits of verbs and adjectives. Accurately assigning part-of-speech tags to participles in Latin heavily relies on contextual cues, demanding a unified consensus in their classification.

% Comparative analyses akin to \citet{Chu2023} highlighted similarities and disparities in performance among GPT models, accentuating the importance of prompting in part-of-speech tagging. Additionally, this study unveiled challenges in applying taggers to the Bullinger corpus due to disparities in tagging standards and word usage.

% Despite demonstrating competitive performance, especially post fine-tuning, the pragmatic utilization of Large Language Models in historical text analysis raises concerns about resource scalability. The time-intensive nature of tagging procedures, notably with models like GPT-4, presents challenges that affect the efficiency and scalability of employing such models for extensive historical language analyses.

% In conclusion, these findings underline the potential and constraints of both traditional POS taggers and LLMs in historical Latin text analysis. This research is a pivotal impetus for future studies, prompting advancements in tagging precision and adaptability within historical and resource-limited language contexts.

In conclusion, these multifaceted findings contribute to our understanding of part-of-speech tagging in historical Latin texts and pave the way for nuanced and targeted advancements in natural language processing within this domain. We could show that fine-tuning Large Language Models like OpenAI's GPT-3.5-Turbo can significantly heighten accuracy in part-of-speech tagging performance. We also provided insights into different prompting techniques for obtaining optimal results. However, the challenges related to stability and resource scalability, especially with time-intensive tagging procedures, raise considerations for the pragmatic utilization of Large Language Models in large-scale historical language studies.

\section*{Limitations}
The study faced the following limitations: Tests, especially with GPT-4, were time-consuming, limiting the number of test runs due to extended processing times influenced by global demand and usage peaks. Performing only one test run per model with a single-epoch testing approach constrained a more thorough assessment of fine-tuning capabilities. Furthermore, formatting complexities in the output of models posed challenges, impeding the ability to adjust incorrectly formatted passages manually. This limitation hindered a more comprehensive analysis that could have been achieved through re-tagging sections if the process had been less time-intensive. Additionally, it is important to note that our fine-tuning was exclusively conducted on the treebank data due to the absence of large-scaled gold standards for the 16th-century dataset. The 16th-century test set, comprising only 200 sentences, might not have fully represented that era's language complexities. Consequently, this limitation might have introduced an abundance of edge cases, potentially leading to decreased accuracy in the assessment.

% Entries for the entire Anthology, followed by custom entries
\bibliography{custom}
\bibliographystyle{acl_natbib}

% \appendix

% \section{Example Appendix}
% \label{sec:appendix}

% This is a section in the appendix.

\end{document}
