% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
%\usepackage[review]{EACL2023}
\usepackage{EACL2023}


% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{graphicx}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

\usepackage{paralist}


% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{People and Places of the Past - Named Entity Recognition in Swedish Labour Movement Documents from Historical Sources}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{
   Crina Tudor \\ 
   Stockholm University,  Sweden \\ 
   {\tt first.last@ling.su.se} \\\And
   Eva Pettersson \\ 
   Uppsala University,  Sweden \\ 
   {\tt first.last@lingfil.uu.se}
}

\begin{document}
\maketitle
\begin{abstract}
Named Entity Recognition (NER) is an important step in many Natural Language Processing tasks. The existing state-of-the-art NER systems are however typically developed based on contemporary data, and not very well suited for analyzing historical text. In this paper, we present a comparative analysis of the performance of several language models when applied to Named Entity Recognition for historical Swedish text. The source texts we work with are documents from Swedish labour unions from the 19th and 20th century. We experiment with three off-the-shelf models for contemporary Swedish text, and one language model built on historical Swedish text that we fine-tune with labelled data for adaptation to the NER task. Lastly, we propose a hybrid approach by combining the results of two models in order to maximize usability. We show that, even though historical Swedish is a low-resource language with data sparsity issues affecting  overall performance, historical language models still show very promising results. Further contributions of our paper are the release of our newly trained model for NER of historical Swedish text, along with a manually annotated corpus of over 650 named entities.
\end{abstract}

\section{Introduction}
\label{sec:intro}

The Swedish labour movement is strong by tradition and has played a crucial role in the development of the Swedish welfare society and in shaping the structure of the labour market. The Swedish trade union federations play an important role internationally, and their archives offer a unique possibility to study the development of the trade unions and their key topics over time, and thereby also the social development nationally and internationally. 

In the project {\it Labour's Memory. Digitization of annual and financial reports of blue-collar worker unions 1880-2020}
, we aim to collect and digitize annual and financial reports from local, regional, national and international trade union organisations from 1880 onwards. The collection is to be stored in a database, and will be made searchable for people with an interest in diachronic labour movement documents through a user portal. This is achieved in collaboration between labour history experts, archivists, computational linguists and image analysis specialists.

In this paper, we aim to investigate to which extent current state-of-the-art models for Swedish can be used to extract named entities from historical sources, a key topic for enhanced searchability in the trade union documents. Secondly, we focus on maximizing usability for the intended end product by combining the strengths of different models. Last but not least, we evaluate the performance of these models in terms of accuracy, as well as F1 score. On a more practical level, we also release a new model that is fine-tuned for NER and trained on historical Swedish text, as well as a manually annotated gold-corpus of named entities extracted from Swedish labour union documents dated between 1892 and 1974. 
%In this paper, we aim to investigate the computational linguistic aspects of this project, focusing on Named Entity Recognition techniques for enhanced searchability options and for automatic metadata generation. Since the documents in the collection span from 1880 over the 1906 spelling reform for the Swedish language \cite{reform} and onwards to present times, we focus on tools and methods used for handling old and non-standard spelling.

The choices that we made in order to optimize the results and usability of our system were made in consultations with a group of experts from the Labour's Memory project, whose competence overlaps with that of the intended user. 
 
\section{Background}
\label{sec:background}
Named Entity Recognition (NER) is the process of automatically identifying and classifying name-like entities in text, such as names of persons, organizations and locations \cite{nadeau:2007}. NER is an important subtask in many Natural Language Processing (NLP) applications, e.g., in information extraction/retrieval (see for example \citet{CanBertDigIt}) and for anonymisation/pseudonymisation of sensitive personal data in a text (e.g. \citet{Bridal1566701} or \citet{papadopoulou-etal-2022-neural}). 

For Swedish, researchers have recently worked with developing a gold standard for Swedish named entity recognition \cite{afo:20b}, trying to merge and accommodate previous NER annotation schemes used for Swedish. There are also initiatives to adapt this standard to the task of annotating named entities in historical Swedish text, where the needs and features to consider differ slightly  \cite{borin-etal-2007-naming}.

Outside the topic of NER itself, it is important to acknowledge that Swedish is still a low resource language, which does not have the same large-scale NLP infrastructure as other high-resource languages such as English, Spanish or Chinese. This is evident in terms of data sets, language models and tools, and even more so in the case of historical Swedish text. While there are efforts currently being made to build large language models for Swedish by organizations such as AI Sweden,\footnote{\url{https://www.ai.se/en}} or historical resources from the side of SWE-CLARIN (e.g. \citet{swe-diachronic}), it is still a tough undertaking to achieve high benchmark scores for NLP tasks on Swedish.



\section{Method}
\label{sec:method}

The aim of our work is to perform Named Entity Recognition (NER) for trade union documents from the late 1800s and onwards, with the goal of enhancing searchability in the documents by automatically extracting metadata on persons, locations, organisations, events etc. An important subtask is therefore to adapt our tools to handling historical text, which is further elaborated on in Section \ref{sec:historical}. We move on by describing the different language models we use for the NER task in Section \ref{sec:models}. In Section \ref{sec:evaluation}, we introduce the evaluation method that we use, and the gold standard created for this purpose. 

\subsection{Handling historical text}
\label{sec:historical}
With the aim of improving the performance of our chosen language models on historical text, we apply several pre-processing steps in order to modernize the original text and bring it closer to the kind of text the readily-available models were originally trained on, as illustrated in Figure \ref{figure:workflow} and further described below.


\begin{figure*}[ht]
\begin{center}
\includegraphics[scale=0.32]{Figures/workflow-new.png}
\end{center}
\caption{Handling historical texts among the labour movement documents.} 
\label{figure:workflow}
\end{figure*}

%\href{http://www.overleaf.com}{Something Linky}
The first step in our pipeline concerns abbreviations. The abbreviations used in historical times do not always follow the same standards as present-day abbreviations, meaning that NLP tools trained on contemporary sources may be confused by these. Therefore, we use a dictionary of abbreviations taken from Swedish family history sources\footnote{\url{https://www.familysearch.org/en/wiki/Sweden\_Abbreviations\_in\_Family\_History\_Sources}} to automatically expand as many abbreviations as possible. 
Since our data contains a large amount of names in the form of \textit{initial + surname} (e.g. {\it F. Linden}), we remove one-letter abbreviations from the list of abbreviations, in order to avoid confusion. For example, the list contains abbreviations for Swedish counties using one upper-case letter, such as 'F' for {\it Jönköping}, which would coincide with the 'F' in {\it F. Linden}, so expanding these would be counterproductive. 

In the second step, we use the dictionary {\it Ordbok Öfver Svenska Språket} by Dalin to map historical Swedish spellings to their contemporary counterpart. This dictionary is available in digital format, with over 62,000 entries of words with their historical spelling mapped to its modern version \cite{dalin2011}. 

Lastly, we perform spelling normalization of the words not covered by the Dalin dictionary. Spelling normalization is the process of automatically transforming historical spelling to a more modern, standard spelling. This can be done in several ways. In this paper, we choose to use a rule-based approach, with rules based on the Swedish spelling reform in 1906 and previously implemented in \citet{pettersson2016}. The motivation for using this approach, is that since the documents are not several hundreds of years old, the spelling differences are rather modest and assumingly pretty well covered by a rule-based approach. Furthermore, the rules are already defined and described, and thereby easy to include in our pipeline.

Throughout the rest of the paper, when we mention normalization, we refer to the inclusion of all the pre-processing steps described above with the purpose of normalizing the original text. 

\subsection{NER modelling}
\label{sec:models}
After the historical texts have been pre-processed, the actual NER process takes place, through the use of language models. We try three off-the-shelf models for contemporary Swedish, and one model trained on historical Swedish text, as further described below.

As our point of departure, we choose an off-the-shelf model for contemporary Swedish developed by spaCy.\footnote{\url{https://spacy.io/}} SpaCy is an open-source NLP software library which provides language models for over 65 languages, for a wide array of practical applications. In our case, we had three different pipelines to choose between which all can perform NER for Swedish. We ended up going for \textit{sv\_core\_news\_lg}, as that one had slightly better reported F1 score for the task at hand in comparison to the other two. This model is trained on data from news and media sources, as well as the Stockholm-Umeå corpus, v3.0 (SUC3), a balanced corpus of texts from different genres \cite{SUCcorpus}. 

The second model we try is also built on the spaCy infrastructure, but produced by the Swedish National Library.\footnote{\url{https://github.com/Kungbib/swedish-spacy}} The training data for this model largely overlaps with that of the previously mentioned model, but the reported F1 score is 4 percentage points higher. 

The third model was selected after further investigating the work done by the NLP lab (i.e. KB lab) at the Swedish National Library. It is an updated version of the aforementioned model that makes use of Hugging Face\footnote{\url{https://huggingface.co/}} and their transformer architecture. For this model, they are experimenting with Hyper Parameter Optimization (HPO) leading to additional increases in F1 score for NER, reaching up to  91\%.

Due to the fact that language models for historical text are in short supply, even more so when it comes to languages such as Swedish, there is no readily available model that can perform NER for historical Swedish text, to the best of our knowledge. In order to overcome this, we try a recently developed BERT model for historical Swedish text and fine-tune it using the same SUCX 3.0 corpus as the model developed by KBLab, so that we can run the same NER experiments and compare accordingly. The original model is released by the National Archives of Sweden and is using  data from the 15th up to the 19th century, as well as the Hugging Face ecosystem. Our fine-tuned NER model is freely available to the public on Hugging Face.\footnote{\url{https://huggingface.co/crina-t/histbert-finetuned-ner}} The self-reported training statistics for our model are available in Appendix A.

For the sake of readability, we will refer to these models by an acronym for the rest of the paper, as follows:\\
\begin{compactdesc}

    \item [DEF] the default spaCy model for Swedish
    \item [BIB] the spaCy model built by the Swedish National Library (i.e. Kungliga Biblioteket in Swedish)
    \item [KB] the Hugging Face model developed by the KBLab group at the Swedish National Library
    \item [RA] the model developed by the Swedish National Archives (i.e. Riksarkivet in Swedish) and fine-tuned for NER by us\\
\end{compactdesc}

We partially summarize the attributes of all the models that we experiment with in Table \ref{tbl:models}.
\begin{table}[ht]
\small
\begin{tabular}{l|l|l|l}
\textbf{Model} & \textbf{Platform}    & \textbf{Time}                                                         & \textbf{NER corpus} \\
\hline
\hline
DEF   & spaCy       & Contemporary                                                       & SUC3.0     \\
BIB   & spaCy       & Contemporary                                                       & SUC3.0     \\
KB    & Hugging Face & Contemporary                                                       & SUCX 3.0   \\
RA    & Hugging Face & \begin{tabular}[c]{@{}l@{}}15th-19th \\ century\end{tabular} & SUCX 3.0  
\end{tabular}
\caption[models]{\label{tbl:models} Summary of the models we use for NER.}
\end{table}


\subsection{Evaluation}
\label{sec:evaluation}
%Describe and motivate the creation of gold standard. 
To be able to compare the performance of language models on equal grounds, we create a gold standard dataset that is manually labelled by a human annotator, and validated by a second annotator in order to settle eventual uncertainties. For the annotation, we use the same standard as presented by \citet{borin-etal-2007-naming}. We use a total of 8 labels, as follows:\\
    
\begin{compactdesc}
    \item [PRS ``person''] names of people
    \item [LOC ``location''] names of locations and other types of geographical entities
    \item [TME ``time''] temporal expressions
    \item [EVN ``event''] well-known events and celebrations
    \item [ORG ``organisation''] names of corporations and other kinds of organisations
    \item [WRK ``work of art''] names of movies, sculptures, periodicals etc.
    \item [MSR ``measure''] numerical expressions, such as monetary expressions or distances 
    \item [OBJ ``artifact''] names of food/wine products, vehicles etc.\\
\end{compactdesc}


For the gold corpus, we select a total of 50 pages of sample text. In order to account for the shift in spelling conventions introduced through the 1906 Swedish spelling reform \cite{reform}, we select 25 pages which are dated before 1906, and the remaining 25 pages from years dated post-reform. To the best of our ability, we attempt to make sure that these pages are equally spaced out in terms of time elapsed between the dating of each one, as well as that they contain a reasonable body of text (i.e. at least half a page), and not just a few lines. 

It can be noted that quite general phrases, not referring to a well-defined point in time, such as {\it under året} 'during the year' or {\it de senaste åren} 'in recent years' are labelled as time expressions ('TME') by most models. However, we choose to omit them from our analysis since our group of experts deem them irrelevant. We therefore only keep those that contain numerical expressions and/or names of months or their respective acronyms (e.g. {\it December} or {\it dec}). We take a similar approach when it comes to other kinds of named entities as well in the cases where they are too vague and do not point to a specific, individual entity (e.g. {\textit{förbund} 'trade union' is too vague, but {\it Typografförbundet} 'Typographers' Union' would be included in the manual annotation).

Other than the aforementioned 50 pages, we annotate an additional 10 pages from the same time span as the original gold standard (i.e. 1892--1974). We do this in order to be able to evaluate our final hybrid approach on unseen data so that we can more accurately assess its performance, following the same principles for data selection and annotation as the gold corpus. 

In total, our gold corpus contains 570 manually annotated entities, plus an additional 85 entities in the test set, which we summarize by label in Table \ref{tbl:gold}. Both of these are freely available to the public on Hugging Face.\footnote{\url{https://huggingface.co/datasets/crina-t/UnioNER}} We mention here that a total of 35 entities representing names of people were replaced with a placeholder in the released version of the corpus at the request of the archive in order to comply with their privacy policy. The documents containing placeholders are clearly pointed out in the description of the dataset.  


\begin{table}[ht]
\begin{tabular}{l|c|c}
\textit{\textbf{}} & \multicolumn{1}{l}{\textbf{Gold set}} & \multicolumn{1}{l}{\textbf{Test set}} \\
\hline
\hline
EVN                & 19                                    & 6                                     \\
LOC                & 86                                    & 19                                    \\
MSR                & 97                                    & 7                                     \\
ORG                & 71                                    & 13                                    \\
PRS                & 162                                   & 17                                    \\
TME                & 134                                   & 22                                    \\
WRK                & 1                                     & 1                                    
\end{tabular}
\caption[gold]{\label{tbl:gold} Label count for manually annotated entities.}
\end{table}



%Describe the different types of matches we identify and motivate. Give examples
When comparing the gold standard with the automatically extracted entities, we identified some consistent differences between the different kinds of matches we encountered. For this reason, we create 8 distinct categories to differentiate between them in the evaluation phase. While this is a more fine-grained evaluation when compared to what is more widely used in the field (e.g. \citet{muc1993}, \citet{conll2003} or \citet{semeval}), we believe that our evaluation schema can greatly help in easily identifying the source of prediction errors generated by the model. We define and exemplify these categories below:\\

\begin{compactitem}
    \item \textbf{EXM} -- exact match\\
     Both the entity and the label overlap exactly between the model and the annotator.
    
    \item \textbf{PAM} -- partial match\\
    A substring of the gold standard entity is identified by the system, with the same label -e.g. {\it J.E Blomkvist} (PRS) in the gold standard, while the system outputs {\it Blomkvist} (PRS).  

     \item \textbf{ENM} -- entity match\\
    The exact same string is annotated by both the annotator and the system, with different labels (e.g. manually annotated {\it Harg} (LOC) is output as {\it Harg} (PRS) by the system). 
    
    \item \textbf{VAM} -- vague match\\
    The system predicts part of the gold standard entity, but with a different label - e.g. the annotator would label {\it E Lund} (PRS), while the system outputs {\it Lund} (LOC).
    
    \item \textbf{COM} -- compound match\\
    The system merges several entities that the annotator identified as being separate units. E.g. {\it Hilmer Johansson} (PRS) and {\it Ernst Hörngren} (PRS) in the gold standard are predicted as {\it Hilmer Johansson Ernst Hörngren} (PRS).
    
    
    \item \textbf{SPM} -- split match\\
    The system splits an entity from the gold standard into separate entities (e.g. gold standard {\it Uppsala Typografiska Förening} (ORG) vs.	the automatically annotated {\it Uppsala} (LOC) and {\it Typografiska Förening} (ORG), referring to the Uppsala branch of the Typographers' Union).   

    \item \textbf{FP} -- false positive\\
    The system labels a unit that is not in fact a named entity.%, such as {\it Förbundsstadgarna} (WRK) 'the union regulations'.
    
    \item \textbf{FN} -- false negative \\
    The system fails to identify an entity that the annotator has identified.
    

\end{compactitem}
%All the examples given above are extracted from the results.


\section{Results}
\label{sec:results}

In order to identify which approach gives us the best results, we calculate the accuracy of each model, and in doing so we also look at the count for each type of match per individual model. By accuracy we mean the percentage of entities from the gold standard which have a counterpart in the output predicted by the model, regardless of the type of match. The matches we look at are detailed in Section \ref{sec:evaluation}, and we do not include the counts for FP and FN in calculating accuracy. We also calculate the (accuracy) count for all the different kinds of matches, as well as NER labels, which we present in Figures \ref{fig:KB} through \ref{fig:HYB_NORM_labels}. We calculate both these metrics first on the original text, then on the normalized version of the text in the gold standard. Our goal is to reach as many exact matches as possible (at best) and to minimize the number of cases where the model returns no match for a given entity in the gold standard (FN). 

As a secondary step, we provide calculations for F1 score as applied to our text, since F1 score is a widely used metric for NER, and which also accounts for the FP and FN tags in our case. We present our results for F1 score in Section \ref{f1}

\subsection{Baseline}
\label{subsec:baseline}
After applying the default spaCy model (DEF) to historical text, we obtain an accuracy of only 57.54\%, due to the fact that the number of entities that are left unannotated (i.e. FN) by the DEF model exceeds the number of exact matches. In this case, normalization is detrimental to the model's performance, bringing accuracy down to 51.57\%. Given the fact that more than half of our entities are not recognized by the system, we opt out of using this model further.
 

%\begin{figure}[t!]
%    \centering
%    \includegraphics[width=1\columnwidth]{Figures/DEF+NORM.png}
%    \caption{Accuracy for the default SpaCy model. DEF = applied to  original text, DEF\_NORM = applied to  normalized version.}
%    \label{fig:DEF}
%\end{figure}

Moving onto the next spaCy model developed by the Swedish National Library, we immediately notice an improvement in performance. The BIB model predicts significantly more exact and partial matches than its predecessor, while the occurrence of false negatives is also decreased by almost 30\% when compared to the DEF model, leading to an overall accuracy of 69.64\%.

Normalization does help in this case, but it is barely enough to get the model over the 70\% mark. We suspect that this jump in performance between the DEF and the BIB model is due to the fact that the DEF model was trained mainly on news and media text, while the BIB model was trained on a more balanced corpus of text from different sources. While this is clearly an improvement, it is not  satisfactory enough to motivate using this model for our purposes.

%\begin{figure}[t!]
%    \centering
%    \includegraphics[width=1\columnwidth]{Figures/BIB+NORM.png}
%    \caption{Accuracy for the spaCy model from the Swedish National Library. BIB = applied to  original text, BIB\_NORM = applied to normalized version.}
%    \label{fig:BIB}
%\end{figure}


The Swedish National Library KB model \cite{KBmodel} is an updated version of the previous BIB model. In comparison with its predecessor, the KB model, while not much better at extracting exact matches, does help decrease the number of FNs, as demonstrated in Figure \ref{fig:KB}. This happens as a result of the increase we see when it comes to partial matches, and at a smaller level in entity matches, vague matches, compound matches and split matches. The overall accuracy for the model lands at 77.34\%, with normalization ebbing this number by only 0.29\%. 

\begin{figure}[t!]
    \centering
    \includegraphics[width=1\columnwidth]{Figures/KB+NORM.png}
    \caption{Accuracy count for the Hugging Face model from the Swedish National Library. KB = original text, KB\_NORM = normalized text.}
    \label{fig:KB}
\end{figure}

Lastly, we evaluate the performance of our fine-tuned RA model, which is based on a model created by the National Archives of Sweden and trained exclusively on historical data, but fine-tuned for NER on the same SUCX3.0 corpus as the previously evaluated KB model. In this case, we do expect an increase in performance due to the fact that the model is trained on data from the same time period as our gold standard corpus, as opposed to the aforementioned models which are all trained on contemporary text. 

When conducting the analysis of this model, we were surprised to see that the number of exact matches dropped significantly. Even more surprising is the fact that despite this, the model shows the least amount of false negatives among all the models we evaluated, which means that it manages to capture (to some extent) about 80\% of the entities from the gold corpus. After normalization, the overall accuracy of the model reaches up to 83.41\%, which is the highest we were able to reach in our experiments. However, it is worth noting that this high accuracy does not account for the doubled amount of false positives compared to previous models, or the staggering increase in split matches. We investigate this further in Section \ref{subsec:closer}.

\begin{figure}[ht!]
    \centering
    \includegraphics[width=1\columnwidth]{Figures/RA+NORM.png}
    \caption{Accuracy count for the model from the Swedish National Archives. RA = original text, RA\_NORM = normalized text.}
    \label{fig:RA}
\end{figure}



We summarize the accuracy of each model in Table \ref{tbl:accuracy}.

\begin{table}[ht]
\begin{tabular}{l|l|l}
\textbf{Model} & \textbf{Original} & \textbf{Normalized} \\
\hline
\hline
DEF            & 57.54\%           & 51.57\%             \\
BIB            & 69.64\%           & 70.35\%             \\
KB             & 77.34\%           & 77.05\%             \\
RA             & 82.60\%           & 83.41\%            \\
\end{tabular}
\caption[models]{\label{tbl:accuracy} Overall accuracy for each NER model.}
\end{table}


\subsection{A closer look}
\label{subsec:closer}
After the evaluation described in Section \ref{subsec:baseline}, it is clear that our two front runners are the KB and the RA models. Were it not for the notable increases in split matches and false positives, the RA model would take precedence, but as it stands, it is worth investigating more in-depth what could be causing these fluctuations. We therefore take a closer look at the way it performs for each individual label, while also extracting the same statistics for the KB model as a point of comparison. 

Figure \ref{fig:KB_labels} shows how the KB model consistently extracts more exact matches for all different label types, with partial matches being a close second. Same pattern is visible after normalization as well, which can be observed in Figure \ref{fig:KB_NORM_labels}, with slight improvements in exact matches for the person (PRS) and organisation (ORG) labels.
\begin{figure}[t!]
    \centering
    \includegraphics[width=1\columnwidth]{Figures/KB_labels.png}
    \caption{Accuracy count per label for the Hugging Face model from the Swedish National Library, applied to the original text.}
    \label{fig:KB_labels}
\end{figure}

\begin{figure}[ht!]
    \centering
    \includegraphics[width=1\columnwidth]{Figures/KB_NORM_labels.png}
    \caption{Accuracy count per label for the Hugging Face model from the Swedish National Library, applied to the normalized version of the text.}
    \label{fig:KB_NORM_labels}
\end{figure}

In the case of the RA model, there is a clear decrease in the number of exact matches across labels, and significantly more split matches, as shown in Figure \ref{fig:RA_labels}. Among all the labels, time expression (TME) is the one that is most affected by the high number of split matches, which we suspect is due to a tokenization issue. There is a high chance that, since the original model is trained on a smaller corpus, there were not enough numerical values for the model to know how to handle years, dates etc. From Figure \ref{fig:RA_NORM_labels}, we can see that the trend remains the same even after normalization, which reinforces the theory that the errors are stemming from the tokenization process. 

\begin{figure}[ht!]
    \centering
    \includegraphics[width=1\columnwidth]{Figures/RA_labels.png}
    \caption{Accuracy per label for the model from the Swedish National Archives, applied to the original text.}
    \label{fig:RA_labels}
\end{figure}

\begin{figure}[ht!]
    \centering
    \includegraphics[width=1\columnwidth]{Figures/RA_NORM_labels.png}
    \caption{Accuracy count per label for the model from the Swedish National Archives, applied to the normalized version of the text.}
    \label{fig:RA_NORM_labels}
\end{figure}




%\subsection{A chronological perspective}

%Furthermore, we want to investigate the effect of normalization throughout time in order to see if it has any negative effects for documents from more recent years. As previously stated, the Swedish spelling reform was enforced in 1906, but it could be safe to assume that the old spellings trickled their way into the following years - maybe even decades. It is however not a straightforward question to set such an exact cutoff point for when to stop normalizing, which is why we wanted to look at the accuracy for each document throughout the years in order to assess its practical impact. 

%While it is safe to say that normalization does affect the performance of the model, it is hard to find a pattern for a presupposed negative impact of normalization, as the results fluctuate regardless of the anchoring point that is 1906.

%We provide exact figures for this in Appendix B.

%In Figure \ref{fig:KB_chronological} we present the accuracy for the KB model for each page in the gold standard, in chronological order. While it is safe to say that normalization does affect the performance of the model, it is hard to find a pattern for a presupposed negative impact of normalization, as the results fluctuate regardless of the anchoring point that is 1906. 

%Figure \ref{fig:RA_chronological} shows a similar analysis as applied to the RA model. In this case, it is even more difficult to discern a pattern as the normalization shows even more of a positive impact, even as late as 1969.




\subsection{A hybrid approach}
After looking in-depth at the strengths and weaknesses of the KB and RA models, we want to investigate to which extent combining their outputs could benefit the end results. More specifically, we aim to avoid overgenerating split matches in the RA model and to try to increase the accuracy of the KB model. For this reason, we prioritize high counts of exact and partial matches, and take specific labels from the RA model (PRS, ORG), merging them with the rest of the labels from the KB model.



By doing this hybrid approach (HYB), we reduce the number of false negatives that were initially present in the KB model, and we also manage to drop the number of split matches that were problematic for the RA model, as shown in Figure \ref{fig:HYB}. 

From Figures \ref{fig:HYB_labels} and \ref{fig:HYB_NORM_labels}, we can clearly see that this approach is beneficial in reducing the number of split matches for those categories that are prone to having numerical expressions, such as MSR and TME, and which the RA model could not handle very well. 

For accuracy, we obtain an increase from the KB model, reaching 79.82\% on the original text, which drops slightly after normalization - to 79.47\%.

\begin{figure}[ht!]
    \centering
    \includegraphics[width=1\columnwidth]{Figures/HYB+NORM.png}
    \caption{Accuracy count for the hybrid approach. HYB = original text, HYB\_NORM = normalized version text.}
    \label{fig:HYB}
\end{figure}

\begin{figure}[ht!]
    \centering
    \includegraphics[width=1\columnwidth]{Figures/HYB_labels.png}
    \caption{Accuracy count per label for the hybrid approach, applied to the original text.}
    \label{fig:HYB_labels}
\end{figure}

\begin{figure}[ht!]
    \centering
    \includegraphics[width=1\columnwidth]{Figures/HYB_NORM_labels.png}
    \caption{Accuracy count per label for hybrid approach, applied to the normalized version of the text.}
    \label{fig:HYB_NORM_labels}
\end{figure}

%\section*{Ethics Statement}
%Scientific work published at EACL 2023 must comply with the \href{https://www.aclweb.org/portal/content/acl-code-ethics}{ACL Ethics Policy}. We encourage all authors to include an explicit ethics statement on the broader impact of the work, or other ethical considerations after the conclusion but before the references. The ethics statement will not count toward the page limit (8 pages for long, 4 pages for short papers).

\subsection{F1}
\label{f1}
As a last evaluation step, we calculate F1 score for the KB, RA and HYB models using seqeval \cite{seqeval}, which is presented in Table \ref{tbl:F1}. The reason why we do this is two-fold - on the one hand, we want to assess performance on new, unseen data, while on the other hand we want to see how the score is affected by a metric that accounts for false negatives and false positives. 

\begin{table}[ht]
\begin{tabular}{l|r|r}
\textbf{Model} & \multicolumn{1}{l}{\textbf{Gold}} & \multicolumn{1}{l}{\textbf{Test}} \\
\hline
\hline
KB                   & 75.68\%                           & 66.23\%                           \\
KB\_NORM             & 76.79\%                           & 71.14\%                           \\
RA                   & 71.23\%                           & 67.36\%                           \\
RA\_NORM             & 71.68\%                           & 66.66\%                           \\
HYB                  & 76.25\%                           & 68.23\%                           \\
HYB\_NORM            & 76.66\%                           & 71.85\%                          
\end{tabular}
\caption[models]{\label{tbl:F1} F1 score for the KB, RA and HYB models, before and after normalization, from the gold corpus as well as the test set.}
\end{table}

Not surprisingly, the RA model performs the worst in this case, due to the high volume of false positives that it predicts. Even though it has a lower false negative count, this is not enough to counterbalance the effect of PAM where the predicted entities did not match the gold standard entirely, or the split entities where the gold entity span was split into several different ones. 

It is however interesting to see that the HYB model overtakes the KB model on almost all accounts, and while normalization did not help for accuracy, it does increase F1 score in this case. The HYB model with normalization manages to obtain the highest score on the test set at 71.85\%.

\section{Discussion}
\label{sec:discussion}
Our study focuses on the application of NER to historical Swedish text, specifically documents sourced from Swedish labor unions dating back to the 19th and 20th centuries. The primary challenge lies in adapting contemporary state-of-the-art NER systems to effectively process and extract entities from historical text, which often differs significantly in linguistic norms, vocabulary, spelling, and syntactic structures from contemporary Swedish.

Our research delves into a comparative analysis of multiple language models applied to NER for historical Swedish text. Three off-the-shelf models designed for contemporary Swedish text were experimented with, alongside a custom-built language model trained on historical Swedish text. This unique approach allowed us to explore the adaptability of existing models and assess the feasibility of fine-tuning historical language models for NER tasks. Through our experiments, we show that current off-the-shelf models have the capability to extract named entities from historical text, but at the same time they can benefit from training on historical data, as shown by the high accuracy of our RA model. 

Moreover, we believe that the inconsistent effect of the normalization rules could be partly due to the rather small amount of normalization rules, as well as the nature of a rule-based approach to spelling normalization, where it is hard to write efficient rules without risking overgeneration. Another, more data-driven approach, might have given more consistent results. 


It is also important to keep in mind that our evaluation metrics were customized according to the needs of our future users.
Since our target groups are looking for as many named entities as possible, we attempt to adapt our approach in order to maximize the usability for the end product -  which is the archival database of the Labour's Memory project, while at the same time maintaining a good level of quality for the automatically extracted named entities.

For future work, we would like to investigate the way different data augmentation methods can improve our results, since previous work done on English text shows promising results when it comes to applications on pretrained transformer models (see, for example, \citet{data_augmentation_2020}), such as the RA model we propose in this paper. Moreover, given the fact that our source material comes from labour union documents, it could also be interesting to look at a more fine-grained analysis of the PRS label in order to be able to identify potential biases in NER systems - similar to the work conducted by \citet{social-groups-2023} for Danish text. 

%Possible improvements - social groups\citet{social-groups-2023}, data augmentation\citet{data_augmentation_2020}  
 
\section{Conclusion}
\label{sec:conclusion}

In this paper, we show that current off-the-shelf models for Swedish can perform NER on historical text, but using a historical language model shows more promising results. However, data from historical sources could also be beneficial for training in order to achieve better F1 score and reduce errors. An alternative path we would like to explore in the future is training on multilingual data from other Scandinavian languages, given that multilingual models show great promise when it comes to cross-lingual transfer learning (see, for exasmple, \citet{multilingual-findings} or \citet{Katsarou1618310}), with the added bonus that Scandinavian languages have similar vocabulary and structure. 

A significant contribution of this study lies in the release of the newly trained RA model tailored for NER of historical Swedish text. Additionally, we introduce a manually annotated corpus comprising over 650 named entities, offering a valuable resource for future research endeavors. We also show that combining the strengths of multiple models can be beneficial to our NER task. %and aids in finding a satisfactory middle ground for use within a search platform for labour union documents.  
%Potentially fine-tuning on a historical corpus of named entities. 

%Lastly, while we do show that spelling nomalization can affect the performance of NER systems, it is hard to find a reliable pattern of its impact on our data. We believe that experimenting with a more machine learning focused approach instead of the rule-based one in this present study can show more robust results.

In conclusion, while our study provides valuable insights and tools for NER in historical Swedish text, it also underscores the necessity for further advancements and novel methodologies to address the challenges posed by data sparsity in low-resource languages.

\section*{Acknowledgements} 
%\textcolor{red}{We need to remove the acknowledgements to preserve anonymity during the review process}
We would like to extend our heartfelt gratitude to our colleagues in the Labour's Memory project for their invaluable historical expertise and support. 

This work has been supported by Riksbankens Jubileumsfond, grant  IN20-0040, \textit{Labour's Memory. Digitization of annual and financial reports of blue-collar worker unions 1880-2020}.

% Entries for the entire Anthology, followed by custom entries
\bibliography{anthology,custom}
\bibliographystyle{acl_natbib}


\newpage
\appendix
\onecolumn
\section*{Appendix A. Self-reported raining results for the RA model}
\label{sec:appendix}
\begin{table}[ht]
\begin{tabular}{l|l|l|l|l|l|l|l}
\textbf{Training Loss} & \textbf{Epoch} & \textbf{Step} & \textbf{Validation Loss} & \textbf{Precision} & \textbf{Recall} & \textbf{F1} & \textbf{Accuracy} \\
\hline
\hline
0.0403                          & 1.0            & 5391          & 0.0316                   & 0.8496             & 0.8866          & 0.8677      & 0.9903            \\
0.0199                          & 2.0            & 10782         & 0.0308                   & 0.8814             & 0.9034          & 0.8923      & 0.9915            \\
0.0173                          & 3.0            & 16173         & 0.0372                   & 0.8698             & 0.9197          & 0.8940      & 0.9913            \\
0.0066                          & 4.0            & 21564         & 0.0397                   & 0.8783             & 0.9239          & 0.9005      & 0.9921            \\
0.0029                          & 5.0            & 26955         & 0.0454                   & 0.8855             & 0.9181          & 0.9015      & 0.9923            \\
0.0035                          & 6.0            & 32346         & 0.0454                   & 0.8834             & 0.9211          & 0.9019      & 0.9922            \\
0.0009                          & 7.0            & 37737         & 0.0495                   & 0.8784             & 0.9261          & 0.9017      & 0.9922           
\end{tabular}
\end{table}

%\section*{Appendix B. Normalization through time}
%\begin{figure}[t!]
%    \centering
%    \includegraphics[width=1\columnwidth]%{Figures/KB_chronological.png}
   % \caption{Accuracy per document for the Hugging Face model %from the Swedish National Library. KB = applied to  %original text, KB\_NORM = applied to normalized version.}
  %  \label{fig:KB_chronological}
%\end{figure}

%\begin{figure}[t!]    \centering
%    \includegraphics[width=1\columnwidth]%{Figures/RA_chronological.png}
%    \caption{Accuracy per document for  model from the Swedish %National Archives. RA = applied to  original text, %RA\_NORM = applied to normalized version.}
%    \label{fig:RA_chronological}
%\end{figure}


\end{document}
