% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
\usepackage{EACL2023}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{graphicx}
\usepackage{multirow}
\usepackage{booktabs}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}


% If the title and author information does not fit in the area allocated, uncomment the following
%
\setlength\titlebox{7cm}
%
% and set <dim> to something 5cm or larger.

\title{Perplexing Canon: \\ A study on GPT-based perplexity \\ for canonical and non-canonical literary works}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Yaru Wu \\
  Uppsala University\\
  \texttt{yaru.wu.6038@student.uu.se} \\\And
  Pascale Feldkamp Moreira \\
  Center for Humanities Computing\\
  Aarhus University\\
  \texttt{pascale.moreira@cc.au.dk} \\\AND
 Kristoffer L. Nielbo \\
  Center for Humanities Computing\\
  Aarhus University\\
  \texttt{kln@cas.au.dk} \\\And
 Yuri Bizzoni \\
  Center for Humanities Computing\\
  Aarhus University\\
  \texttt{yuri.bizzoni@cc.au.dk} \\}

\begin{document}
\maketitle
\begin{abstract}
This study extends previous research on literary quality by using information theory-based methods to assess the level of perplexity recorded by three large language models when processing 20th-century English works deemed to have high literary quality, recognized by experts as canonical, compared to a broader control group. %In this context, ``quality'' refers to texts . 
We find that canonical texts appear to elicit a higher perplexity in the models and we explore which textual features might concur to create such effect. We find that the usage of a more heavily nominal style, together with a more diverse vocabulary, is one leading cause for the difference between the two groups. These traits could reflect ``strategies'' to achieve a more informationally dense literary style in the canonical groups.
\end{abstract}

\section{Introduction}
The question of what ``literary quality'' is has been at the center of a millennia-long debate in aesthetics and literary studies.%, although recent developments in quantitative corpus studies are making it possible to approach the topic from a completely new perspective  
%The challenge arises from the diversity of narratives, the difficulty of determining a ``gold standard", and the extensive range of features potentially relevant in literary works. 
While literary judgment is almost by definition subjective, reflecting individual reader preferences, quantitative studies have shown that such preferences tend to converge at the large scale, %, and qualitative and quantitative studies have shown 
and that both textual features, like coherence and style \cite{bizzoni_sentimental_2023, bizzoni_good_2023, riddlers, cranenburgh_data-oriented_2017, archer_bestseller_2017}, and text-extrinsic features, such as reader or critic demographics \cite{lassen_reviewer_2022, koolen_reading_2018, wang_success_2019}, significantly influence appreciation or perceived quality. 
%Thus, current research focuses on two main questions: whether literary quality can be defined, and what features affect its perception. 
Most schools of thought in literary research tend to see literary quality as either a perceived quality -- an effect of reception and cultural dynamics \cite{bourdieu_field_1993, casanova_world_2007, guillory_cultural_1995} -- or as the effect of certain textual features, %, such as particularly ``literary'' textual features, 
such as, among others, authorial strategies of defamiliarization and foregrounding \cite{shklovsky_art_1917,mukarovsky_standard_1964, peer_quality_2008, attridge_singularity_2004}. While consensus on a single gold standard of quality is hard to achieve \cite{bizzoni_predicting_2022}, reader preferences and expert valuations can offer a range of measurable levels of appreciation.
An often discussed dimension of literary quality is that of the so-called ``literary canon'', a complex concept generally representing a set of works that have survived or/and (by the same token) remain distinguished in the memory of a literary culture \cite{bloom_western_1995}. A community, usually over large periods of time, defines as outstanding and worthy of attention; yet this process is not devolved to any individual authority, which makes the very definition of what is within the canon complex. 
As such, the canon is often scrutinized in cultural approaches to literary quality. Some schools of thought have seen it as representing nothing but entrenched interests \cite{von_hallberg_editors_1983} and thus as the cultural capital of ruling classes \cite{guillory_cultural_1995}, while others have considered ``canonic'' works to excel in terms of intrinsic features \cite{bloom_western_1995}, whether stylistic \cite{brottrager_modeling_2022, barre_operationalizing_2023, algee-hewitt_canonarchive_2016}
or narrative \cite{bizzoni_fractality_2023}. 
%Canonicity may seem to be an intrinsic quality of texts and more easily modelled than general popularity. 
In his work on the dynamics in the literary field, \citet{bourdieu_field_1993} placed ``popular success'' and ``consecration'' %by artists and institutions 
at opposed positions.\footnote{``There are few fields in which the antagonism between the occupants of the polar positions is more total [than in the literary]'' \cite[p.~46]{bourdieu_field_1993}.} Recent quantitative studies of the literary field often follow a similar distinction between popularity and prestige \cite{porter_popularityprestige_2018, manshel_who_2019}, where more prestigious books and genres are what we could call the more ``literary'' ones \cite{porter_popularityprestige_2018, lassen_persistence_2023}. Supporting the idea that such literariness may be distinguished by certain text-intrinsic features, %making a distinction between overall enjoyment and literary quality, 
\citet{riddlers} have shown that readers agree more on the ``literariness'' of books, and that it is easier to model literariness ratings from textual features than overall enjoyment ratings. %%Added this section

In this work, we %use a diverse set of references to better 
approximate what can be considered canonical in a large corpus of around 9,000 novels. Based on this corpus, we ask two main questions: (1) Are canonical novels more ``perplexing'', as measured through different Large Language Models, than a non-canonical control group? (2) If they are, which linguistic and stylistic features might contribute to the difference?

% \begin{enumerate}
%     \item Are canonical novels more ``perplexing", as measured through different LLMs, than a non-canonical control group?
%     \item If yes, which linguistic and stylistic features might be contributing to the difference in perplexity?
% \end{enumerate}

Our reason for using perplexity (see Chapter \ref{sec:perplexity} for the formal definition of perplexity) is at least two-fold. On one hand, canonical works of fiction are often examples of either ``virtuous'' \cite{bloom_western_1995} or defamiliarizing usage of language \cite{mukarovsky_standard_1964,peer_quality_2008}, thus an uncommon usage of language. Such characteristics may make canonical works of fiction more surprising with respect to non-canonical. Even when perplexity is operationalized as a measure of information theory, these works might elicit a higher perplexity on average.
On the other hand, perplexity is a central measure of informativity in information theory \cite{shannon1949communication}. Since perplexity is a function of surprisal, more perplexing texts tend to be more informationally dense. A highly specialized scientific paper -- like a highly complex and articulate page of James Joyce -- is unusually informative in the sense that it constantly relays novel information (highly specialized or new words -- neologisms -- or words in a new order) to the reader. In theory, a communicative system that manages to be as dense as possible without breaking down or being ``too dense'' for its own readers indicates elements of a heightened communicative efficiency -- a feature that communities might tend to optimize over time \cite{rubino-etal-2016-modeling, biber2011historical}. 


% In Section 2 with present a short overview of the main quantitative studies carried on the topic of literary quality. In Section 3 we present our corpus and the way we selected for canonical works. Section 4 presents a short explanation of perplexity as our measure of perplexity and in Section 5 we outline the three Large Language Models that we use for this study. Section 6 presents the differences between canon and non-canon in terms of surprisal. Finally, in Section 7 we report on a series of analyses about which linguistic features might have a role in the perplexing nature of canonical literature. Section 8 presents our conclusions and future works.  




\section{Related works}
Studies seeking to predict literary success or perceived literary quality have often followed the intuitive idea that readers perceive a difference between more difficult and easier texts, and approximate some form of stylistic complexity. Such studies use features related to the readability indices developed in linguistics research, such as sentence length, vocabulary richness, or redundancy \cite{brottrager_modeling_2022, cranenburgh_data-oriented_2017, crosbie_2013, riddlers, maharjan_multi-task_2017, algee-hewitt_canonarchive_2016}. Additionally, readability formulas %hold significance in creative writing and publishing. They 
find integration in editing tools such as the Hemingway or Marlowe applications,\footnote{See \href{https://hemingwayapp.com/help.html}{https://hemingwayapp.com/help.html}, \href{https://authors.ai/marlowe/}{https://authors.ai/marlowe/}} which prioritize more ``readable'' texts. 
Yet the relation between stylistic aspects of text complexity and reader appreciation appears complex: %for at least two reasons: one, there is a wealth of possible features to implement which do not have a concrete theoretical background, % pointing to . Second, 
%and two, studies of reader appreciation% on what is more or less appreciated by readers often show contrasting results.
%For example, 
while it is suggested that readers prefer more stylistically complex or informationally dense texts \cite{algee-hewitt_canonarchive_2016}, it is a widespread conception that bestsellers are easier to read \cite{martin_1996}. In literary studies, reading ease has also been proposed as a marker of “better” style as far back as 1893 \cite{sherman1893analytics}. 
While \citet{martin_1996} and \citet{maharjan_multi-task_2017} found that readability formulas were weak for predicting reader appreciation, more recent work has shown that preference for the type of text difficulty measured by readability formulas may vary across different audiences: novels with higher readability are preferred by raters on large online platforms, while award-winning novels tend to have lower scores 
\cite{bizzoni_good_2023}. 
Measures that are more explicitly related to information density or entropy, such as word and bigram entropy \cite{algee-hewitt_canonarchive_2016}, surprisal \cite{mcgrath_measuring_2018}, and text compressibility \cite{baechler_information-theoretic_2016} have also been used to assess the complexity of literary texts.\footnote{In the latter case, the aim is to approximate Kolmogorov complexity, i.e., the complexity of e.g. a string is defined as the length of the shortest possible description of it, as in \citet{baechler_information-theoretic_2016} and \citet{liddle_could_2019}.} \citet{liddle_could_2019}, for example, shows a diachronic evolution of literary texts towards a greater density of information. Surprisal has been shown to correlate with the cognitive effort of processing words \cite{hale_probabilistic_2001,levy_expectation_based_2008, balling_probability_2012} and is as such a measure of the information density of text. The connection of information density or surprisal with a text's relative ``quality'' (in this case intended as communicative effectiveness) has been linked more explicitly in studies about non-literary domains. For example, %\citet{teich2021less} and 
\citet{degaetano2022toward} found that scientific prose has gradually developed informationally denser prose, optimized for expert-to-expert communication. 


Perplexity, as a closely related measure of the probability of words in context, may be applied as another measure of difficulty or as a measure of the \textit{information density} of a text \cite{rubino-etal-2016-modeling}.
%``strangeness", unusuality and originality of expression - all features that 
While perplexity is primarily used as an internal evaluation metric for the performance of language models, it has also been used variously as a descriptive and predictive metric to distinguish between the domain and style of texts, for example between formal and colloquial tweets \cite{gonzalez_analysis_2015} or between speech production by people with dementia and without \cite{fritsch_automatic_2019}.


Like surprisal, LLMs' perplexity also shows a relationship to human word processing or perception of text difficulty, for example with gaze duration in reading \cite{goodkind_predictive_2018}, though the similarity of model perplexity to, for example, human reading time may change with larger model size \cite{oh-schuler-2021-contributions}. %bit conflicting studies here, not sure where to go - remember this ACL paper Yuri?
Still, the relation between the ``difficulty'' level of a text and perplexity is not clear-cut, and perplexity seems to capture something different than what can be estimated with traditional readability formulas from linguistics research. \citet{dellorletta_is_2020} show no relation between model perplexity and one readability measure, while \citet{martinc_supervised_2021} suggest that models might actually attribute less perplexity to texts aimed at adults compared to texts aimed at younger audiences. Similarly, there seems to be no clear connection of perplexity to stylistic features of texts connected to readability, suggesting that different textual features affect readability and model perplexity \cite{dellorletta_is_2020}.\footnote{ Perplexity appears to be estimated consistently across different (and also smaller) models \cite{goodkind_predictive_2018}}
Some work has been done to estimate surprise or narrative coherence in fiction \cite{mcgrath_measuring_2018, underwood_how_2020, wu2023predicting-key}, still the question of quality or reader appreciation of more or less ``surprising'' texts remains underexplored. In this context, perplexity may constitute an additional measure easily related to different types of reader appreciation. Notably, in text generation, model perplexity is explored to retrieve more or less diverse output, given that a higher likelihood text (with less perplexity) does not necessarily mean that it is of better quality for human raters \cite{zhang_trading_2020}.

\section{Data}

\subsection{Corpus}
We use a corpus spanning 9,089 novels published in the US between 1880 and 2000 (see Table \ref{tab:authors} and Figure \ref{fig:dist_canonworks}). It is a unique resource both in terms of size \footnote{Studies on literary quality often rely on corpora of $<$ 1,000 books \cite{ashok2013success, riddlers}.} and diversity, as it contains relatively recent novels from various genres. 
It was compiled based on the number of libraries holding each novel with a preference for higher holding numbers, i.e., for more circulated works. As library holdings reflect a diverse demand, the corpus is not homogeneous in terms of genre and features both prestigious and popular works ranging from Mystery to Science Fiction \cite{textlabscorpus}.\footnote{While the corpus has no reference publication, several other works have used the same dataset \citep{underwood2018transformation, cheng2020fleshing}. See \url{https://textual-optics-lab.uchicago.edu/us_novel_corpus} for an overview of the corpus.}

\begin{table}[h]
    \centering
    \caption{Number of titles, authors, and average titles per author in the dataset.}
    \begin{tabular}{ccc}
    \toprule
        \textbf{Titles} & \textbf{Authors} & \textbf{Titles per author} \\
    \midrule
        9089 & 3166 & 2.88 \\
    \bottomrule
    \end{tabular}
    \label{tab:authors}
\end{table}

For example, the corpus contains several National Book Award winners (including Don DeLillo, Joyce Carol Oates, and Philip Roth), as well as important works of genre-fiction (i.e., Tolkien or Philip K. Dick), influential authors of mainstream fiction (such as Agatha Christie and Stephen King), and highly canonical names (such as James Joyce and Ernst Hemingway). 
Books in the corpus vary in length, from 341 words (Beatrix Potter's \textit{The Story of Miss Moppet}) to 714,744 words (Ben A. Williams' \textit{House Divided}), though only 255 books -- 2.9\% of the corpus -- are shorter than 35,000 words -- roughly the average length of a novella like Orwell's \textit{Animal Farm}. The total word count of the corpus is 1,060,549,793 words.

\begin{figure}
    \centering
    \includegraphics[width=6.5cm, height=3.5cm]{figures/plot_canon.png}
    \caption{Distribution of canonic titles in the Chicago Corpus over time.}
    \label{fig:dist_canonworks}
\end{figure}

\subsection{Canonical novels}
The definition of ``canonical works'' used here adheres to a comprehensive principle, relying on the amalgamation of different expert perspectives on canonicity. %of what is canonical in the English literature of the 20\textsuperscript{th} century.
We considered four main sources (see Fig. \ref{fig:proxies_different} for the number of works gathered from each source and the overlap between sources):

(i) \textbf{The Norton Anthology}: This is a leading anthology dedicated to authors considered canonical \citep{pope_we_2019, ragen_uncanonical_1992}. We consulted both the English and the American Norton Anthology %, with the English version being the most widespread \cite{}.


(ii) \textbf{College Syllabi}: The frequency with which college syllabi include an author's work can measure their level of canonization \cite{barre_operationalizing_2023}. We used OpenSyllabus, a database that has compiled 18.7 million college syllabi\footnote{\url{https://www.opensyllabus.org}}. Using this data, we tallied all works in our collection from the top 1000 most frequent authors in \textit{English Literature} syllabi.


(iii) \textbf{Classics Series}: Numerous major publishers, such as Vintage and Penguin\footnote{\url{https://www.penguin.com/penguin-classics-overview/}}, feature a series dedicated to ``classic'' (e.g. canonic) literature. Given Penguin's status as a leading publisher of English-language literature \cite{alter_will_2022}, we marked all works in our corpus that featured in this series.


(iv) \textbf{Prizes}: We collected long-listed titles (winners and finalists) for prestigious literary awards: The Nobel Prize in Literature, the Pulitzer Prize, the National Book Award. \citet{manshel_who_2019} have shown that winning an award contributes to the long-term prestige -- but also popularity -- of titles in academia and on GoodReads. The choices of award-committees seem to be in touch with the general public, but prize-winning books also seem to be connected to disagreement between readers at the large scale \cite{kovacs_paradox_2014}.%Interestingly, \cite{kovacs_paradox_2014}, found that while awards may initially make a title more popular and gather more ratings on GoodReads, this may also affect a drop in average rating as the reception of a book becomes more debated.

%Novels listed in any of these aforementioned sources are considered canonical within the context of this paper. 
%We compared the resulting list of titles with crowd-compiled lists of \textit{Classics} and \textit{XX Century Classics} in GoodReads, finding an almost complete overlap. 
These sources divide our dataset in two groups: 745 canonical and 8344 non-canonical works. 
Naturally, we consider this division artificial, as a necessary rule of thumb to make the study possible. In fact, canonicity is not a defined and boolean variable \cite{barre_operationalizing_2023}, but would be best represented as a continuum on several dimensions. To contrast against these proxies, we also collected books in our corpus that are in Publisher's Weekly American 20\textsuperscript{th} century bestseller list.\footnote{Extracted from the database collected by John Unsworth at the University of Illinois: \url{https://web.archive.org/web/20111014055658/http://www3.isrl.illinois.edu/~unsworth/courses/bestsellers/picked.books.cgi}. Publishers Weekly is a trade news magazine which is published once a week (from 1872) and targeted at agents in the field: publishers, literary agents, booksellers, and librarians. Although based on sales numbers, the full set of selection criteria for the list are unknown.}

\begin{figure}[b!]
    \centering
    \includegraphics{canon_rose.png}
    \caption{Number and overlap of the canonicity sources used in this study. Note that the largest overlap appears to be between the Norton Anthology and Opensyllabus, indicating the near relation between the two proxies, possibly due to the institutional affiliation of the Norton Anthology.}
    \label{fig:enter-label}
\end{figure}


\section{Perplexity}
\label{sec:perplexity}
Perplexity is an information-theoretic measurement of how well a probability model predicts a sample \cite{goldberg2022neural}. The perplexity of a well-trained language model on a test text  can be interpreted as the exponential of its average level of surprisal \cite{hao2020probabilistic}, namely
\begin{equation}
    e^{-\frac{1}{N}\sum_{i=1}^{N}ln(P(token_{i}|tokens_{j<i}))}
\end{equation}
where $N$ is the number of tokens of the text and $P(token_{i}|tokens_{j<i})$ is the probability assigned to the $i$th token after the model has processed the first $i-1$ tokens. Thus, lower perplexities indicate that the model is less uncertain about its predictions. In information theory and linguistics, this measure is often used to approximate how surprising or complex a text can be for humans as well. As language models are trained on word sequences, perplexity has the benefit of encapsulating lexical, grammatical and syntactic phenomena alike. Strictly in this sense, perplexity-based approaches are able to model a text more holistically than approaches that focus only on one linguistic dimension.

In this study, we calculate the perplexity of three language model on the same corpus. We based all perplexity calculations on the byte pair encoding tokenization \cite{sennrich2015neural} used in the series of gpt2 models. %, which divides the whole dataset into 50,257 tokens. 
Due to constraints imposed by the maximum input length, we employed a stride-based methodology to gauge perplexity at the text level. This method incorporates a strided sliding window, wherein the context is shifted by fixed-length strides, affording the model an expansive context for making predictions at each iteration. In this specific framework, the window's size is 1024 tokens long, with a stride length of 512 tokens. By doing so, the second half of the previous context window served as the first half of a new context window to calculate perplexity estimates for the remaining 512 tokens \cite{oh2023does}. Therefore, the surprisal for each book is comprised of perplexity values for the initial 1024 tokens, intermediate segments of 512 tokens, and the residual tokens of varying lengths. The aggregate-mean value is designated as the textual perplexity within the confines of this study.  

\begin{figure}%[!htbp]
    \centering
    \includegraphics[width=0.28\textwidth]{figures/heatmap_perplexity.png}
    \caption{Spearman's correlation between the three GPT-2 based models' mean perplexity scores on each novel of our corpus}
    \label{fig:heatmap_perplexity}
\end{figure}

\section{Models}
We use three alternative, large language models to assess the average perplexity of the novels (see Table \ref{table:models} for details). For each novel, we thus have three perplexity measures. 

The two standard versions of GPT2 models \cite{radford2019language}, namely the gpt2 and the gpt2-xl, are used in this paper since they are based on neural networks with billions of parameters and trained on terabytes of text, achieving very good results both in generating natural text and in estimating the perplexity of unseen texts. However, there is a substantial risk that some of the books of the corpus may have been included within the dataset that these models were trained on, especially when OpenAI has not yet published its dataset until now. Therefore, the main methodology is to train a model of the same architecture as the series of gpt2 models from scratch using a dataset outside the corpus (hereinafter referred to as the self-trained model). 

If the corpus perplexity estimation observed from the self-trained model is in close correlation with results from the series of gpt2 models, then the potential bias risk can be excluded.
In this context, a new text generator based on the gpt2 model is trained from the beginning on the ``article'' content of the CNN Dailymail Dataset\footnote{\href{https://huggingface.co/datasets/ccdv/cnn_dailymail}{https://huggingface.co/datasets/ccdv/cnn\_dailymail}}. The primary reason for not employing other literary works as the training set is due to the potential bias associated with the selection of these books. Moreover, the CNN Dailymail dataset is chosen for its compilation of approximately one million news stories designed for reading and comprehension tasks \cite{hermann2015teaching}, offering a narrative consistency more closely aligned with that of novels than other datasets, such as WikiText. Then, we use the Adam optimizer with a learning rate of 5e-5 and a cross-entropy loss criterion to train the model for 10 epochs. 


\begin{table}[h]
\centering
\small
\caption{\label{table:models} Architecture hyperparameters and training set sizes for the three models.}
\begin{tabular}{lccc}
\toprule
              & \textbf{self\_model} & \textbf{gpt2} & \textbf{gpt2-xl} \\ \midrule
parameters    & 117M               & 117M       & 1542M         \\ 
layers        & 12                 & 12         & 48            \\ 
heads        & 12                 & 12         & 25            \\ 
dimensions     & 768                & 768        & 1600          \\ 
dataset size &   535M                & 40G         &   40G            \\ 
\bottomrule
\end{tabular}
% \caption{\label{table:models} Architecture hyperparameters and training set sizes for the three models.}
\end{table}

The Spearmann Correlation test results presented in Figure \ref{fig:heatmap_perplexity} show a robust correlation in perplexity values between the self-trained model and the other gpt2 models, indicating that a potential data bias can be excluded at least within this corpus. Therefore, the models forming the final hierarchy can be viewed as a sequential examination on the hypotheses and the consistency of our results across the models of varying sizes,  ranging from the smallest version of the self-trained model to the largest version of the gpt2-xl model (see Table \ref{table:models}).

\section{Perplexity \& the Canon}
These three variants of GPT2 models based on the Transformer architecture are employed to calculate perplexity values using the stride-based method across the entirety of the novels in the Chicago Corpus. A first mean of evaluation is to observe whether the mean perplexity changes with the models' size, as we would expect larger models to display lower perplexity. Consistently with our expectations, the mean perplexity values decrease when the model size increases, as delineated in Table \ref{table: general perplexity values} \footnote{Also see Figure \ref{fig:histograms} in Appendix}. Largest models are likely to be less perplexed by unusual linguistic structures, as they have been trained on much larger datasets and have, in some sense, ``seen more''. It is also a matter of debate, in this respect, whether larger is always better when it comes to correlations with human intuition. It is possible that very large models are harder to surprise than human readers, and their levels of perplexity may not correspond with human readers' experiences as much as those of smaller models \cite{oh-schuler-2021-contributions}. In our case, we find that the distinction between canonical and non-canonical works, defined by humans, is most strongly reproduced by the smallest of the three models.

Despite some potential fluctuations, the outcomes exhibit general consistency across the three language models. Notably, the highest and lowest perplexity values are elicited from the same two books, namely \textit{The Graduate} by Charles Richard Webb (the least perplexing novel overall) and \textit{Finnegans Wake} by James Joyce (the most perplexing).



\begin{table}[ht]
\centering
\small
\caption{\label{table: general perplexity values} An outlook on the perplexity values estimated by the three models. }
% Where corr means the correlation of perplexity values yielded by the self\_trained model with those of the other two gpt2 models.}
\begin{tabular}{lrrr}
\toprule
                 & \textbf{self\_model} & \textbf{gpt2}     & \textbf{gpt2-xl} \\ 
\midrule
min  & 16.307      & 8.9058   & 6.5862   \\ 
max  & 998.4872    & 306.1784 & 229.1857 \\ 
mean & 67.1944     & 28.8428  & 18.2334  \\ 
% corr & 1           & 0.9314   & 0.8875   \\ 
\bottomrule
\end{tabular}
\end{table}

Then, the Mann-Whitney test is used to examine the perplexity difference between canonical and non-canonical works. As shown in Table \ref{tab:textual_features_c_nc} %and Figure \ref{fig:boxplot_canon_noncanon}
, in terms of perplexity the difference between canonical and  non-canonical novels is significant over all of the three models, with canonical books being more perplexing than non-canonical in all cases.
%Novels considered canonical in our corpus appear more surprising - less usual or less predictable - than their larger non-canonical counterparts. 
This can be in turn surprising: canonical works, due to their status, might influence other works and become more typical. Yet it seems that they retain a unique originality, or a specially distinctive usage of language.
Moreover, there seems to be an internal variation within the canon. 

When inspecting works of different types of canonicity (contrasting literary prizes with other types of collections) we find that works judged canonical by experts and that are more closely affiliated to institutions (the Norton Anthology, Opensyllabus, and Penguin Classics) have a higher perplexity (Table \ref{tab:measures_per_canontype}). If we contrast with bestsellers, we also find that these appear to even have a lower perplexity than non-canon works (Table \ref{tab:measures_per_canontype}).

It is important here to note here once again that perplexity is not an absolute measure, but is the result of a model's training. Models trained on large enough datasets will capture fundamental regularities and find idiosyncratic uses of language more perplexing, but every model will consider elements closer to its training set as more normal. In this paper we assume the large training sets of the models as representative enough of contemporary English. 


\section{Correlations with textual features}

\begin{figure*}
    \centering
    \includegraphics[width=0.8\textwidth]{figures/heatmap_shorter_labels.png}
    \caption{Correlations (Spearman) of perplexity with stylistic and syntactic features.}
    \label{fig:corr_matrix_features}
\end{figure*}


\begin{table}[]
    \centering
    \small
    \caption{Correlation between perplexity and readability with GoodReads' rating count and number of libraries' holdings for each novel - proxies for the popularity or circulation of the works.}
    \begin{tabular}{lccc}
    \toprule
     & GR rating count  &  Libraries\\
     \midrule
    self\_model   & -.23 & -.31 \\
    R Dale-Chall & -.22 & -.25 \\
    \bottomrule
    \end{tabular}
    \label{tab:continuous_quality_correlations}
\end{table}

\begin{table}[h]
\centering
\small
\caption{Correlation Matrix of Readability Metrics and PPLs (Spearmann correlation)}
\begin{tabular}{lccc}
\toprule
& \textbf{self\_model} & \textbf{gpt2} & \textbf{gpt2-xl} \\
\midrule
Flesch Ease & -0.530 & -0.483 & -0.428 \\
Flesch Grade & 0.581 & 0.530 & 0.470 \\
Smog & 0.532 & 0.480 & 0.422 \\
ARI & 0.636 & 0.571 & 0.506 \\
Dale-Chall & 0.608 & 0.603 & 0.550 \\
\bottomrule
\end{tabular}
\label{tab:correlation_readability}
\end{table}

Perplexity is a powerful measure of linguistic predictability, as it results from large-scale modelling of word sequences. It can also be, and usually is, the effect of a composition of several factors, so that it is not always easy to understand what elements are driving its values. The richness of a large corpus of narrative fiction only adds to this difficulty. 
According to all our models, the most perplexing ``novel'' in the corpus is James Joyce's \textit{Finnegans Wake}, while the least perplexing is Webb's \textit{The Graduate}. A look at the first few lines of these books suffices to align our intuition to the models' results.\footnote{\textbf{Finnegans Wake}: ``riverrun, past Eve and Adam's, from swerve of shore to bend of bay, brings us by a commodius vicus of recirculation back to Howth Castle and Environs.''. \textbf{The Graduate}: ``Benjamin Braddock graduated from a small eastern college on a day in June. Then he flew home. The following evening a party was given for him by his parents.''} But the reasons for scoring a high perplexity can be different even among those texts that are ``clearly'' unusual: for example, another high-perplexing novel, Harris' \textit{Nights with Uncle Remus}, often reads as a fairy tale, but is heavily interspersed with %the transcription of a rare, almost unintelligible lingo.
heavy use of almost unintelligible eye dialect.\footnote{``Ez ter dat,'' responded Uncle Remus, ``dey mought stan' on one foot an' drap off ter sleep en fergit deyse'f. Deze yer gooses''.}
While the models' scores clearly pick from the same elements - recording internal Spearman correlations between 0.89 and 0.93 (Fig. \ref{fig:heatmap_perplexity}) - it is not easy to determine which linguistic features have the highest role in determining a given level of perplexity, and, more importantly, in determining which are the perplexing elements in a text that help tell canonical from non-canonical works. In the next sections, we will check the correlation between perplexity and some textual features often considered in the discourse over literary quality and canonicity. We refer to Figure \ref{fig:corr_matrix_features} for a summary of the findings.

\begin{figure*}
    \centering
    \includegraphics[width=0.8\textwidth]{figures/boxplot_C_NC.png}
    \caption{Features distribution for canonical (C) and non-canonical (NC) titles in our corpus. The nominal verb ratio is intended as the ratio of both adjectives and nouns over verbs.}
    \label{fig:boxplot_c_nc}
\end{figure*}




% which features? 
\begin{table*}[h]
\centering
\small
\caption{Mean and standard deviation for canonical (c) and non-canonical (nc) works regarding features displaying the highest correlations with perplexity. Mann-Whitney's \textit{z} score and size effect \textit{r} are reported. * p-value $<$.005. Numbers in parenthesis report the means, stds, z and r values when running the measures on a corpus where we have randomly selected 1 book per author (thus a smaller corpus of 3153 works of which 200 are canonical).%* p-value $<$.05, ** p-value $<$.005.   
}
\begin{tabular}{lcccccc}
\toprule
\textbf{Measure} & \textbf{Mean\_c} & \textbf{Std\_c} & \textbf{Mean\_nc} & \textbf{Std\_nc} & \textbf{z} & \textbf{r} \\ \midrule
Perplexity (self) & \textbf{81.57} (80.17) & 53.27 (76.70) & 65.01 (64.32) & 74.92 (32.26) & 3.1* (2.7*) (m) & .33 (.27) \\ 
Perplexity (gpt2) & \textbf{34.21} (34.33) & 15.17 (22.74) & 28.00 (28.39) & 9.39 (7.91) & 3.1* (2.8*) (m) & .38 (.33) \\
Perplexity (gpt-xl) & \textbf{21.81} (22.06) & 10.70 (16.97) & 17.74 (17.91) & 7.03 (4.87) & 3.2* (2.8*) (m) & .39 (.35) \\
\hline
Adj+Noun/Verb Ratio & \textbf{1.62} (1.60) & 0.29 (0.28) & 1.50 (1.50) & 0.24 (0.25) & 3.0* (2.6*) (m) & .27 (.22) \\ 
Verb/Noun Ratio & 0.874 (0.881) & 0.159 (0.153) & \textbf{0.929} (0.924) & 0.151 (0.15) & 1.8* (1.7*) (m) & .23 (.17) \\ 
Adverb/Verb Ratio & \textbf{0.406} (0.399) & 0.069 (0.067) & 0.386 (0.378) & 0.066 (0.065) & 2.7* (2.5*) (m) & .17 (.18) \\ \hline
%Passive/Active Ratio &  &  &  &  & 2.6(m)** & -.12 \\ \hline
TTR verbs & \textbf{0.253} (0.252) & 0.052 (0.053) & 0.242 (0.245) & 0.042 (0.043) & 2.5* (2.3) (m) & .17 (.10) \\ 
TTR nouns & \textbf{0.312} (0.312) & 0.046 (0.053) & 0.298 (0.299) & 0.037 (0.038) & 2.7* (2.4*) (m) & .12 (.15) \\
%MSTTR-100 & 0.686 (0.685) & 0.02 (0.02) & \textbf{0.689} (0.689) & 0.018 (0.018) & 2.1* (1.9*) (m) & .10 (.11) \\ 
%BZIP & 2.897 (2.892) & 0.168 (0.154) & \textbf{2.915} (2.909) & 0.126 (0.135) & 2.1* (2.6) (m) & .10 (.11) \\
\bottomrule
\end{tabular}
\label{tab:textual_features_c_nc}
\end{table*}

% % Just saving this old table (without the values when dropping for unique authors) for the future
% \begin{table*}[h]
% \centering
% \small
% \caption{Mean value and standard deviation for canonical (c) and non-canonical (nc) works regarding some of the features displaying the highest correlations with perplexity. Mann-Whitney's \textit{z} score and its size effect \textit{r} are reported. All p-values $<$.005.%* p-value $<$.05, ** p-value $<$.005.   
% }
% \begin{tabular}{lcccccc}
% \toprule
% \textbf{Measure} & \textbf{Mean\_c} & \textbf{Std\_c} & \textbf{Mean\_nc} & \textbf{Std\_nc} & \textbf{z} & \textbf{r} \\ \midrule
% Perplexity (self) & 81.57 & 53.27 & 65.01 & 74.92 & 3.1(m)** & -.33 \\ 
% Perplexity (gpt2) & 34.21 & 15.17 & 28.00 & 9.39 & 3.1(m)** & -.38 \\
% Perplexity (gpt-xl) & 21.81 & 10.70 & 17.74 & 7.03 & 3.2(m)** & -.39 \\
% \hline
% Adverb/Verb Ratio & 0.4 & 0.07 & 0.38 & 0.06 & 2.7(m)** & -.17 \\ 
% %Passive/Active Ratio &  &  &  &  & 2.6(m)** & -.12 \\ \hline
% Verb/Noun Ratio & 0.88 & 0.16 & 0.93 & 0.14 & 1.8(m)** & .23 \\ 
% Nominal/Verb Ratio & 1.62 & 0.29 & 1.50 & 0.24 & 3.0(m)** & -.27 \\ 
% MSTTR-100 & 0.686 & 0.02 & 0.689 & 0.018 & 2.1(m)** & .11 \\ \hline
% TTR verbs & 0.253 & 0.052 & 0.242 & 0.042 & 2.5(m)** & -.12 \\ 
% TTR nouns & 0.312 & 0.046 & 0.298 & 0.037 & 2.7(m)** & -.19 \\
% \bottomrule
% \end{tabular}
% \label{tab:textual_features_c_nc}
% \end{table*}

\begin{table*}[h]
\caption{Means and standard deviations (in parentheses) for measures across proxies. Note how bestsellers are closer to non-canonical works than canonicals in terms of overall perplexity.}
\small
\centering
\begin{tabular}{lrrrr}
\toprule
\textbf{} & \textbf{Non-canon}& \textbf{Bestsellers} & \textbf{Prizes} & \textbf{Canon lists} \\
\midrule
Perplexity (self model) & 67.85 (71.44) & 64.1 (24.69) & 73.08 (28.82) & \textbf{85.17} (54.17) \\ \hline
Nominal/verb ratio & 1.51 (0.27) & 1.55 (0.25) & 1.56 (0.28) & \textbf{1.64} (0.29) \\
Verb/noun ratio & \textbf{0.92 (0.15)} & 0.91 (0.15) & 0.90 (0.15) & 0.87 (0.16) \\ \hline
TTR verb & 0.24 (0.04) & \textbf{0.25} (0.05) & \textbf{0.25} (0.05) & \textbf{0.25} (0.06) \\
TTR noun & 0.30 (0.04) & 0.30 (0.04) & \textbf{0.31} (0.05) & \textbf{0.31} (0.05) \\ \hline
Dale-Chall readability & 5.10 (0.32) & 5.01 (0.29) & 5.15 (0.35) & \textbf{5.29} (0.46) \\

%Freq. `of' & 0.02 (0.005) & 0.018 (0.005) & 0.018 (0.004) & 0.019 (0.005) \\
\bottomrule
\end{tabular}
\label{tab:measures_per_canontype}
\end{table*}

\subsection{Stylometric features}
A novel's high perplexity score can be the effect of stylistic complexity. A simple conceptualization of this dimension of style is represented by readability measures, a family of algorithms developed in linguistics that gauge prose difficulty based on simple elements such as sentence and word length, and frequently used in relation to concepts of general literary quality \citep{bizzoni2023good, weigel2016literature, ashok2013success}.\footnote{Explicating the formulas is out of the scope of this paper, but can be consulted via the package we used to extract readability scores: \url{https://pypi.org/project/textstat/}} The models' perplexity shows robust correlations with all readability measures: books with a higher perplexity are harder to read (Table \ref{tab:correlation_readability}), at least to an extent. 

This is not an obvious correlation, as the central elements in readability algorithms, such as sentence length, are not directly factored in the language model's computation of perplexity. Yet, average sentence length alone has $>$.2 correlations with all our models: texts that are challenging at other levels also tend, to an extent, to feature longer sentences. Other features that affect formulae of readability, such as average word length, also show robust correlations with perplexity.%PM:added this last sentence
It seems to indicate that canonical works present on average a prose that is more difficult to read than non-canonical works. The inverse relation of readability and perplexity with some proxies of mere popularity, as shown in Table \ref{tab:continuous_quality_correlations},  additionally indicates that there is at least one ``type'' of novel that aggregates different strategies of simplicity - unsurprising usage of language, shorter sentences, shorter words etc. - to achieve a higher level of diffusion. While this, too, can be considered a distinct form of quality, it appears that canonical works tend to the opposite stylistic pole. 

% Mention the fact that Dale-Chall, one of the best correlatings, is also well-working in other domains e.g. scientific?
Another typical metric often associated with more complex and challenging novels is Type-Token Ratio \cite{kao2012computational}. As TTR shows a significant relation with our perplexity measures, %\footnote{MSTTR is a measure of TTR based on (100-word) segments.} 
it is likely that
more perplexing novels use a more diverse lexicon or more complex lexical structures rather than simpler and more repetitive alternatives, as also shown by the negative correlation with text compressibility, often a proxy for formulaicity or information density (Fig. \ref{fig:corr_matrix_features}).\footnote{As it is used in \cite{baechler_information-theoretic_2016,liddle_could_2019}.} 

%One possible factor driving up both perplexity, TTR and redundancy is extensive use of eye dialect or neologisms -- as is the case in the high-perplexity titles \textit{Nights with Uncle Remus} and \textit{Finnegan's Wake} -- which (in the form recognized as ``nonsense" by human readers) has also been shown to effect very high model perplexity \cite{zhang_trading_2020}. %YB: maybe this works better in the discussion?

\subsection{Syntactic features}
We looked into selected syntactic and grammatical features often considered in discussions about the quality of literary (and general) writing: frequency of passive voice and adverbs \cite{strunk2007elements} and relative ratios of Parts-of-Speech, especially looking for traces of so-called ``nominal style" \cite{mcintosh1975quantities, bostian1983active}. % Yb: Is 'nominal style' a real way of saying it?
% YB: I have to insert at least a couple of citations for each metric
% We should say why, no?

The frequency of the passive voice has a faint positive correlation with perplexity, and the active voice a slight negative correlation, suggesting that the passive is slightly more unusual than the active voice. While the percentage of adverbs and verbs plays no role in the perplexity of the novels, the adverbs-to-verb ratio does show a positive correlation. 

The verb/noun ratio of each novel displays a very robust correlation with the texts' perplexity. %Texts that tend more heavily to devolve to nouns rather than to verbs their informational content are more perplexing. 
This effect is even more pronounced when we compute the ratio of nouns plus adjectives against verbs. It displays one of the strongest correlations with model perplexity along the three models (see Figures \ref{fig:corr_matrix_features} and \ref{fig:perp_nom_corr}) and delineates a significant difference between the canonical and non-canonical groups (Table \ref{tab:textual_features_c_nc}).
We also checked for the relative frequency of the ``function words'' \textit{of} and \textit{that}: the first is associated with the presence of more nominal phrases, while the latter is typical of more declarative and verb-centered prose.
The fact that \textit{of} has a stronger correlation with perplexity than \textit{that}, and is more frequent in the canonical group (Figure \ref{fig:boxplot_c_nc}), is another hint to the larger presence of nominal phrases in more perplexing works. 
Interestingly, these differences can be extended to subcategories \textit{within} the canonical  category: %.Among the non-canons, bestellers are less surprising and less difficult to read, while our models showed higher perplexity over titles longlisted for awards. We also find that there is a difference within the canon: 
longlisted novels exhibit less perplexity, reading difficulty and traces of nominal style than the rest of the canon group (more ``classically'' canonical) but more so than the non-canon group, bestsellers included (Table \ref{tab:measures_per_canontype}).

\begin{figure}[ht]
    \centering
    \includegraphics[width=0.35\textwidth]{figures/nominal_ratio_ppl.png}
    \caption{Correlation between perplexity scores and nominal/verb ratio of texts.}
    \label{fig:perp_nom_corr}
\end{figure}

Nominal style is often considered ``heavier'' \cite{huckin1993stylistic}. Several studies on linguistic and information theory also found that non-fiction domains tend to optimize their communication strategies by increasingly relying on nominal phrases -- a strategy that works for ``expert'' audiences \cite{rubino-etal-2016-modeling, degaetano2019typical, juzek2020exploring, bizzoni2020linguistic}. % see Stefania
It is possible that the canons' higher perplexity is partly due to including more cognitively demanding, ``heavily nominal'' texts. 
One example is Jack Kerouac's \textit{Doctor Sax} which is one of the top books in our corpus in terms of perplexity and also of nominal/verb ratio. Its prose is rich with adjectives and nouns, sometimes skipping verbs altogether, as in: ``not as if idiot but as if sensual or senseless and bitter with venoms of woe''. Combined with its frequent neologisms this work offers a good example of text-intrinsic features of perplexity.

% \subsection{Sentiment Analysis}
% It's entirely possible that the ``strangeness" of the canonicals does not stop at the classical stylometric and syntactic level but characterizes the texts' semantics as well. While a semantic analysis of the novels would grant a study of its own, we have run a preliminary test of this possibility by observing whether there are statistically significant differences in their \textit{sentimental} profile, a narrative and semantic aspect that, in its most basic form, is relatively easy to represent. 
% % Yb: we used VADER ... while not perfect, still grants ...

% %Yb: table.
% The average sentiment of canonical novels appears slightly higher than that of non-canonical novels. This seems true even if we consider subsections of the novels (e.g. their ``beginning" and ``end"). Most importantly, the standard deviation of the sentimental values of the canonical works appears significantly larger than that of non-canonical works. While the difference in sentiment values, at the overall and local scale, is not particularly large, their difference in terms of standard deviation is relatively large, and correlates more strongly with the perplexity of our three models. This might point in an important direction: it is not necessarily the case that canonical texts are ``happier" than their counterpart - e.g. having more positive narratives or displaying a larger amount of positive emotion words or more ``intensely positive" words - but it could be the case that canonical works display a larger sentimental vocabulary and, thus, a broader emotional palette, allowing them to touch upon wider, as well as less predictable, emotional tones. 

\section{Conclusions}
We have explored some features of canonical vs non-canonical works based on a corpus of 9,000 novels from the late 19\textsuperscript{the} and the 20\textsuperscript{th} century. We first found that canonical novels seem to elicit higher perplexity scores based on three LLMs, with the difference remaining significant across different model sizes. Perplexity seems to reflect a  higher complexity in style of canonical novels, compared to that of non-canonical works that enjoy a vast readership. 

We have then explored some specific features that might contribute to this effect. 
Based on our collection, the higher perplexity of the canonical group is linked %less with word or bigram entropy within the novel itself, and more from 
to different distributions of grammatical constructions: heavier use of nominal phrases, paired with average longer sentences, words, and a higher lexical diversity. 
Specifically, the presence of a more marked nominal style might be an important cause for the difference in perplexity between the two groups, although it is clear that the overall effect is a result of an ensemble of features at the syntactic, stylistic, and semantic level. 
The idea that ``canonical'' novels, on average, are more challenging for readers than non-canonical ones, while the opposite holds for widely spread but non-canonical texts (such as texts rated very often on GoodReads), mirrors existent findings \cite{bizzoni2023good}.\footnote{Naturally, there are important overlaps, as these are intermingling categories. \textit{The Hobbit} features in our list of canonical works, and so does Twain's \textit{The Prince And The Pauper}.%, but of course their level of diffusion is not the same.
} The characteristics of this difference are of particular interest as they seem to be at least partly linked to the %development of 
communication efficiency observed in expert-domain prose for other fields. A heavily nominal style has been linked with the development of refined and diverse vocabulary, a higher cognitive load for the reader, and more effective communication of information, as nouns can be highly specific and diverse, bringing a higher amount of information at the cost of higher decoding effort.  
It is a hypothesis worth considering that canonical works might achieve a lower communicative \textit{immediacy} in favor of a higher communicative \textit{efficiency}. 
What this means for a literary work and what it implies for the reader's experience -- given the unique communicative functions of literary texts -- remains an open question to explore in future studies. 

Finally, it is important to consider that in this work we have intentionally ignored the dimension of time: we are interested in which features distinguish canonical, awarded, best-selling texts etc., independently from their distribution within the corpus. This means that we are treating our models as an abstract ``contemporary reader'' who, in front of the corpus, reacts to the canonical group differently from the bestseller group and so on. In future, we intend to observe how the features we have studied in this work correlate with time. 
% The following might go in the Discussion


% \begin{figure*}
%     \centering
%     \includegraphics[width=0.9\textwidth]{figures/heatmap.png}
%     \caption{Correlations (spearman) of perplexity and stylistic as well as sentiment features.}
%     \label{fig:heatmap}
% \end{figure*}



% \section*{Limitations}
% EACL 2023 requires all submissions to have a section titled ``Limitations'', for discussing the limitations of the paper as a complement to the discussion of strengths in the main text. This section should occur after the conclusion, but before the references. It will not count towards the page limit.

% \section*{Ethics Statement}
% Scientific work published at EACL 2023 must comply with the \href{https://www.aclweb.org/portal/content/acl-code-ethics}{ACL Ethics Policy}. We encourage all authors to include an explicit ethics statement on the broader impact of the work, or other ethical considerations after the conclusion but before the references. The ethics statement will not count toward the page limit (8 pages for long, 4 pages for short papers).

%\section*{Acknowledgements}


% Entries for the entire Anthology, followed by custom entries
\bibliography{anthology,custom}
\bibliographystyle{acl_natbib}

\appendix

\section{Appendix}
\label{sec:appendix}



\begin{figure*}[!htbp]
    \centering
    \includegraphics[width=0.9\textwidth]{figures/histograms_perplexity_means.png}
    \caption{Histogram of the distribution of perplexity per model in our corpus. Note that perplexity has a log normal distribution.}
    \label{fig:histograms}
\end{figure*}

% \begin{figure*}
%     \centering
%     \includegraphics[width=1\textwidth]{figures/histogram_ttrs.png}
%     \caption{Distribution of features}
%     \label{fig:ttr_dists}
% \end{figure*}

\begin{figure*}
    \centering
    \includegraphics[width=1\textwidth]{figures/boxplot_canon_proxies_individual.png}
    \caption{Nominal style features for each canon-type: Bestselling books, Prizewinning books, and books contained in one of the Canon lists. Note that outliers (points beyond the 99.5th percentile of our data) have been removed for this visualization.}
    \label{fig:proxies_different}
\end{figure*}



% \begin{figure*}[ht]
%   \begin{minipage}[b]{0.3\textwidth}
%     \centering
%     \includegraphics[width=\linewidth]{figures/corr_read_cont.png}
%     \caption{Caption for Figure 1}
%     \label{fig:figure1}
%   \end{minipage}%
%   \begin{minipage}[b]{0.4\textwidth}
%     \centering
%     \includegraphics[width=\linewidth]{figures/corr_self_cont.png}
%     \caption{Caption for Figure 2}
%     \label{fig:figure2}
%   \end{minipage}
% \end{figure*}



\end{document}





\begin{table}[h]
\centering
\begin{tabular}{lccccc}
\toprule
\textbf{} & \textbf{Perplexity (self model)} & \textbf{Nominal/verb Ratio} & \textbf{TTR verb} & \textbf{TTR noun} & \textbf{Freq. `of'}\\
\midrule
Bestsellers mean & 64.1 & 1.55 & 0.16 & 0.21 & 0.018\\
Bestsellers std & 24.69 & 0.26 & 0.06 & 0.07 & 0.005 \\
Prizes mean & 73.08 & 1.56 & 0.18 & 0.24 & 0.018 \\
Prizes std & 28.82 & 0.28 & 0.06 & 0.06 & 0.004 \\
Canon mean & 85.01 & 1.62 & 0.21 & 0.26 & 0.019 \\
Canon std & 57.88 & 0.30 & 0.08 & 0.07 & 0.005 \\
\bottomrule
\end{tabular}
\caption{Means and standard deviations for measures across proxies}
\label{tab:measures_per_canontype}
\end{table}