    % This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{tcolorbox}
\usepackage{float}
\usepackage{placeins} 

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
\usepackage{amsmath}

%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}
\usepackage{url}
\usepackage{expex}
\usepackage{tablefootnote}
\usepackage{todonotes}
\usepackage{arydshln} % For dashed lines in tables
\usepackage{booktabs}
\usepackage{cancel}
\usepackage{pifont}

\newcommand{\cmark}{\ding{51}}%
\newcommand{\xmark}{\ding{55}}%

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Information-Theoretic and Prompt-Based Evaluation of Discourse Connective Edits in Instructional Text Revisions}
% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\iffalse
\author{First Author \\
  Affiliation / Address line 1 \\
  Affiliation / Address line 2 \\
  Affiliation / Address line 3 \\
  \texttt{email@domain} \\\And
  Second Author \\
  Affiliation / Address line 1 \\
  Affiliation / Address line 2 \\
  Affiliation / Address line 3 \\
  \texttt{email@domain} \\}
\fi
\author{Berfin Akta\c{s} \\
  Natural Language Understanding Lab \\
    University of Technology Nuremberg\\
  \texttt{berfin.aktas@utn.de} \\\And
 Michael Roth \\
  Natural Language Understanding Lab \\
    University of Technology Nuremberg\\
  \texttt{michael.roth@utn.de} \\}
  
%\author{
%  \textbf{First Author\textsuperscript{1}},
%  \textbf{Second Author\textsuperscript{1,2}},
%  \textbf{Third T. Author\textsuperscript{1}},
%  \textbf{Fourth Author\textsuperscript{1}},
%\\
%  \textbf{Fifth Author\textsuperscript{1,2}},
%  \textbf{Sixth Author\textsuperscript{1}},
%  \textbf{Seventh Author\textsuperscript{1}},
%  \textbf{Eighth Author \textsuperscript{1,2,3,4}},
%\\
%  \textbf{Ninth Author\textsuperscript{1}},
%  \textbf{Tenth Author\textsuperscript{1}},
%  \textbf{Eleventh E. Author\textsuperscript{1,2,3,4,5}},
%  \textbf{Twelfth Author\textsuperscript{1}},
%\\
%  \textbf{Thirteenth Author\textsuperscript{3}},
%  \textbf{Fourteenth F. Author\textsuperscript{2,4}},
%  \textbf{Fifteenth Author\textsuperscript{1}},
%  \textbf{Sixteenth Author\textsuperscript{1}},
%\\
%  \textbf{Seventeenth S. Author\textsuperscript{4,5}},
%  \textbf{Eighteenth Author\textsuperscript{3,4}},
%  \textbf{Nineteenth N. Author\textsuperscript{2,5}},
%  \textbf{Twentieth Author\textsuperscript{1}}
%\\
%\\
%  \textsuperscript{1}Affiliation 1,
%  \textsuperscript{2}Affiliation 2,
%  \textsuperscript{3}Affiliation 3,
%  \textsuperscript{4}Affiliation 4,
%  \textsuperscript{5}Affiliation 5
%\\
%  \small{
%    \textbf{Correspondence:} \href{mailto:email@domain}{email@domain}
%  }
%}

\begin{document}
\maketitle
\begin{abstract}
We present a dataset of text revisions involving the deletion or replacement of discourse connectives. Manual annotation of a replacement subset reveals that only 19\% of edits were judged either necessary or should be left unchanged, with the rest appearing optional. Surprisal metrics from \texttt{GPT-2} token probabilities and prompt-based predictions from \texttt{GPT-4.1} correlate with these judgments, particularly in such clear cases.
\end{abstract}

\section{Introduction}

Discourse relations are essential for maintaining coherence and logical flow in text.
%and logical flow between segments of natural language. 
This is especially critical in instructional texts such as how-to guides, where a lack of clarity can lead to misinterpretation and misunderstanding \cite{roth-etal-2022-semeval, aktas-roth-2025-clarifying}. Discourse relations are often signaled explicitly through discourse connectives like ``because'' or ``however.'' In the case of implicit discourse relations, where coherence is inferred from context rather without an overt connective, interpretation becomes more ambiguous. Even %in texts 
with explicit connectives, alternative connectives may sometimes express a relation more clearly or appropriately. Identifying when a connective is truly necessary, when it may be redundant or misleading, and which connective best fits the context remains a challenge in discourse processing.

We investigate revisions in which discourse connectives are inserted, replaced or deleted, and explore whether %transformer-based models, including encoder (BERT, RoBERTa) and decoder 
LLMs (\texttt{GPT-2}, \texttt{GPT-4.1}, \texttt{Mistral-7B}),
%\footnote{We also tested LLaMA-3.1-8B, but it did not follow instructions consistently.} 
can offer useful cues. % for connective insertion decisions. 
Specifically, we explore whether an information-theoretic measure such as \citet{shannon1948mathematical}'s \textit{surprisal} can be used to assess the necessity or appropriateness of a discourse connective in context. Our hypothesis is that connectives inserted during revision (i.e., judged necessary) and those deleted (i.e., judged unnecessary) affect the predictability of the text in different ways and and these effects can be captured through surprisal-based measures such as \textit{average surprisal}, \textit{variance}, and \textit{smoothness} of surprisal change.
%\todo{Actually following the uniform informativity hypothesis, if the connective is informative, then the rest should be less informative. This should be checked.}

%We base our analysis on a dataset of human-authored text revisions involving either the deletion or replacement of discourse connectives. For each original–revised sentence pair, we compute token-level log-likelihoods using BERT, RoBERTa, and GPT-2, from which we derive sentence- and token-level surprisal and entropy scores. These measures serve as indicators of model preference, i.e., whether the model favors the version with or without a connective, and, if present, which connective it prefers.
An initial analysis of information-theoretic measures over \textit{all} revisions regarding connectives revealed inconsistent patterns: the presence or absence of a discourse connective did not consistently affect surprisal-based metrics (see Appendix~\ref{app:surprisal_wholedata}). % and does not produce results in line with our expectations. 
To further investigate this, we conduct a qualitative study on a subset of connective replacements, which we present in Section~\ref{sec:data}.
%from our dataset. 
Manual annotation of this subset reveals that not all replacements serve the same function: some are optional, others are essential for coherence, and some may even be inappropriate. We apply two complementary approaches to uncover underlying patterns computationally: information-theoretic analysis using \texttt{GPT-2} \cite{radford2019language} and prompting-based evaluation with more recent models, namely \texttt{GPT-4.1} \cite{openai2024gpt4technicalreport} and \texttt{Mistral-7B} \cite{jiang2023mistral7b}. We discuss the patterns uncovered by these methods in Section~\ref{sec:pilot_expr_replacements},  arguing that comparing these approaches provides valuable insights into the interpretability of model behavior.

Although our exploration of information-theoretic measures with language models yields largely inconclusive results, we believe this line of investigation remains promising. 
%The absence of strong findings may be attributed to current limitations in model architecture and API constraints, particularly the unavailability of token-level probability outputs in closed source models such as GPT-4.1. 
As language models become more transparent in exposing token-level probabilities and better at modeling discourse, information-theoretic metrics may offer an interpretable framework for assessing connective necessity and discourse coherence. We view this as an interesting direction for future research.

We investigate the following research questions:

\begin{itemize}
\item RQ1: Can information-theoretic metrics reveal when replacing a connective improves coherence?
%\item Under what conditions are connective replacements in revisions driven by functional necessity or stylistic preference?
\item RQ2: To what extent do information-theoretic and prompt-based methods align in their interpretation of discourse relations?
%\item To what extent are connective deletions in text revisions justified or necessary?
%\item Can large language models (LLMs) reliably identify when a connective replacement is necessary?
%\item Can LLMs support the resolution of implicit discourse relations by selecting the better version of a text using information-theoretic measures?
%\item Do information-theoretic metrics (e.g., surprisal, entropy) provide empirical support for the continuity hypothesis in discourse processing?

\end{itemize}

\begin{table*}[th]
    \centering
    \begin{tabular}{p{2cm}p{13cm}}
    \toprule
         Deletion & Cut the four shirt pieces out of the sheet material. \cancel{Then}
place those pieces on the lining material, face side up, and
pin them into place before cutting them out.  \\
\midrule
         Replacement & Pick a music choice that agrees with your style choice.
\cancel{So} \underline{For example}, if your dance is fast, pick a fast song.\\
        \bottomrule
    \end{tabular}
    \caption{Examples of connective edits in our data. In the first example, the connective is deleted (\textit{Then}), while in the second example the connective (\textit{So}) is replaced by another connective (\textit{For example}).}
    \label{tab:examples}
\end{table*}

\section{Related Work}
%Information \textit{surprisal} quantifies how unexpected a word is given its preceding context and serves as a basis for modeling textual coherence and flow. 
Information \textit{surprisal} has been extensively used in computational linguistics to model processing difficulty and expectations in language comprehension \cite[e.g.,][]{levy2008expectation,clark-etal-2023-cross,oh-schuler-2023-surprisal}. 
From a discourse perspective, \citet{torabi-asr-demberg-2015-uniform} examine the role of discourse connectives through the lens of the Uniform Information Density (UID) hypothesis. They show that connectives can help distribute information more evenly.
%suggesting a functional role in smoothing variation in surprisal across discourse segments.

Recently, \citet{aktas-roth-2025-clarifying} examine discourse connective insertions in revision data and find that multiple relations are often plausible, while language models perform inconsistently in detecting ambiguity. This underscores the challenge of identifying the necessity of connectives and motivates the need for complementary metrics.
\iffalse
\section{Research Questions}
\label{sec:intro}


We conduct further analysis on the data, addressing the following research questions:

\section*{Research Questions and Contributions}

\begin{itemize}
    \item \textbf{Can LLMs help make implicit discourse relations explicit?} \\
    Prior work shows LLMs struggle with identifying implicit discourse relations. This study investigates whether \textbf{LLM surprisal and entropy} can serve as a proxy for identifying such relations.

    \item Do the informatıon theoretic measures provide support for the continuity hypothesis?
    \item \textbf{Main Contribution:} \\
    We evaluate whether surprisal and entropy scores from LLMs (specifically BERT and GPT-2) reflect sensitivity to changes in discourse coherence due to connective revisions.

    \item \textbf{Model Justification:} \\
    BERT and GPT-2 were chosen because ...
\end{itemize}

\section*{Experiments and Data}

\begin{itemize}
    \item We evaluate three types of revisions:
    \begin{enumerate}
        \item \textbf{Explicitation:} Inserting a connective into an implicit context.
        \item \textbf{Implicitation:} Removing an explicit connective.
        \item \textbf{Replacement:} Substituting one connective for another.
        \item We can include statistics for the connective explicitation, implicitation and replacement instances (for the appendix?). Together with an analysis on the relation senses they indicate. List of connectives different tan PDTB (e.g., just because)
    \end{enumerate}

    \item For each case, we compare surprisal and entropy between the original and revised sentences:
    \begin{itemize}
        \item Are revisions more or less surprising?
        \item Are entropy and surprisal patterns consistent?
    \end{itemize}
\end{itemize}

\section*{Continuity Hypothesis}

\begin{itemize}
    \item \textbf{Definition:} When clauses follow each other without an explicit connective, comprehenders (and possibly LLMs) tend to infer a coherent discourse relation, commonly \textbf{Contingency} (cause-effect) or \textbf{Temporal} (sequence), unless context signals otherwise.
    
    \item \textbf{Research Question:} Do LLMs show lower surprisal for default relations (e.g., cause, temporal) than for contrastive ones when connectives are removed?

    \item \textbf{Planned Analysis:}
    \begin{itemize}
        \item Examine surprisal in explicitation and implicitation experiments.
        \item Check if model behavior aligns with the continuity hypothesis.
        \item Compare trends across surprisal and entropy metrics.
    \end{itemize}
\end{itemize}

\section*{Connective Replacement Experiments}

\begin{itemize}
    \item \textbf{Method:} In these experiments, the original connective is replaced with a placeholder (e.g., ``<...>''), and the LLM is prompted to generate a suitable connective.

    \item \textbf{Preliminary Results:}
    \begin{itemize}
        \item \textbf{GPT} prefers the crowdsourced revision over the original in approximately 30\% of cases, compared to 15\% for the original.
        \item \textbf{Mistral} shows similar tendencies, though it fills the blanks less reliably.
    \end{itemize}
    
    \item \textbf{Next Steps:}
    \begin{itemize}
        \item Analyze the sense and diversity of suggested connectives.
        \item Include additional models (e.g., DeepSeek).
        \item Investigate if model preferences reflect continuity-based defaults.
    \end{itemize}
\end{itemize}
\fi

\section{Overview}
\label{sec:overview}

In this section, we provide a brief overview of our study design. We introduce the dataset of connective edits extracted from WikiHow revisions (Section~\ref{sec:data}), including deletions, replacements, and manually annotated subsets of these edits used to assess their function. Building on this, we conduct experiments on the annotated replacement instances, comparing information-theoretic measures with prompt-based judgments from large language models (Section~\ref{sec:pilot_expr_replacements}). Together, these components allow us to investigate how connective edits affect discourse coherence and how well computational models capture these preferences.

\section{Data}
\label{sec:data}
As a framework for discourse relations, we adopt the Penn Discourse Treebank (PDTB) \citep{prasad-etal-2018-discourse} and use its inventory of discourse connectives. Building on prior work by \citet{aktas-roth-2025-clarifying}, we use the wikiHowToImprove dataset \citep{anthonio-etal-2020-wikihowtoimprove}, which contains sentence-level revisions. While that prior study primarily focused on connective insertions, we target two complementary operations: the deletion of existing connectives and the replacement of one connective with another. We introduce a subset of instructional text revisions where explicit discourse connectives are either deleted from the beginning of a sentence~(\S\ref{sec:data_deletion}) or replaced with another~(\S\ref{sec:data_replacement}). 
%We describe these two types of data in  and \textsection, respectively.


\subsection{Connective Deletions}
\label{sec:data_deletion}
We identified 13,597 instances where discourse connectives were deleted from the beginning of a sentence, as illustrated in the top row of Table~\ref{tab:examples}.
%Figure~\ref{fig:connective-deletion-example}. 
In a quantitative analysis, we find the most commonly deleted connectives to be
\textit{then}, \textit{also}, and \textit{and}.\footnote{We list the 15 most frequently deleted connectives in Table~\ref{tab:connective-deletions} in Appendix~\ref{sec:appendix_stats}.}
To better understand these deletions, we qualitatively analyzed 100 examples (four for each of the 25 most frequently deleted discourse connectives). In 64\% of cases, the connective was redundant or its removal improved sentence flow. In 21\%, deletion was necessary to resolve syntactic or interpretability issues. In the remaining 15\%, deletion should not happen as it is altering the intended meaning.\footnote{For an example, see Figure~\ref{fig:connective-needed} in Appendix \ref{app:examples} where deleting the contrastive connective `Otherwise'' causes the instruction to lose its conditional framing.}

We examined the connective deletion statistics in comparison to the connective insertion statistics provided by \citet{aktas-roth-2025-clarifying}. Across the dataset, there is a clear asymmetry between these two types of revisions: the number of connective deletions (13,597) significantly exceeds the number of insertions (4,274).

As shown in Appendix~\ref{sec:appendix_stats}, connectives such as \textit{then}, \textit{finally}, \textit{and}, and \textit{so} are deleted far more often than inserted, suggesting they are often perceived as redundant. This supports the findings of \citet{torabi-asr-demberg-2012-implicitness}, who argue that causal, temporal, and additive relations (when not marking an event shift) are frequently left implicit. 
In contrast, connectives like \textit{for example} and \textit{in addition} are more often inserted, pointing to their role in improving clarity. Interestingly, although \textit{for example} signals an “instantiation” relation in PDTB, which \citet{torabi-asr-demberg-2012-implicitness} describe as typically implicit, our data shows frequent explicit realization. This discrepancy suggests the need for further investigation, possibly by analyzing the full dataset in terms of discourse relations, rather than focusing only on those made explicit or implicit through sentence-initial edits.

\setlength\dashlinedash{0.3pt}
\setlength\dashlinegap{2pt}
\setlength\arrayrulewidth{0.3pt}
\begin{table*}[t]
    \centering
    \begin{tabular}{p{1.9cm}p{11.5cm}r}
    \toprule
   \textbf{Revision necessary} & The revised version is the only option that conveys the necessary meaning or fits the syntactic context. & 11 cases
   \\\hdashline
   \textbf{Revision better} & Both options are plausible, but the revision is preferred in terms of clarity, formality, or fluency. & 35 cases
   \\\hdashline
   \textbf{Either way} & Both the original and revised versions are similarly acceptable. & 36 cases
\\\hdashline
   \textbf{Original better} & Both options are plausible, but the original is preferred in terms of clarity, formality, or fluency. & 10 cases
\\\hdashline
  \textbf{Original should stay} & The original version is the only option that conveys the necessary meaning or fits the syntactic context.  & 8 cases\\
  \bottomrule
   \end{tabular}
    \caption{Description of labels for annotation and absolute counts after aggregation by majority vote.}
    \label{tab:labels}
\end{table*}

%(TODO: Can we establish a connection with the previous paper's crowdsourced annotations here? Also, can we establish a connection with the continuity hypothesis?)
\subsection{Connective Replacements}
\label{sec:data_replacement}

We identified 1,841 revisions in which discourse connectives occurring at the beginning of a sentence were replaced by another connective, as illustrated in the bottom row of Table~\ref{tab:examples}. %as in Figure \ref{fig:connective-replacement-example}. 
The most common replacement is \textit{but} $\rightarrow$ \textit{however}.\footnote{Frequent pairs are shown in Table~\ref{tab:replacements} in Appendix~\ref{sec:appendix_stats}.} While both connectives share the same dominant PDTB sense, \textsc{comparison.concession.arg2-as-denier}, \textit{however} is slightly less ambiguous, linked to only 8 different PDTB senses compared to 13 for \textit{but}, and arguably better aligned with the formal tone of instructional texts.


However, reduced ambiguity or stylistic considerations do not fully explain all replacement choices. For instance, \textit{since} is more ambiguous than \textit{because}, yet \textit{because} $\rightarrow$ \textit{since} appears among the top five replacement pairs. Apart from replacements like \textit{if} $\leftrightarrow$ \textit{even if}, which reflect a clear shift in discourse relation, most replacements preserve the original PDTB sense. This suggests many changes are guided more by tone, readability, or syntactic fit rather than discourse semantics. To gain deeper insight into the nature of these replacements, %we carried out a small-scale qualitative analysis, detailed in the following section.
%\paragraph{Qualitative Annotation Study}
\label{sec:qualitative_replacements}
%As with the deletions, 
we analyze four randomly selected instances for each of the top 25 replacement pairs.\footnote{Since this dataset includes only edits at the beginning of sentences, all annotated cases involve sentence-initial edits.}
%We randomly selected 100 examples from the dataset (four instances from most common 25 connective replacements). 
As this data is used for experiments (\S\ref{sec:pilot_expr_replacements}),
%These annotations are used not only for qualitative evaluation but also as gold data for LLM experiments in 
%the data is annotated independently by three annotators: 
we collect three independent annotations for each instance to ensure data quality and aggregate them by majority vote.\footnote{One annotator is an author of this paper, whereas the other two are PhD students in Computational Linguistics.}
%The task was a 5-way classification based on the following categories:


The labels and aggregated counts are listed in Table~\ref{tab:labels}. The average agreement with the majority label is 74\%.
%\marginpar{\footnotesize MR: is this pairwise agreement or average agreement with majority label?}
\if 0
Distribution of majority labels is shown in Table~\ref{tab:majority-distribution}.

\begin{table}[t]
\centering
\begin{tabular}{lr}
\toprule
\textbf{Label} & \textbf{Count} \\
\midrule
Either\_way             & 36 \\
Revision\_better        & 35 \\
Revision\_necessary     & 11 \\
Original\_better        & 10 \\
Original\_should\_stay  & 8 \\
\bottomrule
\end{tabular}
\caption{Distribution of labels for replacements.}
\label{tab:majority-distribution}
\end{table}
\fi
The high frequency of the \textit{Either\_way} label (36\%) is expected as the original and revised connectives convey the same discourse relation in 48\% of cases.% the 100 annotated instances, the original and revised versions conveyed the same discourse relation, 
\footnote{As determined by the most common PDTB sense annotations (e.g., \textit{once} and \textit{after} both signal \textit{temporal.asynchronous.succession}).} 
%This semantic similarity likely led annotators to judge both versions as equally acceptable. 
By contrast, only 11\% of the revisions are annotated as clearly necessary, and 8\% are seen as inappropriate. The remaining cases appear to reflect stylistic or contextual preferences rather than discourse-level necessity.
%\section{Experiments with Information-Theoretic Approach}
%\label{sec:info_theo_expr}


%\section{Information-Theoretic Effects of Connective Edits}
%\label{sec:uid-effects}
%\marginpar{\footnotesize MR: I outcommented the results \& discussion of applying information-theoretic measures on the whole data $\rightarrow$ maybe put into appendix?}
%\marginpar{\footnotesize MR: I put a copy of the definitions themselves into the experiment section (now 4.1).}

%\subsection{Measures of LM Uncertainty}
%We compute word-level surprisal using GPT-2:
\if
To quantify a language model's (LM) uncertainty and text predictability, we employ two normalized\footnote{These normalized measures enable direct comparison of LM predictive behavior and text predictability across varying sequence lengths.} measures for a sequence $T = (t_1, \ldots, t_N)$ of length $N$. 
%All logarithmic values are based on logarithm base 2.

\paragraph{Average Entropy:} At each token $t_i$, the LM predicts a probability distribution over the set of all possible tokens $v$ the model can generate given context $t_{<i}$, yielding \textit{ entropy} $H_i$:
$$H_i = - \sum_{v} P(v | t_{<i}) \log_2 P(v | t_{<i})$$
The \textit{average entropy} for the sequence is then:
$$\text{Avg } H(T) = \frac{1}{N} \sum_{i=1}^{N} H_i$$
This normalized measure reflects the LM's overall uncertainty across the text; higher values indicate greater uncertainty.
\fi

%To measure a language model’s (LM) uncertainty and the predictability of text, we use \textit{surprisal}. For GPT-2, the input is first tokenized, and then, the model computes the probability of each token given its preceding context. The \textit{surprisal} of each token is then calculated as the negative base-2 logarithm of its predicted probability.

\if 0
\paragraph{Average Token Surprisal}
For each token $t_i$, its \textit{surprisal} $S(t_i | t_{<i})$ measures how unexpected it is given the preceding context $t_{<i}$ according to the language model (LM):
\[
S(t_i | t_{<i}) = -\log_2 P(t_i | t_{<i})
\]
The \textit{average token surprisal} for a sequence $T = (t_1, t_2, \ldots, t_N)$ is:
\[
\text{Avg } S(T) = \frac{1}{N} \sum_{i=1}^{N} S(t_i | t_{<i})
\]
This value reflects the overall predictability of the sequence for the LM; lower values indicate greater predictability.

\paragraph{Results}

Insertions and replacements lead to a statistically significant increase in average surprisal, while deletions cause a significant decrease (Figure~\ref{fig:delta_mean} in Appendix~\ref{app:surprisal_figures}). An increase in average surprisal suggests that the resulting text became less predictable overall, whereas a decrease reflects a shift toward greater predictability.

Interestingly, the reverse of insertions (i.e., removing connectives that had been inserted by humans, labeled as \texttt{insertion\_rev}) also leads to a decrease in average surprisal, mirroring the effect of deletions. The difference between \texttt{deletion} and \texttt{insertion\_rev} is not statistically significant, suggesting that these two operations have symmetrical effects on average surprisal despite differing in context.
\fi

\if 0
\subsection{Variance of Surprisal}
\label{sec:uid-variance}

The variance of surprisal, $\text{Var}(S)$, quantifies the spread of token-wise surprisal values around their mean:

\[
\text{Var}(S) = \frac{1}{N} \sum_{i=1}^{N} \left(S(t_i \mid t_{<i}) - \text{Avg}(S)\right)^2
\]
Higher variance indicates larger fluctuations in predictability across tokens, whereas lower variance suggests a more uniform distribution.

\paragraph{Results} As shown in Figure~\ref{fig:delta_var} (Appendix~\ref{app:surprisal_figures}), both deletion and replacement significantly reduce surprisal variance, making the text more uniformly predictable. In contrast, insertions and therefore their reverse functions \texttt{insertion\_rev} do not significantly alter variance. 

This asymmetry between deletions and \texttt{insertion\_rev}, despite their functional similarity, suggests that while the mean surprisal remains stable, the local variance depends more on the editing context.
\fi

\if 0
\subsection{Smoothness of Surprisal}
\label{sec:uid-smoothness}

Surprisal smoothness captures how abruptly the LM's predictability shifts from one token to the next. It is defined as the mean absolute difference between the surprisals of consecutive tokens:

\[
\text{Smoothness}(S) = \frac{1}{N-1} \sum_{i=1}^{N-1} |S(t_{i+1}) - S(t_i)|
\]

A lower value indicates more gradual changes (smoother transitions), while a higher value reflects more abrupt predictability shifts.

\paragraph{Results} Figure~\ref{fig:delta_smooth} (Appendix~\ref{app:surprisal_figures}) shows that insertions, deletions, and replacements each lead to small but statistically significant decreases in surprisal smoothness (i.e., negative values indicate that revisions are less smooth). However, the differences across edit types are not statistically significant, suggesting that all three introduce increases in local unpredictability.

There is a clear contrast between deletions and \texttt{insertion\_rev} in terms of smoothness. Removing human-inserted connectives appears to make the surprisal values change more gradually, yielding a smoother predictability pattern.

\paragraph{Discussion}

These results demonstrate that while \texttt{deletion} and \texttt{insertion\_rev} exhibit similar effects on average surprisal, they diverge substantially in their impact on surprisal variance and smoothness. This suggests that although the overall informativeness of a connective may remain stable across insertions and deletions, the local predictability structure it induces, whether it smooths or disrupts the flow, depends strongly on the context.

Taken together, these metrics provide a nuanced picture of how connective edits influence the predictability of text for an LM (\texttt{GPT-2} in our case).
\fi

\iffalse
\subsection{Experiments}
\label{sec:info-theo-expr-explained}

We compute the difference in entropy and surprisal scores for each sentence pair (original vs. revised) using several transformer models implemented via the Hugging Face transformers library \cite{wolf-etal-2020-transformers}. These included variants of BERT \cite{devlin-etal-2019-bert}, RoBERTa \cite{liu-etal-roberta-2019}, and \texttt{GPT-2} \cite{radford-2019-LanguageMA}\footnote{Since modern models like \texttt{GPT-4.1} do not currently expose token-level log-probabilities via API, we relied on earlier architectures for information-theoretic analysis.}. The version with the lower score was interpreted as the model’s preferred option. 

\todo{Add the results}
\fi


\section{Pilot Study on Replacements}
\label{sec:pilot_expr_replacements}
%Not all connective replacements done in the revisions appeared equally necessary or meaningful. To assess the quality of these replacements, we conducted a small pilot study.

%Given that our large-scale experiments on connective deletions and replacements did not yield conclusive results, we turned to a smaller, manually annotated dataset (described in \textsection~\ref{sec:qualitative_replacements}) for more focused analysis. 

Using the data described in Section~\ref{sec:data_replacement}, we evaluate whether language models can detect subtle discourse preferences between original and revised versions of connectives. We compare two different methodologies: (1) an information-theoretic approach based on token-level log-likelihoods, and (2) prompting-based judgments from more recent large language models. While the former offers an interpretable output grounded in language predictability, the latter captures higher-level discourse reasoning not available through raw probabilities. We describe information-theoretic measures 
(\S\ref{sec:uid-effects})
before turning to a comparative evaluation (\S\ref{subsec:eval}).

%\textit{TODO: Shall we also include the outcome of LLMs in a 5-way classification task>} 
%These methods provide a more nuanced view of model behavior in discourse-sensitive tasks. We discuss their respective advantages and alignment in more depth in \textsection~\ref{sec:discussion}.

\subsection{Information-Theoretic Measures}
\label{sec:uid-effects}

To measure a language model’s uncertainty and the predictability of text, we use \textit{surprisal}. For \texttt{GPT-2}, the input is first tokenized, and then, the model computes the probability of each token given its preceding context. The \textit{surprisal} of each token is then calculated as the negative base-2 logarithm of its predicted probability.

\paragraph{Average Token Surprisal}

For each token $t_i$, its \textit{surprisal} $S(t_i | t_{<i})$ measures how unexpected it is given the preceding context $t_{<i}$ according to the language model:
\[
S(t_i | t_{<i}) = -\log_2 P(t_i | t_{<i})
\]
The \textit{average token surprisal} for a sequence $T = (t_1, t_2, \ldots, t_N)$ is:
\[
\text{Avg } S(T) = \frac{1}{N} \sum_{i=1}^{N} S(t_i | t_{<i})
\]
This value reflects the overall predictability of the sequence for the language model, with lower values indicating greater predictability.

\paragraph{Variance of Surprisal}

The variance of surprisal, $\text{Var}(S)$, quantifies the spread of token-wise surprisal values around their mean:
\[
\text{Var}(S) = \frac{1}{N} \sum_{i=1}^{N} \left(S(t_i \mid t_{<i}) - \text{Avg}(S)\right)^2
\]
Higher variance indicates larger fluctuations in predictability across tokens, whereas lower variance suggests a more uniform distribution.

\paragraph{Smoothness of Surprisal}

Surprisal smoothness captures how abruptly the language model's predictability shifts from one token to the next. It is defined as the mean absolute difference between the surprisals of consecutive tokens:
\[
\text{Smoothness}(S) = \frac{1}{N-1} \sum_{i=1}^{N-1} |S(t_{i+1}) - S(t_i)|
\]
A lower value indicates more gradual changes (smoother transitions), while a higher value reflects more abrupt predictability shifts.

%\subsection{Information-Theoretic Evaluation}
\subsection{Comparative Evaluation}
\label{subsec:eval}

We computed model preferences using \texttt{GPT2}\footnote{Specifically, the \texttt{GPT2-large} model from the \texttt{transformers} library \cite{wolf-etal-2020-transformers}.}, comparing the ``original'' and ``revision'' versions based on surprisal metrics described in Section~\ref{sec:uid-effects}. Among these, only the differences in \textit{average token surprisal} were statistically significant across human annotation categories ($p < 0.001$), suggesting a meaningful correlation between average surprisal changes and human preferences. In contrast, differences in \textit{variance} and \textit{smoothness} did not reach statistical significance.

%\marginpar{\footnotesize MR: Please clarify Which scores were used in the end}
%Among the models evaluated,
%\marginpar{\footnotesize MR: Please clarify Which other models were evaluated}
%\texttt{GPT2} showed the most robust statistical differences between original and revised sentences (p<0.05), with the smallest p-value. 
%\subsection{Prompt-Based Evaluation with Modern LLMs}
For a comparison with a more recent model, we conducted prompting-based experiments using \texttt{GPT-4.1-mini}. In each prompt, the connective was replaced with ``<..>'' and the model was explicitly asked to suggest an appropriate discourse connective for that position (see Appendix~\ref{app:prompts} for examples). Each item was evaluated across five independent runs. If the original connective was predicted more frequently than the revised one, we interpreted this as a preference for the original; if the revised one was more frequent, it was considered preferred. Equal prediction rates (or no prediction) were treated as indicating no clear preference.

\paragraph{Results} 
Table~\ref{tab:gpt2-results} and Table~\ref{tab:gpt4-results} summarize the predictions of \texttt{GPT-2}.
%summarizes the prediction distribution from 
and \texttt{GPT-4.1}, respectively, on the 100 manually annotated instances.\footnote{Note that zero-shot and few-shot prompting did not yield statistically significant differences in \texttt{GPT-4.1}’s output.}
\begin{table}[t]
\centering
\begin{tabular}{lccc}
\toprule
\textbf{Gold label} & \textbf{Equal} & \textbf{Orig} & \textbf{Revi} \\
\midrule
Either\_way & \textbf{8} & 14 & 14 \\
Original\_better & 1 & \textbf{6} & 3 \\
Original\_should\_stay & 2 & \textbf{6} & 0 \\
Revision\_better & 6 & 9 & \textbf{20} \\
Revision\_necessary & 2 & 0 & \textbf{9} \\
\bottomrule
\end{tabular}
\caption{Prediction distribution from \texttt{GPT2-large}. \textbf{Orig} indicates preference for the original connective; \textbf{Revi}, for the revised connective; and \textbf{Equal} indicates both versions received identical surprisal values.}

\label{tab:gpt2-results}
\end{table}
Despite some variation across individual categories, the overall distributional patterns between both models are largely consistent. A \textit{chi-square test} performed across categories revealed no statistically significant differences between two models' predictions.

We also evaluated \texttt{GPT-4.1-mini} as a classifier by prompting it with the same 5-way classification instructions provided to our human annotators. Each of the 100 instances was labeled through 5 runs, and the majority prediction was compared to the human majority label. The model’s predictions matched the human majority in 43\% of the cases, with a moderate association between the two label sets (Cramér’s V = 0.306, p-value = 0.002).


\begin{table}[t]
\centering
\begin{tabular}{lccc}
\toprule
\textbf{Gold label} & \textbf{Equal} & \textbf{Orig} & \textbf{Revi} \\
\midrule
Either\_way & \textbf{11} & 9 & 16 \\
Original\_better & 3 & \textbf{5} & 2 \\
Original\_should\_stay & 2 & \textbf{5} & 1 \\
Revision\_better & 10 & 3 & \textbf{22} \\
Revision\_necessary & 2 & 0 & \textbf{9} \\
\bottomrule
\end{tabular}
\caption{Prediction distribution from \texttt{GPT-4.1} on the manually annotated dataset.}
\label{tab:gpt4-results}
\end{table}

We further evaluated open-source LLMs, specifically \texttt{Mistral-7B} and \texttt{LLaMA-3.1-8B}. Both models had difficulty adhering to the prompt instructions %, frequently producing outputs that are not discourse connectives. When 
unless supplied with a short list of example connectives. 
%their performance improved; 
In consequence, they heavily favored items from the provided list, resulting in reduced lexical diversity. To address this limitation, we also experiment with a few-shot prompting strategy rather than explicitly listing options. Under this configuration, \texttt{Mistral-7B} exhibited improved performance, producing valid connectives more consistently and with greater lexical variety. 
However, no statistically significant correlation with human annotations was found ($p > 0.05$; see Table \ref{tab:mistral_match_majority} in Appendix~\ref{app:mistral_vs_human} for the predictions of \texttt{Mistral-7B}).

\section{Conclusion}
%\textit{TODO: Details on Mistral (it is not properly following the instructions, but its results can be put in the appendix).}
We present a dataset of text revisions involving the deletion or replacement of discourse connectives at the beginning of sentences in the WikiHow text revisions.\footnote{The dataset is publicly available at: \url{https://github.com/berfingit/connective-deletion-replacement}.} From this dataset, we manually annotated 100 instances of connective replacements. Only 11\% of these edits were judged necessary to convey the correct discourse meaning, whereas in 8\% cases, the original connective was favored; the remaining edits appeared to be somewhat optional.

Using \texttt{GPT-2}, we computed information-theoretic metrics (mean surprisal, variance, and smoothness) for these annotations. Of these, only mean surprisal significantly correlated with human judgments (RQ1). A prompt-based evaluation with \texttt{GPT-4.1-mini} showed similar preferences, especially in edge cases of 5-way classification, where a revision was necessary or the original connective should be retained (RQ2).
These results suggest that information-theoretic metrics and prompt-based methods capture some patterns in human decisions on discourse connectives, though their coverage remains limited.


\section*{Acknowledgements}

The research presented in this paper was funded by the DFG Emmy Noether program (RO 4848/2-1).

%Surprisal patterns don't appear to be informative, possibly because the massive size of the training data causes LLM predictions to diverge from human prediction patterns.

%The uniform information density (UID) hypothesis, which holds that information should be spread evenly throughout an utterance. This work studies the uniform information den- sity (UID) hypothesis as an explanatory principle for the speakers’ choices of whether to use connectives and if yes, which one.

 

% Additional results, tables, or analysis to follow...


\section*{Limitations}

Our human annotation analysis is limited to a small subset of the connective replacement data; expanding annotations to include more examples and other edit types (e.g., insertions and deletions) would strengthen the generalizability of our findings. Additionally, we do not explicitly ground our analysis in a theoretical framework such as the Uniform Information Density (UID) hypothesis, leaving open questions about the broader cognitive or linguistic implications of our results.

For computing surprisal-based metrics, we rely on \texttt{GPT-2}, a relatively outdated language model. This choice is motivated by the lack of token-level log-probability access in widely used API-based models like \texttt{GPT-3.5} and \texttt{GPT-4}. While more recent open-source models such as \texttt{Mistral-7B} and \texttt{LLaMA-3.1} provide access to logits (via the \texttt{transformers} library) and can be used for surprisal computation, integrating them was beyond the scope of this pilot study. We leave surprisal-based experiments with more recent models to future work.



% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{anthology,custom}
\appendix

\section{Examples with Additional Context}
\label{app:examples}
\begin{figure}[H]
\centering
\begin{minipage}{0.95\linewidth}
\small
\textbf{Source:} \textit{Choreograph a Great Solo} \\
\textbf{Section:} Steps

\medskip

\textbf{Original:} \\
2. Pick a music choice that agrees with your style choice. \textbf{[So]}, if your dance is fast, pick a fast song.

\medskip

\textbf{Revised:} \\
2. Pick a music choice that agrees with your style choice. \textbf{[For example]}, if your dance is fast, pick a fast song.
\end{minipage}
\caption{Example of connective replacement}
\label{fig:connective-replacement-example}
\end{figure}

\begin{figure}[H]
\centering
\begin{minipage}{0.95\linewidth}
\small
\textbf{Article:} \textit{Make a Summer Dress out of a Bedsheet} \\
\textbf{Section:} Steps

\medskip

\textbf{Original:} \\
9. Cut the four shirt pieces out of the sheet material. \textbf{[Then]} place those pieces on the lining material, face side up, and pin them into place before cutting them out.

\medskip

\textbf{Revised:} \\
9. Cut the four shirt pieces out of the sheet material. Place those pieces on the lining material, face side up, and pin them into place before cutting them out.
\end{minipage}
\caption{Example of connective deletion in revision}
\label{fig:connective-deletion-example}
\end{figure}

\begin{figure}[H]
\centering
\begin{minipage}{0.95\linewidth}
\small
\textbf{Source:} \textit{Chat Using Facebook Messenger App on iOS} \\
\textbf{Section:} Logging into Messenger

\medskip

\textbf{Original:} \\
1. Sign in. If you’re already using the Facebook app and have it installed and running on your mobile device, just tap on the blue “Continue as…” button upon launch. \textbf{[Otherwise]}, input your email and password for your Facebook account.

\medskip

\textbf{Revised:} \\
1. Sign in. If you’re already using the Facebook app and have it installed and running on your mobile device, just tap on the blue “Continue as…” button upon launch. Input your email and password for your Facebook account.
\end{minipage}
\caption{Example of misleading connective deletion.}
\label{fig:connective-needed}
\end{figure}

\begin{figure}[H]
\centering
\begin{minipage}{0.95\linewidth}
\small
\textbf{Source:} \textit{Use Every Nikon Digital SLR} \\

\medskip

\textbf{Original:} \\
\textbf{[While]} there are enough similarities between all Nikon digital SLRs. \\
These categorisations are used here for convenience's sake and have nothing to do with image quality.

\medskip

\textbf{Revised:} \\
There are enough similarities between all Nikon digital SLRs. \\
These categorisations are used here for convenience's sake and have nothing to do with image quality.
\end{minipage}
\caption{Example of connective deletion causing ungrammatical output.}
\label{fig:connective-ungrammatical}
\end{figure}



\section{Information-Theoretic Metrics on the Full Dataset}
\label{app:surprisal_wholedata}
We analyze the whole dataset using the metrics described in \textsection \ref{sec:uid-effects} and present the results in this section. 

\paragraph{Average token surprisal:} Connective insertions and replacements lead to a statistically significant increase in average surprisal, while deletions cause a significant decrease (Figure~\ref{fig:delta_mean}). An increase in average surprisal suggests that the resulting text became less predictable overall, whereas a decrease reflects a shift toward greater predictability.

Interestingly, the reverse of insertions (i.e., removing connectives that had been inserted by humans, labeled as \texttt{insertion\_rev}) also leads to a decrease in average surprisal, mirroring the effect of deletions. The difference between \texttt{deletion} and \texttt{insertion\_rev} is not statistically significant, suggesting that these two operations have symmetrical effects on average surprisal despite differing in context.

\begin{figure}[htbp]
    \centering
    \includegraphics[width=0.5\textwidth]{delta_mean_anova_plot.png}
    \caption{Mean change in surprisal ($\Delta$ Mean) across edit types. Error bars represent 95\% confidence intervals. ANOVA indicates a significant effect of edit type on mean surprisal (F = 59.76, p = 1.4e-26).}
    \label{fig:delta_mean}
\end{figure}

\paragraph{Variance:}
As shown in Figure~\ref{fig:delta_var}, both deletion and replacement significantly reduce surprisal variance, making the text more uniformly predictable. In contrast, insertions and therefore their reverse functions \texttt{insertion\_rev} do not significantly alter variance. 

This asymmetry between deletions and \texttt{insertion\_rev}, despite their functional similarity, suggests that while the mean surprisal remains stable, the local variance depends more on the editing context.
\begin{figure}[htbp]
    \centering
    \includegraphics[width=0.5\textwidth]{delta_var_anova_plot.png}
    \caption{Mean change in variance ($\Delta$ Var) across edit types. Error bars represent 95\% confidence intervals. ANOVA indicates a significant effect of edit type on surprisal variance (F = 55.11, p = 1.4e-24).}
    \label{fig:delta_var}
\end{figure}

\paragraph{Smoothness:}Figure~\ref{fig:delta_smooth} shows that insertions, deletions, and replacements each lead to small but statistically significant decreases in surprisal smoothness (i.e., negative values indicate that revisions are less smooth). However, the differences across edit types are not statistically significant, suggesting that all three introduce increases in local unpredictability.

There is a clear contrast between deletions and \texttt{insertion\_rev} in terms of smoothness. Removing human-inserted connectives appears to make the surprisal values change more gradually, yielding a smoother predictability pattern.

\begin{figure}[htbp]
    \centering
    \includegraphics[width=0.5\textwidth]{delta_smooth_anova_plot.png}
    \caption{Mean change in smoothness ($\Delta$ Smooth) across edit types. Error bars represent 95\% confidence intervals. ANOVA does a not indicate a significant effect of edit type on smoothness (F = 0.42, p = 0.66).}
    \label{fig:delta_smooth}
\end{figure}

\paragraph{Discussion}

These results show that while \texttt{deletion} and \texttt{insertion\_rev} have similar effects on average surprisal, they differ notably in their impact on surprisal variance and smoothness. This suggests that although a connective’s overall informativeness may remain stable, its influence on local predictability (i.e., whether it smooths or disrupts the flow) depends on context.

Overall, the surprisal-based metrics offer a nuanced view of how connective edits affect a language model’s expectations (as measured by \texttt{GPT-2}). Interestingly, connective insertions and replacements increase average surprisal and reduce smoothness, indicating that these edits do not improve predictability as one might expect with explicit connectives. Since some of these findings run counter to intuition and we do not yet provide a deeper analysis, we consider them inconclusive.



\section{Prompts}
\label{app:prompts}

\noindent
\textbf{Prompt Example:}

\begin{quote}
\small
\texttt{TASK: Insert an appropriate discourse connective into the gap marked by <...> in the following text.}

\medskip

\texttt{Text:}

\texttt{Steps 1. To easily interpret your quotation and make a plausible argument and analysis, ask yourself questions. For example, why was this said? What possible situation made this quotation significant? 2. For your main points, ask yourself questions based on your interpretation. Always remember why, what, and how. 3. <...> you have your main points, come up with solid examples. Books, movies, politics, current events, music, art, culture, history, and any other category will work. Just remember to not use all of one category! Mix them up for a more solid analysis.}

\medskip

\texttt{Answer:}
\end{quote}

\section{Mistral-7B's Connective Predictions on Annotated Data}
\label{app:mistral_vs_human}
\FloatBarrier

\begin{table}[H]
\centering
\begin{tabular}{lccc}
\toprule
\textbf{Human Majority} & \textbf{Equal} & \textbf{Orig} & \textbf{Revi} \\
\midrule
Either\_way             & 31            & 2             & 3             \\
Original\_better        & 10            & 0             & 0             \\
Original\_should\_stay  & 5             & 2             & 1             \\
Revision\_better        & 27            & 6             & 2             \\
Revision\_necessary     & 10            & 0             & 1             \\
\bottomrule
\end{tabular}
\caption{Mistral-7B's match breakdown by human majority class}
\label{tab:mistral_match_majority}
\end{table}



\section{Frequency Distribution of Connective Deletions and Replacements}
\label{sec:appendix_stats}

\begin{table}[H]
\centering
\begin{tabular}{lrr}
\toprule
\textbf{Connective} & \textbf{Count} & \textbf{Normalized (\%)} \\
\midrule
Then            & 882 & 20.5\% \\
For example     & 669 & 15.5\% \\
However         & 558 & 13.0\% \\
Also            & 425 & 9.9\%  \\
But             & 237 & 5.5\%  \\
Or              & 185 & 4.3\%  \\
And             & 177 & 4.1\%  \\
So              & 174 & 4.0\%  \\
For instance    & 139 & 3.2\%  \\
If              & 121 & 2.8\%  \\
Finally         & 118 & 2.7\%  \\
Instead         & 106 & 2.5\%  \\
In addition     & 70  & 1.6\%  \\
Otherwise       & 42  & 1.0\%  \\
In fact         & 35  & 0.8\%  \\
\bottomrule
\end{tabular}
\caption{Most frequent connective insertions with raw counts and normalized percentages.}
\label{tab:connective-insertions}
\end{table}


\begin{table}[H]
\centering
\begin{tabular}{lrr}
\toprule
\textbf{Connective} & \textbf{Count} & \textbf{Percentage (\%)} \\
\midrule
Then           & 4287 & 31.5\% \\
Also           & 1690 & 12.4\% \\
And            & 1600 & 11.8\% \\
So             & 971  & 7.1\%  \\
But            & 960  & 7.1\%  \\
However        & 917  & 6.7\%  \\
Finally        & 851  & 6.3\%  \\
Or             & 414  & 3.0\%  \\
For example    & 374  & 2.8\%  \\
Instead        & 147  & 1.1\%  \\
Because        & 120  & 0.9\%  \\
In order       & 114  & 0.8\%  \\
Therefore      & 99   & 0.7\%  \\
If             & 97   & 0.7\%  \\
For instance   & 88   & 0.6\%  \\
\bottomrule
\end{tabular}
\caption{Most frequent connective deletions with raw counts and normalized percentages.}
\label{tab:connective-deletions}
\end{table}


\begin{table}[H]
\centering
\begin{tabular}{lrr}
\toprule
\textbf{Replacement} & \textbf{Count} & \textbf{Percentage (\%)} \\
\midrule
But $\rightarrow$ However         & 314 & 17.1 \\
When $\rightarrow$ If             & 106 & 5.8  \\
If $\rightarrow$ When             & 80  & 4.3  \\
And $\rightarrow$ Also            & 65  & 3.5  \\
Because $\rightarrow$ Since       & 39  & 2.1  \\
However $\rightarrow$ But         & 35  & 1.9  \\
If $\rightarrow$ Even if          & 31  & 1.7  \\
Also $\rightarrow$ In addition    & 29  & 1.6  \\
When $\rightarrow$ Once           & 23  & 1.2  \\
And $\rightarrow$ In addition     & 22  & 1.2  \\
Once $\rightarrow$ When           & 18  & 1.0  \\
Even if $\rightarrow$ If          & 18  & 1.0  \\
While $\rightarrow$ Although      & 18  & 1.0  \\
Then $\rightarrow$ Finally        & 18  & 1.0  \\
So $\rightarrow$ Therefore        & 18  & 1.0  \\
\bottomrule
\end{tabular}
\caption{Most frequent connective replacements with raw counts and normalized percentages.}
\label{tab:replacements}
\end{table}


\end{document}
