\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{hyperref}
\usepackage{caption}
\usepackage{tikz}
\usepackage{pgfplots}
\usepackage{float}
\usepackage{placeins}
\pgfplotsset{compat=1.18}
\usepackage{acl}
\usepackage{times}
\usepackage{latexsym}
\usepackage{microtype}
\usepackage{needspace}


\setcounter{topnumber}{5}
\setcounter{bottomnumber}{5}
\setcounter{totalnumber}{10}
\renewcommand{\textfraction}{0.05}
\renewcommand{\floatpagefraction}{0.8}
\setlength{\floatsep}{6pt plus 2pt minus 2pt}
\setlength{\textfloatsep}{8pt plus 2pt minus 4pt}
\setlength{\intextsep}{6pt plus 2pt minus 2pt}
\captionsetup{skip=4pt}


\title{FER at SemEval-2026 Task 6: Analysis of Different Approaches to
Unmasking Political Question Evasions}
\author{Matija Akrap, Andrija Bilić, Luka Čuturilo, Fran Račić, Roko Šimpraga \\
  Faculty of Electrical Engineering and Computing \\
  University of Zagreb \\
  \texttt{\{first.last\}@fer.hr}
}

\begin{document}
\maketitle

\begin{abstract}
We tackle the problem of classification of political ambiguity within the
context of SemEval-2026 Task 6.  We compare three modeling strategies: a
flat baseline, a hierarchical cascade, and a multitask learning approach.
Our experiments demonstrate that a hierarchical RoBERTa-base model
achieves the best performance, particularly by leveraging the
distinctiveness of the class \textit{Clear Non-Reply}.  Conversely, we
find that standard multitask learning frequently produces structurally
invalid label combinations in approximately 25\% of predictions.  We
demonstrate that applying a constrained inference mask eliminates these
errors entirely while improving $F_1$ performance, whereas a fully joint
training approach falters due to data sparsity.  Finally, we employ
Dataset Cartography to diagnose training dynamics, revealing that while
hierarchical models suffer from error propagation, they offer a more
calibrated representation of uncertainty compared to flat classifiers.
\end{abstract}

\section{Introduction}

Politicians often give evasive answers to sensitive or controversial
questions during interviews.  Such evasions can obscure accountability
and mask true intentions, making their automatic detection an important
problem for political science and natural language processing.

This work focuses on detecting and categorizing evasions of political
questions as defined in SemEval-2026 Task 6
\citep{thomas2026semeval2026task6clarity}.  We follow the taxonomy and
dataset introduced by \citet{thomas-etal-2024-never}.  The task is
structured as a two-level classification problem.  At the first level,
each answer is assigned a \textit{clarity label}, indicating whether the
response is a \textit{Clear Reply}, a \textit{Clear Non-Reply}, or an
\textit{Ambivalent} answer.  At the second level, answers are further
annotated with fine-grained evasion categories, corresponding to the
specific techniques used by the speaker.  These fine-grained labels are
hierarchically dependent on the clarity level; for instance,
\textit{dodging} is only applicable to ambivalent replies (Figure \ref{fig:taxonomy}).

We report accuracy, precision, recall, and macro $F_1$ as mean $\pm$
standard deviation over five runs with different random seeds.  For each
model evaluation, we did five runs with random seeds.

We focus on the approaches described in Figure \ref{fig:methodology_diagram}.
Our code can be accessed on GitHub.\footnote{\url{https://github.com/ma55530/SemEval2026-CLARITY-FER}}

\begin{figure}[t]
    \centering
    \includegraphics[width=\columnwidth]{figures/taxonomy.png}\\[1ex]
    \caption{Two-level taxonomy of political answer types \citep{thomas-etal-2024-never}: three top-level clarity labels (Clear Reply,
    Ambivalent, Clear Non-Reply) and the fine-grained evasion categories. }
    \label{fig:taxonomy}
\end{figure}


\begin{figure}[t]
    \centering
    \includegraphics[width=\columnwidth]{figures/methodology_diagram.png}
    \caption{An overview of the approaches used for the \textit{Clarity} and \textit{Evasion} tasks.}
    \label{fig:methodology_diagram}
\end{figure}

\section{Related Work}

Political evasion detection has transitioned from foundational social science typologies \citep{bull-2003} to formal computational benchmarks. While traditional studies focused on speaker intent and subjective equivocation, \citet{thomas-etal-2024-never} emphasize objective response clarity using fine-tuned transformer encoders or large language models (LLMs) through zero-shot and chain-of-thought prompting. \citet{mancini-etal-2024-multimodal} task is distinct from related phenomena like argumentative fallacy detection, which focuses on logical validity rather than the pragmatic alignment of a response to a question.

Hierarchical text classification (HTC) leverages taxonomic structures to improve performance over flat baselines, typically through local cascades or global models that integrate label dependencies \citep{silla-freitas-2011, dampa-2025-investigating}. Multitask learning (MTL) further enhances generalization by sharing representations across related tasks, a strategy applied in joint sentiment and tone analysis \citep{baric-etal-2023-target}. However, joint training in low-data regimes can suffer from data sparsity and task interference \citep{hanneke-xu-2026-limits}, leading to logically incompatible label predictions. Recent research addresses these inconsistencies through structural topology modeling and constrained inference to enforce logical dependencies across hierarchy levels \citep{wulamu-etal-2025-html}.


\section{Data Analysis and Evaluation}


The hierarchical label structure poses several challenges, including
strong class imbalance, semantic overlap between evasion categories, and
ambiguity in borderline cases \citep{thomas-etal-2024-never}.

Given the nature of the problem, the evaluation methodology requires
specific attention.  Unlike the training dataset, the test dataset
contains three annotator labels.  We follow the official task evaluation
metric: if the model's prediction matches any of the three labels, we
classify that prediction as correct.  This applies to all metrics.

Each input sequence was formed as \texttt{Question:} $Q$ \texttt{Answer:}
$A$, where $Q$ and $A$ denote the interview question and answer,
respectively.

\subsection{Dataset Statistics}

The dataset consists of 3,448 QA pairs with 4 distinct presidents, along with significant class imbalance (~\ref{tab:label_dist}).

\begin{table}[htbp]
\centering
\small
\begin{tabular}{lrr}
\toprule
\textbf{Label} & \textbf{Count} & \textbf{\%} \\
\midrule
\multicolumn{3}{c}{\textit{Clarity Labels}} \\
Clear Non-Reply  &   356 & 10.3\% \\
Clear Reply      & 1,052 & 30.5\% \\
Ambivalent       & 2,040 & 59.2\% \\
\midrule
\multicolumn{3}{c}{\textit{Top Evasion Labels}} \\
Explicit  & 1,052 & 30.5\% \\
Dodging   &   706 & 20.5\% \\
Implicit  &   488 & 14.2\% \\
\bottomrule
\end{tabular}
\caption{Distribution of clarity labels and the three most frequent
fine-grained evasion labels in the training set.}
\label{tab:label_dist}
\end{table}

\subsection{Text Length Distribution}

The average answer length varies significantly across clarity labels.
As shown in Figure~\ref{fig:length_dist}, \textit{Clear Non-Replies} are
substantially shorter than other categories, a feature that likely aids
the model in distinguishing this class.

\begin{figure}[htbp]
    \centering
    \resizebox{\columnwidth}{!}{%
        \begin{tikzpicture}
            \begin{axis}[
                width=8cm, height=4cm,
                xbar,
                xmin=0, xmax=2200,
                xlabel={Average Character Length},
                symbolic y coords={Clear Non-Reply, Clear Reply, Ambivalent},
                ytick=data,
                nodes near coords,
                nodes near coords align={horizontal},
                y axis line style={opacity=0},
                axis x line=bottom,
                tickwidth=0pt,
                enlarge y limits=0.3,
                every y tick label/.style={font=\small}
            ]
            \addplot coordinates {(772,Clear Non-Reply) (1542,Clear Reply) (1882,Ambivalent)};
            \end{axis}
        \end{tikzpicture}%
    }
    \caption{Average answer length (in characters) by clarity label,
    computed over the training set.}
    \label{fig:length_dist}
\end{figure}

\section{Baseline Flat Classification}

As a simple baseline, we train two independent BERT-base
\citep{devlin-etal-2019-bert} classifiers to directly predict the clarity
and evasion labels without exploiting the hierarchical structure of the
taxonomy.  In this setup, the task is treated as a standard multi-class
text classification problem, where each answer is assigned exactly one
category: either one of the three clarity levels or one of the nine
evasion categories. 

As shown in Table~\ref{tab:overall_results}, the flat BERT classifier
achieves a macro $F_1$ of 0.577 on the clarity task and 0.341 on the
evasion task.  These results establish a strong reference for the
structured models described below.  While accuracy on the clarity task
reaches 0.630, the evasion task proves considerably harder, with accuracy
of only 0.405, reflecting the greater number of fine-grained categories
and the inherent semantic overlap among evasion types.  This flat
classification approach serves as a baseline for comparison with the
hierarchical and multitask models presented in the following sections.

\section{Hierarchical Approach}


To improve classification at the clarity level, we employed a
hierarchical strategy and a more powerful BERT model -- RoBERTa \citep{liu2019robertarobustlyoptimizedbert}.  First, we trained a binary model to distinguish
one clarity class from the others.  Then, a second model is trained to
differentiate between the remaining two classes.

We applied this approach in two ways: first, by distinguishing the
\textit{Clear Reply} class from all others, and second, by distinguishing
the \textit{Clear Non-Reply} class from all others.  We hypothesized that
isolating either the \textit{Clear Replies} or \textit{Clear Non-Replies}
first would give better results, rather than separating the ambiguous
\textit{Ambivalent} class.

\subsection{Clear Reply vs. Rest}

In the first hierarchical variant, a BERT-base binary model separates
\textit{Clear Reply} examples from all others before a second model
resolves the remaining two classes.  As reported in
Table~\ref{tab:overall_results}, this pipeline achieves a macro $F_1$ of
0.556 on the clarity task -- slightly below the flat baseline, suggesting
that \textit{Clear Reply} does not provide a sufficiently distinctive
signal to benefit from early isolation.

\subsection{Clear Non-Reply vs. Rest}

In the second variant, the binary stage instead isolates \textit{Clear
Non-Reply} examples.  This choice is motivated by the notably shorter
average answer length of this class (772 characters, versus 1,542 and
1,882 for \textit{Clear Reply} and \textit{Ambivalent}, respectively; see
Figure~\ref{fig:length_dist}), which reduces semantic ambiguity and makes
the binary decision easier.  With BERT-base, this pipeline reaches a
macro $F_1$ of 0.577, matching the flat baseline. Although no significant improvement, switching to RoBERTa-base yields a further improvement to $F_1 = 0.603$, the best
clarity result overall (Table~\ref{tab:overall_results}).  The gain over
the \textit{Clear Reply}-first variant confirms that early isolation of
the most linguistically distinctive class is key to hierarchical
performance.


\section{Multitask Learning Approach}

Following \citet{baric-etal-2023-target}, we also explored a multitask learning (MTL) approach to leverage common
patterns in the data, leading to better generalization and reduced
overfitting.

In our initial baseline setup, we employed a shared encoder to transform
input text into latent embeddings, upon which we attached separate,
independent task-specific heads for predicting clarity and evasion
levels.  Each head has its own output layer and loss function, while the
encoder weights are shared across all tasks.  The overall loss function
is the weighted sum of the individual task losses:
\[
\mathcal{L}_{\text{total}} =
  \lambda_{\text{evasion}} \, \mathcal{L}_{\text{evasion}}
  + \lambda_{\text{clarity}} \, \mathcal{L}_{\text{clarity}}
\]
where $\lambda_{\text{evasion}}$ and $\lambda_{\text{clarity}}$ are loss
weights for each task.

However, we observed that this architecture frequently produced invalid
label combinations (e.g., predicting a ``Clear Reply'' alongside a
``Dodging'' evasion technique), as the independent heads do not
inherently respect the hierarchical constraints of the taxonomy.
To address this structural incoherence, we extended our study to include
two additional modeling strategies:

\begin{itemize}
    \setlength\parskip{0em}
    \item \textbf{Unified label space.}  We replaced the independent heads
    with a single classification head that predicts the Cartesian product
    of all valid label pairs, explicitly modeling the joint distribution
    $P(C, E)$ and making illegal pairs impossible.
    \item \textbf{Constrained inference.}  We retained the efficient
    independent training objective but augmented the model with a post-hoc
    validity mask during inference.  This method computes the joint
    probability $P(C, E) = P(C) \times P(E)$ and forces structurally
    invalid pairs to zero before selecting the final prediction.
\end{itemize}

As shown in Table~\ref{tab:overall_results}, the independent MTL baseline
achieves an evasion $F_1$ of 0.424.  The unified joint model drops to
0.373, suffering from data sparsity introduced by fragmenting labels into
a large Cartesian product space.  The constrained model not only
eliminates all structurally invalid predictions but also achieves the
highest evasion $F_1$ of 0.438, with accuracy 0.435, precision 0.475, and
recall 0.518.  This confirms that enforcing logical constraints during
inference helps filter noise without incurring the sparsity cost of joint
training.


\section{Overall Results}


Table~\ref{tab:overall_results} presents a comprehensive side-by-side
comparison of all modeling approaches across both tasks.  For each system
we report average accuracy, precision, recall, and macro $F_1$ (mean
$\pm$ standard deviation over five seeds).  The best result per task is
highlighted in \textbf{bold}; the $\dagger$ symbol marks the single best
system overall.

\begin{table*}[t!]
\centering
\small
\renewcommand{\arraystretch}{1.15}
\begin{tabular}{ll l cccc}
\toprule
\textbf{Task} & \textbf{Approach} & \textbf{Backbone} &
\textbf{Acc} & \textbf{Prec} & \textbf{Rec} &
\textbf{Macro $F_1$ (Avg $\pm$ Std)} \\
\midrule
\multirow{5}{*}{\textit{Clarity}}
  & Flat baseline
    & BERT-base               & 0.6300 & 0.6450 & 0.5650 & 0.5770 $\pm$ 0.0075 \\
  & Hier.\ (Clear Reply vs.\ Rest)
    & BERT-base               & 0.7091 & 0.6093 & 0.5574 & 0.5564 $\pm$ 0.0255 \\
  & \textbf{Hier.\ (Clear Reply vs.\ Rest)}$^\dagger$
    & \textbf{RoBERTa-base}   & \textbf{0.7318} & \textbf{0.6512}
                              & \textbf{0.6530} & \textbf{0.6507 $\pm$ 0.0108} \\
  & Hier.\ (Clear Non-Reply vs.\ Rest)
    & BERT-base               & 0.7097 & 0.6276 & 0.5671 & 0.5766 $\pm$ 0.0111 \\
  & \text{Hier.\ (Clear Non-Reply vs.\ Rest)}
    & \text{RoBERTa-base}   & \text{0.7039} & \text{0.6216}
                              & \text{0.6270} & \text{0.6029 $\pm$ 0.0143} \\
\midrule
\multirow{5}{*}{\textit{Evasion}}
  & Flat baseline
    & BERT-base               & 0.4053 & 0.4274 & 0.3210 & 0.3406 $\pm$ 0.0128 \\
  & MTL Independent
    & BERT-base               & ---    & ---    & ---    & 0.4241 $\pm$ 0.0216 \\
  & MTL Joint (unified label space)
    & BERT-base               & ---    & ---    & ---    & 0.3732 $\pm$ 0.0124 \\
  & \textbf{MTL Constrained (masked)}
    & \textbf{BERT-base}      & \textbf{0.4349} & \textbf{0.4746}
                              & \textbf{0.5183} & \textbf{0.4380 $\pm$ 0.0225} \\
\bottomrule
\end{tabular}
\caption{Comprehensive comparison of all modeling approaches across the
clarity and evasion tasks.  Each row reports mean accuracy, precision,
recall, and macro $F_1$ ($\pm$ standard deviation across five random
seeds).  Best result per task is in \textbf{bold};
$\dagger$ marks the overall best system.
The best
configuration per task is shown (no weights for clarity; with class weights for
evasion).  ``---''\ indicates that breakdown metrics were not recorded for
these MTL variants.}
\label{tab:overall_results}
\end{table*}


\section{Error Analysis}


To better understand the performance of our models, we employed
\textit{Dataset Cartography}~\citep{swayamdipta2020datasetcartographymappingdiagnosing},
a method for diagnosing datasets based on training dynamics.  This
approach tracks the confidence, variability, and correctness of model
predictions across training epochs, allowing us to categorize samples as
\textit{Easy-to-learn}, \textit{Ambiguous}, or \textit{Hard-to-learn}.

\textit{Easy-to-Learn} samples are associated
with high confidence and low variability, indicating that the model
consistently predicts them correctly.  \textit{Hard-to-Learn} samples
exhibit low confidence and low variability, suggesting systematic
misclassification.  Finally, \textit{Ambiguous} samples are characterized
by higher variability and moderate to high confidence, reflecting
inherent ambiguity or semantic overlap between classes.

By visualizing these categories, we can identify regions in the dataset
where the model struggles and understand which examples contribute most
to errors or uncertainty.


\subsection{BERT Baseline}

\begin{figure}[htbp]
    \centering
    \includegraphics[width=0.9\linewidth]{figures/baseline_Dataset.png}
    \caption{Dataset cartography map for the flat BERT baseline on the
    clarity task, showing the distribution of training instances across
    confidence and variability.}
    \label{fig:dataset_cartography}
\end{figure}

\subsection{Hierarchical Approach}

The hierarchical approach generally improved classification for the
\textit{Clear Non-Reply} class, likely due to the shorter average answer
length, which reduces semantic ambiguity.

\begin{figure}[htbp]
    \centering
    \includegraphics[width=\columnwidth]{figures/hdcm.png}
    \caption{Dataset cartography map for the hierarchical pipeline
    (\textit{Clear Non-Reply} vs.\ Rest, RoBERTa-base), aggregated
    across both stages.}
    \label{fig:ierarchical_cartography_overall}
\end{figure}

Comparing the hierarchical and BERT baseline cartography maps reveals
distinct training dynamics.  The BERT baseline exhibits a characteristic
``V''-shaped distribution, separating easy and hard examples with low
variability.

In contrast, the hierarchical model produces a more diffuse
distribution.
Rather than sharply separating \textit{Easy-to-learn} and
\textit{Hard-to-learn} instances, the hierarchical model produces a
broader band of confidence predictions with slightly decreased variability
compared to the BERT baseline map.  This effect can be attributed to
error propagation in the hierarchical pipeline: a \textit{Clear
Non-Reply} instance misclassified by the first binary model as ``Rest''
is routed to the fine-grained model, where it can no longer be correctly
labeled.  Consequently, mistakes at the top level cascade downstream,
limiting the overall accuracy and $F_1$ scores.  If the binary model
could be improved significantly, or by introducing a \textit{fallback
model}, the propagation errors could be reduced.

Overall, the absence of a ``V'' shape in the hierarchical cartography
highlights a key trade-off: while the hierarchical model sacrifices some
decisiveness, it gains a more calibrated representation of uncertainty,
particularly for semantically overlapping or structurally constrained
labels.

\begin{figure}[htbp]
    \centering
    \includegraphics[width=0.95\columnwidth]{figures/binary_cm.png}
    \caption{Confusion matrix for the binary clarity model
    (\textit{Clear Non-Reply} vs.\ \textit{Rest}) at the top level of
    the hierarchical pipeline.}
    \label{fig:hierarchical_cartography_binary}
\end{figure}

The confusion matrix in Figure~\ref{fig:hierarchical_cartography_binary}
highlights significant class imbalance and several misclassifications.
Nine examples were incorrectly classified as \textit{Reply (Rest)} at
the binary level, which are then routed to the fine-grained model and,
as discussed above, are misclassified again.  Additionally, thirteen
examples predicted as \textit{Clear Non-Reply} never enter the
fine-grained model, effectively bypassing the downstream correction step.

\begin{figure}[htbp]
    \centering
    \includegraphics[width=0.95\columnwidth]{figures/fine_cm.png}
    \caption{Confusion matrix for the fine-grained clarity model
    (\textit{Ambivalent} vs.\ \textit{Clear Reply}) at the second level
    of the hierarchical pipeline.}
    \label{fig:mdcmierarchical_cartography_fine}
\end{figure}


\subsection{Multitask Approach}

Multitask models exhibited distinct training dynamics compared to the
hierarchical approach.  While the initial motivation was to leverage
shared representations to improve generalization, our experiments
revealed that the standard architecture, utilizing independent prediction
heads, suffered from structural inconsistencies.

\begin{figure}[htbp]
    \centering
    \includegraphics[width=\columnwidth]{figures/Clarity vs Evasion Predictions.png}
    \caption{Predicted clarity-evasion label pairs from the independent
    multitask baseline; structurally invalid pairs (e.g., \textit{Clear
    Reply} with \textit{Dodging}) are highlighted in red.}
    \label{fig:Clarity_vs_Evasion_Predictions}
\end{figure}

As shown in Figure~\ref{fig:Clarity_vs_Evasion_Predictions}, the
independent multitask baseline frequently produces structurally invalid
label combinations.  We quantified this error rate and found that
approximately 26.8\% of test instances resulted in ``illegal'' pairs.
This behavior arises because the independent heads optimize separate loss
functions, causing the model to learn independent probabilities $P(\text{clarity}
| x)$ and $P(\text{evasion} | x)$ but fail to model their joint
distribution $P(\text{clarity}, \text{evasion} | x)$.

\begin{figure}[htbp]
    \centering
    \includegraphics[width=\linewidth, keepaspectratio]{figures/dct.png}
    \caption{Dataset cartography map for the multitask model with two
    independent prediction heads (the baseline MTL setup).}
    \label{fig:mdcm}
\end{figure}

The cartography reveals that a substantial proportion of samples fall
into the \textit{Hard-to-learn} region with low confidence and low
variability.
This indicates that despite shared representations, the model struggles
to distinguish between overlapping linguistic cues in the
\textit{Ambivalent} and borderline categories.  The standard independent
model fails due to the structural invalidity shown in
Figure~\ref{fig:Clarity_vs_Evasion_Predictions}.  While joint training
methods are often recommended for related tasks
\citep{zhang-etal-2023-survey}, we found that the joint model fails in
our case due to the data sparsity visible in the ``Hard-to-Learn''
regions of Figure~\ref{fig:mdcm}.  This observation aligns with the
findings of \citet{hanneke2026datadoesnthelplimits}, who show that more
data does not necessarily improve model adaptation in multitask settings.
The constrained approach succeeds because it bridges this gap: it allows
the model to learn efficient independent representations, avoiding
sparsity while enforcing the necessary structural logic during inference.


\section{Conclusion}

In this work, we demonstrated that the inherent hierarchical structure of political question evasion can be effectively exploited to improve classification performance. Our findings show that the linguistic distinctiveness of the \textit{Clear Non-Reply} class makes it a prime candidate for early isolation in cascaded pipelines, leading to our best macro $F_1$ of 0.603. Furthermore, we addressed the prevalent issue of structural inconsistency in multitask learning — affecting 26.8\% of independent predictions — by implementing a constrained inference mask that ensures logical alignment with the taxonomy. While Dataset Cartography revealed significant semantic overlap in the \textit{Ambivalent} category, it also highlighted a more calibrated representation of uncertainty in our hierarchical approach. Future research should leverage instruction-tuned Large Language Models (LLMs) to tackle 'Hard-to-learn' fine-grained evasion techniques and explore more complex ensembles to mitigate error propagation.


\section*{Limitations}

The QEvasion dataset's focus on four U.S. presidents in English restricts the generalisation of the findings to diverse political systems, other languages, or different discourse formats like parliamentary debates. Significant class imbalance, where the Clear Non-Reply class comprises only 10.3\% of the data, further limits accuracy on classifying minority labels and increases evaluation variance. Our hierarchical pipeline is vulnerable to top-level error propagation, where initial clarity misclassifications irrevocably affect downstream evasion labeling. Additionally, our multitask experiments were restricted to standard architectures, omitting advanced structural learning methods such as label-aware attention or graph-based decoding. Further more, the official "match-any" evaluation metric may provide an optimistic estimate of performance compared to stricter agreement-based measures. Future work should transition from BERT-base encoders to larger, instruction-tuned models or LLMs and also explore more sophisticated ensambles.
Finally, we were limited by our computing power and Google Colab runtime limits.
\clearpage
\FloatBarrier
\bibliographystyle{acl_natbib}
\bibliography{references}

\appendix


\section{Traditional Machine Learning Baselines}
\label{app:traditional_ml}

We evaluated a set of traditional machine learning models that do not
rely on transformer-based architectures.  These models --- SVMs, Random
Forests, and others --- serve as classical baselines for the clarity and
evasion classification tasks, establishing a performance reference point
before the transformer-based approaches described in the main paper.
Full results for the clarity task (Table~\ref{tab:clarity_ml_baselines})
and the evasion task (Table~\ref{tab:evasion_ml_baselines}) are provided
below; the best configuration per task is also included in the overall
comparison in Table~\ref{tab:overall_results}.

\subsection{Clarity Classification}

\begin{table}[htbp]
\centering
\resizebox{\columnwidth}{!}{%
\begin{tabular}{llcccc}
\toprule
Scenario & Model & Avg Acc. & Avg Prec. & Avg Rec. & $F_1$ $\pm$ Std \\
\midrule
No Weights & Nearest Neighbors   & 0.6299 & 0.4708 & 0.4044 & \textbf{0.4154 $\pm$ 0.0000} \\
No Weights & Linear SVM          & 0.6688 & 0.2229 & 0.3333 & 0.2672 $\pm$ 0.0000 \\
No Weights & RBF SVM             & 0.6688 & 0.2229 & 0.3333 & 0.2672 $\pm$ 0.0000 \\
No Weights & Decision Tree       & 0.6747 & 0.5322 & 0.3698 & 0.3386 $\pm$ 0.0004 \\
No Weights & Random Forest       & 0.6688 & 0.2229 & 0.3333 & 0.2672 $\pm$ 0.0000 \\
No Weights & Neural Net          & 0.6630 & 0.6963 & 0.3667 & 0.3496 $\pm$ 0.0111 \\
No Weights & AdaBoost            & 0.6526 & 0.6393 & 0.3459 & 0.3119 $\pm$ 0.0000 \\
No Weights & Naive Bayes         & 0.6688 & 0.2229 & 0.3333 & 0.2672 $\pm$ 0.0000 \\
No Weights & Logistic Regression & 0.6721 & 0.7118 & 0.3841 & 0.3795 $\pm$ 0.0000 \\
\midrule
With Weights & Nearest Neighbors   & 0.6299 & 0.4708 & 0.4044 & \textbf{0.4154 $\pm$ 0.0000} \\
With Weights & Linear SVM          & 0.5812 & 0.2953 & 0.4570 & 0.3405 $\pm$ 0.0000 \\
With Weights & RBF SVM             & 0.5779 & 0.6620 & 0.3685 & 0.3682 $\pm$ 0.0000 \\
With Weights & Decision Tree       & 0.5844 & 0.3505 & 0.3892 & 0.3367 $\pm$ 0.0000 \\
With Weights & Random Forest       & 0.2234 & 0.3464 & 0.3819 & 0.1944 $\pm$ 0.0432 \\
With Weights & Neural Net          & 0.6656 & 0.6918 & 0.3638 & 0.3431 $\pm$ 0.0078 \\
With Weights & AdaBoost            & 0.6526 & 0.6393 & 0.3459 & 0.3119 $\pm$ 0.0000 \\
With Weights & Naive Bayes         & 0.6364 & 0.4435 & 0.4023 & 0.4008 $\pm$ 0.0000 \\
With Weights & Logistic Regression & 0.4968 & 0.3919 & 0.4001 & 0.3919 $\pm$ 0.0000 \\
\bottomrule
\end{tabular}%
}
\caption{Performance of traditional machine learning models on the
clarity classification task, with and without class-weights.  Best $F_1$
in each scenario is in \textbf{bold}.}
\label{tab:clarity_ml_baselines}
\end{table}

\clearpage
\subsection{Evasion Classification}

\begin{table}[htbp]
\centering
\resizebox{\columnwidth}{!}{%
\begin{tabular}{llcccc}
\toprule
Scenario & Model & Avg Acc. & Avg Prec. & Avg Rec. & $F_1$ $\pm$ Std \\
\midrule
No Weights & Nearest Neighbors   & 0.3117 & 0.2280 & 0.1522 & \textbf{0.1665 $\pm$ 0.0000} \\
No Weights & Linear SVM          & 0.3734 & 0.0415 & 0.1111 & 0.0604 $\pm$ 0.0000 \\
No Weights & RBF SVM             & 0.3701 & 0.0816 & 0.1142 & 0.0701 $\pm$ 0.0000 \\
No Weights & Decision Tree       & 0.3831 & 0.1207 & 0.1187 & 0.0823 $\pm$ 0.0000 \\
No Weights & Random Forest       & 0.3714 & 0.0562 & 0.1107 & 0.0612 $\pm$ 0.0025 \\
No Weights & Neural Net          & 0.3747 & 0.1295 & 0.1191 & 0.0906 $\pm$ 0.0066 \\
No Weights & AdaBoost            & 0.3636 & 0.1883 & 0.1191 & 0.0894 $\pm$ 0.0000 \\
No Weights & Naive Bayes         & 0.3734 & 0.0415 & 0.1111 & 0.0604 $\pm$ 0.0000 \\
No Weights & Logistic Regression & 0.3636 & 0.2365 & 0.1305 & 0.1203 $\pm$ 0.0000 \\
\midrule
With Weights & Nearest Neighbors   & 0.3117 & 0.2280 & 0.1522 & 0.1665 $\pm$ 0.0000 \\
With Weights & Linear SVM          & 0.1591 & 0.0263 & 0.1240 & 0.0434 $\pm$ 0.0000 \\
With Weights & RBF SVM             & 0.3636 & 0.1578 & 0.1367 & 0.1229 $\pm$ 0.0000 \\
With Weights & Decision Tree       & 0.3636 & 0.1279 & 0.2328 & 0.1235 $\pm$ 0.0000 \\
With Weights & Random Forest       & 0.3045 & 0.1741 & 0.1406 & 0.0904 $\pm$ 0.0100 \\
With Weights & Neural Net          & 0.3779 & 0.1371 & 0.1202 & 0.0903 $\pm$ 0.0067 \\
With Weights & AdaBoost            & 0.3630 & 0.1884 & 0.1191 & 0.0895 $\pm$ 0.0003 \\
With Weights & Naive Bayes         & 0.2825 & 0.1806 & 0.1646 & 0.1425 $\pm$ 0.0000 \\
With Weights & Logistic Regression & 0.3247 & 0.2618 & 0.2380 & \textbf{0.2200 $\pm$ 0.0000} \\
\bottomrule
\end{tabular}%
}
\caption{Performance of traditional machine learning models on the
fine-grained evasion classification task, with and without class-weights.
Best $F_1$ in each scenario is in \textbf{bold}.}
\label{tab:evasion_ml_baselines}
\end{table}

\end{document}