\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{xcolor}
\usepackage{amsmath}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{array}
\usepackage{hyperref}
\usepackage{caption}
\usepackage{tikz}
\usepackage{pgfplots}
\usepackage{float}
\usepackage{placeins}
\pgfplotsset{compat=1.18}
\usepackage{acl}
\usepackage{times}
\usepackage{latexsym}
\usepackage{microtype}
\usepackage{needspace}
\usepackage{stfloats}
\usepackage{subcaption}

\graphicspath{{./}{figures/}{../}}

\newcommand{\todo}[1]{{\color{purple}{\small [TODO: #1]}}}
\newcommand{\fix}[1]{{\color{cyan}{#1}}}

\setcounter{topnumber}{5}
\setcounter{bottomnumber}{5}
\setcounter{totalnumber}{10}
\setcounter{dbltopnumber}{3}
\renewcommand{\textfraction}{0.05}
\renewcommand{\floatpagefraction}{0.8}
\renewcommand{\dbltopfraction}{0.95}
\renewcommand{\dblfloatpagefraction}{0.8}
\setlength{\floatsep}{6pt plus 2pt minus 2pt}
\setlength{\textfloatsep}{8pt plus 2pt minus 4pt}
\setlength{\intextsep}{6pt plus 2pt minus 2pt}
\captionsetup{skip=4pt}


\title{FER at SemEval-2026 Task 6: Analysis of Different Approaches to
Unmasking Political Question Evasions}
\author{Matija Akrap, Andrija Bilić, Luka Čuturilo, Fran Račić, Roko Šimpraga \\
  Faculty of Electrical Engineering and Computing \\
  University of Zagreb \\
  \texttt{\{first.last\}@fer.hr}
}

\begin{document}
\maketitle

\begin{abstract}
We tackle classifying evasive political answers within the
context of SemEval-2026 Task 6 and compare three modeling strategies: a
flat baseline, a hierarchical cascade, and a multitask learning approach.
Our experiments demonstrate that a hierarchical RoBERTa-base model
achieves the best performance, particularly by leveraging the
distinctiveness of the class \textit{Clear Non-Reply}.  Conversely, we
find that standard multitask learning frequently produces structurally
invalid label combinations in a significant fraction of predictions.  Our demonstrations show that applying a constrained inference mask eliminates these
errors entirely while improving $F_1$ performance, whereas a fully joint
training approach underperforms due to data sparsity.  Finally, we employ
dataset cartography to compare training dynamics between the hierarchical and multitask approach.
\end{abstract}

\section{Introduction}

Politicians often give evasive answers to sensitive or controversial
questions during interviews.  Such evasions can obscure accountability
and mask true intentions, making their automatic detection an important
problem for political science and natural language processing.

This work focuses on detecting and categorizing evasions of political
questions as defined in SemEval-2026 Task 6
\citep{thomas2026semeval2026task6clarity}.  We follow the taxonomy and
dataset introduced by \citet{thomas-etal-2024-never}.  The task is
structured as a two-level classification problem.  At the first level,
each answer is assigned a \textit{clarity label}, indicating whether the
response is a \textit{Clear Reply}, a \textit{Clear Non-Reply}, or an
\textit{Ambivalent} answer.  At the second level, answers are further
annotated with fine-grained evasion categories, corresponding to the
specific techniques used by the speaker.  These fine-grained labels are
hierarchically dependent on the clarity level; for instance,
\textit{dodging} is only applicable to ambivalent replies.
The entire two-level taxonomy tree of political answer types is presented in Figure \ref{fig:taxonomy}.

We focus on the approaches described in Figure \ref{fig:methodology_diagram}.
Our code can be accessed on GitHub.\footnote{\href{https://github.com/ma55530/FER-SemEval2026-T6}{https://github.com/ma55530/FER-SemEval2026-T6}}

\begin{figure}[t]
    \centering
    \includegraphics[width=\columnwidth]{figures/Methodology_diagram.pdf}
    \caption{An overview of the approaches used for the \textit{Clarity} and \textit{Evasion} tasks.}
    \label{fig:methodology_diagram}
\end{figure}


\section{Related Work}

Political evasion detection has evolved from early social science typologies \citep{bull-2003} into a well-defined computational task. Whereas prior work focused on speaker intent and subjective interpretations of equivocation, \citet{thomas-etal-2024-never} formalize the problem as response clarity classification, introducing a two-level taxonomy and dataset of question–answer pairs from political interviews, and evaluating both fine-tuned models and large language model-based approaches. Related work has explored multimodal approaches to political discourse, such as argumentative fallacy detection in debates, where incorporating audio signals improves classification of fallacious arguments \citep{mancini-etal-2024-multimodal}.


Hierarchical text classification (HTC) leverages taxonomic structures to improve performance over flat baselines, typically through local cascades or global models that integrate label dependencies \citep{silla-freitas-2011, dampa-2025-investigating}. Multitask learning (MTL) further enhances generalization by sharing representations across related tasks, a strategy applied in joint sentiment and tone analysis \citep{baric-etal-2023-target}. However, joint training in low-data regimes can suffer from data sparsity and task interference \citep{hanneke-xu-2026-limits}, leading to logically incompatible label predictions. Recent work addresses these issues by explicitly modeling label dependencies or applying constrained inference to enforce hierarchical consistency \citep{wulamu-etal-2025-html}.

\begin{figure}
    \centering
    \includegraphics[width=\columnwidth]{taxonomy.png}\\[1ex]
    \caption{A two-level taxonomy of political answer types \citep{thomas-etal-2024-never}: top-level clarity labels (\emph{Clear Reply},
    \emph{Ambivalent}, \emph{Clear Non-Reply}) and the corresponding fine-grained evasion categories. }
    \label{fig:taxonomy}
\end{figure}


\section{Task Description}

The hierarchical label structure (Fig. ~\ref{fig:taxonomy}) poses several challenges, including
strong class imbalance, semantic overlap between evasion categories, and
ambiguity in borderline cases \citep{thomas-etal-2024-never}.

Given the nature of the problem, the evaluation methodology requires
specific attention. Unlike the training dataset, the test dataset
contains three annotator labels. We follow the official task evaluation
metric: if the model's prediction matches any of the three labels, we
classify that prediction as correct.  This applies to all metrics.

Each input sequence was formed as \texttt{Question:} $Q$ \texttt{Answer:}
$A$, where $Q$ and $A$ denote the interview question and answer,
respectively.

%We report accuracy, precision, recall, and macro $F_1$ as mean $\pm$ standard deviation over five runs with different random seeds for each model evaluation.

\paragraph{Dataset Statistics.}

The dataset consists of 3,448 QA pairs with 4 distinct presidents, along with significant class imbalance (Table ~\ref{tab:label_dist}).

\begin{table}
\centering
{\small
\begin{tabular}{lrr}
\toprule
\textbf{Label} & \textbf{Count} & \textbf{\%} \\
\midrule
\multicolumn{3}{c}{\textit{Clarity Labels}} \\
Clear Non-Reply  &   356 & 10.3\% \\
Clear Reply      & 1,052 & 30.5\% \\
Ambivalent       & 2,040 & 59.2\% \\
\midrule
\multicolumn{3}{c}{\textit{Top Evasion Labels}} \\
Explicit  & 1,052 & 30.5\% \\
Dodging   &   706 & 20.5\% \\
Implicit  &   488 & 14.2\% \\
\bottomrule
\end{tabular}}
\caption{Distribution of clarity labels and the three most frequent
fine-grained evasion labels in the training set.}
\label{tab:label_dist}
\end{table}

\paragraph{Text Length Distribution.}

The average answer length varies significantly across clarity labels.
As shown in Figure~\ref{fig:length_dist}, \textit{Clear Non-Replies} are
substantially shorter than other categories, a feature that likely aids
the model in distinguishing this class.

\begin{figure}
    \centering
    \resizebox{\columnwidth}{!}{%
        \begin{tikzpicture}
            \begin{axis}[
                width=8cm, height=4cm,
                xbar,
                xmin=0, xmax=2200,
                xlabel={Average Character Length},
                symbolic y coords={Clear Non-Reply, Clear Reply, Ambivalent},
                ytick=data,
                nodes near coords,
                nodes near coords align={horizontal},
                y axis line style={opacity=0},
                axis x line=bottom,
                tickwidth=0pt,
                enlarge y limits=0.3,
                every y tick label/.style={font=\small}
            ]
            \addplot coordinates {(772,Clear Non-Reply) (1542,Clear Reply) (1882,Ambivalent)};
            \end{axis}
        \end{tikzpicture}%
    }
    \caption{Average answer length (in characters) by clarity label,
    computed over the training set.}
    \label{fig:length_dist}
\end{figure}

\section{Methodology and Results}



This section presents the methodology and experimental results for each of the three 
modeling approaches explored in this work. 

We begin with a flat baseline that treats 
clarity and evasion classification as independent standard multi-class problems, 
establishing a reference point for comparison. 

We then describe two structured 
approaches: a hierarchical model that leverages the linguistic distinctiveness of 
clarity classes through two binary classifiers, and a multitask learning framework 
that jointly optimizes for both tasks using a shared encoder. 

For each approach, we 
report accuracy, precision, recall, and macro $F_1$ averaged over five random seeds. 
Hyperparameter configurations for all models are provided in Appendix~\ref{tab:baselineMTLhyperparamaters}.

\subsection{Baseline Flat Classification}
As a simple baseline, we train two independent BERT-base
 classifiers \citep{devlin-etal-2019-bert} to directly predict the clarity
and evasion labels without exploiting the hierarchical structure of the
taxonomy.  In this setup, the task is treated as a standard multi-class
text classification problem, where each answer is assigned exactly one
category: either one of the three clarity levels or one of the nine
evasion categories. 

As shown in Table~\ref{tab:overall_results}, the flat BERT classifier
achieves a macro $F_1$ of 0.577 on the clarity task and 0.341 on the
evasion task.  These results establish a strong reference for the
structured models described below.  While accuracy on the clarity task
reaches 0.630, fine-grained categories and semantic overlap make it harder for the evasion task, with accuracy
of only 0.405.  This flat
classification approach serves as a baseline for comparison with the
hierarchical and multitask models presented in the following sections.

\begin{table*}[!t]
\centering
\small
\renewcommand{\arraystretch}{1.15}
\begin{tabular}{ll l cccc}
\toprule
\textbf{Task} & \textbf{Approach} & \textbf{Backbone} &
\textbf{Acc} & \textbf{Prec} & \textbf{Rec} &
\textbf{Macro $F_1$ (Avg $\pm$ Std)} \\
\midrule
\multirow{5}{*}{\textit{Clarity}}
  & Traditional ML (KNN)$^\ddagger$
    & TF-IDF features         & 0.6299 & 0.4708 & 0.4044 & 0.4154 $\pm$ 0.0000 \\
  & Flat baseline
    & BERT-base               & 0.6300 & 0.6450 & 0.5650 & 0.5770 $\pm$ 0.0075 \\
  & Hier.\ (Clear Reply vs.\ Rest)
    & BERT-base               & 0.7091 & 0.6093 & 0.5574 & 0.5564 $\pm$ 0.0255 \\
  & Hier.\ (Clear Reply vs.\ Rest)
    & RoBERTa-base            & 0.5883 & 0.5171 & 0.6198 & 0.5355 $\pm$ 0.0155 \\
  & Hier.\ (Clear Non-Reply vs.\ Rest)
    & BERT-base               & 0.7097 & 0.6276 & 0.5671 & 0.5766 $\pm$ 0.0111 \\
  & \textbf{Hier.\ (Clear Non-Reply vs.\ Rest)}$^\dagger$
    & \textbf{RoBERTa-base}   & \textbf{0.7039} & \textbf{0.6216}
                              & \textbf{0.6270} & \textbf{0.6029 $\pm$ 0.0143} \\
\midrule
\multirow{5}{*}{\textit{Evasion}}
  & Traditional ML (Log.\ Reg.)$^\ddagger$
    & TF-IDF features         & 0.3247 & 0.2618 & 0.2380 & 0.2200 $\pm$ 0.0000 \\
  & Flat baseline
    & BERT-base               & 0.4053 & 0.4274 & 0.3210 & 0.3406 $\pm$ 0.0128 \\
  & MTL with Independent Heads
    & BERT-base               & 0.4080 & 0.4266 & 0.5090 & 0.4241 $\pm$ 0.0216 \\
  & MTL Joint (unified label space)
    & BERT-base               & 0.3615 & 0.3577 & 0.4781 & 0.3732 $\pm$ 0.0124 \\
  & \textbf{MTL Constrained (masked)}
    & \textbf{BERT-base}      & \textbf{0.4349} & \textbf{0.4746}
                              & \textbf{0.5183} & \textbf{0.4380 $\pm$ 0.0225} \\
\bottomrule
\end{tabular}
\caption{Comprehensive comparison of all modeling approaches across the
clarity and evasion tasks.  Each row reports mean accuracy, precision,
recall, and macro $F_1$ ($\pm$ standard deviation across five random
seeds).  Best result per task is in \textbf{bold};
$\dagger$ marks the overall best system.
$^\ddagger$Traditional ML results are deterministic (Std~=~0); the best
configuration per task is shown (no weights for clarity; with class weights for
evasion).}
\label{tab:overall_results}
\end{table*}

\subsection{Hierarchical Approach}


To improve classification at the clarity level, we employed a
hierarchical strategy and a more powerful BERT model -- RoBERTa \citep{liu2019robertarobustlyoptimizedbert}.  First, we trained a binary model to distinguish
one clarity class from the others.  Then, a second model is trained to
differentiate between the remaining two classes.

We consider two variants: one where a binary classifier first isolates the \textit{Clear Reply} class, and another where it first isolates the \textit{Clear Non-Reply} class. We hypothesized that
isolating either the \textit{Clear Replies} or \textit{Clear Non-Replies}
first would give better results, rather than separating the ambiguous
\textit{Ambivalent} class.

\subsection{Clear Reply vs. Rest}

In the first hierarchical variant, a BERT-base binary model separates
\textit{Clear Reply} examples from all others before a second model
resolves the remaining two classes.  As reported in
Table~\ref{tab:overall_results}, this pipeline achieves a macro $F_1$ of
0.556 on the clarity task -- slightly below the flat baseline, suggesting
that \textit{Clear Reply} does not provide a sufficiently distinctive
signal to benefit from early isolation.

\subsection{Clear Non-Reply vs. Rest}

In the second variant, the binary stage instead isolates \textit{Clear
Non-Reply} examples.  This choice is motivated by the notably shorter
average answer length of this class (772 characters, versus 1,542 and
1,882 for \textit{Clear Reply} and \textit{Ambivalent}, respectively; see
Figure~\ref{fig:length_dist}), which reduces semantic ambiguity and makes the binary decision easier.  With BERT-base we achieve $F_1$ = 0.577 (on par with the flat baseline). Replacing BERT with RoBERTa increases performance to $F_1$ = 0.603, our highest clarity score overall (Table~\ref{tab:overall_results}).  The gain over
the \textit{Clear Reply}-first variant confirms that early isolation of the most linguistically distinctive class is key to hierarchical
performance.


\subsection{Multitask Learning Approach}

Following the success of multitask learning (MTL) frameworks in capturing cross-task correlations \citep[e.g.,][]{baric-etal-2023-target}, we employ an MTL strategy to leverage commonalities across classification tasks, thereby improving robustness and reducing overfitting.

In our initial baseline setup, we employed a shared encoder to transform input text into latent embeddings, upon which we attached separate, independent task-specific heads for predicting clarity and evasion levels.  Each head has its own output layer and loss function, while the encoder weights are shared across all tasks.  The overall loss function is the weighted sum of the individual task losses:
\[
\mathcal{L}_{\text{total}} =
  \lambda_{\text{evasion}} \, \mathcal{L}_{\text{evasion}}
  + \lambda_{\text{clarity}} \, \mathcal{L}_{\text{clarity}}
\]
where $\lambda_{\text{evasion}}$ and $\lambda_{\text{clarity}}$ are loss
weights for each task.

However, we observed that this architecture frequently produced invalid
label combinations (e.g., predicting a \emph{Clear Reply} alongside a
\emph{Dodging} evasion technique), as the independent heads do not
inherently respect the hierarchical constraints of the taxonomy.
To address this structural incoherence, we extended our study to include
two additional modeling strategies.
Here, $C$ denotes the \emph{clarity label}
and $E$ denotes the \emph{evasion label}:

\begin{itemize}
    \setlength\parskip{0em}
    \item \textbf{Unified label space.}  We replaced the independent heads
    with a single classification head that predicts the Cartesian product
    of all valid label pairs, explicitly modeling the joint distribution
    $P(C, E)$ and making illegal pairs impossible.
    \item \textbf{Constrained inference.}  We retained the efficient
    independent training objective but augmented the model with a post-hoc
    validity mask during inference.  This method computes the joint
    probability $P(C, E) = P(C) \cdot P(E)$ and forces structurally
    invalid pairs to zero before selecting the final prediction.
\end{itemize}

As shown in Table~\ref{tab:overall_results}, the independent MTL baseline
achieves an evasion $F_1$ of 0.424.  The unified joint model drops to
0.373, suffering from data sparsity introduced by fragmenting labels into
a large Cartesian product space.  The constrained model not only
eliminates all structurally invalid predictions but also achieves the
highest evasion $F_1$ of 0.438, with accuracy 0.435, precision 0.475, and
recall 0.518.  This confirms that enforcing logical constraints during
inference helps filter noise without incurring the sparsity cost of joint
training.



\section{Error Analysis}


We analyze errors along three axes: hierarchical error propagation, structural
validity in multitask predictions, and task-specific cartographic
difficulty \citep{swayamdipta2020datasetcartographymappingdiagnosing}.  This separates failures caused by the label hierarchy from
failures caused by pragmatic ambiguity.

\subsection{Hierarchical Approach}
The main weakness of the hierarchical pipeline is its tendency to propagate errors. Figure~\ref{fig:hierarchical_confusion_matrices} illustrates this with the confusion matrices. 

A \textit{Clear Non-Reply} instance misclassified by the first binary model as \emph{Rest} is routed to the fine-grained model, where it can no longer be correctly classified. Consequently, mistakes at the top level cascade downstream, limiting the overall accuracy and $F_1$ scores. If the binary model could be improved significantly, or by introducing a fallback model, the propagation errors could be reduced.


\begin{figure}[!tbp]
    \centering
    \begin{minipage}{0.48\columnwidth}
        \centering
        \includegraphics[width=\linewidth]{figures/binary_cm.png}\\[-0.35em]
        {\scriptsize (a) Binary stage}
    \end{minipage}\hfill
    \begin{minipage}{0.48\columnwidth}
        \centering
        \includegraphics[width=\linewidth]{figures/fine_cm.png}\\[-0.35em]
        {\scriptsize (b) Fine-grained stage}
    \end{minipage}
    \caption{Confusion matrices for the hierarchical pipeline on the RoBERTa model.}
    \label{fig:hierarchical_confusion_matrices}
\end{figure}

Nine \textit{Clear Non-Reply} examples are routed to \textit{Rest} and
therefore cannot recover the correct label, while thirteen examples
predicted as \textit{Clear Non-Reply} never enter the fine-grained
model. 


\begin{figure}[!t]
    \centering

    \includegraphics[width=0.9\columnwidth]{figures/qevasion_clarity_hierarchical_cartography_no_title.png}
    \caption{Dataset cartography map for the hierarchical
pipeline (\emph{Clear Non-Reply} vs.~\emph{Rest}, RoBERTa-base),
aggregated across both stages.}
    \label{fig:hierarchical_cartography}

    \vspace{4pt}

    \includegraphics[width=0.9\columnwidth]{figures/qevasion_clarity_cartography_no_title.png}
    \caption{Task-specific cartography map for the clarity decision, calculated on the multitask model for better comparison between the two tasks, color denotes clarity correctness.}
    \label{fig:clarity_dimension_cartography}
\end{figure}
\FloatBarrier
\subsection{Multitask Approach}
In our multitask analysis, we examined the clarity and evasion labels predicted by the independent-head model. We observed that this model exhibits a distinct failure mode: it often produces locally plausible but structurally invalid clarity-evasion pairs, such as \textit{Clear Reply} with \textit{Dodging}. To address this, we determined whether each predicted pair was legal under our taxonomy and compared these errors against the constrained decoder outputs. This evaluation allowed us to disentangle structural inconsistencies from ordinary semantic classification mistakes.


\begin{figure}[t]
    \centering
    \includegraphics[width=\columnwidth]{figure8_clarity_evasion_predictions_large.png}
    \caption{Predicted clarity--evasion label pairs for the independent
    multitask model in one test run.  Red cells mark structurally
    invalid combinations, while the numbers show the test examples
    assigned to each predicted pair in a single run.}
    \label{fig:clarity_vs_evasion_predictions}
\end{figure}

Figure~\ref{fig:clarity_vs_evasion_predictions} quantifies the issue: 32 of 308 predictions (10.4\%) are illegal in the run shown.  The constrained decoder removes these invalid pairs across all five seeds, but only at the structural level; remaining errors still depend on
distinguishing fine-grained evasion strategies.

Most illegal
predictions are not random combinations: 22 of the 32 involve
\textit{Clear Reply} paired with a non-\textit{Explicit} evasion label.
The largest single illegal cell is \textit{Clear Reply} +
\textit{General} (11 cases), followed by \textit{Clear Reply} +
\textit{Implicit} (6 cases).  This suggests that the clarity head often
recognizes answer-like surface form, while the evasion head still reacts
to topic shifts or generalized political language.  Constrained decoding
therefore improves ontology consistency, but the semantic conflict
remains.

\FloatBarrier

\subsection{Dataset Cartography}

We use dataset cartography
\citep{swayamdipta2020datasetcartographymappingdiagnosing} to aggregate
confidence, variability, correctness, and forgetfulness across epochs and
seeds for the constrained
multitask BERT-base model. For the task-specific maps,
we recompute the same metrics separately from the
clarity and evasion probabilities produced by the
multitask model. Figure \ref{fig:hierarchical_cartography} additionally shows RoBERTa hierarchical clarity cartography for comparison. We use a natural-break split in which high-variability examples
are treated as \textit{Ambiguous}; low-variability examples are then
separated into \textit{Easy-to-learn} and \textit{Hard-to-learn} by
confidence.  This yields 1,345 easy examples (39.0\%), 818 ambiguous
examples (23.7\%), and 1,285 hard examples (37.3\%).
Table~\ref{tab:cartography_qualitative_reasons} summarizes the
qualitative motifs behind these regions, while
Figures~\ref{fig:clarity_dimension_cartography}
and~\ref{fig:evasion_dimension_cartography} show the corresponding
task-specific cartographies.

These categories reflect how the model behaved during training (based on confidence and variability), not just how common the labels are.  Easy examples are not
only more frequent; they are examples where surface form and pragmatic
function agree.  Ambiguous examples often contain competing evidence
across the question and answer, so the model can be correct in some
epochs and wrong in others.  Hard examples form the most important
diagnostic group: they are frequently learned with low confidence and
low variability, which indicates systematic misunderstanding rather than
mere seed instability.


Figure \ref{fig:hierarchical_cartography} shows how a hierarchical model produces a diffuse distribution with high variance of confidence and variability. Rather than sharply separating
\textit{Easy-to-learn} and \textit{Hard-to-learn} instances, the hierarchical model produces a broad band of confidence
predictions.

The task-specific maps (Fig. \ref{fig:clarity_dimension_cartography} and \ref{fig:evasion_dimension_cartography}) reveal distinct difficulty profiles for clarity and evasion.
Clarity is mostly learnable: 2,050 examples fall
in the easy region (59.5\%), while only 520 are hard (15.1\%).  Evasion
is substantially harder, with 1,548 hard examples (44.9\%) and only 642
examples in the ambiguous region (18.6\%), showing that the model struggles much more with the specific evasion nuances than the basic clarity task.

\begin{table*}[t!]
\centering
\scriptsize
\setlength{\tabcolsep}{1.8pt}
\renewcommand{\arraystretch}{1.12}
\begin{tabular}{>{\raggedright\arraybackslash}p{0.055\textwidth}
                >{\raggedright\arraybackslash}p{0.155\textwidth}
                >{\raggedright\arraybackslash}p{0.195\textwidth}
                >{\raggedright\arraybackslash}p{0.295\textwidth}
                >{\raggedright\arraybackslash}p{0.255\textwidth}}
\toprule
\textbf{Region} & \textbf{Motive} &
\textbf{Evidence} & \textbf{Examples} &
\textbf{Effect} \\
\midrule
Easy &
Direct or explicit answers dominate. &
The majority of inspected
examples were direct or explicit
answers. &
\emph{Q:} Would you encourage Congress to pay for school testing? \newline
\emph{A:} I would if they want. \par\vspace{0.45em}
\emph{Q:} Why did you come back, Mr. President? \newline
\emph{A:} I did not even think about not coming back. &
The model learns stable lexical and discourse cues when the answer gives
a clear commitment or a conventional explicit response. \\
\midrule
Ambig. &
The text often supports more than one reading. &
Inspected examples often
combined direct-looking
answers, multi-question context,
or non-replies with answer-like
content. &
\emph{Q:} Did you discuss the election with her? \newline
\emph{A:} No, I did not. I did not. \par\vspace{0.45em}
\emph{Q:} What exactly is un-American about those programs? \newline
\emph{A:} We are going to do a report; those people are gone. &
Ambiguity usually comes from competing targets: a reply can answer one
part of the prompt while evading another, or a refusal can be followed by
substantive policy material. \\
\midrule
Hard &
The label requires pragmatic inference beyond surface form. &
Most inspected hard cases
required implicit rhetorical
inference or exposed model
reliance on surface cues. &
\emph{Q:} Is the peace process over? \newline
\emph{A:} Who? \par\vspace{0.45em}
\emph{Q:} Can the President go around Congress? \newline
\emph{A:} You ever hear the word obstruction? &
Hard examples are not merely noisy; many require identifying whether a
short reply, counterquestion, topic shift, or long answer functions as a
specific evasion strategy. \\
\bottomrule
\end{tabular}
\caption{Qualitative explanation of why examples fall into each cartography region. Examples are cropped to the
minimal span needed to show the relevant behavior.}
\label{tab:cartography_qualitative_reasons}
\end{table*}

\begin{figure}[!t]
    \centering

    \includegraphics[width=\columnwidth]{qevasion_evasion_cartography_no_title.png}
    \caption{Task-specific cartography map for the evasion decision, calculated on the multitask model. The larger hard-to-learn region indicates that fine-grained pragmatic labels are the main source of remaining errors.}
    \label{fig:evasion_dimension_cartography}

    \vspace{2em}

    \includegraphics[width=\columnwidth]{qevasion_region_distribution_combined_no_title.png}
    \caption{MTL distribution of clarity (top) and evasion (bottom) labels across cartography regions.}
    \label{fig:combined_region_distribution}

\end{figure}




The category-level distributions in Figure~\ref{fig:combined_region_distribution} support the same conclusion.
\textit{Dodging} is the most concentrated hard label: 64.4\% of its
examples are hard under evasion-only cartography.  \textit{Implicit} is
also difficult, with 57.0\% in the hard region.  Labels with overt cues,
such as \textit{Clarification}, \textit{Declining to answer}, and
\textit{Claims ignorance}, move more often to the ambiguous region,
suggesting unstable confidence rather than uniformly low learnability.

In general, structural constraints prevent invalid output, but the remaining largest
errors are pragmatic.

By comparing the HTC and MTL cartography maps on the clarity task (Fig. \ref{fig:hierarchical_cartography} \& \ref{fig:clarity_dimension_cartography}) with the results in Table \ref{tab:overall_results}, we observe how HTC produces a more polarized map, with clearly clustered \textit{Easy-to-learn} and \textit{Hard-to-learn} examples, while MTL appears more ambiguous and noisy, forming a large central cluster where confidence is approximately 0.5 and variability remains low. This can be explained by the hierarchical model’s two-stage decision process, which enforces sharper separations between classes, whereas the multitask setup, due to its shared representation and greater flexibility, blends decision boundaries and results in less distinct confidence patterns. As a result, HTC exhibits clearer class separation and interpretability, while MTL trades this structure for improved flexibility on harder, fine-grained~cases.



\section{Conclusion}

In this work, we demonstrate that the inherent hierarchical structure of political question evasion can be effectively exploited to improve classification performance. Our findings show that the linguistic distinctiveness of the \textit{Clear Non-Reply} class makes it a prime candidate for early isolation in cascaded pipelines, leading to our best macro $F_1$ of 0.603. 

Furthermore, we addressed the prevalent issue of structural inconsistency in multitask learning by implementing a constrained inference mask that ensures logical alignment with the taxonomy. 

While dataset cartography revealed significant semantic overlap in the \textit{Ambivalent} category, it also highlighted a more distinct and interpretable representation of uncertainty in our hierarchical approach.


\section*{Limitations}

The QEvasion dataset's focus on four U.S. presidents in English restricts the generalisation of the findings to diverse political systems, other languages, or different discourse formats like parliamentary debates. 

Significant class imbalance, where the Clear Non-Reply class comprises only 10.3\% of the data, further limits accuracy on classifying minority labels and increases evaluation variance. Our hierarchical pipeline is vulnerable to top-level error propagation, where initial clarity misclassifications irrevocably affect downstream evasion labeling. 

Additionally, our multitask experiments were restricted to standard architectures, omitting advanced structural learning methods such as label-aware attention or graph-based decoding. 

The official ``match-any'' evaluation metric may provide an optimistic estimate of performance compared to stricter agreement-based measures. 

Bound by Google Colab's compute limits, this study used BERT-base encoders; however, future research should scale these efforts by adopting larger LLMs and more advanced ensemble architectures.

\section*{Acknowledgements}

We'd like to thank TakeLab for their mentorship, constructive feedback, and support throughout the development of this work.

\bibliographystyle{acl_natbib}
\bibliography{references}


\clearpage

\appendix


\section{Hyperparameters}

Tables \ref{tab:baselineMTLhyperparamaters} \& \ref{tab:hierarchicalhyperparamaters} show the hyperparameters used in experimental setups.

\begin{table}[t]
\centering
\small
\resizebox{\columnwidth}{!}{
\begin{tabular}{l l l}
\toprule
\textbf{Component} &
\textbf{BERT-base} &
\textbf{MTL (DeBERTa-v3)} \\
\midrule

Model
& BERT-base
& DeBERTa-v3-base \\

Optimizer
& AdamW
& AdamW \\

Learning Rate
& $2 \times 10^{-5}$
& $1.5 \times 10^{-5}$ \\

Epochs
& 2
& 8 \\

Batch Size
& 16
& 8 (accum.\ = 4) \\

Weight Decay
& 0.0
& 0.01 \\

Loss
& Cross-entropy
& $\mathcal{L}_{\text{evasion}} + 0.5 \cdot \mathcal{L}_{\text{clarity}}$ \\

Class Weighting
& None
& Inverse-frequency balancing \\

Seeds
& 42, 43, 44, 45, 46
& 42, 123, 456, 789, 1337 \\

\bottomrule
\end{tabular}
}
\caption{Baseline flat classification and multi-task learning configurations.}
\label{tab:baselineMTLhyperparamaters}
\end{table}

\begin{table}[t]
\centering
\small
\resizebox{\columnwidth}{!}{
\begin{tabular}{l l l l}
\toprule
\textbf{Component} &
\textbf{Clear Reply vs Rest} &
\textbf{Clear Non-Reply vs Rest} \\
\midrule

Model
& RoBERTa-base
& RoBERTa-base \\

Optimizer
& AdamW
& AdamW \\

Binary Learning Rate
& $2 \times 10^{-5}$
& $1 \times 10^{-5}$ \\

Fine Learning Rate
& $2 \times 10^{-5}$
& $1 \times 10^{-5}$ \\

Epochs
& 3
& 4 \\

Batch Size
& 16 
& 16  \\

Weight Decay
& 0.0 
& 0.02  \\

Loss
& Cross-entropy 
& Cross-entropy  \\


Seeds
& 42, 43, 44, 45, 46
& Random seeds \\

\bottomrule
\end{tabular}
}
\caption{Hierarchical RoBERTa configurations using two-stage binary and fine-grained classification.}
\label{tab:hierarchicalhyperparamaters}
\end{table}

\section{Traditional Machine Learning Baselines}
We evaluated a set of traditional machine learning models that do not
rely on transformer-based architectures.  These models serve as a performance reference point
before the transformer-based approaches described in the main paper.
Full results for the clarity task (Table~\ref{tab:clarity_ml_baselines})
and the evasion task (Table~\ref{tab:evasion_ml_baselines}) are provided
below; the best configuration per task is also included in the overall
comparison in Table~\ref{tab:overall_results}.

\begin{table}[htbp]
\centering
\resizebox{\columnwidth}{!}{%
\begin{tabular}{llcccc}
\toprule
Scenario & Model & Avg Acc. & Avg Prec. & Avg Rec. & $F_1$ $\pm$ Std \\
\midrule
No Weights & Nearest Neighbors   & 0.6299 & 0.4708 & 0.4044 & \textbf{0.4154 $\pm$ 0.0000} \\
No Weights & Linear SVM          & 0.6688 & 0.2229 & 0.3333 & 0.2672 $\pm$ 0.0000 \\
No Weights & RBF SVM             & 0.6688 & 0.2229 & 0.3333 & 0.2672 $\pm$ 0.0000 \\
No Weights & Decision Tree       & 0.6747 & 0.5322 & 0.3698 & 0.3386 $\pm$ 0.0004 \\
No Weights & Random Forest       & 0.6688 & 0.2229 & 0.3333 & 0.2672 $\pm$ 0.0000 \\
No Weights & Neural Net          & 0.6630 & 0.6963 & 0.3667 & 0.3496 $\pm$ 0.0111 \\
No Weights & AdaBoost            & 0.6526 & 0.6393 & 0.3459 & 0.3119 $\pm$ 0.0000 \\
No Weights & Naive Bayes         & 0.6688 & 0.2229 & 0.3333 & 0.2672 $\pm$ 0.0000 \\
No Weights & Logistic Regression & 0.6721 & 0.7118 & 0.3841 & 0.3795 $\pm$ 0.0000 \\
\midrule
With Weights & Nearest Neighbors   & 0.6299 & 0.4708 & 0.4044 & \textbf{0.4154 $\pm$ 0.0000} \\
With Weights & Linear SVM          & 0.5812 & 0.2953 & 0.4570 & 0.3405 $\pm$ 0.0000 \\
With Weights & RBF SVM             & 0.5779 & 0.6620 & 0.3685 & 0.3682 $\pm$ 0.0000 \\
With Weights & Decision Tree       & 0.5844 & 0.3505 & 0.3892 & 0.3367 $\pm$ 0.0000 \\
With Weights & Random Forest       & 0.2234 & 0.3464 & 0.3819 & 0.1944 $\pm$ 0.0432 \\
With Weights & Neural Net          & 0.6656 & 0.6918 & 0.3638 & 0.3431 $\pm$ 0.0078 \\
With Weights & AdaBoost            & 0.6526 & 0.6393 & 0.3459 & 0.3119 $\pm$ 0.0000 \\
With Weights & Naive Bayes         & 0.6364 & 0.4435 & 0.4023 & 0.4008 $\pm$ 0.0000 \\
With Weights & Logistic Regression & 0.4968 & 0.3919 & 0.4001 & 0.3919 $\pm$ 0.0000 \\
\bottomrule
\end{tabular}%
}
\caption{Performance of traditional machine learning models on the
clarity classification task, with and without class-weights.  Best $F_1$
in each scenario is in \textbf{bold}.}
\label{tab:clarity_ml_baselines}
\end{table}



\begin{table}[htbp]
\centering
\resizebox{\columnwidth}{!}{%
\begin{tabular}{llcccc}
\toprule
Scenario & Model & Avg Acc. & Avg Prec. & Avg Rec. & $F_1$ $\pm$ Std \\
\midrule
No Weights & Nearest Neighbors   & 0.3117 & 0.2280 & 0.1522 & \textbf{0.1665 $\pm$ 0.0000} \\
No Weights & Linear SVM          & 0.3734 & 0.0415 & 0.1111 & 0.0604 $\pm$ 0.0000 \\
No Weights & RBF SVM             & 0.3701 & 0.0816 & 0.1142 & 0.0701 $\pm$ 0.0000 \\
No Weights & Decision Tree       & 0.3831 & 0.1207 & 0.1187 & 0.0823 $\pm$ 0.0000 \\
No Weights & Random Forest       & 0.3714 & 0.0562 & 0.1107 & 0.0612 $\pm$ 0.0025 \\
No Weights & Neural Net          & 0.3747 & 0.1295 & 0.1191 & 0.0906 $\pm$ 0.0066 \\
No Weights & AdaBoost            & 0.3636 & 0.1883 & 0.1191 & 0.0894 $\pm$ 0.0000 \\
No Weights & Naive Bayes         & 0.3734 & 0.0415 & 0.1111 & 0.0604 $\pm$ 0.0000 \\
No Weights & Logistic Regression & 0.3636 & 0.2365 & 0.1305 & 0.1203 $\pm$ 0.0000 \\
\midrule
With Weights & Nearest Neighbors   & 0.3117 & 0.2280 & 0.1522 & 0.1665 $\pm$ 0.0000 \\
With Weights & Linear SVM          & 0.1591 & 0.0263 & 0.1240 & 0.0434 $\pm$ 0.0000 \\
With Weights & RBF SVM             & 0.3636 & 0.1578 & 0.1367 & 0.1229 $\pm$ 0.0000 \\
With Weights & Decision Tree       & 0.3636 & 0.1279 & 0.2328 & 0.1235 $\pm$ 0.0000 \\
With Weights & Random Forest       & 0.3045 & 0.1741 & 0.1406 & 0.0904 $\pm$ 0.0100 \\
With Weights & Neural Net          & 0.3779 & 0.1371 & 0.1202 & 0.0903 $\pm$ 0.0067 \\
With Weights & AdaBoost            & 0.3630 & 0.1884 & 0.1191 & 0.0895 $\pm$ 0.0003 \\
With Weights & Naive Bayes         & 0.2825 & 0.1806 & 0.1646 & 0.1425 $\pm$ 0.0000 \\
With Weights & Logistic Regression & 0.3247 & 0.2618 & 0.2380 & \textbf{0.2200 $\pm$ 0.0000} \\
\bottomrule
\end{tabular}%
}
\caption{Performance of traditional machine learning models on the
fine-grained evasion classification task, with and without class-weights.
Best $F_1$ in each scenario is in \textbf{bold}.}
\label{tab:evasion_ml_baselines}
\end{table}


\section{Baseline Flat Classification}

We additionally analyze the behavior of the flat BERT baseline using
Dataset Cartography.

The flat BERT baseline's cartography (Fig \ref{fig:baseline_cartography} \& \ref{fig:baseline_distribution}) reveals the  Ambivalent samples are predominantly \textit{Easy-to-learn} (75\%), perhaps largely due to their 59.2\% prevalence. In contrast, \textit{Clear Non-Replies} are almost completely \textit{Hard-to-learn}, making up only a tenth of the dataset. This suggests that without hierarchical cues, the model struggles to consistently identify minority classes despite their linguistic and semantic distinctiveness.

\begin{figure}[t]
    \centering
    \includegraphics[width=\columnwidth]{figures/qevasion_clarity_regular_cartography_no_title.png}
    \caption{Dataset cartography map for the baseline model.}
    \label{fig:baseline_cartography}
\end{figure}

\begin{figure}[t]
    \centering
    \includegraphics[width=\columnwidth]{figures/qevasion_clarity_region_distribution_no_title.png}
    \caption{Flat baseline distribution of clarity labels across cartography regions.}
    \label{fig:baseline_distribution}
\end{figure}

\section{Hierarchical Approach}

Figure \ref{fig:hierarchical_distribution}. illustrates the distribution of clarity labels for the HTC model. 
\begin{figure}[htbp]
    \centering
    \includegraphics[width=\columnwidth]{figures/qevasion_clarity_hierarchical_region_distribution_no_title.png}
    \caption{HTC distribution of clarity labels across cartography regions.}
    \label{fig:hierarchical_distribution}
\end{figure}



\end{document}
