\documentclass[11pt]{article}
\usepackage[final]{acl} % \usepackage[review]{acl} 
\usepackage{times}
\usepackage{latexsym}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{microtype}
\usepackage{inconsolata}
\usepackage{graphicx}
\usepackage{svg}
\usepackage{booktabs}
\usepackage{natbib}
\usepackage{hyperref}
\usepackage{refcount}
\setlength{\textfloatsep}{6pt}
\setlength{\intextsep}{6pt}
\setlength{\abovecaptionskip}{4pt}
\setlength{\belowcaptionskip}{0pt}
\usepackage{tabularx}
\usepackage{xcolor}
\usepackage{tikz}
\usepackage{soul}
\usepackage[table]{xcolor}
\usetikzlibrary{positioning, arrows.meta}

\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{orcidlink}



\title{NUST PsyAI at SemEval-2026 Task 10: Parameter-Efficient RoBERTa for Conspiracy Detection and Character-Level Marker Extraction}

\author{M. Husnain Akram \orcidlink{0009-0007-0461-1449} \and Mehwish Fatima \orcidlink{0000-0003-3424-2991} \\
School of Electrical Engineering and Computer Science (SEECS), \\
National University of Sciences and Technology (NUST), Islamabad, Pakistan \\
  \texttt{\{makram.mscs24seecs, mehwish.fatima\}@seecs.edu.pk}
}

\begin{document}
\maketitle
\begin{abstract}
We present the NUST PsyAI system for SemEval-2026 Task 10 (PsyCoMark), targeting document-level conspiracy detection and character-level psycholinguistic marker extraction from Reddit discourse. Our system ranks 7th in Extraction and 8th in Detection on the leaderboard. We benchmark feature-based and transformer approaches, adopting RoBERTa-large with LoRA for parameter-efficient fine-tuning. For detection, RB-DET-LoRA outperforms all baselines, achieving weighted F1 >0.79 (dev) and 0.76 (test), with robust generalization under blinded evaluation. For extraction, we contrast a unified multi-type BIO scheme with a decomposed per-type setup; the latter mitigates cross-label interference and improves boundary consistency, reaching Overlap F1 of 0.16 (dev) and 0.21 (test). Results reveal a clear asymmetry: detection benefits from contextual semantic modeling, while extraction is limited by sparse supervision and boundary-sensitive evaluation.
\end{abstract}

\section{Introduction}
The rapid proliferation of conspiracy narratives on social media undermines public discourse and information integrity. SemEval-2026 Task 10 (PsyCoMark) \citep{samory-etal-2026-semeval} formalizes this problem through two coupled objectives: (i) document-level conspiracy detection and (ii) character-level extraction of psycholinguistic markers—\textit{Actor}, \textit{Action}, \textit{Victim}, \textit{Effect}, and \textit{Evidence}. These objectives impose fundamentally different modeling requirements: detection depends on global semantic coherence and discourse-level signals, while extraction requires boundary-sensitive sequence labeling under sparse supervision. Bridging this gap demands models that jointly capture contextual semantics and fine-grained span structure.
%The rapid proliferation of conspiracy narratives on social media undermines public discourse and information integrity. SemEval-2026 Task 10 (PsyCoMark) \citep{samory-etal-2026-semeval} formalizes this challenge through two coupled objectives: (i) document-level conspiracy detection and (ii) character-level extraction of psycholinguistic markers—\textit{Actor}, \textit{Action}, \textit{Victim}, \textit{Effect}, and \textit{Evidence}. These tasks impose distinct modeling requirements: detection relies on global semantic and discourse-level signals, whereas extraction demands boundary-sensitive sequence labeling under sparse supervision. Bridging this gap requires models that capture both contextual meaning and fine-grained span structure.

We design a unified, transformer-centric framework with parameter-efficient adaptation to address both sub-tasks. We systematically evaluate feature-based baselines and pretrained encoders, and adopt RoBERTa-large with LoRA to balance representational capacity and efficiency. For detection, RB-DET-LoRA consistently outperforms both feature-based methods and fully fine-tuned transformer baselines, demonstrating strong generalization under blinded evaluation. For extraction, we explicitly compare a unified multi-type BIO formulation with a decomposed five-model setup; the latter assigns an independent classifier per marker type, reduces cross-label interference, and improves boundary stability.
%We address both sub-tasks using a unified transformer-centric framework with parameter-efficient adaptation. We evaluate feature-based baselines alongside multiple pretrained encoders and adopt RoBERTa-large with LoRA to balance capacity and efficiency. For detection, RB-DET-LoRA consistently outperforms both feature-based and fully fine-tuned transformer baselines. For extraction, we compare a unified multi-type BIO formulation with a decomposed five-model setup, where independent classifiers per marker type reduce cross-label interference and improve boundary stability.

Our contributions are threefold. First, we provide a systematic comparison of feature-based, augmented, and parameter-efficient transformer models for conspiracy detection. Second, we present an empirical analysis of unified versus decomposed formulations for character-level span extraction. Third, we conduct a focused analysis of boundary errors under IoU-based evaluation, exposing structural limitations of current approaches to fine-grained psycholinguistic marker extraction.
%Our contributions are threefold: (i) a systematic comparison of feature-based, augmented, and parameter-efficient transformer models for conspiracy detection; (ii) an empirical analysis of unified vs.\ decomposed formulations for character-level span extraction; (iii) an analysis of boundary errors under IoU-based evaluation, highlighting structural limitations of current approaches.

\begin{figure*}[t]
\centering
\resizebox{\textwidth}{!}{
\begin{tikzpicture}[
    node distance=1.4cm and 2.0cm,
    arrow/.style={-Latex, thick},
    box/.style={rectangle, draw, rounded corners, align=center, minimum height=1.1cm, text width=4.0cm},
    head/.style={rectangle, draw, rounded corners, align=center, minimum height=1.1cm, text width=4.3cm, fill=gray!12},
    feat/.style={box, fill=blue!10},
    trf/.style={box, fill=green!10},
    ext/.style={box, fill=orange!12},
    every node/.style={font=\LARGE}
]
% ================= INPUT =================
\node[head] (input) {Reddit\\Comments};

% ================= DETECTION (moved left) =================
\node[head, right=2.2cm of input] (det) {\textbf{Detection}};

% ---- symmetric branches with larger separation ----
\node[feat, right=2.0cm of det, yshift=+1.1cm] (feat_extr) {Feature\\Extraction};
\node[feat, right=1.9cm of feat_extr] (ml) {ML\\Models};
\node[trf, right=2.0cm of det, yshift=-1.1cm] (enc) {Transformer\\Encoder};
\node[trf, right=1.9cm of enc] (cls) {Classification\\Head};
% Output label centered between both branches
\node[head, right=2.2cm of ml, yshift=-1.1cm] (dout) {Conspiracy\\Label};
% ================= EXTRACTION (moved lower for clear separation) =================
\node[head, below=3.2cm of input] (exthead) {\textbf{Extraction}};
\node[ext, right=2.2cm of exthead] (tok) {Tokenizer + Offsets};
\node[ext, right=1.9cm of tok] (rext) {RoBERTa + LoRA};
\node[ext, right=1.9cm of rext] (bio) {Token\\Labeling};
\node[ext, right=1.9cm of bio] (rec) {Span Decode\\(Char Offsets)};
\node[head, right=1.9cm of rec] (eout) {Marker Spans};
% ================= ARROWS =================
% Input fan-out
\draw[arrow] (input) -- (det);
\draw[arrow] (input) -- (exthead);
% Detection feature path
\draw[arrow] (det) -- (feat_extr);
\draw[arrow] (feat_extr) -- (ml);
% Detection transformer path
\draw[arrow] (det) -- (enc);
\draw[arrow] (enc) -- (cls);
% ---- MIRRORED arrows into label ----
\draw[arrow] (ml) -- (dout);
% Extraction flow
\draw[arrow] (exthead) -- (tok);
\draw[arrow] (tok) -- (rext);
\draw[arrow] (rext) -- (bio);
\draw[arrow] (bio) -- (rec);
\draw[arrow] (rec) -- (eout);
\end{tikzpicture}
}
\caption{Two-branch conspiracy analysis framework: the detection branch combines feature-based models with transformer encoders for document-level prediction, while the extraction branch performs BIO-based token classification with RoBERTa-large (LoRA) followed by character-level span reconstruction.}
\label{fig:architecture}
\end{figure*}


\section{Related Work}
Conspiracy detection evolves from feature-engineered pipelines to context-aware transformer models. Early approaches use psycholinguistic, lexical, and dictionary-based features with classifiers such as SVMs and Random Forests to capture cognitive and emotional signals in conspiratorial discourse \citep{bessi2015science, klein2019exploring, ferrara2020types, miani2021loco}. These methods remain interpretable and efficient, but depend on static vocabularies and degrade under linguistic drift and platform-specific variation.
%Conspiracy detection has progressed from feature-engineered pipelines to context-aware transformer models. Early approaches leverage psycholinguistic, lexical, and dictionary-based features with classifiers such as SVMs and Random Forests to model cognitive and emotional signals in conspiratorial discourse \citep{bessi2015science, klein2019exploring, ferrara2020types, miani2021loco}. While interpretable and efficient, these methods rely on static vocabularies and degrade under linguistic drift and platform-specific variation.

%Transformer-based models substantially improve performance by capturing contextual semantics and long-range dependencies. Fine-tuned BERT variants achieve strong results on misinformation and social media benchmarks \citep{hoseini2023conspiracy, wadden2020fact, min2022divide}, but largely operate at sentence or document level, offering limited resolution for span-level reasoning. Hybrid approaches combine psycholinguistic features with contextual embeddings to trade off interpretability and representational capacity \citep{giachanou2023psycholinguistic}.
Transformer-based models improve performance by modeling contextual semantics and long-range dependencies. Fine-tuned BERT variants achieve strong results on misinformation and social media benchmarks \citep{hoseini2023conspiracy, wadden2020fact, min2022divide}, but typically operate at sentence or document level and provide limited resolution for span-level reasoning. Hybrid approaches integrate psycholinguistic features with contextual embeddings to balance interpretability and representational capacity \citep{giachanou2023psycholinguistic}.

%Recent work explores large language models, prompting strategies, and graph-based formulations to model narrative structure and information propagation \citep{zhang2024llm, bang2023multitask, pan2023risk, wang2024social}. These approaches enhance semantic coverage and reasoning but incur higher computational cost and remain underexplored for boundary-sensitive extraction tasks.
Recent work explores large language models, prompting strategies, and graph-based formulations to model narrative structure and information propagation \citep{zhang2024llm, bang2023multitask, pan2023risk, wang2024social}. These methods expand semantic coverage and reasoning ability, but increase computational cost and remain underexplored for boundary-sensitive extraction.

%Parameter-efficient fine-tuning methods, particularly LoRA, reduce adaptation cost while retaining the expressive power of large transformers \citep{hu2021lora, ding2023parameter, chen2025efficient}. However, existing work primarily targets coarse-grained classification. Fine-grained, character-level extraction of psycholinguistic roles—especially under strict boundary-based evaluation—remains insufficiently studied.
Parameter-efficient fine-tuning methods, particularly LoRA, reduce adaptation cost while preserving the expressive power of large transformers \citep{hu2021lora, ding2023parameter, chen2025efficient}. However, existing work focuses primarily on coarse-grained classification. Fine-grained, character-level extraction of psycholinguistic roles—especially under strict boundary-based evaluation—remains insufficiently studied.

%==========================================================================================================
\section{Our Framework}
We propose a two-branch architecture addressing (i) document-level detection and (ii) character-level marker extraction (Figure~\ref{fig:architecture}). The design decouples global semantic modeling from boundary-sensitive span prediction. To ensure reproducibility, we release our exploratory data analysis, processing pipelines, and full framework\footnote{\href{https://github.com/husnain-devops/semeval26_psycholinguistic_markers/tree/re-processing/EDA-Rehydrated/notebooks}{NUST PsyAI: EDA-Rehydrated Notebooks}}.

\subsection{Document-Level Conspiracy Detection}
This section discusses document-level detection pipeline.

\subsubsection{Feature-Based Modeling}
We compute lexical diversity (TTR, MATTR, MTLD \citep{mccarthy2010mtld}), readability indices (Flesch, FKGL, SMOG), POS distributions, discourse markers, and sentiment features using standard NLP toolkits \citep{bird2009nltk, honnibal2017spacy, hutto2014vader}. This yields a 62-dimensional feature space capturing stylistic and structural variation beyond surface lexical cues. For example, MA-TTR distributions (Figure~\ref{fig:ma_ttr_density}) highlight clear differences in lexical diversity between conspiracy and non-conspiracy texts, indicating distinct narrative construction patterns. Following task guidelines, we use binary labels (\textit{Yes}, \textit{No}) and exclude ambiguous cases.

\begin{figure}[t]
    \centering
    \includegraphics[width=\columnwidth]{visuals/Gemini_Generated_Image_nd1b3xnd1b3xnd1b.png}
    \caption{MA-TTR density plot showing lexical diversity differences between conspiracy (Yes) and non-conspiracy (No) texts.}
    \label{fig:ma_ttr_density}
\end{figure}

\subsubsection{Statistical Filtering} 
We perform univariate $t$-tests to identify discriminative features ($p<0.05$), including stopword ratio, adverb usage, interrogatives, pronouns, and passive constructions (Table~\ref{tab:univariate-tests}). These patterns reflect stylistic and rhetorical differences in conspiratorial discourse.

\subsubsection{Classifiers} 
We evaluate Linear Regression (LR), Support Vector Machine (SVM), Random Forest (RF), and XGBoost on the engineered feature space to model stylistic separability.

\subsubsection{Transformer-Based Detection} 
We fine-tune pretrained encoders directly on raw text. DistilBERT and BERT serve as full fine-tuning baselines.

\subsubsection{Data-Augmented RoBERTa} 
%We evaluate data augmentation via two configurations: RB-DET-AG2$\times$ (contextual synonym substitution using RoBERTa-base) and RB-DET-AG3$\times$ (a hybrid of synonym substitution and EN$\to$DE$\to$EN back-translation via MarianMT). Augmented samples are accepted only if they pass a surface-form quality gate to filter out degenerate or identical variants. To integrate this synthetic data, we apply a layered fine-tuning strategy. This allows the model to adapt to the expanded feature space while preserving core semantic representations(see Appendix \ref{sec:appendix-bt-analysis}).
We evaluate data augmentation through two configurations: RB-DET-AG2$\times$ (contextual synonym substitution using RoBERTa-base) and RB-DET-AG3$\times$ (a hybrid of synonym substitution and EN$\to$DE$\to$EN back-translation via MarianMT). We accept augmented samples only if they pass a surface-form quality gate that filters degenerate or identical variants. We integrate this synthetic data using a layered fine-tuning strategy, enabling the model to adapt to the expanded feature space while preserving core semantic representations (see Appendix~\ref{sec:appendix-bt-analysis} for pipeline details and failure analysis).

\subsubsection{PEFT with LoRA} 
We adopt LoRA over RoBERTa-large (RB-DET-LoRA) and inject low-rank adapters ($r=16$, $\alpha=32$), reducing trainable parameters while preserving representational capacity.
%\subsubsection{PEFT with LoRA} 
%We adopt LoRA over RoBERTa-large (RB-DET-LoRA), injecting low-rank adapters ($r=16$, $\alpha=32$). This reduces trainable parameters while preserving representational capacity.

%-------------------------------
\begin{table}[t]
\centering
\small
\renewcommand{\arraystretch}{1.05}
\setlength{\tabcolsep}{4pt}
\begin{tabular}{lrr}
\toprule
\textbf{Feature} & \textbf{$t$} & \textbf{$p$} \\
\midrule
stopword\_ratio  & +3.87 & \textbf{0.00011} \\
n\_advs\_x       & +3.81 & \textbf{0.00014} \\
qmarks           & +3.11 & \textbf{0.0019}  \\
commas           & -3.00 & \textbf{0.0027}  \\
n\_nouns\_x      & -2.82 & \textbf{0.0049}  \\
n\_adjs\_x       & -2.67 & \textbf{0.0076}  \\
discourse\_count & +2.45 & \textbf{0.0145}  \\
ttr              & +2.43 & \textbf{0.0150}  \\
n\_pronouns\_x   & +2.34 & \textbf{0.0195}  \\
n\_sentences\_x  & +2.22 & \textbf{0.0267}  \\
allcaps\_ratio   & +2.04 & \textbf{0.0419}  \\
passive\_count   & +2.05 & \textbf{0.0406}  \\
\midrule
hedge\_count     & -1.74 & 0.0810  \\
url\_count       & +0.19 & 0.8500  \\
\bottomrule
\end{tabular}
\caption{Univariate $t$-test results for psycholinguistic features; bold $p$-values denote significance ($p < 0.05$).}
\label{tab:univariate-tests}
\end{table}

%-------------------------------------------
\subsection{Marker Extraction Architecture} 
This section presents how markers are extracted.

\subsubsection{Task Formulation} 
We cast extraction as token-level BIO tagging aligned to character spans via tokenizer offsets. Predictions are projected back to character space for evaluation.

\subsubsection{Decoding and Span Reconstruction} 
Predicted BIO sequences are merged into spans and mapped to character offsets. Invalid fragments are discarded, and overlaps are resolved via confidence-based selection. Outputs conform strictly to required span format.

\subsubsection{Model Variants} 
We compare: (i) a unified multi-type model (\textbf{RB-EXT-MT}) with shared label space, and (ii) a decomposed setup (\textbf{RB-EXT-5M}) with five independent classifiers. We apply LoRA to improve parameter efficiency across both formulations.

%=============================================================================================================
\section{Experimental Design}
This section covers the details of experimental configurations.

\subsection{Dataset}
%We use the PsyCoMark dataset provided for SemEval-2026 Task 10\footnote{\label{fn:gh_semeval}\href{https://github.com/hide-ous/semeval26_task10_starter_pack}{Official Starter Pack}}. The dataset comprises three splits: Train ($N = 2{,}929$), Development (hereafter  \texttt{dev\_rehydrated} )($N = 100$), and Test (hereafter  \texttt{test\_rehydrated}) ($N = 938$). Each instance consists of a Reddit comment with detection labels in $\{\textit{Yes}, \textit{No}, \textit{Can't tell}\}$ and character-level annotations for marker extraction.The Train set was provided with gold labels for supervised learning. The Development set was initially released without labels to support the development and evaluation phase; these labels were subsequently made public to facilitate internal testing and model refinement. Finally, the \texttt{test\_rehydrated} set served as the held-out evaluation set for the official CodaBench leaderboard. For conspiracy detection, we restrict our internal experiments to binary labels (\textit{Yes}, \textit{No}) and discard ambiguous cases. The final evaluations were made on CodaBench\footnote{\label{fn:codabench}\href{https://www.codabench.org/competitions/10749/}{Competition Page}}.
We use the PsyCoMark dataset of SemEval-2026 Task 10\footnote{\label{fn:gh_semeval}\href{https://github.com/hide-ous/semeval26_task10_starter_pack}{Official Starter Pack}}. It comprises three splits: Train ($N = 2{,}929$), Development (hereafter \texttt{dev\_rehydrated}) ($N = 100$), and Test (hereafter \texttt{test\_rehydrated}) ($N = 938$). Each instance contains a Reddit comment with detection labels in $\{\textit{Yes}, \textit{No}, \textit{Can't tell}\}$ and character-level annotations for marker extraction. 

We use the Train set with gold labels for supervised learning. We initially use the Development set without labels for model development and later incorporate released labels for internal evaluation and refinement. We treat \texttt{test\_rehydrated} as the held-out set for official evaluation on CodaBench\footnote{\label{fn:codabench}\href{https://www.codabench.org/competitions/10749/}{Competition Page}}. For detection, we restrict experiments to binary labels (\textit{Yes}, \textit{No}) and discard ambiguous cases.

\subsection{Evaluation Protocol}
We evaluate detection using weighted F1 (W-F1) (official metric) and report Macro F1 (Ma-F1) for additional analysis. For extraction, we use Overlap F1 (O-F1) under a token-IoU protocol (IoU $\geq 0.5$): we map character spans to token sets and enforce type-constrained one-to-one alignment. This setup makes boundary precision the dominant factor (see Appendix~\ref{sec:imp}, Table~\ref{tab:token_iou_eval_summary}).

\subsection{Extraction Setup}
We align subword tokens with character-level spans using tokenizer offset mappings and assign BIO tags. A single model predicts marker-specific BIO labels (\texttt{B/I-Action}, \texttt{B/I-Actor}, \texttt{B/I-Effect}, \texttt{B/I-Evidence}, \texttt{B/I-Victim}). In the decomposed setup, we train five one-vs-type BIO models and aggregate their outputs at inference time. We address label imbalance in decomposed training using class-weighted cross-entropy and apply early stopping based on validation F1.

\subsection{Training and Implementation}
We implement all models in PyTorch and train them in mixed precision (FP16) on 8$\times$ AMD MI250X GPUs. We use an effective batch size of 32 (via gradient accumulation) and a maximum sequence length of 512. We tune learning rates in $[5 \times 10^{-5}, 6 \times 10^{-5}]$ and train for 10–20 epochs depending on configuration.We apply LoRA to reduce trainable parameters and memory footprint, enabling efficient adaptation of RoBERTa-large without full fine-tuning. Additional implementation details, including libraries and preprocessing components, appear in Appendix~\ref{sec:imp}.

\begin{table}[t]
\centering
\small
\renewcommand{\arraystretch}{1.1}
\begin{tabular}{lccc}
\toprule
\multicolumn{4}{c}{\textbf{4-Fold Cross-Validation (Train: $N=2{,}929$)}} \\
%\midrule
\textbf{Model} & \textbf{Acc.} & \textbf{Ma-F1} & \textbf{W-F1} \\
\midrule
GNB   & 0.57 & 0.51 & 0.53 \\
L-SVM & 0.58 & 0.54 & 0.56 \\
RF    & \textbf{0.59} & 0.55 & 0.57 \\
LR    & 0.59 & \textbf{0.56} & \textbf{0.57} \\
XGB   & 0.57 & 0.55 & 0.56 \\
DT    & 0.49 & 0.48 & 0.49 \\
MLP   & 0.56 & 0.50 & 0.55 \\
\midrule
\multicolumn{4}{c}{\textbf{Codabench dev\_rehydrated} ($N=100$)} \\
%\midrule
\textbf{Model} & \textbf{Acc.} & \textbf{Ma-F1} & \textbf{W-F1} \\
\midrule
GNB   & \textbf{0.68} & 0.47 & 0.57 \\
L-SVM & 0.65 & 0.56 & 0.62 \\
RF    & 0.69 & \textbf{0.58} & \textbf{0.64} \\
LR    & 0.61 & 0.46 & 0.54 \\
XGB   & 0.62 & 0.57 & 0.62 \\
DT    & 0.56 & 0.50 & 0.55 \\
MLP   & 0.55 & 0.50 & 0.54 \\
\bottomrule
\end{tabular}
\caption{Performance of feature-based models for binary conspiracy detection under cross-validation and official evaluation settings.}
\label{tab:ml_results}
\end{table}

%-----------------------------------------------------------------------
\section{Results and Analysis}
We present results for both detection and extraction, analyzing model behavior across feature-based and transformer settings.

\subsection{Document-Level Conspiracy Detection}
We evaluate models for document-level prediction under both feature-driven and contextual paradigms.

\subsubsection{Feature-Driven Detection} 
Logistic Regression achieves the strongest cross-validation performance (Ma-F1 = 0.56, W-F1 = 0.57), indicating that linear boundaries capture a meaningful portion of stylistic variation. On the \texttt{dev\_rehydrated} set, Random Forest generalizes better and achieves the highest W-F1 (0.64). Overall, feature-based models capture moderate signal but remain clearly below contextual models. Table~\ref{tab:ml_results} summarizes these results across cross-validation and held-out evaluation. 

We analyze feature relevance using univariate $t$-tests and TreeSHAP for Random Forest (Table~\ref{tab:treeshap_vs_ttest}). The two analyses reveal complementary patterns: $t$-tests highlight frequency-based signals such as \texttt{stopword\_ratio}, \texttt{n\_advs\_x}, and \texttt{n\_pronouns\_x}, while TreeSHAP emphasizes structural ratio features (e.g., \texttt{adj\_ratio}, \texttt{noun\_ratio}). This alignment at the category level—particularly for adverb-related features—indicates that the model captures linguistically meaningful signals rather than spurious correlations.

%Logistic Regression achieves the strongest cross-validation performance (Ma-F1= 0.56, W-F1= 0.57), indicating that linear boundaries capture a meaningful portion of stylistic variation. For \texttt{dev\_rehydrated} set, Random Forest generalizes better and attains the highest W-F1(0.64). Overall, feature-based models provide moderate signal but remain clearly below contextual models.

%We analyze feature relevance using univariate $t$-tests and TreeSHAP for Random Forest (Table~\ref{tab:treeshap_vs_ttest}). Both analyses cover: statistically significant features ($p<0.05$), including \texttt{stopword\_ratio}, \texttt{n\_advs\_x}, and \texttt{n\_pronouns\_x}, also rank highest in SHAP importance. This agreement between statistical significance and model attribution indicates that the classifier relies on linguistically meaningful signals rather than spurious correlations. We further quantify feature contributions by computing global attributions using TreeSHAP on the Random Forest model.

\begin{table}[t]
\centering
\small
\begin{tabular}{l l r r}
\toprule
\textbf{TreeSHAP} & \textbf{t-test} & \textbf{$t$-statistic} & \textbf{$p$-value} \\
\midrule
adj\_ratio & stopword\_ratio & +3.87 & 0.0001  \\
noun\_ratio & n\_advs\_x       & +3.81 & 0.0001  \\
num\_ratio  & qmarks          & +3.11 & 0.0019  \\
adp\_ratio  & commas          & -3.00 & 0.0027  \\
adv\_ratio  & n\_nouns\_x     & -2.82 & 0.0049  \\
\bottomrule
\end{tabular}
\caption{Comparison of top-ranked features from TreeSHAP (by impact) and univariate tests (by significance).}
\label{tab:treeshap_vs_ttest}
\end{table}

TreeSHAP further highlights structural and part-of-speech ratios as dominant signals, complementing the frequency-based patterns identified by univariate tests. The model places greater emphasis on ratio-based features, suggesting that relative distributional patterns provide stronger discriminative power than raw counts. We provide additional analysis of feature attribution for the Random Forest (RF) classifier using TreeSHAP, extending the observations discussed in Appendix~\ref{app:treeshap}.

%TreeSHAP highlights structural and part-of-speech ratios as dominant signals, complementing the frequency-based patterns identified by univariate tests. In particular, adverb-related features appear consistently across both analyses, reinforcing their role as stable indicators of conspiratorial discourse. The model places additional emphasis on ratio-based structural features (e.g., \texttt{adj\_ratio}, \texttt{noun\_ratio}), suggesting that relative distributional patterns provide stronger discriminative power than raw counts.

%---------------------------------------------------------------------
\subsubsection{Transformer-Based Detection} 
Table~\ref{tab:transformer_results} summarizes results on \texttt{dev\_rehydrated}. DistilBERT achieves high accuracy (0.79) but lower W-F1(0.67), indicating imbalance sensitivity. BERT further underperforms, particularly on the minority class. Data augmentation provides inconsistent gains. While RB-DET-AG variants improve recall for the \textit{Yes} class, they introduce noise that degrades overall performance. Qualitative analysis shows that back-translation distorts conspiracy-specific cues, limiting effectiveness.

We provide a detailed analysis of back-translation effects in Appendix~\ref{sec:appendix-bt-analysis}. Table~\ref{tab:bt-samples} illustrates these failure modes, including lexical hallucination and semantic drift introduced by back-translation. %As shown in Table~\ref{tab:bt-samples}, back-translation introduces ``label-preserving but semantics-altering'' noise, including lexical hallucinations and syntactic distortions that remove critical conspiratorial markers. 
Our layered fine-tuning strategy (unfreezing top 12--18 layers) does not compensate for this semantic drift. Consequently, parameter-efficient adaptation via LoRA proves more robust than expanding the training set with noisy synthetic samples. 

%As analyzed in Appendix~\ref{sec:appendix-bt-analysis}, back-translation introduces "label-preserving but semantics-altering" noise, including lexical hallucinations and syntactic distortions that strip away critical conspiratorial markers (see Table~\ref{tab:bt-samples}). Our layered fine-tuning strategy (unfreezing top 12--18 layers) could not compensate for this semantic drift. Consequently, parameter-efficient adaptation via LoRA proved more robust than expanding the training set with noisy synthetic samples.

RB-DET-LoRA achieves a W-F1 of 0.79 during the official development phase, where we evaluate on CodaBench using the organizers' pipeline and hidden labels. After the development phase ends and gold labels for the \texttt{dev\_rehydrated} (\(N=100\)) set become available, we continue task-specific fine-tuning and re-evaluate using the same official script, achieving 0.83 on the same set. Figure~\ref{fig:cm_lora} illustrates the corresponding class-wise prediction behavior.
%RB-DET-LoRA achieved a W-F1 of 0.79 during the official development phase, where scoring was performed on CodaBench using the organizers' evaluation pipeline and hidden labels. After the development phase ended and gold labels for the \texttt{dev\_rehydrated} (\(N=100\)) were released, we continued task-specific fine-tuning and re-evaluated with the same organizer-provided script, obtaining 0.83 on that same set as shown in the Figure~\ref{fig:cm_lora}. 

\begin{figure}[h]
    \centering
    \includegraphics[width=\columnwidth]{visuals/RoBERTa-Large_PEFT_2.png}
    \caption{Confusion Matrix for the RoBERTa-large PEFT (LoRA) model on the \texttt{dev\_rehydrated} set.}
    \label{fig:cm_lora}
\end{figure}


\begin{table}[t]
\centering
%\small
\resizebox{0.95\columnwidth}{!}{%
%\renewcommand{\arraystretch}{1.1}
\begin{tabular}{lccccc}
\toprule
\textbf{Model} & \textbf{Acc.} & \textbf{Ma-F1} & \textbf{W-F1} & \textbf{No-F1} & \textbf{Yes-F1} \\
\midrule
DistilBERT         & 0.79 & 0.75 & 0.67 & 0.85 & 0.64 \\
BERT               & 0.65 & 0.59 & 0.63 & 0.75 & 0.43 \\
RB-DET-AG2$\times$ & 0.70 & 0.63 & 0.68 & 0.79 & 0.47 \\
RB-DET-AG3$\times$ & 0.62 & 0.41 & 0.52 & 0.76 & 0.06 \\
\midrule
\textbf{RB-DET-LoRA} 
& \textbf{0.83} & \textbf{0.82} & \textbf{0.83} 
& \textbf{0.87} & \textbf{0.76} \\
\bottomrule
\end{tabular}}
\caption{Post-submission detection performance on \texttt{dev\_rehydrated} ($N=100$); official system achieves 0.79 (dev) and 0.76 (test, $N=938$) weighted F1.}
\label{tab:transformer_results}
\end{table}

We further validate robustness using the official CodaBench evaluation. As shown in Figure~\ref{fig:codabench_bar}, RB-DET-LoRA achieves a W-F1 of 0.76 on the held-out \texttt{test\_rehydrated} set, confirming stable generalization under a larger, fully blinded distribution.

\begin{figure}[t]
\centering
\includegraphics[width=\columnwidth]{latex/coda_results.png}
\caption{CodaBench scores for extraction (O-F1) and detection (Weighted F1) on dev and test sets.}\label{fig:codabench_bar}
\end{figure}

\subsection{Conspiracy Markers Extraction}
We evaluate span-level extraction under unified and decomposed labeling strategies.

\subsubsection{Transformer-Based Extraction} 
Table~\ref{tab:extraction_comparison} shows that the unified multi-type model (RB-EXT-MT) performs poorly ($F1=0.05$), indicating instability under a shared label space. In contrast, the decomposed formulation (\textbf{RB-EXT-5M}) achieves significantly higher stability. During the official development phase, our system reaches an Overlap $F1=0.16$. After the release of the development \texttt{dev\_public} (\(N=100\)), we refine the model and reach 0.22 F1. We use the same optimized configuration for the final test submission, where it achieves a stable O-F1 of 0.21 on the blinded \texttt{test\_rehydrated} set ($N=938$). Figure~\ref{fig:codabench_bar} highlights that separating label spaces reduces cross-type interference and improves boundary consistency under strict IoU constraints.
%Table~\ref{tab:extraction_comparison} shows that the unified multi-type model (RB-EXT-MT) performs poorly (F1 = 0.05), indicating instability under a shared label space. In contrast, the decomposed formulation (\textbf{RB-EXT-5M}) achieves significantly higher stability. During the official development phase, our system reached an O-F1 of 0.16. Upon the release of the development  \texttt{dev\_public} (\(N=100\)), we refined the model to reach 0.22 F1. The same optimized configuration was used for our final test submission, where it achieved a stable O-F1 of 0.21 on the blinded set \texttt{test\_rehydrated} ($N=938$). These results, summarized in Figure~\ref{fig:codabench_bar}, demonstrate that separating label spaces reduces cross-type interference and improves boundary consistency under strict IoU constraints.

\begin{figure}[t]
\centering
\includegraphics[width=\columnwidth]{semeval_markers_chart.png}
\caption{Per-role extraction performance showing F1 and recall across psycholinguistic categories.}
\label{fig:role_performance}
\end{figure}

\begin{table}[t]
\centering
\small
\resizebox{\columnwidth}{!}{%
\begin{tabular}{lcccccc}
\toprule
\textbf{Role} & \textbf{Precision} & \textbf{Recall} & \textbf{F1} & \textbf{TP} & \textbf{Pred} & \textbf{Gold} \\
\midrule
Action         & 0.1207 & 0.2692 & 0.1667 & 28 & 232 & 104 \\
Actor          & 0.2245 & 0.7279 & 0.3432 & 99 & 441 & 136 \\
Effect         & 0.0806 & 0.1408 & 0.1026 & 10 & 124 & 71 \\
Evidence       & 0.0658 & 0.1370 & 0.0889 & 10 & 152 & 73 \\
Victim         & 0.1659 & 0.5139 & 0.2508 & 37 & 223 & 72 \\
\midrule
\textbf{Micro Avg} & 0.1570 & 0.4035 & 0.2260 & 184 & 1172 & 456 \\
\bottomrule
\end{tabular}%
}
\caption{Per-role token-overlap results (IoU $\geq$ 0.5) on \texttt{dev\_rehydrated} vs.\ \texttt{dev\_public}; TP, Pred, Gold denote true positives, predicted, and gold spans.}
\label{tab:per_role_extraction}
\end{table}

\subsubsection{Boundary Behavior and Error Patterns} 
%Table~\ref{tab:per_role_extraction} presents extraction errors are dominated by boundary misalignment rather than categorical misclassification. While models successfully localize relevant discourse regions, they exhibit systematic over-generation (1,172 predicted vs. 456 gold spans), leading to high recall but depressed precision. This behavior is largely a byproduct of our type-separated decoding strategy, which preserves valid multi-role overlaps but increases false positives under strict matching (see Appendix~\ref{sec:appendix-bt-analysis} for decoding details).
Table~\ref{tab:per_role_extraction} shows that extraction errors are dominated by boundary misalignment rather than categorical misclassification. Models successfully localize relevant discourse regions but systematically over-generate spans (1,172 predicted vs.\ 456 gold), resulting in high recall and low precision. This behavior arises from the type-separated decoding strategy, which preserves valid multi-role overlaps but increases false positives under strict matching (see Appendix~\ref{sec:Error_Analysis} for decoding details).

The token-level Intersection-over-Union (IoU $\geq 0.5$) evaluation protocol (Table~\ref{tab:token_iou_eval_summary}) further amplifies this effect. Mapping character spans to discrete token sets makes the metric highly sensitive to boundary shifts: even minor misalignment or subword fragmentation reduces IoU below the threshold, converting semantic matches into false positives. Per-role analysis (Figure~\ref{fig:role_performance}) shows that concrete roles (\textit{Actor}, \textit{Victim}) remain more robust to these shifts, while abstract roles (\textit{Effect}, \textit{Evidence}) exhibit greater degradation due to diffuse boundaries.
%The rigorous token-level Intersection-over-Union (IoU $\geq 0.5$) protocol used for evaluation (detailed in Table~\ref{tab:token_iou_eval_summary}) further penalizes these results. As character spans are mapped to discrete token sets, even minor shifts in span boundaries or fragmentation during subword reconstruction can drop the IoU below the required threshold, converting a semantic ``hit'' into a false positive. Per-role analysis (Figure~\ref{fig:role_performance}) confirms that concrete roles (\textit{Actor}, \textit{Victim}) are more resilient to these alignment shifts than abstract roles (\textit{Effect}, \textit{Evidence}), which suffer from diffuse linguistic boundaries.

\subsection{Comparative Observations}
Across tasks, transformer models consistently outperform feature-based approaches for detection. RB-DET-LoRA achieves the best trade-off between performance and efficiency. In contrast, extraction remains bottlenecked by sparse supervision and boundary sensitivity; RB-EXT-5M provides a more stable formulation than RB-EXT-MT under these constraints.
%Across tasks, transformer models consistently outperform feature-based approaches for detection. RB-DET-LoRA provides the best trade-off between performance and efficiency. In contrast, extraction remains bottlenecked by sparse supervision and boundary sensitivity; RB-EXT-5M offers a more stable formulation than RB-EXT-MT under these constraints.

\section{Conclusions}
We present the NUST PsyAI system for SemEval-2026 Task 10, where contextual transformers consistently outperform feature-based approaches. For detection, RB-DET-LoRA achieves the best performance, confirming that parameter-efficient adaptation effectively captures conspiratorial semantics while maintaining efficiency. For extraction, task decomposition (RB-EXT-5M) outperforms a unified formulation by reducing cross-type interference under sparse supervision. The results expose a clear asymmetry: document-level detection benefits from semantic modeling, whereas character-level extraction remains constrained by label sparsity and strict boundary evaluation. Future work will focus on unified multi-task learning to couple global and local signals, span-aware objectives (e.g., contrastive or boundary-aware losses) to improve alignment precision, and more robust training strategies to enhance cross-domain generalization without increasing inference complexity.

\begin{table}[t]
%\renewcommand{\arraystretch}{1.15}
\centering
%\small
\resizebox{0.95\columnwidth}{!}{%
\begin{tabular}{lcc}
\toprule
\textbf{Configuration} & \textbf{Multi-Type} & \textbf{Five-Model} \\
\midrule
Label scheme & 11 (joint BIO) & 3 ($O$, $B$, $I$) per type \\
Max sequence length & 256 & 512 \\
Learning rate & $3\times10^{-4}$ & $2\times10^{-4}$ \\
Class weighting ($B$/$I$ vs.\ $O$) & No & Yes \\
Effective batch size & 16 & 16 \\
Epochs / early stopping & 10 / patience 3 & 10 / patience 3 \\
LoRA rank ($r$) & 16 & 16 \\
\midrule
\textbf{Dev O-F1} & 0.05 & \textbf{0.22} \\
\bottomrule
\end{tabular}}
\caption{Comparison of Multi-Type vs.\ Five-Model extraction (O-F1) on \texttt{dev\_rehydrated} ($N=100$); Five-Model improves beyond the official 0.16 F1 after post-submission fine-tuning.}
\label{tab:extraction_comparison}
\end{table}

\section*{Limitations}
The system exhibits several limitations. First, the dataset is sourced from Reddit that biases the model toward long-form, text-only discourse, limiting transfer to platforms with shorter formats, multimodal signals, or different linguistic distributions. Second, psycholinguistic markers exhibit extreme sparsity (exceeding a $1\!:\!100$ marker-to-background token ratio), making character-level boundary detection inherently unstable and prone to fragmentation under strict (IoU $\geq 0.5$) evaluation. Third, while the RB-EXT-5M setup improves precision, it increases inference latency and computational overhead compared to unified architectures. Finally, the system remains sensitive to temporal drift in conspiracy narratives and subjectivity in role annotation, highlighting the need for cross-domain robustness and adaptive modeling.

\section*{Acknowledgements}
We thank the organizers and reviewers of SemEval-2026 Task~10 for the PsyCoMark challenge, dataset curation, and transparent CodaBench evaluation, which together establish a rigorous benchmark for psycholinguistic analysis and conspiracy detection.

\bibliographystyle{plainnat}
\bibliography{custom} 

\appendix
\section{Implementation Details}\label{sec:imp}
We implement all pipelines in Python 3. We use pandas \citep{mckinney-proc-scipy-2010} and NumPy \citep{harris2020array} for data processing; spaCy \citep{honnibal2020spacy} for linguistic analysis; textstat and lexicalrichness for readability and lexical diversity; VADER \citep{hutto2014vader} and NRCLex for sentiment and emotion; Gensim \citep{rehurek_lrec} and BERTopic \citep{grootendorst2022bertopic} for topic modeling; sentence-transformers \citep{reimers-2019-sentence-bert} for embeddings; UMAP \citep{mcinnes2018umap} and HDBSCAN \citep{mcinnes2017hdbscan} for clustering; and scikit-learn \citep{scikit-learn}, SciPy \citep{2020SciPy-NMeth}, and SHAP \citep{lundberg2017unified} for modeling and analysis. We generate visualizations using matplotlib and seaborn.

\section{Random Forest Interpretation with TreeSHAP}\label{app:treeshap}
We report global feature attribution for the Random Forest (RF) conspiracy classifier using TreeSHAP and compare it with univariate statistical analysis.

\subsection{TreeSHAP vs.\ univariate t-tests}
We contrast model-based attribution with marginal statistical association. While univariate tests highlight frequency-based signals (e.g., stopwords, adverbs), TreeSHAP assigns higher importance to ratio-based structural features. Both analyses consistently identify adverb-related signals as salient, indicating robust stylistic markers of conspiratorial discourse.

\paragraph{Interpretation.}
Linguistic categories like adverbs (\texttt{adv\_ratio} and \texttt{n\_advs\_x}) appear at the top of both rankings, marking them as robust signals of conspiratorial discourse. TreeSHAP places higher importance on ratio-based structural features, while $t$-tests highlight specific frequency counts like \texttt{stopword\_ratio}.

%=====================================================================================
\section{Back-Translation Augmentation}\label{sec:appendix-bt-analysis}
We describe the augmentation pipeline and analyze its impact, including failure modes introduced by back-translation.

\begin{table*}[t]
\centering
\small
\setlength{\tabcolsep}{6pt}
\begin{tabularx}{\textwidth}{c l X X >{\raggedright\arraybackslash}X >{\raggedright\arraybackslash}X}
\toprule
\textbf{\#} & \textbf{Label} & \textbf{Original} & \textbf{Backtranslated} \\
\midrule
1 & \textit{yes} &
  Tom Delonge has \underline{accumulated} high level government officials \ldots\ puts it into a chronological timeline \ldots\ It is \underline{either a major} false flag or real UFO \underline{disclosure} \ldots &
  Tom Delonge has \underline{gathered} high-ranking government officials \ldots\ puts it in a chronological timeline \ldots\ It's \underline{either a large} false flag or real UFO \underline{revelation} \ldots \\
\midrule
2 & \textit{no} &
  \underline{Sessions} also indicated that the federal government would soon take steps to license more \underline{entities} to legally grow marijuana \ldots &
  \underline{Meetings} also indicated that the federal government would soon take steps to license more \underline{facilities} to legally grow marijuana \ldots \\
\midrule
3 & \textit{no} &
  \ldots\ two volunteer \underline{medics} \ldots\ rescued people under fire from the most devastated kibbutzim and \underline{moshavim}. &
  \ldots\ two voluntary \underline{doctors} \ldots\ saving people under fire from the most devastated kibbutzim and \underline{Moshabim}. \\
\midrule
4 & \textit{no} &
  Joe has some excellent \underline{rants}. He is hilarious \ldots\ Look \underline{him up}! \ldots\ I love to \underline{listen to} his rants \ldots &
  Joe has some excellent \underline{ranzes}. He is hilarious \ldots\ \underline{Check him out}! \ldots\ I love to \underline{hear} his ranzes \ldots \\
\midrule
5 & \textit{yes} &
  \ldots\ aggressively attempting to \underline{crush competitive primaries}. Pushing Progressives out of Key races. They didn't learn rigging the \underline{primaries} in 2016 \ldots &
  \ldots\ aggressively trying to \underline{crush competition priorities}. Pushing progressives from key races. They didn't learn rigging the \underline{priories} in 2016 \ldots \\
\midrule
\bottomrule
\end{tabularx}
\caption{Some random examples of EN$\rightarrow$DE$\rightarrow$EN back-translation. Highlighted spans show lexical and semantic distortions introduced by the pipeline.}\label{tab:bt-samples}
\end{table*}

\subsection{Pipeline}
We apply per-sample augmentation in the RB-DET-AG series after splitting the training corpus ($N{=}2{,}929$ yes/no samples) from the held-out evaluation set. We evaluate two configurations that differ in the number of augmented variants per sample and the augmentation methods used:

\begin{itemize}
\item \textbf{RB-DET-AG2$\times$} generates \emph{one} augmented variant per original sample (target $2\times$ expansion; total 5,858 samples). The augmentation loop assigns the first slot to contextual synonym substitution; thus, this configuration applies \textbf{contextual synonym substitution only}. We replace approximately 5\% of tokens per sentence with contextually plausible alternatives predicted by \texttt{roberta-base} via \texttt{nlpaug}~\citep{ma2019nlpaug}. We invoke back-translation only as a fallback.
\item \textbf{RB-DET-AG3$\times$} generates \emph{two} augmented variants per sample (target $3\times$ expansion; total 8,787 samples). We alternate augmentation methods across slots, producing roughly equal proportions of (1) contextual synonym substitution (same \texttt{roberta-base} setup) and (2) \textbf{back-translation} via a German pivot using MarianMT~\citep{junczys-dowmunt-etal-2018-marian}: \texttt{opus-mt-en-de} followed by \texttt{opus-mt-de-en}, decoded with default beam search (\texttt{num\_beams=4}).
\end{itemize}

We accept an augmented sample only if it is non-empty and differs from the source, enforcing a minimal surface-form quality gate. We also vary fine-tuning across configurations: RB-DET-AG2$\times$ unfreezes the top 12 of 24 transformer layers (lr = $5{\times}10^{-5}$, no weight decay, 20 epochs), while RB-DET-AG3$\times$ unfreezes the top 18 layers (lr = $2{\times}10^{-6}$, weight decay = 0.01, 25 epochs). We freeze embeddings in both cases and keep classification heads trainable.


\subsection{Failure Analysis}
We observe that neither augmented configuration consistently outperforms non-augmented contextual baselines (Table~\ref{tab:transformer_results}). In RB-DET-AG2$\times$, which relies solely on contextual synonym substitution, augmented samples remain semantically close to the originals but tend to smooth out idiosyncratic lexical cues that encode conspiratorial signals. In RB-DET-AG3$\times$, where half of the augmented data comes from back-translation, we observe additional failure modes (Table~\ref{tab:bt-samples}). For example, \textit{``gun Mason compass''} becomes \textit{``Cannon Mason Compass''} (row~10), altering semantically critical terms.

\begin{itemize}
\item \textbf{Lexical hallucination.} Back-translation introduces non-existent or semantically shifted tokens, particularly for colloquial or domain-specific expressions: \textit{``rants''} $\to$ \textit{``ranzes''} (row~4), \textit{``moshavim''} $\to$ \textit{``Moshabim''} (row~3). These substitutions distort the original vocabulary distribution.
\item \textbf{Syntactic distortion.} Round-trip translation alters sentence structure and argument relations. For instance, \textit{``Is there any method of persuasion \ldots\ that you think can penetrate a community that far off the deep end?''} becomes \textit{``Is there a method \ldots\ that you think that a community that can penetrate far away from the deep end?''} (row~7), changing the underlying meaning.
\end{itemize}

These effects introduce \textit{label-preserving but semantics-altering} noise into training. Since conspiratorial signals depend on specific phrasing, rhetorical patterns, and named entities, such distortions weaken the model's ability to learn reliable cues. The absence of a semantic consistency filter (e.g., SBERT similarity constraints) further amplifies this issue at higher augmentation levels. These observations motivate our use of \textbf{RB-DET-LoRA}, which improves performance by adapting model representations rather than relying on noisy synthetic data.

\section{Error Analysis and Boundary Behavior} \label{sec:Error_Analysis}
As shown in Table~\ref{tab:per_role_extraction}, our extraction models consistently exhibit high recall but low precision, particularly for the \textit{Actor} and \textit{Victim} roles. The models correctly localize relevant text regions but systematically over-generate spans (1,172 predictions vs.\ 456 gold), resulting in a high false positive rate.
%As observed in Table~\ref{tab:per_role_extraction}, a consistent pattern across our extraction models is high recall coupled with low precision, particularly for the \textit{Actor} and \textit{Victim} roles. This indicates that while the models successfully detect relevant regions of text, they heavily over-generate candidate spans (producing 1,172 predictions compared to 456 gold spans), leading to a high false positive rate.

We follow the official CodaBench-compatible token-overlap evaluation from the starter pack. We first map character spans to token index sets using the organizer’s regex-based tokenizer, then perform matching via token-level Intersection-over-Union (IoU). A prediction counts as correct only if it matches the gold span type and achieves IoU $\geq 0.5$ under a one-to-one alignment (best unmatched prediction per gold span). This criterion requires substantial overlap rather than mere boundary contact.
%Regarding boundary behavior, we follow the official CodaBench-compatible token-overlap evaluation used in the starter pack. Concretely, character spans are first mapped to token-index sets using the organizer tokenizer (regex-based tokenization), then matching is performed with token-level Intersection-over-Union (IoU). A predicted span is counted as correct only if it has the same type as a gold span and reaches IoU $\geq 0.5$ under a one-to-one matching scheme (best unmatched prediction per gold span). Therefore, correctness requires substantial token overlap rather than mere boundary contact.

This setup tolerates minor boundary shifts when overlap remains sufficient, but fragmented or partial spans often fall below the IoU threshold and are penalized. We count unmatched predictions as false positives and unmatched gold spans as false negatives. Consequently, precision drops primarily due to over-prediction and span fragmentation, while recall reflects misses under the same IoU-constrained matching.
%This criterion implies that small boundary shifts may still be tolerated when overlap remains high enough, but partial/fragmented spans often fail to reach the IoU threshold and are penalized. Unmatched predictions are counted as false positives and unmatched gold spans as false negatives. Hence, precision loss in our setup is primarily driven by over-prediction and span fragmentation, while recall reflects misses under the same IoU-constrained matching.

\subsection{Evaluation Methods}
We follow the official CodaBench token-overlap protocol. We map character spans to token sets and match them using IoU $\geq 0.5$ under one-to-one, type-constrained alignment. This formulation prioritizes boundary precision: partial overlaps often fail to meet the IoU threshold, producing false positives and false negatives. Consequently, over-generation and span fragmentation drive precision loss in our models. Table~\ref{tab:token_iou_eval_summary} summarizes the protocol.

\begin{table}[t]
\centering
\small
\begin{tabularx}{\columnwidth}{>{\raggedright\arraybackslash}p{0.3\columnwidth} >{\raggedright\arraybackslash}X}
\toprule
\textbf{Evaluation Component} & \textbf{CodaBench/Starter-Pack Criterion} \\
\midrule
Tokenization & Text is tokenized with the regex \texttt{(\textbackslash w+|[\textasciicircum\textbackslash w\textbackslash s])}, so words and punctuation are separate tokens. \\
\addlinespace
Span representation & Each character span is converted to a set of covered token indices (a token is included if character intervals overlap). \\
\addlinespace
Matching policy & One-to-one, type-constrained matching: for each gold span, select the best \emph{unmatched} predicted span of the same marker type. \\
\addlinespace
True Positive (TP) rule & A match is counted as TP only when token-level IoU between gold and predicted token sets is $\geq 0.5$. \\
\addlinespace
False Negatives (FN) & Gold spans left unmatched after the matching step are counted as FN. \\
\addlinespace
False Positives (FP) & Predicted spans left unmatched after the matching step are counted as FP. \\
\addlinespace
Per-role scores & Precision, Recall, and F1 are computed independently for each marker type (Action, Actor, Effect, Evidence, Victim). \\
\addlinespace
Aggregate scores & Micro scores are computed from global TP/FP/FN counts; Ma-F1is the mean of per-type F1 values. \\
\addlinespace
Practical implication & Minor boundary shifts may still match if IoU remains high; fragmented or over-generated spans often fail IoU $\geq 0.5$, reducing precision. \\
\bottomrule
\end{tabularx}
\caption{Concise summary of the token-IoU evaluation protocol used by the official CodaBench-compatible starter-pack script and our \texttt{infer\_5\_models\_dev\_f1.py} pipeline.}
\label{tab:token_iou_eval_summary}
\end{table}


\subsection{Handling of Overlapping Spans During Decoding}
We adopt a type-separated decoding strategy. We predict each marker type independently and merge spans without cross-type suppression. This design preserves valid multi-role overlaps but increases false positives, especially under IoU-based evaluation. We plan to incorporate type-aware NMS and confidence calibration to better balance recall and precision.
      
\end{document}
