\documentclass[11pt]{article}
\usepackage[preprint]{acl} % \usepackage[review]{acl} 
\usepackage{times}
\usepackage{latexsym}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{microtype}
\usepackage{inconsolata}
\usepackage{graphicx}
\usepackage{svg}
\usepackage{tikz}
\usetikzlibrary{positioning, arrows.meta, shapes.multipart}


\title{NUST PsyAI at SemEval-2026 Task 10: A Comparative Evaluation of Psycholinguistic Features, Machine Learning, and LLMs for Conspiracy Marker Extraction and Detection}
\author{M. Husnain Akram \and Mehwish Fatima \\
School of Electrical Engineering and Computer Science (SEECS), \\
National University of Sciences and Technology (NUST), Islamabad, Pakistan \\
  \texttt{\{makram.mscs24seecs, mehwish.fatima\}@seecs.edu.pk}
}

\begin{document}


\maketitle
\begin{abstract}
%This paper describes the system developed by team NUST PsyAI for SemEval-2026 Task 10: PsyCoMark, which focuses on extracting and detecting psycholinguistic conspiracy markers in Reddit submission statements. We address both sub-tasks: document-level conspiracy detection and character-level marker span extraction. While we evaluated multiple methodologies, our most effective approach leverages Parameter-Efficient Fine-Tuning (PEFT) using Low-Rank Adaptation (LoRA) on the RoBERTa-large architecture. For detection, our model effectively distinguishes conspiracy-related discourse from non-conspiracy text. For extraction, we evaluate two primary strategies: a unified single-model approach utilizing the BIO labeling scheme, and an ensemble of five separate models dedicated to individual marker types. Our systems demonstrate significant improvements over the baseline models, achieving a weighted F1 score above 0.79 and 0.76 for detection, and an overlap F1 above 0.16 and 0.21 for extraction in the development and test phases, respectively.

We present the NUST PsyAI system for SemEval-2026 Task~10 (PsyCoMark), which targets document-level conspiracy detection and character-level psycholinguistic marker extraction from Reddit discourse. We evaluate multiple modeling strategies and center our final approach on RoBERTa-large adapted with LoRA for parameter-efficient fine-tuning. For detection, the LoRA-adapted model effectively discriminates conspiracy from non-conspiracy text, outperforming baseline transformers and feature-based classifiers, achieving weighted F1 scores above 0.79 and 0.76 on development and test sets. For extraction, we compare a unified BIO tagging model against a five-model per-type configuration. The per-type strategy yields superior boundary precision, reaching overlap F1 scores above 0.16 and 0.21 in development and test evaluations. 

\end{abstract}

\section{Introduction}
%The proliferation of conspiracy theories on social media platforms poses a significant threat to public discourse and information integrity. SemEval-2026 Task 10 (PsyCoMark) \citep{samory-etal-2026-semeval} formalizes the challenge of combating this phenomenon by targeting the detection of conspiracy-related texts and the extraction of specific psycholinguistic markers (Actor, Action, Victim, Threat, and Evidence/Effect) from a dataset compiled by scraping user comments on Reddit. 
The rapid proliferation of conspiracy narratives on social media undermines public discourse and weakens information integrity. SemEval-2026 Task~10 (PsyCoMark) \citep{samory-etal-2026-semeval} operationalizes this challenge through two complementary objectives: (i) detecting conspiracy-related text and (ii) extracting fine-grained psycholinguistic markers—\textit{Actor}, \textit{Action}, \textit{Victim}, \textit{Threat}, and \textit{Evidence/Effect}—from Reddit comments annotated with character-level spans. The task therefore requires both global semantic discrimination and precise boundary-sensitive extraction.

Team \textbf{NUST PsyAI} participates in both Sub-task~1 (Extraction) and Sub-task~2 (Detection). We systematically evaluate feature-driven models and multiple transformer configurations, ultimately centering our approach on RoBERTa-large adapted with LoRA to balance representational capacity and computational efficiency. We posit that parameter-efficient fine-tuning enables large pretrained encoders to capture subtle narrative structures, rhetorical framing, and discourse-level signals characteristic of conspiratorial language.
%Our team, NUST PsyAI, participated in both Sub-task 1 (Extraction) and Sub-task 2 (Detection). We systematically evaluated multiple approaches, ultimately centering our main strategy on robust pre-trained language models---specifically RoBERTa-Large. We enhanced this model with Low-Rank Adaptation (LoRA) to maintain computational efficiency while maximizing predictive performance. We hypothesized that large language models (LLMs), when fine-tuned efficiently, can accurately capture the nuanced discourse markers and sentiment features indicative of conspiratorial text.

%For Sub-task 2, our LoRA-adapted RoBERTa-large model successfully outperforms the DistilBERT baseline, BERT and ROBERTa-large, as well as all other psycholinguistic feature-based and traditional Machine Learning (ML) approaches tested during our evaluation phase. For Sub-task 1, we discovered that decomposing the extraction task into five distinct models yields a higher overlap F1 score compared to a unified BIO-tagging approach. Our code and exploratory data analysis pipelines are publicly available. \footnote{\url{https://github.com/NUST-PsyAI/SemEval2026-Task10}}.
For Sub-task~2, the LoRA-adapted RoBERTa-large model consistently outperforms DistilBERT, BERT, standard RoBERTa fine-tuning, and feature-based classifiers. For Sub-task~1, we observe that decomposing span extraction into five independent BIO classifiers yields higher Overlap F1 than a unified multi-type tagging scheme, suggesting reduced cross-label interference under sparse supervision. Our implementation, including preprocessing and exploratory analysis pipelines, is publicly available.\footnote{\url{https://github.com/NUST-PsyAI/SemEval2026-Task10}}


\section{Related Work}
%Recent efforts to detect the spread of conspiracies in social discourse have increasingly relied on a blend of psycholinguistic analysis, traditional machine learning, and transformer-based models. Early approaches heavily favored psycholinguistic feature engineering. For instance, several studies \cite{bessi2015science, klein2019exploring, ferrara2020types, miani2021loco} utilized linguistic dictionaries (e.g., LIWC) paired with Support Vector Machines (SVMs) and Random Forests to identify cognitive and emotional markers of conspiratorial thinking. While highly efficient and requiring minimal compute power, these approaches suffer from rigid vocabularies and struggle to generalize across evolving internet slang, presenting a significant research gap in zero-shot contexts.
Research on conspiracy detection progresses from feature-engineered models to large-scale transformers. Early work relies on psycholinguistic feature extraction. Studies leverage linguistic dictionaries such as Linguistic Inquiry and Word Count (LIWC) with classifiers including Supoprt Vector Machines (SVM)s and Random Forests to capture cognitive and emotional correlates of conspiratorial thinking \cite{bessi2015science, klein2019exploring, ferrara2020types, miani2021loco}. These approaches remain computationally efficient and interpretable, but depend on fixed vocabularies and generalize poorly to evolving online language, particularly in zero-shot settings.

Subsequent work adopts transformer architectures to model deeper semantic context. Fine-tuned BERT variants achieve strong performance on social media and claim verification benchmarks at moderate computational cost \cite{hoseini2023conspiracy, wadden2020fact, min2022divide}. However, these models often lack transparent explanations for their predictions. To improve interpretability, \citet{giachanou2023psycholinguistic} integrate psycholinguistic features with CNNs and contextual embeddings. This hybrid design balances contextual modeling and feature-level insight, though it increases model complexity.
%To capture deeper semantic context, researchers transitioned to transformer architectures. Fine-tuning standard BERT variants on social media graphs or claim verification datasets yielded high accuracy and operated at moderate computational costs accessible to standard GPUs \cite{hoseini2023conspiracy, wadden2020fact, min2022divide}. However, they often lacked explainability regarding why a text was flagged. Bridging this gap, \citet{giachanou2023psycholinguistic} combined extracted psycholinguistic features with Convolutional Neural Networks (CNNs) and BERT embeddings. This hybrid approach offered a strong balance of context and interpretability, though it slightly increased computational overhead.

Recent advances like zero- and few-shot prompting with models such as GPT-4 and LLaMA-2 captures nuanced conspiracy narratives and long-range dependencies \cite{zhang2024llm, bang2023multitask, pan2023risk}. Despite strong semantic reasoning, these models incur substantial inference cost and frequently exhibit instability in boundary-sensitive span detection. In parallel, graph-based approaches combine RoBERTa with GNNs to model propagation dynamics \cite{wang2024social}. While effective for modeling spread patterns, these architectures demand high memory and limit practical scalability.
%More recently, the advent of Large Language Models (LLMs) has shifted the paradigm. Studies applying zero-shot and few-shot prompting using massive models like GPT-4 and LLaMA-2 to detect conspiracy narratives demonstrate an excellent grasp of complex semantic structures \cite{zhang2024llm, bang2023multitask, pan2023risk}. However, these massive models incur prohibitively high computational and inference costs, making real-time social media moderation economically unviable, and frequently suffer from hallucinations in boundary-span detection. Alternatively, \citet{wang2024social} introduced Graph Neural Networks (GNNs) combined with RoBERTa to capture topological spread, but this multi-modal method demands extreme VRAM, severely limiting scalability. 
To balance efficiency and performance, recent work explores Parameter-Efficient Fine-Tuning (PEFT), particularly LoRA, for misinformation detection \cite{hu2021lora, ding2023parameter, chen2025efficient}. PEFT reduces trainable parameters and VRAM requirements while retaining competitive accuracy relative to full fine-tuning. However, prior studies largely focus on document- or sentence-level classification. They rarely address fine-grained, character-level extraction of narrative components, such as explicit \emph{Victim} or \emph{Threat} spans.

%To resolve the tension between computational cost and performance, literature demonstrates the efficacy of Parameter-Efficient Fine-Tuning (PEFT) techniques, such as LoRA \cite{hu2021lora, ding2023parameter}, on mid-sized language models for misinformation detection \cite{chen2025efficient}. This drastically reduces VRAM requirements while matching full fine-tuning performance. However, a persistent limitation across all these works is the lack of granular, character-level extraction of specific narrative elements (e.g., identifying the exact span of a "Victim" or "Threat"). Though we have applied different statistical, machine learning as well, but our work directly addresses this gap by applying LoRA-optimized RoBERTa architectures not just to document-level detection, but specifically to the psycholinguistic span extraction formalized by the PsyCoMark task.

\begin{figure*}[t]
    \centering
    \includegraphics[width=0.75\textwidth]{visuals/Architecture.pdf}
    \caption{Psycholinguistic strategy, ML and Transformer Models for Conspiracy Detection}
    \label{fig:eda}
\end{figure*}


%------------------------

\begin{figure*}[t]
\centering
\resizebox{\textwidth}{!}{%
\begin{tikzpicture}[
    node distance=1.0cm and 1.6cm,
    every node/.style={font=\small},
    block/.style={rectangle, draw, rounded corners, align=center, minimum width=3.2cm, minimum height=0.85cm},
    smallblock/.style={rectangle, draw, rounded corners, align=center, minimum width=2.8cm, minimum height=0.75cm},
    arrow/.style={-Latex, thick}
]

% Top input
\node[block] (input) {Raw Reddit Comment};

% =========================
% Detection branch (left)
% =========================
\node[block, below left=1.2cm and 4.0cm of input] (det_head) {\textbf{Detection Branch}};

\node[smallblock, below=0.8cm of det_head] (feat) {Feature Extraction\\(62D)};
\node[smallblock, below=0.8cm of feat] (ml) {ML Classifiers\\(LR, SVM, RF, XGB, MLP)};
\node[smallblock, right=1.8cm of feat] (det_trf) {Transformer Encoder\\(DistilBERT/BERT)};
\node[smallblock, below=0.8cm of det_trf] (det_cls) {Classification Head};
\node[block, below=1.0cm of ml, xshift=1.9cm] (det_out) {Conspiracy Label\\(Yes / No)};

% =========================
% Extraction branch (right)
% =========================
\node[block, below right=1.2cm and 4.0cm of input] (ext_head) {\textbf{Extraction Branch}};

\node[smallblock, below=0.8cm of ext_head] (tok) {Tokenizer\\+ Offset Mapping};
\node[smallblock, below=0.8cm of tok] (ext_trf) {RoBERTa-large + LoRA};
\node[smallblock, below=0.8cm of ext_trf] (bio) {Token Classification\\(BIO Tags)};
\node[smallblock, below=0.8cm of bio] (recon) {Span Reconstruction\\(Char Offsets)};
\node[block, below=0.8cm of recon] (ext_out) {Marker Spans\\Actor, Action, Victim,\\Evidence, Effect};

% =========================
% Shared arrows from input
% =========================
\draw[arrow] (input) -- (det_head);
\draw[arrow] (input) -- (ext_head);

% Detection arrows
\draw[arrow] (det_head) -- (feat);
\draw[arrow] (feat) -- (ml);
\draw[arrow] (ml) -- (det_out);

\draw[arrow] (feat) -- (det_trf);
\draw[arrow] (det_trf) -- (det_cls);
\draw[arrow] (det_cls) -- (det_out);

% Extraction arrows
\draw[arrow] (ext_head) -- (tok);
\draw[arrow] (tok) -- (ext_trf);
\draw[arrow] (ext_trf) -- (bio);
\draw[arrow] (bio) -- (recon);
\draw[arrow] (recon) -- (ext_out);

\end{tikzpicture}%
}
\caption{Hierarchical architecture of our two-branch framework: document-level conspiracy detection (left) and character-level psycholinguistic marker extraction (right).}
\label{fig:architecture}
\end{figure*}


%==========================================================================================================
\section{Our Framework}
We design a two-branch framework that addresses (i) document-level conspiracy detection and (ii) character-level psycholinguistic span extraction. The detection branch combines feature-based models and transformer architectures. The extraction branch formulates marker identification as token-level sequence labeling. Figure~\ref{fig:eda} summarizes the pipeline.

\subsection{Document-Level Conspiracy Detection}
\textbf{Feature-Driven Detection Pipeline.} We construct a 62-dimensional feature space derived from lexical, syntactic, discourse, readability, and sentiment signals. For binary classification, we restrict the dataset to \textit{Yes} and \textit{No} labels and exclude \textit{Can't tell} instances.

\textbf{Psycholinguistic and Surface Features.}  We compute lexical diversity metrics (TTR, Hapax ratio, Shannon entropy, MATTR, MTLD), readability indices (Flesch, FKGL, Gunning Fog, SMOG, ARI), fine-grained POS ratios, discourse and hedge counts, passive constructions, and VADER sentiment scores. These features encode stylistic and cognitive signals associated with conspiratorial language.

\textbf{Statistical Filtering.}  We conduct independent univariate $t$-tests to identify features that significantly differ between classes. Twelve features reach statistical significance at $p<0.05$ as shown in Table~\ref{tab:univariate-tests}, with conspiracy texts exhibiting higher stopword ratios, adverb frequency, interrogative markers, pronoun usage, and passive constructions.

%-------------------------------
\begin{table}[t]
  \centering
  \small
  \begin{tabular}{lrrl}
    \hline
    \textbf{Feature} & \textbf{$t$-statistic} & \textbf{$p$-value} & \textbf{Sig.} \\
    \hline
    stopword\_ratio   & $+3.87$ & $0.00011$ & Yes \\
    n\_advs\_x        & $+3.81$ & $0.00014$ & Yes \\
    qmarks            & $+3.11$ & $0.0019$  & Yes \\
    commas            & $-3.00$ & $0.0027$  & Yes \\
    n\_nouns\_x       & $-2.82$ & $0.0049$  & Yes \\
    n\_adjs\_x        & $-2.67$ & $0.0076$  & Yes \\
    discourse\_count  & $+2.45$ & $0.0145$  & Yes \\
    ttr               & $+2.43$ & $0.0150$  & Yes \\
    n\_pronouns\_x    & $+2.34$ & $0.0195$  & Yes \\
    n\_sentences\_x   & $+2.22$ & $0.0267$  & Yes \\
    allcaps\_ratio    & $+2.04$ & $0.0419$  & Yes \\
    passive\_count    & $+2.05$ & $0.0406$  & Yes \\
    hedge\_count      & $-1.74$ & $0.081$   & No  \\
    url\_count        & $+0.19$ & $0.85$    & No  \\
    \hline
  \end{tabular}
  \caption{Univariate $t$-test results (\textit{Yes} vs. \textit{No}). A positive $t$-statistic indicates the feature is higher in the conspiracy class; negative indicates higher in non-conspiracy. Significance is defined as $p<0.05$.}
  \label{tab:univariate-tests}
\end{table}
%-------------------------------

\textbf{Transformer-Based Detection.} We fine-tune pretrained transformers directly on raw text. For full fine-tuning, we train distilbert base-uncased and bert base-uncased with classification heads. 

\textbf{Data-Augmented RoBERTa.} To mitigate class imbalance, we apply English-German back-translation and synonym replacement. We fine-tune \texttt{roberta-large} with partially unfrozen upper layers under 2$\times$ and 3$\times$ augmented settings.

\textbf{Parameter-Efficient Fine-Tuning (LoRA).} 
We apply LoRA to a frozen RoBERTa-large backbone, injecting low-rank adapters ($r=16$, $\alpha=32$, dropout $=0.1$) into the attention (\textit{query}, \textit{key}, \textit{value}) and feed-forward dense projections. This configuration reduces trainable parameters while preserving contextual modeling capacity.

%-------------------------------------------
\subsection{Marker Extraction Architecture}
We formulate marker extraction as token classification with BIO tagging and character-offset reconstruction.

\textbf{Task Formulation.} For each document, the system predicts spans defined by \texttt{startIndex}, \texttt{endIndex}, and \texttt{type} for five categories: \textit{Actor}, \textit{Action}, \textit{Victim}, \textit{Evidence}, and \textit{Effect}. We align subword tokens to gold character spans and exclude padding and special tokens from the loss.

\textbf{Character-Level Span Extraction Pipeline.}  Although training operates at the subword-token level using RoBERTa, supervision and evaluation are defined at the character level. Each gold marker provides \texttt{startIndex} and \texttt{endIndex} offsets over raw text. During preprocessing, we align subword tokens to character spans using tokenizer offset mappings. Tokens whose character ranges overlap a gold span receive BIO labels corresponding to the marker type. 

At inference, the model predicts BIO tags at the token level. We then reconstruct character-level spans by merging contiguous $B$/$I$ predictions of the same type and projecting them back to their original character offsets via the stored mappings. We discard inconsistent fragments (e.g., isolated $I$ without preceding $B$) and resolve overlapping spans by retaining the highest-confidence sequence. The final output strictly follows the required character-offset format.

\textbf{Transformer Backbone for Extraction.}  We use roberta-large as the encoder and attach a token-classification head to predict BIO labels for each subword token. To ensure parameter efficiency, we apply LoRA with rank $r=16$ and scaling factor $\alpha=32$, injecting adapters into the attention (\textit{query}, \textit{key}, \textit{value}) and feed-forward dense projections while freezing the backbone weights. We train the model using class-weighted cross-entropy to address label imbalance, and predictions are decoded into contiguous spans before projection back to character-level offsets.


%=============
%Logistic Regression with 5-fold stratified cross-validation on the reduced feature subset yields mean F1 in the 0.36--0.41 range. TreeSHAP analysis on a Random Forest confirms the importance of syntactic and discourse-level signals.

%\textbf{Machine Learning Models.}  We evaluate Gaussian Naive Bayes, linear and RBF SVMs, Decision Trees, Random Forests, Logistic Regression (4-fold CV), XGBoost, and an MLP trained exclusively on the 62 EDA features. All models share a fixed stratified validation split of 100 documents to ensure controlled comparison.

%This configuration reduces trainable parameters while preserving contextual capacity. The LoRA model achieves 0.79 F1 in Phase 1 and improves to 0.81 F1 after refinement (Table~\ref{tab:transformer_results}).


%\textbf{Backbone and Optimization.}  
%All extraction models use \texttt{roberta-large} with LoRA ($r=16$, $\alpha=32$) applied to attention and dense layers. We use class-weighted cross-entropy to address severe label imbalance and monitor macro F1 on a held-out validation split with early stopping.

%The provided DistilBERT baseline reports approximately 0.76 weighted F1 for detection and 0.15 for extraction.

%=============================================================================================================
\section{Experimental Design}
\subsection{Dataset}
%Both tasks utilize the shared PsyCoMark from SemEval 2026 Task 10. For detection, each instance features a \textit{conspiracy} label: \textit{Yes}, \textit{No}, or \textit{Can't tell}. For extraction, each instance contains \texttt{markers}: a list of spans with \texttt{startIndex}, \texttt{endIndex}, \texttt{type}, and \texttt{text}. The five marker types are: \textit{Actor}, \textit{Action}, \textit{Victim}, \textit{Evidence}, and \textit{Effect}.
We use the PsyCoMark dataset released for SemEval-2026 Task~10. Each instance contains a Reddit comment and a conspiracy label in \{\textit{Yes}, \textit{No}, \textit{Can't tell}\}. For the marker extraction task, each instance additionally provides character-level spans annotated with \textit{startIndex}, \textit{endIndex}, and \textit{type}. The marker categories are \textit{Actor}, \textit{Action}, \textit{Victim}, \textit{Threat}, and \textit{Evidence/Effect}.

For binary detection experiments, we retain only \textit{Yes} and \textit{No} instances and discard ambiguous labels. We train detection models on the official training split and evaluate on the corresponding development and test sets. For marker extraction, we train on the rehydrated training data. We create a held-out validation set by randomly partitioning the training data into 90\% training and 10\% validation using a fixed seed (42). The official development and test sets are reserved strictly for final inference and evaluation.

\subsection{Evaluation Protocol}
We evaluate conspiracy detection using weighted F1 under the official scoring scheme and additionally report binary F1 on the filtered \textit{Yes}/\textit{No} subset. For marker extraction, we follow the official CodaBench Overlap F1 metric, which measures boundary-sensitive agreement between predicted and gold character spans. 

\subsection{Detection Experiments}
\textbf{Feature-Based Models.}  We train Gaussian Naive Bayes, Logistic Regression, linear and RBF SVMs, Decision Trees, Random Forests, XGBoost, and an MLP on the 62-dimensional feature matrix. Continuous features are standardized and missing values are mean-imputed. We perform univariate $t$-tests on the training set to identify discriminative psycholinguistic signals and compute TreeSHAP attributions for Random Forest models to support feature-level analysis.

\textbf{Transformer Models.}  We fine-tune DistilBERT-base-uncased and BERT-base-uncased as full fine-tuning baselines. We then evaluate RoBERTa-large under two regimes: (i) data augmentation via English–German back-translation and synonym replacement, and (ii) parameter-efficient adaptation using LoRA with $r=16$ and $\alpha=32$ applied to attention and feed-forward projections. We select checkpoints using early stopping based on development weighted F1.

%\subsection{Baselines \& Evaluation}
%All detection model variants are compared against the DistilBERT binary baseline ($\sim$0.76 weighted F1 for detection and 0.15 for extraction). Evaluation protocols compare predictions on dev/test to ground truth; for binary evaluation, only instances with label \textit{Yes} or \textit{No} are included.

%\subsection{Binary Classification Strategy}
%Our binary (yes/no) conspiracy classification builds in stages. We first establish a statistical and baseline-prediction layer using only the merged lexical, discourse, sentiment, and POS feature set from the EDA pipeline. On that matrix we run univariate $t$-tests to identify features that differ significantly between classes, train a Logistic Regression with 5-fold stratified cross-validation (reporting mean F1), and fit a Random Forest on the full binary data. We then apply TreeSHAP to the Random Forest to obtain a ranked list of the most predictive psycholinguistic features. This step yields the discriminative markers and a first predictive baseline (e.g., mean F1 in the 0.36--0.41 range on this reduced feature set), as summarized later in \autoref{tab:univariate-tests} and the surrounding discussion.

%We then move to a full machine-learning evaluation on the complete EDA feature matrix (62 dimensions), including readability indices, MATTR/MTLD, and fine-grained POS features. All of these models are trained on the same training subset and evaluated on the same stratified validation set of 100 documents. The feature-based models are Naive Bayes (GaussianNB) with mean imputation and standardization, SVMs with scaled features and linear or RBF kernels, a single Decision Tree for interpretable splits, Random Forest with 5-fold stratified CV and feature importance, Logistic Regression with 4-fold cross-validation, XGBoost on the EDA feature space, and a dense neural network (MLP) trained on the same 62 EDA features rather than raw text.

%The transformer-based models are fine-tuned directly on the raw submission text. We use DistilBERT with full fine-tuning, BERT in base and large variants, RoBERTa with LoRA ($r{=}16$, $\alpha{=}32$) for parameter-efficient fine-tuning, and RoBERTa-Large with LoRA (top 12--14 layers and classifier head unfrozen), the latter also evaluated with 1$\times$ and 2$\times$ English--German back-translation data augmentation.
%---------------------------------------------------------------------

\subsection{Psycholinguistic Marker Extraction}
\textbf{Label Construction and Alignment.}  We tokenize each comment using RoBERTa subword tokenization and compute offset mappings to align tokens with gold character spans. Tokens overlapping a span receive BIO labels corresponding to the marker type. Special tokens and padding positions are excluded from loss computation.

\textbf{Model Variants.}  We compare a unified multi-type token classifier with an 11-label BIO scheme ($O$ plus $B/I$ for each marker type) against a five-model setup that trains one 3-label BIO classifier per type. At inference, we decode token sequences into contiguous spans and project them to character offsets. We discard malformed fragments and resolve overlapping spans by retaining higher-confidence predictions.

\textbf{Optimization.}  Marker labels are highly sparse relative to the $O$ class; therefore, we use class-weighted cross-entropy to mitigate class imbalance. We monitor validation F1 and apply early stopping to select the optimal checkpoint.

%We treat marker extraction as a sequence labeling task. Using a BIO (Begin, Inside, Outside) scheme, we identify subword-level boundaries for five marker categories: Action, Actor, Effect, Evidence, and Victim. The backbone is \texttt{roberta-large}, with Low-Rank Adaptation (LoRA) applied to the attention and dense layers to keep training efficient while preserving contextual representation.

%We compare two setups. In the first, a single unified model is trained on all marker types at once, with a token-classification head over the full set of type-specific BIO tags so that the model learns a joint representation. In the second, we train five separate models, one per marker type, each performing a 3-class boundary task ($O$, $B$-type, $I$-type). At inference, the five models' span predictions are combined into one extracted set.

%In both cases, subwords are aligned to gold spans and padding and special tokens are excluded from the loss. Because markers are sparse relative to the rest of the text, we use a class-weighted cross-entropy loss to penalize missed spans more strongly. We monitor macro F1 on a held-out validation set and use early stopping to select the best checkpoint.

\subsection{Training}
%All computational experiments, including the parameter-efficient fine-tuning (LoRA) of the transformer models and the extraction of psycholinguistic features, were conducted on a high-performance compute node optimized for distributed deep learning. The system is equipped with an array of 8 $\times$ AMD Instinct MI250X GPUs. Each logical GPU provides 64 GB of High Bandwidth Memory (VRAM), yielding a total of 512 GB of distributed VRAM across the node, though not all 8 were always available. The hardware acceleration is managed via the AMD ROCm software stack (Driver Version 6.12.12), enabling efficient, mixed-precision (FP16) training for our PyTorch workloads.
We run all experiments on a high-performance compute node with 8$\times$ AMD Instinct MI250X GPUs (64\,GB HBM each) using the ROCm 6.12.12 stack. We train transformer models in mixed precision (FP16) with PyTorch. LoRA significantly reduces trainable parameters and memory usage while preserving performance.

TODO by HUSNAIN: learning rate
batch size
max sequence length
epochs? details please

%-----------------------------------------------------------------------
\section{Results and Analysis}

\subsection{Document-Level Conspiracy Detection}
\textbf{Feature-Driven Detection.} Table~\ref{tab:ml_results} reports performance for models trained on the 62-dimensional psycholinguistic feature space. \textbf{Four-fold stratified cross-validation is conducted on $N=2{,}929$ training instances, and final evaluation is performed on a fixed held-out subset ($N=82$).} Logistic Regression achieves the strongest cross-validation performance (Macro F1 = 0.5628), indicating that linear decision boundaries capture a meaningful portion of stylistic variation. On the held-out subset, ensemble methods show improved generalization, with XGBoost achieving the highest weighted F1 (0.4724). Overall, feature-driven models provide moderate discriminative power but remain substantially below contextual transformer models.

\textbf{Transformer-Based Detection.} Table~\ref{tab:transformer_results} summarizes results across transformer configurations. \textbf{DistilBERT, BERT, and augmented RoBERTa models are evaluated on the held-out subset ($N=82$).}
DistilBERT-base achieves balanced performance (Weighted F1 = 0.6895), while BERT-base underperforms, particularly on the minority class. Data-augmented RoBERTa variants improve recall for the \textit{Yes} class but do not substantially exceed baseline contextual models.

The LoRA-adapted RoBERTa-large configuration yields the strongest overall detection performance (Weighted F1 = 0.8319, Macro F1 = 0.8162). \textbf{This model is evaluated on the official dev\_public split ($N=77$), which differs from the held-out subset used for the other transformer models.} The improvement suggests that parameter-efficient adaptation preserves contextual capacity while strengthening class separation.

\textbf{Model Comparison.} Table~\ref{tab:extraction_comparison} reports Overlap F1 on the official development set. 
\textbf{We compute overlap F1 using the CodaBench character-level overlap criterion.} The unified multi-type BIO model achieves 0.05 Overlap F1, indicating instability under a shared label space with sparse supervision. In contrast, the five-model configuration reaches approximately 0.21, suggesting that separating marker types reduces cross-type interference and improves boundary consistency.




\begin{table*}[t] % [t] forces both to the top of the page
  \centering
  \small
  
  % --- FIRST TABLE ---
  \begin{tabular}{l ccc | ccc}
    \hline
    & \multicolumn{3}{c|}{\textbf{4-Fold Cross-Validation (Train: $N=2,929$)}} & \multicolumn{3}{c}{\textbf{Held-Out Test ($N=82$)}} \\
    \textbf{Model} & \textbf{Accuracy} & \textbf{F1 (Macro)} & \textbf{F1 (Wgt)} & \textbf{Accuracy} & \textbf{F1 (Macro)} & \textbf{F1 (Wgt)} \\
    \hline
    Naive Bayes      & 0.5596 $\pm$ 0.045 & 0.5530 $\pm$ 0.040 & 0.5588 $\pm$ 0.041 & 0.4268 & \textbf{0.4267} & 0.4276 \\
    SVM              & 0.5756 $\pm$ 0.030 & 0.5299 $\pm$ 0.030 & 0.5496 $\pm$ 0.029 & 0.4700 & 0.3135 & 0.3988 \\
    Random Forest    & 0.5852 $\pm$ 0.032 & 0.5439 $\pm$ 0.041 & 0.5624 $\pm$ 0.037 & \textbf{0.5200} & 0.3572 & 0.4508 \\
    Logistic Reg.    & \textbf{0.5927 $\pm$ 0.034} & \textbf{0.5628 $\pm$ 0.024} & \textbf{0.5782 $\pm$ 0.026} & 0.4600 & 0.3238 & 0.4069 \\
    XGBoost          & 0.5695 $\pm$ 0.021 & 0.5500 $\pm$ 0.024 & 0.5626 $\pm$ 0.022 & \textbf{0.5200} & 0.3929 & \textbf{0.4724} \\
    \hline
  \end{tabular}
  \caption{Performance metrics for traditional ML binary classification. Cross-validation scores report the mean $\pm$ 2 standard deviations across 4 stratified folds. Held-out test metrics evaluate a fixed 82-sample subset.}
  \label{tab:ml_results}

  \vspace{0.5cm} % Adjusts the vertical gap between the two tables

  % --- SECOND TABLE ---
  \begin{tabular}{l l ccc | cc}
    \hline
    & & \multicolumn{3}{c|}{\textbf{Aggregate Metrics}} & \multicolumn{2}{c}{\textbf{Per-Class F1}} \\
    \textbf{Model Architecture} & \textbf{Training Strategy} & \textbf{Accuracy} & \textbf{F1 (Macro)} & \textbf{F1 (Wgt)} & \textbf{F1 (No)} & \textbf{F1 (Yes)} \\
    \hline
    DistilBERT-base & Full Fine-tuning & 0.6951 & 0.6814 & 0.6895 & 0.75 & 0.62 \\
    BERT-base & Top 10 Layers & 0.6098 & 0.5661 & 0.5829 & 0.70 & 0.43 \\
    \hline
    RoBERTa-large (BT1) & Augmentation (2$\times$) + Top 12 & 0.6707 & 0.6667 & 0.6712 & 0.70 & 0.63 \\
    RoBERTa-large (BT2) & Augmentation (3$\times$) + Top 18 & 0.6463 & 0.5958 & 0.6132 & 0.74 & 0.45 \\
    \hline
    \textbf{RoBERTa-large (LoRA)}* & \textbf{PEFT} ($r=16$) & \textbf{0.8312} & \textbf{0.8162} & \textbf{0.8319} & \textbf{0.87} &\textbf{ 0.76}  \\
    \hline
  \end{tabular}
  \caption{Performance comparison of transformer-based models on the binary conspiracy detection task (\textit{Yes} vs. \textit{No}). Metrics for the first four models evaluate a held-out set ($N=82$). *The LoRA model evaluates the official \texttt{dev\_public} split ($N=77$).}
  \label{tab:transformer_results}
\end{table*}


\begin{table}[htbp]
    \centering
    \renewcommand{\arraystretch}{1.35}
    % This scales the table to exactly fit one column width
    \resizebox{\columnwidth}{!}{%
    \begin{tabular}{l cc}
        \hline
        \textbf{Feature} & \textbf{Multi-type} & \textbf{Five-model} \\
        \hline
        Label scheme & 11 (joint BIO) & 3 ($O$, $B$, $I$) per type \\
        Max sequence length & 256 & 512 \\
        Learning rate & $3\times10^{-4}$ & $2\times10^{-4}$ \\
        Class weights ($B$/$I$ vs.\ $O$) & No & Yes \\
        Effective batch size & 16 & 16 \\
        Epochs / early stopping & 10 / patience 3 & 10 / patience 3 \\
        LoRA rank $r$ & 16 & 16 \\
        \hline
        Dev Overlap F1 & \textit{0.05} & \textit{$\sim$0.21} \\
        \hline
    \end{tabular}%
    }
    \caption{Comparative summary of the two marker extraction strategies. Dev Overlap F1 is computed on the official dev set using the CodaBench overlap criterion.}
    \label{tab:extraction_comparison}
\end{table}

\textbf{Boundary Behavior and Error Patterns.} 
Span extraction remains substantially more challenging than document-level detection due to extreme label sparsity and boundary sensitivity. Most errors arise from partial span misalignment rather than incorrect type prediction. The model frequently identifies semantically relevant segments but fails to align exact character offsets, which directly lowers Overlap F1. Overlapping narrative roles, particularly between \textit{Actor} and \textit{Victim} and between \textit{Evidence} and \textit{Effect}, further reduce precision. Longer clauses containing multiple embedded entities increase span fragmentation and negatively affect recall for rare marker categories.

\subsection{Comparative Observations}

Across both tasks, contextual transformer models substantially outperform feature-driven approaches in detection. LoRA adaptation provides the strongest overall results while maintaining parameter efficiency. In contrast, span extraction remains constrained by sparse supervision and strict boundary evaluation, with per-type modeling demonstrating greater robustness than unified tagging under current training conditions.


\section{Conclusions}
In this paper, we presented the NUST PsyAI team's system for SemEval-2026 Task 10, addressing the complex challenges of conspiracy text detection and psycholinguistic marker extraction. Through a systematic evaluation of traditional machine learning algorithms, baseline transformers, and large language models, we demonstrated that deep semantic representations are crucial for decoding the nuanced language of conspiratorial discourse. Our leading approach, which utilized Parameter-Efficient Fine-Tuning (LoRA) on the RoBERTa-large architecture, consistently outperformed the standard DistilBERT baselines. For document-level detection, this optimized architecture proved highly effective at distinguishing conspiracy narratives from standard text. For granular marker extraction, we discovered that deploying an ensemble of five specialized models yielded superior boundary overlap scores compared to a unified sequence-labeling approach. Future work will focus on resolving the inherent latency of multi-model ensembles and improving model generalizability across diverse social media ecosystems.

\section*{Limitations}
Despite the robust performance of our LoRA-optimized architectures, this study presents several limitations. First, because our models were fine-tuned exclusively on Reddit submission statements, they may struggle to generalize to the distinct linguistic nuances and shorter, fragmented formats of other social media platforms like Twitter or Telegram. Second, the extreme sparsity of psycholinguistic markers in the dataset makes precise character-level boundary extraction highly challenging, frequently resulting in partial span overlaps rather than exact matches. Finally, while our five-model ensemble approach maximizes boundary detection accuracy for individual marker types, it inherently introduces higher inference latency and computational overhead compared to a unified single-model architecture.

\section*{Acknowledgements}
We extend our gratitude to the organizers of SemEval-2026 Task 10 for conceptualizing the PsyCoMark challenge, curating the dataset, and facilitating a rigorous evaluation via CodaBench. Their efforts in establishing this benchmark have been invaluable in advancing our research in psycholinguistic analysis and conspiracy detection.


\begin{thebibliography}{99}

\bibitem[Samory et al.(2026)]{samory-etal-2026-semeval}
Mattia Samory, Felix Soldner, and Veronika Batzdorfer.
\newblock 2026.
\newblock SemEval-2026 Task 10: PsyCoMark -- Psycholinguistic Conspiracy Marker Extraction and Detection.
\newblock \emph{Proceedings of the 20th International Workshop on Semantic Evaluation (SemEval-2026)}.

\bibitem[Bang et al.(2023)]{bang2023multitask}
Yejin Bang, Samuel Cahyawijaya, Nayeon Lee, Wenliang Dai, Dan Su, et al.
\newblock 2023.
\newblock A multitask, multilingual, multimodal evaluation of ChatGPT on reasoning, hallucination, and interactivity.
\newblock \emph{arXiv preprint arXiv:2302.04023}.

\bibitem[Bessi et al.(2015)]{bessi2015science}
Alessandro Bessi, Mauro Coletto, George Alexandru Davidescu, Antonio Scala, Guido Caldarelli, and Walter Quattrociocchi.
\newblock 2015.
\newblock Science vs conspiracy: Collective narratives in the age of misinformation.
\newblock \emph{PloS one}, 10(2):e0118093.

\bibitem[Chen et al.(2025)]{chen2025efficient}
Wei Chen, Yuxiao Dong, and Jie Tang.
\newblock 2025.
\newblock Efficient fine-tuning of large language models for misinformation detection.
\newblock \emph{Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (ACL)}.

\bibitem[Ding et al.(2023)]{ding2023parameter}
Ning Ding, Yujia Qin, Guang Yang, Fuchao Wei, Zonghan Yang, et al.
\newblock 2023.
\newblock Parameter-efficient fine-tuning of large-scale pre-trained language models.
\newblock \emph{Nature Machine Intelligence}, 5(3):220-235.

\bibitem[Ferrara(2020)]{ferrara2020types}
Emilio Ferrara.
\newblock 2020.
\newblock What types of COVID-19 conspiracies are populated by Twitter bots?
\newblock \emph{First Monday}, 25(6).

\bibitem[Giachanou et al.(2023)]{giachanou2023psycholinguistic}
Anastasia Giachanou, Bilal Ghanem, and Paolo Rosso.
\newblock 2023.
\newblock A psycholinguistic and transformer-based approach to conspiracy theory detection.
\newblock \emph{Information Processing \& Management}, 60(3):103288.

\bibitem[Hoseini et al.(2023)]{hoseini2023conspiracy}
Mahmoud Hoseini, Shirin Nilizadeh, and A. E. Çelik.
\newblock 2023.
\newblock Conspiracy theory detection on social media using transformers and linguistic features.
\newblock \emph{IEEE Transactions on Computational Social Systems}, 10(4):1854-1865.

\bibitem[Hu et al.(2021)]{hu2021lora}
Edward J Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, and Weizhu Chen.
\newblock 2021.
\newblock LoRA: Low-rank adaptation of large language models.
\newblock \emph{International Conference on Learning Representations (ICLR)}.

\bibitem[Klein et al.(2019)]{klein2019exploring}
Colin Klein, Peter Clutton, and Adam Dunn.
\newblock 2019.
\newblock Pathways to conspiracy: The social and linguistic precursors of involvement in Reddit’s conspiracy theory forum.
\newblock \emph{PloS one}, 14(11):e0225098.

\bibitem[Miani et al.(2021)]{miani2021loco}
Alessandro Miani, Thomas Hills, and Adrian Bangerter.
\newblock 2021.
\newblock LOCO: The 88-million-word language of conspiracy corpus.
\newblock \emph{Behavior Research Methods}, 53(4):1790-1805.

\bibitem[Min et al.(2022)]{min2022divide}
Sewon Min, Kalpesh Krishna, Xinxi Lyu, Mike Lewis, et al.
\newblock 2022.
\newblock Divide and conquer: A program representation for fact-checking.
\newblock \emph{Proceedings of the Association for Computational Linguistics (ACL)}.

\bibitem[Pan et al.(2023)]{pan2023risk}
Yikang Pan, et al.
\newblock 2023.
\newblock On the risks of large language models in generating misinformation.
\newblock \emph{Proceedings of the Conference on Empirical Methods in Natural Language Processing (EMNLP)}.

\bibitem[Wadden et al.(2020)]{wadden2020fact}
David Wadden, Shanchuan Lin, Kyle Lo, et al.
\newblock 2020.
\newblock Fact or fiction: Verifying scientific claims.
\newblock \emph{Proceedings of the Conference on Empirical Methods in Natural Language Processing (EMNLP)}.

\bibitem[Wang et al.(2024)]{wang2024social}
Liang Wang, Jinyan Li, and Huan Liu.
\newblock 2024.
\newblock Social network and discourse: Graph neural networks for early conspiracy detection.
\newblock \emph{Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing (EMNLP)}.

\bibitem[Zhang et al.(2024)]{zhang2024llm}
Yue Zhang, Ming Li, and Chenhao Tan.
\newblock 2024.
\newblock LLM zero-shot capabilities for conspiracy theory narrative extraction.
\newblock \emph{Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics (NAACL)}.

\end{thebibliography}




\appendix

\section{Appendix}
\label{sec:appendix}

\fig
\begin{figure}[h]
    \centering
    \includegraphics[width=1\linewidth]{visuals/figures/density/ma_ttr_density.png}
    \caption{MA-TTR Density plot from the EDA}
    \label{fig:placeholder}
\end{figure}


\begin{figure}[h]
    \centering
    \includegraphics[width=1\linewidth]{visuals/figures/density/sentiment_compound_density.png}
    \caption{Compound Sentiment Score Distribution from the EDA}
    \label{fig:placeholder}
\end{figure}


\begin{figure}[t]
    \centering
    \includegraphics[width=1\linewidth]{visuals/figures/decision_tree_feature_importance.png}
    \caption{Top important features for Decision Tree}
    \label{fig:placeholder}
\end{figure}

\begin{figure}[t]
    \centering
    \includegraphics[width=1\linewidth]{visuals/confusion_matrix_lora.png}
    \caption{Confusion Matrix for the \texttt{dev\_public} split used as a Test Set for RoBERTA-large LoRA finetuning}
    \label{fig:placeholder}
\end{figure}

\end{document}
