% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}

%CUSTOM PACKAGES
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{listings}
\usepackage{xcolor}
\usepackage{nameref}
\usepackage{natbib}
\colorlet{punct}{red!60!black}
\definecolor{background}{HTML}{EEEEEE}
\definecolor{delim}{RGB}{20,105,176}
\colorlet{numb}{magenta!60!black}
\usepackage{amsmath}
\usepackage{tabularx}
\usepackage{amssymb} % For \checkmark
\usepackage{caption}

\lstdefinelanguage{json}{
    basicstyle=\fontsize{9}{11}\ttfamily,
    numbers=left,
    numberstyle=\scriptsize,
    stepnumber=1,
    numbersep=4pt,
    showstringspaces=false,
    breaklines=true,
    frame=lines,
    backgroundcolor=\color{background},
    literate=
     *{0}{{{\color{numb}0}}}{1}
      {1}{{{\color{numb}1}}}{1}
      {2}{{{\color{numb}2}}}{1}
      {3}{{{\color{numb}3}}}{1}
      {4}{{{\color{numb}4}}}{1}
      {5}{{{\color{numb}5}}}{1}
      {6}{{{\color{numb}6}}}{1}
      {7}{{{\color{numb}7}}}{1}
      {8}{{{\color{numb}8}}}{1}
      {9}{{{\color{numb}9}}}{1}
      {:}{{{\color{punct}{:}}}}{1}
      {,}{{{\color{punct}{,}}}}{1}
      {\{}{{{\color{delim}{\{}}}}{1}
      {\}}{{{\color{delim}{\}}}}}{1}
      {[}{{{\color{delim}{[}}}}{1}
      {]}{{{\color{delim}{]}}}}{1},
}



% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{SemEval-2026 Task 10: PsyCoMark -- Psycholinguistic Conspiracy Marker Extraction and Detection}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Mattia Samory \\
  Sapienza University of Rome \\
  \small\texttt{mattia.samory@uniroma1.it} \\\And
  Felix Soldner \\
  Independent Researcher\\
  \small\texttt{felix.soldner@pm.me} \\\And
  Veronika Batzdorfer \\
  Karlsruhe Institute of Technology \\
  \small\texttt{veronika.batzdorfer@kit.edu } 
  }

%\author{
%  \textbf{First Author\textsuperscript{1}},
%  \textbf{Second Author\textsuperscript{1,2}},
%  \textbf{Third T. Author\textsuperscript{1}},
%  \textbf{Fourth Author\textsuperscript{1}},
%\\
%  \textbf{Fifth Author\textsuperscript{1,2}},
%  \textbf{Sixth Author\textsuperscript{1}},
%  \textbf{Seventh Author\textsuperscript{1}},
%  \textbf{Eighth Author \textsuperscript{1,2,3,4}},
%\\
%  \textbf{Ninth Author\textsuperscript{1}},
%  \textbf{Tenth Author\textsuperscript{1}},
%  \textbf{Eleventh E. Author\textsuperscript{1,2,3,4,5}},
%  \textbf{Twelfth Author\textsuperscript{1}},
%\\
%  \textbf{Thirteenth Author\textsuperscript{3}},
%  \textbf{Fourteenth F. Author\textsuperscript{2,4}},
%  \textbf{Fifteenth Author\textsuperscript{1}},
%  \textbf{Sixteenth Author\textsuperscript{1}},
%\\
%  \textbf{Seventeenth S. Author\textsuperscript{4,5}},
%  \textbf{Eighteenth Author\textsuperscript{3,4}},
%  \textbf{Nineteenth N. Author\textsuperscript{2,5}},
%  \textbf{Twentieth Author\textsuperscript{1}}
%\\
%\\
%  \textsuperscript{1}Affiliation 1,
%  \textsuperscript{2}Affiliation 2,
%  \textsuperscript{3}Affiliation 3,
%  \textsuperscript{4}Affiliation 4,
%  \textsuperscript{5}Affiliation 5
%\\
%  \small{
%    \textbf{Correspondence:} \href{mailto:email@domain}{email@domain}
%  }
%}

\begin{document}
\maketitle
\begin{abstract}
Despite the need to address the proliferation of conspiracy theories in online discussions, there is a lack of benchmarks for effectively detecting conspiracy-related content in everyday conversational settings. We introduce a novel dataset of comments from Reddit, ranging from politics to TV series, as well as two synergetic tasks: (1) extracting five psycholinguistic markers, grounded in evolutionary psychology, and (2) detecting conspiracy content. The data enable multi-task approaches, allowing testing of whether marker extraction improves detection performance. 
\end{abstract}

\section{Introduction}
% This task introduces novel aspects to the computational analysis of conspiracy theory language. 
Most related NLP work aiming to detect conspiracy theories focuses on detecting specific claims, such as the belief that vaccines cause autism \cite{mitra2016understanding}. However, conspiracy theories may adapt to a variety of topical contexts, complicating their detection in realistic settings \cite{samory2018government}. Yet, psychology research points to invariants in the structure of conspiracy theory narratives due to the functions that they serve in group life, which are independent of the specific claims of each theory \cite{10.1177/1745691618774270,10.3389/fpsyg.2013.00406}. We propose extracting the markers of conspiracy theory narratives and detecting conspiracy theories as two synergistic tasks. We introduce two novel aspects to the computational analysis of conspiracy theory language: psychologically grounded markers and topic-agnostic conversations. 

% We introduce span-level markers that are expressly informed by the evolutionary psychology of conspiracy theories, and a topic-agnostic dataset of Reddit comments annotated for such markers.

\paragraph{Psychologically-Grounded Markers} Evolutionary psychology literature posits that conspiracy beliefs evolved as adaptive mechanisms for detecting dangerous coalitions \cite{10.1177/1745691618774270}. To shield oneself and one's in-group from threats, conspiracy thinking trains the individual to anticipate the secret scheming of malicious opponents. We introduce a novel dataset that maps the psychological mechanisms of conspiracy beliefs to linguistic markers in conspiracy theory discussions. Table \ref{tab:makers} summarizes the relationship between psychological mechanisms and linguistic markers in our annotation schema. This grounding ensures that the annotations capture the underlying psychological drivers of conspiracy thinking, affording deeper insights and explainability. 
\begin{figure}[tbp!]
    \centering
    \includegraphics[width=\linewidth]{latex/summary_statement.png}
    \caption{Example of a submission statement on Reddit (anonymized). 
    }
    \label{fig:summary_statement}
\end{figure}
\paragraph{Topic-Agnostic Conversations} Existing corpora focus on pre-determined conspiracy topics --- in fact, there is criticism about the influence of topicality even in the psychological measurement of conspiracy thinking. Few resources, like the LOCO dataset \cite{miani2021loco}, encompass a broad range of topics but do not include conspiracy theories as they are discussed on social media. Our dataset offers a contextually rich resource for analysis across a broad range of topics in everyday social media conversations---including subreddits both overtly about conspiracy theories (e.g., \texttt{conspiracy}, \texttt{ufobelievers}, \texttt{epstein}) and of general interest (e.g., \texttt{hongkong}, \texttt{truecrime}, \texttt{news}). The diversity ensures that models cannot rely on topic-specific shortcuts (e.g., associating "vaccines" with conspiracy theories) but need to learn markers that generalize across domains.

\section{Task Structure}

By combining detection and marker extraction, the PsyCoMark task enables the development of generalizable and interpretable models of conspiracy theory language. %To this end, participants were allowed to leverage external data and pre-trained models.


\paragraph{Subtask 1: Conspiracy Marker Extraction}

Participants were tasked to develop models to extract the spans indicative of conspiracy theory markers. Each document may contain zero or multiple spans corresponding to each marker, which may overlap with each other. %We will evaluate the models' ability to detect each marker using an overlap-based F1-score. The overall performance for the task will be the macro-average of the per-marker F1 scores.

\paragraph{Subtask 2: Conspiracy Detection}Participants in this subtask have developed models to classify comments as either conspiracy-related or non-conspiracy-related. 

Participants were encouraged but not required to contribute end-to-end systems that address both tasks jointly, and were allowed to leverage external data and pre-trained models to remove barriers to participation. %We will evaluate performance with macro-averaged F1-score, restricting the test set to instances classified as ``conspiracy'' or ``not conspiracy''.



\section{Data}




\begin{table}[]
\centering\small
\begin{tabular}{@{}lll@{}}

\toprule
\textbf{Psychological driver}& \textbf{Marker}& \textbf{Documents} \\ \midrule
Alliance detection& Actor& 4012                 \\
\multirow{2}{*}{Threat management}& Effect& 3240                 \\
                                   & Victim& 2603                  \\
Pattern perception& Evidence& 3178                  \\
Agency detection& Action& 3961                 \\ \bottomrule
\end{tabular}
\caption{Relationship between the psychological mechanisms underpinning conspiracy beliefs and the linguistic markers used in our annotation schema.\label{tab:makers}}

\end{table}

\begin{table}[htbp]
\centering\small
\begin{tabular}{lr}
\toprule
\textbf{Metric} & \textbf{Value} \\
\midrule
Documents & 5354 \\
Unique documents & 4720 \\
Median tokens per document & 61 \\
Annotators & 138 \\
Subreddits & 226 \\
Documents labeled ``conspiracy'' & 1925\\
Documents labeled ``not conspiracy'' & 2457\\
Documents labeled ``can't tell'' & 972\\
Documents with any marker & 4511 \\
Total markers  & 27891 \\
\bottomrule
\end{tabular}
\caption{Statistics of the annotated dataset.\label{tab:data}}
\end{table}


\paragraph{Submission Statements}
To include texts that cover a wide variety of topics and include rich narrative information, we focused our data on submission statements on Reddit. Submission statements stem from the practice of accompanying submissions that contain only media---like a picture, video, or URL---with a comment describing why the media relates to a subreddit's topic. For example, the submission statement accompanying a newspaper article in the subreddit \texttt{conspiracy} may summarize how the article supports a conspiracy theory (see, e.g., Fig. \ref{fig:summary_statement}), whereas in the subreddit \texttt{law} could connect news events to a broader legal discourse. Since submission statements encode users' summaries of the narrative in media, they represent an ideal format from which to extract linguistic markers of conspiracy theories. This may elicit higher-density marker usage than spontaneous comments, where conspiracy narratives may remain implicit. Submission statements are common practice in a wide range of subreddits, which encompass a broad topical range. 

\begin{lstlisting}[language=json,firstnumber=1,label={lst:example}, caption=Example data point]
{
    "_id": "t1_xxxyyyzzz",
    "conspiracy": "Yes",
    "markers": [{
            "type": "Actor",
            "text": "CNN",
            "startIndex": 11, 
            "endIndex": 13 
        }//, ...
    ],
    "subreddit": "conspiracy",
    "annotator": "annotator_1"
}
\end{lstlisting}

\paragraph{Sampling}
We selected first-level comments, authored by the discussion starter, that began with \verb:ss|((submission )?statement)[^a-zA-Z\d]:, which is the standard format for submission statements. We manually excluded subreddits where submission statements were used for alternative purposes. We converted markdown to text and converted URLs to special tokens. We discarded comments containing quotes to limit the text by the comment's author. To balance content richness with the feasibility of annotation, we retained comments between 160 and 1000 characters. Since conspiracy theories are relatively rare across Reddit, we oversampled comments that likely contain them: we sampled 1500 comments uniformly at random from the subreddit \texttt{conspiracy} and the remaining comments from the remaining subreddits with at least 10 submission statements, sampling an equal number per subreddit iteratively. 

\paragraph{Annotation}
We annotated the data through the crowdwork platform Prolific. Annotators (N=138) were native English speakers recruited from the U.S., U.K., Canada, and Australia. The annotation task comprised 1) a consent form, 2) a pre-survey about conspiracy thinking and related psychological predispositions and sociodemographics, 3) task instructions and codebook description, 4) a training task, 5) annotation of 10 documents for markers and three-way classification (conspiracy, not conspiracy, can't tell), and 6) a debriefing page. Annotators were compensated with £9/h on average. To ensure quality, whenever an annotator did not annotate more than two markers in at least one document of the batch, we discarded the entire batch from the final data.


\begin{figure}[tbp!]
    \centering
    \includegraphics[width=\linewidth]{latex/label_composition.png}
    \caption{Conspiracy annotations for the top 20 subreddits in the dataset. Even self-professed conspiracy theory communities and those associated with them in the public discourse show a mixture of labels, e.g., the \texttt{conspiracy} and \texttt{UFO} subreddits show fewer than 50\% "Yes" labels.  
    }
    \label{fig:label_composition}
\end{figure}

\paragraph{Statistics} 
The annotated dataset comprises 5,354 summary statements (see Tab. \ref{tab:data} for summary statistics). Annotators substantially agreed, at a Krippendorff's $\alpha=0.58$ for the binary conspiracy labels, and unitizing $\alpha=0.15$ (macro F1=0.19 at IoU=0.5) for marker spans. Data encompasses 226 subreddits, with 1256 comments coming from the \texttt{conspiracy} subreddit. 
The dataset is available \footnote{\url{https://dx.doi.org/10.5281/zenodo.15114172}} under a CC-BY license. A sample of 20\% of the data was retained as a private test set. Listing \ref{lst:example} showcases an example data point.



\begin{figure}[tbp!]
    \centering
    \includegraphics[width=\linewidth]{latex/marker_vs_labels.png}
    \caption{Disaggregation of conspiracy labels based on the observed combinations of marker types in the dataset. The more complete the set of markers a narrative contains, the more likely it relates a conspiracy theory, thus validating the hypothesized association between theoretical drivers of conspiracy thinking and their pragmatics in conspiracy narratives.}
    \label{fig:marker_vs_labels}
\end{figure}
\subsection{Data Exploration}

Next, we describe the annotated dataset to surface opportunities for modeling and to provide due context for our evaluation methodology. 



\paragraph{Label Composition}
Labels are far from homogeneous within communities, and the "hardest" ones are not always the most conspiratorial (see Fig. \ref{fig:label_composition}). Some cases follow expectations about their conspiracy theory content, such as \texttt{media\_criticism} and \texttt{conspiracy\_commons} standing out as high-Yes communities (>60\% Yes), while \texttt{Futurology}, \texttt{Africa}, and \texttt{news} skew heavily No. Yet, most subreddits show a balanced mixture of labels: e.g., the majority of comments from the \texttt{conspiracy} community are labeled either No or Can't tell. Furthermore, the substantial Can't tell slices in subreddits like \texttt{centrist}, \texttt{CoronavirusCirclejerk}, \texttt{HongKong}, and \texttt{Israel\_Palestine} signal that ambiguity is frequent, and may be structurally tied to political and epistemic gray zones --- exactly where a binary conspiracy/non-conspiracy framing strains most visibly. Once closely inspected, conspiracy theories appear present in a broader range of topics than may be expected --- and perhaps surprisingly, their frequency is not substantially different between fringe and mainstream conversational spaces. Therefore, modeling topical and community contexts may provide some, but limited information for the task.

\paragraph{Marker Distribution across Labels}


The presence of all five marker types simultaneously is strongly associated with the conspiracy label (see Fig. \ref{fig:marker_vs_labels}): items with all five markers (n=1,256) yield the highest Yes rate of any configuration, while items with zero markers (n=749) are predominantly labeled No, with varying proportions in between. Yet marker count alone is not a sufficient signal of the document's label: e.g., the Actor-only configuration (n=64) is overwhelmingly No, and several four-marker combinations show substantial No and Can't tell fractions, revealing that it is the specific combination of markers, not sheer quantity, that constitutes a conspiracy narrative. Yet, this association opens up opportunities for joint modeling of marker extraction and document classification.



\paragraph{Marker Co-Occurrence}
\begin{figure}[tbp]
    \centering
    \includegraphics[width=\linewidth]{latex/marker_cooccurrence.png}
    \caption{Network of annotated spans based on their marker type. Node size reflects the number of annotated spans; edge weight corresponds to the number of documents containing each pair of marker types. }
    \label{fig:marker_cooccurrence}
\end{figure}
Figure \ref{fig:marker_cooccurrence} shows the co-presence of marker spans within documents. Actor (8,295 instances) and Action (6,151) are the backbone of the annotation space, and their co-occurrence edge (3,178) is the heaviest in the network, confirming that most annotated spans describe an agent doing something. Effect (4,748) and Evidence (4,609) cluster tightly with both Actor and Action, forming a near-complete subgraph that mirrors the rhetorical spine of a conspiracy claim: someone does something, with evidence, causing harm. Victim co-occurs least with Evidence (1,725) and most with Action (2,078), suggesting that victim framing in conspiracy text is structurally decoupled from evidentiary language --- harm is asserted, not argued. Therefore, beyond the presence and number of markers, their relations appear to reflect the rhetorical structure of conspiracy narratives, information which may be leveraged in the marker extraction task.


\paragraph{Marker Position in Texts}
\begin{figure}[tbp]
    \centering
    \includegraphics[width=\linewidth]{latex/marker_position.png}
    \caption{Distribution of marker positions in submission statements' text. Narratives typically start with Actor, Action, and Evidence.
}
    \label{fig:marker_position}
\end{figure}
The rhetorical relations between markers are further elucidated by their position in the narrative's progression. Actors, Actions, and Evidence front-load their claims: the three marker types peak sharply in the first 20\% of a document and decay monotonically toward the end, suggesting that conspiracy narratives tend to establish "who did what" and "why we know" early, before elaborating consequences. Effect markers, by contrast, show a mild right skew, accumulating through the middle and tail of texts, consistent with a rhetorical structure where harm or outcome is narrated after the conspiratorial premise has been established. Victim markers remain near-uniform throughout, indicating they are opportunistically placed rather than structurally motivated.

\section{Evaluation Methodology}
\label{sec:eval_methodology}

In light of our data analysis, our evaluation framework is designed to assess both the local precision of marker extraction (Subtask 1) and the global accuracy of conspiracy detection (Subtask 2). We supplement standard lexical metrics with a Bayesian hierarchical analysis to account for the inherent variance in subreddit difficulty and participant performance.

\subsection{Subtask 1: Marker Extraction (Span Identification)}
Evaluating the extraction of psycholinguistic markers presents a distinct challenge due to the "fuzzy" boundaries of spans representing abstract concepts like \textit{Threat}. Following recent span-labeling tasks \cite{vazquez-etal-2025-semeval}, we adopt a \textbf{Token-based Intersection over Union (IoU)} matching strategy.

\paragraph{Span Matching} 
We represent each predicted span $S_p$ and ground-truth span $S_g$ as sets of token indices. A prediction is classified as a True Positive ($TP$) if it satisfies an overlap threshold $\tau \ge 0.5$ with a ground-truth span of the same category:
$$ \text{IoU}(S_p, S_g) = \frac{|S_p \cap S_g|}{|S_p \cup S_g|} \ge 0.5 $$

Unmatched predictions are counted as False Positives ($FP$), and unmatched ground-truth spans as False Negatives ($FN$). 

\paragraph{Aggregation} 
We calculate Precision, Recall, and $F_1$-score per marker type. The primary metric for Subtask 1 is the \textbf{Macro-averaged $F_1$}, which ensures that models are evaluated on their ability to identify rare, reasoning-heavy markers (e.g., \textit{Evidence}) as effectively as more frequent entity-based markers (e.g., \textit{Actor}).

\subsection{Subtask 2: Conspiracy Detection}
Subtask 2 is evaluated as a binary classification problem (\textit{Conspiracy} vs. \textit{Non-Conspiracy}). 

\paragraph{Primary Metric} 
The official ranking is determined by weighted-$F_1$. Given that our dataset features a nearly balanced label distribution, weighted-$F_1$ serves as a robust indicator of overall system performance, effectively capturing the balance between precision and recall while respecting the specific class frequencies within the test set. This choice ensures that the final rankings reflect a model's true generalizability across the topic-agnostic corpus, providing a stable measure that accounts for every prediction made by the systems.

% \subsection{Robustness and Difficulty Analysis}
% To provide a deeper diagnostic of model performance, we utilize \textbf{Markov Chain Monte Carlo (MCMC)} sampling to fit a Bayesian hierarchical model to the submission data.


% \section{Baseline Systems}
% To establish a performance floor and illustrate the feasibility of the proposed tasks, we provide two transformer-based baselines. Our choice of architecture, DistilBERT \cite{sanh2019distilbert}, serves as a computationally efficient yet robust foundation, lowering the barrier to entry while maintaining high sensitivity to the nuanced linguistic patterns inherent in conspiracy-related discourse.


% \begin{table}[tp]
% \centering\small
% \begin{tabular}{lr}
% \toprule
% \textbf{Hyperparameter} & \textbf{Value} \\
% \midrule
% Architecture & DistilBERT \\
% Learning Rate & $2 \times 10^{-5}$ \\
% Batch Size & 16 \\
% Max Sequence Length & 128 tokens \\
% Optimizer & AdamW ($\epsilon=1e-8$) \\
% Training Epochs & 10 \\
% Weight Decay & 0.01 \\
% \bottomrule
% \end{tabular}
% \caption{Hyperparameter configurations for the baseline models.}
% \label{tab:params}
% \end{table}

\begin{table}[tp]
\centering
\small
\begin{tabular}{lccc}
\toprule
\textbf{Marker Type} & \textbf{Precision} & \textbf{Recall} & \textbf{F1-Score} \\
\midrule
Action   & 0.1342 & 0.1031 & 0.1166 \\
Actor    & 0.3772 & 0.2277 & 0.2840 \\
Effect   & 0.1139 & 0.0797 & 0.0938 \\
Evidence & 0.1214 & 0.0884 & 0.1023 \\
Victim   & 0.2541 & 0.1520 & 0.1902 \\
\midrule
\textbf{Macro Average} & --- & --- & 0.1574 \\
\textbf{Micro Average} & 0.2110 & 0.1426 & \textbf{0.1702} \\
\bottomrule
\end{tabular}
\caption{Baseline results for Subtask 1 (Token-level IoU with 0.5 threshold). The \textit{Actor} and \textit{Victim} categories appear more linguistically distinct, while reasoning-heavy markers like \textit{Evidence} prove significantly more challenging for the baseline model.}
\label{tab:subtask1_baseline}
\end{table}

\subsection{Baseline Model Setup}Both baselines leverage the \texttt{distilbert-base-uncased} model as their backbone. We utilize the hidden state of the special \texttt{[CLS]} token for sequence-level representation in Subtask 2, and the full sequence of hidden states for token-level boundary detection in Subtask 1. All models were implemented using the \textit{Hugging Face Transformers} library and were optimized using the AdamW optimizer with a linear learning rate scheduler. We maintained a consistent set of hyperparameters across both subtasks to ensure a controlled comparison.
% (see Table \ref{tab:params}).


\begin{table*}[htp!]
\centering
\caption{Subtask 1: System performance and approach summary. DA columns represent: (1) training-time augmentation and (2) use of external or additional data. \checkmark and -- denote use and non-use, respectively.}
\label{tab:subtask1_results}
\small
\begin{tabularx}{\textwidth}{l c cc X c l}
\toprule
\textbf{Team} & \textbf{F1}  & \multicolumn{2}{c}{\textbf{DA}} & \textbf{LLM} & \textbf{Conspiracy Specificities} & \textbf{Reference} \\ 
& & \textit{1} & \textit{2} & & & \\
\midrule
HU & 0.26 & \checkmark & -- & -- & Marker descriptions w/ GLiNER & \cite{kashaf-etal-2026-hu} \\
CredenceAI & 0.24 & -- & -- & -- & Cross-marker attn., joint pred. & \cite{karan-2026-CredenceAI} \\
CCNU & 0.24 & \checkmark & -- & Qwen-2.5 & Joint marker prediction & \cite{wang-chen-2026-ccnu} \\
dangphuduy & 0.24 & -- & -- & -- & -- & \cite{dang-2026-dangphuduy} \\
UMUTeam & 0.24 & -- & -- & -- & -- & \cite{gomez-navalon-2026-umuteam} \\
UCSC NLP & 0.22 & -- & -- & -- & Joint marker prediction & \cite{marhoefer-etal-2026-ucsc} \\
NUST PsyAI & 0.21 & \checkmark & -- & -- & -- & \cite{akram-fatima-2026-nust} \\
AILS-NTUA & 0.21 & -- & -- & GPT-5.2, Claude & DD-CoT w/ counter-arguments & \cite{spanakis-etal-2026-ails} \\
YNU-HPCC & 0.21 & \checkmark & -- & -- & -- & \cite{chen-2026-YNU} \\
Team A & 0.19 & -- & -- & -- & -- & \cite{pan-2026-teama} \\
CuriosAI & 0.18 & -- & -- & Qwen3-14B & Multi-task token classification & \cite{yamaga-etal-2026-curiosai} \\
CUET\_320 & 0.18 & -- & -- & Gemma-3-4B  & Marker descriptions  & \cite{fariha-2026-CUET} \\
wangkongqiang & 0.15 & -- & -- & -- & -- & \cite{wang-tan-2026-wangkongqiang} \\
zhangpeng & 0.14 & -- & -- & -- & -- & \cite{zhang-lu-2026-zhangpeng} \\
Jia & 0.08 & -- & -- & Qwen2.5-7B,-14B & Joint generative \& discriminative & \cite{zhu-2026-jia} \\
AGAI & 0.07 & -- & -- & -- & -- & \cite{ranxiandong-2026-agai} \\
\bottomrule
\end{tabularx}
\end{table*}
\paragraph{Subtask 1: Sequence Labeling Baseline}For the marker extraction task, given that markers may overlap or appear with varying frequencies, we simplify the problem into a ``one-vs-rest'' binary token classification for each psychological marker type. For a sentence with $n$ tokens, the model produces a sequence of predictions $y = (y_1, y_2, \dots, y_n)$, where $y_i \in \{0, 1\}$. The model is trained using a standard Cross-Entropy loss calculated over the valid sub-token offsets, effectively identifying spans that align with the evolutionary markers defined in Table \ref{tab:makers}.

\paragraph{Subtask 2: Sequence Classification Baseline}
The binary conspiracy detection baseline model treats the comment as a single unit of analysis, mapping the final layer representation of the \texttt{[CLS]} token to a bipartite output space: $\mathcal{Y} = \{\text{Conspiracy, Non-Conspiracy}\}$. To ensure the model focuses on the structural and psycholinguistic invariants rather than topic-specific keywords, the training data is filtered to remove ``Can't Tell'' or ambiguous labels, focusing the optimization on the most prototypical conspiratorial narratives.





\subsection{Baseline Performance}
Model 1 achieves 0.15 F1 in the Conspiracy Marker Extraction subtask, whereas model 2 achieves 0.70 F1 in the Conspiracy Detection subtask. 

While the models effectively learn information about each task, they leave substantial room for model improvement. In particular, the marker extraction baseline model performs best on Actor ($F1=0.28$) and Victim ($F1=0.19$) (see Table \ref{tab:subtask1_baseline}). This suggests that the baseline is primarily picking up on named entities or referential pronouns that often characterize conspiratorial ``us vs. them'' narratives. The low performance on Effect ($0.09$) and Evidence ($0.10$) suggests that these markers are not merely lexical but require understanding causal superfluity or rhetorical structure, which a vanilla DistilBERT baseline lacks. With a Micro F1 of 0.1702, the task is clearly "hard", which is an open invitation for participants to use multi-task learning (leveraging Subtask 2) or reasoning-aware LLMs.

% Moreover, comments in broader discussion venues, especially those related to politics, news, and general conspiracy theories, appear harder to classify.

\begin{figure*}[tp!]
    
    \centering
    \includegraphics[width=\linewidth]{latex/system_marker_analysis.png}
    \caption{Per-marker system performance. Parallel coordinate trajectories for the top-performing systems are overlaid on the global distribution (represented by violin plots) of all submissions. }
    \label{fig:system_marker_analysis}
\end{figure*}

\subsection{Difficulty and Robustness Analysis}

\paragraph{Latent Difficulty Estimation} 
We model the probability of a system $i$ correctly classifying an instance $j$ from subreddit $k$ as a function of system ability $\theta_i$ and subreddit difficulty $\beta_k$:
$$ P(y_{ij} = 1) = \text{logit}^{-1}(\theta_i - \beta_k) $$
By estimating the posterior distribution of $\beta_k$, we identify "minefield" subreddits where high misclassification rates are statistically significant rather than being artifacts of sampling. This allows us to distinguish between systems that are globally robust and those that overfit to "easy" topics.

\paragraph{Significance Testing} 
Following best practices \cite{dror-etal-2018-hitchhikers}, we apply Approximate Randomization testing with $R=10,000$ iterations to verify that the performance deltas between the top-three teams and the DistilBERT baseline are statistically significant ($p < 0.05$).



% \section{Baselines and Preliminary Results}
% We developed baseline models for both subtasks. We fine-tuned distilbert-base models, model 1 adding a token classification head for the Conspiracy Marker Extraction Task, and model 2 adding a sequence classification head for the Conspiracy Detection task. Models were trained for 10 epochs.

% Model 1 achieves 0.2 F1 in the Conspiracy Marker Extraction subtask, whereas model 2 achieves 0.67 F1 in the Conspiracy Detection subtask. 

% While the models effectively learn information about each task, there is substantial room for model improvement. Baseline models identify ``actor'' markers twice as well as the others. Moreover, comments in broader discussion venues, especially those related to politics, news, and general conspiracy theories, appear harder to classify. %We look forward to the task participants' solutions to these challenges.

% \section{Task Organizers}
% \paragraph{Mattia Samory -- Lead Organizer} is an assistant professor at the Department of Computer Science at the Sapienza University of Rome. He co-organized the SemEval-2022 Task 8: Multilingual news article similarity \cite{chen-etal-2022-semeval}. 
% \paragraph{Felix Soldner -- Co-Organizer} is a postdoctoral researcher at the Computational Social Science department of GESIS Cologne. His research includes fraud and deception detection. 
% \paragraph{Veronika Batzdorfer -- Co-Organizer} is a postdoctoral researcher at the Department of Sociology and Computational Social Science at the Karlsruhe Institute of Technology. Her research includes socio‑psychological phenomena e.g., reinforcement loops of conspiracy beliefs.



\section{Task Results}
PsyCoMark attracted substantial interest from the NLP community, with participation numbers confirming the relevance and timeliness of the task. Across both subtasks, a combined total of 291 registered participants engaged with the competition platform, generating an aggregate of 3,370 submissions throughout the evaluation period (October 2025 -- January 2026).

\subsection{Subtask 1: Conspiracy Marker Extraction}


Subtask 1 drew \textbf{134 registered participants}, who collectively produced \textbf{977 submissions} across the development and test phases. The final test leaderboard on CodaBench comprised 30 teams, of which 16 are included in the official leaderboard, shown in Tab. \ref{tab:subtask1_results}. The top-performing system, submitted by team \textbf{HU}, achieved an F1 of \textbf{0.26}, followed by \textbf{CredenceAI} and \textbf{CCNU}, both at \textbf{F1~=~0.24}. The score distribution across the leaderboard (0.07--0.26) reflects the inherent difficulty of fine-grained conspiracy marker extraction, a task that requires nuanced linguistic understanding well beyond standard sequence labeling benchmarks. Notably, though, the top-performing models surpassed individual human annotators (F1~=~0.19). Figure \ref{fig:system_marker_analysis} shows that a challenge shared by all systems was maintaining consistent performance across marker types. While most systems cluster around the task mean, the highlighted systems demonstrate consistent performance across most narrative markers. Notably, the extraction of the NER-style markers "Actor" and "Victim" shows both the highest performance and the highest variance, suggesting these categories were the primary differentiators in system ranking, whereas "Evidence" and "Effect" remained a bottleneck for even the highest-ranked models.

Most systems used pretrained transformer encoders fine-tuned for token-level sequence labeling, typically with BIO tagging to identify marker roles. Stronger submissions extended this approach with explicit span-aware modeling, including boundary-aware classifiers, span-consistency mechanisms, and structural constraints that encourage coherent contiguous spans and better alignment with the IoU-based evaluation metric. Several teams also explored joint modeling of multiple marker types, allowing the model to capture dependencies between roles within a unified representation. For example, team HU utilized GLiNER 2.0 to match text spans against semantically descriptive prompts for markers; CredenceAI incorporated a cross-marker attention module to model interdependencies between marker types; CCNU predicted markers on a joint BIO tagging space to capture cross-role contextual dependencies. Ensemble strategies and cross-validation averaging were common techniques for improving robustness. Overall, the most promising patterns combine transformer encoders with explicit span modeling, joint role prediction, and robust inference strategies, suggesting that architectures that capture both span structure and cross-role dependencies are particularly well suited for conspiracy marker extraction. This attention to the specifics of conspiracy narratives results in significantly higher performance than the baseline, and we will see next, will prove essential also in the detection subtask.

\subsection{Subtask 2: Conspiracy Detection}
Subtask 2 attracted \textbf{157 registered participants} and a substantially higher submission volume of \textbf{2,393 entries}, indicating sustained and active engagement throughout the competition. The final test leaderboard on CodaBench comprised 52 teams, of which 28 qualified for the official leaderboard. Performance on this binary classification subtask was considerably higher than on Subtask~1, with the leading team, \textbf{NJUST\_KMG}, achieving an F1 of \textbf{0.89}. The second and third positions were held by \textbf{mdok-style} and \textbf{dangphuduy} (F1~=~\textbf{0.78}), respectively. The top of the leaderboard was densely populated, with most teams scoring F1~$\geq$~0.74, suggesting that the binary detection framing, while challenging, proved more tractable for participating systems than the extraction task.

\begin{table*}[t!]
\centering
\caption{Subtask 2: System performance and approach summary. DA columns represent: (1) training-time augmentation and (2) pre-trained/external data fine-tuning. \checkmark and -- denote use and non-use, respectively.}
\label{tab:subtask2_results}
\small
\begin{tabularx}{\textwidth}{l c cc l c l}
\toprule
\textbf{Team} & \textbf{F1}  & \multicolumn{2}{c}{\textbf{DA}} & \textbf{LLM} & \textbf{Conspiracy Specificities} & \textbf{Reference} \\ 
& & \textit{1} & \textit{2} & & & \\
\midrule
NJUST\_KMG & 0.89  & \checkmark & \checkmark & GPT-5.2 & Keywords, subreddit & \cite{zheng-yang-2026-njustkmg} \\
mdok-style & 0.78  & \checkmark & \checkmark & Qwen3-32B & -- & \cite{macko-2026-mdok} \\
dangphuduy & 0.78  & \checkmark & \checkmark & GPT-OSS-120B & Sentiment analysis & \cite{dang-2026-dangphuduy} \\
VARH-AI & 0.78  & \checkmark & -- & -- & Task 1 markers as input & \cite{solanki-etal-2026-varh} \\
UMUTeam & 0.77  & -- & -- & -- & -- & \cite{gomez-navalon-2026-umuteam} \\
HU & 0.76  & \checkmark & -- & -- & Marker descriptions, PAN24 & \cite{kashaf-etal-2026-hu} \\
CuriosAI & 0.76  & \checkmark & \checkmark & GPT-OSS, Qwen3 & Empath/VAD features & \cite{yamaga-etal-2026-curiosai} \\
NUST PsyAI & 0.76  & \checkmark & -- & -- & LIWC, NRC Lexicons & \cite{akram-fatima-2026-nust} \\
CSECU-DSG & 0.75  & -- & -- & -- & Sentiment analysis & \cite{chakraborty-etal-2026-csecu} \\
LATE-iimas & 0.75  & -- & -- & -- & -- & \cite{vazquez-cerrillo-etal-2026-late} \\
davidinfotec & 0.75  & -- & -- & -- & -- & \cite{rodriguez-guierrez-2026-davidinfotec} \\
psy\_detectives & 0.75  & -- & \checkmark & TinyLlama-1.1B & LIWC & \cite{carabas-etal-2026-psydetectives} \\
AILS-NTUA & 0.75  & -- & \checkmark & GPT-5, Claude & Topic/Stance separation & \cite{spanakis-etal-2026-ails} \\
TruthGradient & 0.75  & -- & -- & -- & -- & \cite{goyal-2026-truthgradient} \\
wangkongqiang & 0.74  & \checkmark & \checkmark & Gemma-3-27B & -- & \cite{wang-tan-2026-wangkongqiang} \\
YNU-HPCC & 0.74  & \checkmark & -- & -- & -- & \cite{chen-2026-YNU} \\
Unibuc-NLP & 0.74  & -- & \checkmark & Llama-3.1, Mistral & LIWC & \cite{marchitan-2026-unibuc} \\
Hidetsune & 0.73  & \checkmark & \checkmark & -- & -- & \cite{takahashi-2026-hidetsune} \\
Team A & 0.73  & -- & -- & -- & -- & \cite{pan-2026-teama} \\
CCNU & 0.73  & \checkmark & \checkmark & Qwen-2.5 & Multitask marker extraction & \cite{wang-chen-2026-ccnu} \\
UCSC NLP & 0.73  & -- & -- & -- & -- & \cite{marhoefer-etal-2026-ucsc} \\
zhangpeng & 0.73  & -- & -- & -- & -- & \cite{zhang-lu-2026-zhangpeng} \\
Macaroni & 0.73  & -- & -- & -- & Stylistic cues & \cite{rabehi-etal-2026-macaroni} \\
TTLab & 0.72  & -- & \checkmark & Qwen2.5, Llama-3.2 & -- & \cite{richter-marreddy-2026-ttlab} \\
AGAI & 0.70  & \checkmark & \checkmark & Qwen3-14B & -- & \cite{ranxiandong-2026-agai} \\
CUET\_320 & 0.59  & -- & -- & Gemma-3-4B & Task 1 markers as input & \cite{fariha-2026-CUET} \\
Jia & 0.37  & -- & \checkmark & Qwen2.5-Instruct & Multitask marker extraction & \cite{zhu-2026-jia} \\
GUNLP & 0.34  & -- & \checkmark & GPT-5, Claude & Sentiment analysis & \cite{ziaei-etal-2026-GUNLP} \\
\bottomrule
\end{tabularx}
\end{table*}

Several trends emerge from comparing approaches: LLM usage, data augmentation, and explicit modeling of conspiracy narratives over general classification. First, transformer-based model backbones are almost ubiquitous, and transformer fine-tuning still outperforms pure prompting setups. Yet, most top performers integrate LLMs (often GPT/Qwen-style) in hybrid architectures. Furthermore, data augmentation correlates positively with higher performance, and is especially adopted by systems in the top quartile in the form of training-time augmentation. Pre-training or finetuning on external data, on the other hand, does not appear to provide clear performance benefits. Finally, explicit consideration of the peculiarities of conspiracy narratives, while pursued by many participants, is more common among top-quartile systems. The approaches, though, vary substantially: from identifying known conspiracy topics to separating them from stances toward conspiracy theories (e.g., to avoid the ``reporter trap'', in which the model incorrectly classifies objective reporting on a conspiracy theory as endorsing it), to incorporating emotional cues, and to account for ambiguity. 

A recipe for success, therefore, emerges: transformer fine-tuning + LLM component + data augmentation + marker-aware modeling. These trends are embodied by the top-performing system, which combines a transformer backbone, an LLM-based filter for comments not pertaining to conspiracy keywords, data augmentation, and explicit conspiratorial cue modeling. 
More specifically, \cite{zheng-yang-2026-njustkmg} use retrieval augmentation to update the prompt, which is then fed into the model for fine-tuning. First, samples are divided between those containing and not containing relevant keywords --- unrelated topics (health, art, lifestyle) are directly classified as "No". A weighted classification is performed on the remaining conspiracy theory-related content, together with the subreddit tags obtained from the query.




\begin{figure*}[tp!]
    
    \centering
    \includegraphics[width=\linewidth]{latex/domain_difficulty_minimal_leaderboard.png}
    \caption{Top and bottom 10 most difficult-to-predict subreddits (positive $\beta_s \rightarrow$ difficult), disaggregated into "conspiracy" (left panel) and "mainstream" (right panel) based on whether they contain more "Yes" than "No" labels. 
}
    \label{fig:domain_difficulty_minimal_leaderboard}
\end{figure*}


\paragraph{Subreddit Difficulty}
Across all submissions, patterns emerge about which discussion communities are harder to predict correctly (Fig. \ref{fig:domain_difficulty_minimal_leaderboard}; to ease interpretation, we divide subreddits based on whether they contained more comments labeled as "Yes" than "No", and refer to using the shorthands of "conspiracy" and "mainstream", respectively.  

Mainstream subreddits are substantially easier to predict than conspiracy subreddits. Easy-to-predict mainstream subreddits have broader appeal than corresponding easy-to-predict conspiracy subreddits (\texttt{Health} vs. \texttt{AntiVaxxers}, \texttt{EverythingScience} vs. \texttt{ScienceUncensored}, \texttt{NEWPOLITIC} vs. \texttt{NewPatriotism}). Conversely, easy-to-predict conspiracy subreddits position themselves as alternatives to a mainstream (e.g., \texttt{outsideofthebox}, \texttt{awfuleverything}, \texttt{CoronavirusConspiracy}). This suggests that mainstream \textit{positioning} may be a telltale sign of both conspiracy and non-conspiracy narratives.  

Subreddits that explicitly elicit a diversity of viewpoints are among the hardest-to-predict, like \texttt{TheMotte}, \texttt{Foodforthought} (conspiracy), \texttt{TruthSeekers}, and \texttt{Israel\_Palestine} (mainstream). The same is true for inherently contentious topics, such as political figures (\texttt{WayOfTheBern} and \texttt{samharris}), economics (\texttt{Wallstreetsilver} and \texttt{LateStageCapitalism}), and esoteric topics (\texttt{ufo} and \texttt{TheSaturnTimeCube}). It appears that, more than the topic, what drives difficulty is a polarizing interaction space, which may lead to the unsuspected presence or absence of conspiracy theories.  



\section{Related Work}
Computational studies of online conspiracy theories often resort to distant-labeled data, such as, by reference to the platform or community of origin (e.g., \citealt{samory2018government,10.1177/1745691618774270,10.1371/journal.pone.0233879,10.1007/s11280-021-00862-x}). Recent research contributed annotated datasets (e.g., \citealt{faddoul2020longitudinalanalysisyoutubespromotion,Liaw_Huang_Benevenuto_Kwak_An_2023,10.48550/arxiv.2310.18545}), the majority focusing on content from Twitter and on conspiracy theories related to COVID-19 (e.g., \citealt{10.1145/3487553.3524665,doi:10.1177/20563051211043212,GAMBINI202425,10.1111/exsy.13671,batzdorfer2022conspiracy,10.1007/s42001-020-00086-5,10.1007/s42001-023-00200-3}). Few datasets include conspiracy theories without topical restrictions. Most relevant to our work, \citeauthor{10.1111/exsy.13671} present a span-based annotation schema to differentiate between conspiratorial and critical narratives \cite{10.1111/exsy.13671}. Our dataset takes stock of past work and extends it by 1) refining the annotation schema by grounding it in evolutionary psychology and 2) diversifying and expanding the topical focus of the data to a wide range of conversations.

% This proposal complements SemEval's prior tasks related to the moderation of online content, such as toxic span detection \cite{pavlopoulosSemEval2021Task52021}, sexism detection \cite{kirkSemEval2023Task102023}, misogyny detection \cite{fersiniSemEval2022Task52022}, and general offensive language detection \cite{meaneySemEval2021Task2021}. Previous tasks shared our goal of tackling misinformation through fact-checking \cite{mihaylovaSemEval2019Task82019}, or aimed to detect persuasive language \cite{dimitrovSemEval2021Task62021,piskorskiSemEval2023Task32023}.



\section{Ethics}
The raw data in this study were retrieved from publicly accessible archives. Although exempt, we obtained IRB approval from GESIS Cologne for the annotation task. Data is released without personal identifiers and raw comment text to preserve privacy, the right to erasure, and data minimization. We provide a script to rehydrate the comment text.\footnote{\url{https://github.com/hide-ous/semeval26_starter_pack}}  

% \cite{samory-etal-2026-semeval}
% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only

\section{Conclusions} The PsyCoMark task was particularly aimed at NLP researchers focused on explainability, misinformation detection, and fact-checking, as well as researchers on content moderators seeking granular interventions for conspiracy-related content. 
The high number of participants and submissions across both subtasks demonstrates strong community interest in computational approaches to conspiracy theory analysis and underscores the value of PsyCoMark as a shared evaluation framework for this emerging research area.

The contrast in difficulty between the two subtasks is clearly reflected in the performance distributions: Subtask 1 (extraction) presented a harder challenge with scores clustered in a lower range. Participants showed how accurately modeling the characteristics of conspiracy narratives yields significant gains over baselines and may achieve impressive performances. Much information, however, remains untapped. The participants explored a variety of approaches that operate at distinct lexical, semantic, local-distributional, and meta-textual levels: a principled integration is likely an avenue for further improvement. In particular, a minority of participants successfully leveraged the extracted conspiracy markers, despite their wealth of information and their strong association with conspiracy narratives found in our data exploration; further integration of the two tasks is expected to further boost systems' overall performance.

\bibliography{custom}

\appendix


\end{document}
