% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}

%CUSTOM PACKAGES
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{listings}
\usepackage{xcolor}
\usepackage{nameref}
\usepackage{natbib}
\colorlet{punct}{red!60!black}
\definecolor{background}{HTML}{EEEEEE}
\definecolor{delim}{RGB}{20,105,176}
\colorlet{numb}{magenta!60!black}

\lstdefinelanguage{json}{
    basicstyle=\fontsize{9}{11}\ttfamily,
    numbers=left,
    numberstyle=\scriptsize,
    stepnumber=1,
    numbersep=4pt,
    showstringspaces=false,
    breaklines=true,
    frame=lines,
    backgroundcolor=\color{background},
    literate=
     *{0}{{{\color{numb}0}}}{1}
      {1}{{{\color{numb}1}}}{1}
      {2}{{{\color{numb}2}}}{1}
      {3}{{{\color{numb}3}}}{1}
      {4}{{{\color{numb}4}}}{1}
      {5}{{{\color{numb}5}}}{1}
      {6}{{{\color{numb}6}}}{1}
      {7}{{{\color{numb}7}}}{1}
      {8}{{{\color{numb}8}}}{1}
      {9}{{{\color{numb}9}}}{1}
      {:}{{{\color{punct}{:}}}}{1}
      {,}{{{\color{punct}{,}}}}{1}
      {\{}{{{\color{delim}{\{}}}}{1}
      {\}}{{{\color{delim}{\}}}}}{1}
      {[}{{{\color{delim}{[}}}}{1}
      {]}{{{\color{delim}{]}}}}{1},
}



% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{PsyCoMark -- Psycholinguistic Conspiracy Marker Extraction and Detection}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Mattia Samory \\
  Sapienza University of Rome \\
  \small\texttt{mattia.samory@uniroma1.it} \\\And
  Felix Soldner \\
  GESIS Cologne\\
  \small\texttt{felix.soldner@gesis.org} \\\And
  Veronika Batzdorfer \\
  Karlsruhe Institute of Technology \\
  \small\texttt{veronika.batzdorfer@kit.edu } 
  }

%\author{
%  \textbf{First Author\textsuperscript{1}},
%  \textbf{Second Author\textsuperscript{1,2}},
%  \textbf{Third T. Author\textsuperscript{1}},
%  \textbf{Fourth Author\textsuperscript{1}},
%\\
%  \textbf{Fifth Author\textsuperscript{1,2}},
%  \textbf{Sixth Author\textsuperscript{1}},
%  \textbf{Seventh Author\textsuperscript{1}},
%  \textbf{Eighth Author \textsuperscript{1,2,3,4}},
%\\
%  \textbf{Ninth Author\textsuperscript{1}},
%  \textbf{Tenth Author\textsuperscript{1}},
%  \textbf{Eleventh E. Author\textsuperscript{1,2,3,4,5}},
%  \textbf{Twelfth Author\textsuperscript{1}},
%\\
%  \textbf{Thirteenth Author\textsuperscript{3}},
%  \textbf{Fourteenth F. Author\textsuperscript{2,4}},
%  \textbf{Fifteenth Author\textsuperscript{1}},
%  \textbf{Sixteenth Author\textsuperscript{1}},
%\\
%  \textbf{Seventeenth S. Author\textsuperscript{4,5}},
%  \textbf{Eighteenth Author\textsuperscript{3,4}},
%  \textbf{Nineteenth N. Author\textsuperscript{2,5}},
%  \textbf{Twentieth Author\textsuperscript{1}}
%\\
%\\
%  \textsuperscript{1}Affiliation 1,
%  \textsuperscript{2}Affiliation 2,
%  \textsuperscript{3}Affiliation 3,
%  \textsuperscript{4}Affiliation 4,
%  \textsuperscript{5}Affiliation 5
%\\
%  \small{
%    \textbf{Correspondence:} \href{mailto:email@domain}{email@domain}
%  }
%}

\begin{document}
\maketitle
\begin{abstract}
Despite the need to address the proliferation of conspiracy theories in online discussions, there is a lack of benchmarks for effectively detecting conspiracy-related content in everyday conversational settings. Informed by evolutionary psychology, we introduce a novel dataset and span identification task to uncover psycholinguistics markers of conspiracy theories in online discussions. Markers are expected to improve performance in conspiracy detection: we will put this to the test in a parallel task using comments from everyday discussions on Reddit ranging from politics to TV series. 
\end{abstract}

\section{Introduction}
% This task introduces novel aspects to the computational analysis of conspiracy theory language. 
Most related NLP work aiming to detect conspiracy theories focuses on detecting specific claims, such as the belief that vaccines cause autism \cite{mitra2016understanding}. However, conspiracy theories may adapt to a variety of topical contexts, complicating their detection in realistic settings \cite{samory2018government}. Yet, psychology research points to invariants in the structure of conspiracy theory narratives due to the functions that they serve in group life, which are independent of the specific claims of each theory \cite{10.1177/1745691618774270,10.3389/fpsyg.2013.00406}. We propose extracting the markers of conspiracy theory narratives and detecting conspiracy theories as two synergistic tasks. We introduce two novel aspects to the computational analysis of conspiracy theory language: psychologically grounded markers and topic-agnostic conversations. 

% We introduce span-level markers that are expressly informed by the evolutionary psychology of conspiracy theories, and a topic-agnostic dataset of Reddit comments annotated for such markers.

\paragraph{Psychologically-Grounded Markers} Evolutionary psychology literature posits that conspiracy beliefs evolved as adaptive mechanisms for detecting dangerous coalitions \cite{10.1177/1745691618774270}. To shield oneself and one's in-group from threats, conspiracy thinking trains the individual to anticipate the secret scheming of malicious opponents. We introduce a novel dataset that maps the psychological mechanisms of conspiracy beliefs to linguistic markers in conspiracy theory discussions. Table \ref{tab:makers} summarizes the relationship between psychological mechanisms and linguistic markers in our annotation schema. This grounding ensures that the annotations capture the underlying psychological drivers of conspiracy thinking, affording deeper insights and explainability. 

\paragraph{Topic-Agnostic Conversations} Existing corpora focus on pre-determined conspiracy topics.\footnote{There is criticism about the influence of topicality even in the psychological measurement of conspiracy thinking.} Few resources, like the LOCO dataset \cite{miani2021loco}, encompass a broad range of topics but do not include conspiracy theories as they are discussed on social media. Our dataset offers a contextually rich resource for analysis across a broad range of topics in everyday social media conversations---including subreddits both overtly about conspiracy theories (e.g., \texttt{conspiracy}, \texttt{ufobelievers}, \texttt{epstein}) and of general interest (e.g., \texttt{hongkong}, \texttt{truecrime}, \texttt{news}).

\section{Task Structure}

By combining detection and marker extraction, this proposal encourages the development of generalizable and interpretable models of conspiracy theory language. To this end, participants will be allowed to leverage external data and pre-trained models.


\paragraph{Subtask 1: Conspiracy Marker Extraction}

Participants will develop models to extract the spans indicative of conspiracy theory markers. Each document may contain zero or multiple spans corresponding to each marker, which may overlap with each other. We will evaluate the models' ability to detect each marker using an overlap-based F1-score. The overall performance for the task will be the macro-average of the per-marker F1 scores.

\paragraph{Subtask 2: Conspiracy Detection}Participants will develop models to classify comments as either conspiracy-related or non-conspiracy-related. Participants will be encouraged but not required to contribute end-to-end systems that address both tasks jointly, to remove barriers to participation. We will evaluate performance with macro-averaged F1-score, restricting the test set to instances classified as ``conspiracy'' or ``not conspiracy''.

\paragraph{Audience} The tasks are particularly aimed at NLP researchers focused on explainability, misinformation detection, and fact-checking, as well as researchers on content moderators seeking granular interventions for conspiracy-related content.

\section{Data}

% Please add the following required packages to your document preamble:
% \usepackage{booktabs}
% \usepackage{multirow}
\begin{table}[]
\begin{tabular}{@{}lll@{}}
\toprule
\textbf{Psychological driver}& \textbf{Marker}& \textbf{\# comments} \\ \midrule
Alliance detection& Actor& 1276                 \\
\multirow{2}{*}{Threat management}& Effect& 1009                 \\
                                   & Victim& 929                  \\
Pattern perception& Evidence& 989                  \\
Agency detection& Action& 1239                 \\ \bottomrule
\end{tabular}
\caption{Relationship between the psychological mechanisms underpinning conspiracy beliefs and the linguistic markers used in our annotation schema.\label{tab:makers}}

\end{table}


\paragraph{Submission Statements}
To include texts that cover a wide variety of topics and include rich narrative information, we focused our data on submission statements on Reddit. Submission statements stem from the practice of accompanying submissions that contain only media---like a picture, video, or URL---with a comment describing why the media relates to a subreddit's topic. For example, the submission statement accompanying a newspaper article in the subreddit \texttt{conspiracy} may summarize how the article supports a conspiracy theory, whereas in the subreddit \texttt{law} could connect news events to a broader legal discourse. Since submission statements encode users' summaries of the narrative in media, they represent an ideal format from which to extract linguistic markers of conspiracy theories. Submission statements are common practice in a wide range of subreddits, which encompass a broad topical range. 

\paragraph{Sampling}
We selected first-level comments, authored by the discussion starter, that began with \verb:ss|((submission )?statement)[^a-zA-Z\d]:, which is the standard format for submission statements. We manually excluded subreddits where submission statements were used for alternative purposes. We converted markdown to text and converted URLs to special tokens. We discarded comments containing quotes to limit to text by the comment's author. To balance content richness with the feasibility of annotation, we retained comments between 160 and 1000 characters. Since conspiracy theories are relatively rare across Reddit, we oversampled comments that likely contain them: we sampled 1150 comments uniformly at random from the subreddit \texttt{conspiracy} and 1150 comments from the remaining subreddits with at least 10 submission statements (6 comments each). 

\paragraph{Annotation}
We annotated the data through the crowdwork platform Prolific. Annotators were selected from the U.S., U.K., Canada, and Australia, and their first language was English to ensure that participants were native speakers. The annotation task comprised 1) a consent form, 2) a pre-survey about conspiracy thinking and related psychological constructs, 3) task instructions and codebook description, 4) a training task, 5) annotation of 10 documents for markers and three-way classification (conspiracy, not conspiracy, can't tell), and 6) a debriefing page. Annotators were compensated with £9/h on average. To ensure quality, whenever an annotator did not annotate more than two markers in at least one document of the batch, we discarded the entire batch from the final data.

\paragraph{Statistics} 
The annotated dataset presently consists of 1646 summary statements; we are expanding data annotation and expect around 4000 comments to be in the final dataset, but the task would be feasible with the already collected data, which we describe next. Classes are well balanced: 630 were annotated as ``conspiracy'', 655  ``not conspiracy'', and 361 ``can't tell''. Krippendorff's $\alpha=0.43$ for the binary conspiracy labels. Data is also densely annotated for conspiracy markers: 1407 contain at least one marker, and 563 contain all five. The distribution of markers per document is the following: 1276 contain at least one actor marker, 1239 action, 1009 effect, 989 evidence, and 929 victim. Data spans 191 subreddits, with 781 comments coming from the \texttt{conspiracy} subreddit. 

\paragraph{Sharing}
The dataset is publicly and permanently available through Zenodo\footnote{\url{https://dx.doi.org/10.5281/zenodo.15114172}} under a CC-BY license. A random sample of 20\% of the data was retained as a private test set for both subtasks. Listing \ref{lst:example} showcases an example data point.



\section{Baselines and Preliminary Results}
We developed baseline models for both subtasks. We fine-tuned distilbert-base models, model 1 adding a token classification head for the Conspiracy Marker Extraction Task, and model 2 adding a sequence classification head for the Conspiracy Detection task. We further modified this latter model into model 3, encoding conspiracy markers with special tokens to explore their contribution to the detection task. Models were trained for 10 epochs.

Model 1 achieves 0.2 F1 in the Conspiracy Marker Extraction subtask, whereas models 2 and 3 achieve 0.63 and 0.67 F1 respectively in the Conspiracy Detection subtask. Thus, the models effectively learn information about each task and can exploit markers for detecting conspiracy theories. 

Yet, there is substantial room for model improvement. Baseline models identify ``actor'' markers twice as well as the others. Moreover, comments in broader discussion venues, especially those related to politics, news, and general conspiracy theories, appear harder to classify. We look forward to the task participants' solutions to these challenges.

\section{Task Organizers}
\paragraph{Mattia Samory -- Lead Organizer} is an assistant professor at the Department of Computer Science at the Sapienza University of Rome. He co-organized the SemEval-2022 Task 8: Multilingual news article similarity \cite{chen-etal-2022-semeval}. 
\paragraph{Felix Soldner -- Co-Organizer} is a postdoctoral researcher at the Computational Social Science department of GESIS Cologne. His research includes fraud and deception detection. 
\paragraph{Veronika Batzdorfer -- Co-Organizer} is a postdoctoral researcher at the Department of Sociology and Computational Social Science at the Karlsruhe Institute of Technology. Her research includes socio‑psychological phenomena e.g., reinforcement loops of conspiracy beliefs.


\section{Related Work}
Computational studies of online conspiracy theories often resort to distant-labeled data, such as, by reference to the platform or community of origin (e.g., \citealt{samory2018government,10.1177/1745691618774270,10.1371/journal.pone.0233879,10.1007/s11280-021-00862-x}). Recent research contributed annotated datasets (e.g., \citealt{faddoul2020longitudinalanalysisyoutubespromotion,Liaw_Huang_Benevenuto_Kwak_An_2023,10.48550/arxiv.2310.18545}), the majority focusing on content from Twitter and on conspiracy theories related to COVID-19 (e.g., \citealt{10.1145/3487553.3524665,doi:10.1177/20563051211043212,GAMBINI202425,10.1111/exsy.13671,batzdorfer2022conspiracy,10.1007/s42001-020-00086-5,10.1007/s42001-023-00200-3}). Few datasets include conspiracy theories without topical restrictions. Most relevant to our work, \citeauthor{10.1111/exsy.13671} present a span-based annotation schema to differentiate between conspiratorial and critical narratives. Our dataset takes stock of past work and extends it by 1) refining the annotation schema by grounding it into evolutionary psychology and 2) diversifying and expanding the topical focus of the data to a wide range of conversations on Reddit.

This proposal complements SemEval's prior tasks related to the moderation of online content, such as toxic span detection \cite{pavlopoulosSemEval2021Task52021}, sexism detection \cite{kirkSemEval2023Task102023}, misogyny detection \cite{fersiniSemEval2022Task52022}, and general offensive language detection \cite{meaneySemEval2021Task2021}. Previous tasks shared our goal of tackling misinformation through fact-checking \cite{mihaylovaSemEval2019Task82019}, or aimed to detect persuasive language \cite{dimitrovSemEval2021Task62021,piskorskiSemEval2023Task32023}.




\section{Ethics}
The raw data in this study was retrieved from publicly accessible archives. Although exempt, we obtained IRB approval from GESIS Cologne for the annotation task. Data is released without personal identifiers and raw comment text, to preserve privacy, the right to erasure, and data minimization. We provide a script to rehydrate the comment text.\footnote{\url{https://github.com/hide-ous/semeval26_starter_pack}}  

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only

\begin{lstlisting}[language=json,firstnumber=1,label={lst:example}, caption=Example data point]
{
    "_id": "t1_xxxyyyzzz",
    "conspiracy": "Yes",
    "markers": [{
            "type": "Actor",
            "text": "CNN",
            "startIndex": 11, 
            "endIndex": 13 
        }//, ...
    ],
    "subreddit": "conspiracy",
    "annotator": "annotator_1"
}
\end{lstlisting}

\bibliography{custom}

\appendix


\end{document}
