% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\usepackage[english]{babel}
\usepackage{algpseudocode}
\usepackage{algorithm}
\usepackage{tabularx}
\usepackage{amsmath}
\usepackage{float}
\usepackage{pifont}
\usepackage{amssymb}
\usepackage{xurl}
\usepackage{booktabs}
% \usepackage[square,numbers]{natbib}
% \bibliographystyle{abbrvnat}
% \bibliographystyle{unsrtnat}
\usepackage{tikz}
\usetikzlibrary{shapes,arrows,positioning,fit,backgrounds,calc,3d}
\usepackage{multirow}
% \usepackage{flushend}  % Balances columns on the last page
% \usepackage{geometry}  % Adjust margins if needed

% % After your template packages
% \usepackage[breakable]{tcolorbox}
\usepackage{geometry}
\usepackage[skins,breakable,theorems]{tcolorbox}
\usepackage{fontawesome5}

\usepackage{subcaption}

% After your template packages
\usepackage{xcolor}

\newcounter{mybox}
% Create a counter for our boxes
\newenvironment{simplebox}[2][]{%  % #1 is caption (optional), #2 is label
    \refstepcounter{mybox}%
    \begin{tcolorbox}[
        colback=white,
        colframe=black!30,
        width=\columnwidth,
        arc=3mm,
        title={Prompt \themybox: #1},
        fonttitle=\bfseries,
        breakable,
        break at=20cm,
        before={\par\noindent},
        after={\par},
        % Break styling
        pad at break=2mm,
        % Consistent spacing
        top=3mm,
        bottom=3mm,
        left=3mm,
        right=3mm
    ]
    \label{box:#2}%
}{%
    \end{tcolorbox}
}

% For conversation environment, if needed
\definecolor{humanColor}{RGB}{70,70,70}
\definecolor{assistantColor}{RGB}{25,95,140}
\definecolor{systemColor}{RGB}{100,60,100}


\newcommand{\human}[1]{%
    {\color{humanColor}\faUser} \textbf{Human:}\\
    #1\par\medskip
}

\newcommand{\assistant}[1]{%
    {\color{assistantColor}\faRobot} \textbf{Assistant:}\\
    #1\par\medskip
}

\newcommand{\system}[1]{%
    {\color{systemColor}\faCog} \textbf{System:}\\
    #1\par\medskip
}

% \setlength{\belowcaptionskip}{-11pt}

% \usepackage{titlesec}
% \titlespacing\section{0pt}{8pt plus 4pt minus 2pt}{6pt plus 2pt minus 2pt}
% \titlespacing\subsection{0pt}{4pt plus 4pt minus 2pt}{2pt plus 2pt minus 2pt}

\title{Corpus-Oriented Stance Target Extraction}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}


\author{
  Benjamin D. Steel\\
  McGill University\\
  \texttt{benjamin.steel@mail.mcgill.ca}
  \\\And
  Derek Ruths\\
  McGill University\\
  \texttt{derek.ruths@mcgill.ca}
}

%\author{
%  \textbf{First Author\textsuperscript{1}},
%  \textbf{Second Author\textsuperscript{1,2}},
%  \textbf{Third T. Author\textsuperscript{1}},
%  \textbf{Fourth Author\textsuperscript{1}},
%\\
%  \textbf{Fifth Author\textsuperscript{1,2}},
%  \textbf{Sixth Author\textsuperscript{1}},
%  \textbf{Seventh Author\textsuperscript{1}},
%  \textbf{Eighth Author \textsuperscript{1,2,3,4}},
%\\
%  \textbf{Ninth Author\textsuperscript{1}},
%  \textbf{Tenth Author\textsuperscript{1}},
%  \textbf{Eleventh E. Author\textsuperscript{1,2,3,4,5}},
%  \textbf{Twelfth Author\textsuperscript{1}},
%\\
%  \textbf{Thirteenth Author\textsuperscript{3}},
%  \textbf{Fourteenth F. Author\textsuperscript{2,4}},
%  \textbf{Fifteenth Author\textsuperscript{1}},
%  \textbf{Sixteenth Author\textsuperscript{1}},
%\\
%  \textbf{Seventeenth S. Author\textsuperscript{4,5}},
%  \textbf{Eighteenth Author\textsuperscript{3,4}},
%  \textbf{Nineteenth N. Author\textsuperscript{2,5}},
%  \textbf{Twentieth Author\textsuperscript{1}}
%\\
%\\
%  \textsuperscript{1}Affiliation 1,
%  \textsuperscript{2}Affiliation 2,
%  \textsuperscript{3}Affiliation 3,
%  \textsuperscript{4}Affiliation 4,
%  \textsuperscript{5}Affiliation 5
%\\
%  \small{
%    \textbf{Correspondence:} \href{mailto:email@domain}{email@domain}
%  }
%}

\begin{document}
\maketitle
\begin{abstract}

Understanding public discourse through the frame of stance detection requires effective extraction of issues of discussion, or stance targets. Yet current approaches to stance target extraction are limited, only focusing on a single document to single stance target mapping. We propose a broader view of stance target extraction, which we call corpus-oriented stance target extraction. This approach considers that documents have multiple stance targets, those stance targets are hierarchical in nature, and document stance targets should not be considered in isolation of other documents in a corpus. We develop a formalization and metrics for this task, propose a new method to address this task, and show its improvement over previous methods using supervised and unsupervised metrics, and human evaluation tasks. Finally, we demonstrate its utility in a case study, showcasing its ability to aid in reliably surfacing key issues of discussion in large-scale corpuses.
\end{abstract}

\vspace{-2mm}
\section{Introduction}
\label{sec:intro}
\vspace{-1mm}

Disagreement is a critical part of discourse. As such, understanding discourse requires inferring the constituent disagreements. This task becomes increasingly complex as discussions scale to online environments \cite{gottfried2024americans}, where the pressing need to ensure healthy dialogue is compounded by threats from inauthentic influence attempts and harmful platform mechanisms  \cite{saurwein2021automated,goldstein2023generative,european2024commission}. To address these challenges, we need both easy-to-use analytical methods and clear representations of discussion data. However, developing such tools presents challenges, given that online media documents typically mix many different related issues, topics, and contexts.

Stance detection (i.e. the task of identifying the attitude of the author of a text on a stance target (a claim, entity etc.) \cite{mohammad2016semeval}) is a well-developed method for understanding disagreement. But the current state of stance detection is such that, unless one knows a priori the stance targets one wants to know the documents' stance on, one must undertake the difficult task of defining those stance targets oneself via the arduous task of understanding the entire corpus. While there are initial methods available for finding targets in documents, we propose that they are insufficient at the corpus-level, and that such a method needs four key features in order to faithfully and clearly capture stance in a discussion corpus:
\vspace{-2mm}
\begin{enumerate}
    \item The stance targets need not be known a priori to the researcher - avoiding human bias in issue selection, and improving scalability. \label{itm:not-known}
    \vspace{-2mm}
    \item A single document can articulate a position on multiple, or hierarchical (i.e. more abstract, or more general), targets - which frequently occurs in the real-world - and as such, the method should map the document to these targets.  \label{itm:multiple-targets}
    \vspace{-2mm}
    \item Targets should be determined in the context of the corpus - meaning both that the discussion as a whole aids the inference of the targets of a document, and that documents should be clustered to targets to allow aggregation for downstream application. \label{itm:contextualized-targets}
    \vspace{-2mm}
    \item Documents should be mapped to clear representations of these stance targets, to aid understanding, and allow use in downstream tasks \label{itm:document-target-map}
    \vspace{-2mm}
\end{enumerate}
\vspace{-2mm}

\begin{figure*}[h]
\begin{tikzpicture}[
    topic/.style={draw, ellipse, minimum width=1.2cm, minimum height=0.5cm, align=center},
    statement/.style={align=center, text width=8cm},
    cluster/.style={draw, ellipse, minimum width=1.2cm, minimum height=0.5cm, align=center},
    textbox/.style={draw, rounded corners=10pt, align=center, text width=6cm,font=\small}
]

% Left side - Topics
\node[topic] (s_ct) at (-5.5, 0.65) {carbon tax};
\node[topic] (s_t) at (-5.5, -0.65) {tariffs};

% Center - Text boxes (stacked vertically)
\node[textbox] (ct_text) at (-0.8, 0.65) {A carbon tax is critical for\\Canada's green future};
\node[textbox] (t_text) at (-0.8, -0.65) {Removing tariffs between provinces is necessary to help increase free trade between them and boost the Canadian economy.};

% Arrows from topics to text boxes
\draw[->, thin]  (ct_text.west) -- (s_ct);
\draw[->, thin]  (t_text.west) -- (s_t);

% Right side - Clusters
% Large economic regulations cluster (light blue)
\node[cluster, minimum width=4cm, minimum height=1.3cm, fill=blue!10] (econ) at (6.5, -0.9) {};
\node[font=\small, yshift=3mm] at (econ.south) {economic regulations};

% Environmental action cluster (light green)
\node[cluster, minimum width=2.5cm, minimum height=1.1cm, fill=green!10] (env) at (5.0, 0.85) {};
\node[font=\small, align=center] at (env.center) {environmental\\action};

% Inside economic regulations
\node[cluster, fill=blue!20] (tariffs) at (6.0, -0.7) {tariffs};
\node[cluster, fill=blue!20] (freetrade) at (7.8, -0.3) {free\\trade};

% Carbon tax overlapping both clusters (light yellow)
\node[cluster, minimum width=0.8cm, minimum height=0.5cm, fill=yellow!10] (ct) at (5.7, 0.07) {carbon tax};

% Canada cluster (light gray)
\node[cluster, fill=gray!10] (canada) at (3.3, 0.0) {canada};

% Connecting arrows from text boxes to clusters
% From carbon tax text
\draw[->, thin] (ct_text.east) -- (ct);
\draw[->, thin] (ct_text.east) -- (env);
\draw[->, thin] (ct_text.east) to[bend left=4] (econ);
\draw[->, thin] (ct_text.east) -- (canada);

% From tariffs text  
\draw[->, thin] (t_text.east) -- (tariffs);
\draw[->, thin] (t_text.east) to[bend left=8] (freetrade);
\draw[->, thin] (t_text.east) -- (canada);
\draw[->, thin] (t_text.east) to[bend right=5] (econ);

\end{tikzpicture}
\caption{Comparison of assigning a single stance target to each document (left), versus assigning multiple hierarchical stance targets that overlap with other texts as proposed here (right).}
\label{fig:stance-targets}
\end{figure*}

Existing approaches do not address all of these features. Most stance target extraction methods produce a single stance target for a single given document, without attending to the broader context of a discussion, or allowing for multiple issues to be addressed in a document \cite{irani2024wiba, akash2024can, li2023new, zhang2021towards}. Disagreement discovery methods from outside the stance-detection literature that \textit{do} consider a corpus as a whole \cite{paschalides2021polar,he2021detecting} do not produce a clear mapping of documents to stance targets.

We make four contributions. We formalize this task of mapping issues/targets of disagreements in a corpus into a computational task which we call {\it corpus-oriented stance target extraction} (COSTEx). We then provide a metric for evaluating a method's performance on this task. We present a method which addresses the task, and show it outperforms existing methods on our task. Finally, we conduct a case study using our method, which shows that it can retrieve key issues (stance targets and stances) from a discussion represented by a corpus.

With the evaluation and development of a method that performs well on the task we outline here, we can unlock powerful insights in large-scale media corpora, giving us new tools to understand large-scale natural language behaviour such as polarization and public opinion. We release a library for this method at \url{https://github.com/bendavidsteel/stancemining}

\vspace{-1mm}
\section{Background}
\label{sec:background}
\vspace{-1mm}
\paragraph{Subjectivity Detection}

The fields of stance detection, aspect-based sentiment detection, and argument mining have produced methods to identify targets of subjective perspective, and classifying the subjective judgement of documents towards those targets. \citet{li2023new} and \citet{steel2024multi} look at stance target extraction, but both methods require a priori knowledge and manual target choice at a point in the method, and therefore do not fulfill feature \ref{itm:not-known}. \citet{akash2024can}, \citet{irani2024wiba} and \citet{zhang2021towards}, all look at open-target extraction (where there are no predefined targets) in stance detection, argument mining, and sentiment detection respectively. All three focus on inferring targets for documents in isolation and, as a result, none of these methods consider the multiple or hierarchical stance targets possible from a document (as represented in Fig. \ref{fig:stance-targets} and defined as a required feature \ref{itm:multiple-targets}), or the need for large stance target clusters \textemdash groups of documents mapped to the same stance target \textemdash if we want to aggregate the data for further analysis (feature \ref{itm:contextualized-targets}). Nevertheless, we compare our developed method against WIBA \cite{irani2024wiba} in this work.

\vspace{-2mm}
\paragraph{Polarized and Controversial Topics}

Topic modelling derived methods are a common approach to this problem space, and naturally handle the desired aggregation process from feature \ref{itm:contextualized-targets}. But converting topic clusters to stance target clusters is not trivial. Topic and stance targets clusters don't map neatly one-to-one, as demonstrated in Fig. \ref{fig:hierarchy_targets}. Work on topic cluster representation, such as \citet{pham2023topicgpt} and \citet{grootendorst2022bertopic}, uses large language models (LLMs) to improve the interpretability of cluster names, working towards feature \ref{itm:document-target-map}. But mapping a topic cluster to a stance target is difficult, as it requires domain knowledge and reasoning to convert topic descriptions into a stance target (Fig. \ref{fig:target_naming}).

\begin{figure*}
    \centering
    \begin{subfigure}[b]{0.48\textwidth}
        \centering
        \begin{tikzpicture}[
            cluster/.style={rounded corners=8pt, draw, minimum width=4.0cm, minimum height=2cm},
            subcluster/.style={draw, minimum width=3cm, minimum height=0.6cm, font=\scriptsize, text width=3.5cm, align=center},
            annotation/.style={->, thick}
        ]
        % Reason 1: Hierarchy of stance targets
        \begin{scope}[shift={(-4,1.6)}]
            \node[cluster, fill=blue!10] (r1main) {};
            \node[subcluster, fill=blue!25] (r1s1) at (0,0.4) {Greece needs to pay its debt otherwise the eurozone is finished!};
            \node[subcluster, fill=blue!35] (r1s2) at (0,-0.4) {My heart goes out to Greece in these times.};
            \node[rectangle, draw, above=0.3cm of r1main, font=\scriptsize] (topic) {Topic: "greece, bankers, eurozone"};
            \node[rectangle, draw, below left=0.3cm and -0.8cm of r1main, font=\scriptsize] (stance1) {Stance Target: "eurozone"};
            \node[rectangle, draw, below right=0.3cm and -0.8cm of r1main, font=\scriptsize] (stance2) {Stance Target: "greece"};
            \draw[annotation] (r1main.north) -- (topic);
            % Use curved paths to avoid overlapping
            \draw[annotation] (r1s1.south west) .. controls +(-0.5,-0.3) and +(0.3,0.3) .. (stance1.north);
            \draw[annotation] (r1s1.south east) .. controls +(0.5,-0.3) and +(-0.3,0.3) .. (stance2.north);
            \draw[annotation] (r1s2.south) -- (stance2.north);
        \end{scope}
        \end{tikzpicture}
        \caption{Same high-level topic, but different specific stance targets}
        \label{fig:hierarchy_targets}
    \end{subfigure}
    \hfill
    \begin{subfigure}[b]{0.48\textwidth}
        \centering
        \begin{tikzpicture}[
            cluster/.style={rounded corners=8pt, draw, minimum width=4.0cm, minimum height=2cm},
            smalldoc/.style={draw, minimum width=1.6cm, minimum height=0.5cm, font=\scriptsize, text width=3.5cm, align=center},
            annotation/.style={->, thick}
        ]
        
        % Reason 3: Different cluster naming approaches
        \begin{scope}[shift={(6.5,1.6)}]
            \node[cluster, fill=gray!10] (r2main) {};
            \node[smalldoc, fill=gray!20] (r4doc1) at (0,0.4) {Requiring that the police wear body cameras is common sense.};
            \node[smalldoc, fill=gray!30] (r4doc2) at (0,-0.4) {Cops using body cameras at all times is just more surveillance.};
            \node[rectangle, draw, above=0.3cm of r2main, font=\scriptsize] (topic) {Topic: "cameras, police, camera, cops"};
            \node[rectangle, draw, below=0.3cm of r2main, font=\scriptsize] (stance) {Stance Target: "police body cameras"};
            \draw[annotation] (r2main.north) -- (topic);
            \draw[annotation] (r2main.south) -- (stance);
        \end{scope}
        
        \end{tikzpicture}
        \caption{Stance targets need more specific names than topics.}
        \label{fig:target_naming}
    \end{subfigure}
    \caption{Representation of the differences between stance target clusters and topic clusters, showing hierarchical relationships, one-to-many mappings, and different cluster naming requirements, as discovered in manual analysis.}
    \label{fig:stance-clusters}
\end{figure*}

\citet{fukuma2022many} use a network method to find polarized topics, but this method is designed for X/Twitter specific features. \citet{garimella2018quantifying} use hashtags to define conversational graphs, and find partitions in those graphs in order to find controversial topics. This method however relies on hashtags, limiting it to corpuses with heavy hashtag usage. \citet{paschalides2021polar} and \citet{he2021detecting} produce methods to find polarized topics, and we evaluate these methods in this work.

\section{Problem Definition}
\label{sec:problem-definition}
\vspace{-1mm}

Motivated by our desired features from Section \ref{sec:intro}, we define COSTEx as follows: given a corpus of documents, we seek to identify labeled clusters of those documents where all documents in a cluster share the same stance target, which is captured by the label of the cluster. Crucially, clusters can be overlapping, allowing a document to be assigned more than one stance target. Formally, for a corpus of documents $D=\{d_{1},..., d_{N}\}$, we want to find a set stance target clusters, $C=\{c_1, c_2, ... c_M\}$ where $c_i \subset D$, and their corresponding stance targets $T =\{t_1, t_2, ..., t_M\}$. As stance detection has not previously considered corpus-aware methods, we propose new criteria that define success in COSTEx, that measure the extent to which a method that implements this task fulfills the desired features. As such, the COSTEx problem seeks $C$ and $T$ such that they reflect the following criteria:

\vspace{-2mm}
\begin{enumerate}
    \item{\textbf{Clusters with Large Stance Variance:}}\label{itm:stance_variance} Given the stance of each document on the stance target $stance(t_i, d_j) \rightarrow \{-1, 0, 1\}$, we want to find stance targets that maximize the stance variance for all related documents:
    \vspace{-2mm}
    \[\frac{1}{|C|}\sum_{c_i \in C} Var(\{stance(t_i, d) : d \in c_i\})\]
    \vspace{-6mm}
    
    This is a metric for picking ``controversial'' stance targets. Intuitively, stance targets that no-one disagrees on are less interesting than stance targets that people disagree on.
    \vspace{-3mm}
    \item {\textbf{Stance Target Range and Relevance:}}\label{itm:targets} We want to find many stance targets that are relevant to the documents. We can measure relevance of targets by ensuring that the stance targets adhere to human judgments of stance targets, via comparison to labeled datasets and custom human annotation (as in \citet{akash2024can}), and we can measure `many stance targets' by measuring the mean number of targets per document: 
    \vspace{-2mm}
    \[\frac{1}{|D|}\sum_{d \in D} |\{c_i \in C : d \in c_i \}|\]
    \vspace{-8mm}
    \item {\textbf{Balanced Stance Target Clusters:}} \label{itm:cluster_size} We want to optimize for clusters of a range of sizes, including large clusters to allow for useful aggregations, that still capture clusters of meaningful grouping. To measure meaningful clusters, we will use human evaluation. And to measure cluster size, we will use the cluster size Shannon entropy multipled by normalized cluster size range, to ensure there are a balanced number of clusters of a range of sizes:
    \vspace{-3mm}
    \[\frac{\max_i c_i - \min_i c_i}{\max_i c_i} \left(-\sum_{i=1}^{n} \frac{c_i}{C} \log_2\frac{c_i}{C}\right)\]
    where $C=\sum_{k=1}^{n}c_k$
    \vspace{-2mm}
\end{enumerate}
% \vspace{-2mm}
Naturally, in most situations it will be impossible to perfectly satisfy all of these. Solutions to this task will have to make careful trade-offs between these criteria. In practice, some of these metrics are trivially measurable, and some of them are much harder to measure (i.e. the ones requiring human evaluation). We will seek to do so via quantitative supervised and unsupervised metrics, and metrics from human evaluation tasks. 

Finally, we must address the question of what we mean by {\it stance targets} in the formulation above. In the literature, it is common to define stance targets either as noun-phrases (e.g., ``police body cameras''), or claims (``police should wear police body cameras'') \cite{zhao2024ez}. A document assigned to this stance target contains content that takes a position on it. Note that, where stance targets are concerned, the problem definition requires only a means of scoring a document's position on a stance target (i.e., $stance(t_i,d)$). As a result, the problem admits either noun-phrases or claims as stance targets.

\vspace{-2mm}

\section{Methods}
\vspace{-1mm}

Here, we propose our method that fulfills all the features in Section \ref{sec:intro}. Rather than clustering documents directly (which conflates topics with stance targets as discussed in Section \ref{sec:background}), we first extract multiple stance targets (fulfilling feature \ref{itm:multiple-targets}), then cluster those targets. This lets us find large stance target clusters, where documents can naturally belong to multiple large clusters. We then generate stance targets for each of those clusters to find higher-level stance targets (i.e. targets that are hierarchically more abstract, or more general to the cluster). Collecting all these stance targets together for each document, we then have small, specific stance target clusters, and larger, high level stance target clusters. We call this method \textit{ExtractCluster} (EC), formally define it in Algorithm \ref{alg:extract_cluster}, and show a simplified system diagram of it in Fig. \ref{fig:ec-diagram}. Our method aims to meaningfully achieve each criteria from Section \ref{sec:problem-definition}.

The base stance targets are produced using an LLM fine-tuned on document - stance target pairs, using diverse beam search \cite{vijayakumar2016diverse} to generate multiple targets. We cluster the targets using BERTopic \cite{grootendorst2022bertopic}, which provides an easy-to-use and configurable topic modelling solution. The default clustering configuration of BERTopic gives us one layer of clusters, meaning there are two hierarchical levels of stance targets. The higher-level stance targets are generated using an LLM with a few-shot prompt (shown in Appendix \ref{sec:prompts}). To avoid producing stance targets for each document that are paraphrases of each other, we remove stance targets where their sentence embeddings have high cosine similarity based on a configurable threshold \cite{reimers-2019-sentence-bert} (detailed in Appendix \ref{sec:baseline_implementation}).

\begin{figure}[t]
\begin{algorithm}[H]
\begin{algorithmic}[1]
\Function{ExtractCluster}{$D$}
    \For{each document $d \in D$}
        \State $T_d \gets$ ExtractStanceTargets($d$)
        \State $T_d \gets$ RemoveSimilarTargets($T_d$)
    \EndFor
    \State $C \gets$ TopicModelTargets($T$)
    \For{each cluster $c \in C$}
        \State $T_c \gets$ GenerateHigherLevelTargets($c$)
        \State $T_c \gets$ RemoveSimilarTargets($T_c$)
        \For{each $d : \exists t \in T_d : t \in c$}
            \State $T_d \gets T_d \cup T_c$
        \EndFor
    \EndFor
    \For{each document $d \in D$}
        \For{each target $t \in T_d$}
            \State $S_{d,t} \gets$ ClassifyStance($d, t$)
        \EndFor
    \EndFor
    \State \Return D, T, S
\EndFunction
\end{algorithmic}
\end{algorithm}
\vspace{-2mm}
\caption{Algorithm used by \textit{EC}. Topic modelling is done on the flat list of stance targets using BERTopic. Removal of similar targets is based on high cosine similarity between stance target sentence embeddings.}
\label{alg:extract_cluster}
\end{figure}

\begin{figure*}
\begin{tikzpicture}[
    doc/.style={rectangle, rounded corners, draw=black!50, fill=green!20, text width=5cm, minimum height=1.0cm, align=center, font=\scriptsize},
    extracted/.style={rectangle, rounded corners, draw=black!50, fill=blue!15, text width=3cm, minimum height=0.8cm, align=center, font=\small},
    concat/.style={rectangle, rounded corners, draw=black!50, fill=yellow!30, text width=3cm, minimum height=0.8cm, align=center, font=\small},
    process/.style={rectangle, rounded corners, draw=black!50, fill=yellow!30, text width=3.5cm, minimum height=1.2cm, align=center, font=\small},
    cluster/.style={rectangle, rounded corners, draw=black!50, fill=blue!15, minimum width=1.8cm, minimum height=0.7cm, align=center, font=\small},
    table/.style={rectangle, rounded corners, draw=black!50, fill=green!15, minimum width=2cm, minimum height=0.5cm, align=center, font=\footnotesize},
    stance/.style={rectangle, rounded corners, draw=black!50, fill=pink!30, minimum width=1.2cm, minimum height=0.5cm, align=center, font=\footnotesize},
    arrow/.style={->,>=stealth,thick}
]
% Input documents
\node[doc] (doc1) at (0,0) {A carbon tax is critical for\\Canada's green future};
\node[doc] (doc2) at (0,-1.2) {Removing tariffs between provinces is necessary to help increase free trade between them and boost the Canadian economy.};
% Extracted targets - now positioned relative to documents
\node[extracted, right=0.5cm of doc1] (extract1) {carbon tax, Canada};
\node[extracted, right=0.5cm of doc2] (extract2) {tariffs, free-trade,\\Canada};
% Draw arrows first to establish midpoints
\draw[arrow] (doc1) -- coordinate[midway] (mid1) (extract1);
\draw[arrow] (doc2) -- coordinate[midway] (mid2) (extract2);
% LLM extraction process positioned above midpoints
\node[process, above=1.2cm of $(mid1)!0.5!(mid2)$] (llmextract) {(1) Extract stance targets\\from documents using\\LLM};
% Cluster process
\node[process] (clusterbox) at (11,1.5) {(2) Cluster targets};
% Cluster visualization
\begin{scope}[shift={(11,-1)}]
    % Draw ellipse for cluster
    \draw[fill=orange!30] (0,0) ellipse (1cm and 0.5cm);
    
    % Cluster nodes
    \node[cluster] (c1) at (-1.8,0.3) {carbon tax};
    \node[cluster] (c2) at (-1.8,-0.5) {tariffs};
    \node[cluster] (c3) at (1.2,1.0) {free-trade};
    \node[cluster] (c4) at (2,-0.5) {Canada};
    
    % Draw dots for cluster points
    % Define dots as named coordinates
    \coordinate (dot1) at (-0.3,0.2);
    \coordinate (dot2) at (-0.3,-0.2);
    \coordinate (dot3) at (0.4,0.0);
    \coordinate (dot4) at (1.3,0.2);
    
    % Draw the dots
    \fill (dot1) circle (2pt);
    \fill (dot2) circle (2pt);
    \fill (dot3) circle (2pt);
    \fill (dot4) circle (2pt);
\end{scope}
% Concat process
\node[concat] (concat) at (7.3,-3.5) {(4) Concat to target\\lists with targets\\in this cluster};
% Stance detection table
\node[process] (getstance) at (3,-2.8) {(5) Get stance of each document\\on each of its discovered\\stance targets};
% Table entries
\begin{scope}[shift={(0,-3.75)}]
    \node[table] (t1) at (0.5,-0.5) {A carbon tax...};
    \node[table] (t2) at (0.5,-1.2) {A carbon tax...};
    \node[table] (t3) at (0.5,-1.9) {A carbon tax...};
    \node[table, text width=2cm, minimum height=0.5cm, fill=blue!15] (target1) at (2.8,-0.5) {carbon tax};
    \node[table, text width=2cm, minimum height=0.5cm, fill=blue!15] (target2) at (2.8,-1.2) {canada};
    \node[table, text width=2cm, minimum height=0.5cm, fill=blue!15] (target3) at (2.8,-1.9) {economic reg...};
    \node[stance] (s1) at (4.8,-0.5) {favor};
    \node[stance] (s2) at (4.8,-1.2) {neutral};
    \node[stance] (s3) at (4.8,-1.9) {favor};
\end{scope}
% Prompt LLM process
\node[process] (promptllm) at (11,-3.5) {(3) Prompt LLM using\\cluster description for\\targets general to\\the cluster};
% Final output
\node[extracted] (output) at (11,-5) {economic\\regulations};
% Draw remaining arrows
% Arrows from llmextract to midpoints
\draw[arrow] (llmextract.south) -- (mid1);
\draw[arrow] (llmextract.south) -- (mid2);
\draw[arrow] ([yshift=3pt]extract1.east) -- ++ (1.2,0) |- (clusterbox.west);
\draw[arrow] ([yshift=3pt]extract2.east) -- ++ (1.2,0) |- (clusterbox.west);
\draw[arrow] (clusterbox) -- (11,0.3);
\draw[arrow] (c1) -- (dot1);
\draw[arrow] (c2) -- (dot2);
\draw[arrow] (c3) -- (dot3);
\draw[arrow] (c4) -- (dot4);
\draw[arrow] (11,-2.2) -- (promptllm);
\draw[arrow] (concat.north) -- ++ (0,0.3) |- ([yshift=-3pt]extract1.east);
\draw[arrow] (concat.north) -- ++ (0, 0.3) |- ([yshift=-3pt]extract2.east);
\draw[arrow] (getstance) -- (3,-4);
\draw[arrow] (promptllm) -- (output);
\draw[arrow] (output) -| (concat);
\draw[arrow] (target1) -- (s1);
\draw[arrow] (target2) -- (s2);
\draw[arrow] (target3) -- (s3);
\draw[arrow] (doc2.south) -- ++ (0, -0.1) -| (getstance.north);
\draw[arrow] (extract2.south) -- ++ (0, -0.21) -| (getstance.north);
\end{tikzpicture}
\caption{Simplified system diagram of the \textit{ExtractCluster} (EC) method. Green boxes represent documents, yellow boxes represent system component steps, blue boxes represent stance targets, the orange circle represents a cluster, and pink boxes represent stance classifications. Numbers in each component step indicate the sequence of operations. We have excluded the stance target de-duplication step for brevity.}
\label{fig:ec-diagram}
\end{figure*}

\subsection{Comparison Methods}
We selected three methods to compare to EC on our task COSTEx. Although these methods do not fully address our proposed task, they address it sufficiently to warrant evaluation.


\vspace{-2mm}

\paragraph{POLAR} \cite{paschalides2021polar} uses entity extraction and network methods to find polarized topics. While this method is designed to find polarized topics, we apply it here to the similar but more general COSTEx task. Though the method does not explicitly map documents to stance targets, we extend it to use any entities or noun phrases that are tagged as part of a polarized topic as stance targets for their respective documents.

\vspace{-2mm}
\paragraph{PaCTE} \cite{he2021detecting} combines topic modeling and a partisanship classification model to find topics of partisan disagreement. We adapt it here to finding targets of stance disagreement.

\vspace{-2mm}
\paragraph{WIBA} \cite{irani2024wiba} uses three fine-tuned LLMs to determine whether a document features an argument, extracts the claim topic of the argument, then determines the stance of the document on that argument. In this application we remove the argument detection step, instead relying on the neutral label in stance classification. While this method is defined for argument detection, it maps neatly to stance detection. Although a more stance detection-centric method is now available \cite{akash2024can}, we use \citet{irani2024wiba} because it was available with an implementation at the time of this work's inception. However, the two methods are functionally similar enough as to be interchangeable in this context.
\vspace{-2mm}
\paragraph{Comparison}

To summarize, these three methods from the literature fulfill different features of the COSTEx task as defined in Section \ref{sec:intro}. We summarize the ways in which the representative methods \textemdash which we will evaluate here \textemdash fulfill those requirements in Table \ref{tab:method-comparison}. As shown, none of the methods achieve all of the necessary attributes, but they each achieve most aspects of the desired method.

\begin{table*}[h]
\centering
\small
\begin{tabular}{lccccc}
\toprule
\textbf{Feature} & \textbf{PaCTE} & \textbf{POLAR} & \textbf{WIBA} & \textbf{EC}\\
\midrule
Stance target discovery through aggregation & \checkmark & \checkmark & \ding{55} & \checkmark \\
Multiple stance targets per document & \checkmark & \checkmark & \ding{55} & \checkmark \\
Map documents to stance targets & \ding{55} & \ding{55}  & \checkmark & \checkmark \\
\bottomrule
\end{tabular}
\caption{Comparison of different methods against our method, \textit{EC}, for each of features \ref{itm:contextualized-targets}, \ref{itm:multiple-targets}, and \ref{itm:document-target-map}, as defined in Section \ref{sec:intro}. All of the methods fulfill feature \ref{itm:not-known}.}
\label{tab:method-comparison}
\end{table*}

\vspace{-1mm}
\section{Experiments}
\vspace{-1mm}

With our method in hand, we now want to see to what extent it fulfills COSTEx by testing it using metrics and human evaluation methods derived from our formulation, and comparing it to our comparison methods. 
\vspace{-2mm}
\paragraph{Datasets}

We use two large stance detection datasets to evaluate the methods, VAST \cite{allaway2020zero} and EZ-STANCE \cite{zhao2024ez}. These datasets come from two domains, New York Times comments and Twitter respectively, enabling testing across diverse text types. Importantly, both datasets derive their stance targets from each document \textemdash as opposed to a dataset designed around a specifically chosen set of stance targets \textemdash allowing us to grade the produced stance targets against the annotated stance targets from the datasets. We report statistics from the datasets (Tab. \ref{tab:dataset-stats}).

\begin{table}[]
    \centering
    \footnotesize
    \begin{tabular}{c|cccc}
        \toprule
        Dataset & Num. & Mean.  & Stance & Lang. \\
        & Ex. & Num. & Split & \\
        & & Targets & (F/N/A) & \\
        \midrule
        VAST & 784 & 2.45 & 0.47/0.02/0.51 & en\\
        EZ-STANCE & 1561 & 1.71 & 0.36/0.35/0.29 & en\\
        \bottomrule
    \end{tabular}
    \caption{Statistics from the datasets used for testing.}
    \label{tab:dataset-stats}
\end{table}

\paragraph{Configuration} 

We fine-tune a stance target extraction model and a stance detection model, on both VAST and EZ-STANCE for both tasks, using  \textit{Llama-3.2-1B-Instruct} as a base model (an open-weight 1B parameter model) \cite{meta2024llama}. We use these fine-tuned models for both EC and WIBA. For diverse beam generations in EC, we sample 3 generations, as this is the ceiling integer above the highest mean number of targets in each dataset (Tab. \ref{tab:dataset-stats}). We use a cosine similarity value of 0.8 for EC, as through manual validation, this de-duplicates stance targets that are functionally identical. We use \textit{phi-3.5-mini-instruct} \cite{abdin2024phi3} to generate higher-level stance targets for EC, a 4B model suitable for few-shot prompting. We list all other experimental implementation details of each comparison method in Appendix \ref{sec:implementation}.

\subsection{Automated Evaluation}
\paragraph{Metrics}

As previously highlighted (Sec. \ref{sec:problem-definition}), some of the outcomes that we want to optimize in our method are trivially measurable, and some are much more difficult to measure. We therefore propose a set of metrics that assess the extent to which the method outputs optimize for the objectives defined above. While our method does produce hierarchical stance targets, in evaluation we will treat them as a flat list, while maintaining the valuable property of higher-level stance targets aggregating across more documents.
\vspace{-2mm}
\begin{itemize}

\item \textbf{Target F1:} The BERTScore F1 \cite{zhang2019bertscore} of the discovered targets, compared to the annotated dataset, as in \cite{akash2024can}. As we have a set of annotated stance targets for each document in our labelled dataset, we compute the precision by comparing each predicted stance target to all gold stance targets, and the recall by comparing each gold stance target to all predicted stance targets, and compute the F1 from the resulting precision and recall, as defined in Appendix \ref{sec:target_f1}. This metric measures adherence to Criterium \ref{itm:targets}.
\vspace{-2mm}

\item \textbf{Stance Retrieval F1:} The F1 of the discovered stance of the documents, compared to a labeled dataset. Seeing as we have a potentially different set of predicted stance targets as the gold stance targets, we create a mapping of predicted stance targets to gold stance targets where the sentence embedding cosine similarity is greater than 0.9, then compute the precision by comparing each predicted stance to all gold stances, the recall by comparing each gold stance to all predicted stances, and the F1 score from the precision and recall, as defined in Appendix \ref{sec:stance_f1}.
\vspace{-2mm}
\item \textbf{Stance Variance:}
    See Criterium \ref{itm:stance_variance}.
\vspace{-2mm}
\item \textbf{Mean Num. of Targets:}
    See Criterium \ref{itm:targets}
\vspace{-2mm}
 \item \textbf{Balanced Cluster Sizes:}
    See Criterium \ref{itm:cluster_size}
\vspace{-2mm}
\item \textbf{Walltime:}
    Method run-time duration (s).
\end{itemize}
\vspace{-2mm}
The supervised metrics, the target F1 and stance retrieval F1, are measuring the adherence of the method to a typical stance detection dataset. However, we also want to optimize for multi-target, hierarchical, and clustered stance targets. Optimizing for metrics that measure these aspects will reduce our target F1 score, as the stance targets will be further from the stance targets given in the base datasets. We need to assess our results holistically, and consider that, as part of our task formalization, any solution to this problem is making a trade-off between objectives. We will therefore determine the overall ranking of the methods via a summed rank order: we find the rank of each method on every metric, sum all the ranks for each method, and the lowest summed rank order is the best method. 
\vspace{-2mm}
\paragraph{Results}

\begin{table*}[t]
\centering
\small
\begin{tabular}{l|ccc|ccc|cccc}
\toprule
\multicolumn{1}{c}{} & \multicolumn{3}{c}{\textbf{Target}} & \multicolumn{3}{c}{\textbf{Stance}} & \textbf{Mean Num.} & \textbf{Stance} & \textbf{B. Cluster} & \textbf{Wall} \\
\textbf{Method} & \textbf{F1 ↑} & \textbf{P. ↑} & \textbf{R. ↑} & \textbf{F1 ↑} & \textbf{P. ↑} & \textbf{R. ↑} & \textbf{Targets ↑} & \textbf{Variance ↑} & \textbf{Sizes ↑} & \textbf{time ↓} \\
\midrule
\multicolumn{11}{c}{\textbf{VAST}}\\
\midrule
PaCTE & 0.775 & 0.779 & 0.771 & 0.000 & 0.000 & 0.000 & 1.212 & \textbf{0.226} & 2.314 & \textbf{155.2} \\
POLAR & 0.512 & 0.524 & 0.501 & - & - & - & \underline{2.140} & - & 3.028 & 327.7 \\
WIBA & \textbf{0.910} & \textbf{0.930} & \underline{0.891} & \underline{0.116} & \underline{0.190} & \underline{0.089} & 1.000 & 0.108 & \underline{7.753} & \underline{246.9} \\
EC & \underline{0.897} & \underline{0.889} & \textbf{0.907} & \textbf{0.143} & \textbf{0.210} & \textbf{0.119} & \textbf{3.190} & \underline{0.136} & \textbf{8.031} & 1569.1 \\
\midrule
\multicolumn{11}{c}{\textbf{EZSTANCE}}\\
\midrule
PaCTE & 0.766 & 0.768 & 0.763 & 0.000 & 0.000 & 0.000 & \underline{1.038} & \textbf{0.208} & 4.311 & \textbf{213.7} \\
POLAR & 0.038 & 0.038 & 0.037 & - & - & - & 0.218 & - & 0.168 & \underline{582.7} \\
WIBA & \textbf{0.884} & \textbf{0.899} & \textbf{0.871} & \underline{0.145} & \underline{0.200} & \underline{0.120} & 1.000 & 0.019 & \underline{9.495} & 766.4 \\
EC & \underline{0.859} & \underline{0.851} & \underline{0.867} & \textbf{0.158} & \textbf{0.202} & \textbf{0.141} & \textbf{3.380} & \underline{0.039} & \textbf{9.520} & 3349.8 \\
\midrule
\bottomrule
\end{tabular}
\caption{Metrics comparison across datasets and methods averaged across 5 runs for each method and dataset. Best metrics are indicated with arrows. P. and R. stand for precision and recall respectively. We do not include stance results for POLAR as it does not assign stance to individual documents. Bold numbers indicate the best performance, underline indicates second best.}
\label{tab:results-supervised}
\end{table*}

We report the supervised and unsupervised metrics from the mean of 5 runs for each method on each dataset (Tab. \ref{tab:results-supervised}). EC generally outperforms other methods, except on stance target F1 and precision, and wall-time. Stance retrieval rankings are robust to varying cosine similarity, see Appendix \ref{sec:appendix-retrieval-sensitivity}. We conducted ablations of cosine similarity threshold and number of beam generations and confirm that our chosen values yield the best results, see Appendix \ref{sec:appendix-ablations}. We also tested a version of the method that removes lower variance stance targets and find that it achieves higher mean stance variance but worse on all other metrics (See Appendix \ref{sec:appendix-variance-ablation}), showing our method could not be trivially improved in this manner.

\vspace{-1mm}
\paragraph{Human Evaluation}
\vspace{-1mm}

We created two human evaluation tasks to evaluate the method outputs. The first task presents a triad of documents (a base document, another from the same cluster, and one from a different cluster) and has the annotator select which two documents go in the same stance target cluster. We measure how often the annotators agree with the document clustering chosen by each method. A second task presents a base document, and two stance target sets provided by two different methods, and a prompt asks the labeler to choose between the two stance target sets, or neither if neither are suitable. Each annotator received an evaluation guide prior to their evaluation task to explain the concepts in use in the task. We obtained 483 and 492 annotations for each task respectively, from 6 annotators who were students in the authors' lab. We show the prompts and evaluation guide given to annotators, and example generation process in Appendix \ref{sec:human-eval}.



\begin{table}[]
    \centering
    \small
    \begin{tabular}{c|c|c|c}
        \toprule
        Method & LSR & Agree Pct. & Example Output \\
        \midrule
        PaCTE & -2.23 & 0.19 & school,health,covid...\\
        POLAR & -2.79 & 0.00 & anyone \\
        WIBA & \underline{1.51} & \textbf{0.62} & medical law\\
        EC & \textbf{2.23} & \underline{0.34} & jerusalem\\
        \bottomrule
    \end{tabular}
    \caption{Luce Spectral Ranking (LSR) pairwise comparison score, calculated by comparing different methods' stance target sets for each document, alongside an example stance target output from each method for reference (PaCTE example shown truncated). And percentage of examples where annotators agreed with the clustering of a document triad, for each method.}
    \label{tab:human_eval_scores}
\end{table}


To ensure there was agreement between annotators, we had two annotators evaluate the same set of 20 examples from each task. The Fleiss' Kappa \cite{fleiss1981measurement} of the stance target cluster task was 0.53, and for the stance target task it was 0.83, indicating inter-annotator agreement. For the stance target set comparison task, we use the Luce spectral ranking (LSR) \cite{maystre2015fast} (via Choix \footnote{github.com/lucasmaystre/choix}) to determine the output stance targets most preferred by human annotators. EC and WIBA are rated the highest, with POLAR and PaCTE rated poorly (Tab. \ref{tab:human_eval_scores}). For stance target cluster agreement scores, we simply record the number of times the human evaluator agreed with the method. WIBA, EC, and PaCTE obtain the best results for cluster evaluation, and POLAR obtains no agreement from evaluators (Tab. \ref{tab:human_eval_scores}). 

\vspace{-1mm}
\paragraph{Summary}
\vspace{-1mm}

We show the summed rank order of each method, for each metric, in Fig. \ref{fig:summed-rank-order}. This demonstrates the overall rank of the methods on the COSTEx task we introduce in this work.

\begin{figure}
    \centering
    \includegraphics[width=0.85\linewidth]{figs/rank_order.png}
    \caption{Summed rank order across all metrics for each method. \textit{EC} outperforms the other methods we trial from the literature across our metrics.}
    \label{fig:summed-rank-order}
\end{figure}

\vspace{-1mm}
\section{Discussion}
\vspace{-1mm}

POLAR needs to find many named entities to find polarized topics (being designed for news articles), and as such performs poorly on the short text datasets used here, especially the EZ-STANCE dataset (Tab. \ref{tab:results-supervised}). We observe poor evaluations of naming and clustering performance (Tab. \ref{tab:human_eval_scores}).

PaCTE's use of LDA topic modeling and a small classifier model means that it can quickly find large stance target clusters with high stance variance (Tab. \ref{tab:results-supervised}). However, the naming of clusters with topic keywords results in a low evaluation score (Tab. \ref{tab:human_eval_scores}), and the stance target clusters are only moderately agreed with (Tab. \ref{tab:human_eval_scores}).

WIBA's stance target extraction produces good stance targets (Tab. \ref{tab:results-supervised} and \ref{tab:human_eval_scores}), and performs highest on our cluster agreement evaluation (Tab. \ref{tab:human_eval_scores}). But the small stance target clusters it produces \textemdash due to only producing one stance target per document \textemdash result in lower stance retrieval F1, and low stance variance and cluster size (Tab. \ref{tab:results-supervised}).

EC outperforms WIBA in stance target cluster size, stance variance, stance retrieval, and stance target set preference (Tab. \ref{tab:human_eval_scores}). However, we also see that it under-performs WIBA on our cluster agreement evaluation (Tab. \ref{tab:human_eval_scores}), and stance target precision. We infer that as EC maps higher-level stance targets to each document \textemdash that have no parallel in the annotated datasets we use \textemdash which results in large clusters defined by abstract stance targets that are too general for annotators to spot in our cluster agreement exercise. Nonetheless, from the summed rank order (Fig. \ref{fig:summed-rank-order}), EC is the most effective method tested here.

\vspace{-1mm}
\section{Case Study}
\vspace{-1mm}

Having empirically shown our method outperforms other methods from the literature, we chose to assess its effectiveness at identifying key characteristics of a discourse under real-world conditions. Topic modelling is frequently used for exploratory analysis of discourse corpuses \cite{hobson2024story, falkenberg2022growing}. Although it does not disaggregate expressed valence \textemdash a key part of separating discourse \cite{ghafouri2024love}\textemdash the previously missing step of stance detection \textemdash stance target discovery \textemdash makes it labour-intensive to run as a go-to exploratory step. Crucially, both EC and BERTopic \cite{grootendorst2022bertopic} require no notable parameter tuning and, so, are of equal complexity for a domain researcher to use.

We assumed the role of a researcher studying the political views present in a social media dataset. We chose a 2024 Twitter dataset consisting of 1.4m tweets (81\% English, 9\% French, 10\% other languages) from 1.9k prominent Canadian media accounts \cite{pehlivan2025can}. See Appendix \ref{sec:case_implementation} for implementation details. Table \ref{tab:topic-compare} shows the largest stance target vs. topic clusters.

\begin{table*}[t!]
    \centering
    \small
    \begin{tabular}{c|c|c|c}
        \toprule
        \multicolumn{2}{c|}{Stance Targets} & \multicolumn{2}{c}{Topics} \\
        \midrule
        Name & Count & Name & Count \\
        \midrule
        canada & 76k & gaza, israel, israeli, hamas & 22k \\
        j. trudeau & 54k & olympics, game, olympic, athletes & 15k \\
        trudeau & 39k & hes, guy, coyne, mrstache9 & 10k \\
        trump presidency & 29k & url, juliemarienolke, thejagmeetsingh, saudet80 & 9k \\
        liberal party & 22k & healthcare, nurses, doctors, doctor & 9k \\
        israeli & 17k & shes, shell, shed, quelle & 8k \\
        trump & 17k & housing, rent, rental, homes & 7k\\
        b.c. ndp & 16k & trudeau, justin, trudeaus, resign & 6k \\
        \bottomrule
    \end{tabular}
    \caption{Comparing largest stance target clusters to largest topic clusters.}
    \label{tab:topic-compare}
\end{table*}


\vspace{-2mm}
\paragraph{Meaningful clusters.} Both stance target and topic modeling methods can produce nonsensical clusters. How do we quickly remove the noise? In topic modeling, this is messy: as seen in Table \ref{tab:topic-compare}, some of the largest topic clusters are meaningless (e.g., ``shes, shell, shed, quelle''). In contrast, with EC, an easy way of filtering weak stance targets is by simply dropping small stance clusters, with the intuition being that modal stance targets are more frequently good stance targets. In this case study, we found removing stance targets with less than 50 data-points to be a good level. At this border, there are some good stance targets (`Organic Food Movement', and `US Col. Lawrence Wilkerson') but also many non-specific or nonsensical stance targets (`which will', `candidate nomination'). 
\vspace{-2mm}
\paragraph{Cluster informativeness.} Table \ref{tab:topic-compare} highlights the informativeness of EC clusters in several ways. First, stance target clusters capture more of the documents than the largest topics, due to EC allowing documents to belong to multiple stance target clusters. If we kept just the first extracted stance target (as previously \cite{akash2024can}), the `trudeau' stance target would only be assigned to 22k documents, with our method allowing us to know the stance of more documents on `trudeau' where he may be referred to implicitly. Second, the stance targets capture the large ongoing issues of Canadian public discourse (Canada, Justin Trudeau, the Liberal Party), and topical issues (Donald Trump's presidency, the B.C. election, the Israel-Palestine conflict), where these large issues are missed by the topics - instead emphasizing smaller topics like the Olympics. Even for topic clusters that are not ``noise'', the stance target names are consistently more specific, and therefore more usable for further analysis. However, EC needs improved stance target de-duplication, as shown by the presence of `j. trudeau' and `trudeau'.

\begin{figure*}[t!]
    \includegraphics[width=\textwidth]{figs/target_map.png}
    \caption{Map of top stance targets, sized by frequency, coloured by average stance. In general, the major issues that Canadian social media users tend to have attitudes on are represented. However, we can also see that improved stance target de-duplication is necessary, along stance target clarity (`climate change denialism').}
    \label{fig:target-map}
\end{figure*}
\vspace{-2mm}
\paragraph{Understanding stance on the target clusters.} We show a map of the 30 largest stance target clusters in Fig. \ref{fig:target-map}. Having stance classifications on so many targets surfaces key aspects of the discourse: allowing us to compare mean stance on party leaders (-0.57 for Trudeau vs. -0.44 for Poilievre), parties (-0.45 for the NDP vs. -0.62 for the Liberal Party), and foreign policy issues (-0.46 for Israel vs. -0.79 for Hamas) with one method application (where we have substituted `favor' for 1, `neutral' for 0, and `against' for -1).

This case study highlights how EC gave the researcher a larger and more detailed map of the discussion in our dataset, alongside more specific and understandable cluster names.

\vspace{-1mm}
\section{Conclusion}
\vspace{-1mm}

We have motivated and conceptualized the task of \textit{COSTEx}, and shown that our new method for this task, \textit{EC}, outperforms previous methods for similar tasks. We then used a large-scale real-world dataset to demonstrate that our method reliably captures and represents clusters of stance target discussion. We hope that this method can aid practitioners in quickly understanding discourse in large and wide-ranging real-world datasets, and help improve understanding of complex behaviors such as polarization and public opinion in our quickly changing information environments.

\section{Limitations}

\vspace{-2mm}
\paragraph{Datasets}

We found that the lack of hierarchical stance targets in the stance-detection datasets used in this work made it difficult to evaluate the ability of the methods to find a full breadth of hierarchical, clustered stance targets for each document. We can only use these datasets to assess the extent to which the method found the base stance targets for each document. Future work should develop new datasets to evaluate higher-level stance targets.

\vspace{-2mm}
\paragraph{Methods}

Stance target de-duplication became an issue when we applied our method to a larger corpus. We experimented with using DBSCAN to some success, but de-duplicating different ways of spelling names (`j. trudeau', `trudeau') while avoiding false positives requires a carefully set distance threshold between embeddings. Additionally, our method of using diverse generation to generate multiple stance targets for each document \textemdash while not requiring re-training of our stance target generation model \textemdash could be made faster and more flexible by generating targets as a list.

\vspace{-2mm}
\paragraph{Task Formulation}

Optimizing for stance variance deprioritizes stance targets that are generally agreed upon, but when disagreed upon, are interesting, such as conspiracy theories, so optimizing for this metric is a trade-off.

Another key limitation was a lack of a principled framework for defining a hierarchy of targets. In practice, LLM prompting produced sufficiently useful results here, but a more well-defined definition could produce stronger results. 

\bibliography{custom}

\appendix

\section{Methods}
\label{sec:appendix-methods}

In addition to the method we propose in this work, we also trialled a method we call \textit{ClusterExtract}, inspired by PaCTE. It starts by finding hierarchical topics in the corpus using BERTopic \cite{grootendorst2022bertopic}, then assigns stance targets to each topic. It is described in Algorithm \ref{alg:cluster_extract}.  However, we found that it produced inferior results to \textit{EC}, and so do not detail it in the main results of the work.


\begin{algorithm}
\caption{Algorithm used by \textit{ClusterExtract}.}
\begin{algorithmic}[1]
\Require Documents $D$
\Function{ClusterExtract}{$D$}
    \State $C \gets$ TopicModelDocs($D$)
    
    \State \Comment{Handle outlier documents (Topic = -1)}
    \State $D_{out} \gets$ FilterOutliers($D, C$)
    \For{each document $d \in D_{out}$}
        \State $T_d \gets$ ExtractStanceTargets($d$)
        \State $T_d \gets$ RemoveSimilarTargets($T_d$)
    \EndFor
    
    \State \Comment{Handle non-outlier documents}
    \For{each cluster $c \in C$}
        \State $T_c \gets$ ExtractClusterStanceTargets($c$)
        \State $T_c \gets$ RemoveSimilarTargets($T_c$)
    \EndFor
    
    \State \Comment{Generate hierarchical topic targets}
    \State $H \gets$ GetHierarchicalTopics($T$)
    \For{each parent cluster $c \in H$}
        \State $C_p \gets$ GetChildTopics($c$)
        \State $T_p \gets$ AggregateChildTargets($C_p$)
        \State $T_p \gets$ RemoveSimilarTargets($T_p$)
    \EndFor
    
    \State \Comment{Combine targets and remove duplicates}
    \For{each document $d \in D$}
        \If{$d \not \in D_{out}$}
            \State $c \gets$ GetDocumentCluster($d$)
            \State $p \gets$ GetParentCluster($c$)
            \State $T_d \gets T_c \cup T_p$
            \State $T_d \gets$ RemoveSimilarTargets($T_d$)
        \EndIf
        \For{each target $t \in T_d$}
            \State $S_{d,t} \gets$ DetermineStance($d, t$)
        \EndFor
    \EndFor
    
    \State \Return D, T, S
\EndFunction
\end{algorithmic}
\label{alg:cluster_extract}
\end{algorithm}


\section{Implementations}
\label{sec:implementation}

\subsection{POLAR}

We used all of the default parameter settings and models for POLAR for the VAST dataset, but for EZ-STANCE, we reduce the noun phrase clustering threshold from 0.8 to 0.6, as the default value was resulting in no found clusters given that the EZ-STANCE dataset is composed of low word count tweets, which have low entity mention counts.

In adapting this method, we need to extend it by mapping the chosen polarized topics back to the documents, to allow our metrics to be applied to the results. We do so by considering a document to be in a stance target cluster when it features a polarized entity, and the discovered noun phrases as the stance targets.

\subsection{PaCTE}
\label{sec:pacte_implementation}

We train the PaCTE BERT model \cite{devlin2018bert} using the combined training sets from VAST and EZ-STANCE, removing all neutral examples as the original implementation was only trained on partisan news.

We use online latent dirichlet allocation (LDA) \cite{hoffman2010online} as a drop in method speed-up, instead of the original single-core method. Other implementation details are all the same as the original implementation. 

\subsection{WIBA}
\label{sec:wiba_implementation}

We used Llama 3.2 1B \cite{meta2024llama} as the base LLM for our implementation of WIBA, for its trade-off of performance with small size. Training used the combined VAST and EZ-STANCE train/validation sets. On the combined test sets, it achieved a stance detection F1 of 71.5\%, and for stance target extraction it obtained a BERTScore of 90.3\%, comparable with the metrics achieved in the original work.

We replaced the system and instruction tuning tokens with a chat template as appropriate for the Llama model. We used a cosine learning rate with warmup that increments every step \cite{loshchilov2016sgdr}, and NEFTune to improve fine-tuned accuracy \cite{jain2023neftune}. We trained on a 24GB NVIDIA GPU, training took roughly 8 hours.

\subsection{EC}
\label{sec:baseline_implementation}

For diverse generation, we generate 3 return sequences, by exploring 3 beam groups using 6 beams, with a diversity penalty of 10.0. We use a no repeat n-gram size of 2 to prevent repetition. 

We use the \textit{paraphrase-MiniLM-L6-v2} sentence transformer model \cite{reimers-2019-sentence-bert} to embed candidate stance targets, and remove a target from pairs that have a cosine similarity of higher than 0.8.

We run ablation experiments on the number of beam groups and the cosine similarity threshold used for stance target de-duplication in Section \ref{sec:appendix-ablations}.

\subsection{Datasets}

When using VAST as a comparison dataset for the methods, we remove the synthetic neutral examples, as these targets aren't specific for each document. We do however use the synthetic neutral examples to train our stance detection model.

\subsection{Prompts}
\label{sec:prompts}

We include the few-shot prompt used for stance target extraction from topic clusters in Prompt \ref{box:topic-stance-target}:

\begin{simplebox}[Prompt used for extracting stance targets from a topic cluster.]{topic-stance-target}
    \system{You are an expert at analyzing discussions across multiple documents.}
    \human{Your task is to identify a common stance target that multiple documents are expressing opinions about.

        Instructions:
        
        1. Read all provided documents
        
        2. Identify topics that appear across multiple documents
        
        3. Determine if there is a shared target that documents are taking stances on
        
        4. Express the target as a clear noun phrase

        Input:
        
        Documents: [list of texts]

        Output:
        
        Stance target: [noun phrase or "None"]
        
        Reasoning: [2-3 sentences explaining the choice]

        Examples:

        Example 1:
        
        Documents:
        
        "The council's new parking fees are excessive. Downtown businesses will suffer as shoppers avoid the area."
        
        "Increased parking rates will encourage public transit use. This is exactly what our city needs."
        
        "Local restaurant owners report 20\% fewer customers since the parking fee increase."
        
        Output:
        
        Stance target: downtown parking fees
        
        Reasoning: All three documents discuss the impact of new parking fees, though from different angles. The documents show varying stances on this policy change's effects on business and transportation behavior.""",
        
        Example 2:
        
        Documents:
        
        "Beijing saw clear skies yesterday as wind cleared the air."
        "Traffic was unusually light on Monday due to the holiday."
        "New subway line construction continues on schedule."
        
        Output:
        
        Stance target: None
        
        Reasoning: While all documents relate to urban conditions, they discuss different aspects with no common target for stance-taking. The texts are primarily descriptive rather than expressing stances.
        
        Example 3:
        
        Documents:
        
        "AI art tools make creativity accessible to everyone."
        
        "Generated images lack the soul of human-made art."
        
        "Artists demand proper attribution when AI models use their work."
        
        Output:
        
        Stance target: AI-generated art
        
        Reasoning: The documents all address AI's role in art creation, discussing its benefits, limitations, and ethical implications. While covering different aspects, they all take stances on AI's place in artistic creation.
        
        ---
        
        Documents:
        
        \{formatted\_docs\}
    }
    \assistant{Output:
    
        Stance target: 
    }
\end{simplebox}

We include the few-shot prompt used for aggregating stance targets in Prompt \ref{box:aggregate-stance-targets}:

\begin{simplebox}[3-shot in-context prompt for aggregating stance target clusters.]{aggregate-stance-targets}
    \system{You are an expert at analyzing and categorizing topics.}
    \human{Your task is to generate a generalized stance target that best represents a cluster of related specific stance targets.

        Instructions:
        
        1. Review the provided stance targets and keywords that characterize the topic cluster
        
        2. Identify the common theme or broader issue these targets relate to
        
        3. Generate a concise noun phrase that:
        
        - Captures the core concept shared across the targets
        
        - Is general enough to encompass the specific instances
        
        - Is specific enough to be meaningful for stance analysis

        Input:
        
        Representative stance targets: [list of stance targets]
        
        Top keywords: [list of high tf-idf terms]

        Output format:
        
        Generalized target: [noun phrase]
        
        Reasoning: [1-2 sentences explaining why this generalization fits]

        Examples:

        Input:
        
        Representative stance targets: ["vaccine mandates", "mandatory covid shots", "required immunization for schools"]
        
        Top keywords: ["mandatory", "requirement", "public health", "immunization", "vaccination"]
        
        Output:
        
        Generalized target: vaccination requirements
        
        Reasoning: This captures the common theme of mandatory immunization policies while being broad enough to cover various contexts (workplace, school, public spaces).
        
        Input:
        
        Representative stance targets: ["EVs in cities", "gas car phase-out", "zero emission zones"]
        
        Top keywords: ["emissions", "vehicles", "transportation", "electric", "fossil-fuel"]
        
        Output:
        
        Generalized target: vehicle electrification
        
        Reasoning: This encompasses various aspects of transitioning from gas to electric vehicles, including both the technology and policy dimensions.
        
        Input:
        
        Representative stance targets: ["content moderation", "online censorship", "platform guidelines"]
        
        Top keywords: ["social media", "guidelines", "content", "moderation", "posts"]
        
        Output:
        
        Generalized target: social media content control
        
        Reasoning: This captures the broader issue of managing online content while remaining neutral on the specific approach or implementation.
        
        ---
        
        Representative stance targets: \{repr\_docs\}
        
        Top keywords: \{keywords\}}
    \assistant{
    Output:
    
    Generalized target: 
    }
\end{simplebox}

\section{Ablations}
\label{sec:appendix-ablations}

We re-ran EC for varying values of the number of beam generations and cosine similarity threshold for stance target de-duplication to explore the impact it had on method outputs. Figs. \ref{fig:vast-beam-ablation} and \ref{fig:ezstance-beam-ablation} shows that 3 generated beam groups produces consistently the best results on our automated metrics (other the number of beam groups and wall-time being linearly directly correlated), out of 2, 3, and 5 as possible values. Figs. \ref{fig:vast-cosine-sim-ablation} and \ref{fig:ezstance-cosine-sim-ablation} shows that varying the cosine similarity threshold between values of 0.8, 0.9, and 0.95 has minimal effect on the final metrics.

\begin{figure*}
    \centering
    \includegraphics[width=\textwidth]{figs/vast_num_beam_groups_ablation.png}
    \caption{Ablation of the number of beam group generations used for EC for VAST.}
    \label{fig:vast-beam-ablation}
\end{figure*}

\begin{figure*}
    \centering
    \includegraphics[width=\textwidth]{figs/ezstance_num_beam_groups_ablation.png}
    \caption{Ablation of the number of beam group generations used for EC for EZ-STANCE.}
    \label{fig:ezstance-beam-ablation}
\end{figure*}

\begin{figure*}
    \centering
    \includegraphics[width=\textwidth]{figs/vast_cosine_similarity_threshold_ablation.png}
    \caption{Ablation of the cosine similarity threshold used for stance target de-duplication in EC for VAST.}
    \label{fig:vast-cosine-sim-ablation}
\end{figure*}
    
\begin{figure*}
    \centering
    \includegraphics[width=\textwidth]{figs/ezstance_cosine_similarity_threshold_ablation.png}
    \caption{Ablation of the cosine similarity threshold used for stance target de-duplication in EC for EZ-STANCE.}
    \label{fig:ezstance-cosine-sim-ablation}
\end{figure*}

\subsection{Stance Variance}
\label{sec:appendix-variance-ablation}

Our stance variance metric could be optimized to through de-generate solutions. We wanted to determine the effect that solely optimizing for this metric would have on other metrics. We ran an experiment where we kept only stance targets that had over the 75$^{th}$ and 90$^{th}$ percentiles. Keeping only stance targets over the 75$^{th}$ stance variance percentile did not impact the mean stance variance (0\%) change, and reduced stance retrieval F1 by 4.5\%. Keeping only stance targets over the 90$^{th}$ stance variance percentile increased mean stance variance by 33\%, but decreased stance retrieval F1 (-58\%), stance target F1 (-37\%), balanced cluster size (-27\%), and the mean number of targets per document (-53\%).  

\section{Human Evaluation}
\label{sec:human-eval}

Human evaluators were fellow students from the authors' lab.

We provided each annotator this explanatory document prior to their evaluation task to help them understand the concepts in Prompt \ref{box:evaluation-guide}.

\begin{simplebox}[Evaluation Guide]{evaluation-guide}
\textbf{What is a stance target?}
A stance target is a concept that one can have an opinion on. While one can technically have an opinion on almost anything (i.e. one can technically be for or against atoms, but we generally do not consider atoms to be an issue that one is for or against), there are a more constrained set of concepts that we generally put forth opinions, or stances, on.

\textbf{What is a topic?}
In computer science, defined as a set of frequently co-occurring words. More generally, synonymous with a theme, or subject that a document can reference or be about. A document can have multiple topics. It is an abstract concept.

\textbf{Stance Targets}
So for example, the text: ‘I discussed my preference for tariffs over free trade while playing golf today at mar-a-lago’
There are 4 prominent concepts: tariffs, free trade, golf, and mar-a-lago. Two topics for this text would be trade policy (tariffs, free trade), and golf (golf, mar-a-lago), as these are frequently co-occurring words/concepts. The two prominent stance targets are tariffs and free-trade, as they are discussed in the context of having a position on them, and are things that one generally has a stance on. 

Golf could also be considered a stance target in this context, but is discussed with less emphasis on stance. 

Stance targets can also exist at a higher conceptual level. For example, here the author is expressing not only their preference for tariffs, but economic regulation, and protectionism. In this way, the most representative set of stance targets for this text would be ‘tariffs’, ‘free trade’, ‘economic regulations’, and ‘protectionism’

One can discuss a stance target while staying neutral. For example: ‘I read about the idea of tariffs recently. Undecided on whether or not they’re effective’. This author is neutral on the stance target of tariffs.


\textbf{Stance Target Clusters}
Two documents fit in the same stance target cluster, if they discuss the same stance target, whatever the conceptual level of that stance target. The two documents may both be favoring the stance target, on opposing sides of the stance target, or both neutral on the stance target. 

For example, the texts:
‘I think tariffs are a terrible idea’
‘Taxes should be much higher!’
Are not in the same ‘tariffs’ stance target cluster, but are in a stance target cluster: ‘economic regulations’
\end{simplebox}

The exact text prompt given to human evaluators for the stance target cluster comparison task is shown in Prompt \ref{box:cluster-comparison}:

\begin{simplebox}[Stance target cluster comparison prompt]{cluster-comparison}
    Which document discusses a stance target that the base document is also discussing? If both documents discuss completely different stance targets from the base document, choose neither.
\end{simplebox}

To generate triads, for each method and document from both datasets, we randomly sample a document that is a stance target cluster that the base document is also in, and randomly sample a document that is not in any of the same stance target clusters. If the method does not place the base document in a stance target cluster with any other document, then two documents that are not in the same stance cluster are sampled. The order of the two comparison documents is randomly swapped to prevent the chosen document being inferred from the order. We then simply check if the annotator agrees with the method.

The exact text prompt given to human evaluators for the stance target comparison task is shown in Prompt \ref{box:stance-target-comparison}:

\begin{simplebox}[Stance target comparison prompt]{stance-target-comparison}
    Compare the two sets of stance targets, and choose the set that better covers the stance targets the document discusses. If neither sets fit at all, choose neither.
\end{simplebox}

We sample comparisons from the set of all pairwise stance target set comparisons between methods for all documents from both methods. We randomly swap the order of these sets to ensure the same method does not always appear on the same side.

\section{Metrics}
\subsection{Stance Target F1}
\label{sec:target_f1}

For the stance target BERTScore, given a set of documents $D$ where each document $d$ has predicted targets $P_d$ and gold targets $G_d$, we compute the precision, recall and F1 as:
$$P = \frac{1}{|D|} \sum^{D} \frac{\sum^{P_d} \max_{g \in G_d} \text{BERTScore}(p,g)}{|P_d|}$$
$$R = \frac{1}{|D|} \sum^D \frac{\sum^{G_d} \max_{p \in P_d} \text{BERTScore}(g,p)}{|G_d|}$$
$$F1 = \frac{2 \cdot P \cdot R}{P + R}$$

\subsection{Stance Retrieval F1}
\label{sec:stance_f1}

Given a set of documents $D$, where each document $d$ has predicted target-stance pairs $P_d = \{(t, s)\}$, and gold target-stance pairs $G_d = \{(t, s)\}$, where stance can be any of $\{favor, against, neutral\}$.

We define a mapping between predicted stance targets and gold stance targets, where stance targets are only mapped to each other if their sentence embedding cosine similarity is higher than $\theta=0.9$:
$$M = \{(t_p, t_g) : \max_{t' \in G}\text{sim}(t_p, t') \wedge \text{sim}(t_p, t_g) \geq \theta\}$$

For each document $d$, define the set of correct predictions:
$$C_d = \{(t_p, s) \in P_d : \exists (t_g, s) \in G_d, (t_p, t_g) \in M\}$$

Then:
$$\text{P} = \frac{1}{|D|}\sum_{d \in D} \frac{|C_d|}{|P_d|}$$

$$\text{R} = \frac{1}{|D|}\sum_{d \in D} \frac{|C_d|}{|G_d|}$$

$$\text{F1} = \frac{2\text{P}\text{R}}{\text{P} + \text{R}}$$

\subsubsection{Threshold Sensitivity}
\label{sec:appendix-retrieval-sensitivity}

We looked at the sensitivity of our stance retrieval metrics to the chosen cosine similarity parameter, as seen in Fig. \ref{fig:retrieval-sensitivity}. The rankings of the method are robust to varying values of the chosen cosine similarity.

\begin{figure*}[b]
    \centering
    \includegraphics[width=\textwidth]{figs/stance_retrieval_threshold_sensitivity.png}
    \caption{Varying values of the cosine similarity parameter used for calculating stance retrieval against the final value of the metric. Method ranking remains robust to the varying parameter. Shown are the median and quartiles for 5 outputs of each method.}
    \label{fig:retrieval-sensitivity}
\end{figure*}

\section{Case Study Implementation}
\label{sec:case_implementation}

When deploying EC at scale in the case study, we use smaller models: \textit{SmolLM2-360M-Instruct}\footnote{huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct} to generate the base targets, and \textit{SmolLM2-135M-Instruct}\footnote{huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct} to classify stance. Although this makes applying this method to large datasets more tractable, it occasionally results in poor stance targets. This problem is alleviated by using a strong model for the higher-level stance target generation (\textit{huggingface.co/microsoft/Phi-4-mini-instruct}).

\end{document}
