\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{tcolorbox}
\usepackage{booktabs}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{The Counterfactuals at SemEval-2026 Task 9: Can Counterfactually-Inspired Preprocessing help Detect Polarization?}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Teagan Johnson \\
  Department of Computer Science, University of Colorado Boulder \\
  \texttt{teagan.johnson@colorado.edu} \\}
  % Manikandan Gunaseelan \\
  % University of Colorado, Boulder \\
  % \texttt{manikandan.gunaseelan@colorado.edu} \\

%\author{
%  \textbf{First Author\textsuperscript{1}},
%  \textbf{Second Author\textsuperscript{1,2}},
%  \textbf{Third T. Author\textsuperscript{1}},
%  \textbf{Fourth Author\textsuperscript{1}},
%\\
%  \textbf{Fifth Author\textsuperscript{1,2}},
%  \textbf{Sixth Author\textsuperscript{1}},
%  \textbf{Seventh Author\textsuperscript{1}},
%  \textbf{Eighth Author \textsuperscript{1,2,3,4}},
%\\
%  \textbf{Ninth Author\textsuperscript{1}},
%  \textbf{Tenth Author\textsuperscript{1}},
%  \textbf{Eleventh E. Author\textsuperscript{1,2,3,4,5}},
%  \textbf{Twelfth Author\textsuperscript{1}},
%\\
%  \textbf{Thirteenth Author\textsuperscript{3}},
%  \textbf{Fourteenth F. Author\textsuperscript{2,4}},
%  \textbf{Fifteenth Author\textsuperscript{1}},
%  \textbf{Sixteenth Author\textsuperscript{1}},
%\\
%  \textbf{Seventeenth S. Author\textsuperscript{4,5}},
%  \textbf{Eighteenth Author\textsuperscript{3,4}},
%  \textbf{Nineteenth N. Author\textsuperscript{2,5}},
%  \textbf{Twentieth Author\textsuperscript{1}}
%\\
%\\
%  \textsuperscript{1}Affiliation 1,
%  \textsuperscript{2}Affiliation 2,
%  \textsuperscript{3}Affiliation 3,
%  \textsuperscript{4}Affiliation 4,
%  \textsuperscript{5}Affiliation 5
%\\
%  \small{
%    \textbf{Correspondence:} \href{mailto:email@domain}{email@domain}
%  }
%}

\begin{document}
\maketitle
\begin{abstract}
% This document is a supplement to the general instructions for *ACL authors. It contains instructions for using the \LaTeX{} style files for ACL conferences.
% The document itself conforms to its own specifications, and is therefore an example of what your manuscript should look like.
% These instructions should be used both for papers submitted for review and for final versions of accepted papers.
This paper presents the English-language submissions of ``The Counterfactuals" team for the three subtasks of Task 9 at SemEval 2026. The task aims to detect multicultural online polarization, how it is expressed, and in what contexts. The task provides a high-quality annotation dataset of posts that follows a three-level schema: polarized or not (subtask 1), polarization type classification (subtask 2), and manifestation identification (subtask 3). I construct a pointwise mutual information-based lexicon that identifies highly-correlated words with the polarized class as labeled in subtask 1. Using this lexicon, I implement a large language model data augmentation technique. I then use the preprocessed datasets to finetune a BERT model (BERTweet) for each subtask. My highest performing models placed 48th out of 60, 35th out of 36, and 17th out of 24 on subtasks 1, 2, and 3 respectively. All code is available on GitHub.\footnote{https://github.com/johnsont4/semeval-task9-polarization-detection/tree/main}

\end{abstract}

\begin{center}
\textbf{Warning: This paper contains examples that may be offensive or upsetting.}
\end{center}

\section{Introduction}

Online polarization refers to deepening divisions and hostility between social, political, or identity groups \cite{Iyengar2019affective,Garimella2018polarization}. It has become a growing concern, as prior work shows that polarized discourse often precedes or co-occurs with hate speech, toxic language, and broader social conflict \cite{Waller2021context,Mathew2020polarization}. In its extreme form, polarization can produce fragmented online communities where individuals struggle to engage in constructive dialogue, undermining social cohesion and mutual understanding \cite{Tucker2018socialmedia,Williams2015networkpolitics}. Detecting and mitigating polarization before it escalates is therefore crucial for promoting safer and more inclusive online environments. While existing approaches have attempted large-scale detection of polarized or partisan content online, this task has commonly been framed as a coarse binary classification task (polarized or not polarized) \cite{conover2011political,Garimella2016quantifying}. More fine-grained frameworks that identify how polarization is expressed and who it targets could provide greater interpretability and practical value for researchers, platform moderators, and users.
Recently, SemEval-2026 Task 9: Detecting Multilingual, Multicultural, and Multievent Online Polarization \cite{naseem2026polar} has been proposed to encourage research related to detecting online polarized language. By participating in this task, my goal is to identify and categorize the type of polarization in online dialogue.

\begin{figure}[t]
    \begin{small}
    \hfill
    \begin{tcolorbox}[colback=gray!5,colframe=gray!40,title=Spurious Correlation Example]

    \textbf{Polarized Example}\\
    \emph{``They constantly accuse others of violence while using their own people as \textbf{human shields}—it’s pure hypocrisy."}\\[6pt]

    \textbf{Non-polarized Example}\\
    \emph{``Women are not \textbf{human shields} for men."}

    \end{tcolorbox}
    \caption{Example of a spurious correlation: the model predicts both of these examples as polarized even though only the first example is really labeled as polarized.}
    \label{fig:spurious}
    \end{small}
\end{figure}

I initially fine-tuned BERTweet~\citep{nguyen-etal-2020-bertweet} on the annotated training data provided by the task organizers for subtask 1 (more details on the tasks in Section~\ref{sec:task_desc}) to classify polarization \cite{naseem2026polar}. An initial inspection of the fine-tuned model’s output revealed that the presence of certain keywords seemingly disproportionately influenced the classifier’s decisions. In many cases, the model appeared to rely on specific unigrams or bigrams rather than learning the deeper contextual patterns that signal polarization. One example of this behavior can be seen in Figure~\ref{fig:spurious} regarding the spurious term ``human shields." This motivates my effort to reduce the model’s dependence on specific spurious correlations and encourage more robust, context-sensitive representations.

My work is guided by two research questions.
\textbf{(1)} How effective is pointwise mutual information (PMI) for identifying a lexicon of spurious, polarization-inducing tokens?
\textbf{(2)} Can large language models (LLMs) be used to augment the training data in ways that reduce reliance on specific unigrams and bigrams and promote learning of contextual signals?

\section{Related Work}

\paragraph{Political polarization and social media.}
A large body of work studies political polarization in social media, especially on Twitter (now X), finding that polarization is tightly coupled with specific topical and group-related vocabularies~\citep{conover2011political, garimella2017longterm, barbera2015birds}. This suggests that purely lexical models are especially vulnerable to learning spurious correlations between particular terms and polarized labels.

\paragraph{Spurious lexical correlations and identity-term bias.}
Similar issues have been documented in toxic language detection, where hate speech datasets over-represent certain identity terms in the positive class, leading models to conflate mentions of marginalized groups with abusive content~\citep{waseem2016hateful, davidson2017automated}. Dixon et al.~\citep{dixon2018measuring} formalize this as \emph{identity term bias}, showing that toxicity classifiers over-predict toxicity for benign sentences containing terms such as ``gay''. My work treats highly associated terms as potential sources of unintended bias and explicitly targets them during preprocessing.

\paragraph{LLM-based data augmentation.}
Recent work leverages LLMs as generators of synthetic training data for text classification, showing improvements particularly for minority classes, though naive augmentation may introduce distributional shifts that harm robustness~\citep{zhao2024llmaugment, chai2025textaugment}. My counterfactual augmentation procedure fits within this broader paradigm.

\section{Task Description}
\label{sec:task_desc}
SemEval 2026 Task 9 consists of 3 subtasks covering 22 languages. I am participating in all 3 subtasks in English \cite{naseem2026polarbenchmarkmultilingualmulticultural}.

\subsection{Polarization Detection Task}
Subtask 1 is a binary classification task: given a post, determine whether it contains polarized content. Polarized posts display a negative attitude toward outgroups while showing blind support for ingroups, encompassing language that incites division, groupism, hatred, conflict, or intolerance. Label counts are shown in Table~\ref{tab:subtask1_counts}.

\begin{table}[h!]
\centering
\begin{tabular}{cc}
\hline
\textbf{Polarized} & \textbf{Non-Polarized} \\
\hline
1,175 & 2,047 \\
\hline
\end{tabular}
\caption{Counts for the subtask 1 dataset.}
\label{tab:subtask1_counts}
\end{table}

\subsection{Polarization Type Classification Task}
Subtask 2 is a multi-label classification task to identify which of five polarization targets are expressed: political, racial/ethnic, religious, gender/sexual, or other. Each category is an independent binary label, and a post may be associated with multiple targets simultaneously. Label counts are shown in Table~\ref{tab:subtask2_counts}.

\begin{table}[h!]
\centering
\begin{tabular}{lc}
\hline
\textbf{Polarization Type} & \textbf{Count} \\
\hline
Political      & 1,150 \\
Racial/Ethnic  & 281  \\
Religious      & 112  \\
Gender/Sexual  & 72   \\
Other          & 126  \\
\hline
\end{tabular}
\caption{Counts per label in the subtask 2 dataset.}
\label{tab:subtask2_counts}
\end{table}

\subsection{Manifestation Identification Task}
Subtask 3 is a multi-label classification task to identify which of six manifestation types are present: stereotype, vilification, dehumanization, extreme language, lack of empathy, and invalidation. As with subtask 2, each label is independent and multiple labels may apply simultaneously. Label counts are shown in Table~\ref{tab:subtask3_counts}.

\begin{table}[h!]
\centering
\begin{tabular}{lc}
\hline
\textbf{Polarization Manifestation} & \textbf{Count} \\
\hline
Stereotype       & 487 \\
Vilification     & 858 \\
Dehumanization   & 391 \\
Extreme Language & 770 \\
Lack of Empathy  & 357 \\
Invalidation     & 586 \\
\hline
\end{tabular}
\caption{Counts per label in the subtask 3 dataset.}
\label{tab:subtask3_counts}
\end{table}

\section{Methods}
Upon initial examination of the training data, I identified strong correlations between certain words (e.g. Human Shields, Zionist, Leftist) and the polarization classification. These specific words are features that allow the model to classify based on topic-specific words rather than actual indicators of polarization. In prior work these have been referred to as \textit{spurious correlations}. To reduce reliance on these learned shortcuts, I construct a lexicon of spurious tokens using Pointwise Mutual Information (PMI) and introduce LLM-assisted counterfactual data augmentation to explicitly break correlations between spurious tokens and labels.

\subsection{Creating a Spurious Lexicon with PMI}

Pointwise mutual information (PMI) has long been used to quantify word--context association and to construct lexical resources~\citep{church1990word}. To quantify the degree to which words are associated with polarized posts, I compute pointwise mutual information (PMI) between each token $w$ and the polarized class label $y \in \{0,1\}$. PMI measures how much more frequently a word co-occurs with a class than would be expected under independence:

\[
\text{PMI}(w, y) = \log \frac{p(w, y)}{p(w) \, p(y)}.
\]

Probabilities are estimated from frequencies in the training data where $p(w,y)$ signifies the fraction of posts containing $w$ with label $y$, and $p(w)$ and $p(y)$ signify the marginal word and class probabilities. I consider words whose PMI with the polarized class exceeds 0.75 to form my \textit{spurious lexicon}. This threshold was selected to balance lexicon size against noise: I evaluated several candidate thresholds and found that 0.75 yielded a lexicon of reasonable size without admitting too many low-frequency terms. Importantly, I don't manually curate the spurious lexicon at all. I want to evaluate the models' performance relying purely on the lexicon created by PMI. The terms included in the lexicon often encode political identities, geographic entities, or demographic descriptors that correlate with polarization in the dataset but are not in themselves evidence of polarized framing. See Table~\ref{tab:sample_top_pmi_words} for a sample of the top 10 unigrams with the highest PMI and Table~\ref{tab:sample_top_bigrams} for the top 10 bigrams.

I construct a single global lexicon from the subtask 1 binary labels and apply it uniformly across all three subtasks. I chose this design because subtask 1 provides the cleanest signal for spurious correlations: the binary polarized/non-polarized distinction maps directly onto the PMI formulation. Computing separate label-conditional lexicons for subtasks 2 and 3 is complicated by the small per-label instance counts in those multi-label settings (e.g., only 72 Gender/Sexual instances in subtask 2), which would make per-label PMI estimates unreliable. I treat this as a deliberate simplification and acknowledge it as a limitation in the discussion.

\begin{table*}[!ht]
\begin{small}
\centering
\begin{tabular}{lccrr}
\hline
\textbf{Unigram} & \textbf{PMI} & \textbf{Count in Polarized Label} & \textbf{Total Unigram Count} \\
\hline
felon & 1.2728 & 10 & 10 \\
fuck & 1.2728 & 29 & 29 \\
agenda & 1.1732 & 14 & 15 \\
fascist & 1.1658 & 26 & 28 \\
kids & 1.1573 & 12 & 13 \\
let & 1.1573 & 12 & 13 \\
liberal & 1.1415 & 21 & 23 \\
genocidal & 1.1352 & 10 & 11 \\
stolen & 1.1284 & 19 & 21 \\
antisemitism & 1.1207 & 9 & 10 \\
\hline
\end{tabular}
\caption{Top 10 unigram PMI scores associated with polarization. Full spurious unigram lexicon can be seen in Table~\ref{tab:all_top_pmi_words}.}
\label{tab:sample_top_pmi_words}
\end{small}
\end{table*}

\subsection{Finetuning BERTweet Model for Classification}
I used BERTweet, a RoBERTa-based model trained on $\sim$850M English tweets, as a baseline classifier for this task \citep{nguyen-etal-2020-bertweet}. The motivation behind using BERTweet was the similarity of its training corpus and my task data: social media text involves fragmented grammar, slang, sarcasm, and short noisy sequences that BERTweet is specifically adapted to handle. Each model consists of the BERTweet encoder with a linear classification head over the pooled \texttt{[CLS]} representation. For subtask 1 (binary), I use a single output unit with binary cross-entropy loss. For subtasks 2 and 3 (multi-label), I use one output unit per label with independent sigmoid activations and per-label binary cross-entropy loss. I fine-tuned BERTweet for all 3 subtasks, training on 3 different preprocessed datasets per subtask, resulting in 9 fully fine-tuned models in total.

% \subsection{Blindness Preprocessing}

% To reduce the classifier’s dependence on the PMI-derived lexicon, we implement a \textit{blindness} procedure that randomly masks a proportion of lexicon tokens in each training example. Let $L$ denote the lexicon and let $p$ denote the masking probability. For each token $w \in L$ appearing in a training example, we replace it with the sentinel token \texttt{[MASKED\_TOKEN]} with probability $p$.

% Formally, for each input sequence $x = (w_1, \dots, w_n)$, we generate a blinded sequence $\tilde{x}$ via:
% \[
% \tilde{w}_i =
% \begin{cases}
% \texttt{[MASKED\_TOKEN]} & \text{if } w_i \in L \text{ with prob } p\\
% w_i & \text{otherwise}.
% \end{cases}
% \]

% This stochastic masking is a form of data noising. Across epochs, the model is repeatedly exposed to partially blinded versions of the same input, encouraging it to extract polarization-relevant cues from discourse structure, stance, and linguistic framing rather than from topic-identifying tokens.

% For \textbf{subtask 1}, the stochastic process ultimately masks 516 instances with 689 mask tokens added in total. For \textbf{subtask 2}, the stochastic process ultimately masks 512 instances with 691 mask tokens added in total. For \textbf{subtask 3}, the stochastic process ultimately masks 531 instances with 697 mask tokens added in total.

\subsection{LLM-Assisted Counterfactual Augmentation}

To teach the model that the \textit{label should be invariant} to changes in group-specific or topic-specific tokens, I generate counterfactual training examples using a large language model. I use the Llama-3.1-8B-Instruct language model with the following decoding parameters: temperature$=0.8$, top-$p=0.9$, and \texttt{max\_new\_tokens}$=128$.

\textbf{For subtask~1} (binary classification), my augmentation procedure generates both polarized and non--polarized counterfactuals. For each training example labeled as ``Polarized’’ and containing a token from the spurious lexicon, I prompt the LLM to produce six perturbations:
(1) three semantically comparable versions in which the spurious term is replaced by a \textit{different but comparable} group such that the LLM still judges the example to be polarized, and
(2) three semantically altered versions that the LLM considers non--polarizing but which retain the original spurious term.
This design provides the model with balanced evidence that the presence of a spurious token is neither necessary nor sufficient for polarization, which is essential for learning invariance in the binary setting. A concrete example of my augmentation process for subtask 1 is shown in Figure~\ref{fig:llm_example}.

\textbf{For subtasks~2 and~3} (multilabel classification), however, it is substantially more difficult to generate non--polarized counterfactuals. These subtasks involve 5 and 6 labels respectively, and replacing a spurious term does not naturally yield a controlled non-polarized counterpart. As a result, my augmentation for these subtasks only generates \textit{polarized} counterfactuals. Specifically, alternative versions of the input where the spurious term is swapped for another entity while preserving the polarization label. \textbf{This allows the model to observe variation in which entity is referenced, but without the label contrast available in subtask~1.} I return to this in the discussion.

For \textbf{subtask 1}, I ultimately cap the number of augmentations produced to 100 instances, which means the LLM produced 600 total augmentations (300 polarized, 300 non-polarized). For \textbf{subtask 2}, the LLM produced 228 total augmentations. For \textbf{subtask 3}, the LLM produced 228 total augmentations.

Malformed or placeholder outputs from the LLM (an example of which is shown in Figure~\ref{fig:llm_example_mistake}) were identified by checking for template phrases (e.g., ``first new polarized post here'') and discarded automatically. No formal human validation of label correctness was performed; I relied solely on the LLM's own judgment to preserve labels during generation. I acknowledge this as a limitation: label noise in the augmented data may have dampened the effectiveness of the augmentation strategy, and human validation or automatic filtering based on a secondary classifier is left for future work.

\subsection{Blindness Preprocessing}

As a point of comparison and to help with error analysis, I run a second preprocessing step that I call ``blindness" or ``masked" preprocessing. Instead of replacing the identity terms found with the spurious lexicon, I replace the terms with a [BLIND] token 85\% of the time (remaining 15\% of the time the token is unchanged). The 85\% rate was chosen to ensure that spurious tokens are masked in the majority of training examples while retaining some stochasticity, preventing the model from simply learning to rely on the absence of the token.

\section{Results}

For each of the fine-tuned BERT models I use the same hyperparameters which can be found in Table~\ref{tab:bertweet_hparams}. Results tables include a \textbf{POLAR Baseline}, which is the official baseline score provided by the SemEval-2026 Task 9 organizers \cite{naseem2026polar} for comparison. I find the optimal thresholds for each label in subtasks 2 and 3 by performing a simple grid search over thresholds in [0.05, 0.95] (step size 0.05) and choosing, for each label independently, the value that maximizes its binary F1 score. I also split the annotated training data provided by the organizers into train, validation, and test sets, using the same split across all three subtasks (Table~\ref{tab:data_splits}). Full evaluation results on my internal splits are reported in Appendix Tables~\ref{tab:subtask1_metrics}--\ref{tab:subtask3_perclass}.

\subsection{Binary Polarization Detection Task}

I use the LLM-augmented training data to train BERTweet. The organizers provided a train set which I partition into separate train/val/test splits. The f1, precision, and recall scores on my splits are provided in Table~\ref{tab:subtask1_metrics} in the Appendix. The f1 score achieved after uploading my results to the SemEval Task 9 server are shown in Table~\ref{tab:semeval_f1_scores_1}. These f1 scores were calculated from a separate label-blind test set provided by the organizers. All models cluster around 0.77--0.78 macro-F1, indicating that BERTweet is already a strong baseline for binary polarization detection and that the augmentation provided minimal additional gain on the organizer test set.

\begin{table}[ht]
\centering

    \begin{tabular}{lc}
    \hline
    \textbf{Model} & \textbf{SemEval Macro-F1} \\
    \hline
    POLAR Baseline & 0.78 \\
    LLM-Aug BERT & 0.77 \\
    \hline
    \end{tabular}

\caption{Final test set F1 scores obtained from predictions submitted to the organizer-provided evaluation server for \textbf{subtask 1}.}
\label{tab:semeval_f1_scores_1}
\end{table}

\subsection{Polarization Type Classification Task}

I extend the spurious lexicon created from subtask 1 to this subtask. The f1 scores achieved after uploading my results to the SemEval Task 9 server are shown in Table~\ref{tab:semeval_f1_scores_2}. The macro f1 and f1 scores per label are shown in Table~\ref{tab:subtask2_macro} and Table~\ref{tab:subtask2_perclass}, respectively. Both models match the baseline at 0.33 macro-F1, reflecting the difficulty of this task: the severe class imbalance across types (e.g., 1,150 Political vs.\ 72 Gender/Sexual instances) makes it challenging to generalize across all five categories.

\begin{table}[ht]
\centering

    \begin{tabular}{lc}
    \hline
    \textbf{Model} & \textbf{SemEval Macro-F1} \\
    \hline
    POLAR Baseline & 0.33 \\
    LLM-Aug BERT & 0.33 \\
    \hline
    \end{tabular}

\caption{Final test set F1 scores obtained from predictions submitted to the organizer-provided evaluation server for \textbf{subtask 2}.}
\label{tab:semeval_f1_scores_2}
\end{table}


\subsection{Manifestation Identification Task}

I extend the spurious lexicon created from task 1 to this task. The f1 scores achieved after uploading my results to the SemEval Task 9 server are shown in Table~\ref{tab:semeval_f1_scores_3}. The macro f1 and f1 scores per label are displayed in Table~\ref{tab:subtask3_macro} and Table~\ref{tab:subtask3_perclass} respectively. The LLM-augmented model marginally outperforms the POLAR baseline (0.43 vs.\ 0.41), making subtask 3 the only setting where augmentation produced a measurable gain on the organizer evaluation.

\begin{table}[ht]
\centering

    \begin{tabular}{lc}
    \hline
    \textbf{Model} & \textbf{SemEval Macro-F1} \\
    \hline
    POLAR Baseline & 0.41 \\
    LLM-Aug BERT & 0.43 \\
    \hline
    \end{tabular}

\caption{Final test set F1 scores obtained from predictions submitted to the organizer-provided evaluation server for \textbf{subtask 3}.}
\label{tab:semeval_f1_scores_3}
\end{table}

\section{Discussion}
\subsection{Effect of LLM-Assisted Counterfactual Augmentation}
For \textbf{subtask 1}, overall macro-F1 remained almost the same as seen in Table~\ref{tab:semeval_f1_scores_1}. Although my augmentation strategy explicitly created contrastive polarized/non-polarized pairs designed to teach label invariance, two factors likely limited its impact. First, the augmentation volume was small: 600 generations from 100 source instances is modest relative to the 1,175 polarized training examples already available. Second, the dataset's class imbalance (1,175 polarized vs. 2,047 non-polarized; Table~\ref{tab:subtask1_counts}) means the model has access to substantial non-polarized signal already, reducing the marginal benefit of generated non-polarized counterfactuals.

For \textbf{subtask 2}, macro-F1 remained almost the same as baseline as seen in Table~\ref{tab:semeval_f1_scores_2}. Looking at the per-class F1, Table~\ref{tab:subtask2_perclass} shows the LLM-Aug model substantially improved performance on the \textit{Religious} class (0.37 $\rightarrow$ 0.53) and modestly improved \textit{Gender/Sexual} (0.16 $\rightarrow$ 0.22), while performance decreased for \textit{Racial/Ethnic} (0.62 $\rightarrow$ 0.45). A plausible explanation is that religious and gender/sexual terms are lexically entangled with polarized framing across many contexts, so swapping them with comparable terms genuinely teaches invariance. Racial/ethnic terms, by contrast, may be semantically central enough that replacing them introduces distributional noise rather than reducing spurious reliance.

I achieve a performance boost compared to baseline for \textbf{subtask 3}. The LLM-Aug model achieved improvements in specific fine-grained categories as seen in Table~\ref{tab:subtask3_perclass}, notably \textit{Dehumanization} (0.42 $\rightarrow$ 0.47) and \textit{Lack of Empathy} (0.23 $\rightarrow$ 0.24). These categories are characterized more by linguistic stance and framing than by the presence of specific identity tokens, so reducing the model's reliance on particular entities may push it toward learning the deeper discourse cues that actually define these manifestations. This pattern is consistent with the hypothesis that entity-swapping augmentation is most effective for label categories that are structurally rather than lexically defined.

\subsection{Masking vs.\ Augmentation: A Key Contrast}

The divergent effects of masking and augmentation reveal something important about the nature of the spurious correlations in this dataset. Plain masking from the PMI-derived lexicon consistently reduced performance, particularly in fine-grained subtasks. For example, as seen in Table~\ref{tab:subtask2_perclass}, masking dramatically harmed the \textit{Other} category (0.24 $\rightarrow$ 0.07) and reduced \textit{Dehumanization} (0.42 $\rightarrow$ 0.36). These results indicate that many high-PMI tokens are not purely spurious but carry genuine semantic signal for polarization; removing them deprives the model of crucial contextual information. Augmentation, by contrast, keeps the tokens present while varying the labels around them, teaching the model that the token alone is insufficient evidence — a fundamentally different inductive signal than erasure.

This contrast suggests that the right strategy for this task is not to suppress identity-related vocabulary but to diversify the label contexts in which it appears. A key limitation of the PMI lexicon in both settings is that the purely threshold-based approach introduces noise (e.g., function-word N-grams such as ``let'' or ``is just'') and inflates scores for rare terms: several top-ranked unigrams (e.g., ``felon'', ``antisemitism'') appear only 9--10 times in the polarized class (Tables~\ref{tab:sample_top_pmi_words},~\ref{tab:all_top_pmi_words}), making their PMI values statistically fragile. Future work should apply minimum frequency thresholds and linguistic filtering to better distinguish spurious cues from meaningful semantic signals.

\section{Conclusion}

I presented a counterfactually-inspired preprocessing pipeline for polarization detection in English social media posts. Using a PMI-derived spurious lexicon built from subtask 1 binary labels, I generated LLM-assisted counterfactual training examples and applied stochastic token masking as a complementary debiasing strategy, then fine-tuned BERTweet separately for each of the three subtasks. My results show that LLM augmentation produced modest but meaningful gains on subtask 3 (manifestation identification), particularly for fine-grained categories such as Dehumanization and Lack of Empathy, while offering little benefit for the binary and type classification subtasks. Plain masking, by contrast, consistently degraded performance, suggesting that many high-PMI tokens carry genuine semantic signal rather than acting as pure spurious shortcuts. Returning to the research questions: \textbf{(1)} PMI proves effective at surfacing a lexicon of candidate spurious tokens, though it is prone to inflating scores for rare terms and cannot distinguish genuinely spurious cues from semantically central vocabulary — making it a useful but noisy signal. \textbf{(2)} LLM augmentation can reduce reliance on specific tokens, but only partially: gains were concentrated in the most fine-grained subtask and were limited by small augmentation volume and unvalidated label quality. Key limitations include the use of a single global lexicon across subtasks, unvalidated LLM label quality, and small augmentation volume. Future work should explore label-conditional PMI lexicons, frequency-thresholded PMI estimation, and human or automatic validation of augmented examples to better realize the potential of counterfactual data augmentation for polarization detection.

\section{Acknowledgements}

This work utilized the Blanca condo computing resource at the University of Colorado Boulder. Blanca is jointly funded by computing users and the University of Colorado Boulder \cite{ucb_blanca_2021}. I also want to extend my appreciation to Dr. Jim Martin for his support and his instruction during the CSCI-LING 5832 course at the University of Colorado, Boulder.

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

\appendix

\section{Appendix}
\label{sec:appendix}



\begin{table*}[!ht]
\centering
\begin{small}
\begin{tabular}{lccc}
\hline
\textbf{Bigram} & \textbf{PMI} & \textbf{Count in Polarized Label} & \textbf{Total Bigram Count} \\
\hline
liberal agenda & 1.2576 & 11 & 11 \\
stolen election & 1.2576 & 18 & 18 \\
ukrainian nazis & 1.1645 & 15 & 16 \\
is just & 1.1201 & 10 & 11 \\
as human & 1.1056 & 18 & 20 \\
the radical & 1.0650 & 14 & 16 \\
the woke & 1.0650 & 21 & 24 \\
rigged election & 1.0232 & 17 & 20 \\
woke mob & 1.0125 & 27 & 32 \\
human shields & 1.0013 & 36 & 43 \\
\hline
\end{tabular}
\caption{Top 10 bigram PMI scores associated with polarization. Full spurious bigram lexicon can be seen in Table~\ref{tab:all_top_pmi_bigrams}.}
\label{tab:sample_top_bigrams}
\end{small}
\end{table*}

\begin{table}[h!]
\centering
\resizebox{\columnwidth}{!}{
    \begin{tabular}{lccc}
    \hline
    \textbf{Model} & \textbf{Macro F1} & \textbf{Precision} & \textbf{Recall} \\
    \hline
    Base BERT     & \textbf{0.78} & 0.75 & \textbf{0.82} \\
    Masked BERT   & 0.77 & \textbf{0.77} & 0.77 \\
    LLM-Aug BERT  & 0.77 & \textbf{0.77} & 0.77 \\
    \hline
    \end{tabular}
}
\caption{Subtask 1 overall metrics.}
\label{tab:subtask1_metrics}
\end{table}

\begin{table}[!ht]
\centering
\resizebox{\columnwidth}{!}{
    \begin{tabular}{lccc}
    \hline
    \textbf{Model} & \textbf{Macro F1} & \textbf{Precision} & \textbf{Recall} \\
    \hline
    Base BERT     & 0.41 & 0.39 & \textbf{0.76} \\
    LLM-Aug BERT   & \textbf{0.42} & \textbf{0.42} & \textbf{0.65} \\
    Masked BERT  & 0.33 & 0.43 & 0.65 \\
    \hline
    \end{tabular}
}
\caption{Subtask 2 overall metrics.}
\label{tab:subtask2_macro}
\end{table}

\begin{table}[!ht]
\centering
\resizebox{\columnwidth}{!}{
    \begin{tabular}{lccc}
    \hline
    \textbf{Class} & \textbf{Base F1} & \textbf{LLM-Aug F1} & \textbf{Masked F1} \\
    \hline
    Political       & 0.66 & \textbf{0.67} & 0.64 \\
    Racial/Ethnic   & \textbf{0.62} & 0.45 & 0.46 \\
    Religious       & 0.37 & \textbf{0.53} & 0.33 \\
    Gender/Sexual   & 0.16 & \textbf{0.22} & 0.16 \\
    Other           & \textbf{0.24} & \textbf{0.24} & 0.07 \\
    \hline
    \end{tabular}
}
\caption{Subtask 2 per-class F1 scores.}
\label{tab:subtask2_perclass}
\end{table}

\begin{table}[!ht]
\centering
\resizebox{\columnwidth}{!}{
    \begin{tabular}{lccc}
    \hline
    \textbf{Model} & \textbf{Macro F1} & \textbf{Precision} & \textbf{Recall} \\
    \hline
    Base BERT     & \textbf{0.47} & 0.38 & 0.70 \\
    LLM-Aug BERT   & 0.44 & 0.31 & \textbf{0.82} \\
    Masked BERT  & 0.46 & \textbf{0.41} & 0.61 \\
    \hline
    \end{tabular}
}
\caption{Subtask 3 overall metrics.}
\label{tab:subtask3_macro}
\end{table}

\begin{table}[!ht]
\centering
\resizebox{\columnwidth}{!}{
    \begin{tabular}{lccc}
    \hline
    \textbf{Class} & \textbf{Base F1} & \textbf{LLM-Aug F1} & \textbf{Masked F1} \\
    \hline
    Stereotype & 0.46 & 0.42 & \textbf{0.50} \\
    Vilification & \textbf{0.62} & 0.54 & \textbf{0.62} \\
    Dehumanization & 0.42 & \textbf{0.47} & 0.36 \\
    Extreme Language & \textbf{0.58} & \textbf{0.49} & 0.56 \\
    Lack of Empathy & 0.23 & \textbf{0.24} & 0.20 \\
    Invalidation & 0.48 & 0.48 & \textbf{0.50} \\
    \hline
    \end{tabular}
}
\caption{Subtask 3 per-class F1 scores.}
\label{tab:subtask3_perclass}
\end{table}


\begin{figure}[!ht]
    \centering
    \begin{small}
    \begin{tcolorbox}[
        colback=gray!5,
        colframe=gray!40,
        title=Example of LLM-Augmented Training Data,
        left=2mm, right=2mm, top=1mm, bottom=1mm
    ]
    
    \textbf{Original Input:} \\
    I love the smell of \textbf{Christian Nationalism} on New Years Eve blakeshelton
    
    \vspace{3pt}
    \textbf{Polarized Entity-Swapping Augmentations:}
    \begin{itemize}
        \item There's something nauseating about the smell of \textbf{Far Right Fundamentalism} wafting through the air on New Years Eve blakeshelton
        \item The stench of \textbf{Nationalist Populism} is overwhelming on this New Years Eve, a vile reminder of our country's darker side blakeshelton
        \item I hate the smell of \textbf{White Christian Conservatism} on New Years Eve, it's choking our democracy blakeshelton
    \end{itemize}
    
    \textbf{Non-Polarized Entity-Preserving Augmentations:}
    \begin{itemize}
        \item New Year's Eve gatherings often bring people together to share common values and traditions, including those related to \textbf{Christian Nationalism}.
        \item \textbf{Christian Nationalism} has become a significant aspect of American culture, influencing various aspects of society.
        \item The concept of \textbf{Christian Nationalism} is worth exploring, especially during New Year's Eve celebrations.
    \end{itemize}
    
    \end{tcolorbox}
    \caption{
    An example of my LLM augmentation procedure. Given an input containing a spurious entity (\emph{Christian Nationalism}), the model generates (1) \textbf{polarized augmentations} by swapping the entity with semantically comparable alternatives, and (2) \textbf{non-polarized augmentations} that preserve meaning while removing inflammatory framing.
    }
    \label{fig:llm_example}
    \end{small}
\end{figure}

\begin{figure}[!ht]
    \centering
    \begin{small}
    \begin{tcolorbox}[
        colback=gray!5,
        colframe=gray!40,
        title=Example of a Mistake from the LLM,
        left=2mm, right=2mm, top=1mm, bottom=1mm
    ]
    
    \textbf{Original Input:} \\
    ``They" are now the \textbf{deep state}.
    
    \vspace{3pt}
    \textbf{Polarized Entity-Swapping Augmentations:}
    \begin{itemize}
        \item third new polarized post here
        \item second new polarized post here
        \item first new polarized post here
    \end{itemize}
    \end{tcolorbox}
    \caption{
    An example of where the LLM makes a mistake when generating additional polarized examples. The spurious token is ``deep state" (a bigram), but the model only returns placeholder text instead of actual polarization examples.
    }
    \label{fig:llm_example_mistake}
    \end{small}
\end{figure}

\begin{table}[!ht]
    \centering
    \small
    \begin{tabular}{lrr}
    \toprule
    \textbf{Split} & \textbf{Number of Examples} & \textbf{Percentage} \\
    \midrule
    Train      & 2,577 & 80\% \\
    Validation & 332 & 10\% \\
    Test       & 332 & 10\% \\
    \bottomrule
    \end{tabular}
    \caption{Dataset split statistics for all three subtasks.}
    \label{tab:data_splits}
\end{table}

\begin{table*}[!ht]
    \centering
    \small
    \begin{tabular}{@{}lcrr@{}}
    \toprule
    \textbf{Unigram} & \textbf{PMI} & \textbf{Count in Polarized} & \textbf{Total Count} \\
    \midrule
    felon            & 1.2728 & 10 & 10 \\
    fuck             & 1.2728 & 29 & 29 \\
    agenda           & 1.1732 & 14 & 15 \\
    fascist          & 1.1658 & 26 & 28 \\
    kids             & 1.1573 & 12 & 13 \\
    let              & 1.1573 & 12 & 13 \\
    liberal          & 1.1415 & 21 & 23 \\
    genocidal        & 1.1352 & 10 & 11 \\
    stolen           & 1.1284 & 19 & 21 \\
    antisemitism     & 1.1207 & 9  & 10 \\
    full             & 1.1207 & 9  & 10 \\
    rigged           & 1.0711 & 20 & 23 \\
    racism           & 1.0504 & 12 & 14 \\
    hes              & 1.0504 & 18 & 21 \\
    hate             & 1.0504 & 18 & 21 \\
    shields          & 1.0164 & 36 & 43 \\
    nazis            & 1.0097 & 20 & 24 \\
    progressive      & 1.0097 & 10 & 12 \\
    nazi             & 1.0097 & 10 & 12 \\
    traitor          & 0.9971 & 19 & 23 \\
    mob              & 0.9926 & 28 & 34 \\
    musk             & 0.9890 & 23 & 28 \\
    woke             & 0.9873 & 32 & 39 \\
    feel             & 0.9832 & 9  & 11 \\
    literally        & 0.9832 & 9  & 11 \\
    radical          & 0.9773 & 22 & 27 \\
    everyone         & 0.9508 & 12 & 15 \\
    apartheid        & 0.9508 & 40 & 50 \\
    using            & 0.9508 & 12 & 15 \\
    away             & 0.9508 & 12 & 15 \\
    cleansing        & 0.9191 & 36 & 46 \\
    human            & 0.9143 & 39 & 50 \\
    ethnic           & 0.9060 & 38 & 49 \\
    being            & 0.8984 & 27 & 35 \\
    trumpism         & 0.8942 & 20 & 26 \\
    ukrainian        & 0.8894 & 23 & 30 \\
    enemy            & 0.8804 & 16 & 21 \\
    them             & 0.8668 & 40 & 53 \\
    talking          & 0.8577 & 9  & 12 \\
    shit             & 0.8253 & 11 & 15 \\
    fascism          & 0.8253 & 11 & 15 \\
    claim            & 0.8133 & 8  & 11 \\
    facts            & 0.8133 & 8  & 11 \\
    elon             & 0.8033 & 13 & 18 \\
    use              & 0.8033 & 13 & 18 \\
    always           & 0.7988 & 18 & 25 \\
    left             & 0.7947 & 28 & 39 \\
    care             & 0.7873 & 10 & 14 \\
    keep             & 0.7873 & 10 & 14 \\
    something        & 0.7873 & 10 & 14 \\
    \bottomrule
    \end{tabular}
    \caption{All unigrams in the corpus with a PMI greater than 0.75. This ultimately becomes the first part of my spurious lexicon.}
    \label{tab:all_top_pmi_words}
\end{table*}


\begin{table*}[!ht]
    \centering
    \small
    \begin{tabular}{@{}lccc@{}}
    \toprule
    \textbf{Bigram} & \textbf{PMI} & \textbf{Count in Polarized Label} & \textbf{Total Count} \\
    \midrule
    liberal agenda        & 1.26 & 11 & 11 \\
    stolen election       & 1.26 & 18 & 18 \\
    ukrainian nazis       & 1.16 & 15 & 16 \\
    is just               & 1.12 & 10 & 11 \\
    as human              & 1.11 & 18 & 20 \\
    the radical           & 1.06 & 14 & 16 \\
    the woke              & 1.06 & 21 & 24 \\
    rigged election       & 1.02 & 17 & 20 \\
    woke mob              & 1.01 & 27 & 32 \\
    human shields         & 1.00 & 36 & 43 \\
    the deep              & 0.99 & 10 & 12 \\
    when the              & 0.99 & 10 & 12 \\
    apartheid state       & 0.99 & 34 & 41 \\
    is an                 & 0.98 & 14 & 17 \\
    the election          & 0.97 & 9  & 11 \\
    they are              & 0.97 & 18 & 22 \\
    a traitor             & 0.97 & 9  & 11 \\
    voted for             & 0.97 & 9  & 11 \\
    enemy of              & 0.96 & 13 & 16 \\
    are the               & 0.96 & 13 & 16 \\
    radical left          & 0.95 & 17 & 21 \\
    of israel             & 0.94 & 8  & 10 \\
    christian nationalism & 0.94 & 8  & 10 \\
    israel is             & 0.91 & 11 & 14 \\
    ethnic cleansing      & 0.90 & 36 & 46 \\
    trump is              & 0.90 & 18 & 23 \\
    should be             & 0.84 & 12 & 16 \\
    elon musk             & 0.84 & 9  & 12 \\
    and his               & 0.84 & 9  & 12 \\
    deep state            & 0.82 & 17 & 23 \\
    to take               & 0.80 & 8  & 11 \\
    an apartheid          & 0.77 & 15 & 21 \\
    the people            & 0.76 & 17 & 24 \\
    defund the            & 0.76 & 12 & 17 \\
    \bottomrule
    \end{tabular}
    \caption{All bigrams in the corpus with a PMI greater than 0.75. This ultimately becomes the second part of my spurious lexicon.}
    \label{tab:all_top_pmi_bigrams}
\end{table*}

\begin{figure*}[!ht]
    \centering
    \small
    \begin{verbatim}
    We are studying polarization in social media posts.

    - A "polarized" post clearly expresses a strong, divisive attitude or opinion,
      usually framing an "us vs them" dynamic, strongly supporting one side and/or
      attacking another.
    - Here, all original posts you will see are labeled as POLARIZED (1).
    
    You will be given a social media post that is labeled as POLARIZED (1).
    It expresses a strong, divisive attitude.

    Original post:
    {text}

    Groups or entities mentioned in the original post:
    [{mentions_str}]

    Your task:
    - Generate 3 new posts that:
      1. Remain clearly POLARIZED (strong, divisive opinion).
      2. Keep a similar *type* of stance (e.g., critical, supportive, mocking),
         but you must change the specific groups, parties, countries, or movements
         mentioned to DIFFERENT BUT SOCIALLY COMPARABLE ones.
         For example:
         - If the original attacks "Democrats", you might attack "Republicans" or "liberals".
         - If the original talks about one country, you may switch to another country
           in a similar geopolitical context.
      3. The intensity of polarization should remain similar (don't make it neutral).
      4. Do NOT copy the original sentences; produce 
            genuinely rephrased posts following the same syntactical structure.
      5. Do NOT mention exactly the same groups/entities as in the original.
      6. Do NOT add explanations; only return JSON.
    \end{verbatim}
    \caption{Prompt used for \textbf{polarized} LLM-based data augmentation for subtask 1.}
    \label{fig:llm_prompt_1_polarized}
\end{figure*}

\begin{figure*}[!ht]
    \centering
    \small
    \begin{verbatim}
    We are studying polarization in social media posts.

    - A "polarized" post clearly expresses a strong, divisive attitude or opinion,
      usually framing an "us vs them" dynamic, strongly supporting one side and/or
      attacking another.
    - Here, all original posts you will see are labeled as POLARIZED (1).

    You will be given a social media post that is labeled as POLARIZED (1),
    meaning it expresses a strong, divisive attitude.

    Original post:
    {text}

    Groups or entities mentioned in the original post:
    [{mentions_str}]

    Your task:
    - Generate 3 new posts that:
      1. Are clearly NON-POLARIZED (neutral, balanced, or mildly opinionated).
      2. Should NOT frame a strong "us vs them" conflict or use aggressive, hostile language.
      3. Stay roughly on the same topic as the original.
      4. Mention the groups found ([{mentions_str}]),
         but only in a descriptive, informational, or balanced way.
      5. Do NOT copy the original sentences; 
            produce genuinely rephrased posts that may have syntactically different structure.
      6. Do NOT add explanations; only return JSON.
    \end{verbatim}
    \caption{Prompt used for \textbf{non-polarized} LLM-based data augmentation for subtask 1.}
    \label{fig:llm_prompt_1_non_polarized}
\end{figure*}

\begin{figure*}[!ht]
    \centering
    \small
    \begin{verbatim}
    "You generate counterfactual training data for polarization type classification.\n"
    "Given a sentence, you rewrite it so that it preserves the same meaning and "
    "polarization type, but replaces references to specific countries, political groups, "
    "or identity groups with different but comparable groups.\n\n"
    "STRICT OUTPUT RULES:\n"
    "- Return ONLY the rewritten sentence.\n"
    "- Do NOT provide any explanation or commentary.\n"
    "- Do NOT number or bullet the output.\n"
    "- Output MUST be a single line of text.\n"
    \end{verbatim}
    \caption{Prompt used for LLM-based data augmentation for subtask 2.}
    \label{fig:llm_prompt_2}
\end{figure*}

\begin{figure*}[!ht]
    \centering
    \small
    \begin{verbatim}
    "You generate counterfactual training data for polarization type classification.\n"
    "Given a sentence, you rewrite it so that it preserves the same meaning and "
    "polarization type, but replaces references to specific countries, political groups, "
    "or identity groups with different but comparable groups.\n\n"
    "STRICT OUTPUT RULES:\n"
    "- Return ONLY the rewritten sentence.\n"
    "- Do NOT provide any explanation or commentary.\n"
    "- Do NOT number or bullet the output.\n"
    "- Output MUST be a single line of text.\n"
    \end{verbatim}
    \caption{Prompt used for LLM-based data augmentation for subtask 3.}
    \label{fig:llm_prompt_3}
\end{figure*}

\begin{table*}[!ht]
\centering
\small
\begin{tabular}{ll}
\hline
\textbf{Setting} & \textbf{Value} \\
\hline
Base model & \texttt{vinai/bertweet-base} \\
Max sequence length & 128 \\
Train batch size & 8 \\
Eval batch size & 8 \\
Number of epochs & 3 \\
Threshold during training metrics & 0.5 (per label) \\
Evaluation strategy & \texttt{epoch} \\
Checkpoint saving & \texttt{save\_strategy=no} (final model only) \\
Logging frequency & every 50 training steps \\
Metrics & macro precision, macro recall, macro F1 \\
\hline
\end{tabular}
\caption{Model architecture and training hyperparameters for BERTweet fine-tuning on all three subtasks.}
\label{tab:bertweet_hparams}
\end{table*}


\begin{figure}[!htbp]
    \centering
    \includegraphics[width=\columnwidth]{latex/visuals/train_val_loss_base_model_1.png}
    \caption{Loss curves for the base BERT model for subtask 1.}
    \label{fig:loss_curves_base_1}
\end{figure}
\begin{figure}[!htbp]
    \centering
    \includegraphics[width=\columnwidth]{latex/visuals/train_val_loss_masked_model_1.png}
    \caption{Loss curves for the masked BERT model for subtask 1.}
    \label{fig:loss_curves_masked_1}
\end{figure}
\begin{figure}[!htbp]
    \centering
    \includegraphics[width=\columnwidth]{latex/visuals/train_val_loss_llm_aug_model_1.png}
    \caption{Loss curves for the LLM-augmented BERT model for subtask 1.}
    \label{fig:loss_curves_llm_aug_1}
\end{figure}

\begin{figure}[!htbp]
    \centering
    \includegraphics[width=\columnwidth]{latex/visuals/train_val_loss_base_model_2.png}
    \caption{Loss curves for the base BERT model for subtask 2.}
    \label{fig:loss_curves_base_2}
\end{figure}
\begin{figure}[!htbp]
    \centering
    \includegraphics[width=\columnwidth]{latex/visuals/train_val_loss_masked_model_2.png}
    \caption{Loss curves for the masked BERT model for subtask 2.}
    \label{fig:loss_curves_masked_2}
\end{figure}
\begin{figure}[!htbp]
    \centering
    \includegraphics[width=\columnwidth]{latex/visuals/train_val_loss_llm_aug_model_2.png}
    \caption{Loss curves for the LLM-augmented BERT model for subtask 2.}
    \label{fig:loss_curves_llm_aug_2}
\end{figure}

\begin{figure}[!htbp]
    \centering
    \includegraphics[width=\columnwidth]{latex/visuals/train_val_loss_base_model_3.png}
    \caption{Loss curves for the base BERT model for subtask 3.}
    \label{fig:loss_curves_base_3}
\end{figure}
\begin{figure}[!htbp]
    \centering
    \includegraphics[width=\columnwidth]{latex/visuals/train_val_loss_masked_model_3.png}
    \caption{Loss curves for the masked BERT model for subtask 3.}
    \label{fig:loss_curves_masked_3}
\end{figure}
\begin{figure}[!htbp]
    \centering
    \includegraphics[width=\columnwidth]{latex/visuals/train_val_loss_llm_aug_model_3.png}
    \caption{Loss curves for the LLM-augmented BERT model for subtask 3.}
    \label{fig:loss_curves_llm_aug_3}
\end{figure}

\end{document}
