\def\year{2024}\relax
%File: formatting-instructions-latex-2022.tex
%release 2022.1
\documentclass[letterpaper]{article} % DO NOT CHANGE THIS
\usepackage{aaai22}  % DO NOT CHANGE THIS
\usepackage{times}  % DO NOT CHANGE THIS
\usepackage{helvet}  % DO NOT CHANGE THIS
\usepackage{courier}  % DO NOT CHANGE THIS
\usepackage[hyphens]{url}  % DO NOT CHANGE THIS
\usepackage{graphicx} % DO NOT CHANGE THIS
\urlstyle{rm} % DO NOT CHANGE THIS
\def\UrlFont{\rm}  % DO NOT CHANGE THIS
\usepackage{natbib}  % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\usepackage{caption} % DO NOT CHANGE THIS AND DO NOT ADD ANY OPTIONS TO IT
\DeclareCaptionStyle{ruled}{labelfont=normalfont,labelsep=colon,strut=off} % DO NOT CHANGE THIS
\frenchspacing  % DO NOT CHANGE THIS
\setlength{\pdfpagewidth}{8.5in}  % DO NOT CHANGE THIS
\setlength{\pdfpageheight}{11in}  % DO NOT CHANGE THIS
%
% These are recommended to typeset algorithms but not required. See the subsubsection on algorithms. Remove them if you don't have algorithms in your paper.
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{xcolor}
%\usepackage{emoji}
\usepackage{color, colortbl}
%\definecolor{name}{system}{definition}

\definecolor{Gray}{gray}{0.9}

\definecolor{LightCyan}{rgb}{0.88,1,1}
% Checklist macros

\newcommand{\answerYes}[1]{\textcolor{blue}{#1}} 
\newcommand{\answerNo}[1]{\textcolor{teal}{#1}} 
\newcommand{\answerNA}[1]{\textcolor{gray}{#1}} 
\newcommand{\answerTODO}[1]{\textcolor{red}{#1}} 
%
% These are are recommended to typeset listings but not required. See the subsubsection on listing. Remove this block if you don't have listings in your paper.
\usepackage{newfloat}
\usepackage{listings}
\lstset{%
	basicstyle={\footnotesize\ttfamily},% footnotesize acceptable for monospace
	numbers=left,numberstyle=\footnotesize,xleftmargin=2em,% show line numbers, remove this entire line if you don't want the numbers.
	aboveskip=0pt,belowskip=0pt,%
	showstringspaces=false,tabsize=2,breaklines=true}
\floatstyle{ruled}
\newfloat{listing}{tb}{lst}{}
\floatname{listing}{Listing}

\usepackage{booktabs}
\usepackage{multirow}
%
%\nocopyright
%
% PDF Info Is REQUIRED.
% For /Title, write your title in Mixed Case.
% Don't use accents or commands. Retain the parentheses.
% For /Author, add all authors within the parentheses,
% separated by commas. No accents, special characters
% or commands are allowed.
% Keep the /TemplateVersion tag as is
\pdfinfo{
/Title (AAAI Press Formatting Instructions for Authors Using LaTeX -- A Guide)
/Author (AAAI Press Staff, Pater Patel Schneider, Sunil Issar, J. Scott Penberthy, George Ferguson, Hans Guesgen, Francisco Cruz, Marc Pujol-Gonzalez)
/TemplateVersion (2022.1)
}

% DISALLOWED PACKAGES
% \usepackage{authblk} -- This package is specifically forbidden
% \usepackage{balance} -- This package is specifically forbidden
% \usepackage{color (if used in text)
% \usepackage{CJK} -- This package is specifically forbidden
% \usepackage{float} -- This package is specifically forbidden
% \usepackage{flushend} -- This package is specifically forbidden
% \usepackage{fontenc} -- This package is specifically forbidden
% \usepackage{fullpage} -- This package is specifically forbidden
% \usepackage{geometry} -- This package is specifically forbidden
% \usepackage{grffile} -- This package is specifically forbidden
% \usepackage{hyperref} -- This package is specifically forbidden
% \usepackage{navigator} -- This package is specifically forbidden
% (or any other package that embeds links such as navigator or hyperref)
% \indentfirst} -- This package is specifically forbidden
% \layout} -- This package is specifically forbidden
% \multicol} -- This package is specifically forbidden
% \nameref} -- This package is specifically forbidden
% \usepackage{savetrees} -- This package is specifically forbidden
% \usepackage{setspace} -- This package is specifically forbidden
% \usepackage{stfloats} -- This package is specifically forbidden
% \usepackage{tabu} -- This package is specifically forbidden
% \usepackage{titlesec} -- This package is specifically forbidden
% \usepackage{tocbibind} -- This package is specifically forbidden
% \usepackage{ulem} -- This package is specifically forbidden
% \usepackage{wrapfig} -- This package is specifically forbidden
% DISALLOWED COMMANDS
% \nocopyright -- Your paper will not be published if you use this command
% \addtolength -- This command may not be used
% \balance -- This command may not be used
% \baselinestretch -- Your paper will not be published if you use this command
% \clearpage -- No page breaks of any kind may be used for the final version of your paper
% \columnsep -- This command may not be used
% \newpage -- No page breaks of any kind may be used for the final version of your paper
% \pagebreak -- No page breaks of any kind may be used for the final version of your paperr
% \pagestyle -- This command may not be used
% \tiny -- This is not an acceptable font size.
% \vspace{- -- No negative value may be used in proximity of a caption, figure, table, section, subsection, subsubsection, or reference
% \vskip{- -- No negative value may be used to alter spacing above or below a caption, figure, table, section, subsection, subsubsection, or reference

\setcounter{secnumdepth}{0} %May be changed to 1 or 2 if section numbers are desired.

% The file aaai22.sty is the style file for AAAI Press
% proceedings, working notes, and technical reports.
%

% Title

% Your title must be in mixed case, not sentence case.
% That means all verbs (including short verbs like be, is, using,and go),
% nouns, adverbs, adjectives should be capitalized, including both words in hyphenated terms, while
% articles, conjunctions, and prepositions are lower case unless they
% directly follow a colon or long dash
\title{Hostility Detection in UK Politics: A Dataset on Online Abuse Targeting MPs}
\author {
    % Authors
    Mugdha Pandya,
    Mali Jin, 
    Kalina Bontcheva,
    Diana Maynard    
}
\affiliations {
    % Affiliations
    Department of Computer Science, University of Sheffield\\
    \{mugdha.pandya, m.jin, k.bontcheva, d.maynard\}@sheffield.ac.uk
}
% \author{
%     %Authors
%     % All authors must be in the same font size and format.
%     Written by AAAI Press Staff\textsuperscript{\rm 1}\thanks{With help from the AAAI Publications Committee.}\\
%     AAAI Style Contributions by Pater Patel Schneider,
%     Sunil Issar,\\
%     J. Scott Penberthy,
%     George Ferguson,
%     Hans Guesgen,
%     Francisco Cruz\equalcontrib,
%     Marc Pujol-Gonzalez\equalcontrib
% }
% \affiliations{
%     %Afiliations
%     \textsuperscript{\rm 1}Association for the Advancement of Artificial Intelligence\\
%     % If you have multiple authors and multiple affiliations
%     % use superscripts in text and roman font to identify them.
%     % For example,

%     % Sunil Issar, \textsuperscript{\rm 2}
%     % J. Scott Penberthy, \textsuperscript{\rm 3}
%     % George Ferguson,\textsuperscript{\rm 4}
%     % Hans Guesgen, \textsuperscript{\rm 5}.
%     % Note that the comma should be placed BEFORE the superscript for optimum readability

%     2275 East Bayshore Road, Suite 160\\
%     Palo Alto, California 94303\\
%     % email address must be in roman text type, not monospace or sans serif
%     publications22@aaai.org
% %
% % See more examples next
% }

%Example, Single Author, - \textgreater \textgreater remove \iffalse,\fi and place them surrounding AAAI title to use it
\iffalse
\title{My Publication Title --- Single Author}
\author {
    Author Name
}
\affiliations{
    Affiliation\\
    Affiliation Line 2\\
    name@example.com
}
\fi

\iffalse
%Example, Multiple Authors, - \textgreater \textgreater remove \iffalse,\fi and place them surrounding AAAI title to use it
\title{My Publication Title --- Multiple Authors}
\author {
    % Authors
    First Author Name,\textsuperscript{\rm 1}
    Second Author Name, \textsuperscript{\rm 2}
    Third Author Name \textsuperscript{\rm 1}
}
\affiliations {
    % Affiliations
    \textsuperscript{\rm 1} Affiliation 1\\
    \textsuperscript{\rm 2} Affiliation 2\\
    firstAuthor@affiliation1.com, secondAuthor@affilation2.com, thirdAuthor@affiliation1.com
}
\fi


% REMOVE THIS: bibentry
% This is only needed to show inline citations in the guidelines document. You should not need it and can safely delete it.
\usepackage{bibentry}
% END REMOVE bibentry

\begin{document}

\maketitle

\begin{abstract}
%Numerous politicians use social media platforms, particularly X, to engage with their constituents. This interaction allows constituents to pose questions and offer feedback but also exposes politicians to a barrage of hostile responses, especially given the anonymity afforded by social media. They are typically targeted in relation to their governmental role, but the comments also tend to attack their personal identity. 
%This can discredit politicians and reduce public trust in the government. It can also incite anger and disrespect, leading to offline harm and violence.
%While numerous models exist for detecting hostility in general, they lack the specificity required for political contexts. Furthermore, addressing hostility towards politicians demands tailored approaches due to the distinct language and issues inherent to each country (e.g., Brexit for the UK).
%To bridge this gap, we construct a dataset of 3,320 English tweets spanning a two-year period manually annotated for hostility towards UK MPs. Our dataset also captures the targeted identity characteristics (race, gender, religion, none) in hostile tweets. We perform linguistic and topical analyses to delve into the unique content of the UK political data. 
%Finally, we evaluate the performance of pre-trained language models and large language models on binary hostility detection and multi-class targeted identity type classification tasks. Our study offers valuable data and insights for future research on the prevalence and nature of politics-related hostility specific to the UK.


Social media platforms, particularly X, enable direct interaction between politicians and constituents but also expose politicians to hostile responses targetting both their governmental role and personal identity. This online hostility can undermine public trust and potentially incite offline violence. While general hostility detection models exist, they lack the specificity needed for political contexts and country-specific issues. We address this gap by creating a dataset of 3,320 English tweets directed at UK MPs over two years, annotated for hostility and targeted identity characteristics (race, gender, religion). Through linguistic and topical analyses, we examine the unique features of UK political discourse and evaluate pre-trained language models and large language models on binary hostility detection and multi-class targeted identity type classification tasks. Our work provides essential data and insights for studying politics-related hostility in the UK.
\end{abstract}


\section{Introduction}
With the rise of social media use among politicians, especially on X, there has been an increase in direct interaction with the public \cite{agarwal2019tweeting}. This interaction, while beneficial for communication and feedback, also exposes politicians to a significant amount of hostile replies due to the anonymity of online platforms \cite{solovev2022hate}. Such hostility is considered a major concern as it erodes public trust in political processes and institutions, which disrupts constructive communication \cite{gross2023online}. Furthermore, it affects the personal lives and mental health of politicians, with online abuse sometimes leading to real-world threats and violence \cite{enock2023tracking}. In extreme cases, sustained hostility has driven some politicians to step down from their roles and retreat from public life altogether \cite{scott2019women}.

Hostility targeting politicians is a global phenomenon characterised by widespread misogyny, sexism, and racism. %Political and social scientists use surveys, interviews, and extensive quantitative and qualitative social media data analysis to investigate it \cite{haakansson2024explaining,collignon2021increasing}.%,scott2019women}. 
Political and social science research indicates that all politicians receive hostility, but those from minority groups (e.g., Black, female, LGBTQ+) often face increased hostility based on their identity characteristics \cite{carson2024online}.%,hua2020characterizing}.
%, the underlying reasons for these attacks, and the profiles of the perpetrators \cite{hua2020characterizing,haakansson2024explaining,carson2024online}.

In NLP, sentiment analysis tools have been used to identify negative tweets and facilitate studies on abuse trends \cite{hua2020characterizing,ward2020turds}. Although general hostility detection is prevalent, identifying political hostility requires specialised approaches 
%This is due to the fact that 
as political discussions often reflect a country's unique linguistic and cultural characteristics, incorporating regional colloquialism, profanity and prejudices. For example, hostility towards people of colour is more prevalent in the US \cite{lavalley2022occupation}, while the phenomenon of 
Islamophobia is more severe in India \cite{amarasingam2022fight}. Furthermore, hostile posts are frequently tied to popular issues at that time. %For instance, recent hostility towards politicians has largely centred around %Brexit and illegal immigration in the UK \cite{goodman2024supporting}.%\cite{burnett2017racial}.

As the body of work on hate speech, abuse and hostility detection in NLP grows \cite{jahan2023systematic}, there has been a move towards developing resources specifically for political hate speech detection across different countries \cite{grimminger2021hate,guellil2020detecting, jafri2023uncovering}. In the UK, Members of Parliament (MPs) represent a wide range of backgrounds, and this diversity is mirrored in the nature of the abusive comments they receive \cite{gorrell2020politicians}. 
Studies have compiled datasets to analyse abuse trends specific to UK politics \cite{southern2021twitter,bakir2024abuse,gorrell2018twits}. %As discussed in the related work section, most of these datasets lack hostility-related labels. Among them, only two datasets include labels suitable for automatic political hostility detection, but they do not take into account identity characteristics \cite{agarwal2021hate, ward2020turds}. A third dataset was created in the context of UK political hostility and used to train a classifier, but with a focus on detecting Islamophobia alone \cite{vidgen2020detecting}.
While existing political datasets are available, only two contain hostility-related labels suitable for automated detection \cite{agarwal2021hate, ward2020turds}, and one focuses solely on Islamophobia in UK politics \cite{vidgen2020detecting}. None comprehensively capture identity-based hostility.

We aim to bridge this gap by constructing a high-quality hostility dataset spanning a two-year period to cover diverse political topics in the UK. Our main contributions are:

\begin{itemize}
    \item A publicly available dataset specific to political hostility towards UK MPs, consisting of 3,320 tweets with expert annotations for hostility and the targeted identity characteristics (race, gender, religion, none), including individual annotations with confidence scores and gold labels;\footnote{Dataset is available at \url{https://zenodo.org/records/10809695}}

    \item In-depth linguistic and topical analyses to identify linguistic patterns and trending topics in the dataset;

    \item Demonstrating its usefulness for political hostility detection by evaluating pre-trained language models (PLMs) and large language models (LLMs) on binary hostility classification and multi-class  identity type classification. %to  over regular hsd datasets/models
\end{itemize}



%Our work distinguishes itself from others by creating a dataset specifically designed for training models to automatically detect political hostility towards UK politicians based on targeted identity characteristics. it enables us to demonstrate through topic analysis that political hostility is closely tied to contemporaneous events. While this is unsurprising, it nevertheless has important ramifications for training models \cite{jin2023examining}. Linguistic analysis and dataset statistics reveal that the governing political party receives the most hostility proportionally, with race-based hostility being the most prominent among the identity characteristics studied. The extended two-year data collection period of our dataset thus provides a broader range of topics than existing datasets, improving both topic diversity and the generalisability of classifiers trained on this data \cite{jin2023examining}. Critically, our dataset includes labels for identity characteristics and their combinations, as intersectional abuse is a significant and particularly damaging feature of online hostility \cite{kuperberg2018intersectional,kuperberg2021incongruous}.

Our work is distinctive in creating a dataset specifically for training models to detect identity-based political hostility towards UK MPs. Through topic analysis, we show how political hostility correlates with current events, which has crucial implications for model training \cite{jin2023examining}. Our analysis reveals that the governing party faces proportionally more hostility, with race-based attacks being most prevalent. The dataset's two-year span offers greater topic diversity and generalisability than existing datasets \cite{jin2023examining}, while uniquely capturing intersectional abuse through identity characteristic labels—a particularly harmful form of online hostility \cite{kuperberg2018intersectional,kuperberg2021incongruous}.
%the importance of intersectionality has been shown in previous work \cite{kuperberg2018intersectional,kuperberg2021incongruous}.

%Thus, the main contributions of this paper are (i) a publicly available dataset specific to political hostility towards UK MPs consisting of 3,320 tweets with expert annotations for hostility and identity characteristics, including individual annotations with confidence scores and gold labels;\footnote{Dataset available at \url{https://anonymous.4open.science/r/ohtukmp-21D8}} (ii) 2 sets of transformer-based models - flat classification and hierarchical classification - that have been fine-tuned on this data; and (iii) in-depth linguistic and topical analyses to identify linguistic patterns and trending topics in our hostility dataset towards UK MPs.



%In this work, we develop an English hostility dataset specific to UK MPs which consists of 3,320 tweets spanning over two years. These tweets are manually annotated with hostility labels (hostile, not hostile) as well as the targeted identity type labels (race, gender, religion, none) in hostile tweets. We conduct extensive linguistic and topical analyses to identify linguistic patterns and trending topics in our hostility dataset towards UK MPs. 
%Our findings indicate that the governing political party receives the most hostility, with race-based hostility being the most prominent among the identity characteristics studied. Finally, we examine the performance of hostility detection models using both flat and two-level hierarchical classification approaches.

%Our work differs from \cite{agarwal2021hate} in that we generate high-quality labels manually annotated by experts. While their data collection only spans 2 months, ours spans two years. Hostility is quite dependent on the topic so short durations have an effect on the generalisability of the dataset. Our work differs from \cite{vidgen2020detecting} because it looks at multiple dimensions of hostility. Based on the findings in \cite{kuperberg2018intersectional, kuperberg2021incongruous}, hostility is due to different identity characteristics and their intersection. In our dataset, we try to capture intersectionality by allowing annotators to select multiple identity characteristic labels and then train models on this data. 
%Our work differs from the existing political hostility detection in the UK in that we generate high-quality labels manually annotated by experts. The previous datasets' data collection periods are short and focused on topics like elections, COVID, Islamophobia, etc. Political hostility is highly correlated with the goings-on at the time of collection, as is revealed through our topic analysis. Previous work does not have the same breadth of topics due to a longer data collection period (2 years), which affects the generalisability of the dataset. Based on the findings in \cite{kuperberg2018intersectional,kuperberg2021incongruous}, hostility is not always one dimensional i.e., gender-based hostility is not always independent of race and intersectional hostility is common in UK politics. To this end, our dataset also contains labels for identity characteristics and their intersection.
%Our work distinguishes itself from others through the generation of high-quality labels by experts created for the specific purpose of creating training material to develop tools for the automatic detection of political hostility in the UK. Through our topic analysis, we demonstrate that political hostility is closely tied to contemporaneous events. The extended two-year data collection period of our dataset thus provides a broader range of topics, enhancing its generalisability. Furthermore, our dataset includes labels for identity characteristics and their intersection, as the importance of intersectionality has been shown in previous work \cite{kuperberg2018intersectional,kuperberg2021incongruous}.
%Furthermore, according to findings by Kuperberg et al. \cite{kuperberg2018intersectional,kuperberg2021incongruous}, hostility is often multidimensional. Therefore, our dataset includes labels for identity characteristics and their intersections.

%Thus, the main contributions of this paper are (i) a publicly available dataset with expert annotations for hostility and identity characteristics, including individual annotations with confidence scores and gold labels; (ii) 2 sets of transformer-based models that have been fine-tuned on this data; and (iii) in-depth linguistic and topical analyses to better understand the hostility towards UK MPs.



\section{Related Work}

\subsection{Online Hostility}

%The rise in social media usage has resulted in a growing amount of hostility \cite{walther2022social,macavaney2019hate}. Consequently, this has prompted NLP research into different hostility detection tasks such as \cite{mansur2023twitter, jahan2023systematic} hate speech, abuse, toxicity, offence, trolling, cyberbullying, etc \cite{pavlopoulos2020toxicity,mathew2021hatexplain}. Hate speech and abuse datasets have labels for hate, abuse, and offence with additional labels for the targeted groups like gender and race \cite{zampieri2019semeval,basile2019semeval}. Toxicity and cyberbullying datasets have labels such as harassment, aggression, toxic, etc. \cite{rosa2019automatic,hartvigsen2022toxigen}
%However, the definitions of these tasks are very similar \cite{fortuna2020toxic,basile2019semeval}, making it challenging for annotation and for comparison of datasets \cite{zampieri2019semeval, waseem2017understanding}
%To avoid this problem, we combine the definitions of these terms in NLP literature into one umbrella term - hostile. While lots of work has been done to combat general hostility detection issues on different social media like Gab, Reddit, X (formerly Twitter), etc, \cite{jahan2023systematic,mollas2022ethos,rieger2021assessing} the problem of the specificity of political hostility data (i.e., language, topic, country) needs more specialised research. 


The rise in social media usage has led to growing hostility \cite{walther2022social,macavaney2019hate}, spurring NLP research into online hostility tasks \cite{mansur2023twitter, jahan2023systematic} like detecting hate speech, abuse, toxicity, and cyberbullying \cite{pavlopoulos2020toxicity,mathew2021hatexplain}. While existing datasets include labels for targeted groups and various forms of harassment \cite{rosa2019automatic,hartvigsen2022toxigen}, overlapping definitions complicate annotation and dataset comparison \cite{fortuna2020toxic,waseem2017understanding}. We address this by using ``hostile'' as an umbrella term. Though general hostility detection has been studied across social media platforms like Gab, Reddit, X (formerly Twitter), etc, \cite{,mollas2022ethos,rieger2021assessing}, political hostility requires specialised research due to the distinct characteristics of the data (e.g. language, topic, country).


\subsection{Online Hostility towards Politicians}
%Existing work in political hostility typically focuses on qualitative insights or analysis of summary statistics. This work is widespread, and there seems to be an overarching theme of sexism, racism and religious hostility. For instance, female politicians face negative sentiments and attitudes in Japan \cite{fuchs2021normalizing}; in the US, people of colour from the Democratic party and female politicians receive disproportionate hate \cite{solovev2022hate,grimminger2021hate, hua2020characterizing}; MPs in the UK face substantial racial and gender-based abuse \cite{bakir2024abuse, kuperberg2018intersectional}. 

%Datasets and machine learning models to detect political hate speech have been created for different countries and their corresponding languages (Arabic in Algeria \cite{guellil2020detecting}, Chinese in Taiwan \cite{wang2022political}, Hindi in India \cite{jafri2023uncovering}). While these datasets can be used to detect country-specific political hate speech, they do not take identity characteristics into account despite their prominence in political hate speech.% correlation between identity characteristics and political hate speech. % not focus on the role of politicians' identity characteristics in political hostility.


Existing work in this kind of data typically focuses on qualitative insights or analysis of summary statistics, revealing overarching themes of sexism, racism and religious hostility. Studies document gender-based hostility in Japan \cite{fuchs2021normalizing}, disproportionate hate towards Democratic politicians of colour and women in the US \cite{solovev2022hate,grimminger2021hate,hua2020characterizing}, and racial and gender-based abuse of UK MPs \cite{bakir2024abuse,kuperberg2018intersectional}. While country-specific political hate speech detection models exist (Arabic in Algeria \cite{guellil2020detecting}, Chinese in Taiwan \cite{wang2022political}, Hindi in India \cite{jafri2023uncovering}), they typically overlook identity characteristics despite their prominence in political hate.% speech.



\subsection{UK-Specific Hostility towards MPs}
%In the UK, political hostility has been studied based on topics and identity characteristics. \citet{bakir2024abuse} and \citet{farrell2021mp} found that abuse towards MPs was at an all-time high during the first year of the COVID-19 pandemic and that women MPs, especially from non-white backgrounds, received higher abuse. 
%\citet{gorrell2019race} investigated trends in racial and religious abuse towards MPs around Brexit and also abuse trends leading up to the 2015, 2017 \cite{gorrell2018twits} and 2019 \cite{gorrell2020politicians} General Elections. They found that prominence, Parliamentary events, and MP identity characteristics correlated with receiving abuse. 

%A large body of work has focused on gender-based hostility showing that the hostility female MPs face is often in the form of othering, belittling, discrediting, and stereotyping. For example, female electoral candidates' lower success rates were correlated with gender-based harassment \cite{collignon2021increasing}; gender stereotypes and misogyny are reinforced on YouTube through hateful videos and comments \cite{esposito2021dare}; female MPs face more incivility, including stereotyping and questioning credibility than male MPs \cite{southern2021twitter}. However, gender alone is not the only dimension where MPs face hostility. Rather, it is intertwined with other identity characteristics such as age, class, race, and religious beliefs. \cite{kuperberg2021incongruous,esposito2022gender}. 
In the UK, studies of political hostility have examined both topics and identity characteristics. \citet{bakir2024abuse} and \citet{farrell2021mp} found abuse towards MPs peaked during the first year of COVID-19, with women MPs, particularly those from non-white backgrounds, receiving higher levels of abuse.
\citet{gorrell2019race} examined racial and religious abuse trends towards MPs relating to Brexit, along with abuse patterns before the 2015, 2017 \cite{gorrell2018twits} and 2019 \cite{gorrell2020politicians} General Elections. Their research revealed correlations between abuse and MPs' prominence, Parliamentary events, and identity characteristics.
Research on gender-based hostility shows female MPs face othering, belittling, discrediting, and stereotyping. Gender-based harassment correlates with lower success rates for female electoral candidates \cite{collignon2021increasing}, while YouTube reinforces gender stereotypes and misogyny through hateful videos and comments \cite{esposito2021dare}. Female MPs encounter more incivility, including stereotyping and credibility challenges, than their male counterparts \cite{southern2021twitter}. Gender intersects with other identity characteristics—age, class, race, and religious beliefs—in shaping hostility towards MPs \cite{kuperberg2021incongruous,esposito2022gender}.

\subsection{Existing Datasets for UK Political Hostility}
%Despite so much awareness about political hostility in the UK, only a small amount of work has been done in developing NLP datasets and models specifically for its automatic detection. To the best of our knowledge, there are only 3 datasets that are suitable for this task. Details of these are in Table \ref{tab:datasets}. 
Despite widespread awareness of UK political hostility, few NLP datasets and models exist for its automatic detection. To the best of our knowledge, only 3 suitable datasets are currently available, detailed in Table \ref{tab:datasets}.
%\citet{agarwal2021hate} created a dataset of 2.5 million tweets collected over 2 months. They used 18 existing social media hate speech classifiers to generate binary hate labels and then analysed topic and MP characteristic trends in political hate speech. However, these hate speech classifiers were not trained specifically on political hate speech data.
%\citet{vidgen2020detecting} created a dataset and classifier to detect Islamophobia in a political hate speech context. The dataset consists of 4000 expert-annotated tweets collected over 1.5 years. While this study uses political social media data, the labels pertain specifically to Islamophobia rather than to hostility.
%\citet{ward2020turds} used sentiment analysis to collect negative tweets from which they created a dataset of 3000 tweets collected over 2.5 months manually annotated for hate and abuse. They used this data to study abuse trends and found that other than identity characteristics, one of the main causes of abuse was reacting to political topics and issues.
\citet{agarwal2021hate} compiled 2.5 million tweets over 2 months, using 18 existing social media hate speech classifiers to generate binary hate labels and analyse topic and MP characteristic trends in political hate speech. The hate speech classifiers, however, were not trained on political hate speech data.
\citet{vidgen2020detecting} developed a dataset and classifier for detecting Islamophobia in political contexts, comprising 4000 expert-annotated tweets collected over 1.5 years. While using political social media data, the labels only specifically address Islamophobia rather than general hostility.
\citet{ward2020turds} examined abuse trends by collecting 3000 negative tweets over 2.5 months through sentiment analysis, manually annotating them for hate and abuse. Their findings showed abuse stemmed from both identity characteristics and reactions to political topics and issues.


%The work presented in this paper differs from existing datasets as it is specifically designed to facilitate the automatic detection of political hostility in the UK, focusing on multiple identity characteristics. Unlike the limited suitable existing datasets, our data collection spans two years, covering a broad range of topics over an extended period. This timeframe is crucial for creating classifiers that generalise more effectively \cite{jin2023examining}. Additionally, we use the dataset to show some preliminary findings about the nature of this hostility, as well as methods to identify it best.
Our work differs by specifically targetting the automatic detection of UK political hostility across multiple identity characteristics. Unlike existing datasets, our two-year collection period covers diverse topics over an extended timeframe, enabling more effective classifier generalisation \cite{jin2023examining}. Additionally, we use the dataset to show some preliminary findings about the nature of this hostility, as well as best methods to identify it.

%that have shorter data collection periods and focus on specific topics like elections \cite{gorrell2018twits}, COVID \cite{bakir2024abuse}, and Islamophobia \cite{vidgen2020detecting}

%Furthermore, according to findings by Kuperberg et al. \cite{kuperberg2018intersectional,kuperberg2021incongruous}, hostility is often multidimensional; for example, gender-based hostility can intersect with race-based hostility \cite{kwarteng2022misogynoir}, and intersectional hostility is common in UK politics. Therefore, our dataset includes labels for identity characteristics and their intersections. We also use the dataset to show some preliminary findings about the nature of this hostility, as well as methods to best identify it.

%distinguishes itself from other data and tools for understanding political hostility in the UK through the generation of high-quality labels manually annotated by experts created for the specific purpose of creating training material for the development of tools for the automatic detection of political hostility in the UK. Previous datasets have shorter data collection periods and focus on specific topics like elections \cite{gorrell2018twits}, COVID \cite{bakir2024abuse}, and Islamophobia \cite{vidgen2020detecting}. Through our topic analysis, we demonstrate that political hostility is closely tied to contemporaneous events. The extended two-year data collection period of our dataset thus provides a broader range of topics, enhancing its generalisability. Furthermore, according to findings by Kuperberg et al. \cite{kuperberg2018intersectional,kuperberg2021incongruous}, hostility is often multidimensional; for example, gender-based hostility can intersect with race-based hostility \cite{kwarteng2022misogynoir}, and intersectional hostility is common in UK politics. Therefore, our dataset includes labels for identity characteristics and their intersections. We also use the dataset to show some preliminary findings about the nature of this hostility, as well as methods to best identify it.

\begin{table*}[t!]
    \centering
    
    \small
    \begin{tabular}{|c|c|c|c|}
    \toprule
         \bf{Dataset} & \bf{Time} & \bf{Tweets} & \bf{Labels} \\\midrule
         Agarwal et al. (2021) & 1 Oct 2017 - 29 Nov 2017 & 2.5 M & hate, not hate\\\bottomrule
         Vidgen et al. (2020) & Jan 2017 - June 2018 & 4000 & none, weak islamophobia, strong islamophobia\\\bottomrule
         Ward et al. (2020) & 14 Nov 2016 - 28 Jan 2017 & 3000 & non-abusive, not-directed, abusive, hate-speech\\\bottomrule
         %Gorrell et al. (2018) & 6 May 2015 - 6 June 2015 & 1.5 M &\\
         %& 7 April 2017 - 7 May 2017& & \\\bottomrule
         \textbf{Our dataset} & \textbf{Nov 2020 - Dec 2022} & \textbf{3320} & \textbf{hostile-none, hostile-religion, hostile-gender, hostile-race, non-hostile}\\
         %&&&\bf{hostile-race, non-hostile}\\
    \bottomrule
    \end{tabular}
    \caption{Datasets for automatic UK political hostility detection.}
    \label{tab:datasets}
\end{table*}
    

%only a small amount of work has been done in developing high-quality datasets and models specifically for automatically detecting such abuse and hostility. To the best of our knowledge, there are only 4 datasets that serve this purpose. \cite{agarwal2021hate}, which is the most similar to ours, develops a dataset of 2.5 million tweets with metadata information and synthetic labels. They collect entire threads of conversations to maximise the context. For each tweet, they add labels for the identity characteristics of the MPs mentioned and topic labels based on the hashtags. Interestingly they also include characteristics such as political leaning of other users associated with the tweet (i.e, mentioned in the tweet, author, etc.). They use 18 different hate speech classifiers and choose the hate/not-hate label based on the majority label. Then, based on this, they look at topic trends in hateful and not-hateful tweets and also trends of when MPs receive hate and on what basis. \cite{vidgen2020detecting} develops a dataset and classifier to detect the severity/obviousness of islamophobia in a political hate speech context. Their dataset, consisting of 4000 tweets, was manually annotated by experts in UK politics and prejudice. While this study uses political social media data, it focuses only on hostility in one dimension: Islamophobia. \cite{gorrell2018twits} and also \cite{ward2020turds} looked at targets and triggers. They performed sentiment analysis to get 3000 negative tweets which they then manually annotate with hate and abuse labels, and then further labeled for target MP. They found that while political affiliation and gender play a role, the main reason for targeting MPs was reactionary (in response to parliamentary discussions).


\section{Data}
We develop our dataset in 3 steps: data collection, data sampling, and annotation.

%\begin{enumerate}
%    \item Data Collection: a set of tweets (C) related to UK MPs was collected via the Twitter streaming API in real-time by following the accounts of UK MPs. 
%    \item Data Sampling: A subset of MPs was chosen from C, and a set of tweets (S) %was sampled for annotation.
%    \item Data Annotation: The sampled set S was annotated to generate labels.
%\end{enumerate}

\subsection{Data Collection}
Following the method used by \citet{bakir2024abuse}, the Twitter (now X) Streaming API was used to follow the accounts of all MPs (568) with active X accounts. We collected four types of tweets relating to each MP between November 2020 and December 2022: the original tweets posted by the MPs, replies to them, retweets of tweets posted by them and retweets created by them. This collection contains over 30 million tweets, which we denote as ${C}$. 


\begin{table}[t!]
\centering
    
    \small
    \begin{tabular}{|c|c|c|c|c|}
    \toprule
    \bf{Party}&  \bf{Conservative}&  \bf{Labour}&  \bf{SNP}& \bf{Total}\\
    \midrule    
         Female&  6&  4&  1 & 11 \\ \bottomrule 
         Male& 3 & 4 &  0& 7 \\ \bottomrule% \bottomrule
         Non-white& 7 & 4 & 1 & 12\\ \bottomrule 
         White& 2 & 4 & 0 & 6\\ \bottomrule% \bottomrule
         Not Christian& 5 &  2&  1& 8\\ \bottomrule 
         Christian&4  & 6 & 0 & 10\\ \bottomrule
         %Total&  9&  8& 1 & 18\\
    %\bottomrule
    \end{tabular}
    \caption{Statistics of MP identity characteristics and parties.}
    \label{tab:dataset}
\end{table}

\subsection{Data Sampling}
The sheer volume of tweets in ${C}$ makes manual annotation infeasible. Therefore, we sample a subset of ${C}$ for the annotation task covering diverse time periods and topics, which we denote as ${S}$. We employ the following sampling steps:

\begin{itemize}
    \item We choose a \textbf{subset of 18 MPs} to ensure an annotated dataset with a diverse representation of identities and political affiliations. The MPs are selected to ensure the overall pool includes both minority (race: non-White; gender: female; religion: non-Christian) and majority identity groups (race: White; gender: male; religion: Christian).\footnote{The MPs' identity characteristics are based on self-declared public information.} The selected MPs are from the Conservative Party (9 MPs), the Labour Party (8 MPs) and the Scottish National Party (1 MP). Table~\ref{tab:dataset} presents the distribution of identities and parties of these 18 MPs.
    %Information about the MPs, including their political affiliation and position held at the time of data collection, can be found in Table~\ref{tab:mps} in Appendix~\ref{app:dataset}       
    \item A \textbf{long temporal span} was ensured by sampling tweets from the 5 highest posting activity days for each MP, which occur in ${C}$. 
    \item We exclude duplicate tweets and use an abusive language classifier from \citet{gorrell2020politicians} to identify \textbf{hostility} of all 2.54M individual tweets. For each of the 5 days, we sample 17 hostile and 20 non-hostile tweets. Therefore, there are potentially 85 hostile and 100 non-hostile tweets per MP, which are then manually annotated.
\end{itemize}
In total, ${S}$ contains 3,330 tweets in English. 
\subsection{Data Annotation}
%The first step in data annotation is defining the labels and setting up the annotation guidelines.
The data annotation process consists of defining the guidelines, performing the annotation task and quality control.

\subsubsection{Annotation Guidelines}
To address the challenge of differentiating between the closely related concepts such as hate, abuse and toxicity, we combined the definitions of these terms from NLP literature into an umbrella term, hostile (see Table~\ref{tab:definitions}). We revised the definitions multiple times to strike a balance between the prescriptive and descriptive paradigm \cite{rottger2021two}. The former enforces definitions and rules that annotators must abide to, while the latter provides guidelines that allow annotators to apply their own understanding. To this end, small focus groups were conducted to discuss the rigidity of the label definitions.

\begin{table*}[t!]
    
    \small
    \begin{tabular}{|l|p{0.4\linewidth}|p{0.4\linewidth}|}
    \toprule
    \bf{Label}&\bf{Definition} &\bf{Example}\\
    \midrule   
    \textbf{Hostile} &  \textbf{Hostility towards a target group or individual. Intended to be derogatory, abusive, threatening, humiliating, inciting violence or hatred towards an individual/members of the group.}&\textit{\textless USER \textgreater and \textless USER \textgreater Put back on your leash were you? There’s a good boy}\\
    \hspace{4mm}Race & \hspace{4mm}Hostility directed at a person/group based on racial background/ethnicity. Including discrimination based on somatic traits (e.g. skin colour), origin, cultural traits, language, nationality, etc..& \hspace{4mm}\textit{\textless USER \textgreater Your in England speak bloody ENGLISH!}\\
    \hspace{4mm}Gender & \hspace{4mm}Hostility directed at a person/group based on their gender. Including negative stereotyping, objectification, using gendered slurs to insult, and threats of a sexual nature. & \hspace{4mm}\textit{\textless USER \textgreater If you can't stand the heat get the hell out of the kitchen next time elect a man to be prime minister, Liz Truss just showed us there are things women can't do.}\\
    \hspace{4mm}Religion & \hspace{4mm}Hostility directed at a person/group based on their religious beliefs. including misrepresenting the truth and criticism of a religious group without a well-founded argument. & \hspace{4mm}\textit{\textless USER \textgreater sick of you tweeting about muslims or any other religion. Your silence speaks the same bullshit, but its ok as Ramadan is over?!?!}\\
    \hspace{4mm}None & \hspace{4mm}Do not refer to gender, race/ethnicity or religion.& \hspace{4mm}\textit{\textless USER \textgreater is the worst human being. I wish someone would shoot her}\\\bottomrule
    \textbf{Not hostile} & \textbf{Posts that are not hostile. A tweet containing profanity is not hostile unless its context makes it so.} & \textit{\textless USER \textgreater will make a bad PM. Don’t turn this into a race war. Please notice that he is a terrible politician}\\\bottomrule
    %None & if the posts are not hostile & @Conservatives and @Jeremy\_Hunt and having the \\
    %&&highest level of taxation in over 70 years. You’ve lost my vote!\\         
    %\bottomrule
    \end{tabular}
    \caption{Hostility taxonomy with targeted identity type definitions and examples.}
    \label{tab:definitions}
    \end{table*}

%Each message was labelled as hostile or not hostile. Further, each hostile tweet was labelled according to targeted identity characteristics - religion, gender, and race. A ``none'' label was also provided when tweets were hostile but did not target any of the identity characteristics in our study. For cases where tweets were targeting multiple identity characteristics (\textit{``No no no and no. After 2 disastrous female PMs the last thing we need is a Paki!''}), annotators were encouraged to select as many identity characteristics as applicable. Label definitions with examples are given in Table~\ref{tab:definitions}. Annotators were also required to provide a confidence score on a scale of 1 to 5, from very low confidence to extreme confidence. If confidence was lower than 3, annotators also had to leave a comment explaining their decision. Commenting was optional for higher confidence scores. Confidence score information can be found in Table~\ref{tab:confidence} in the Appendix.

We consider political hostility detection as a hierarchical classification task. Given a tweet $t$, the aim is to classify $t$ based on hostility (binary classification) and the target identity characteristics (multiclass classification). We formulate the task in a hierarchical manner similar to existing datasets like OffensEval \cite{zampieri2019semeval} and HatEval \cite{basile2019semeval}. First, $t$ is classified into two hostility labels: hostile and not hostile. If $t$ is classified as hostile, then it will be further classified into one of the four target identity characteristic labels: religion, gender, race and none. Table~\ref{tab:definitions} shows the definitions of each category and example tweets. Note that hostility can be intersectional (i.e., target multiple identity characteristics simultaneously), so a tweet can have more than one identity label. To provide a measure of reliability of each annotation, we include a confidence score of 1 to 5, from very low confidence to extreme confidence for both hostility and identity characteristic labels. %Table~\ref{tab:confidence} in the appendix presents the confidence scores with explanations.


%\begin{table*}[t!]
%
%    \small
%\begin{tabular}{|c|l|l|}
%    \toprule
%    \bf{Score} & \bf{Meaning} & \bf{Explanation}\\
%    \midrule
%        5 & Extremely confident & I'm certain without a doubt.  \\
%        4 & Fairly confident & I’m confident, but there might be a small chance other annotators may label it as a different category. \\
%        %& & annotators may label it as a different category \\
%        3 & Pretty confident & I’m pretty sure, but there might be a high chance other annotators may label it as a different category. \\
%        %& & annotators may label it as a different category \\
%        2 & Not confident & I’m not sure; it could belong to this or another category. \\
%        1 & Very low confidence & I’m really unsure; it might belong to another category instead.\\
%        \bottomrule
%    \end{tabular}
%    \caption{Confidence Scores used in the annotation task.}
%    \label{tab:confidence}
%\end{table*}
\begin{figure}[t!]
  \centering
  \includegraphics[width=0.9\linewidth,height=0.9\linewidth]{figures/interface_new.png}
  \caption{Annotation platform user interface.}
  %\Description{example of the annotation task UI}
  \label{fig:UI}
\end{figure}
\subsubsection{Annotation Method}
The annotation task was conducted in three steps: training, testing, and annotation. Steps 1 and 2 ensured high-quality annotations. The entire annotation process was conducted using the collaborative web-based annotation tool Teamware 2 \cite{wilby2023gate}.

\begin{enumerate}
    \item \textbf{Training sessions}: Training sessions were conducted in which annotators received in-person presentations explaining label definitions with detailed examples. Annotators were also guided on setting up their annotator account and familiarising themselves with the platform.
    \item \textbf{Testing sessions}: Each annotator then underwent a short test to ensure a proper understanding of the task and guidelines, consisting of 20 tweets covering all the labels. Annotators were required to label at least 14 of the 20 annotations correctly. Finally, annotators were provided with the correct answers as well as explanations. 
    \item \textbf{Annotation}: Once annotators passed the test, they were added to the actual annotation task. % and were shown tweets sequentially. 
    Figure~\ref{fig:UI} shows the platform user interface. 
\end{enumerate}

\subsubsection{Annotation Task Quality}
A number of steps were taken to ensure high-quality manual annotations. Annotators were recruited from postgraduate courses in Politics and Computer Science. The only prerequisite was that they had to be familiar with UK politics and colloquialisms. We placed no restriction on age, gender, ethnicity, etc. so as to not bias the labels. We contacted potential annotators by emailing the respective course groups. Each annotator was paid 30 GBP for the annotation of 200 tweets. We recruited a total of 48 annotators. Each tweet in ${S}$ is labelled by 3 annotators. 

During the task, annotators were instructed to look up unfamiliar terms and slang. Each annotator was allowed to annotate only 200 tweets in total, and the task did not need to be completed in one sitting. This allowed annotators to take breaks and prevented them from getting overly desensitised to the hostile content. 

A manual analysis of the annotation results established that some annotators had incorrectly confused the race and religion labels in cases where Muslims and Jews were being targeted. Therefore, expert annotators corrected this small number of annotations.

\subsection{Dataset}
The fully annotated dataset consists of 3,320 tweets, after removing posts that contain URLs or user mentions only. We use 3 sets of gold labels for our modelling experiments:
\begin{itemize}
    \item \textbf{Set 1:} The gold labels were assigned based on the majority vote, i.e. the label which has at least 2 annotations out of 3. For the cases where multiple identity labels were chosen (intersectional), two of the authors manually assigned the dominant label on consensus. For example, the dominant label of ``\textit{\textless USER \textgreater Are you allowed to wear a dress? I thought your `community' didn’t like women dressing how they wanted?}'' is Religion, which was annotated with Religion and Gender labels. 
    \item \textbf{Set 2:} Annotations with confidence \textless 3 were removed, and gold labels are derived from the remaining annotations. In the cases where only one annotation remained for a tweet, that was selected as the gold label. When there were 2 annotations, the annotation with the higher confidence was selected. If both had equal confidence, two of the authors manually assigned the gold label on consensus. If all 3 annotations remained, we used the majority vote method as used in Set 1. Note that there is no case where all confidence scores of three annotations are \textless 3. Additionally, annotations with intersectional labels have lower confidence scores than those with one label, so there are no intersectional labels in Set 2.
    \item \textbf{Set 3:} To investigate intersectionality in the data and model performance, we used the same method as Set 2 for high-level hostility labels (hostile or not hostile). For the lower-level identity labels, if there was an intersectional label with confidence \textgreater 2, we chose that as the gold label. We had no cases of different intersectional labels with confidence \textgreater 2. Also, there were no cases where an intersectional label included three different identity labels simultaneously. As a result, a hostile tweet would be annotated with one of the following labels: \{Religion, Gender, Race, None, [Religion, Gender], [Religion, Race], [Gender, Race]\}.
\end{itemize}
\begin{table}[t!]
    \small
    \centering
    \begin{tabular}{|l|l|c|c|c|}
    \toprule
    \bf{Hostility}&\bf{Identity} & \bf{Set 1} & \bf{Set 2} & \bf{Set 3}\\
    \midrule
    \multirow{ 5}{*}{Hostile}&Religion & 36 & 41 &52 (26)\\
    &Gender & 108 & 119 & 119 (22)\\
    &Race & 188 & 182&205 (38)\\
    &None & 1135 & 1112&1121 (0)\\
    &Total & 1467 & 1454 & 1454 (43)\\\bottomrule
    Not Hostile & Total & 1853 & 1866 & 1866\\\bottomrule
    \bottomrule
    \multirow{ 2}{*}{Fleiss' $\kappa$} & Hostility &0.68 &0.79 & 0.79\\
    & Identity & 0.51 & 0.65 & 0.47\\\bottomrule
    \end{tabular}
    \caption{Label counts for each set. For Set 3, the value in parentheses shows the count of identity-based hostility that comes from intersectional labels.}
    \label{tab:hostile}    
\end{table}

%\begin{table}[t!]
%    \caption{Numbers and percentages of hostile tweets mentioning MPs from each party for Set 1 and Set 2/Set 3.}
%    \label{tab:party}
%    \centering
%    \begin{tabular}{|c|c|c|}
%    \toprule
%    %\bf{Party}&\multicolumn{2}{c|}{\bf{Set 1}} & \multicolumn{2}{c|}{\bf{Set 2}}\\
%    \bf{Party} & \bf{Set 1} & \bf{Set 2/Set 3}\\
%    \midrule
%    
    %& count & \% & count & \% \\\bottomrule
%     Conservative & 706 (48.13\%) &696 (47.87\%)\\\bottomrule
%     Labour & 618 (42.13\%) & 612 (42.09\%) \\\bottomrule
%     SNP & 89 (6.07\%) & 88 (6.05\%) \\\bottomrule
%   \end{tabular}
    
%\end{table}




Table~\ref{tab:hostile} shows the statistics of each set. The top 6 rows present the frequency of each label for each set. Non-hostile tweets are predominant, followed by no identity and race-based hostile tweets. Set 3 includes the 43 intersectional labels, of which 5 target religion and gender, 21 religion and race, and 17 gender and race. The bottom 2 rows present the Fleiss' $\kappa$ annotator agreement score \cite{fleiss1971measuring} for hostility and target identity annotation. Set 2 exhibits the highest $\kappa$-value for both hostility (0.79) and identity (0.65) annotation, indicating substantial agreement \cite{artstein2008inter}. This suggests selecting annotations based on confidence scores helps to improve the quality of the dataset. %The differences in the amount and type of hostility MPs receive based on the political party and identity groups they belong to can be found in the appendix. 



%A more detailed look at the data shows that for set 1, 1467 (44.19\%) are hostile, while 1853 (55.81\%) are not hostile. Of the hostile tweets, 332 (22.63\%) target an identity characteristic. For set 2 and set 3, 1454 (43.79\%) are hostile, while 1866 (56.21\%) are not hostile. In set 2, of the hostile tweets, 342 (23.52\%) target an identity characteristic. In set 3, of the hostile tweets, 333 (22.90\%) target an identity characteristic. Of these 333, 43 are intersectional, with 5 (1.50\%) targeting both religion and gender, 21 (6.31\%) targeting both religion and race and  17 (5.11\%) targeting both gender and race.

%We calculate the Fleiss' kappa ($\kappa$) to measure the inter-annotator agreement. For Set 1, the $\kappa$ for hostility labels is 0.68, while the $\kappa$ for identity characteristic labels is 0.51. For set 2, we see that hostility-label $\kappa$ increases to 0.89 and the identity-characteristic-label $\kappa$ increases to 0.70. For set 3, $\kappa$ for hostility labels remains 0.89 (same as set 2) but identity-characteristics-label $\kappa$ drops to 0.47. 


%Table \ref{tab:party} presents the numbers and percentages of hostile tweets towards UK MPs from each party for Set 1 and Set 2. We observe that the Conservative Party receives more hostility than the Labour Party and the SNP. In Set 1, 48.13\% of the hostile tweets are directed at MPs from the Conservative Party, while only 42.13\% and 6.07\% are directed at MPs from the Labour Party and the SNP, respectively. In Set 2 and Set 3, 47.87\% of the hostile tweets are directed at MPs from the Conservative Party, while only 42.09\% and 6.05\% are directed at MPs from the Labour Party and the SNP, respectively. It is important to note at this point that due to only one MP in our study belonging to SNP, we have a very small percentage of tweets representing them in the data. 

Figures~\ref{fig:stats_party} and~\ref{fig:stats} show the %differences in the 
amount and type of hostile tweets MPs receive based on their political party and identity group. The horizontal pink (Figure~\ref{fig:stats_party}) and black (Figure~\ref{fig:stats}) lines mark the mean value for each group. 
%they belong to, respectively. %MPs belonging to the Conservative party receive more hostility in general (Figure~\ref{fig:stats_party}). 
On average, Conservative MPs receive more race-based hostility. %, but there are some MPs from the Labour Party who receive more race-based hostility (e.g. David Lammy). 
For gender and religion-based hostility, on average, MPs from both parties receive similar levels. However, there are some Labour MPs who receive more identity-based hostility than others (e.g. Diane Abbott, David Lammy). Due to only one SNP MP in our study, we do not include SNP in this comparison.  

In Figure~\ref{fig:stats}, as expected, we see that while male (M) MPs receive more hostile tweets, %(including non-identity related),
female (F) MPs are disproportionately subjected to gender-based hostility. % compared to their male counterparts. 
Similarly, non-white (NW) and non-Christian (NC) MPs face significantly higher levels of general, race- and religion-based hostility. Interestingly, we see that MPs from racial and religious minority groups consistently receive more general hostility and identity-based hostility (consistently higher mean values for all types of hostile tweets), than their white (W) or Christian (C) counterparts. This highlights the issues of intersectional hostility \cite{kwarteng2022misogynoir}, where individuals belonging to multiple minority groups experience compounded forms of discrimination and harassment.%This highlights the issues of intersectional hostility \cite{kwarteng2022misogynoir} wherein different minority groups intersect with and reinforce each other. %For example, white women and coloured women do not receive the same amount of gender-based hostility. The intersection of race and gender leads to coloured women being the target of hostility more so than white women.  
\begin{figure}
    \centering
    \includegraphics[width=0.9\linewidth,height=0.9\linewidth]{figures/stats_party.pdf}
    \caption{Comparing political party-based differences in the amount and type of hostility received}
    \label{fig:stats_party}
\end{figure}
\begin{figure}
    \centering
    \includegraphics[width=0.9\linewidth,height=0.9\linewidth]{figures/stats.pdf}
    \caption{Comparing identity-based differences in the amount and type of hostility received}
    \label{fig:stats}
\end{figure}

\section{Data Characterisation}
\subsection{Linguistic Analysis}


We conduct a comparative linguistic analysis to investigate differences between content and language of hostile and non-hostile tweets. We use the Bag of Words (BOW) model and Linguistic Inquiry and Word Count (LIWC) Dictionary \cite{boyd2022development} to identify linguistic patterns. Then, we use a univariate Pearson's correlation test to identify which of these linguistic patterns significantly correlate with hostile and not hostile tweets. %, respectively. 
Tweets are pre-processed to replace URLs and user @mentions with special tokens (\textless URL \textgreater and \textless USER \textgreater, respectively) and remove stop words using NLTK \cite{bird2009natural}.
%During pre-processing, we replace URLs and user @mentions in the tweets with special tokens (\textless URL \textgreater and \textless USER \textgreater, respectively), and remove stop words using NLTK \cite{bird2009natural}.


\begin{figure}[t!]
  \centering
  \includegraphics[width=0.9\linewidth]{figures/bow_uni.pdf}
  \caption{Top 100 BOW unigrams associated with \textcolor{red}{hostile} and \textcolor{blue}{non-hostile} tweets. The larger the text size, the higher the Pearson correlation coefficient $r$, and vice versa.}
  %\Description{BOW unigrams}
  \label{fig:bow_uni}
\end{figure}

\begin{figure}[t!]
  \centering
  \includegraphics[width=0.9\linewidth]{figures/bow_bi.pdf}
  \caption{Top 100 BOW bigrams associated with \textcolor{red}{hostile} and \textcolor{blue}{non-hostile} tweets. The larger the text size, the higher the Pearson correlation coefficient $r$, and vice versa.}
  %\Description{BOW bigrams}
  \label{fig:bow_bi}
\end{figure}


\subsubsection{BOW}
We employ the BOW model to represent each post as a TF-IDF weighted distribution over a vocabulary 
of the 3,000 most frequent unigrams and bigrams. Figures~\ref{fig:bow_uni} and~\ref{fig:bow_bi} show the differences in BOW features associated with hostile and non-hostile tweets as word clouds.% (see Figure~\ref{fig:bow_uni} and Figure~\ref{fig:bow_bi}).% for unigrams and bigrams, respectively). 
These two figures show the top 100 BOW features for each category, with different text colours indicating the type of tweet and text size representing the Pearson correlation (${r}$) with the respective category (the larger the font, the higher the correlation).


Unsurprisingly, we observe that hostile tweets are characterised by negative and abusive phrases (e.g.``scum'', ``vile'', ``incompetent'', ``evil'', ``stupid'', 
``nothing good'', ``absolute disgrace''). They %are mostly posted to vent 
express anger or dissatisfaction at politicians, from questioning their abilities and distrusting their policies to insulting their personal traits. We also see emojis, e.g. ``face\_with\_symbols\_on\_mouth'' and ``face\_vomiting'', representing the use of profanity and disgust. Here is an example of such tweets from our dataset:
\begin{quote}
    Tweet 1: \textit{\textless USER\textgreater Some in the Cabinet are incompetent, others corrupt or evil. %. Some are corrupt. Some are evil. 
    You are all three. %You stand for nothing good. 
    I have nothing but contempt and disgust for you.}% and the ordure you spew forth.}
\end{quote}
Secondly, phrases such as ``go away'', 
``shame resign'' and ``run country'' in hostile tweets suggest that much of the hostility is directed at the Conservative Party, which is the ruling party. Here is an example requesting the MP to resign:
\begin{quote}
    Tweet 2: \textit{%\textless USER \textgreater \textless USER \textgreater \textless USER \textgreater 
Too late with \textless USER\textgreater in charge \& his cabinet of mendacious halfwits. Demand his resignation.}
\end{quote}
Phrases such as ``liar'', ``hypocrite'', 
`corrupt'', ``never trust'', ``know nothing'' and ``another lie'' 
indicate general distrust in the MPs. Also, we notice some trending topics in hostile tweets (e.g. ``vaccine passports'', ``illegal immigrants''), which reveal specific issues that cause dissatisfaction. The example tweet expresses the anger at policies relating to illegal immigration:
\begin{quote}
    Tweet 3: \textit{\textless USER \textgreater Just what would you do about the illegal immigration welcome them with open arms just wish we could send you to Rwanda and your filthy son}
\end{quote}


%In Figure~\ref{fig:bow_uni}, we observe that hostile tweets are characterised by negative words and phrases such as ``toryscum'', ``disgrace'', ``vile'', ``evil'', ``liar'', etc. In Figure~\ref{fig:bow_bi}, we see ``illegal immigration'', ``absolute disgrace'', ``go away'', ``shame resign'', ``run country'' etc. These suggest that much of the hostility is directed at the Conservative Party, which is the ruling party. There are also emojis like ``face\_with\_symbols\_on\_mouth,'' which represents the use of profanity, and ``face\_vomiting,'' which represents throwing up. Both unigrams and bigrams have words calling for resignations and questioning credibility (incompetent, hypocrite, corrupt) indicating a general distrust in the MPs. Here are two examples from our dataset: 

%Tweet 1: \textit{\textless USER \textgreater Some in the Cabinet are incompetent. Some are corrupt. Some are evil. You are all three. You stand for nothing that is good. I have nothing but contempt and disgust for you and the ordure you spew forth.}

%Tweet 2: \textit{\textless USER \textgreater \textless USER \textgreater You're right they should be hung until dead and stopped as soon as they step foot on our land. When does our right to live in peace come before the scums rights?}

For non-hostile tweets, the correlation $r$ is lower (as can be seen from the text size in Figure~\ref{fig:bow_uni}). However, they are correlated with words and phrases such as ``excellent'', %``recovery'', 
``best wishes'' and ``well done''. These suggest that non-hostile tweets often contain appreciative and positive emotions towards MPs. Some phrases (e.g. ``asked questions'', ``free movement'') indicate users' attempts to voice their political concerns. The following tweet is an example conveying appreciation to the MP:
\begin{quote}
    Tweet 4: \textit{\textless USER \textgreater was on fire! Another spectacular debate. Well done sir!}
\end{quote}


\subsubsection{LIWC}
Each tweet is also characterised using psycho-linguistic categories from the LIWC 2022 dictionary \cite{boyd2022development}. 
Table~\ref{tab:liwc} presents the top 10 categories most strongly correlated with hostile and non-hostile tweets. %We find similar associations between linguistic patterns and hostile and not hostile tweets for LIWC as we did for BOW.

\begin{table}[t!]
    \small
    \centering
    \begin{tabular}{|l|c|l|c|}
    \toprule
    \bf{Hostile} & \bf{\textit{r}} & \bf{Not hostile} & \bf{\textit{r}}\\
    \midrule
    socrefs & 0.186 & Tone & 0.192\\\bottomrule
    you & 0.181 & OtherP & 0.189 \\\bottomrule
    swearwords & 0.162 & AllPunc & 0.183\\\bottomrule
    clout & 0.160 & focuspast & 0.133\\\bottomrule
    tone\_neg & 0.160 & comm &0.104\\\bottomrule
    moral & 1.51 & prosoc & 0.084 \\\bottomrule
    affect & 0.142 & polite & 0.064\\\bottomrule
    ppron & 0.131& i & 0.063\\\bottomrule
    ethnicity & 0.111 & work & 0.062\\\bottomrule
    sex & 0.109 & tone\_pos & 0.061\\
    \bottomrule
    \end{tabular}
    \caption{Top 10 LIWC categories associated with hostile and non-hostile tweets sorted by Pearson's correlation ($r$) between the normalised frequency and the labels. All correlations are significant at $p$ \textless .001, two-tailed t-test.}
    \label{tab:liwc}
\end{table}

Similar to the BOW findings, we see that hostile tweets tend to have a negative tone (\textit{tone\_neg}) and convey negative emotions like anger and sadness (\textit{affect}). They contain more assertive and judgemental language (\textit{clout} and \textit{moralisation}). Unsurprisingly, they also contain more swear words (\textit{swearwords}) and sexual terms (\textit{sex}). Interestingly, race-related (\textit{ethnicity}) terms are frequent, suggesting that hostility is often related to race. The following example shows race-based hostility towards the MP from the dataset:
\begin{quote}
    Tweet 5: \textit{\textless USER \textgreater What about black violence! Your just another race divider.Marxists like you have ruined this country and divided it further!}
\end{quote}

%This indicates that hostile tweets can often be aggressive and negative comments directed at the MPs. We give two examples from the dataset:

%Tweet 3: \textit{\textless USER \textgreater What about black violence! Your just another race divider. Marxists like you have ruined this country and divided it further!}

%Tweet 4: \textit{\textless USER \textgreater do you have a heart?! Did you even stop to think about what this could mean to them, Putrid Patel?! How can you be an MP and not be able to empathise with people? Fucking cold bitch!}

We notice that non-hostile tweets correlate highly with the tone marker (\textit{tone}), particularly a positive tone (tone\_pos). They are polite, more communicative (\textit{comm}), and adhere to social norms (\textit{prosoc} and \textit{polite}), often consisting of explanations, feedback and questions. They also express concerns about work, jobs, schooling, etc. (\textit{work}). Below is an example tweet expressing concerns about the new scheme: 
\begin{quote}
    Tweet 6: \textit{\textless USER \textgreater please consider the effect of this new scheme before putting it into action. We have lost jobs and suffered a lot during covid. The UK economy will not recover. Please think carefully!}%We need to think of our next steps very carefully}
\end{quote}

\begin{table}[t!]
\small
\centering
    \begin{tabular}{|p{0.2\linewidth}|p{0.7\linewidth}|}
    \toprule
         \bf{Topic} & \bf{Representative Words}\\
    \midrule
         Brexit& brexit,  uk, ireland, eu, europe, leave, deal, citizens, free, border\\ \bottomrule 
         Illegal immigration & refugees, illegal, boats, rwanda, immigrants, asylum, raped, terrorists, seekers, migrants \\ \bottomrule 
         Conservative party & tory, conservative, resign, vote, rishi, scum, tories, johnsonout, torysewageparty, cabinet\\ \bottomrule 
         Labour party& labour, starmer, voters, corbyn, party, win, mps, election, abbott, protest\\ \bottomrule 
         COVID-19& covid, virus, vaccine, lockdown, died, pandemic, mask, vulnerable, jab, nhs\\ \bottomrule 
         Cost of living crisis & economy, bills, winter, job, tax, inflation, energy, nhs, heating, gas\\
         \bottomrule
    \end{tabular}   
    \caption{Topic groups and representative words derived from BERTopic}
    \label{tab:topic_censored_nomps}
\end{table}


\subsection{Topic Analysis}
We perform topic analysis using BERTopic \cite{grootendorst2022bertopic} after removing stop words with NLTK \cite{bird2009natural}. Due to the frequency of MP names and profanity, the topics are rather unclear. Once we remove these, clear topics and their representative words emerge. Table \ref{tab:topic_censored_nomps} shows the top 6 topic groups and their representative words after removing these terms. 
The topics relate to major events and issues in the UK, like Brexit (e.g., ``europe'', ``border''), illegal immigration (e.g., ``refugees'', ``terrorists''), and the cost of living crisis (e.g., ``bills'', ``tax'', ``inflation''). The following example is a hostile tweet expressing anger due to increased costs of bills: 
\begin{quote}
    Tweet 7: \textit{\textless USER \textgreater What planet do you live on? You haven’t saved the day. Fuel is still +40\%. Energy bills are +50\%. We’re still fucked. Make it make sense!!!}
\end{quote}

Other popular topics are the two main political parties in our dataset (Conservative and Labour). However, the ruling Conservative party is likely to receive more hostility based on the larger proportion of negative terms we find, such as ``scum'', ``johnsonout''. Here is an example of hostile tweets mentioning the Conservative Party:
\begin{quote}
    Tweet 7: \textit{\textless USER \textgreater this you? You ludicrous pork Hay-bale. You bin bag full of custard. \#ToryCriminalsUnfitToGovern \textless USER \textgreater shouldn't apologise you scum}
\end{quote}

Based on the representative words, we see that the word ``scum'' is frequently used to describe MPs from the Conservative party and illegal immigration is considered a threat due to ``terrorist'' refugees entering the UK. Here are two examples from the data:


Tweet 8: \textit{\textless USER \textgreater please hurry up and stop fake migrants landing here daily costing millions to tax payers at a time when thousands losing jobs due to covid money can go to better use than ungrateful future terrorists this country is turning into a middle eastern shithole wake up now}


 %From this, we can see that the dataset covers various topics. One thing to note is that due to the sampling method (activity spikes), the tweets tend to be about the most pressing issue on those days. %While the tweets all refer to MPs, they will still contain issue related information. %a portion of tweets are general comments about the MP, not about specific events or current affairs, most tweets will be about the event at the time of posting. %However, this helps to 

Most topics appear in the same proportion in both hostile tweets %as they do in
and non-hostile tweets. The exception is ``illegal immigration'' which appears twice as much in hostile tweets than in non-hostile tweets. 
Figure~\ref{fig:topic_id} shows the proportion of topic-related tweets belonging to identity-based hostility. Looking at the distribution of ``illegal immigration'' and ``Brexit'', they appear mainly in race-based hostile tweets. While the ``Conservative party'' and ``Labour party'' topics contribute to race-based hostile tweets, they appear much more frequently in non-race, gender or religion-based hostility.

\begin{figure}[t!]
  \centering
  \includegraphics[width=0.9\linewidth]{figures/topic_id.pdf}
  \caption{Proportion of topic-related tweets belonging to each identity characteristic label}
  %\Description{topics identity breakdown}
  \label{fig:topic_id}
\end{figure}

While all the tweets relate to MPs, they still naturally fall into topics related to current issues at the time. Due to its 2-year span, the dataset thus covers a wide range of topics, since issues discussed on social media can change rapidly. This topic characterisation means that the dataset could eventually be used for analysis and comparison of hostility in relation to different issues over time. 
 
\section{Online Hostility Detection}
We finetune PLMs for political hostility detection to test how they perform on our dataset.
%We finetune PLMs for political hostility detection using our dataset to demonstrate its use.
We also evaluate the ability of LLMs to identify political hostility on our dataset, demonstrating its usefulness.%to  over regular hsd datasets/models

%Given a text snippet, we define online hostility detection as two classification tasks: (1) binary hostility identification (if a tweet contains hostility or not) and (2) multi-class classification for classifying if a tweet contains one of the four identity-based hostility types (i.e., religion, gender, race, none) or no hostility at all. For multi-class classification, we compare two classification methods, namely flat and two-level hierarchical classification.

%\paragraph{Flat Classification} Given a tweet, models classify it as religion-based hostility, gender-based hostility, race-based hostility, hostility with no protected identity characteristics (none-hostility) or not hostility. 

%\paragraph{Two-level Hierarchical Classification} Given a tweet, the first classifiers identify if it is hostile or not. Then, the second classifiers classify the identity types of the hostile tweets (religion, gender, race or none) based on predictive results from the first classifiers.

Given a text snippet, we define online hostility detection as two classification tasks: (1) binary hostility identification (if a tweet contains hostility or not) and (2) multi-class classification to see if it contains one of  the four identity-based hostility types (religion, gender, race, none) or no hostility at all. For multi-class classification, we use a two-level hierarchical classification method.\footnote{We also tried a flat classification method, but we exclude the results as it performs slightly worse.} The first classifiers classify tweets as hostile or not, while the second classifiers then classify the identity types of those identified as hostile.


\subsection{Predictive Models}
We use three PLMs for binary hostility identification and multi-class classification. %Also, we evaluate two widely used LLMs on identifying hostile tweets as well as their targeted identity types.
We fine-tune \textbf{BERT} \cite{devlin2018bert}, \textbf{RoBERTa} \cite{liu2019roberta}, and a domain adaptation model, \textbf{RoBERTa-Hate}\footnote{\url{https://huggingface.co/cardiffnlp/twitter-roberta-base-hate-latest}}, (trained on 13 different hate speech datasets in English including political content) by adding a classification layer with softmax activation function on top of the [CLS].

%\paragraph{BERT} We fine-tune Bidirectional Encoder Representations from Transformers (BERT) \cite{devlin2018bert}  by adding a classification layer with softmax activation %function on top of the [CLS].

%\paragraph{RoBERTa} We fine-tune RoBERTa \cite{liu2019roberta} model in a similar way to BERT.

%\paragraph{RoBERTa-Hate} Similarly, we fine-tune a domain adaptation model, RoBERTa-Hate\footnote{\url{https://huggingface.co/cardiffnlp/twitter-roberta-base-hate-latest}}, which is trained on 13 different hate speech datasets in the English language including political content.
Also, we evaluate two widely used LLMs on identifying hostile tweets and their targeted identity types. We use the \textbf{instruction-tuned LLaMA 3 8B model} through the Hugging Face platform\footnote{\url{https://huggingface.co/meta-llama/Meta-Llama-3-8B}}, providing the model with a sequence of texts and a prompt with a task description to guide its output (see Experimental Set-up for detailed prompts). Similarly, we use the \textbf{GPT-3.5 model}\footnote{\url{https://platform.openai.com/docs/models/gpt-3-5-turbo}} with an API key providing texts and corresponding prompts.
%\paragraph{LLaMA-3-8B (LLaMA)} We use the instruction tuned LLaMA 3 8B model through the Hugging Face platform\footnote{\url{https://huggingface.co/meta-llama/Meta-Llama-3-8B}}. We provide the model with a sequence of texts and a prompt with a task description to guide its output.

%\paragraph{GPT-3.5 (GPT)} Similarly, we use the GPT-3.5 model\footnote{\url{https://platform.openai.com/docs/models/gpt-3-5-turbo}} with an API key providing texts and corresponding prompts.

\subsection{Experimental Set-up}
Tweets are pre-processed, replacing  URLs and user @mentions with special tokens \textless URL \textgreater and \textless USER \textgreater. 
%For BERT, we use base uncased model, and for RoBERTa, we use base model. The maximum sequence length is set to 256 tokens, and the batch size is set to 32. We run all models using a 5-fold cross-validation method where 4-fold data is used for training and 1-fold data is used for testing. We split 4-fold data into training and validation sets with a ratio of 9:1. We use Cross Entropy Loss as the training loss function with the AdamW optimizer. For flat classification, we train using a learning rate of lr = 3e-6, while hierarchical classification is trained with lr = 5e-5. During training, we choose the model with the smallest validation loss over 15 epochs. All models are trained on an NVIDIA A100 GPU. 
For evaluation, we report average Accuracy, Precision, Recall and macro F1 over 5 folds with standard deviations. %Details of model parameters can be found in the appendix.
We use BERT-base-uncased and RoBERTa-base models with a maximum sequence length of 256 tokens and batch size of 32. Training uses 5-fold cross-validation (4-fold training, split 9:1 for validation, 1-fold testing) with Cross Entropy Loss and AdamW optimizer at 5e-5 learning rate. Models are selected based on minimum validation loss over 15 epochs and trained on an NVIDIA A100 GPU. 

For LLMs, we input the prompt to specify the task for binary hostility identification and multi-class flat classification: (1) \textit{Classify the tweet as hostile or not hostile} and (2) \textit{Classify the tweet into religion-based hostile, gender-based hostile, race-based hostile, other-hostile or not hostile} with (\textbf{LLaMA w/ Def.}, \textbf{GPT w/ Def.}) or without definitions (\textbf{LLaMA}, \textbf{GPT}) of each category (see Table \ref{tab:definitions} for definitions). For 2-level hierarchical classification, we input the prompt based on the outputs from the binary hostility identification: \textit{Classify the tweet as hostility based on race, gender, religion or other}. For all experiments of LLMs, the temperature is set to 0.1.
%For LLMs, we input the prompt to specify the task for binary hostility identification:  \textit{Classify the tweet as hostile or not hostile} with (\textbf{LLaMA w/ Def.}, \textbf{GPT w/ Def.}) or without definitions (\textbf{LLaMA}, \textbf{GPT}). For 2-level hierarchical classification, we input the prompt based on the outputs from the binary hostility identification: \textit{Classify the tweet as hostility based on race, gender, religion or other}. For all experiments of LLMs, the temperature is set to 0.1.
For a fair comparison, we also report the average performance over 5 folds with the same data in each fold.


\begin{table}[t!]
   
    \centering
    \scriptsize
    \begin{tabular}{|l|c|c|c|c|}
    \toprule
    Model & Accuracy & Precision & Recall & F1\\
    %\midrule
    %\multicolumn{5}{|c|}{Set 1}\\
    \bottomrule
    \rowcolor{orange}
    BERT & 66.96$\pm$1.35 & 66.55$\pm$1.45 & 65.75$\pm$1.32 & 65.84$\pm$1.35\\
    \rowcolor{orange}
    RoBERTa& 68.13$\pm$0.83 & 68.04$\pm$0.62 & 67.55$\pm$0.51 &67.44$\pm$0.48\\
    \rowcolor{orange}
    RoBERTa-Hate & 67.38$\pm$1.51 & 67.47$\pm$1.15 & 67.10$\pm$0.66 & 66.84$\pm$1.09\\
    \bottomrule   
    %\multicolumn{5}{|c|}{Set 2}\\
    %\bottomrule
    \rowcolor{pink}
    BERT & 72.47$\pm$3.56 & 72.27$\pm$3.82 & 71.62$\pm$3.22 & 71.77$\pm$3.37\\
    \rowcolor{pink}
    RoBERTa & 71.77$\pm$3.37 & 72.26$\pm$2.05 & 69.15$\pm$2.99 & 68.86$\pm$3.65\\
    \rowcolor{pink}
    RoBERTa-Hate & 72.27$\pm$3.82 & \bf{73.44}$\pm$1.00 & \bf{73.16}$\pm$1.44 & \bf{73.03}$\pm$1.27\\
    \bottomrule
    \rowcolor{pink}
    LLaMA & 71.30$\pm$0.96 & 71.17$\pm$0.86 & 71.44$\pm$0.86 & 71.11$\pm$0.91\\
    \rowcolor{pink}
    LLaMA w/ Def. & \bf{73.55}$\pm$1.39 & 73.21$\pm$1.42 & 72.76$\pm$1.43 & 72.91$\pm$1.43\\
    \rowcolor{pink}
    GPT & 60.57$\pm$1.93 & 69.97$\pm$1.21 & 64.20$\pm$1.72 & 58.67$\pm$2.41 \\
    \rowcolor{pink}
    GPT w/ Def. & 70.69$\pm$1.27 & 71.90$\pm$1.29 & 71.85$\pm$1.28 & 70.69$\pm$1.27 \\
    \bottomrule
    \end{tabular}
     \caption{Accuracy, Precision, Recall and macro F1-Score (F1) for binary hostility identification ($\pm$ std. dev.) for \textcolor{orange}{Set 1} and \textcolor{pink}{Set 2}. Best results are in bold.}
    \label{tab:binary_predictive_results}
\end{table}






\begin{table}[t!]
    \centering
    \scriptsize
    \begin{tabular}{|l|c|c|c|c|}
    \toprule
    Model & Accuracy & Precision & Recall & F1\\
    \midrule
    %\multicolumn{5}{|c|}{Flat Classification}\\
    %\bottomrule
    %\multicolumn{5}{|c|}{Set 1}\\
    %\bottomrule
    %\rowcolor{Gray}
    %BERT & 61.93$\pm$2.03 & 31.42$\pm$2.14 & 28.31$\pm$2.36 & 28.18$\pm$2.91\\
    %\rowcolor{Gray}
    %RoBERTa & 63.13$\pm$0.35 & 24.40$\pm$0.32 & 26.03$\pm$0.20 & 20.79$\pm$0.21\\
    %\rowcolor{Gray}
    %RoBERTa-Hate & 64.01$\pm$1.41 & 45.81$\pm$6.34 & 34.74$\pm$3.90 & 36.67$\pm$4.54\\
    %\bottomrule
    %\multicolumn{5}{|c|}{Set 2}\\
    %\bottomrule
    %\rowcolor{LightCyan}
    %BERT & 65.93$\pm$1.68 & 37.12$\pm$1.62 & 34.77$\pm$2.79 & 35.07$\pm$2.72\\
    %\rowcolor{LightCyan}
    %RoBERTa & 67.26$\pm$1.05 & 33.01$\pm$5.96 & 32.25$\pm$3.95 & 32.04$\pm$4.68\\
    %\rowcolor{LightCyan}
    %RoBERTa-Hate & \bf{69.64}$\pm$1.24 & 50.82$\pm$7.61 & 41.66$\pm$4.90 & 43.31$\pm$5.65\\
    %\bottomrule
    %\rowcolor{LightCyan}
    %LLaMA & 55.48$\pm$2.29 & 42.01$\pm$5.38 & 46.60$\pm$8.18 & 38.09$\pm$5.69\\
    %\rowcolor{LightCyan}
    %LLaMA w/ Def. & 61.33$\pm$1.41 & 48.61$\pm$3.58 & 52.39$\pm$3.68 & 47.02$\pm$3.25\\
    %\rowcolor{LightCyan}
    %GPT & 65.57$\pm$1.77 & 52.64$\pm$4.21 & 55.02$\pm$5.21 & 52.65$\pm$4.45\\
    %\rowcolor{LightCyan}
    %GPT w/ Def. & 65.03$\pm$1.71 & 53.70$\pm$2.71 & 56.89$\pm$2.69 & 54.74$\pm$2.22 \\
    %\bottomrule
    %\multicolumn{5}{|c|}{Set 3}\\
    %\bottomrule
    %BERT & 65.03$\pm$2.31 & 19.45$\pm$3.80 & 18.43$\pm$2.03 & 18.23$\pm$2.66\\
    %RoBERTa & 66.66$\pm$1.72 & 20.30$\pm$3.68 & 20.43$\pm$3.44 & 20.08$\pm$3.64 \\
    %RoBERTa-Hate & 68.70$\pm$2.64 & 28.55$\pm$8.12 & 23.51$\pm$3.70 & 23.98$\pm$4.49\\
    %\bottomrule
    %\bottomrule
    %\multicolumn{5}{|c|}{Hierarchical Classification}\\
    %\bottomrule
    %\multicolumn{5}{|c|}{Set 1}\\
    %\bottomrule
    \rowcolor{orange}
    BERT & 60.78$\pm$1.00 & 27.44$\pm$5.76 & 25.60$\pm$2.01 & 24.79$\pm$2.14\\
    \rowcolor{orange}
    RoBERTa & 61.99$\pm$1.32 & 37.66$\pm$9.18 & 27.53$\pm$2.08 & 28.87$\pm$2.44\\
    \rowcolor{orange}
    RoBERTa-Hate & 62.47$\pm$2.29 & 38.77$\pm$5.79 & 28.42$\pm$1.58 & 31.21$\pm$2.39\\
    \bottomrule
    %\multicolumn{5}{|c|}{Set 2}\\
    %\bottomrule
    \rowcolor{pink}
    BERT & 66.30$\pm$4.52 & 32.42$\pm$2.08 & 28.41$\pm$2.95 & 29.09$\pm$3.08\\
    \rowcolor{pink}
    RoBERTa & 66.14$\pm$1.70 & 40.77$\pm$8.44 & 30.47$\pm$6.38 & 32.85$\pm$7.03\\
    \rowcolor{pink}
    RoBERTa-Hate & 68.10$\pm$1.57 & 39.93$\pm$4.37 & 32.18$\pm$4.57 & 33.81$\pm$4.63\\
    \bottomrule
    \rowcolor{pink}
    LLaMA & 64.79$\pm$1.97 & 54.62$\pm$3.75 & 51.77$\pm$3.83 & 52.15$\pm$3.65\\
    \rowcolor{pink}
    LLaMA w/ Def. & 64.70$\pm$2.37 & 53.11$\pm$11.04 & 53.98 $\pm$3.67 & 54.16$\pm$4.43\\
    \rowcolor{pink}
    GPT & 54.19$\pm$2.77 & \bf{55.61}$\pm$5.11 & 54.29$\pm$5.79 & 50.53$\pm$5.08\\
    \rowcolor{pink}
    GPT w/ Def. & 64.43$\pm$1.52 & 54.15$\pm$3.42 & \bf{60.02}$\pm$3.11 & \bf{55.98}$\pm$3.08\\
    \bottomrule
    %\multicolumn{5}{|c|}{Set 3}\\
    %\bottomrule
    \rowcolor{green}
    BERT & 66.30$\pm$4.32 & 21.53$\pm$2.29 & 19.14$\pm$1.51 & 19.49$\pm$1.60\\
    \rowcolor{green}
    RoBERTa & 65.84$\pm$2.24 & 30.52$\pm$8.89 & 23.01$\pm$6.86 & 23.60$\pm$6.55\\
    \rowcolor{green}
    RoBERTa-Hate & \bf{67.80}$\pm$2.07 & 26.00$\pm$2.28 & 25.09$\pm$3.29 & 24.22$\pm$2.96\\
    \bottomrule
    \end{tabular}
    \caption{Accuracy, Precision, Recall and macro F1-Score (F1) for hostility type classification in 2-level hierarchical method ($\pm$ std. dev.) for \textcolor{orange}{Set 1}, \textcolor{pink}{Set 2} and \textcolor{green}{Set 3}. Best results are in bold.}
    \label{tab:multiclass_predictive_results}
\end{table}


\subsection{Results}

\paragraph{Binary Hostility Identification} Table \ref{tab:binary_predictive_results} presents the predictive results of all models on binary hostility identification using Set 1 and Set 2. We exclude Set 3 because the intersectional labels in identity type annotation do not affect the binary labels. Overall, RoBERTa-Hate on Set 2 achieves the best performance among all models, reaching a macro F1 score up to 73.03 (in bold). Then, we observe that models trained on Set 2 achieve better performance than those trained on Set 1 (e.g., 68.86 vs. 67.44 for RoBERTa on Set 2 and Set 1), highlighting the importance of selecting annotations based on confidence scores. Also, the domain adaptation model (i.e., RoBERTa-Hate) outperforms the vanilla models on Set 2 (e.g., 68.86 F1 for RoBERTa vs. 73.03 F1 RoBERTa-Hate).\footnote{We also evaluate Set 1 and Set 2 on the same test set with the same labels (i.e. a subset in Set 2). Note that in Table \ref{tab:binary_predictive_results}, we evaluate Set 1 and Set 2 using the test set with different labels due to the different methods we used to choose labels for Set 1 and Set 2. We exclude Set 3 since adding intersectional labels leads to different test sets. RoBERTa and RoBERTa-Hate using Set 2 achieve better results than using Set 1 (72.46 vs. 71.11 and 74.10 vs. 73.26 accordingly).}






We apply LLMs on Set 2, where better results are achieved. Among four LLM settings, LLaMA w/ Def. achieves the best performance with a macro F1 score of 72.91, followed by GPT w/ Def (70.69 F1). We notice that adding label definitions in the prompt leads to performance improvement (+1.80 F1 for LLaMA and +12.02 F1 for GPT). We argue that advanced LLMs do not show significant advantages over PLMs on binary hostility identification as it is a simple and straightforward 2-class classification task.


%\paragraph{Multi-class Hostility Classification} Table \ref{tab:multiclass_predictive_results} presents the results of all models on multi-class hostility type classification using three sets of data in flat (top) and 2-level hierarchical (bottom) ways. Among all PLMs, the best performing model is RoBERTa-Hate on Set 2 in the flat classification method with an F1 score of 43.31 (in bold). Similar to the binary hostility identification, models in Set 2 achieve the best predictive results compared with the same models trained on other sets (e.g., 32.04 F1 for RoBERTa in flat classification, 33.81 F1 for RoBERTa-Hate in hierarchical classification), followed by Set 1  (e.g., 20.79 F1 for RoBERTa in flat classification, 31.21 F1 for RoBERTa-Hate in hierarchical classification). The domain adaptation model, RoBERTa-Hate, outperforms the vanilla RoBERTa model with a larger difference compared to binary hostility identification (e.g., +4.17 F1 vs. +11.27 F1 on Set 2 in binary hostility identification and in multi-class hostility classification using the flat method). Additionally, BERT and RoBERTa exhibit comparable performance, with BERT sometimes outperforming RoBERTa and vice versa depending on different settings. 

%Furthermore, the flat classification method outperforms the 2-level hierarchical classification method (e.g., 36.67 F1 in flat classification vs. 27.65 F1 in hierarchical for RoBERTa-Hate on Set 1). This may be explained by the fact that after training the first classifier, predictive errors introduce more noise in the hierarchical method. Also, the second classifier is trained on a smaller data set (non-hostile tweets are excluded) compared with that in the flat classification.

%Similar to the hostility identification task, we only apply LLMs on Set 2. First of all, GPT w/ Def. in hierarchical classification outperforms all PLMs and LLMs, reaching a macro F1 score up to 55.98, which is 12.67 higher than the best performing PLM, RoBERTa-Hate. Secondly, in general, adding definitions of each hostility type boosts the performance. Moreover, prompts with definitions result in a larger improvement on the multi-class classification than the binary one (e.g., +5.85 F1 for LLaMA in flat classification, +5.45 F1 for GPT in hierarchical classification). Furthermore, LLMs in flat and hierarchical settings achieve comparable results. This might be explained by the fact that we use zero-shot classification, where they do not rely on our training data. Future research may extend it to few-shot learning. 



\paragraph{Multi-class Hostility Classification} Table \ref{tab:multiclass_predictive_results} presents the results of all models on multi-class hostility type classification using three sets of data in 2-level hierarchical method. Among all PLMs, the best performing model is RoBERTa-Hate on Set 2 with an F1 score of 33.81 (in bold). Similar to the binary hostility identification, models in Set 2 achieve the best predictive results compared with the same models trained on other sets (e.g., 32.85 F1 for RoBERTa ), followed by Set 1  (e.g., 31.21 F1 for RoBERTa-Hate). The domain adaptation model, RoBERTa-Hate, outperforms the vanilla RoBERTa model with a smaller difference compared to binary hostility identification (e.g., +4.17 F1 vs. +0.96 F1 on Set 2 in binary hostility identification and in multi-class hostility classification). Additionally, RoBERTa outperforms BERT across three sets of data (e.g., 32.85 vs. 29.09 F1 on Set 2)


Similar to the hostility identification task, we only apply LLMs on Set 2. First of all, GPT w/ Def. outperforms all PLMs and LLMs, reaching a macro F1 score up to 55.98, which is 22.17 higher than the best-performing PLM, RoBERTa-Hate. Secondly, in general, adding definitions of each hostility type boosts the performance. Moreover, prompts with definitions result in a larger improvement on the multi-class classification than the binary one (e.g., +5.45 F1 for GPT in hierarchical classification).  

\section{Conclusion}
This work focuses on the creation of data for investigating online hostility towards UK politicians. We developed an English dataset of 3,320 tweets, manually annotated with hostility as well as their targeted identity characteristics: religion, gender, and race. Also, we conducted extensive linguistic and topical analyses to provide deeper insights into the specific content of these hostile interactions. By constructing and analysing such a dataset, we identify key patterns, such as the prevalence of race-based hostility, especially regarding immigration issues in the UK. Also, our findings suggest that there is a general lack of trust in MPs in the UK. Furthermore, we evaluate the performance of various PLMs and LLMs on binary hostility identification and multi-class targeted identity type classification using our dataset. This study not only offers valuable data but also lays the groundwork for future research aimed at understanding and mitigating the impact of online hostility in political contexts specific to the UK. 

\section{Dataset Availability}
Our dataset is publicly available in accordance with the FAIR principles \cite{fair}:
\begin{itemize}
    \item \textbf{Findable:} Our dataset will be published in the Zenodo dataset-sharing service with a unique DOI.%digital object identifier. %For now it can be found at \url{https://anonymous.4open.science/r/ohtukmp-21D8}.
    \item \textbf{Accessible:} Original tweets can be retrieved using their tweet IDs via the standard Twitter API.\footnote{https://developer.twitter.com/en/docs/twitter-api/tweets/lookup/api-reference/get-tweets-id}
    \item \textbf{Interoperable:} File structure and column descriptions are detailed in a readme file and the CSV format ensures broad compatibility across data processing tools. %The readme file in the repository explains the dataset structure and the description of each column in the CSV data file. CSV datasets are easily imported and processed by most widely used data processing tools.
    \item \textbf{Re-usable:} Our dataset can be re-used by anyone who has a Twitter developer account.
\end{itemize}




%\section{new results}

%To better understand how hostility may be distributed across identity groups and political parties, we compare the amount of hostile tweets received by MPs belonging to them. Figure~\ref{fig:stats_party} shows that MPs belonging to the Conservative party receive more hostility in general. On average, MPs belonging to the Conservative Party receive more race-based hostility, but there are some MPs from the Labour Party who receive more race-based hostility (e.g. David Lammy). For both gender-based hostility and religion-based hostility, on average, MPs from both parties receive a similar amount of hostility. However, there are some MPs from the Labour party who receive more than the others (e.g. Diane Abbott). Due to only one MP in our study belonging to SNP, we do not include it in this analysis.   
%\begin{figure}
%    \centering
%    \includegraphics[width=0.8\linewidth]{figures/stats_party.png}
%    \caption{Comparing political party based differences in the amount and type of hostility received}
%    \label{fig:stats_party}
%\end{figure}


%In Figure~\ref{fig:stats}, we observe that while male MPs receive more hostile tweets (including non-identity related), female MPs are disproportionately subjected to gender-based hostility compared to their male counterparts. Similarly, non-white and non-Christian MPs face significantly higher levels of both race-based and religion-based hostility. Additionally, it is interesting to see that MPs belonging to [dominant or majority] identity groups consistently receive less hostility than their minority counterparts. 

%\begin{figure}
%    \centering
%    \includegraphics[width=0.9\linewidth]{figures/stats.png}
%    \caption{Comparing identity-based differences in the amount and type of hostility received}
%    \label{fig:stats}
%\end{figure}

%Due to the imbalance in the number of MPs with different identity characteristics, a direct comparison of the counts of tweets they receive is not feasible. To better understand how hostility may be distributed across identity groups, we analyse the ratios of male to female, non-white to white, and non-Christian to Christian MPs for each tweet category. Table~\ref{tab:ratio} presents the ratio of MPs with differing identity characteristics within our dataset alongside the corresponding ratios for tweets directed at them. If the ratio of MPs is equal to the ratio observed for a specific type of tweet, this would indicate that MPs with those characteristics are receiving comparable amounts of that type of tweet. For instance, equal ratios suggest that Christian and non-Christian MPs are equally targeted with gender-based hostility.

%From Table~\ref{tab:ratio},we observe that Female MPs are disproportionately subjected to both gender-based and religion-based hostility compared to their male counterparts, with ratios of 6.60 and 3.80, respectively. Similarly, non-white MPs face significantly higher levels of race-based hostility, with a ratio of 8.75 
%The dataset contained only a single instance of religion-based hostility directed at a white MP, rendering the ratio insufficiently reliable for meaningful analysis. Additionally, MPs who identify as non-Christian experience more religion-based hostility than Christian MPs. 
%Notably, the non-white:white ratio for religion-based hostility (32.67) suggests a marked disparity in how hostility is distributed among MPs of differing racial and religious backgrounds.


%\begin{table}
 %   \centering
 %   \footnotesize
 %   \begin{tabular}{|c|c|c|c|} \hline 
 %        ratio&  female:male&  not-white:white& not-christ:christ\\ \hline 
  %       \# of MP's&  1.57&  2.00& 0.80\\ \hline 
  %       hostile&  1.46&  2.60& 1.04\\ \hline 
 %        non-hostile&  1.20&  1.92& 0.83\\ \hline 
 %        race&  1.64&  8.75& 1.48\\ \hline 
 %        gender&  6.60&  3.17& 0.84\\ \hline 
 %        religion&  3.80&  32.67& 3.85\\ \hline 
 %        none&  1.25&  2.13& 0.96\\ \hline
 %   \end{tabular}
 %   \caption{ratios of female to male, not-white to white and not-Christian to Christian for the types of tweets they receive}
 %   \label{tab:ratio}
%\end{table}


%\begin{table}
%    \caption{Accuracy, Precision, Recall and macro F1-Score (F1) for binary (top) and multi-class (bottom) hostility classification ($\pm$ std. dev.). Best results are in bold.}
%    \label{tab:predictive_results_same_testing}
%    \centering
%    \scriptsize
%    \begin{tabular}{|l|c|c|c|c|}
%    \toprule
%    Model & Accuracy & Precision & Recall & F1\\
%    \midrule
%    \multicolumn{5}{|c|}{Binary}\\
%    \bottomrule
%    \multicolumn{5}{|c|}{Set 1}\\
%    \bottomrule
%    BERT & 70.63$\pm$1.96 & 70.86$\pm$2.31 & 69.96$\pm$1.89 & 69.96$\pm$1.94\\
%    RoBERTa & 71.93$\pm$1.83 & 73.12$\pm$3.22 & 71.28$\pm$1.46 & 71.11$\pm$1.37 \\
%    RoBERTa-Hate & \bf{75.42}$\pm$1.20 & \bf{75.87}$\pm$1.00 & 74.66$\pm$1.44 & 73.26$\pm$1.57\\
%    \bottomrule   
%    \multicolumn{5}{|c|}{Set 2}\\
%    \bottomrule
%    BERT & 69.49$\pm$0.91 & 70.18$\pm$2.11 & 68.61$\pm$0.54 & 68.50$\pm$0.63 \\
%    RoBERTa & 73.10$\pm$1.79 & 73.50$\pm$1.68 & 72.51$\pm$2.36 & 72.46$\pm$2.26 \\
%    RoBERTa-Hate & 75.21$\pm$1.03 & 75.36$\pm$1.21 & \bf{74.69}$\pm$1.08 & \bf{74.10}$\pm$1.80 \\
%    \bottomrule
%    \bottomrule
%    \multicolumn{5}{|c|}{Multi}\\
%    \bottomrule
%    \multicolumn{5}{|c|}{Set 1}\\
%    \bottomrule
%    BERT & 66.39$\pm$1.55 & 37.25$\pm$1.54 & 33.01$\pm$1.54 & 33.23$\pm$1.11 \\
%    RoBERTa & 67.62$\pm$1.34 & 39.65$\pm$3.20 & 35.14$\pm$2.87 & 35.87$\pm$2.91 \\
%    RoBERTa-Hate & \bf{68.67}$\pm$1.35 & \bf{45.60}$\pm$5.96 & \bf{41.79}$\pm$2.80 & \bf{42.34}$\pm$3.87 \\
%    \bottomrule
%    \multicolumn{5}{|c|}{Set 2}\\
%    \bottomrule
%    BERT & 65.93$\pm$1.29 & 36.10$\pm$2.38 & 31.72$\pm$1.52 & 32.30$\pm$1.82\\
%    RoBERTa & 66.90$\pm$2.04 & 45.59$\pm$6.92 & 36.49$\pm$2.93 & 37.00$\pm$2.80\\
%    RoBERTa-Hate & 68.55$\pm$2.32 & 41.56$\pm$7.20 & 39.18$\pm$3.22 & 38.85$\pm$4.06\\
%    \bottomrule
%    \end{tabular}
%\end{table}

%\section{new experiments}
%We further evaluate Set 1 and Set 2 using the same testing set on binary and multi-class hostility classification. We split our dataset into training and test sets first. For training sets, we create Set 1 and Set 2 by assigning labels based on the majority vote and confidence scores separately, which is similar to the previous experiment. For test sets, we assign labels based on confidence scores for both Set 1 and Set 2 to ensure the labels to be the same. We are unable to include Set 3 in this experiment since adding intersectional labels leads to different test sets.

%Table \ref{tab:predictive_results_same_testing} shows the predictive results using Set 1 and Set 2 for binary hostility identification (top) and multi-class targeted identity type classification (bottom). We observe that models have comparable performance using Set 1 and Set 2 (for binary classification, using Set 2 performs better while for multi-class classification, using Set 1 performs better). This is because the difference in each label count between Set 1 and Set 2 is minor, especially for targeted identity types (see Table \ref{tab:hostile}). For multi-class classification, we further look into the predictive results for each class. We find that the number of tweets with gender-based hostile is only around 20 in the test set. Most of the models obtain 0 f1 scores on this class while only a few models predict some of these tweets accurately which leads to a better overall results.  



% Use \bibliography{yourbibfile} instead or the References section will not appear in your paper
\bibliography{aaai22}

\subsection{Paper Checklist}

\begin{enumerate}

\item For most authors...
\begin{enumerate}
    \item  Would answering this research question advance science without violating social contracts, such as violating privacy norms, perpetuating unfair profiling, exacerbating the socio-economic divide, or implying disrespect to societies or cultures?
    \answerYes{Yes}
  \item Do your main claims in the abstract and introduction accurately reflect the paper's contributions and scope?
    \answerYes{Yes}
   \item Do you clarify how the proposed methodological approach is appropriate for the claims made? 
    \answerYes{Yes}
   \item Do you clarify what are possible artifacts in the data used, given population-specific distributions?
    \answerYes{Yes}
  \item Did you describe the limitations of your work?
    \answerYes{Yes. In the Data section}
  \item Did you discuss any potential negative societal impacts of your work?
    \answerYes{Yes}
      \item Did you discuss any potential misuse of your work?
    \answerYes{Yes}
    \item Did you describe steps taken to prevent or mitigate potential negative outcomes of the research, such as data and model documentation, data anonymization, responsible release, access control, and the reproducibility of findings?
    \answerYes{Yes. In the Data section}
  \item Have you read the ethics review guidelines and ensured that your paper conforms to them?
    \answerYes{Yes}
\end{enumerate}

\item Additionally, if your study involves hypotheses testing...
\begin{enumerate}
  \item Did you clearly state the assumptions underlying all theoretical results?
    \answerNA{NA}
  \item Have you provided justifications for all theoretical results?
    \answerNA{NA}
  \item Did you discuss competing hypotheses or theories that might challenge or complement your theoretical results?
    \answerNA{NA}
  \item Have you considered alternative mechanisms or explanations that might account for the same outcomes observed in your study?
    \answerNA{NA}
  \item Did you address potential biases or limitations in your theoretical framework?
    \answerNA{NA}
  \item Have you related your theoretical results to the existing literature in social science?
    \answerNA{NA}
  \item Did you discuss the implications of your theoretical results for policy, practice, or further research in the social science domain?
    \answerNA{NA}
\end{enumerate}

\item Additionally, if you are including theoretical proofs...
\begin{enumerate}
  \item Did you state the full set of assumptions of all theoretical results?
    \answerNA{NA}
	\item Did you include complete proofs of all theoretical results?
    \answerNA{NA}
\end{enumerate}

\item Additionally, if you ran machine learning experiments...
\begin{enumerate}
  \item Did you include the code, data, and instructions needed to reproduce the main experimental results (either in the supplemental material or as a URL)?
    \answerYes{Yes. In the supplemental material, URL and in the Data Availability section}
  \item Did you specify all the training details (e.g., data splits, hyperparameters, how they were chosen)?
    \answerYes{Yes. In the Model Parameters section}
     \item Did you report error bars (e.g., with respect to the random seed after running experiments multiple times)?
    \answerYes{Yes}
	\item Did you include the total amount of compute and the type of resources used (e.g., type of GPUs, internal cluster, or cloud provider)?
    \answerYes{Yes. In the Model Parameters section}
     \item Do you justify how the proposed evaluation is sufficient and appropriate to the claims made? 
    \answerYes{Yes}
     \item Do you discuss what is ``the cost`` of misclassification and fault (in)tolerance?
    \answerYes{Yes}
  
\end{enumerate}

\item Additionally, if you are using existing assets (e.g., code, data, models) or curating/releasing new assets, \textbf{without compromising anonymity}...
\begin{enumerate}
  \item If your work uses existing assets, did you cite the creators?
    \answerYes{Yes}
  \item Did you mention the license of the assets?
    \answerYes{Yes}
  \item Did you include any new assets in the supplemental material or as a URL?
    \answerYes{Yes. In the supplemental material and As a URL specified in footnote 1 and in the Data Availability section in the appendix}
  \item Did you discuss whether and how consent was obtained from people whose data you're using/curating?
    \answerYes{Yes. In the Data section}
  \item Did you discuss whether the data you are using/curating contains personally identifiable information or offensive content?
    \answerYes{Yes. In the Data section}
\item If you are curating or releasing new datasets, did you discuss how you intend to make your datasets FAIR (see \citet{fair})?
\answerYes{Yes. See Dataset Availability section}% in the Appendix}
\item If you are curating or releasing new datasets, did you create a Datasheet for the Dataset? 
\answerYes{Yes}
\end{enumerate}

\item Additionally, if you used crowdsourcing or conducted research with human subjects, \textbf{without compromising anonymity}...
\begin{enumerate}
  \item Did you include the full text of instructions given to participants and screenshots?
    \answerYes{Yes. In the Data section}
  \item Did you describe any potential participant risks, with mentions of Institutional Review Board (IRB) approvals?
    \answerYes{Yes. In the Data section}
  \item Did you include the estimated hourly wage paid to participants and the total amount spent on participant compensation?
    \answerYes{Yes. In the Data section}
   \item Did you discuss how data is stored, shared, and deidentified?
   \answerYes{Yes. In the Data section and the Data Availability section}% in the appendix}
\end{enumerate}

\end{enumerate}

%\appendix

%\section{Confidence Scores}
%Table \ref{tab:confidence} shows the meaning and explanation of different levels of confidence scores (1-5) used in the annotation task.

%\begin{table}[t!]

%    \small
%\begin{tabular}{|c|l|p{0.5\linewidth}|}%p{0.4\linewidth|}
%    \toprule
%    \bf{Score} & \bf{Meaning} & \bf{Explanation}\\
%    \midrule
%        5 & Extremely confident & I'm certain without a doubt.  \\
%        4 & Fairly confident & I’m confident, but there might be a small chance other annotators may label it as a different category. \\
%        %& & annotators may label it as a different category \\
%        3 & Pretty confident & I’m pretty sure, but there might be a high chance other annotators may label it as a different category. \\
%        %& & annotators may label it as a different category \\
%        2 & Not confident & I’m not sure; it could belong to this or another category. \\
%        1 & Very low confidence & I’m really unsure; it might belong to another category instead.\\
%        \bottomrule
%    \end{tabular}
%    \caption{Confidence Scores used in the annotation task.}
%    \label{tab:confidence}
%\end{table}
\end{document}
