% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}
\usepackage{hyperref}


% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
%\usepackage[review]{acl}
\usepackage{acl} 

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{float}
\usepackage{makecell}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
\usepackage{graphicx}
% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.


\usepackage{url}

\title{914isthebest at SemEval-2024 Task 4: CoT-based Data Augmentation Strategy for Persuasion Techniques Detection}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line, use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Dailin Li\textsuperscript{1}, 
Chuhan Wang\textsuperscript{1}, Xin Zou \textsuperscript{1}, Junlong Wang\textsuperscript{2}, Peng Chen\textsuperscript{1}\\\textbf{Jian Wang \textsuperscript{1}\footnotemark[2]},\textbf{Liang Yang\textsuperscript{1}},\textbf{Hongfei Lin\textsuperscript{1}}  \\
\textsuperscript{1}School of Computer Science and Technology, Dalian University of Technology, China \\
\textsuperscript{2}School of Software, Dalian University of Technology, China \\
 \texttt{\{ldlbest,wangchuhan,zouxin,jlwang,pengchen\}@mail.dlut.edu.cn} \\
 \texttt{\{wangjian,liang,hflin\}@dlut.edu.cn } \\ }

%\author{First Author \\
%  Affiliation / Address line 1 \\
%  Affiliation / Address line 2 \\
%  Affiliation / Address line 3 \\
%  \texttt{email@domain} \\\And
%  Second Author \\
%  Affiliation / Address line 1 \\
%  Affiliation / Address line 2 \\
%  Affiliation / Address line 3 \\
%  \texttt{email@domain} \\}

\begin{document}
\maketitle
\begin{abstract}
%\footnotetext{\equalcontribution}
%\footnotetext{\corr}
Memes are commonly used in online disinformation campaigns, particularly on social media platforms. They are primarily effective on social media platforms since they can easily reach many users. Semeval2024-Task4\citep{semeval2024task4}, "Multilingual detection of persuasion techniques in memes", focuses on detecting persuasive methods across four languages: English, Bulgarian, North Macedonian and Arabic. Subtask 1 aims to identify the given text fragments of memes and which of the 20 persuasion techniques it uses, organized in a hierarchy. For the difficulty of this task and the fundamental role of text in the artificial intelligence area, we concentrate solely on this task.
We develop a system using CoT-based data augmentation methods,in-domain pretraining and ensemble strategy that combines the strengths of both RoBERTa and DeBERTa models. Our solution achieved the top ranking among \textbf{33} teams in the English track during the official assessments. We also analyze the impact of architectural decisions, data construction and training strategies. We release our code at \href{https://github.com/ldlbest/semeval2024-task4}{https://github.com/ldlbest/semeval2024-task4}


\end{abstract}

\section{Introduction}

In the present digital era, persuasive communication is pivotal across diverse arenas, from political discourses to the viral spread of content on social media platforms. A nuanced comprehension of the intricacies of persuasion is indispensable in safeguarding against misinformation, upholding the integrity of information, and nurturing a constructive digital discourse.

In online communication, memes have become decisive for disseminating information and influencing opinions. The focus of this task centres on addressing the intricate task of identifying persuasive techniques within the textual content of memes. This paper addresses the "Textual Persuasion Technique Identification" task, emphasizing recognizing persuasive techniques within meme text. Our approach aims to deliver a robust multi-label classification system tailored to navigate the intricate challenges posed by this task.

We employ the Transformer \citep{vaswani2017attention} architecture. We introduce ensemble learning \citep{breiman1996bagging}, integrating one DeBERTa \citep{he2021DeBERTa} model and four RoBERTa \citep{Liu2019RoBERTaAR} models, each trained with different random seeds. 
In the pretraining phase of our system development, we utilize the in-domain pretraining method to improve our model's context and semantic comprehension. To bolster our dataset, we incorporate additional data from similar past tasks.
Furthermore, we implemented the data augmentation technique, enhancing data diversity by employing data augmentation techniques.

Below is a summary of our contributions:
\begin{itemize}

\item[$\bullet$] We augment the training dataset with a Chain-of-Thought (CoT) based data augmentation method and improve our model's performance.

\item[$\bullet$] Our system utilizes in-domain pretraining to enhance performance and leverages ensemble learning to combine DeBERTa and RoBERTa for further improvements.

\item[$\bullet$] In task 1, we achieve first place on the English test set among \textbf{33} participants with an F1 score of \textbf{0.752}.
\end{itemize}


\section{Background}
\subsection{Persuasion Techniques}
Persuasive communication wields a critical influence across various sectors, including political rhetoric and the spread of online content. Such communication is instrumental in guiding public discourse and moulding opinions, ensuring its significance in the modern digital landscape \citep{yu-etal-2021-interpretable}. This task extends these concepts to analyzing memes, an increasingly prevalent medium on social media and internet platforms. With their distinctive blend of fun and brevity, memes deftly navigate the web to share insights, provoke conversation, and distribute knowledge.

In our task, we concentrate on identifying persuasive techniques within textual content. According to the work of \citet{piskorski2023}, this task involves categorizing textual persuasive techniques into three subtypes: ethos, pathos, and logos. This taxonomy is further amplified to include 20 subordinate precise methods, providing an extensive framework for understanding and interpreting the art of persuasion in digital content.




\subsection{Data Augmentation}

Data Augmentation (DA)  techniques are usually initially explored in computer vision (CV), but they have been relatively slow to gain traction in NLP. Challenges arise due to the discrete nature of language, which rules out continuous noise and makes it hard to maintain diversity \citep{Feng2021ASO}. Although challenges exist, the evolution of NLP has led to an increasing demand for exploring tasks and domains with insufficient training data.
Consequently, this trend has resulted in the proliferation of research studies utilizing DA techniques. 
One classical DA method is back translation \citep{sennrich-etal-2016-improving}, which involves translating the text into another language and then back into the original language.
\citet{wei-zou-2019-eda} proposed EDA to improve the performance of text classification tasks and exhibit solid results on smaller datasets. 
These techniques are helpful for augmenting data, but they only modify the original text in fundamental ways, sometimes even changing the entire meaning of the sentence.
Additionally, \citet{Chen2023Few-shot} proposed knowledge-guided data augment based on the semantic relations of the knowledge graph. 


Recently, Large Language Models (LLMs) can provide a unified solution for various NLP tasks and achieve competitive performance \citep{Zhao2023ASO}. 
For example, GPT-3 \citep{gpt30} and ChatGPT \citep{ouyang2022training} have demonstrated strong performance in various NLP tasks and benchmark tests \citep{qin2023chatgpt}.
Furthermore, LLMs play a role in data augmentation, enhancing their utility in multiple applications.
\citet{dai2023chataug} introduced a text data augmentation approach based on ChatGPT, which can be used in downstream model training.
\citet{abaskohi-etal-2023-lm} proposed Contrastive Paraphrasing-guided Prompt-based Fine-tuning of Language Models (LM-CPPF). To enhance the capacity of LLMs for intricate reasoning tasks, \citet{wei2022chain} proposed Chain-of-Thought (CoT). Inspired by the effectiveness of the CoT method, we leverage CoT prompts to generate paraphrases used for data augmentation, ensuring the preservation of semantic consistency while significantly expanding our dataset. This augmentation strategy contributed to the enhanced performance of our model.






\section{Our System}
As depicted in Figure~\ref{fig:system}, our system comprises the following parts: Data Model, in-domain Pretraining, RoBERTa encoder, DeBERTa encoder and Soft Voting. The final prediction is obtained as  \( \hat{y}\).
We ignore the hierarchical structure of the labels and define it as a multi-label classification problem \citep{tsoumakas2007multi} for the labels of the training dataset are all final nodes of the graph.
\begin{figure*}
    \centering
    \includegraphics[width=1\linewidth]{latex/oursystem.png}
    \caption{The overall architecture of our system.}
    \label{fig:system}
\end{figure*}


\subsection{Dataset Construction}

We construct different data augmentation datasets based on various data augmentation strategies. In practice, while efforts to balance label distribution (such as using techniques like Nlpaug and CoT) aim to increase the number of samples for less frequent labels, it is essential to note that, since the data typically involves multiple labels, they can also result in the expansion of more frequent labels.

\textbf{Nlpaug:}
We identify labels corresponding to train data with fewer than 1000 entries. Subsequently, we employ the nlpaug \citep{ma2019nlpaug} 
 library for these data points to implement data augmentation. Specifically, we utilize the method of synonym replacement, generating new training samples by substituting words in the text with their synonyms. This approach enhances the diversity of training data, thereby enhancing the model's robustness to different text inputs. Using this method, we augment more than 5700 data entries in total.

\textbf{CoT-based Paraphrasing-Guided Data Augmentation:}
We filter data corresponding to labels that occupy less than 0.16 of the entire label distribution and rewrote these entries using GPT-3.5, generating 10,000 entries through this method. The LLM can fully understand the context and focus on improving the targeted content by explaining the labels and tasks and providing a specific description of the problem and data that need to be rewritten. Applying the CoT technique enables the model to acquire more information and generate improved augmented data. 


\begin{figure}[H]
    \centering
    \includegraphics[width=1\linewidth]{latex/cot.png}
    \caption{An example of using CoT for data augmentation.}
    \label{fig:COT}
\end{figure}

We illustrate Figure~\ref{fig:COT}. In the third round of our conversation with GPT-3.5, we use the instruction "Generate a paraphrase of the following text using different words and sentence structures while still conveying the same meaning" because it accurately describes the task with its instructions. \citet{abaskohi-etal-2023-lm} proved its effectiveness as an instruction template.


\begin{table*}
\centering
\setlength{\tabcolsep}{20pt}
\begin{tabular}{lccc}
\hline
\textbf{Method}& \textbf{Recall}& \textbf{Precision}& \textbf{F1-score}
\\
\hline
BCAmirs& 0.732 &   0.668&   0.699 \\
OtterlyObsessedWithSemantics&  0.755&   0.648&   0.697 \\
TUMnlp&  0.714&   0.638 &  0.674 \\
GreyBox& 0.688 &  0.652 &  0.670 \\
BCAmirs	&0.690	&0.640 &0.664 \\
LomonosovMSU&0.632 &0.674 &0.652\\
NLPNCHU	&0.706&	0.604 &0.651 \\
Baseline& 0.300& 0.477	&0.369	\\
Our System&	\textbf{0.836}& \textbf{0.684}& \textbf{0.752}\\
\hline
\end{tabular}
\caption{
Comparison of the performance between other team's models on Task 1 English test dataset.
}\label{table1}
\end{table*}

\textbf{Pseudo-labeling:} We use our model to classify 1000 data points on the dev dataset and 1500 on the test dataset. Pseudo-labelling \citep{Pseudo-labeling} employs labelled data for training and utilizes information from unlabeled data to enhance the model's performance. The objective is to make more complete use of available data resources and improve the model's performance.


\subsection{In-domain Pretraining}
We utilize Masked Language Model (MLM) pertaining to all data, including data from SemEval 2023 task 3, which injects in-domain knowledge of our training datasets, thereby encouraging better learning outcomes for the model.

For a given input text \( x \), we first tokenize it to obtain the tokenized representation \( x_{tokenized} \) and truncate or pad them according to the maximum sequence length. 
For each text, a certain proportion of tokens are randomly masked based on the MLM probability and replaced with the masked token \([MASK]\).
We use cross-entropy loss as the loss function for the masked language model. We compare the model's predicted probabilities for each token position with the actual token's one-hot encoding and calculate the cross-entropy loss.

\begin{equation}
    \label{eq:loss_function}
    L(t, \hat{t}) = -\frac{1}{N}\sum_{i=1}^N t_i \log(\hat{t}_i)
\end{equation}

where \( t\) is the encoding of the true token, and \(\hat{t} \) is the probability distribution of the model's predictions.

\subsection{Ensemble Learning}

We train four RoBERTa models and one DeBERTa model using different random seeds. We integrate them using the soft voting approach, which averages the predicted probabilities of each label from all five models. Given predictions {\( p_{i1}, ..., p_{iN}\) } for class \(i\) from these models, we employ the following formula to obtain the final prediction  \( \hat{p_i}\) 

\begin{equation}
    \hat{p_i} =\frac{1}{N}\sum_{j=1}^Np_{ij} 
\end{equation}

We then set a threshold of 0.25, where  \( \hat{p_i}\) greater than the threshold is chosen as the predicted label.


\begin{table*}
\centering
\setlength{\tabcolsep}{20pt}
\begin{tabular}{lccc}
\hline
\textbf{Method}& \textbf{Recall}& \textbf{Precision}& \textbf{F1-score}\\
\hline
GPT-3.5& 0.457 &0.385 &0.418\\
GPT-4& 0.432 &  0.482& 0.456\\
\hline
CLaC&  \textbf{0.967}&   \textbf{0.808}&   \textbf{0.881}\\
OtterlyObsessedWithSemantics & 0.754& 0.636& 0.690\\
GreyBox& 0.716 & 0.657 & 0.685\\
EURECOM& 0.702 & 0.650 & 0.675\\
Baseline& 0.291 & 0.466 & 0.358\\
 \hline
Our System&	0.727 & 0.636 & 0.678   \\
\hline
\end{tabular}
\caption{
Comparison of the performance between LLM and other team models on Task 1 English dev.
}\label{tabledev}
\end{table*}
\subsection{Low-Resource Languages}
Since our training data is limited to English, we utilize GPT-3.5 to translate Bulgarian, North Macedonian and Arabic datasets into English. Subsequently, we perform inference on the translated data. The results obtained from these experiments can be found in the \ref{sec:appendix2}. For the loss 
 of information during translation, our system gets a relatively low F1 score in these languages.

\section{Experimental Setup}
The completion is based on PyTorch, Transformers and Pytorch-Lighting. During training, we set the batch size as 16, the learning rate as 3e-5, and the warmup steps ratio as 0.3.
Five seeds(42,3407,114514,4096,1234) are used for the label ensemble. We use the AdamW optimizer and the cosine decay scheduler with a power of 0.01. We set a maximum epoch of 7. All experiments are run on one RTX 4090 GPU.

We create three additional datasets for the experiment: GPTDataset, PseudoDataset, and GPT-PseudoDataset (GPT-PDataset). The GPTDataset contains 7,500 training data and 10,686 sentences generated by LLM. PseudoDataset contains 7500 training data and test dev dataset labelled by the Ensemble model. GPT-PDataset is a union of GPTDataset and PseudoDataset.
\section{Result and Analysis}
In this section, we display our results and analyze the impact of each component through ablation
studies.
\subsection{Results}
In this competition with 33 teams, we achieve first place with a hierarchical F1 score of 0.75247. We outperform the official baseline by 0.38382. The result is shown in Table ~\ref{table1}.


We conduct a comparative analysis between our system and other models, including LLMs on the Task 1 English dev set, revealing the superior performance of our approach.

As illustrated in Table~\ref{tabledev}, 
GPT-3.5 \citep{ouyang2022training} and GPT-4 \citep{openai2023gpt4} utilized zero-shot learning, where only label meanings were provided in textual form without specific examples. 
We compare ourselves with other participating teams; the results are shown in Table~\ref{tabledev}. We achieve fourth place with a Hierarchical F1 score of 0.67833. Our performance is significantly better than the official baseline by 0.32010.





\subsection{Ablation Study}
We also conduct ablation experiments to validate our designs, including the encoder model, data modules, training strategy and ensemble.\\


\textbf{Encoder Model}
We build our baseline model
with BaselineDataset and BCE loss and run experiments to find out the best encoder model
among RoBERTa \citep{Liu2019RoBERTaAR}, BERT \citep{devlin-etal-2019-bert} , DeBERTa \citep{he2021DeBERTa}
, etc. As shown in Table ~\ref{table2}, the large
version of DeBERTav3 achieves the best score.
Due to limited computility, we chose RoBERTa as our base model.
\begin{table}[H]
\centering
\begin{tabular}{lcll}
\hline
\textbf{Method}& \textbf{F1-score}\\
\hline
$BERT_{base}$&0.542 \\
$BERT_{large}$  &0.576 \\ 
$RoBERTa_{base}$ &0.614 \\
$RoBERTa_{large}$ & 0.632 \\
$DeBERTav3_{large}$ & \textbf{0.649} \\
\hline
\end{tabular}
\caption{F1 score of different Transformer-based models.}
\label{table2}
\end{table}

\textbf{Training strategy}
We apply the in-domain pretraining on all encoder-based models to facilitate their performance on the downstream task. The result is shown in Table~\ref{tablemlm3}.
For the five models, the F1 score improved by 0.2.

\begin{table}[H]
\centering
\begin{tabular}{lcll}
\hline
\textbf{Method} & \textbf{F1-score}\\
\hline
$BERT_{base}$&0.599 \\
$BERT_{large}$&0.613 \\ 
$RoBERTa_{base}$ &0.630 \\
$RoBERTa_{large}$ &0.664 \\
$DeBERTav3_{large}$& \textbf{0.667} \\
\hline
\end{tabular}
\caption{F1 score of models after MLM training.}
\label{tablemlm3}
\end{table}

\textbf{Data Module}
We use the best encoder model based on the result of dev datasets for ablation experiments on different datasets, including PseudoDataset, GPTDataSet, and GPT-PseudoDataset. The results are shown in Table ~\ref{table4}. 

\begin{table}[H]
\centering
\begin{tabular}{lcc}
\hline
\textbf{Method}& \textbf{dataModule} &\textbf{F1-score}\\
\hline
$RoBERTa_{large}$ & GPTDataSet &0.685 \\
$DeBERTav3_{large}$ & GPTDataSet &0.700 \\
$RoBERTa_{large}$ &PseudoDataset & 0.707 \\
$DeBERTav3_{large}$ & PseudoDataset& 0.704 \\
$RoBERTa_{large}$ & GPT-PDataset& 0.718 \\
$DeBERTav3_{large}$ &GPT-PDataset& \textbf{0.719} \\
\hline
\end{tabular}
\caption{results on different dataset.}
\label{table4}
\end{table}
 
\textbf{Ensemble}
Our ensemble approach can significantly improve performance. We integrate the results of different seeds and models based on their performance on the dev set. The result is shown in \ref{sec:appendix}, where we can see our ensemble approach outperforms the best single model by
0.15 F1 score over the dev set. 

\section{Conclusion}

This paper details the architecture and performance of our multi-label classification system designed for the Persuasion Techniques Detection task. Our system achieves the highest rank for English in the leaderboard, signalling a notable accomplishment in the competitive framework. A comprehensive analysis of the data characteristics and model dynamics informs the strategic modifications we institute to the dataset construction and model training strategy. The efficacy of these refinements is corroborated by extensive empirical evaluation. 



For future research, exploring methods to integrate the informational richness of hierarchical labels within the multi-label classification framework and fully exploiting LLMs to identify persuasion techniques remain promising avenues for further exploration.


% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

\appendix

\section{Appendix}


    \subsection{Model Result on Multilingual Datasets}
    \label{sec:appendix2}
\begin{table}[H]
\centering

\begin{tabular}{lCCC}
\hline
\textbf{Method}& \textbf{Recall}& \textbf{Precision}& \textbf{F1-score}\\
\hline
English& \textbf{0.836}&\textbf{0.684}&\textbf{0.752}\\
Bulgarian& 0.450&  0.477& 0.463\\

North Macedonian&  0.340&   0.401&   0.369
\\
 Arabic& 0.436& 0.285&0.345\\
 \hline
\end{tabular}
\caption{
Performance Metrics on Multilingual Datasets
}\label{otherlanguage}
\end{table}

\subsection{Ensemble Model Result}
    \label{sec:appendix}
    \begin{table}[H]
    \centering
    \begin{tabular}{lcll}
    \hline
    \textbf{Method}& \textbf{seeds}& \textbf{F1-score}\\
    \hline
    $RoBERTa_{large}$&42 & 0.698 \\
    $RoBERTa_{large}$ &3407 &  0.697 \\ 
    $RoBERTa_{large}$ &4096 & 0.694 \\
    $RoBERTa_{large}$ &1234 &   0.695\\
    $RoBERTa_{large}$ &1145145 & 0.696\\
    $RoBERTa_{large}$ &42,3407,4096 &  0.709 \\
    $RoBERTa_{large}$ &42,3407,114514 &   0.710 \\ 
    $RoBERTa_{large}$ &3407,4096,114514 & 0.710 \\
    $RoBERTa_{large}$ &42,4096,114514 &  0.712\\
    $RoBERTa_{large}$ &42,3407,4096,114514 &   0.713\\
    \makecell[l]{$RoBERTa_{large}$\\$DeBERTav3_{large}$} &\makecell[c]{42,3407,4096,114514 \\42} &\textbf{0.718} \\
    \hline
    \end{tabular}
    \caption{Ensemble of different methods and seeds }
    \label{table5}
    \end{table}

\end{document}
