% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[review]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}
\usepackage{booktabs}

\usepackage{subfig, graphicx} 

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
\usepackage{makecell}


% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{UMUTeam at SemEval-2024 Task 4: Multimodal identification of persuasive techniques in memes through LMM}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{First Author \\
  Affiliation / Address line 1 \\
  Affiliation / Address line 2 \\
  Affiliation / Address line 3 \\
  \texttt{email@domain} \\\And
  Second Author \\
  Affiliation / Address line 1 \\
  Affiliation / Address line 2 \\
  Affiliation / Address line 3 \\
  \texttt{email@domain} \\}

\begin{document}
\maketitle
\begin{abstract}
In this manuscript we describe the UMUTeam's participation in SemEval-2024 Task 4, a collaborative task to identify different persuasion techniques within the textual content of a meme alone, as well as in a multimodal setting where both textual and visual content are analysed together. The task was divided into three related subtasks, and we were involved in all three. Our approach is based on a fine-tuning approach with the pre-trained RoBERTa-large model. In addition, for multimodal cases with both textual and visual content, we used the LMM called LlaVa to extract image descriptions and combine them with the meme text. Our system performed well in three subtasks, achieving the tenth-best result with an H-F1 of 0.64774, the fourth best in Subtask 2a with an H-F1 of 0.69003, and the eighth best in Subtask 2b with a macro-F1 of 0.78660.
\end{abstract}

\section{Introduction}
The rise of social media has greatly facilitated the dissemination of information, but its spontaneous nature has also led to the rapid spread of inaccurate information. As a result, misinformation and disinformation have emerged as serious issues on social media platforms, given the challenge of verifying their accuracy. For example, during the COVID-19 pandemic, while social media enabled healthcare professionals to quickly communicate professional information to the public, studies also revealed the detrimental effects of spreading inaccurate health-related information \cite{ferrara2020misinformation}.

Of particular concern is the proliferation of memes, which are used to spread misinformation. Memes, which typically consist of images overlaid with text, are created by Internet users and have become one of the primary forms of content in online disinformation campaigns. Designed specifically to actively spread inaccurate information, ``disinformation memes'' are particularly effective on social media platforms, where they can quickly reach large audiences \cite{qu2022disinfomeme}. Using various rhetorical and psychological techniques such as causal oversimplification, name-calling, and smear tactics, memes play a pivotal role in influencing users' perceptions and beliefs.

In order to counter this phenomenon, the Multilingual Detection of Persuasion Techniques in Memes shared task has been organised at SemEval-2024. The objective of this shared task is to develop models for identifying these techniques solely within the textual content of a meme, as well as in a multimodal setting where both textual and visual content are analysed together. The task is organised around three main sub-tasks: 
\begin{itemize}
    \item \textbf{Subtask 1}. It is a hierarchical multi-label classification problem that consists of identifying which of the 20 persuasion techniques, organized in a hierarchy, are used through textual content. 
    \item \textbf{Subtask 2a}. This subtask involves hierarchical multi-label classification, aiming to identify which of the 20 persuasion techniques, organized hierarchically, is present in multimodal content, including both textual and visual elements.
    \item \textbf{Subtask 3b}. It is a basic task of identifying persuasion techniques, where we determine whether a meme contains a persuasion technique or not.
\end{itemize}

In this paper, we propose an approach based on fine-tuning Transformer models for hierarchical multi-label and binary classification problems of persuasion techniques using textual and visual content. During training for subareas 2a and 2b, we propose to use a large multimodal model (LMM) called LlaVa (Liu et al., 2023) to extract textual features and combine them with the textual content of the meme. Then, we refine the monolingual model, as RoBERTa-large, to identify persuasion techniques and their type.

Besides the system description, we make the following observations based on our experiments: 
\begin{enumerate}
    \item In a multimodal classification problem, image description mixed with textual content improves overall performance. 
    \item Our approach does not constrain the embedding of images and text in the same vector space for fine-tuning or training persuasion classification techniques. Instead, we combine the text of the meme with its description and use this combined set as input for fine-tuning the pre-trained transformer-based model.
\end{enumerate}

The remainder of this paper is organized as follows. Section \ref{sec:background} provides a summary of important details about the task setup.  Section \ref{sec:system-overview} offers an overview of our system for two subtasks. Section \ref{sec:experimental-setup} presents the specific details of our systems. Section \ref{sec:results} discusses the results of the experiments, and finally, the conclusions are shown in Section \ref{sec:conclusion} 

\section{Background}
\label{sec:background}
% Rapido desarrollo de Transformers 
In recent times, there has been a significant increase in social media consumption, with memes being a means of spreading misinformation. Memes consist of a combination of text and images that collectively present some meaning, making it difficult to verify their veracity. They are compact, widely used across social media sites, and often impart a humorous or informative message.  For example, audio, image, or text may convey a benign meaning, but their combination may be derogatory, or vice versa. Hence, criminals and negative entities sometimes exploit this medium as a tool for spreading hateful content. They may spread hate to create unrest among the people, and such hateful content may target communities or individuals based on religion, ethnicity, race, national origin, affiliation, sexual orientation, gender, sex, disability, and disease \cite{10.1145/3623396}.

For this reason, a number of studies have emerged in recent years as a common task to identify memes that contain negative content or misinformation. For example, in \cite{10.1145/3623396}, a dataset of religiously hateful memes has been created and published, and an approach has been proposed that compares and fine-tunes VisualBERT pre-trained on the Conceptual Caption (CC) dataset for the top-down classification task. Visual features have been extracted using ResNeXT-152 Aggregated Residual Transformations based Masked Regions with Convolutional Neural Networks (R-CNN) and Bidirectional Encoder Representations from Transformers (BERT) without textual encoding for the early fusion model. Some other tasks related to this topic, but that did consider the same data and a multimodal approach have previously organised in the same area of interest, i.e. Multimedia Automatic Misogyny Identification of SemEval-2022 \cite{fersini-etal-2022-semeval}, which explores the detection of misogynistic memes on the web using available text and images; DravidianLangTech of EACL-2021, which explores the detection of the offensive language and troll memes classification. 

% Referencia de tareas de meme 
Unlike other tasks, this task focuses on disinformation propaganda through memes. Propaganda uses psychological and rhetorical techniques to achieve its goal. These techniques include the use of logical fallacies and appeals to the audience's emotions. Logical fallacies are often difficult to detect because the argument seems correct and objective at first glance. However, careful analysis reveals that the conclusion cannot be deduced from the premise without the misuse of logical rules. Therefore, memes are a perfect medium for spreading disinformation because they consist of an image superimposed on text, and the image can be deceptive, reinforcing or complementing one or more persuasive techniques in the text or image. Thus, the goal of this task is to identify the existence and type of persuasion techniques through memes with different subtasks. The persuasion techniques can be viewed on the official task page (\url{https://propaganda.math.unipd.it/semeval2024task4/}). 

% Explicar el dataset 
The dataset used for this task is the one provided by the organisers. It consists of a set of texts and images labelled with their corresponding persuasion techniques, or in binary form for Subtask 2b. The data set provided by the organisers is divided into three subsets: train, dev and validation. Note that we do not actually need two datasets for validation (dev and validation), as the dev set was used for the development phase. Therefore, we have combined the train and dev sets into a single training set. Finally, the training set for subtasks 1 and 2a has 8000 examples and 1499 examples in subtask 2b. Figures \ref{fig:sub1-2a-distribution} and \ref{fig:sub2b-distribution} show the distribution of the training set for subtasks 1, 2a and 2b.

\begin{figure}[ht]
\centering
\includegraphics[width=0.5\textwidth]{Subtask 1 y Subtask 2a training set distribution.png}
\caption{Subtask 1 y Subtask 2a training set distribution}
\label{fig:sub1-2a-distribution}
\end{figure}

\begin{figure}[ht]
\centering
\includegraphics[width=0.5\textwidth]{Subtask 2b training set distribution.png}
\caption{Subtask 2b training set distribution}
\label{fig:sub2b-distribution}
\end{figure}

\section{System overview}
\label{sec:system-overview}

\begin{figure*}[htb]
\centering
\includegraphics[width=1\textwidth]{semaval4-arquitectura.png}
\caption{System architecture approach}
\label{fig:arquitectura}
\end{figure*}

Figure \ref{fig:arquitectura} shows the architecture of our system for the three subtasks. We can see that for subtask 1, only the context of the memes is used, which is a multi-label classification problem of different persuasion techniques. To address this, we have adopted a fine-tuning approach of a pre-trained model called RoBERTa-large \cite{DBLP:journals/corr/abs-1907-11692}. This approach involves using RoBERTa-large as a base and adapting it with the training set for persuasion technique classification. Fine-tuning involves adjusting the weights of the model so that it can more effectively learn relevant features from the input data and make accurate predictions for multiple output labels. With this approach, we can use the prior knowledge captured by RoBERTa-large to improve performance in the classification task.

RoBERTa-large is a Transformer-based model that was pre-trained on a large corpus of English data in a self-supervised manner, i.e. it was pre-trained only on raw text without any human labelling. Specifically, it was pre-trained with the Masked Language Modeling (MLM) goal, where the model randomly masks 15\% of the words in the input and then runs the entire masked sentence through the model, predicting the masked words. In this way, the model learns an internal representation of the English language, which can then be used to extract useful features for downstream tasks such as classification. 

For Subtasks 2a and 2b, we have used the same approach as in Subtask 1, but in this case both the textual and visual content of the memes are used. As shown in Figure \ref{fig:arquitectura}, we used a Large Multimodal Model (LLM) called LlaVa \cite{liu2023llava} to extract the image description and then concatenate this information with the textual content of the memes, as shown in Figure \ref{fig:input-multimodal}. LlaVa is an end-to-end trained large multimodal model that combines a vision encoder and a large language model (LLM) for general purpose visual and language understanding. In addition, it has demonstrated impressive multimodal conversational capabilities, sometimes exhibiting behaviour similar to the multimodal GPT-4 on unseen images/instructions, and achieving a relative score of 85.1\% compared to GPT-4 on a synthetic multimodal instruction-following dataset. It is worth noting that, unlike Subtask 2a, Subtask 2b is a binary classification problem that aims to identify whether a meme contains persuasion techniques or not.

\begin{figure*}[htb]
\centering
\includegraphics[width=1\textwidth]{input-multimodal.png}
\caption{Example of multimodal input for Subtasks 2a and 2b.}
\label{fig:input-multimodal}
\end{figure*}

\section{Experimental setup}
\label{sec:experimental-setup}
The dataset used for training and validation of the three subtasks is provided by the organisers. It consists of a collection of texts and images labelled with their corresponding persuasion techniques, or in binary form for Subtask 2b. The dataset provided by the organisers is divided into three subsets: train, dev and validation. It is worth noting that we do not actually need two datasets for validation (dev and validation), as the dev set was used during the development phase. Therefore, we have combined the train and dev sets into a single training set. Finally, the training set for Subtasks 1 and 2a has 8000 examples, and there are 1499 examples in Subtask 2b, as shown in Figures \ref{fig:sub1-2a-distribution} and \ref{fig:sub2b-distribution}. 

Prior to fine-tuning, a pre-processing was performed to remove line breaks, hashtags, symbols such as "*", references, and possible URLs. For subtasks 1, 2a and 2b, we performed the fine-tuning process using an epoch-based evaluation strategy with the Huggingface Trainer library\footnote{\url{https://huggingface.co/docs/transformers/main_classes/trainer}}. This involves training the pre-trained model for a certain number of epochs and performing an evaluation with the evaluation set after each epoch. Once all epochs have been completed, the model with the best macro F1 score in the evaluation set is selected. In this way, overfitting or underfitting leading to low variance and high bias can be avoided. 

For all three subtasks (1, 2a, and 2b), we used the same hyperparameters for fine-tuning, namely: a batch size of 8 for both training and validation, 10 epochs, a learning rate of 2e-5, and a weight decay of 0.01. During training, we used macro-F1 as a reference. For the evaluation of subtasks 1 and 2a, the organisers used hierarchical-F1 as the primary evaluation metric, and for subtask 2b, macro-F1. Hierarchical-F1 is an evaluation metric used in hierarchical classification problems where classes are organised in a hierarchical structure. This metric considers both precision and recall of the prediction for each class in the hierarchy, taking into account the relationship between parent and child classes in the hierarchy. Meanwhile, the macro F1 score is an evaluation metric used in classification problems to measure the precision and recall of a model in predicting multiple classes. It assigns equal weight to each class, meaning that all classes have the same impact on the final metric, regardless of their size or distribution in the data.

\section{Results}
\label{sec:results}

Table \ref{tab:official-results-subtask1} shows the official ranking table for Subtask 1. We can see that we have achieved the tenth position with a hierarchical F1 score of 0.64774 and a hierarchical precision of 0.70817. We have exceeded the baseline by almost 50\% in terms of hierarchical F1 and are 10.473\% away from the best result, which is 0.75247, achieved by the group 914isthebest.
\begin{table}[htb!]
\centering
\caption{Official results for the subtask 1.}\label{tab:official-results-subtask1}
\begin{tabular}{lrrrr}
\toprule
\textbf{Team} &  \textbf{Rank} &  \textbf{H-F1} &    \textbf{H-P} &    \textbf{H-R}\\
\midrule
914isthebest                                      &    1 & 75.247 & 68.419 & 83.590 \\
BCAmirs                                           &    2 & 69.857 & 66.786 & 73.223 \\
\makecell[lc]{Otterly\\Obsessed\\With\\Semantics} &    3 & 69.738 & 64.801 & 75.490 \\
TUMnlp                                            &    4 & 67.384 & 63.781 & 71.419 \\
GreyBox                                           &    5 & 66.998 & 65.248	& 68.844 \\
\multicolumn{5}{c}{\ldots} \\
UMUTeam                                           &    10 & \textbf{64.774} & \textbf{70.817} & \textbf{59.681} \\
\multicolumn{5}{c}{\ldots} \\
Baseline                                          &    - & 36.865 & 47.711 & 30.036 \\
\bottomrule
\end{tabular}
\end{table}

For Subtask 2a, we achieved a hierarchical F1 of 0.69003 and a hierarchical precision of 0.76763, which is the fourth-best result according to the official ranking table (see Table \ref{tab:official-results-subtask2a}). With our approach, we outperformed the baseline by 24.297\% and are only 5.589\% away from first place, which achieved an H-F1 of 0.74592.

\begin{table}[htb!]
\centering
\caption{Official results for the Subtask 2a.}\label{tab:official-results-subtask2a}
\begin{tabular}{lrrrr}
\toprule
\textbf{Team} &  \textbf{Rank} &  \textbf{H-F1} &    \textbf{H-P} &    \textbf{H-R}\\
\midrule
\makecell[lc]{Hierarchy\\Everywhere} &    1 & 74.592 & 86.682 & 65.461 \\
NLPNCHU &     2 & 70.677 & 78.164 & 64.498 \\
BCAmirs &     3 & 70.497 & 78.374 & 64.059 \\
UMUTeam &     4 & \textbf{69.003} & \textbf{76.763} & \textbf{62.669} \\
\multicolumn{5}{c}{\ldots} \\
Baseline &     - & 44.706 & 68.778 & 33.116\\
\bottomrule
\end{tabular}
\end{table}

For subtask 2b, which is a binary classification problem to identify the presence of persuasion techniques in memes, we obtained a macro-F1 score of 0.78660, which puts us in eighth place according to the official ranking table (see Table \ref{tab:official-results-subtask2b}). Furthermore, we can see that our system has improved by up to 53.66\% compared to the baseline and is only 2.37\% behind the first place (LMEME with a M-F1 of 0.81030).

\begin{table}[htb!]
\centering
\caption{Official results for the Subtask 2b.}\label{tab:official-results-subtask2b}
\begin{tabular}{lrrrr}
\toprule
\textbf{Team} &  \textbf{Rank} &  \textbf{M-F1} &    \textbf{m-F1} \\
\midrule
LMEME        &     1 & 81.030 & 82.500 \\
SuteAlbastre &     2 & 80.964 & 83.500  \\
DUTIR938     &     3 & 80.910 & 83.667  \\
BCAmirs      &     4 & 80.337 & 82.500  \\
Snarci       &     5 & 79.860 & 82.667 \\
\multicolumn{4}{c}{\ldots} \\
UMUTeam      &     8 & \textbf{78.660} & \textbf{80.667} \\
\multicolumn{4}{c}{\ldots} \\
Baseline     &    - & 25.000 & 33.333  \\
      
\bottomrule
\end{tabular}
\end{table}

Based on the results obtained, it's clear that combining image descriptions with textual content improves overall performance in a multimodal setting. This approach doesn't impose any restrictions on embedding images and text together in the same vector space when fine-tuning or training persuasion classification techniques. Rather, we merge the text from the meme with its description and use this combined dataset as input for fine-tuning the pre-trained transformer-based model.

\section{Conclusion}
\label{sec:conclusion}
In this paper, we describe the UMUTeam's participation in the common task 4 of SemEval 2024. The goal was to develop models for identifying persuasion techniques in meme text alone, as well as in a multimodal setting where both text and visual content are analysed. The task consists of three subtasks: Subtask 1, a hierarchical multi-label classification problem to identify among 20 persuasion techniques; Subtask 2a, also a hierarchical multi-label classification aimed at identifying techniques in multimodal content, including text and visuals; and Subtask 2b, focused on determining whether memes contain persuasion techniques. For all three subtasks, we used the fine-tuning approach with the pre-trained RoBERTa-large model. In addition, for multimodal cases with both textual and visual content, we used the LMM named LlaVa to extract image descriptions and combine them with the meme text. In terms of results, our system achieved the tenth-best result with an H-F1 of 0.64774, the fourth-best in Subtask 2a with an H-F1 of 0.69003, and the eighth-best in Subtask 2b with a macro-F1 of 0.78660.



\section*{Acknowledgements}

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

\appendix


\end{document}
