% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[preprint]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\usepackage{listings}
\lstdefinelanguage{json}{
    basicstyle=\small\ttfamily,
    numbers=left,
    numberstyle=\scriptsize,
    stepnumber=1,
    numbersep=8pt,
    showstringspaces=false,
    breaklines=true,
    frame=lines,
    xleftmargin=15pt
}

\usepackage{hyperref}
\usepackage{enumitem}
\setlist[itemize]{noitemsep, nolistsep}


\title{LomonosovMSU at SemEval-2024 Task 4: Comparing LLMs and embedder models to identifying propaganda techniques in the content of memes in English for subtasks №1, №2a, and №2b}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Gleb Skiba${}^1$${}^,$${}^2$ \hspace{12pt} Mikhail Pukemo${}^2$ \hspace{12pt} Dmitry Melikhov${}^2$ \hspace{12pt} Konstantin Vorontsov${}^1$${}^,$${}^2$ \\
Institute of AI, Lomonosov Moscow State University${}^1$, \\
The faculty of Computational Mathematics and Cybernetics, \\
Lomonosov Moscow State University${}^2$ \\
\texttt{gleb-skiba@mail.ru, iMan.assistance@gmail.com,} \\
\texttt{melikhov.dmitry.a@gmail.com, Voron@mlsa-iai.ru}}


% \author{ath1 \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\\And
%   Second Author \\
%   Affiliation / Address line 1 \\
%   Affiliation / Address line 2 \\
%   Affiliation / Address line 3 \\
%   \texttt{email@domain} \\}

\begin{document}

\maketitle
\begin{abstract}
This paper presents the solution of the LomonosovMSU team for the SemEval-2024 Task 4 "Multilingual Detection of Persuasion Techniques in Memes" competition for the English language task. During the task solving process, generative and BERT-like (training classifiers on top of embedder models) approaches were tested for subtask №1, as well as an BERT-like approach on top of multimodal embedder models for subtasks №2a/№2b. The models were trained using datasets provided by the competition organizers, enriched with filtered datasets from previous SemEval competitions. The following results were achieved: 18th place for subtask №1, 9th place for subtask №2a, and 11th place for subtask №2b. The code for the solutions is available at github\footnote{\url{https://github.com/pansershrek/Semeval2024_LomonosovMSU}}.
\end{abstract}

\section{Introduction}

In the modern world, memes are one of the most popular forms of delivering information to social media users. Unfortunately, memes created using a variety of rhetorical and psychological techniques are also used to conduct disinformation campaigns.

The overall goal of the SemEval-2024 Task 4 competition is to build models to detect rhetorical and psychological propaganda techniques in memes. The competition itself consists of three subtasks:

1. Build a model to detect rhetorical and psychological techniques only in the textual content of the meme. This is a hierarchical multilabel classification problem.

2a. Build a model to detect rhetorical and psychological techniques in both textual and visual contexts of the meme (multimodal task). This is a hierarchical multilabel classification problem.

2b. Build a model to identify the presence of rhetorical and psychological techniques in both textual and visual contexts of the meme in general. This is a binary classification problem.

In this work, experiments were conducted with generative and BERT-like models to solve subtask №1 and classifiers on top of multimodal models to solve subtasks №2a/№2b. BERT-like approaches refer to the creation classifiers over embedder models and will be used further in this article. All tasks were solved for datasets in English. The following results were achieved: 18th place for subtask №1, 9th place for subtask №2a, and 11th place for subtask №2b. The code for the solutions is available in the repository at GitHub\footnote{\url{https://github.com/pansershrek/Semeval2024_LomonosovMSU}}.

\section{Related Works}

Supervised fine-tuning (SFT)~\citep{ouyang2022training} is one of the most popular methods for fine-tuning LLMs to solve various tasks. In this work, we conducted fine-tuning of LLMs, such as LLAMA\footnote{\url{https://huggingface.co/meta-llama/Llama-2-7b-chat-hf}} and Mistral ~\citep{jiang2023mistral}, for detecting propaganda techniques. Unfortunately, SFT is very resource-intensive, and conducting frequent experiments with fine-tuning LLMs is time-consuming and expensive. To address this problem, a less resource-intensive approach, LoRA~\citep{hu2021lora}, was used, but unfortunately, the results of this approach were not satisfactory.

In parallel with SFT and LoRA approaches, experiments were conducted on fine-tuning simple classifier layers on top of embedding models: debert~\citep{he2021deberta}, CLIP~\citep{radford2021learning}, BLIP~\citep{li2022blip}. The results of these experiments yielded comparable results to experiments with LLMs.


\section{Tasks solutions}

\subsection{Subtask №1}

This subtask represents a hierarchical classification task. Two approaches were used to solve this task:

1. Generative approach: This approach involves training a generative model to generate explicit responses to questions in JSON format.

2. BERT-like approach: This approach involves training a simple fully connected network on top of a frozen pre-trained embedding model to solve the hierarchical classification task.
 \subsubsection{Generative approach}

\textbf{Idea.}
The main idea of this approach is to train a generative model, both in SFT mode~\citep{ouyang2022training} and trained using the LoRA technique~\citep{JRC132862}, to answer the question: "Does the provided text contain any propaganda techniques, and if so, which ones?"

\textbf{Dataset.}
The following combinations of data were used for training the models:

1. For selecting the best candidate model for the final solution, the training dataset provided by the authors was used, along with the dataset from the previous semeval competition~\footnote{\url{https://propaganda.math.unipd.it/semeval2023task3/}}. This dataset was filtered, and only samples containing propaganda techniques matching those from the current competition were taken.

2. For training the model for the final solution, the same dataset described earlier was used, along with all samples from the gold dataset added to it.	

\textbf{Data Format.}
The following prompt phrase was used: "Your goal is to identify rhetorical and psychological techniques in the given text." The model was required to output JSON with propaganda techniques or an empty JSON. The model received texts in the following format: "Your goal is to identify rhetorical and psychological techniques in the given text.\textbackslash nInput:…\textbackslash nOutput:”.

%\begin{table*}
\begin{figure*}
%\begin{tabular}{c}
%\centering
\begin{lstlisting}[frame=single, language=json,firstnumber=1]
{
"id": "train_71410",
"input": "Your goal is to identify rhetorical and psychological techniques in the given text.\nInput:Tunnel to Schiff\nBunker where our next President will be chosen.\nOutput:",
"output": [
        "Causal Oversimplification",
        "Smears"
    ],
"full_input": "Your goal is to identify rhetorical and psychological techniques in the given text.\nInput:Tunnel to Schiff\nBunker where our next President will be chosen.\nOutput:[\"Causal Oversimplification\", \"Smears\"]"
}
\end{lstlisting}
\label{test}
\caption{Sample from the dataset example}
%\end{table*}
%\end{tabular}
\end{figure*}


All texts from the dataset were filtered as follows:

1. All unnecessary line breaks in the texts were removed.

2. All unnecessary "\textbackslash" characters in line breaks were removed. That is, if the text contained a construction of the following form "\textbackslash \textbackslash n", it was replaced with "\textbackslash n".

3. Texts with prompts and responses longer than 4096 characters were not included in the dataset.

% \textbf{This sample demonstrates:}

% \textbf{-} ID: "train\_71410"

% \textbf{-} Input: "Your goal is to identify rhetorical and psychological techniques in the given text.\textbackslash Input:Tunnel to Schiff\textbackslash Bunker where our next President will be chosen."

% \textbf{-} Output: ["Causal Oversimplification", "Smears"]

% \textbf{-} Full Input: "Your goal is to identify rhetorical and psychological techniques in the given text. \textbackslash Input:Tunnel to Schiff\textbackslash Bunker where our next President will be chosen.\textbackslash Output:["Causal Oversimplification", "Smears"]"


\textbf{Models.} LLaMA-2-7b-chat~\footnote{\url{https://huggingface.co/meta-llama/Llama-2-7b-chat-hf}} and Mistral-7b-Instruct-v0.2 ~\citep{jiang2023mistral} were used as models.

\textbf{Training and Inference Parameters.}
Both models were trained in both SFT mode and using the LoRA technique, using the huggingface transformers~\footnote{\url{https://huggingface.co/docs/transformers/index}} and torch frameworks on 4 A100 GPUs for no more than 12 hours per experiment.

In SFT mode, both models were trained with the parameters, presented in Appendix~\ref{sec:appendix_1}.

When training with the LoRA technique, the models were trained with the same parameters, but with the following LoRA parameters, presented in Appendix~\ref{sec:appendix_2}.

For inference, the vLLM framework ~\citep{kwon2023efficient} was used with the following parameters presented in Appendix~\ref{sec:appendix_3}.

\subsubsection{BERT-like approach}

As a result of training with deberta-base-cased, comparable results to the generative approach were achieved: F1 score of 0.638.

% \textbf{Final Prediction.}
% For the final prediction, the LLaMA 2 7b chat model was fine-tuned on dataset #2 with the same parameters as dataset #1, and with the following parameters (\textit{Appendix\_4}).

% It achieved an F1 score of 0.61339 and secured the 18th position on the leaderboard.

\quad \textbf{Idea.}
The main idea of this approach is to train a fully connected layer for hierarchical classification on top of a frozen embedding model for the task of hierarchical multilabel classification.

\textbf{Dataset.}
For model training, we utilized the training dataset provided by the authors. The dev dataset was used for selecting the final model. For the final prediction, the model was trained on a mixture of the train and dev datasets.

\textbf{Data Format.}
To predict propaganda techniques, we represented the data labels in the format of an acyclic graph, where the nodes of this graph are generalized technique types ("Ethos", "Pathos", ..., etc.), and the leaves are concrete techniques ("Name calling", "Doubt", ..., etc.). Instead of predicting only specific techniques, we predict generalized techniques as well. For example, if the current sample needs to predict the technique \{"Whataboutism"\}, the model should predict Anc(y) = \{"Whataboutism", "Distraction", "Reasoning", "Ad Hominem", "Ethos", "Logos", "Persuasion"\}. Thus, datasets are formed as sets of pairs: (x, Anc(y)), where x is the text, and y is the set of techniques contained in the text x.

\textbf{Model.}
The frozen model used to obtain embeddings was deberta-base-cased ~\citep{he2021deberta}. The embedding of the entire text from the embedding model was taken, followed by the application of several dropout layers in parallel, and the results were averaged. At the end, trainable linear layers were used for classification.

\textbf{Training Parameters.}
Training parameters of the discussed models can be seen in Appendix~\ref{sec:appendix_4}.



\subsubsection{Results}

\textbf{Generative approach.}
Experiments with pre-training models using the LoRA technique did not yield the desired result. Consistent responses from the models in JSON format could not be achieved, and there was not enough time to develop rules for formatting their outputs.

The results of the models trained with SFT are presented in the Table~\ref{tab:results_1}.

\begin{table}[h!]
\centering
\begin{tabular}{|c|c|}
\hline
Mistral & LLaMa 2 \\
\hline
0.56 F1 & 0.65 F1 \\
\hline
\end{tabular}
\caption{Results for subtask №1}
\label{tab:results_1}
\end{table}

\textbf{BERT-like approach.}
As a result of training with deberta-base-cased, comparable results to the generative approach were achieved: F1 score of 0.638.

\textbf{Final Prediction.}
For the final prediction, the LLaMA-2-7-chat model was fine-tuned on dataset #2 with the same parameters as dataset #1, and with the following parameters, shown in Appendix~\ref{sec:appendix_5}.

It achieved an F1 score of 0.61339 and secured the 18th position on the leaderboard.

\subsection{Subtask №2a}

\quad \textbf{Idea.}
This task resembles subtask №1, but besides text, it involves meme images. It was tackled using a trainable linear layer for hierarchical classification atop frozen multimodal text-to-image embedding models.

\textbf{Dataset.}
For model training, we utilized the training dataset provided by the authors. The dev dataset was used for selecting the final model. For the final prediction, the model was trained on a mixture of train and dev datasets.

\textbf{Data Format.}
To represent propaganda techniques, we employed the format of an acyclic graph from the first subtask. The dataset is presented as triples: (img, x, Anc(y)), where img and x represent the image and text of the current meme, and y is the set of techniques contained in the current meme.

\textbf{Models.}
CLIP ~\citep{radford2021learning} and BLIP ~\citep{li2023blip2} were used as models to obtain embeddings for texts and images. The embeddings of the full text and the entire image were concatenated, and several dropout layers were applied in parallel, with the results averaged. At the end, trainable linear layers were used for classification.

\textbf{Training Parameters.}
The models were trained on a single P100 GPU on the Kaggle platform.
Parameters for BLIP/CLIP were frozen.

Other parameters are shown in Appendix~\ref{sec:appendix_6}.

\textbf{Results.}
The results of the models trained on dataset #1 are presented in the Table~\ref{tab:results_2}.


\begin{table}[h!]
\centering
\begin{tabular}{|c|c|}
\hline
CLIP & BLIP \\
\hline
0.648 F1 & 0.633 F1 \\
\hline
\end{tabular}
\caption{Results for subtask №2a}
\label{tab:results_2}
\end{table}


\subsection{Subtask №2b}

This subtask differs from subtask №1 only in that it requires predicting whether there is any propaganda technique present in the text at all. We used the same approach, the same models with the same hyperparameters, trained on the same datasets as in subtask №1. The only difference is in the data format - the dataset consists of triples (img, x, y), where img and x are the image and text of the current meme, and y is a flag indicating whether the current meme contains a propaganda technique.

\textbf{Results.}
The results of the models trained on dataset #1 are presented in the Table~\ref{tab:results_3}.

\begin{table}[h!]
\centering
\begin{tabular}{|c|c|}
\hline
CLIP & BLIP \\
\hline
0.72 F1 & 0.748 F1 \\
\hline
\end{tabular}
\caption{Results for subtask №2b}
\label{tab:results_3}
\end{table}

For the final prediction, the CLIP model was selected, achieving an F1 score of 0.772 and securing the 11th position on the leaderboard.



\section{Further research}

We did not have time to take more powerful models and experiment with them to solve subtask №1, but we are confident that Mixtral 8x7b or Llama-2-13b-chat would yield better results. Additionally, we did not have time to add data from the PTC ~\citep{he2021deberta} corpus to the dataset, but we are sure that it would have provided an even greater improvement. We also did not have time to test generative models that take text with images as input and generate text responses, for example, RUDOLPH ~\citep{radford2021learning}.


% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

\appendix

\section{Appendix}
\label{sec:appendix}

This appendix shows the training and generation parameters for the models described above in the text.

\subsection{Hyperparameters for models training in SFT mode}
\label{sec:appendix_1}

\begin{itemize}
    \item BATCH\_SIZE = 4
    \item GRADIENT\_ACCUMULATION = 4
    \item LEARNING\_RATE = 1e-5
    \item MAX\_LEN = 4
    \item WARMUP\_STEPS = 4
    \item in fp32 and for 2 epochs
\end{itemize}


\subsection{Hyperparameters for models training with LoRA technique}
\label{sec:appendix_2}

\begin{itemize}
    \item LORA\_R = 8
    \item LORA\_ALPHA = 16
    \item LORA\_DROPOUT = 0.05
    \item TARGET\_MODULES = ["q\_proj", "k\_proj", "v\_proj", "o\_proj"]
\end{itemize}


\subsection{Hyperparameters for models output generation with vLLM}
\label{sec:appendix_3}

\begin{itemize}
    \item TOP\_K = 50
    \item TOP\_P = 1 and 0.9
    \item MAX\_TOKENS = 600
    \item TEMPERATURE = 1, 0.8, 0.6, and 0.2
\end{itemize}


\subsection{Hyperparameters for BERT-like models training for subtask №1}
\label{sec:appendix_4}

\begin{itemize}
    \item Models were trained on a single P100 GPU on the Kaggle platform \footnote{\url{https://www.kaggle.com/}}.
    \item Optimizer: AdamW with linear scheduler
    \item Learning rate (LR): 2e-5
    \item Batch size: 8
    \item Warmup steps: 100
\end{itemize}


\subsection{Hyperparameters for models output generation with vLLM for final prediction}
\label{sec:appendix_5}

\begin{itemize}
    \item TOP\_K = 50, TOP\_P = 0.9
    \item MAX\_TOKENS = 600
    \item TEMPERATURE = 0.8
\end{itemize}

\subsection{Hyperparameters for BERT-like models training for subtask №2a and №2b}
\label{sec:appendix_6}

\begin{itemize}
    \item Optimizer: Adam
    \item Learning rate (LR): 2e-3
    \item Batch size: 10
\end{itemize}

\end{document}