% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

\usepackage{verbatim}
\usepackage{graphicx} % Required for including images
\usepackage{xcolor,lipsum,subcaption}
\usepackage{tikz}
\usepackage{pgfplots}
\pgfplotsset{compat=1.18}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[preprint]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% remove page numbers
\pagenumbering{gobble}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
\usepackage{adjustbox} % Make sure to include this package
% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\usepackage{listings}
\lstset{
  showstringspaces=false,
  basicstyle=\ttfamily,
  columns=fullflexible,
  frame=single,
  breaklines=true,
  breakindent=0pt,
  breakatwhitespace=false,
  % postbreak=\mbox{\textcolor{red}{$\hookrightarrow$}\space},
}


\title{DeepPavlov at SemEval-2024 Task 3: Multimodal Large Language Models in Emotion Reasoning}
\author{Julia Belikova \and Dmitrii Kosenko\\
Moscow Institute of Physics and Technology\\
\texttt{belikova.iua@phystech.edu, kosenko.dp@mipt.ru}}
  

\begin{document}
\maketitle
\begin{abstract}
This paper presents the solution of the DeepPavlov team for the Multimodal Sentiment Cause Analysis competition in SemEval-2024 Task 3, Subtask 2~\cite{wang-EtAl:2024:SemEval20244}. In the evaluation leaderboard, our approach ranks 7th with an F1-score of 0.2132. Large Language Models (LLMs) are transformative in their ability to comprehend and generate human-like text. With recent advancements, Multimodal Large Language Models (MLLMs) have expanded LLM capabilities, integrating different modalities such as audio, vision, and language. Our work delves into the state-of-the-art MLLM Video-LLaMA, its associated modalities, and its application to the emotion reasoning downstream task, Multimodal Emotion Cause Analysis in Conversations (MECAC). We investigate the model's performance in several modes: zero-shot, few-shot, individual embeddings, and fine-tuned, providing insights into their limits and potential enhancements for emotion understanding.
\end{abstract}


\section{Introduction}
In the dynamic domain of artificial intelligence, the emergence of MLLMs has gained significant interest due to integrating input from different modalities, such as audio, vision and language, opens up in-depth perceptual and interpretive capabilities in dialogues instead of chat-based dialogue systems~\cite{konovalov2016collecting}.

These models exhibit impressive potential in resolving a lot of challenges and have been deployed across various sectors, including banking support systems, social services, and as adjuncts in psychological assistance. In these applications, the deciphering of user intent and emotions is crucial for generating pertinent responses. Consequently, one of the most important domains within MLLM research is emotion reasoning.

This paper explores a specific facet of emotion reasoning: Multimodal Emotion Cause Analysis in Conversations (MECAC)~\cite{wang-EtAl:2024:SemEval20244}. This task involves emotion recognition and matching emotional states with their causes in the context of a conversation, leveraging inputs from different modalities such as text, audio, video and more.

Despite the remarkable advancements in MLLMs, their capabilities and limitations in the high-potential area of emotion reasoning remain active topics for research. One of the seminal contributions to the investigation of MLLMs capabilities within the area of emotional reasoning is delineated in~\cite{lian2023explainable}. The authors introduce a novel task, Explainable Multimodal Emotion Reasoning (EMER), and proceed to evaluate the efficacy of modern multimodal models in addressing EMER. Their research focuses on the integration and interpretability of emotional cues across diverse modalities, thereby advancing the understanding of emotion reasoning.

To address the described issue, we propose to continue research of the MLLMs capabilities in emotion reasoning by evaluating one of the most promising models, Video-LLaMA~\cite{zhang2023videollama}, also explored in~\citet{lian2023explainable}, for MECAC on the Emotion-Cause-in-Friends dataset.

The work evaluates the performance of the model in three modes:
\begin{enumerate}
    \item \textit{Zero-shot and Few-shot modes}. These modes are utilized to evaluate the model's initial capabilities in emotion reasoning.
    \item \textit{Individual Embeddings mode}. In this mode, embeddings from individual modalities are employed alongside trained basic heads to address MECAC.
    \item \textit{Fine-tuned mode}. This mode is used to evaluate specialized emotion reasoning capabilities.
\end{enumerate}

Experimental results demonstrate the considerable potential of MLLMs in the domain of emotion reasoning.

\section{Related Work}

\subsection{Multimodal Large Language Models}
The ascent of Large Language Models such as LLaMA2~\cite{touvron2023llama}, Qwen~\cite{bai2023qwen}, Mistral~\cite{jiang2023mistral} has marked a significant milestone in the field of artificial intelligence. These models have demonstrated exceptional capabilities in language reasoning and decision-making, closely mirroring human-level performance.

The integration of adapters to align pre-trained encoders from different modalities with textual LLMs has given rise to a new class of MLLMs such as:
Flamingo~\cite{alayrac2022flamingo}, BLIP-2~\cite{li2023blip2}, MiniGPT-4~\cite{zhu2023minigpt4}, mPLUG-Owl~\cite{ye2023mplugowl},
VideoChat~\cite{li2024videochat}, InstructBLIP~\cite{dai2023instructblip}, Video-ChatGPT~\cite{maaz2023videochatgpt}, Video-LLaVA~\cite{lin2023videollava}, VideoChat2~\cite{li2024mvbench}, Video-LLaMA~\cite{zhang2023videollama} (Table \ref{tab:mllms_table}).

\begin{table}[h!]
    \centering
    \begin{tabular}{lc}
        \hline
        \textbf{Model} & \textbf{Modality} \\ % & \textbf{Tasks}\\
        \hline
        Flamingo & I, V, T \\
        BLIP-2 & I, T \\
        MiniGPT-4 & I, T \\
        mPLUG-Owl & I, V, T \\
        InstructBLIP &  I, T \\
        Video-ChatGPT & I, V, T \\
        VideoChat2 & I, V, T \\
        Video-LLaMA & I, V, A, T \\
        Video-LLaVA & I, V, T \\\hline         
    \end{tabular}
    \caption{Multimodal Large Language Models. T, A, I, and V stand for text, audio, image and silent video, respectively}
    \label{tab:mllms_table}
\end{table}

These models have gained impressive results in well-known general domains~\cite{multimodal_survey}: temporal perception and reasoning, casual inference, and spatial perception and analysis.

\subsection{Emotion Reasoning in Conversation}

Our work delves into MECAC, a derivation of ECPE~\cite{xia2019emotion}, the downstream task of emotion reasoning. Given a conversation sequence consisting of $N$ utterances, $U = \{U_1, U_2, \ldots, U_N\}$, where each utterance $U_i$ is accompanied by a corresponding speaker identity, textual content, and an associated audio-visual clip.

The task is to output a set of emotion-cause pairs $E = \{(e_i, c_i)\}_{i=1}^{M}$, where each pair contains:
\begin{itemize}
    \item[--] $e_i$: an emotion utterance $U_j$ that expresses an emotion.
    \item[--] $c_i$: a cause utterance $U_k$ that is identified as the cause of the emotion expressed in $U_j$.
\end{itemize}

Additionally, each emotion utterance $e_i$ is tagged with an emotion category $EC$ from a predefined set of emotion categories $EC = \{EC_1, EC_2, \ldots, EC_K\}$.

The exploration of emotion reasoning within the context of conversations has traditionally been addressed using various classical approaches. Recurrence-based or graph-based methods have been particularly popular due to their ability to capture sequential and relational data effectively. Notable methods in this domain include:  MC-ECPE-2steps~\cite{wang2021multimodal}, which focuses on two-step recurrence-based emotion-cause pair extraction; Joint-GCN~\cite{9926166}, which leverages recurrent and graph convolutional networks for joint emotion-cause detection; ECQED~\cite{zheng2023ecqed}, which extends the emotion-cause pair extraction to a quadruple extraction task and structural and semantic heterogeneous graph for conversation representation; CORECT~\cite{Nguyen_2023}, which enhances conversation understanding through relational temporal graph neural networks; and COGMEN~\cite{joshi2022cogmen}, which utilizes contextualized graph neural networks for multimodal emotion recognition.

In this paper, we investigate the capabilities of MLLMs for solving MECAC by evaluating the state-of-the-art model, Video-LLaMA. It is pertinent to acknowledge the application of MLLMs in a variety of sentiment and emotion recognition \cite{ASLAM2023110494}, in the domain of EMER. Previous works indicate that MLLMs demonstrate notable efficacy in these complex tasks, which highlights their potential for advancing the frontier of emotion reasoning research.

\subsection{Video-LLaMA architecture}
Next, we summarize the key points of Video-LLaMA's architecture.

Video-LLaMA is a multimodal framework designed to extend the capabilities of frozen LLMs by enabling them to process and respond to audio-visual content.

\textbf{Visual Encoder}. The visual encoding component employs a pre-trained image encoder to compute representations from individual frames of a video. It introduces a frame embedding layer to provide temporal information and incorporates a video Q-former to generate visual query tokens that encapsulate the temporal dynamics of visual scenes. A linear layer is introduced to transform video embedding vectors into query vectors that are compatible with the embedding space of LLMs.

\textbf{Audio Encoder}. For audio processing, Video-LLaMA leverages ImageBind~\cite{girdhar2023imagebind}. It also uses a similar architecture to the visual encoder to obtain audio embeddings for the LLM module.

\textbf{Cross-Modal Training}. 
The training process involves multi-branch, cross-modal pre-training to achieve both vision-language and audio-language alignment. The vision-language pre-training includes a video-clips-to-text generation task and static image-caption learning. The audio-language pre-training leverages the audio encoder and vision-text data to align with the LLM's embedding space.

\textbf{Standard Inference}. During inference, Video-LLaMA is capable of zero-shot video and audio understanding. It processes video frames and audio signals, converts them into query representations that are concatenated with textual input embeddings of LLMs, and generates responses grounded in the video's visual and auditory content.

\section{Methods}

\subsection{Zero-shot and Few-shot}
Today's LLMs are developed using extensive datasets and are further fine-tuned to comprehend and follow instructions, granting them the capacity to perform certain tasks in a zero-shot fashion~\cite{tirskikh2023zero}. Investigating how these capabilities are exhibited in multimodal models represents an active area of research.

To evaluate the capabilities of Video-LLaMA in the zero-shot emotion reasoning subtasks, we use structured templates such as the one detailed in Appendix \ref{sec.foo_0}, Listing~\ref{lst:zero-shot_1}

While LLMs demonstrate remarkable zero-shot capabilities, they still fall short on more complex tasks within the zero-shot setting. Consequently, it is essential to evaluate their few-shot capabilities as well. To achieve this, we employ prompt templates, such as the one described in Appendix \ref{sec.foo_0}, Listing~\ref{lst:zero-shot_2}.


\subsection{Individual Embeddings}
In this work, we also investigate the capabilities of embeddings obtained from the output of Video-LLaMA. Specifically, we extract multimodal embeddings corresponding to each fragment of the conversation. These embeddings are derived from the last semantic token of the last hidden state during the generation of responses to prompts formatted as shown in Listing~\ref{lst:ind_embed_format}.

\begin{lstlisting}[caption={Prompt templates for multimodal embeddings generation}, label={lst:ind_embed_format}, captionpos=b]
# First option
<Item Value> Describe the behavior of the speaker in this <Item Name> in one word:
# Second option
<Item Value> Describe what is happening in this <Item Name> in one word: 
# Third option
<Item Value> Describe the emotional state of the speaker in this <Item Name>
in one word:
\end{lstlisting}

To utilize these embeddings for our task, we integrate classical heads based on Multi-Layer Perceptron (MLP), Bidirectional Long Short-Term Memory (BiLSTM), and Self-Attention mechanisms. In the context of MLP, we adopt a straightforward approach for multi-class classification of emotions and binary classification of causes. Importantly, for the binary classification of causes, we consider all possible pairs of utterances. The probability that a pair belongs to a specific class is computed based on the output from the linear layer, which receives a concatenated representation of the utterance pairs as its input.

The BiLSTM head is implemented similarly to the MLP. For the self-attention mechanism, we employ multiple layers of a classical architecture. For both approaches — the MLP and BiLSTM-based heads — we utilize Cross-Entropy as the loss function. 

It is also worth noting that there is a class imbalance in the case of binary causes classification. According to the authors of the dataset about 55.73\% of the utterances are annotated with one of the six basic emotions, and 91.34\% of the emotions are annotated with the corresponding causes in the ECF dataset. As a result, the matrices of some conversations divided into utterances become quite sparse. To mitigate the impact of imbalance, we propose several balancing methods: simple weighting of the loss function and adaptive weighting. In the first case, a constant scaling factor is chosen to increase the influence of the minority class, while in the second case, balancing is done for each individual batch based on the current class frequency.

\subsection{Fine-tuning}
The fine-tuning stage employs Low-Rank Adaptation (LoRA)~\cite{hu2021lora} to modify the pre-trained parameters of the LLaMA module within Video-LLaMA, while the visual and audio encoders remain unchanged. We design prompts for fine-tuning, outlined in Listings \ref{lst:ind_embed_format_1} and \ref{lst:ind_embed_format_2}, that closely align with the format proposed in \cite{lei2023instructerc}.

\begin{lstlisting}[caption={Instruction format for the fine-tuning stage}, label={lst:ind_embed_format_1}, captionpos=b]
You are expert of multimodal emotion classification and emotion cause recognition. 

The following is a conversation that involves several speakers. 

Here is a conversation that is described
in several fragments and includes subtitles, video, and audio:

Utterance_1
<Speaker Name>: <Speaker Text>
Video: <Video>
Audio: <Audio>
...

Select the emotion label of each utterance from <neutral, surprise,
fear, sadness, joy, anger, disgust>
and predict the ids of utterances that caused this emotion.
\end{lstlisting}

\begin{lstlisting}[caption={Response format for the fine-tuning stage}, label={lst:ind_embed_format_2}, captionpos=b]
Utterance_1
Emotion: <Emotion>
Causes: 1
...
\end{lstlisting}

\section{Experiments}
For the experiments described below, we use the Emotion-Cause-in-Friends (ECF) dataset (the official train part), which is divided into train, validation, and test sets in accordance with the proportion 8:1:1. We used train part due to test split is not officially available for extensive experiments.

\subsection{Zero-shot and Few-shot}
In the zero-shot experiments, the model exhibits a loss of ability to follow general instructions and ceases responding to the guidelines provided, instead demonstrating a tendency for a detailed description of the events observed in the video. Examples of this behavior are visible in the experimental data presented in Appendix~\ref{sec.foo_1}. In few-shot experiments with Video-LLaMA, we observe the same pattern.

\subsection{Individual Embeddings}       

\textbf{Metrics}. 
In evaluating the model's performance on the emotion classification subtask, we utilize two principal metrics: the macro F1-score, which provides a balanced measure of precision and recall across all classes, and Accuracy, reflecting the overall proportion of correctly identified instances. For the causal classification subtask, we similarly measure performance using the binary F1-score, which is tailored to binary classification problems, alongside Accuracy to determine the proportion of true results in the dataset.

\textbf{Training configuration}.
Each training session is run in 50 epochs. For emotion classification, 32 utterances are used as one batch. For cause classification, one batch describe one conversation and an accumulation of 6 batches for gradient optimization is used.

To address the challenges presented by MECAC, our approach encompassed two distinct training schemas: joint and separate training for the dual classification objectives, namely emotions and causes. The joint training final loss function was composed as a linear combination of the individual losses from both classification heads as in MTL systems~\cite{karpov2023knowledge}.

Initial observations from the joint training indicated that the combination of loss functions from the emotion and cause components was instrumental in enhancing the model's generalization capabilities. However, this joint strategy appeared to reach a plateau, failing to deliver the maximum attainable performance in the later stages of training.

Further experimentation yielded additional insights, particularly in the domain of model convergence. For the emotion classification task, the MLP head emerged as the superior architecture, leading to the most optimal model convergence. Conversely, the BiLSTM head demonstrated a marked advantage in the cause classification domain.

Also, as mentioned above for the training of cause classification it is suggested to perform balancing of the loss function. In practice, the assumption to mitigate class imbalance has proven to be highly significant. According to the experimental results, the best convergence was provided by the use of a constant weight coefficient, notably a value of 3, to give greater emphasis to predictions of the minor classes within the loss function.

\textbf{Prompt optimization}. We evaluate three distinct prompt configurations to derive embeddings for each modality under consideration. The experimental results reinforce the notion of textual content as a leading modality, with the third prompt configuration demonstrating particular efficacy. Accordingly, the tables in Appendix \ref{sec.foo_2} present the training results as evaluated on the test subset, including all combinations of the prompts applied to the audio and video modalities.

\textbf{Modality impact}. To evaluate the contribution of each modality to the overall effectiveness of the classification tasks, we conducted several experiments. The validation results are depicted in Figure~\ref{fig:modalities}, and the test results confirm the observed trend. The text modality emerges as the most influential, exerting the greatest effect on the model's predictive accuracy. In a secondary position, the audio modality is found to have a considerable impact, albeit less than that of text. The video modality, while still contributing to the overall model performance, is observed to have the least influence among the three. 

The leading role of textual modality can also be substantiated by the information provided by the ECF authors, who state that approximately 8\% of the emotion causes in the dataset are the events mainly reflected in the acoustic or visual modalities. It's also important to note the least valuable modality in these experiments: the visual modality. We suppose that this is justified by the lack of confidence of the models in visual feature space, which, most likely, can be eliminated by fine-tuning of the visual encoding branch.

\begin{figure*}[htb!]
  \centering
  \begin{adjustbox}{center} % Center the content
  \begin{subfigure}{0.47\textwidth}
  \begin{adjustbox}{center} % Center the content
      \begin{tikzpicture}
        \begin{axis}[
            title={Cause classification: F1-score},
            xlabel={Steps},
            ylabel={F1-Score},
            ylabel style={shift={(0ex,-1ex)}},
            xmin=0, xmax=200,
            ymin=0, ymax=0.5,
            xtick={0,50,100,150,200},
            ytick={0,0.1,0.2,0.3,0.4,0.5},
            legend pos=south east,
            legend style={font=\small},
            ymajorgrids=true,
            grid style=dashed,
        ]
        
        \addplot[color=blue, mark=square] table [x=y, y=x1, col sep=comma] {latex/tables/causal_f1.csv};
        \addlegendentry{Text}
        
        \addplot[color=red, mark=triangle] table [x=y, y=x2, col sep=comma] {latex/tables/causal_f1.csv};
        \addlegendentry{Text-Video}
        
        \addplot[color=green, mark=*] table [x=y, y=x4, col sep=comma] {latex/tables/causal_f1.csv};
        \addlegendentry{Text-Audio}
        
        \addplot[color=orange, mark=o] table [x=y, y=x3, col sep=comma] {latex/tables/causal_f1.csv};
        \addlegendentry{Text-Video-Audio}
        
        \end{axis}%
      \end{tikzpicture}
      \end{adjustbox}
  \end{subfigure}%
  \hspace{1.8mm}
  \begin{subfigure}{0.47\textwidth}
  \begin{adjustbox}{center}
      \begin{tikzpicture}
        \begin{axis}[
            title={Cause classification: Accuracy},
            xlabel={Steps},
            ylabel={Accuracy},
            ylabel style={shift={(0ex,-1.2ex)}},
            xmin=0, xmax=190,
            ymin=0.5, ymax=1,
            xtick={0,50,100,150,200},
            ytick={0.5,0.6,0.7,0.8,0.9,1},
            legend pos=south east,
            legend style={font=\small},
            ymajorgrids=true,
            grid style=dashed,
        ]
        
        \addplot[color=blue, mark=square] table [x=y, y=x1, col sep=comma] {latex/tables/causal_accuracy.csv};
        \addlegendentry{Text}
        
        \addplot[color=red, mark=triangle] table [x=y, y=x2, col sep=comma] {latex/tables/causal_accuracy.csv};
        \addlegendentry{Text-Video}
        
        \addplot[color=green, mark=*] table [x=y, y=x4, col sep=comma] {latex/tables/causal_accuracy.csv};
        \addlegendentry{Text-Audio}
        
        \addplot[color=orange, mark=o] table [x=y, y=x3, col sep=comma] {latex/tables/causal_accuracy.csv};
        \addlegendentry{Text-Video-Audio}
        
        \end{axis}%
      \end{tikzpicture}%
    \end{adjustbox}
  \end{subfigure}
  \end{adjustbox}

  \begin{adjustbox}{center}
  \begin{subfigure}{0.47\textwidth}
  \begin{adjustbox}{center}
      \begin{tikzpicture}
        \begin{axis}[
            title={Emotion classification: F1-score},
            xlabel={Steps},
            ylabel={F1-Score},
            ylabel style={shift={(0ex,-2ex)}},
            xmin=0, xmax=200,
            ymin=0.4, ymax=0.6,
            xtick={0,50,100,150,200},
            ytick={0.4,0.45,0.5,0.55,0.6},
            legend pos=south east,
            legend style={font=\small},
            ymajorgrids=true,
            grid style=dashed,
        ]
        
        \addplot[color=blue, mark=square] table [x=y, y=x1, col sep=comma] {latex/tables/emotion_f1.csv};
        \addlegendentry{Text}
        
        \addplot[color=red, mark=triangle] table [x=y, y=x2, col sep=comma] {latex/tables/emotion_f1.csv};
        \addlegendentry{Text-Video}
        
        \addplot[color=green, mark=*] table [x=y, y=x4, col sep=comma] {latex/tables/emotion_f1.csv};
        \addlegendentry{Text-Audio}
        
        \addplot[color=orange, mark=o] table [x=y, y=x3, col sep=comma] {latex/tables/emotion_f1.csv};
        \addlegendentry{Text-Video-Audio}
        
        \end{axis}%
      \end{tikzpicture}
      \end{adjustbox}
  \end{subfigure}%
  \hspace{1.9mm}
  \begin{subfigure}{0.47\textwidth}
  \begin{adjustbox}{center}
      \begin{tikzpicture}
        \begin{axis}[
            title={Emotion classification: Accuracy},
            xlabel={Steps},
            ylabel={Accuracy},
            ylabel style={shift={(0ex,-2ex)}},
            xmin=0, xmax=190,
            ymin=0.5, ymax=0.7,
            xtick={0,50,100,150,200},
            ytick={0.5,0.55,0.6,0.65,0.7},
            legend pos=south east,
            legend style={font=\small},
            ymajorgrids=true,
            grid style=dashed,
        ]
        
        \addplot[color=blue, mark=square] table [x=y, y=x1, col sep=comma] {latex/tables/emotion_accuracy.csv};
        \addlegendentry{Text}
        
        \addplot[color=red, mark=triangle] table [x=y, y=x2, col sep=comma] {latex/tables/emotion_accuracy.csv};
        \addlegendentry{Text-Video}
        
        \addplot[color=green, mark=*] table [x=y, y=x4, col sep=comma] {latex/tables/emotion_accuracy.csv};
        \addlegendentry{Text-Audio}
        
        \addplot[color=orange, mark=o] table [x=y, y=x3, col sep=comma] {latex/tables/emotion_accuracy.csv};
        \addlegendentry{Text-Video-Audio}
        
        \end{axis}%
      \end{tikzpicture}%
      \end{adjustbox}
  \end{subfigure}
  \end{adjustbox}
  \caption{Impact of different modalities on the classification tasks}
  \label{fig:modalities}
\end{figure*}

\subsection{Fine-tuning}
In the fine-tuning phase of our experiments, we employ LoRA technique to fine-tune the parameters of the language model component, specifically, the 4-bit quantized Llama-2 7b model, it's selected due to resource constraints. We configure LoRA with an alpha value of 16 and a low-rank factor of 8, while a dropout rate of 0.1 is utilized to prevent overfitting. Our method focuses on selectively adapting only the self-attention projection modules within the transformer architecture. This refines the model's focus on salient features for the tasks at hand without necessitating a comprehensive retraining of the entire network. % For optimization, we chose the AdamW optimizer.
Training batches are set to a size of 4, and the fine-tuning process is conducted over a single epoch, covering the full training dataset.

The fine-tuning strategy yields notable improvements in the model's performance across two distinct classification tasks. For the emotion classification task, the model achieves a macro F1-score of 0.6500 and an Accuracy of 0.7412. This represents a significant enhancement in the model's ability to discern and categorize emotional content within the input data accurately. In the causal classification task, the model demonstrates a binary F1-score of 0.3824 and an Accuracy of 0.9220. While the binary F1-score appears modest, the high Accuracy underscores the model's effectiveness in identifying causal relationships within the tested dataset.

\section{Conclusion}
In this paper, we conduct an analysis of the MLLM Video-LLaMA with an emphasis on its emotion understanding capabilities in MECAC. Our experiments show that multimodal models, in their current iteration, exhibit limitations in deciphering emotional states under zero-shot and few-shot modes.

To enhance the capabilities of such models in emotion understanding, our findings indicate that task-specific dataset fine-tuning is an essential step. Despite the challenges observed, the raw embeddings generated by the Video-LLaMA model show promising potential as a foundation for improving emotion recognition performance.

The implications of this research highlight the necessity for continued development and refinement of multimodal learning frameworks. Future work may concentrate on expanding the diversity of the datasets used for fine-tuning to include a broader spectrum of emotional expressions and cultural contexts. This could mitigate existing biases and enhance the model's generalizability across various demographics and scenarios. Moreover, incorporating advanced techniques such as transfer learning and domain adaptation could further enhance the model's proficiency in interpreting nuanced emotional states~\cite{chizhikova2022multilingual}.

\bibliography{custom}

\clearpage
\appendix
\onecolumn
\section{Appendix}\label{sec.foo_0}
\label{sec:appendix_prompt_0}
\begin{lstlisting}[caption={Zero-shot prompt template}, label={lst:zero-shot_1}, captionpos=b]
Video: <Conversation Video>
Speaker: <Speaker-1 Name>
Text: <Speaker-1 Text>
...
Speaker: <Speaker-i Name>
Text: <Speaker-i Text>

Predict last emotion of <Speaker Name-i>.
Choose emotion from this list: surprise, fear, sadness, neutral, joy, anger, disgust.
\end{lstlisting}

\label{sec:appendix_resp}
\begin{lstlisting}[caption={Few-shot prompt template}, label={lst:zero-shot_2}, captionpos=b]
Video: <Conversation Video>
Speaker: <Speaker-1 Name>
Text: <Speaker-1 Text>
Emotion: <Speaker-1 Emotion>
...
Speaker: <Speaker-i Name>
Text: <Speaker-i Text>
Emotion: <Speaker-i Emotion>

Predict last emotion of <Speaker-i Name>.
Choose emotion from this list: surprise, fear, sadness, neutral, joy, anger, disgust.
\end{lstlisting}


\onecolumn
\section{Appendix}\label{sec.foo_1}
\label{sec:appendix_prompt}
\begin{lstlisting}[caption={Prompt for checking zero-shot capabilities}, label={lst:zero_shot_few_shot}, captionpos=b]
Speaker: Joey
Text: Let it go, Ross.
Speaker: Ross
Text: Yeah, well, you did not know Chi Chi.
Speaker: Monica
Text: Do you all promise?
Speaker: All
Text: Yeah! We promise! We will be good!
Emotion: _____

Describe the last emotion of all. Choose from the list: surprise, fear, sadness, neutral, joy, anger, disgust.
\end{lstlisting}

\label{sec:appendix_resp}
\begin{lstlisting}[caption={Model response for zero-shot prompt}, captionpos=b]
First, the video features a man, woman, and child sitting in a living room with a green glass table, couch, and a flower vase. The mother is holding a blue blanket while the man is eating a bowl of cereal. Next, the video switches to a shot of the mother holding a yellow flower while the daughter is holding a yellow tube and talking on her phone. The mother is also seen holding a yellow flower in a different shot.

The video then shows a couple holding a cake in a kitchen, with a wooden table and chairs. The couple is standing in front of a counter with a white bowl and a white ceramic plate. Then, the video shows a man, woman, and child sitting in a living room with a flower vase on a green glass table, a couch, and a chair. The man and woman are eating cereal while the child is holding a yellow tube and talking on the phone.

The video then shows a shot of the mother, daughter, and son sitting in a living room with a green glass table, couch, and a chair. The mother is holding a blue blanket while the son is eating cereal. Finally, the video features the mother holding a blue blanket while the daughter is holding a yellow tube and talking on her phone.
\end{lstlisting}

\newpage
\section{Appendix}\label{sec.foo_2}
\label{sec:appendix_metrics}
\begin{table}[h!]
\centering
\begin{tabular}{lcc}
\hline
\textbf{Prompt format} & \textbf{macro F1-score} & \textbf{Accuracy}\\
\hline
Audio-1 Video-1 & 0.4983 & 0.6554 \\
Audio-1 Video-2 & 0.4992 & 0.6560 \\
Audio-1 Video-3 & 0.5057 & 0.6568 \\
Audio-2 Video-1 & 0.5102 & 0.6703 \\
Audio-2 Video-2 & \textbf{0.5251} & \textbf{0.6734} \\
Audio-2 Video-3 & 0.5131 & 0.6720 \\
Audio-3 Video-1 & 0.5010 & 0.6566 \\
Audio-3 Video-2 & 0.5105 & 0.6541 \\
Audio-3 Video-3 & 0.5078 & 0.6575 \\
\hline
\end{tabular}
\caption{Prompt optimization results for emotion classification, where Modality-i is an i-th prompt option}
\label{tab:prompts_1}
\end{table}

\begin{table}[h!]
\centering
\begin{tabular}{lcc}
\hline
\textbf{Prompt format} & \textbf{F1-score} & \textbf{Accuracy}\\
\hline
Audio-1 Video-1 & \textbf{0.3505} & \textbf{0.8898} \\
Audio-1 Video-2 & 0.3496 & 0.8872 \\
Audio-1 Video-3 & 0.3494 & 0.8850 \\
Audio-2 Video-1 & 0.3480 & 0.8743 \\
Audio-2 Video-2 & 0.3327 & 0.8735 \\
Audio-2 Video-3 & 0.3194 & 0.8755 \\
Audio-3 Video-1 & 0.3360 & 0.8806 \\
Audio-3 Video-2 & 0.3325 & 0.8739 \\
Audio-3 Video-3 & 0.3184 & 0.8799 \\
\hline
\end{tabular}
\caption{Prompt optimization results for cause classification, where Modality-i is an i-th prompt option}
\label{tab:prompts_2}
\end{table}
\twocolumn 

\end{document}
