% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[review]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{multirow}
\usepackage{array}
\usepackage{graphicx}
\usepackage{float}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{QFNU\_CS at SemEval-2024 Task 3: A Hybrid Pre-trained Model based Approach for Multimodal Emotion-Cause Pair Extraction Task}

\author{Zining Wang, Yanchao Zhao, Guanghui Han, Yang Song\\
        School of Computer Science, Qufu Normal University, Ri Zhao, China}

\begin{document}
\maketitle
\begin{abstract}
This article presents the solution of Qufu Normal University for the Multimodal Sentiment Cause Analysis competition in SemEval2024 Task 3.The competition aims to extract emotion-cause pairs from dialogues containing text, audio, and video modalities. To cope with this task, we employ a hybrid pre-train model based approach. Specifically, we first extract and fusion features from dialogues based on BERT, BiLSTM, openSMILE and  C3D. Then, we adopt BiLSTM and Transformer to extract  the candidate emotion-cause pairs. Finally, we design a filter to identify the correct emotion-cause pairs. The evaluation results show that, we achieve a weighted average F1 score of 0.1786 and an F1 score of 0.1882 on CodaLab.
\end{abstract}

\section{Introduction}
The competition of multimodal emotion cause analysis\cite{gandhi2023multimodal} involves not only understanding linguistic content but also recognizing and comprehending various forms of information such as emotional expressions, sounds, and images. The significance of this competition lies in its ability to comprehensively understand and interpret emotions and motivations in human communication. By analyzing various forms of information in conversations, we can more accurately identify the sources and reasons for emotions, thereby enhancing our understanding of human behavior and communication methods. This holds importance across various fields including psychology, human-computer interaction, and affective computing, aiding in the development of more intelligent and human-centric technologies and systems, improving communication efficiency and quality, and promoting better understanding and communication among individuals.

This paper details our contribution to SemEval-2024 Task 3: Multimodal Emotion Cause Analysis in Conversations\cite{wang-EtAl:2024:SemEval20244}, encompassing two sub-tasks: extracting emotion-cause pairs\cite{xia2019emotion} from text-only dialogues and from multimodal dialogues that include text, audio, and video modalities. In this task, we place a particular emphasis on implementing Sub-task 2.

\vspace{0.8mm}
For Sub-task 2: Multimodal Emotion-Cause Pair Extraction, we aim to extract emotion-cause pairs from dialogues that contain representations in text, audio, and video. Each pair includes an emotional utterance, its emotion category, and a cause utterance. The challenge lies in integrating information from multiple modalities to accurately identify emotional expressions and their related causes.

\begin{figure*}[h]
    \centering
    \includegraphics[width=13cm]{Fig.1.png}
    \caption{Framework diagram of the E-MECPE methodology}
    \label{fig:my-label}
\end{figure*}


\vspace{0.8mm}
In our approach to task 2, we first preprocessed the dataset, mapping the text, audio, and video data of the Emotion Cause in Friends (ECF) dataset to a unified feature space. 
Then, we utilized a baseline model with a two-stage training scheme: emotion recognition and cause pair extraction. This approach focused on utilizing modalities, selecting models such as Bert\cite{devlin2018bert} and LSTM\cite{yu2019review}, and adjusting parameters for two phases of model training. After that, we predicted on test data in two stages using the trained models and evaluated the results through CodaLab to obtain corresponding F1 scores. 

\vspace{0.8mm}
Our best-performing solution involved using Bert for emotion recognition followed by LSTM for cause pair extraction across all three modalities, achieving an F1 score of 0.1882. This methodological progression demonstrates our systematic approach to tackling the complexities of multimodal emotion cause analysis, highlighting our efforts in dataset preprocessing, model experimentation, and performance evaluation, while also proving the effectiveness of the baseline model\cite{wang2023multimodal}.

\vspace{1mm}
\section{Methodology}
In this section, we describe the E-MECPE method  

\noindent in depth. In general, this method is divided into three main parts: multimodal fusion, emotion and cause extraction through multi-task learning and emotion-cause pairing and filtering. The methodology of this paper is summarized in Fig. 1.

\subsection{Multimodal Fusion}
First, we obtain the representations of the three modalities from the text, audio and video modalities for their respective modalities. Then, the three modalities are stitched together in the order of text-audio-video to obtain the joint representation of the three modalities. The feature extraction method for each of these modalities is as follows:

For text, each token is initialized with pre-trained 300-dimensional GloVe vectors\cite{pennington2014glove}. Subsequently, we used two different models to extract text features: the BiLSTM (Bidirectional Long Short-Term Memory Network) and the BERT (Bidirectional Encoder Representation Transformer).BiLSTM is a classical recurrent neural network that can effectively capture long-term dependencies in text sequences by a bidirectional structure that considers both forward and backward information. BERT, on the other hand, is a pre-trained language model based on the Transformer architecture, which is pre-trained on large-scale textual data and is able to capture rich semantic information. In this study, BiLSTM is used as a textual feature extractor to capitalize on its representational learning ability in sequential data; while BERT, as another textual feature extractor, acquires deeper semantic information by pre-training the model\cite{kim2023aobert}. These two models are independently applied to discourse-level feature extraction tasks to evaluate their performance on sentiment and cause extraction tasks.

In the audio domain, we extract the 6373-dimensional acoustic features ($a_i$) using the openSMILE toolkit, leveraging the feature set designed for the INTERSPEECH 2013 Emotion Challenge. This comprehensive approach allows us to capture nuanced acoustic characteristics, providing a rich foundation for our subsequent analyses. 

For video processing, we use a 3D-CNN network variant called C3D\cite{tran2015learning,rao2020three} to extract 16 frames from each video and process them through the C3D network to obtain 4096-dimensional video descriptors optimized for dimensionality reduction and to extract 128-dimensional visual features from each speech video. 
\subsection{Emotion Extraction}
Our aim in sentiment extraction is to derive sentiment-related features from the discourse. We process each discourse to obtain sentiment-specific representations ($re_i$) by means of a specific bidirectional long short-term memory network (BiLSTM). The BiLSTM processes the forward and reverse sequences of the discourse separately by means of its two LSTM units, and ultimately combines the outputs of these two directions. This allows the network to take into account both the forward and backward information of the discourse, thus providing a more comprehensive understanding of the emotional content of the discourse.

Next, the sentiment-specific representation ($re_i$) is fed into a softmax layer, the output of which can be considered as the probability that the discourse belongs to each sentiment category. the formula for the softmax layer is as follows:

\begin{equation}
\hat{\mathbf{y}}_{i}^{e}=\mathrm{softmax}\big(\mathbf{W}^{e}\mathbf{r}_{i}^{e}+\mathbf{b}^{e}\big)
\end{equation}

where \(W_e\) and \(b_e\) are the weights and biases of the softmax layer, respectively, and \(\hat{y}^e_i\) is the predicted sentiment distribution.

\subsection{Cause Extraction}
The purpose of the cause extraction part is to recognize causal relationships in discourse. We use another BiLSTM to extract cause-specific representations ($rc_i$). This BiLSTM works in a similar way to the BiLSTM used in sentiment extraction, but the parameters are not shared to ensure that the network learns the specific features for the cause extraction task.

The reason-specific representation ($rc_i$) is then fed into another softmax layer that focuses on determining the probability of different reason categories in the discourse. The formula for this softmax layer is as follows:

\begin{equation}
\hat{\mathbf{y}}_{i}^{c}=\mathrm{softmax}\big(\mathbf{W}^{c}\mathbf{r}_{i}^{c}+\mathbf{b}^{c}\big)
\end{equation}

Here, \(W_c\) and \(b_c\) are the weights and biases of this softmax layer, and \(\hat{y}^c_i\) denotes the predicted cause distribution.

\subsection{Loss calculation}
Our goal is to minimize the cumulative loss of the model on the emotion extraction and cause extraction tasks. The total loss \(L_{\text{total}}\) is the sum of the losses of the two tasks and is calculated by the following formula:

\begin{tiny}
\begin{equation}
\mathcal{L}_{\text{total}} = -\sum_{i=1}^{N} \left( \sum_{j=1}^{C} y_{i}^{e,j} \log(\hat{y}_{i}^{e,j}) + \sum_{k=1}^{K} y_{i}^{c,k} \log(\hat{y}_{i}^{c,k}) \right)
\end{equation}
\end{tiny}

Where \(y^e_{j,i}\) and \(y^c_{k,i}\) denote the uniquely hot encoding of the true emotion and cause labels, respectively, \(N\) is the number of training samples, and \(C\) and \(K\) are the number of emotion and cause categories, respectively. This loss function ensures that the model learns to extract features related to emotions and reasons efficiently, thus improving the model's performance on both tasks.

\subsection{Emotion-Cause Pairing and Filtering}
Following the acquisition of Candidate Emotion Utterances and Candidate Cause Utterances, the pivotal task is to discern the existence of a causal relationship between sets E and C, ensuring the extraction of valid emotion-cause pairs. Initially, E and C are organized into a dot matrix, depicted in the third segment of Fig. 1, resulting in the generation of all conceivable candidate pairs denoted as 
$x(U_j^e; U_{k}^c)$. This vector amalgamates the self-contained multimodal representations of the emotion and cause expressions, along with a distance vector capturing the relational nuances between the two expressions.

The composite representation is then inputted into a softmax layer to determine the validity of the pairing $x$, filtering and extracting relevant emotion-cause pairs from numerous possibilities.

\vspace{-0.5cm}
\begin{equation}\hat{\mathbf{y}}_{j,k}=\mathrm{softmax}\left(\mathsf{W}\mathbf{x}_{\left(\hat{U}_{j}^{e},\hat{U}_{k}^{e}\right)}+\mathbf{b}\right)\end{equation}

\section{Experiments}

\subsection{Data Resources}

The official dataset consists of three modalities: text, audio, and video clips, and includes 1,374 conversations and 13,619 utterances annotated for 9,794 emotion-cause pairs across the three modalities. The relevant connections are stored in a JSON file and correspond to independent video segments through specified IDs. In order to fully utilize all the multimodal data, we first preprocess and reduce the dimensionality of the data according to the methods described in the paper.

Specifically, during the preprocessing stage, for the audio data in the video, we use the ffmpeg tool to extract the corresponding audio files for each video segment. We then utilize the open-source tool called openSMILE \cite{31} and apply The INTERSPEECH 2013 ComParE feature set \cite{32}, which is the default feature set of openSMILE, to extract features from the audio data. As a result, we obtain a 6373-dimensional acoustic feature vector.For video data, we refer to the C3D model structure to extract video features and obtain a 4096-dimensional representation.As for text data, following the same approach as described in the paper, we utilize pre-trained Glove word vectors to obtain text embeddings.

\subsection{Training}

The training process is divided into two parts: the first part is emotion extraction and cause extraction, and the second part is the extraction of emotion-cause pairs. We explored three different training conditions: utilizing only textual modalities, combining textual and audio modalities, combining textual and video modalities, and leveraging all data modalities and fine-tune the model parameters based on the baseline to select the appropriate parameters to obtain the best score.

\textbf{Emotion extraction and cause extraction: }The initial phase of our experiment compared emotion extraction using Bert and BiLSTM model architectures, conducted on an RTX 4070Ti Super GPU setup. Key training parameters were carefully selected to enhance model performance. The batch size for BiLSTM was fixed at 16, while for Bert, it was set to 4, with the training spanning 15 epochs. The loss weights for both emotion extraction and cause extraction tasks were set to 1.0, indicating their equal importance in our training objectives.

\textbf{Emotion-cause pairs extraction: }In the subsequent phase focusing on cause pair identification, the same model architecture was employed, trained under identical conditions to assess the effect of data modality on performance. The batch size was increased to 200 to potentially improve generalization, with a learning rate of 0.005 aimed at optimal convergence. A 0.5 dropout keep probability for word embeddings was introduced for added regularization, while maintaining a 1.0 keep probability for the softmax layer. The l2 regularization coefficient remained at 1e-5, consistent with our approach to model complexity control.

\subsection{Evaluation}

Similar to baseline, we utilize the macro-averaged F1 score \cite{22} as the primary evaluation metric for our task. This metric accounts for both precision and recall, providing a balanced assessment of model performance. The F1 score is calculated using the following formula:

\begin{equation}  
	F_1 =\frac{2 \times P \times R}{P+R}
\end{equation}

where:
\begin{itemize}
    \item \( P \) denotes precision, calculated as the ratio of correctly predicted emotion-cause pairs to the total predicted pairs.
    \item \( R \) denotes recall, calculated as the ratio of correctly predicted emotion-cause pairs to the total annotated pairs.
\end{itemize}

In our evaluation, \( F_1 \) is the harmonic mean of precision and recall, indicating the model's balance in detecting emotion-cause pairs: a higher \( F_1 \) score signifies better performance.

\begin{table}[h]
\centering
\caption{Experimental Results}
\renewcommand\arraystretch{1}
\resizebox{\linewidth}{!}{
\begin{tabular}{cccccc}
\hline
Model & Modality & $F1_{emotion}$ & $F1_{caution}$ & $F1_{pair}$ \\
\hline
\multirow{4}{*}{BiLSTM} & T    & 0.7441 & 0.7008 & 0.5041 \\
                        & TA   & 0.7398 & 0.6986 & 0.5104 \\
                        & TV   & 0.7431 & 0.7016 & 0.5162 \\
                        & TAV  & 0.7422 & 0.6993 & 0.5226 \\
\hline
\multirow{4}{*}{Bert}   & T    & 0.7362 & 0.6687 & 0.5104 \\
                        & TA   & 0.7356 & 0.6637 & 0.5160 \\
                        & TV   & 0.7365 & 0.6700 & 0.5104 \\
                        & TAV  & 0.7363 & 0.6648 & 0.5246 \\
\hline
\end{tabular}}
\end{table}

\subsection{Results and analysis}

We assessed Bert and BiLSTM models on various modalities: text (T), text-audio (TA), text-video (TV), and their combination (TAV), as shown in Table 1. Results underline the models' proficiency in extracting sentiment-cause pairs from multimodal dialogues, with distinct performance variations across modalities.

The BiLSTM model demonstrates incremental improvements in $F1_{pair}$ scores from T to TAV, indicating the advantage of utilizing multimodal data. The highest performance is observed in the TAV setup with a score of 0.5226, underscoring the benefits of combining text, audio, and video.

Conversely, the Bert model showcases superior performance in the TAV modality, achieving an $F1_{pair}$ score of 0.5246. This performance highlights Bert's ability to effectively leverage deep contextual embeddings across modalities for more accurate extraction of sentiment-cause pairs. The robustness of Bert, particularly in the multimodal TAV setup, confirms its efficacy in handling complex multimodal data.

Overall, Bert emerges as the preferred model for extracting sentiment-cause pairs across all modalities, with a peak performance in the TAV configuration, reflected by a weighted average F1 score of 0.1786 and an F1 score of 0.1882 on CodaLab. These findings advocate for the continued exploration of multimodal approaches, particularly leveraging models like Bert that excel in contextual understanding and integration of multimodal data.

\section{Conclusion}
In this paper, we present Effective Multimodal
Emotion-Cause Pair Extraction (E-MECPE) method. We used this method to perform emotional cause analysis on the Emotion-Cause-in-Friends (ECF) dataset. Ablation experiments were conducted for text unimodal and multimodal under different text encoders, respectively, and the relevant parameters associated with the experiments were tuned. The experimental results show that BERT encoding-based text representation and multimodal joint representation help in the extraction of emotional cause pairs, and that the parameter settings are crucial for the performance enhancement of this task. This finding not only validates the effectiveness of our method, but also points out an important direction for future research in the field of sentiment analysis kresearch by pointing out an important direction.

\section{Prospects for Advancement}
Due to the late entry time, limited hardware resources, and short submission period, we only had time to fine-tune and conduct ablation experiments based on the baseline. However, we believe that there is still a lot of room for improvement in adjusting this model. For example, further attempts can be made in aligning and filtering methods for multimodal data, selecting more encoders, and enhancing the model's understanding of causal relationships. We will also continue exploring on top of this model to continuously advance the development of this research direction.

\section{Acknowledgements}

This research is supported by Natural Science Foundation of Shandong Province [grant number ZR2022QF088]; Natural Science Foundation of Rizhao [grant number RZ2021ZR30]; and ShanDong Students' Platform for innovation and entrepreneurship training program [grant number S2023104460124]

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

\end{document}
