% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.

\usepackage["preprint"]{acl}
\usepackage{graphicx}
% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{TECHSSN1 at SemEval-2024 Task 10: Emotion Classification in Hindi-English Code-Mixed Dialogue using Transformer-based Models}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%       \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}


\author{ 
Venkatasai Ojus Yenumulapalli, 
Pooja Premnath, Parthiban Mohankumar, \\  \textbf{Rajalakshmi Sivanaiah and Angel Deborah Suseelan}\\
Department of Computer Science and Engineering, \\
  Sri Sivasubramaniya Nadar College of Engineering, \\ Chennai - 603110, Tamil Nadu, India\\
  \texttt{\{venkatasai2110272, pooja2110152, parthiban2110207\}@ssn.edu.in}, \\ \texttt{\{rajalakshmis, angeldeborahs\}@ssn.edu.in} 
}


% \author{
%   {\bf Venkatasai Ojus Yenumulapalli} \\
%   \texttt{venkatasai2110272@ssn.edu.in} \\
%   {\bf Pooja Premnath} \\
%   \texttt{pooja2110152@ssn.edu.in} \\
%   {\bf Parthiban Mohankumar} \\
%   \texttt{parthiban2110207@ssn.edu.in} \\
%   {\bf Rajalakshmi Sivanaiah} \\
%   \texttt{rajalakshmis@ssn.edu.in}
% }


%   {\bf Pooja Premnath} \\
%   \texttt{pooja2110152@ssn.edu.in} \\
%   {\bf Parthiban M} \\
%   \texttt{parthiban2110207@ssn.edu.in} \\
%   {\bf Rajalakshmi Sivanaiah} \\
%   \texttt{rajalakshmis@ssn.edu.in} \\
%   Sri Sivasubramaniya Nadar College of Engineering 

% }












\begin{document}
\maketitle
\begin{abstract}
The increase in the popularity of code mixed languages has resulted in the need to engineer language models for the same . Unlike pure languages, code-mixed languages lack clear grammatical structures, leading to ambiguous sentence constructions. This ambiguity presents significant challenges for natural language processing tasks, including syntactic parsing, word sense disambiguation, and language identification. This paper focuses on emotion recognition of conversations in Hinglish, a mix of Hindi and English, as part of Task 10 of SemEval 2024. The proposed approach explores the usage of standard machine learning models like SVM, MNB and RF, and also BERT-based models for Hindi-English code-mixed data- namely, HingBERT, Hing mBERT and HingRoBERTa for subtask A.
\end{abstract}

\section{Introduction}

Code-mixed Hindi and English, also referred to as 'Hinglish', has gained widespread usage, especially in the realm of social media. With the increasing prevalence of code-mixed languages like Hinglish, there arises a necessity to analyze and understand this linguistic material. While language models designed for individual languages like English or Hindi \cite{survey}
 are quite robust and effective, they often struggle to perform well with code-mixed languages. This difficulty stems from the colloquial nature of the conversations in code-mixed dialogue, with no formal grammar rules.

Traditional machine learning models perform well on code-mixed data only when the nature of the classification task is simple, like in the form of sentiment analysis (classification into positive, neutral, and negative emotions). Task 10 of SemEval 2024 \cite{kumar2024semeval} contains emotions from the extended Ekman model\cite{ekman1992argument}, which contain emotions that are more complex to discern and distinguish between like contempt versus anger. 

This paper explores the usage of both classical machine learning models as well as Transformer-based BERT models, specifically designed for Hinglish data. 


\section{Related Work}
 \citet{thakur2020current} delve into the current landscape of Hindi-English code-mixed natural language processing and their work meticulously surveys the progress made in sentiment analysis within this domain while also dissecting the inherent issues and challenges it encounters.


Sentiment analysis in code-mixed data is done in a plethora of ways, spanning from machine translation to corpus processing based on sentence structure. \citet{jadhav2022code} introduced a framework employing a pipeline for the conversion of Hinglish to English, offering a structured approach to the task. Similarly, \citet{sinha2005machine} present a method for translating Hinglish to both English and Hindi, leveraging Hindi and English morphological analyzers and implementing cross-morphological analysis to achieve accurate conversion. Ensemble learning for identifying emotions in contextual texts was proposed by \cite{angel2020contextual}. Additionally, \cite{s-etal-2022-varsini} proposed a lexicon-based solution for recognising emotions in Tamil texts.

\citet{das2023sentiment} embraced a deep learning paradigm, implementing convolutional neural networks (CNN), long short-term memory (LSTM), and bi-directional long short-term memory (Bi-LSTM) for sentiment analysis.  Meanwhile, \citet{ravi2016sentiment} conclusively identified a combination of TF-IDF vectorizer, gain ratio-based feature selection, and a Radial Basis Function Neural Network (RBFN) as the optimal pipeline for sentiment analysis of Hinglish data.   \citet{patwa2020semeval} utilized M-BERT and the Transformers framework, diverging from traditional methods. \citet{singh2021sentiment} employed diverse techniques for sentiment analysis of Hinglish, leveraging various embeddings such as count vectorizer and word2vec across different machine learning algorithms including SVM, KNN, and Decision Trees. A similar work by \cite{rajendram2022contextual} focused on recognizing emotions using Gaussian Process and decision trees.


However, the task of emotion classification poses a much greater challenge compared to the simpler task of sentiment analysis. It necessitates the utilization of specific techniques to process and balance data across a broader spectrum of classes. This paper attempts to utilize both traditional and Transformer based approaches for Hinglish emotion classification. 


\section{Dataset}
The SemEval 2024 Task 10 dataset \cite{kumar-etal-2023-multilingual} comprises 8056 samples, featuring fields such as ID, speaker, utterance, and emotion. The ID uniquely identifies each episode of the conversation, while the speaker field denotes the person speaking. The utterance field represents the dialogue, expressed in Hinglish, and the emotion field indicates the corresponding emotion conveyed in the utterance. Adding on, the validation dataset contains 1354 samples while the test dataset contains 1580 samples. Table \ref{distribution} shows the distribution of labels in the dataset.

\begin{table}[htbp]
\centering
\begin{tabular}{ll}
\hline
\textbf{Emotion}  & \textbf{Count}    \\
\hline
\textbf{Anger}    & 819           \\
\textbf{Contempt} & 542           \\
\textbf{Disgust}  & 127           \\
\textbf{Fear}     & 514         \\
\textbf{Joy}      & 1596                    \\
\textbf{Neutral}  & 3909           \\
\textbf{Sadness}  & 558           \\
\textbf{Surprise} & 441              \\  
\hline
\end{tabular}
\caption{Distribution of emotions and their respective counts.}
\label{distribution}
\end{table}




\section{Data Preprocessing}
In the domain of code-mixed emotion recognition, preprocessing the utterances is essential for effective model training. The emotion column, representing a spectrum of eight distinct emotions—'disgust', 'contempt', 'anger', 'neutral', 'sadness', 'fear', and 'surprise'—is encoded using a label encoder for standardized representation. 
Code-mixed data inherently presents spelling ambiguities, demanding robust normalization techniques. For example, the word 'friend' in Hindi could be spelled as 'dost', 'dhosth', 'dhost' etc. Spelling correction is done using a phonetic similarity assessment. For each word, a phonetic code is computed and identifies feasible correction candidates from a dynamically created phonetic dictionary. The Levenshtein distance metric is used to evaluate the dissimilarity between the input word and potential corrections. This procedure is applied to all the utterances, on each word. The resultant corrected words are subsequently merged to form a spell-corrected utterance. A dictionary of all the speakers is also created, and the speaker names present in the utterances are removed, along with numbers and symbols. 

\section{Proposed Methodology}
\subsection{Support Vector Machine, Multinomial Naive Bayes and Random Forest}
To classify the utterances into one of the eight emotion classes, emotion labels were encoded using LabelEncoder. The CountVectorizer transformed text into numerical features. Initially, standard classification models like Support Vector Machines (SVM), Multinomial Naive Bayes (MNB), and Random Forest(RF) were utilized. These models were chosen based on their suitability for text classification tasks and their potential effectiveness in handling emotion classification within Hindi-English code-mixed data.
%SVMs are known to identify decision boundaries that effectively separate the data points belonging to different classes while minimizing misclassifications. MNBs work on the idea of a frequency-based representation of words or features. It calculates the conditional probabilities of each emotion class based on observed word frequencies in the code-mixed corpus. Random Forest classifiers adopt an ensemble learning approach, constructing multiple decision trees during the training phase and amalgamating their predictions to derive a final classification outcome. 
These models were trained on the training set and evaluated on the validation set using accuracy and the weighted F1 score metrics. Table \ref{ml_pre} and \ref{ml_acc} shows the precision scores and other performance metrics of each of the standard machine learning models.





\begin{table}[]
\centering
\begin{tabular}{llll}
\hline
\textbf{Emotion}  & \textbf{SVM}  & \textbf{MNB}  & \textbf{RF}   \\
\hline
\textbf{Anger}    & 0.00          & 0.12          & \textbf{0.19} \\
\textbf{Contempt} & \textbf{0.33} & 0.00          & 0.17          \\
\textbf{Disgust}  & 0.00          & 0.00          & \textbf{1.00} \\
\textbf{Fear}     & \textbf{0.33} & 0.00          & 0.24          \\
\textbf{Joy}      & 0.55          & \textbf{0.58} & 0.55          \\
\textbf{Neutral}  & 0.43          & 0.43          & \textbf{0.44} \\
\textbf{Sadness}  & 0.00          & 0.27          & \textbf{0.28} \\
\textbf{Surprise} & 0.22          & \textbf{0.29} & 0.27       \\  
\hline
\end{tabular}
\caption{Precision scores of standard machine learning models}
\label{ml_pre}
\end{table}


\begin{table}[]
\centering
\begin{tabular}{cccc}
\hline
                            \textbf{Metric} & \textbf{SVM}  & \textbf{MNB} & \textbf{RF}   \\
\hline
\textbf{Testing Accuracy}    & \textbf{0.44} & 0.40         & 0.43          \\
\textbf{Testing Weighted F1} & 0.31          & 0.30         & \textbf{0.33}\\
\hline
\end{tabular}
\caption{Performance metrics of standard machine learning models}
\label{ml_acc}
\end{table}





\subsection{Long Short Term Memory (LSTM)}
A Bidirectional LSTM model was then leveraged to address the challenges that could not be resolved by the SVM, MNB, and RF models. This model architecture is well-suited for sequential data processing tasks due to its inherent ability to capture long-range dependencies in text sequences. Figure \ref{fig:lstm_op}
shows the architecture diagram of the Bidirectional LSTM model.
\begin{figure}[h]
  \centering
  \includegraphics[height=8cm]{lstm.png}  
  \captionsetup{justification=centering}  
  \caption{Architecture diagram of the Bidirectional LSTM model}
  \label{fig:lstm_op}
\end{figure}

This bidirectional processing allows the model to effectively capture contextual information from preceding and succeeding words. The model architecture is described as follows:

\textbf{Embedding Layer:} This layer transforms input words into dense vectors of fixed size. It facilitates the representation of words in a continuous vector space, where similar words have similar representations.

\textbf{Spatial Dropout1D Layer:} This layer applies dropout to the input features with a dropout rate of 0.2. It helps prevent overfitting by randomly dropping input units during training.

\textbf{Bidirectional LSTM Layers:} The model consists of two Bidirectional LSTM layers. Each layer comprises 64 units and processes input sequences in both forward and backward directions. 

\textbf{Dense Layers:} Two dense layers follow the LSTM layers. The first dense layer has 64 units and uses the ReLU activation function. The final dense layer has 8 units (equal to the number of emotion classes) and uses the softmax activation function for multi-class classification.


The training parameters are as follows:

\textbf{Optimizer:} The model is optimized using the Adam optimizer, a popular choice for training neural networks due to its adaptive learning rate.


\textbf{Loss Function:} Sparse categorical cross-entropy is used as the loss function, suitable for multi-class classification tasks with integer-encoded target labels.

\textbf{Early Stopping:} Training includes early stopping with a patience of 3 epochs. It monitors the loss metric and restores the best weights when no improvement is observed after the specified number of epochs.


\textbf{Batch Size:} Training is performed with a batch size of 32.

\textbf{Epochs:} The model is trained for a maximum of 10 epochs. 

\begin{table}[]
\centering
\begin{tabular}{lc}
\hline
\textbf{Emotion }          & \textbf{LSTM Precision Values} \\
\hline
\textbf{Anger}    & 0.06                           \\
\textbf{Contempt} & 0.08                           \\
\textbf{Disgust}  & 0.017                          \\
\textbf{Fear}     & 0.48                           \\
\textbf{Joy}      & 0.38                           \\
\textbf{Neutral}  & 0.12                           \\
\textbf{Sadness}  & 0.12                           \\
\textbf{Surprise} & 0.21              \\
\hline
\end{tabular}
\caption{Precision scores of LSTM model}
\label{lstm_scores}
\end{table}

The Bidirectional LSTM model achieves a test accuracy of \textbf{0.35} with a weighted F1 score of \textbf{0.43} on the testing set. Table \ref{lstm_scores} shows the precision scores of LSTM model.
 
\subsection{Hindi-English Code Mixed BERT Models}
The usage of BERT (Bidirectional Encoder Representations from Transformers) models tailored for Hindi-English code-mixed data can significantly enhance the accuracy and effectiveness of emotion classification tasks. These models are pre-trained on large corpora of code-mixed text and can be fine-tuned for specific classification tasks. In this section, three models from the  L3Cube Pune team \cite{nayak-joshi-2022-l3cube}, are utilized-  namely HingBERT, Hing-mBERT, and HingRoBERTa.

\subsubsection{HingBERT}
HingBERT, akin to its BERT counterpart, comprises a stack of transformer blocks, typically 12 in number, with self-attention mechanisms and feed-forward neural networks. The model's architecture includes special tokens such as [CLS] and [SEP] to denote sentence boundaries and separation.

\subsubsection{Hing mBERT}
Hing mBERT inherits the architecture of BERT but is trained across a multitude of languages, including Hindi and English. Its architecture remains consistent with BERT's stack of transformer blocks, each equipped with self-attention mechanisms for capturing contextual information.


\subsubsection{Hing RoBERTa}

Hing RoBERTa, an extension of the RoBERTa architecture, delves into the intricacies of Hindi-English code-mixed text by integrating advanced architectural modifications. Built upon the foundation of RoBERTa's transformer-based architecture, Hing RoBERTa leverages deeper stacks of transformer layers, intricate attention mechanisms, and optimized weight initialization strategies to handle the nuances of bilingual conversations. With augmented batch sizes and increased learning rates, Hing RoBERTa optimizes gradient descent algorithms to navigate the vast parameter space effectively\cite{liu2019roberta}. Figure \ref{fig:hingrob_ar} shows the architecture diagram of the Transformer-based models.

\begin{figure}[h]
  \centering
  \includegraphics[height=9cm]{architecture.png}  
  \captionsetup{justification=centering}  
  \caption{Architecture diagram of Transformer-based models}
  \label{fig:hingrob_ar}
\end{figure}

\subsubsection{Implementation}
The implemented framework revolves around fine-tuning the HingBERT, Hing mBERT, and Hing RoBERTa Transformer-based models.

\textbf{Architecture:}
The architecture is characterized by the transformer’s ability to capture long-range dependencies and intricate contextual nuances within text sequences.
Each model comprises a series of transformer blocks, with HingBERT and Hing mBERT featuring 12 transformer layers, while HingRoBERTa encompasses a more extensive architecture with 12 or more layers, as per its pre-defined configuration.
Within each transformer block, self-attention mechanisms enable the model to dynamically weigh the importance of individual tokens based on their contextual relevance, facilitating effective feature extraction and representation learning.


\textbf{Multi-Head Attention Mechanism:}
The attention mechanism, a pivotal component of the transformer architecture, is augmented with multi-head attention, allowing the model to attend to different parts of the input sequence simultaneously.


\textbf{Feed-Forward Neural Networks (FFNN):}
Following the self-attention mechanism, token representations are fed through feed-forward neural networks (FFNN) within each transformer block.
FFNNs consist of multiple layers of linear transformations, interspersed with non-linear activation functions, such as the Rectified Linear Unit (ReLU), facilitating nonlinear transformations and feature extraction at each layer.


\textbf{Gradient Clipping:}
Gradient clipping is employed during the backpropagation phase to alleviate the issue of exploding gradients, ensuring stable training dynamics and promoting convergence.

\textbf{Embedding Layers:}
Token embeddings are employed to represent individual tokens within the input sequences, with dimensions determined by the pre-trained embedding matrices.
Positional encodings are added to the token embeddings to convey positional information, allowing the model to differentiate between tokens based on their relative positions within the sequence.
\\
\\
\textbf{Activation Functions and Layer Normalization:}
Activation functions such as the GELU (Gaussian Error Linear Unit) are applied within the feed-forward neural networks to introduce non-linearity and enable the modeling of complex relationships within the data.

Tables 5 and 6 show the precision value across emotions and the accuracy and weighted F1-scores for the three Transformer-based models. 



\begin{table}[]
\centering
\begin{tabular}{llll}
\hline
\textbf{Emotion}  & \textbf{\begin{tabular}[c]{@{}l@{}}Hing\\ BERT\end{tabular}} & \textbf{\begin{tabular}[c]{@{}l@{}}Hing\\ mBERT\end{tabular}} & \textbf{\begin{tabular}[c]{@{}l@{}}Hing\\ RoBERTa\end{tabular}} \\
\hline
\textbf{Anger}    & 0.28                                                         & 0.27                                                          & \textbf{0.33}                                                   \\
\textbf{Contempt} & 0.19                                                         & 0.16                                                          & \textbf{0.26}                                                   \\
\textbf{Disgust}  & \textbf{0.25}                                                & 0.20                                                          & 0.20                                                            \\
\textbf{Fear}     & 0.24                                                         & 0.23                                                          & \textbf{0.34}                                                   \\
\textbf{Joy}      & 0.45                                                         & 0.49                                                          & \textbf{0.54}                                                   \\
\textbf{Neutral}  & \textbf{0.52}                                                & \textbf{0.52}                                                 & \textbf{0.52}                                                   \\
\textbf{Sadness}  & 0.35                                                         & 0.28                                                          & \textbf{0.36}                                                   \\
\textbf{Surprise} & 0.31                                                         & \textbf{0.34}                                                 & 0.30                                                          \\
\hline
\end{tabular}
\caption{Precision scores of BERT based models}
\label{bert-precision}
\end{table}


\begin{table}[]
\centering
\begin{tabular}{llll}
\hline
                             & \textbf{\begin{tabular}[c]{@{}l@{}}Hing\\ BERT\end{tabular}} & \textbf{\begin{tabular}[c]{@{}l@{}}Hing\\ mBERT\end{tabular}} & \textbf{\begin{tabular}[c]{@{}l@{}}Hing\\ RoBERTa\end{tabular}} \\
\hline
\textbf{Accuracy}    & 0.45                                                         & 0.44                                                          & \textbf{0.47}                                                   \\
\textbf{Weighted F1} & 0.42                                                         & 0.43                                                          & \textbf{0.45}      \\
\hline
\end{tabular}
\caption{Performance metrics of BERT based models}
\label{bert-metric}
\end{table}


\section{Results and Analysis}
\subsection{SVM, MNB and RF}
The Support Vector Machine (SVM) classifier demonstrates varying performance across different emotions. Notably, it achieves relatively high precision for Contempt and Fear classes, scoring 0.33 for each. However, its precision is very low for Anger, Disgust, and Sadness, achieving 0.00 precision for these emotions. SVM's performance seems to struggle particularly with emotions characterized by intensity and subtlety. 
Multinomial Naive Bayes (MNB) exhibits competitive performance, particularly evident in its precision for Joy and Surprise emotions, achieving 0.58 and 0.29 respectively, which are among the highest precision values across all models. 

Random Forest (RF) emerges as a robust performer across various emotions, demonstrating balanced precision values across the emotion spectrum. RF achieves perfect precision (1.00) for Disgust, indicating its capability to discern this emotion accurately within code-mixed text. Additionally, RF performs consistently well for Neutral and Sadness emotions. 

While SVM and MNB show specific strengths for certain emotions, such as Fear and Joy respectively, RF emerges as a more balanced performer across the emotion spectrum, particularly excelling in capturing nuances associated with Disgust. 


\subsection{LSTM}
The LSTM model's precision values exhibit notable variations across different emotions. While it achieves relatively high precision in classifying Fear (0.48) and Joy (0.38), its performance significantly diminishes in categorizing Disgust (0.017) and Anger (0.06). Despite its recurrent nature and ability to retain sequential information, the LSTM model appears to struggle with the contextual intricacies present in the emotion classification task. It achieves a weighted F1-score of 0.43.

\subsection{Hindi-English Code-Mixed BERT Models}
The BERT-based models showcase more consistent and generally higher precision values across various emotions. Specifically, Hing RoBERTa emerges as the top performer among the BERT-based models, achieving the highest precision scores in several emotional categories, including Contempt (0.26), Fear (0.34), Joy (0.54), and Sadness (0.36). Hing BERT and Hing mBERT also demonstrate competitive precision values, albeit slightly lower than Hing RoBERTa. HingRoBERTa achieves the highest weighted F1-score of 0.45. Table \ref{bert-precision} and \ref{bert-metric} shows the precision scores and other performance metrics of BERT-based models.

Our team, TechSSN1, placed 7\textsuperscript{th} out of 39 participating teams in the shared subtask A.


\section{Conclusion}
The future scope of this work entails improving and enhancing the proposed models to handle a wider variety of data. The unstructured nature of Hinglish poses a challenge to the model's performance. By understanding the nuances, fine-tuning can be implemented to enhance the model's efficacy. Additionally, the work can be extended to encompass the classification of other types of emotions apart from the traditional Ekman model and refined to undertake tasks such as sarcasm or humor detection.

\bibliography{latex/References}


\end{document}
