% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[preprint]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{graphicx}
\usepackage{mdframed}
\usepackage{verbatim}
\usepackage{tabularx}
\usepackage{makecell}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.
\title{SCaLAR at SemEval-2024 Task 8: Unmasking the machine : Exploring the power of RoBERTa Ensemble for Detecting Machine Generated Text}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{Anand Kumar M \\
  Department of IT  \\
  NITK, India\\ \\\And
  Abhin B \\
  Artificial Intelligence\\
  Department of IT\\
  NITK, India\\ \\\And
  Sidhaarth Sredharan \\
  Artificial Intelligence\\
  Department of IT\\
  NITK, India\\}

\begin{document}
\maketitle
\begin{abstract}
SemEval SubtaskB, a shared task that is concerned with the detection of text generated by one out of the 5 different models - davinci, bloomz, chatGPT, cohere and dolly. This is an important task considering the boom of generative models in the current day scenario and their ability to draft mails, formal documents, write and qualify exams and many more which keep evolving every passing day. The purpose of classifying text as generated by which pre-trained model helps in analyzing how each of the training data has affected the ability of the model in performing a certain given task.  In the proposed approach, data augmentation was done in order to handle lengthier sentences and also labelling them with the same parent label. Upon the augmented data three RoBERTa models were trained on different segments of data which were then ensembled using a voting classifier based on their R2 score to achieve a higher accuracy than the individual models itself. The proposed model achieved an overall validation accuracy of 97.05\% and testing accuracy of 76.25\%.
\end{abstract}

\section{Introduction}
In the current day scenario, AI has noticed a major boom due the emergence of Large Language Models (LLMs) in the field of Natural Language Processing (NLP). These LLMs are capable of generating text with any given context that they have been trained on making them versatile to a lot of applications. LLMs have also showcased their unrivaled ability to code basic to complex programs. Many Large Language Models (LLMs) depend heavily on the data used for their training. Consequently, they may occasionally provide inaccurate information, especially in contexts where precision is crucial, such as sensitive or professional advice. Hence AI-generated text classification has become increasingly important due to the surge in the use of language models for content creation. Accurately identifying the source of a text, whether human-written or generated by a specific language model, is crucial for various applications, such as combating misinformation and plagiarism detection. Subtask B -  Multi-Way Machine-Generated Text Classification shared task aims to not only detect text generated by these language models, but also specifically distinguish between outputs generated by different models. This in a real life scenario helps in determining the transparency and vulnerability of a model to attacks and reasoning as to why particular models perform in certain ways.  Different contributions of the paper is as follows :-
\begin{itemize}
    \item \textbf{Data Augmentation:} Data augmentation is a crucial task of increasing the volume of available data with specific manipulations which also helps build a more robust model able to tackle edge case scenarios. We propose a novel approach to handle long texts. We initially set a specific threshold to split them into smaller segments while preserving label information, ensuring efficient model training. 
    \item \textbf{Ensemble Learning:} Ensemble learning as name suggests weaker models are brought together to achieve a better model with enhanced performance. We employ a weighted ensemble voting classifier that combines the predictions of multiple models trained on diverse validation sets, leading to improved generalizability and robustness.
\end{itemize}

We observe how effective and relevant Language models are in tackling Natural Language Processing tasks such as the current shared task when compared to other neural network based LSTM or other sequence models. Our final submission had a test accuracy of 76.25\% and our standing was 18th position in the leaderboard.

\section{Background}
Our work improves model generalizability in large-scale tasks by utilising insights from \cite{wang2024m4} and building on recent studies. Previously methods proposed by authors in \cite{ma2023ai} collected 500 scientific articles from 10 domains including biology, chemistry, IT and others and used chatGPT to paraphrase texts for each article. The authors extracted certain features such as perplexity, semantic document and six others to use classifiers on these extracted features. The authors used three classifiers: XGBoost, random forest, and multi-layer perceptron, to train and test models for detecting human-generated and AI-generated texts, as well as human-generated and AI-rephrased texts. They performed a 5-fold cross-validation and evaluated the models using accuracy and F1-score which majorly motivated our approach. Another work \cite{mindner2023classification} used similar techniques to the previous one while using school topics as their dataset. In their work \cite{abburi2023generative} the authors use ensemble neural model that generates probabilities from different pre-trained LLMs which are used as features to a TML classifier following it. Author in \cite{huimin2018ensemble} presents his work on text classification ensemble learning method based on multi-angle perturbation heterogeneous base classifiers and validates the effectiveness of the algorithm through experiments. In a similar work \cite{mohammed2022effective} the authors propose a new meta-learning ensemble method that fuses baseline deep learning models using 2-tiers of meta-classifiers.

Furthermore, our method for comprehending model decision-making in short text classification—particularly in identifying AI-generated content—is influenced by methods from works on short text classification. Authors in their work \cite{tang2022long} use a sliding window to align the sentences with the labels and preserve the edge characteristics of the long text. Another work in the same field \cite{shorten2021text} categorizes text data augmentations into symbolic and neural methods. Symbolic methods use rules or discrete data structures to form new examples, while neural methods use auxiliary neural networks to sample new data.
Our research aims to advance the development of robust and adaptable machine learning models customised to particular tasks through this synthesis of diverse viewpoints. These viewpoints are then combined back again with the help of emsembling ensuring no loss of data. 

\section{Methodology}

\begin{figure*}[!h]
\centering
\includegraphics[width=1.1\textwidth]{flowchartroberta.png}
\caption{Proposed methodology for AI vs Human generated text Detection using weighted voting ensembling of RoBERTa classifiers}
\label{fig:example}
\end{figure*}


Our methodology majorly focuses on exploiting Pre-trained language models such as the RoBERTa model \cite{liu2019roberta} and enhancing its performance through a much simpler traditional approach of ensemble learning. We worked on the M4 based dataset with our methodology \cite{wang2023m4} The ensemble model shows better performance compared to all 3 RoBERTa-base models which were trained on different segments of augmented data. It reduces over-fitting and increases interpret-ability for any given task.

\subsection{RoBERTaForSequenceClassification}

RoBERTa which stands for Robustly Optimized BERT Approach is a variant of the famous BERT model \cite{devlin2018bert} which was developed by Google in 2018. RoBERTa was later introduced by researchers at Facebook AI in 2019. It builds upon the architecture of BERT while bringing in few major changes. It uses Dynamic masking strategies and removes the Next Sentence Prediction (NSP) in its pre-training step. It is further pre-trained with larger amounts of data with larger mini-batch size. The novelty of RoBERTa lies in its ability to achieve state-of-the-art performance on various natural language understanding benchmarks by leveraging advancements in pre-training techniques and model architectures. RoBERTa continues to employ similar tokenizing technique as BERT with WordPiece Encoder (WPE). RoBERTa as a base model in itself gives out embedding for a given sentence or a word as it is only composed of Encoder architecture. 

RoBERTaForSequenceClassifcation consists of a classification head on top of the base RoBERTa model. This classification head maps the backbone outputs to logits suitable for a classification task based on the number of labels provided. 

\subsection{Ensemble Learning}
Ensemble learning is a machine learning technique that combines multiple individual  models to obtain a model with enhanced performance which is more robust as well. It involves training several individual base models which are often referred to as experts on similar data and producing an aggregation out of those models based on their individual performances. The benefits of ensembling include improved generalization, more robustness compared to single models, and efficient as it compensates for the loss in performance of the poor learning algorithms. The common techniques in Ensemble learning including Bagging and Boosting. 
We specifically used the concept of Voting classifier which takes predictions from different models and has a specified weighting parameter based on which it gives out the final prediction. We implemented our own Voting classifier which scores the three RoBERTa models based on the R2-scores achieved by their predictions. R2-scores here are used as the weighting parameter for the prediction and thus we derive our final prediction out of the voting classifier.

\section{System Overview}
As mentioned, we first perform data augmentation.  Before we get into the details of our experimental setup, we want to elaborate on different measures we took in order to augment our data. Data augmentation for training data was performed by carefully splitting the validation data while noting that there is no major imbalance in the class distribution. This was followed by training three different RoBERTa models on different combinations of training and validation dataset. We had 3 validation data splits namely \textit{val1}, \textit{val2} and \textit{val3}. For \textit{model1} we used \textit{val2} and \textit{val3} in training and \textit{val1} for validating the \textit{model1} and so on.

\subsection{Data Augmentation}
We implemented a data augmentation strategy to address instances in our dataset exceeding the token limit, ensuring no information loss while maintaining model compatibility. Given a dataset comprising 71,027 instances for training and 3,000 for validation, with some instances surpassing the 512-token limit, we devised a method to split these instances into $k$ different segments. Utilizing the modulo operation, if an instance contains $n$ tokens, $[n / 512]$ determines the number of segments it will be divided into, while the remainder represents the number of tokens in the last augmented segment of the instance. This process yielded approximately 9985 additional instances for training and 188 for validation.

Subsequently, we merged the augmented training and validation sets to form a combined dataset of 81,012 training instances and 3188 validation instances. This validation dataset was then partitioned into three parts of which two-thirds are used for training alone with the complete training data and the remaining one-third is used for validation. Notably, each RoBERTa model was provided with a distinct subset of one-third of the validation data, thus adhering to a different $k$-fold validation scheme to enhance generalizability.

\subsection{Implementation Details}
The implementation of our method includes three vanilla RoBERTaForSequenceClassification models with 12 encoder layers with a classification head at the end were used. These models were trained on three different splits of two-thirds of validation data coupled with the training data. Each model was effectively trained on roughly 82000 samples with roughly 1060 validation instances. The voting classifier first takes in all three fine-tuned RoBERTa models and predicts on the complete validation set and analyzes the performance of each of the models based on their R2-score and constructs a weighted voting classifier which gives our final predictions. The R2 scores observed for each of the three models were 0.36, 0.29 and 0.35 which had roughly similar weight given to each of their predictions. 
The performance of each of the models were analyzed with the help of training and validation loss plots across training epochs.

\begin{figure}
    \centering
    \includegraphics[width=1\linewidth]{model1.png}
    \caption{Training and validation loss observed over the RoBERTa model-1.}
    \label{fig:enter-label}
\end{figure}

\begin{figure}
    \centering
    \includegraphics[width=1\linewidth]{model2.png}
    \caption{Training and validation loss observed over the RoBERTa model-2.}
    \label{fig:enter-label}
\end{figure}

\begin{figure}
    \centering
    \includegraphics[width=1\linewidth]{model3.png}
    \caption{Training and validation loss observed over the RoBERTa model-3.}
    \label{fig:enter-label}
\end{figure}

\section{Experimental Results}
As a part of our experimental setup we used P100 GPU which is available through kaggle. Further we used the \texttt{RobertaForSequenceclassification} available thorugh transformers library along with \texttt{RobertaTokenizerFast} for the modelling aspect. The learning rate used was a fixed one and we found it optimal at $1e-5$ along with $Cross Entropy Loss$. $AdamW$ optimizer was used with weight decay coefficient of $1e-2$ and $\beta_1 = 0.9$ and $\beta_2 = 0.999$. Batch size of 20 was used for training and validation. 

Our proposed methodology beat the baseline model which was a RoBERTa model with an average accuracy of 0.75. Our experimental results with respect to each of the RoBERTa model is displayed along with the improvement in performance with the use of R2-score based weighted voting classifier. In testing phase our model gave an accuracy of 76.25\% which shows clear signs of over-fitting compared to 97.05\% in validation accuracy. 

\begin{table}[h!]
\renewcommand{\arraystretch}{1.2}
\setlength{\tabcolsep}{4pt} % for the horizontal padding
\caption{Proposed methodology performance comparison}
\label{results-legal}
\begin{minipage}{0.48\textwidth}
\begin{center}
\begin{tabular}{p{3.1cm}ccc}
\hline
\textit{Models}& \textit{Train Acc (\%)}&\textit{Val Acc (\%)}\\
\hline
Baseline & 75 & 75\\
%\hline
RoBERTa-1 & 96.40  & 95.06\\
%\hline
\makecell[l]{RoBERTa-2} & 93.64 & 92.10   \\
%\hline
RoBERTa-3 & 97.21 & 96.62 \\
%\hline
\textbf{\makecell[l]{Voting Classifier \\ (\textit{proposed})}} & \textbf{97.26} & \textbf{97.05}\\
\hline
\end{tabular}
\end{center}
\end{minipage}
\end{table}

\section{Conclusion}
In the proposed methodology, we beat the baseline RoBERTa model and further enhance the performance of the model using R2-score based Voting classifier. The model has performed well on the training data when compared to testing data which shows slight signs of over-fitting. In the light of ensemble learning for Pre-trained language models we see that the models are very sensitive to over-fitting hence should be used with caution. Techniques like early stopping and using data augmentation. Further on embeddings from LLMs can be used to tackle this task more effectively. 

% To produce a PDF file, pdf\LaTeX{} is strongly recommended (over original \LaTeX{} plus dvips+ps2pdf or dvipdf). Xe\LaTeX{} also produces PDF files, and is especially suitable for text in non-Latin scripts.

% \section{Preamble}

% The first line of the file must be
% \begin{quote}
% \begin{verbatim}
% \documentclass[11pt]{article}
% \end{verbatim}
% \end{quote}

% To load the style file in the review version:
% \begin{quote}
% \begin{verbatim}
% \usepackage[review]{acl}
% \end{verbatim}
% \end{quote}
% For the final version, omit the \verb|review| option:
% \begin{quote}
% \begin{verbatim}
% \usepackage{acl}
% \end{verbatim}
% \end{quote}

% To use Times Roman, put the following in the preamble:
% \begin{quote}
% \begin{verbatim}
% \usepackage{times}
% \end{verbatim}
% \end{quote}
% (Alternatives like txfonts or newtx are also acceptable.)

% Please see the \LaTeX{} source of this document for comments on other packages that may be useful.

% Set the title and author using \verb|\title| and \verb|\author|. Within the author list, format multiple authors using \verb|\and| and \verb|\And| and \verb|\AND|; please see the \LaTeX{} source for examples.

% By default, the box containing the title and author names is set to the minimum of 5 cm. If you need more space, include the following in the preamble:
% \begin{quote}
% \begin{verbatim}
% \setlength\titlebox{<dim>}
% \end{verbatim}
% \end{quote}
% where \verb|<dim>| is replaced with a length. Do not set this length smaller than 5 cm.

% \section{Document Body}

% \subsection{Footnotes}

% Footnotes are inserted with the \verb|\footnote| command.\footnote{This is a footnote.}

% \subsection{Tables and figures}

% See Table~\ref{tab:accents} for an example of a table and its caption.
% \textbf{Do not override the default caption sizes.}

% \begin{table}
% \centering
% \begin{tabular}{lc}
% \hline
% \textbf{Command} & \textbf{Output}\\
% \hline
% \verb|{\"a}| & {\"a} \\
% \verb|{\^e}| & {\^e} \\
% \verb|{\`i}| & {\`i} \\ 
% \verb|{\.I}| & {\.I} \\ 
% \verb|{\o}| & {\o} \\
% \verb|{\'u}| & {\'u}  \\ 
% \verb|{\aa}| & {\aa}  \\\hline
% \end{tabular}
% \begin{tabular}{lc}
% \hline
% \textbf{Command} & \textbf{Output}\\
% \hline
% \verb|{\c c}| & {\c c} \\ 
% \verb|{\u g}| & {\u g} \\ 
% \verb|{\l}| & {\l} \\ 
% \verb|{\~n}| & {\~n} \\ 
% \verb|{\H o}| & {\H o} \\ 
% \verb|{\v r}| & {\v r} \\ 
% \verb|{\ss}| & {\ss} \\
% \hline
% \end{tabular}
% \caption{Example commands for accented characters, to be used in, \emph{e.g.}, Bib\TeX{} entries.}
% \label{tab:accents}
% \end{table}

% \subsection{Hyperlinks}

% Users of older versions of \LaTeX{} may encounter the following error during compilation: 
% \begin{quote}
% \tt\verb|\pdfendlink| ended up in different nesting level than \verb|\pdfstartlink|.
% \end{quote}
% This happens when pdf\LaTeX{} is used and a citation splits across a page boundary. The best way to fix this is to upgrade \LaTeX{} to 2018-12-01 or later.

% \subsection{Citations}

% \begin{table*}
% \centering
% \begin{tabular}{lll}
% \hline
% \textbf{Output} & \textbf{natbib command} & \textbf{ACL only command}\\
% \hline
% \citep{Gusfield:97} & \verb|\citep| &  \\
% \citealp{Gusfield:97} & \verb|\citealp| & \\
% \citet{Gusfield:97} & \verb|\citet| &  \\
%   \citeyearpar{Gusfield:97} & \verb|\citeyearpar| &  \\
%   \citeposs{Gusfield:97}	&	& \verb|\citeposs|\\
% \hline
% \end{tabular}
% \caption{\label{citation-guide}
% Citation commands supported by the style file.
% The style is based on the natbib package and supports all natbib citation commands.
% It also supports commands defined in previous ACL style files for compatibility.
% }
% \end{table*}

% Table~\ref{citation-guide} shows the syntax supported by the style files.
% We encourage you to use the natbib styles.
% You can use the command \verb|\citet| (cite in text) to get ``author (year)'' citations, like this citation to a paper by \citet{Gusfield:97}.
% You can use the command \verb|\citep| (cite in parentheses) to get ``(author, year)'' citations \citep{Gusfield:97}.
% You can use the command \verb|\citealp| (alternative cite without parentheses) to get ``author, year'' citations, which is useful for using citations within parentheses (e.g. \citealp{Gusfield:97}).

% A possessive citation can be made with the command \verb|\citeposs|.
% This is not a standard natbib command, so it is generally not compatible
% with other style files.

% \subsection{References}

% \nocite{Ando2005,andrew2007scalable,rasooli-tetrault-2015}

% The \LaTeX{} and Bib\TeX{} style files provided roughly follow the American Psychological Association format.
% If your own bib file is named \texttt{custom.bib}, then placing the following before any appendices in your \LaTeX{} file will generate the references section for you:
% \begin{quote}
% \begin{verbatim}
% \bibliography{custom}
% \end{verbatim}
% \end{quote}

% You can obtain the complete ACL Anthology as a Bib\TeX{} file from \url{https://aclweb.org/anthology/anthology.bib.gz}.
% To include both the Anthology and your own .bib file, use the following instead of the above.
% \begin{quote}
% \begin{verbatim}
% \bibliography{anthology,custom}
% \end{verbatim}
% \end{quote}

% Please see Section~\ref{sec:bibtex} for information on preparing Bib\TeX{} files.

% \subsection{Appendices}

% Use \verb|\appendix| before any appendix section to switch the section numbering over to letters. See Appendix~\ref{sec:appendix} for an example.

% \section{Bib\TeX{} Files}
% \label{sec:bibtex}

% Unicode cannot be used in Bib\TeX{} entries, and some ways of typing special characters can disrupt Bib\TeX's alphabetization. The recommended way of typing special characters is shown in Table~\ref{tab:accents}.

% Please ensure that Bib\TeX{} records contain DOIs or URLs when possible, and for all the ACL materials that you reference.
% Use the \verb|doi| field for DOIs and the \verb|url| field for URLs.
% If a Bib\TeX{} entry has a URL or DOI field, the paper title in the references section will appear as a hyperlink to the paper, using the hyperref \LaTeX{} package.

% \section*{Acknowledgements}

% This document has been adapted
% by Steven Bethard, Ryan Cotterell and Rui Yan
% from the instructions for earlier ACL and NAACL proceedings, including those for 
% ACL 2019 by Douwe Kiela and Ivan Vuli\'{c},
% NAACL 2019 by Stephanie Lukin and Alla Roskovskaya, 
% ACL 2018 by Shay Cohen, Kevin Gimpel, and Wei Lu, 
% NAACL 2018 by Margaret Mitchell and Stephanie Lukin,
% Bib\TeX{} suggestions for (NA)ACL 2017/2018 from Jason Eisner,
% ACL 2017 by Dan Gildea and Min-Yen Kan, 
% NAACL 2017 by Margaret Mitchell, 
% ACL 2012 by Maggie Li and Michael White, 
% ACL 2010 by Jing-Shin Chang and Philipp Koehn, 
% ACL 2008 by Johanna D. Moore, Simone Teufel, James Allan, and Sadaoki Furui, 
% ACL 2005 by Hwee Tou Ng and Kemal Oflazer, 
% ACL 2002 by Eugene Charniak and Dekang Lin, 
% and earlier ACL and EACL formats written by several people, including
% John Chen, Henry S. Thompson and Donald Walker.
% Additional elements were taken from the formatting instructions of the \emph{International Joint Conference on Artificial Intelligence} and the \emph{Conference on Computer Vision and Pattern Recognition}.

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

% \appendix

% \section{Example Appendix}
% \label{sec:appendix}

% This is an appendix.

\end{document}
