% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% \usepackage[symbol]{footmisc}
\usepackage{todonotes}
\usepackage{amsmath}
\usepackage{tabularray}
\usepackage{array}
\usepackage{graphicx}
\usepackage{multirow}
\usepackage{ragged2e}
\usepackage{hhline}
\usepackage{arydshln}


% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

\usepackage{cleveref}
\crefname{section}{\S}{\S}
%\crefname{section}{s}{ss}
\crefname{table}{Table}{Tables}
\crefname{figure}{Fig.}{Figs.}
\crefname{algorithm}{Alg.}{}
\crefname{ALC@unique}{Line}{Lines}
\crefname{equation}{Eq.}{Eqs.}
\crefname{appendix}{App.}{Apps.}
\crefformat{section}{\S#2#1#3} 

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

% \newcommand{\missing}[1]{\textcolor{red}{(missing #1)}}

\title{HU at SemEval-2024 Task 8A: Can Contrastive Learning Learn Embeddings to Detect Machine-Generated Text?}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

% \renewcommand*{\thefootnote}{\fnsymbol{footnote}}
% \renewcommand{\thefootnote}{\arabic{footnote}}

\author{Shubhashis Roy Dipta \\
    Department of CSEE \\ 
    University of Maryland, Baltimore County \\
    Baltimore, Maryland, USA \\
  \texttt{sroydip1@umbc.edu} \\
  \And
  Sadat Shahriar \\
    University of Houston \\
    Houston, Texas, USA \\ 
  \texttt{sshahria@cougarnet.uh.edu} \\
}

\begin{document}


\maketitle


\begin{abstract}
This paper describes our system developed for SemEval-2024 Task 8, ``Multigenerator, Multidomain, and Multilingual Black-Box Machine-Generated Text Detection'' Machine-generated texts have been one of the main concerns due to the use of large language models (LLM) in fake text generation, phishing, cheating in exams, or even plagiarizing copyright materials. A lot of systems have been developed to detect machine-generated text. Nonetheless, the majority of these systems rely on the text-generating model. This limitation is impractical in real-world scenarios, as it's often impossible to know which specific model the user has used for text generation. In this work, we propose a \textbf{single} model based on contrastive learning, which uses \textbf{$\approx$40\% of the baseline's parameters} (149M vs. 355M) but shows a comparable performance on the test dataset (\textit{\textbf{21st out of 137 participants}}). Our key finding is that even without an ensemble of multiple models, a single base model can have comparable performance with the help of data augmentation and contrastive learning. \footnote{Our code is publicly available at \url{https://github.com/dipta007/SemEval24-Task8}}
\end{abstract}

\section{Introduction}
In recent years, Natural Language Processing (NLP) has been totally dependent on Deep Learning rather than statistical machine learning. With multi-task learning \citep{caruana1997multitask}, attention-based transformers \citep{vaswani2017attention}, and the use of Reinforcement Learning in NLP \citep{christiano2017deep}, it has been used in our day-to-day life from mathematical calculations \citep{yang2023gpt} to email writing. But with huge help, it has also been used to generate fake news \citep{Zellers_2019}, to plagiarize copyright materials \citep{dehouche2021plagiarism}, and also to cheat in exams or assignments \citep{cotton2023chatting, fyfe2023cheat}. Humans can identify machine-generated text only at the chance level \citep{Jawahar_2020}. There has been a dire need to develop a system to detect machine-generated text.

Though a lot of works \citep{Badaskar_2008, Gehrmann_2019, Zellers_2019, Jawahar_2020, Ippolito_2020, Chakraborty_2023, Pu_2023, Mitchell_2023, He_2023, Guo_2023} have already been deployed for detecting machine-generated text, with the current development of LLMs, most of the systems are failing to find out which one is human-generated vs. machine-generated (mostly due to the improvement of coherency, fluency and usage of real-world dataset \citep{radford2019language}). In this context, the task "Multigenerator, Multidomain, and Multilingual Black-Box Machine-Generated Text Detection" provides a dataset for training models to classify machine-generated texts. The shared task consists of three sub-tasks: Binary Classification (Machine vs. Human), Multi-class Classification (Which model/human generated this?), and Span Detection (Which part of the text is machine-generated?). A detailed description of the task can be found in the shared task paper \citep{semeval2024task8}. 

In this paper, we describe our final submission on Subtask A (Binary Classification). There were two big challenges of this task: \textbf{First,} five Different models have been used to generate the machine-generated text. \citet{Zellers_2019} has shown that the best defense for machine-generated text is the model itself that was used for generation. However, in reality, there is a massive surge in large language models (LLMs), each with its own unique style of text generation. The challenge in this particular subtask has heightened due to the utilization of five different LLMs. This complexity demands a versatile, model-agnostic architecture capable of detecting text generated by LLMs in a generalized manner. \textbf{Second,} Following the previous challenge, the organizers have employed a different model for generating the validation and test datasets compared to those used in the training set. This implies that the text was drawn from a completely distinct distribution. As a result, participants must develop a generalized model capable of performing effectively regardless of the specific model used in the text generation process.


In response to the key challenges, we have investigated the performance of contrastive learning for this particular task. Contrastive learning has been used as a valuable technique across various domains, including Text Embedding \citep{neelakantan2022text}, Document Embedding \citep{luo2021unsupervised}, Event Embedding \citep{roy-dipta-etal-2023-semantically}, vision \citep{chen2020simple} and Language-Vision learning \citep{radford2021learning}. Notably, unlike the majority of submissions in any shared task like competition, Our final submission utilized a \textbf{single} model to classify the machine-generated texts rather than an ensemble of multiple models. Hence, our contributions to this paper are as follows,

\begin{enumerate}
    \item We proposed a novel data augmentation technique, which nearly makes the data $X$ times bigger ($X$ is the number of models used for data augmentation).
    \item We propose a single unified model that shows a comparable performance on the test dataset.
    \item We have shown that even with a single model, contrastive learning with data augmentation shows a comparable performance, which opens up a door for future exploration.
\end{enumerate}


\section{Related Works}
In this section, we will provide the prior works that have been done in the realm of machine-generated text detection (\cref{rel_fake_text}) and contrastive learning (\cref{rel_fake_text}).

\subsection{Machine Generated Text detection} \label{rel_fake_text}
With the progress of LLMs, much prior research has been done to counter-attack the misuse of the LLMs. Before the attention and transformers, \citet{Badaskar_2008} has shown how the syntactic and semantic features can help in classifying between human and machine-generated text. Later, \citet{Gehrmann_2019} has provided a statistical detection system based on the assumption that the machine samples from the high probability words through max sampling \citep{gu2017trainable}, k-max sampling \citep{fan2018hierarchical}, beam search \citep{shao2017generating}. So, the authors used the probability, rank, and entropy of words as features to classify a machine-generated text. \citet{Jawahar_2020} has shown that state-of-the-art LLM can generate texts with human-like fluency and coherence without grammatical or spelling errors. Lastly, \citet{Mitchell_2023} have used the change of log-probability between the original text and after random perturbation. 

\subsection{Contrastive Learning} \label{rel_con_learn}
Contrastive learning was first introduced in the visual domain \citep{chen2020simple}. Later, it has been widely used in NLP for representation learning \citep{xu2023contrastive, wang2023sncse}, event similarity tasks \citep{gao2022improving} and event modeling \citep{roy-dipta-etal-2023-semantically}. Inspired by the latter works, we have explored whether contrastive learning can help in machine-generated text detection.


\section{System Overview}
Our system is divided into three parts: where the first part is data augmentation (described on \cref{system:data_aug}), the second part is contrastive learning (described on \cref{system:con_learning}), and the last part is the classification head (described on \cref{system:cls}) over the document embeddings.

\subsection{Data Augmentation} \label{system:data_aug}
The dataset provided in the shared task has text and their corresponding label. However, we need a positive and a (hard) negative pair to use contrastive learning. Our main inspiration for using contrastive learning is that as the texts come from two different entities (machine vs. human), the embedding space should also be different. To facilitate the task, we have used a paraphrase model to generate alternate texts for each text in the dataset. In that way, now, every instance of the dataset has one human/machine-generated text and one machine-generated text. We have utilized the human-generated text as the hard negative \footnote{Hard negatives are the total opposite of the given text} and the machine-generated text as the soft positive \footnote{Soft positives expressed the same idea but might not be the exact one}.

Another challenge we faced during the paraphrasing of the dataset is that the texts are long. If we give the whole text to the paraphrase model and ask for alternate text, it gives a much shorter text (an issue we observed in the used paraphrase model). In our primary validation, that gives bad results due to the loss of information while shortening the text. So, instead of giving the whole text at once, we have split the data by end-of-sentence or newline. Then, each sentence was paraphrased on its own and then joined together again to get the previous structure. The technical details behind generating paraphrases and using them for contrastive learning have been discussed in \cref{data_aug} and \cref{pre_trained_enc}, respectively.

\subsection{Contrastive Learning} \label{system:con_learning}
With the availability of an appropriate dataset for contrastive learning, we proceeded to develop our model. Our main assumption was that the embedding of the machine-generated text and human-generated text would exhibit significant differences. A simple overview of the model is shown in the \cref{fig:main_model}.

\begin{figure*}[!ht]
    \centering
    \includegraphics[width=\textwidth]{main_model.pdf}
    \caption{Overview of our model architecture. The same color weights are shared (encoder \& classifier head). Diamond boxes represent the loss function, and the plus sign represents the summation of the three losses. The input to the contrastive loss depends on the original label (y=$+1$ if human, else $-1$).}
    \label{fig:main_model}
\end{figure*}

The positive and negative data go through the same shared encoder to generate an embedding. This embedding is then used in contrastive learning. We have used the following loss formulation for our contrastive learning:

\begin{equation}
    \begin{aligned}
        \mathcal{L}_{con} &= (1 - y) * cos(x_1, x_2) \\ &+ y * max(0, cos(x_1, x_2)) 
    \end{aligned}
\end{equation}

Here, $x_1$ and $x_2$ are the embeddings of two different pairs, respectively. $cos(x_1, x_2)$ is the cosine-similarity score between two embeddings. $y$ is $+1$ for positive-positive pairs and $-1$ otherwise. In our task, $y$ is $+1$ if the data instance contains text from a machine and the other is paraphrased text and $-1$ if the given text is from a human and the other is paraphrased.

\subsection{Classification Loss} \label{system:cls}
In contrastive learning, our primary objective is to acquire meaningful embeddings containing sufficient information for distinguishing between human-generated and machine-generated text. However, we also need to use a classifier model for the downstream task of outputting the actual label. Keeping that in mind, we have used a simple two-linear layer classifier head on top of the embeddings generated by the encoder. During inference time, we used this classifier head to output the labels. We have optimized our model using a simple binary cross-entropy (BCE) loss.

The total loss of our model is defined as,

\begin{equation}
    \mathcal{L} = \alpha * \mathcal{L}_{con} + \beta * \mathcal{L}_{cls_+} + \gamma * \mathcal{L}_{cls_-}
\end{equation}

Here, $\mathcal{L}_{cls_+}$ is the BCE loss of the positive example, and $\mathcal{L}_{cls_-}$ is the BCE loss of the negative sample of the data instance. $\alpha$, $\beta$, and $\gamma$ are hyperparameters that were set to 0.7, 0.8, and 0.1, respectively, based on validation data.


\section{Experimental Setup}
The following sections are used to describe the technical details behind our data augmentation technique (\cref{data_aug}), Encoder (\cref{pre_trained_enc}), Classifier Head (\cref{cls_head}) and Hyperparameters (\cref{hparams}).

\subsection{Data Augmentation \& Pre-processing} \label{data_aug}
We preprocess the raw input, splitting each document into multiple sentences for paraphrasing. After the preprocessing, we got $\approx$ 3.6 million sentences. Even if we are splitting by new lines or end-of-sentences, we kept exactly the same format during joining, i.e., two new lines rather than 1, to keep most information intact. As the paraphrasing is done on the sentence level rather than the paragraph level, the number of paraphrased sentences is the same as the input sentences (3.6M). So, ideally, we got double the number of training data just by using the data augmentation.

We have tried multiple models from HuggingfaceHub \footnote{\href{https://huggingface.co/ibm/qcpg-sentences}{ibm/qcpg-sentences} \label{footnote:2}} \footnote{\href{https://huggingface.co/ceshine/t5-paraphrase-paws-msrp-opinosis}{ceshine/t5-paraphrase-paws-msrp-opinosis}} to generate paraphrase. In our final submission, we have used \citet{bandel-etal-2022-quality}'s model $^{\ref{footnote:2}}$ for our data augmentation. Use of multiple models or use of prompt-based models \citep{achiam2023gpt, touvron2023llama} for data augmentation has been left out for future exploration due to time and compute constraints. For data split, we use the official train \& dev data split. Only train data is used for data augmentation, and the dev data is used to calculate evaluation metrics.

\subsection{Pre-trained Encoder} \label{pre_trained_enc}
To encode the document, we have used a pre-trained version of longformer-base \citep{beltagy2020longformer} \footnote{\href{https://huggingface.co/jpwahle/longformer-base-plagiarism-detection}{jpwahle/longformer-base-plagiarism-detection}}. The reason behind using this encoder rather than others is, \textbf{One,} longformer is good for getting embeddings for long documents because of using global vs. local attention (more details in \citet{beltagy2020longformer}). \textbf{Second, } the pre-trained version was fine-tuned for paraphrase detection, which is kind of similar to our task.

\subsection{Classifier Head} \label{cls_head}
We have used two linear layers for classifier heads with \textit{tanh} activation loss between them. We also have used a dropout layer between them with a probability of 60\%. The primary rationale for using a high dropout rate was to enhance the model's generalization ability and reduce its dependence on the training data.

\subsection{Hyperparameters} \label{hparams}
For training our model, we have used AdamW \citep{loshchilov2017decoupled} optimizer with a learning rate of 1e-5. We have used a batch size of 2 with gradient accumulation for 8 steps (effective batch size 16). We have used early stopping on the validation data with patience 10. Maximum document length was set to 4096 as most of the documents are large. We use the PyTorch-lightning \footnote{\url{https://lightning.ai/}} library to run the experiments and Weight \& Biases \footnote{\url{https://wandb.ai/}} for logging. All of our experiments are run on NVIDIA Quadro RTX 8000 48GB.

\section{Results}
In this section, we report our results on subtask A and discuss our analysis. Our evaluation is based on the accuracy metric, but we have provided the micro and macro-f1 for better comparison. All the results are averaged on 3 runs with 3 different random seeds.

\subsection{Baseline \& Our Model}
We use the official baseline provided by the task organizers. They have used RoBERTa-large \citep{liu2019roberta} as the encoder and fine-tuned on the train data. Throughout the paper, we refer to this model as $baseline_{rob}$.

We have fine-tuned our model (shown in \cref{fig:main_model}) on the training dataset. Throughout the paper, we refer to this model as $ours_{con}$

In the \cref{tab:results}, we have reported the results on the official test file. $Ours_{con}$ is the final submission, and $Ours_{con}$+ is the modified version of our final model for more analysis (not official results; used for ablation study - details on \cref{sec:ablation}). We can get a comparable result using $~60\%$ fewer parameters than the baseline. In the next section, we will see that after hyperparameter tuning, we can get around 5.7\% improvement over the baseline. This supports our assumption that using a contrastive learning-based method can help machine-generated text identification.

\subsection{Ablation Study} \label{sec:ablation}

\paragraph{Effect of Maximum Sentence Length:} The maximum sentence length is used to tokenize the document. The optimal test score is achieved with a maximum sentence length of 256. This demonstrates that the model can effectively identify machine-generated text even with documents as large as 256 words. This underscores the effectiveness and adaptability of our model's learning capabilities.

\paragraph{Effect of Classification Dropout:} The classification dropout is applied between the two classification layers. Contrary to our initial assumption, the results presented in \cref{tab:results} indicate that using a low dropout rate (as low as 0.0) contributes positively to the model's learning process. This suggests that, even without dropout, the model's generalization to unseen data (text generated by a new model) is enabled primarily through contrastive learning and data augmentation.

\paragraph{Effects of (Effective) Batch Size:} Due to computational constraint, we have used a fixed batch size of 2 and gradient accumulation steps of $\{1, 2, 4, 8, 16, 32, 64\}$ resulting in an effective batch size of $\{2, 4, 8, 16, 32, 64, 128\}$. From the results report on \cref{tab:results}, we found that using only an effective batch size of 2 yielded superior performance compared to gradient accumulation. Notably, this configuration represents the most optimal result obtained following hyperparameter tuning, positioning us at the $8^{th}$ rank in the final standings. This suggests that, in this particular context, the benefits of gradient accumulation may be limited compared to simply using a smaller batch size.


\begin{table}
\centering
\resizebox{\linewidth}{!}{%
\begin{tabular}{l|cccccc}
\multicolumn{1}{l}{} & \begin{tabular}[c]{@{}c@{}}Max Sen\\Length\end{tabular} & \begin{tabular}[c]{@{}c@{}}CLS\\Dropout\end{tabular} & \begin{tabular}[c]{@{}c@{}}Effective\\Batch Size\end{tabular} & Macro-f1 & Micro-f1 & Accuracy \\ 
\hhline{~======}
$Ours_{con}$ & 4096 & 0.6 & 16 & \textbf{88.81} & \textbf{89.07} & \textbf{89.07} \\
$baseline_{rob}$ & - & - & - & - & - & 88.47 \\
 & \multicolumn{6}{c}{Maximum Sentence Length} \\ 
\cdashline{2-7}
$Ours_{con}$+ & 128 & \multirow{6}{*}{0.6} & \multirow{6}{*}{16} & 88.88 & 89.14 & 89.14 \\
$Ours_{con}$+ & 256 &  &  & \textbf{93.30} & \textbf{93.37} & \textbf{93.36} \\
$Ours_{con}$+ & 512 &  &  & 88.78 & 89.04 & 89.04 \\
$Ours_{con}$+ & 1024 &  &  & 90.99 & 91.13 & 91.13 \\
$Ours_{con}$+ & 2048 &  &  & 91.81 & 91.93 & 91.93 \\
$Ours_{con}$ & 4096 &  &  & 88.81 & 89.07 & 89.07 \\
 & \multicolumn{6}{c}{Classification Layer Dropout} \\ 
\cdashline{2-7}
$Ours_{con}$+ & \multirow{5}{*}{4096} & 0 & \multirow{5}{*}{16} & \textbf{92.73} & \textbf{92.81} & \textbf{92.81} \\
$Ours_{con}$+ &  & 0.2 &  & 90.16 & 90.33 & 90.33 \\
$Ours_{con}$+ &  & 0.4 &  & 78.98 & 80.21 & 80.21 \\
$Ours_{con}$ &  & 0.6 &  & 88.81 & 89.07 & 89.07 \\
$Ours_{con}$+ &  & 0.9 &  & 82.60 & 83.31 & 83.31 \\
 & \multicolumn{6}{c}{Effective Batch Size} \\ 
\cdashline{2-7}
$Ours_{con}$+ & \multirow{7}{*}{4096} & \multirow{7}{*}{0.6} & 2 & \textbf{\underline{93.80}} & \textbf{\underline{93.86}} & \textbf{\underline{93.86}} \\
$Ours_{con}$+ &  &  & 4 & 70.79 & 73.52 & 73.52 \\
$Ours_{con}$+ &  &  & 8 & 76.82 & 78.43 & 78.43 \\
$Ours_{con}$ &  &  & 16 & 88.81 & 89.07 & 89.07 \\
$Ours_{con}$+ &  &  & 32 & 79.72 & 80.83 & 80.83 \\
$Ours_{con}$+ &  &  & 64 & 90.64 & 90.80 & 90.80 \\
$Ours_{con}$+ &  &  & 128 & 91.39 & 91.51 & 91.51
\end{tabular}
}
\caption{Macro-f1, Micro-f1, and Accuracy score on the test result. $Ours_{con}$ - final submitted model on the shared task, $baseline_{rob}$ - official baseline model, and $Ours_{con}+$ - modified versions of our final model with more hyperparameter tuning. The \textbf{bold} value signifies the best score within a specific section, whereas the \textbf{\underline{underlined}} value denotes the best score across all sections.}
\label{tab:results}
\end{table}


\section{Conclusion \& Future Work}
In this work, we introduce our contrastive learning-based system, which shows a comparable performance. We demonstrate that a model with half the parameters and without an ensemble of large models or hand-engineered features can show a comparable performance, which requires more exploration in this field. For future work, the use of recent prompt-based models \footnote{\url{https://chat.openai.com/}} can be used for data augmentation. Also, the effect of more advanced contrastive loss, i.e., Triplet loss \citep{chechik2010large} or InfoNCE loss \citep{oord2018representation}, need to be explored.


% \section{Limitations}

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

\appendix

% \section{Example Appendix}
% \label{sec:appendix}

% This is an appendix.

\end{document}
