% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{booktabs}
\usepackage{graphicx}
\usepackage{amssymb}
\usepackage{tabularx}
\usepackage{subfigure}
\usepackage{placeins}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{NewbieML at SemEval-2024 Task 8: Ensemble Approach for Multidomain Machine-Generated Text Detection}
% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}
\author{Bao Tran \and Nhi Tran \\
  Ho Chi Minh City University of Technology, VNU-HCM, Ho Chi Minh City, Vietnam \\
  \texttt{\{bao.tran2003, nhi.tranluongyen\}@hcmut.edu.vn} \\}

\begin{document}
\maketitle
\begin{abstract}
Large Language Models (LLMs) are becoming popular and easily accessible, leading to a large growth of machine-generated content over various channels. Along with this popularity, the potential misuse is also a challenge for us. In this paper, we use subtask A monolingual dataset with comparative study between some machine learning model with feature extraction and develop an ensemble method for our system. Our system achieved 84.31\% accuracy score in the test set, ranked 36th of 137 participants.
Our code is available at: \url{https://github.com/baoivy/SemEval-Task8}
\end{abstract}

\section{Introduction}
In our current time, large language models (LLMs), such as ChatGPT \citep{ouyang2022training}, GPT-4\footnote{\url{https://openai.com/}},  LLaMA \citep{touvron2023llama} and BLOOMz \citep{muennighoff2023crosslingual} can be easily observed to be becoming increasingly prevalent from diverse forms ranging of news, multimedia to education. How outstandingly LLMs answer to user's problems makes them appealing for automatic missions as well as diminishing human labor in many scenarios. Nevertheless, this also unexpectedly leads to problems with regard to human's misuses, spreading misinformation and causing disruptions in the education system in particular. Therefore, it is necessary to develop systems that can automatically distinguish AI contexts from human-written ones.

Recently, with the exponential growth of LLMs, many researchers have attempted to distinguish human-written texts from machine- generated ones. \citealp{uchendu-etal-2021-turingbench-benchmark}, \citealp{wang2024m4}, \citealp{he2024mgtbench}, \citealp{liu2023argugpt} have shown us about machine-generated and human writing data from various source. \citealp{mitchell2023detectgpt}, \citealp{bao2023fastdetectgpt}, \citealp{deng2023efficient} used zero-shot classification method to calculate probabilities from perturbated input text. \citealp{bhattacharjee2023fighting} leveraged prompt-base method to utilize LLMs as detector. \citealp{gehrmann2019gltr} used statistical method to detect machine-generated paragraph with language model to compute conditional probability. For fine-tuning language model method, \citealp{Fagni_2021} had a comparative study among pre-trained language model, feature-base and character level classification on DeepFake dataset and showed that pre-trained language model has a best result than others. \citealp{liu2023check} used feature-base classification with RoBERTa as embedding and LSTM + Self-Attention in classification head. \citealp{bhattacharjee-etal-2023-conda} used unsupervised and self-supervised learning by leveraging domain adaptation on unlabeled dataset and contrastive learning belonging with pre-trained language model to learn domain representations. \citealp{pmlr-v202-kirchenbauer23a}, \citealp{liu2023unforgeable} used a novel approach with watermark embedding to detect LLMs text that employed by LLMs or neural network.

Being inspired by feature-based classification technique, we propose to have a comparative study for simple and lightweight machine learning method beside the trend of LLM. Our system compares various machine learning models among with ensemble method for multiple machine learning method to find the best combination for our system. 

The rest of the paper is as follows.
The section \ref{sec:dataset} generalizes task description and dataset for our experiment.
The section \ref{sec:sys_des} shows the description of our system.
The experimental setup and results are presented in section \ref{sec:res}.
Finally, section \ref{sec:conclu} is the conclusion and discussion about our work.

\section{Task description \& dataset} \label{sec:dataset}
\subsection{Task description}
In SemEval 2024- Task 8 \citep{semeval2024task8}, the topic for subtask A is \textbf{\textit{Binary Human-Written vs. Machine-Generated Text Classification}}. The  full text is to determine whether an essay is human-written or machine-generated. There are two tracks for subtask A: monolingual (only English sources) and multilingual. On this subtask, we only focus on monolingual dataset.


\subsection{Dataset overview}
The SubTask-A monolingual dataset originated from various sources of content, including Wikipedia, WikiHow, Reddit, arVix, PeerRead, and OutFox \citep{koike2023outfox}. According to the author, this is an extended version of the M4 dataset \citep{wang2024m4}. All paragraphs in the dataset of subtask A monolingual are written in English. This dataset contains a total of 159,029 essays, which were split into a three-part train set, development set (dev set), and test set. The monolingual dataset contains two types of labels, 0 represented by human writing and 1 represented by machine-generated. In particular, the train set was constructed from 5 different generator (Human, ChatGPT, Dolly-v2, CoHere and Davinci003) and the development set was constructed only from Bloomz. For the test set, GPT-4 had been added along with the remaining generator to generate essays. The overview statistical and distribution of labels will be detailed in Table \ref{tab:label-stat} and an example of the dataset is represented in Table \ref{tab:sample}. Additionally, the distribution between human labels and machine-generated labels on the train set is almost equal so the class imbalance technique is not used for this task. 

Moreover, a pre-processing step was applied to the dataset by the following deletions and changes:
\begin{itemize}
    \item  Removing punctuation in sentence
    \item  Lower casing text
    \item  Removing any leading and trailing whitespace
    \item  Remove URLs 
\end{itemize}
\begin{table}[h!]
\centering
\begin{tabular}{@{}lcccc@{}}
\toprule
\textbf{Dataset}     & \textbf{\textit{\#Number}} & \multicolumn{2}{c}{\textbf{Label Distribution (\%)}} \\ \cmidrule(r){3-4}
            &                       & Human            & Machine               \\ 
\midrule
Train set          & 119,757                   & 52.9             & 47.1                \\
Dev set   & 5,000               & 50.0        & 50.0          \\
Test set     & 34,272               & 47.5           & 52.5                \\
\bottomrule
\end{tabular}%
\caption{Dataset statistical}
\label{tab:label-stat}
\end{table}

\begin{table*}
\centering
\begin{tabular}{@{}lll@{}}
\toprule
ID       & Essays                                                                 & Label       \\ \midrule
1 & ...Step 10. Pause The Game. To pause the game, just press the "start" button...& Machine    \\
56406 & ...If you haven't used it in the last six months there is little  & Human      \\
& chance you'll use it in the next six months. Toss it. & \\
\bottomrule
\end{tabular}%
\caption{Example dataset}
\label{tab:sample}
\end{table*}

\section{System Description} \label{sec:sys_des}
%Motivate for system (như là dạo này llm trend thì mình thử xem các mo hình ML có thể handle dc hay ko 
We will describe our developed model in this section. On Section \ref{sec:encoder}, we will discuss how we embedded sentences in essays using a pre-trained language model. Then, for the crucial section, we would like to present the detail of our model on \ref{sec:model}. We perform our architecture based on Figure \ref{fig:model}. First step, the essays need to be embedded through pre-trained language model. Next, we use ensemble method with base model and then give a final prediction by using meta-model. 
\begin{figure*}[h!]
    \centering
    \includegraphics[width=\textwidth]{Image/SemEvalTask8_pipeline.pdf}
    \caption{Overview of our system architecture. We demonstrate the best combination model from our experiment}
    \label{fig:model}
\end{figure*}
\subsection{Embedding}\label{sec:encoder}
%Gioi thieu embedd
%Giai thich tai sao dung longformer
%Gioi thieu longformer 
% Trinh bay so luoc pp embedded
For the embedding stage, each token needs to be represented in a vector. Some essays have more than 512 tokens, which will lead to exceed at original BERT \citep{devlin-etal-2019-bert}, we determine to utilize Longformer \citep{beltagy2020longformer} model to capture semantic embedding of each word within essays of dataset. Given the essay X, the vector embedding of each token will be calculated in the essay. The input will be formed as ($w_i$ is a word in essay):
\begin{center}
\tt <s> $w_{1}$ $w_{2}$ ... $ w_{i}$  </s>
\label{eq:input1}
\end{center}

This produces an embedding matrix $\mathbf{E} \in \mathbb{R}^{N \times K}$ ($K$ is a hidden size of word, $N$ is a number of token) by taking the last hidden layer. After that, mean pooling is applied to each vector embedding of the matrix to flatten into a standard vector to aggregate feature of token. The dimension of the vector for each essay will be  $\textbf{X} \in \mathbb{R}^{N}$ where $N$ is a number of tokens in essay and $\mathbf{X}$ is a feature vector of essay X.

\subsection{Ensemble model}\label{sec:model}
After text embedding, we develop our classification stage for the system. We will discuss each base model in section \ref{sec:basemodel} and how we ensemble various base models in section  \ref{model:ensemble}
\subsubsection{Base model} \label{sec:basemodel}
We utilized Support Vector Machine (SVM), XGBoost, Logistic Regression, and K-nearest neighbors (KNN) as the base model for our ensemble method.  

\textbf{Support Vector Machine (SVM)} \citep{708428} is a supervised learning model that is used for classification and regression tasks. SVM maximizes the hyperplane or set of hyperplanes to find the best boundary that separates different classes in a dataset.

Due to the non-linearity of the dataset, we decided to use the Radial basis function (rbf) kernel for SVM, which is defined as Formula \ref{eq:1}. Moreover, to find the best parameter for the SVM model, we listed all the hyperparameter values of $C$ and $\lambda$ used in the grid search as Table \ref{tab:svm}. 
\begin{equation}
    K = Ce^{-\gamma||x - z||^{2}}
    \label{eq:1}
\end{equation}
\begin{table}[!tb]
\centering
% \begin{minipage}{.85\linewidth}
\centering
% \begin{tabular}{lccc}
\begin{tabularx}{0.7\linewidth}{lXX}
\toprule
Hyperparameter\hspace{0.3cm}       & Value          \\
\midrule
C      & $1.0, 10.0$       \\
$\lambda$           & 0.0001, 0.001, 0.1, 0.2, 0.5, 1, 10   \\
% Optimizer       & AdamW                \\
\bottomrule
\end{tabularx}
% \end{tabular}
% \end{minipage}\hfill
\caption{SVM configuration} 
\label{tab:svm}
\end{table}

\textbf{XGBoost} \citep{Chen_2016} is a scalable end-to-end tree boosting technique which allows to correct the error of the previous tree by creating multiple trees sequentially. The classifier also assigned a weight value to each independent variable and used some techniques to prevent overfit like tree pruning, sparsity awareness, etc.

Same as SVM, we construct candidate hyperparameter values of max depth of the tree and $\lambda$ used in the grid search as Table \ref{tab:xgboost}. 

\begin{table}[!tb]
\centering
% \begin{minipage}{.85\linewidth}
\centering
% \begin{tabular}{lccc}
\begin{tabularx}{0.7\linewidth}{lXX}
\toprule
Hyperparameter\hspace{0.3cm}       & Values        \\
\midrule
Learning rate & $0.1,0.2$        \\
Estimators      & $60,80,100$       \\
Max depth           & $2,4,6$       \\
% Optimizer       & AdamW                \\
\bottomrule
\end{tabularx}
% \end{tabular}
% \end{minipage}\hfill
\caption{XGBoost configuration} \label{table:param1}
\label{tab:xgboost}
\end{table}

\textbf{Logistic Regression (LR)} is a simple technique for binary classification. Given feature variables, the output is a probability from $[0;1]$. This can be achieved by applying of a sigmoid function to the linear combination of the independent variables. In our system, we only use the default configuration.

\textbf{K-nearest neighbour (KNN)} is a simple technique for classification which uses majority vote on k closest data point to target point. Grid search is also utilized to find the best value of k, where $k \in \{3,5,7,9\}$.
\subsubsection {Stack ensemble} \label{model:ensemble}
Ensemble different machine learning models is the way to improve prediction accuracy to leverage the strengths and mitigate the weaknesses of the individual base models. In our system, we choose stack ensemble as an ensemble method for our system. For this technique, applicable in scenarios with N base model $(M_1, M_2, ... , M_n)$ and meta-model $M$, determine meta feature for meta model by predicting each base model $\mathbf{\hat{X}} = (M_1(X), ..., M_n(X))$. Then predict the final output by calculating meta model $y = M(\mathbf{\hat{X}})$. Many different base models and meta model have been evaluated and compared, including Naive Bayes, k-nearest neighbors, SVM, XGBoost, and Logistic Regression as section 
\ref{sec:basemodel}

\subsection{Experiment setup}
We describe our system setup procedure. As GPU, we use a single RTX 4090 24GB to train and infer our system for both stages. In the embedding stage, we use Hugging Face\footnote{\url{https://huggingface.co/}} library for the Longformer model. For maximum token length in the pre-trained language model, because some essays are longer than 512 tokens, we set it to a maximum of 1024 tokens with padding and truncating. We infer each essay without any training on it. 

For the classification stage, with SVM, Logistic Regression, and KNN model, due to the large dimension of the dataset, we proposed to use cuML\footnote{\url{https://github.com/rapidsai/cuml}} library, which supports GPU-accelerated for machine learning algorithms. For XGBoost we use xgboost \footnote{\url{https://github.com/dmlc/xgboost}} library and sklearn\footnote{\url{https://scikit-learn.org/}} for stack ensemble. Hyperparameter tuning is used in each machine learning model to find the best parameter for each model (all candidate parameters are defined in section \ref{sec:basemodel}). A 5-fold cross-validation is used to find the optimal configuration for the ensemble.

\section{Results \& Discussion} \label{sec:res}

\subsection{Evaluation metric}
For subtask A monolingual task, the metrics used to evaluate our result for the dataset are Marco-F1, Mirco-F1, and Accuracy. The main metric for ranking submission is Accuracy. In more detail, the accuracy metric is given by the ratio of the total number of correct predictions to the total predictions done by the model, regardless of true or false predictions. Micro F1-score is the harmonic mean of precision and recall and macro-F1 score is defined as the average of Mirco F1-score across different classes. 

\subsection{Results}
%Viet tiep so sanh indiviual
%XGBoost + SVM -> KNN : Test accuracy : macro-F1=0.68727	micro-F1=0.70580	accuracy=0.70580
%XGBoost + SVM -> LR : Test accuracy : macro-F1=0.69249	micro-F1=0.71080	accuracy=0.71080
%XGBoost + SVM + KNN -> LR : Test accuracy : macro-F1=0.68402	micro-F1=0.70320	accuracy=0.70320
%SVM + LR + KNN -> XGBoost : Test accuracy : macro-F1=0.68319	micro-F1=0.70280	accuracy=0.70280
%SVM : Test accuracy : macro-F1=0.67584	micro-F1=0.69860	accuracy=0.69860
%KNN : Test accuracy : macro-F1=0.50565	micro-F1=0.54920	accuracy=0.54920
%LR: Test accuracy : macro-F1=0.65489	micro-F1=0.67740	accuracy=0.67740
%XGBoost: Test accuracy : macro-F1=0.62057	micro-F1=0.64860	accuracy=0.64860

\begin{table*}[h]
\centering
\begin{tabular}{@{}lcccccc@{}}
\toprule
\textbf{Method}       & \multicolumn{3}{c}{\textbf{Development phase}} & \multicolumn{3}{c}{\textbf{Test phase}}\\
              & Accuracy & Micro F1 & Macro F1 & Accuracy & Micro F1 & Macro F1\\ 
\midrule
\textbf{SVM} & \textbf{0.6986}  & \textbf{0.6986}  & \textbf{0.6758} & \textbf{\textit{0.8399}} & \textbf{\textit{0.8399}} & \textbf{\textit{0.8360}} \\
XGBoost          & 0.6486  & 0.6486  & 0.6206 & \textit{0.8319} & \textit{0.8319} & \textit{0.8293}\\
KNN      & 0.5492  & 0.5392  & 0.5057 & \textit{0.8244} &  \textit{0.8244} & \textit{0.8228}\\
LR       & 0.6774   & 0.6774  & 0.6549 & \textit{0.8155} & \textit{0.8155} &  \textit{0.8114} \\
\midrule
Model 1 & \textbf{0.7108}  & \textbf{0.7108}  & \textbf{0.6925} & \textit{0.8329} & \textit{0.8329} & \textit{0.8279}\\ %XGBoost + SVM -> LR
Model 2     &     0.7032	& 0.7032	& 0.6840  & 0.8439 & 0.8439 & 0.8401\\ %XGBoost + SVM + KNN -> LR
\textbf{Model 3}      & 0.7028  & 0.7028  & 0.6832 & \textbf{\textit{0.8458}} & \textbf{\textit{0.8458}} & \textbf{\textit{0.8422}}  \\ %SVM + LR + KNN-> XGBoost
\bottomrule
\end{tabular}
\caption{Result on different method on dev set and test set. Test result with italicized have been run after test phase deadline. Denoted that Model 1 is XGBoost + SVM as base model, LR as meta model, Model 2 is XGBoost + SVM + KNN as base model, LR as meta model, Model 3 is LR + SVM + KNN as base model, XGBoost as meta model.}
\label{tab:dev_res}
\end{table*}

In this section, we present the result of our model, focusing on its accuracy in the dev set and test set. Table \ref{tab:dev_res} shows the performance of our models with some combination with the base model when using the ensemble method mentioned in section \ref{sec:model}, compared to the dev set. All results were running on the best hyperparameter value of each base model. We first compare the efficiency of each individual model. SVM gives the best performance among all (0.6986). Surprisingly, LR have better performance than XGBoost in monolingual task. 

From the result of each model,  we also compare our main model with some combinations of the base model when using the stack ensemble method which is represented in Table \ref{tab:dev_res}. SVM is used as base models for all ensemble experiments since they give better performance than others. The results do not significantly differ from the three models in our experiment. For model 3, the result is slightly better than model 1 and model 2 which achieved 0.7101 accuracy score.

%SVM + LR + KNN -> XGBoost : Test accuracy : macro-F1=0.84220	micro-F1=0.84582	accuracy=0.84582
%XGBoost + SVM + LR -> KNN : Test accuracy : macro-F1=0.84004	micro-F1=0.84387	accuracy=0.84387
%XGBoost + SVM -> LR : Test accuracy : macro-F1=0.82794	micro-F1=0.83287	accuracy=0.83287
%XGBoost + SVM -> KNN : Test accuracy : macro-F1=0.82188	micro-F1=0.82691	accuracy=0.82691
%XGBoost + SVM + KNN -> LR : Test accuracy : macro-F1=0.84005	micro-F1=0.84387	accuracy=0.84387
%%SVM : Test accuracy : macro-F1=0.83600	micro-F1=0.83993	accuracy=0.83993
%KNN: Test accuracy : macro-F1=0.82280	micro-F1=0.82435	accuracy=0.82435
%LR: Test accuracy : macro-F1=0.81141	micro-F1=0.81548	accuracy=0.81548
%XGBoost: Test accuracy : macro-F1=0.82928	micro-F1=0.83193	accuracy=0.83193
For the test set, we can notice that all results from 7 methods are not significant differences. The result of SVM (0.8399) still outperformed on individual tests. However, XGBoost has surpassed the performance of KNN and LR on the test set (0.8319 compared to 0.8244 and 0.8155). The LR has the worst performance among 4 models. Surprisingly, after evaluating the test set on the ensemble method, model 1 inferior when compared to the rest. In contrast, model 3 has the best result at the test set with an accuracy of 0.8458. 
We also visualize our performance of model by representing the confusion matrix in Figure \ref{fig:foobar}

Table \ref{tab:test_res} shows the result at the stage. We evaluate our result on model 2. Our system achieves 0.8438 which is ranked 36th out of 137. Unfortunately, we can not surpass the result of baseline (achieves 0.8847), which is using RoBERTa model \citep{zhuang-etal-2021-robustly} for classification. Besides, this is a prospective result that can achieve to acceptable score when comparing the traditional machine learning method with the pre-train language model and LLMs. Moreover, we can have an insight into training on traditional machine learning methods and language models nowadays. We believe that if we have a better strategy on hyperparameter tuning, the result could be higher than our official submission.

%Test accuracy : macro-F1=0.84004	micro-F1=0.84387	accuracy=0.84387
\begin{table}[ht]
\centering
\begin{tabular}{@{}lc@{}}
\toprule
\textbf{Team}       & \multicolumn{1}{c}{\textbf{Subtask A - Monolingual}} \\
              & \textbf{Accuracy}  \\ 
\midrule
Baseline & 0.8847   \\
\midrule
\textbf{\#1 Team} & \textbf{0.9688}   \\
Ours (36th)    & 0.8438   \\
\bottomrule
\end{tabular}
\caption{Result and ranking on test set}
\label{tab:test_res}
\end{table}

\begin{figure*}
    \centering
    \subfigure[]{\includegraphics[width=0.49\textwidth]{Image/confusion_matrix_dev.pdf}} 
    \subfigure[]{\includegraphics[width=0.49\textwidth]{Image/confusion_matrix_test.pdf}} 
    \caption{Confusion matrices for (a) the development set and (b) the test set on official submission}
    \label{fig:foobar}
\end{figure*}

\section{Conclusion} \label{sec:conclu}
\label{sec:bibtex}
In subtask A monolingual of SemEval task 8, we have represented our system for machine-generated detection. We proposed to develop our system based on ensemble of multiple traditional machine learning method with hyperparameter tuning. We found that XGBoost, SVM and KNN as base model and Logistic Regression in meta model would give the highest result. Our official system was ranked the 36th to 137 in test set of subtask A monolingual with 0.8439 accuracy score and 0.7032 accuracy score in dev set. From our result, traditional machine learning methods still have been proven effective in classification, with some training strategy, compared to other methods such as LLMs.

\section*{Acknowledgements}
We acknowledge Ho Chi Minh City University of Technology (HCMUT), VNU-HCM for supporting this research.

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}


\end{document}
