% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[preprint]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}
\usepackage{booktabs}

\usepackage{subfig, graphicx} 

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
\usepackage{makecell}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{UMUTeam at SemEval-2024 Task 8: Combining Transformers and Syntax Features for Machine-Generated Text Detection}


\author{
  Ronghao Pan\(^1\),
  José Antonio García-Díaz\(^1\),
  Pedro José Vivancos-Vicente\(^2\),
  \textbf{Rafael Valencia-García}\(^1\)\\
\(^1\) Facultad de Informática, Universidad de Murcia, Campus de Espinardo, 30100 Murcia, Spain\\
\(^2\)VÓCALI Sistemas Inteligentes S.L., Parque Científico de Murcia, \\ Carretera de Madrid km 388. Complejo de Espinardo, 30100 Murcia, España\\
{\{\tt ronghao.pan, joseantonio.garcia8, valencia\}@um.es}  \\ {\tt pedro.vivancos@vocali.net} \\
}

\begin{document}
\maketitle
\begin{abstract}
In this manuscript we describe the UMUTeam's participation in Task 8 of SemEval-2024 entitled ``Multigenerator, Multidomain, and Multilingual Black-Box Machine-Generated Text Detection''. This is a shared task that aims to identify automatic systems for the detection of machine-generated text in order to mitigate its potential misuse. This shared task is divided into three subtasks: Subtask A, a binary classification task to determine whether a given full-text was written by a human or generated by a machine; Subtask B, a multi-class classification problem to determine, given a full-text, who generated it. It can be written by a human or generated by a specific language model; and Subtask C, mixed human-machine text recognition. We participated in Subtask B, using an approach based on fine-tuning a pre-trained model, such as RoBERTa, combined with syntactic features of the texts. Our system placed 23rd out of a total of 77 participants, with a score of 75.350\%, outperforming the baseline.

\end{abstract}

\section{Introduction}
In the area of natural language generation (NLG), advances such as GPT-2 \cite{radford2019language}, GPT-3 \cite{NEURIPS2020_1457c0d6} and InstructGPT \cite{ouyang2022training} have improved writing aids, providing support for various writing tasks. The widespread adoption of Large Language Models (LLMs) such as ChatGPT and GPT-4 \cite{achiam2023gpt} has led to an increase in machine-generated content across various platforms, including news, social media, education and science. While these models produce remarkably fluid responses, concerns have arisen about their potential to spread misinformation and disrupt established systems. Concerns remain about their misuse, particularly in scenarios such as academic dishonesty and scientific research, where AI-generated content may be presented as original work. The emergence of AI-generated scientific texts raises ethical and integrity concerns in academic publishing, requiring tools or models to distinguish between human-generated and AI-generated content \cite{ma2023ai}.

Efforts to detect AI-generated text have primarily involved fine-tuning pre-trained models and developing detection systems. Recent studies have presented datasets and methods specifically designed to detect AI-generated scientific documents. However, challenges remain in achieving high performance and interpretability across different domains and models \cite{ma2023ai}.

For this reason, Task 8 of SemEval, entitled ``Multigenerator, Multidomain, and Multilingual Black-Box Machine-Generated Text Detection'', aims at identifying automatic systems for the detection of machine-generated text in order to mitigate its potential misuse. To this end, the task is divided into three subtasks that address two text generation paradigms: (i) full text, where a text is considered to be entirely written by a human or generated by a machine; and (ii) mixed text, where a machine-generated text is refined by a human, or a text written by a human is paraphrased by a machine. 

The subtasks are: 

\begin{itemize}
    \item \textbf{Subtask A: Binary Human-Written vs. Machine-Generated Text Classification}. Determine whether a given full-text was authored by a human or generated by a machine. It offers two tracks: monolingual (English source only) and multilingual. 
    \item \textbf{Subtask B: Multi-Way Machine-Generated Text Classification}. Given a full text, determine who generated it. It can be human-written or generated by a specific language model.
    \item \textbf{Subtask C. Human-Machine Mixed Text Detection}. Given a mixed text containing both human-generated and machine-generated segments, identify the boundary where the transition from human-generated to machine-generated content occurs.
\end{itemize}

In this shared task, the UMUTeam contributed to \textbf{Subtask B} with a approach based on fine-tuning a pre-trained model such as RoBERTa combined with syntax features of the text. syntax features of the text refer to the writing style, such as token-level features (e.g. word length, part of speech, function word frequency and stop word ratio) and sentence-level features (e.g. sentence length).

During our experiments, we found that the syntactic features of texts can complement and improve the performance of pre-trained Transformer-based models and RoBERTa is more suitable for this type of task.

The rest of this paper is organized as follows. Section \ref{sec:background} provides a summary of important details about the task setup.  Section \ref{sec:system-overview} provides an overview of our system for two subtasks. Section \ref{sec:experimental-setup} presents the specific details of our systems. Section \ref{sec:results} discusses the results of the experiments, and finally, the conclusions are presented in section \ref{sec:conclusion}. 


\section{Background}
\label{sec:background}
Recent advances in Artificial Intelligence (AI) technology, particularly in the field of Natural Language Processing (NLP), have led to the emergence of many models capable of generating natural language using LLM. These can produce remarkably fluent responses, and this has led to an increase in machine-generated content across multiple domains and platforms, including news, social media, education, and science.

Large Language Models (LLMs) face several technical and social challenges as they advance in NLP tasks. Recent research has shown that pre-trained LLMs can not only learn linguistic knowledge, but also reason about large amounts of acquired knowledge \cite{lewis2020retrieval}. However, LLMs have other problems, such as hallucination, producing texts that contain information or details that are not based on reality or are completely invented; and assertion of falsehoods as facts, which means that they can involuntarily produce texts that present false information as true.

The latest generative LLMs, such as GPT-3 developed by OpenAI, are capable of producing highly fluent text, but they can produce inaccurate, toxic or unhelpful content. Some researchers have explored the use of reinforcement learning from human feedback (RLHF) \cite{ouyang2022training} to adjust language models to better match user intent. ChatGPT, one of OpenAI's models based on GPT-3 and trained with RLHF, performs well in conversations with humans, demonstrating an understanding of user instructions and generating useful, reliable, honest and harmless text content. 

Therefore, a growing number of studies have been conducted to analyze, recognize and identify text generated by AI, especially text generated by GPT. Current research focuses on two main areas: human behavior for recognizing text generated by AI and recognition models for identifying text generated by LLMs. For example, in \cite{guo2023close}, an approach was proposed to determine whether a text (in English and Chinese) was generated by ChatGPT or written by a human across different domains, while in \cite{shijaku2023chatgpt}, a model was developed to identify whether TOEFL essays were written by humans or generated by ChatGPT on a small dataset (126 essays for each).

There are other studies that focus on detecting fake information or fake news generated by LLMs. For example, in \cite{zellers2019defending}, the Grover model was proposed to generate and detect examples of fake news. After the release of GPT-2, OpenAI proposed the GPT-2 generated text detector, which achieved a high F1 score. This detector was fine-tuned based on RoBERTa in a binary text classification format.

For this shared task, we used a fine-tuning approach of transformer-based models such as RoBERTa to create a detector for text generated by different LLMs. Unlike other existing studies on LLM-generated text detection, we have concatenated syntactic features during the fine-tuning process to improve its performance. The models evaluated for Subtask B are \textbf{RoBERTa} \cite{DBLP:journals/corr/abs-1907-11692}, a model based on Transformers, which was pre-trained on a large corpus of English data with Masked Language Model (MLM) goal. This means that it was pre-trained only on the raw texts, without humans labeling them in any way, with an automatic process to generate inputs and labels from these texts. For this task we evaluated the \textit{base} version.

\section{System overview}
\label{sec:system-overview}
Figure \ref{fig:arquitectura} shows the architecture of our system. First, we extracted the syntax features of the texts using the syntax feature extractor and encoded the texts into embeddings using the tokenizers of the pre-trained models. Second, once the embeddings and syntax features were obtained, we normalized the syntax feature values and concatenated them with the text embeddings. Third, the fine-tuning process is performed and a sequence classification layer is added on top of the pre-trained model. This layer takes the sequence representation generated by the pre-trained model and performs a classification based on the labels of the specific classification task. Finally, a performance evaluation is performed using the validation set.

\subsection{Syntax feature extractor}
\label{sec:syntax-extractor}
Sintaxis linguistic features are those aspects related to the grammatical structure and organization of words in a sentence or paragraph \cite{garcia2022umutextstats}. This can include elements such as sentence length, the frequency of certain parts of speech, the presence of function words, the number of stop words, etc. All of these features reflect the writing style that distinguishes different texts. 

The features used in this task are:

\begin{itemize}
    \item \textbf{Average word length}. This is the average number of characters the words in the text have. It is calculated by adding the length of all words in the text and dividing that sum by the total number of words in the text.
    
    \item \textbf{POS tag frequency}. This is the frequency of Part of Speech (POS) grammatical tags in the text. Grammatical tags represent the grammatical categories of words in a text, such as nouns, verbs, adjectives, and so on. The frequency of POS tags indicates how often different grammatical categories occur in the text and can provide information about the structure and style of the text.

    \item \textbf{Average sentence length chars}. This is the average length of the sentences in the text, measured in characters. It is a measure of the complexity and readability of the text.
    
    \item \textbf{Average sentence length words}.  This is the average length of sentences in a text, measured in words. This metric shows the average number of words per sentence in the text.
    
    
    \item \textbf{Percentage of stopwords}. This is the percentage of stopwords in a text, relative to the total number of words in the text. Stopwords are common words that are often filtered out or eliminated during natural language processing because they occur so frequently and have little contextual meaning.
    
    \item \textbf{Punctuation Frequency}. Refers to the number of times different punctuation marks, such as commas, periods, semicolons, etc., occur in a given text.

    \item \textbf{Special character Frequency}. Refers to the number of times characters other than letters and numbers appear in a given text. These special characters can include punctuation marks, mathematical symbols, control characters, emoticons, and other non-alphabetic symbols.
\end{itemize}

To the syntax features, we used an open source tool called \textit{authorstyle}\footnote{\url{https://github.com/mullerpeter/authorstyle}}, a package that allows to treat PAN corpora and extract stylometric features from text documents.

The embeddings of the texts refer to the numerical representation of the words or tokens in a high-dimensional vector space obtained by the tokenizers of the models. For this task, we normalized the syntax feature values and concatenated them with the embeddings obtained by the tokenizers to perform RoBERTa fine-tuning to identify the author. It can be written by humans or generated by a specific language model.

\begin{figure*}[htb]
\centering
\includegraphics[width=1\textwidth]{arquitectura.png}
\caption{System architecture}
\label{fig:arquitectura}
\end{figure*}

\section{Experimental setup}
\label{sec:experimental-setup}
For Subtask B, we used the dataset provided by the organizers, which consists of two subsets: training and validation. Figure \ref{fig:distribution} shows the distribution of the training and validation sets. We can see that both the training and validation sets are balanced and that there are a total of 5 types of texts generated by different LLMs or by humans. The types are: davinci, bloomz, human, chatGPT, dolly and cohere. 

We used the following fine-tuning hyperparameters: a batch size of 16 for both training and validation, 10 epochs, a learning rate of 2e-5, and a weight decay of 0.01. 

During training, we used Macro-F1 as a reference. Macro-F1 is a measure used to evaluate the performance of a model in a multi-class classification problem. It calculates the average F1 score for each class individually, and then averages these scores to obtain an overall score. The macro F1 Score assigns equal weight to each class, regardless of its size or distribution in the data set. This means that all classes are equally important in the final scoring metric.


\begin{figure}[htb]
\centering
\includegraphics[width=0.5\textwidth]{chart.png}
\caption{Training and validation set distribution of Subtask B.}
\label{fig:distribution}
\end{figure}



\section{Results}
\label{sec:results}
In the Table \ref{tab:official-results-subtaskb} we can see the official ranking of Subtask B. With a total of 77 participants, we have achieved the twelfth-third best result, with an accuracy of 75.350, which is 0.744\% higher than the baseline and 15.5\% higher than the first.

\begin{table}[htb!]
\centering
\caption{Official results for the Subtask B.}\label{tab:official-results-subtaskb}
\begin{tabular}{lrrr}
\toprule
\textbf{Team} &  \textbf{Rank} &  \textbf{Accuracy}\ \\
\midrule
joeblack        &     1 & 90.850 \\
tmarchitan      &     2 & 86.955  \\
farawayxxc      &     3 & 84.328 \\
halwhat         &     4 & 83.955  \\
dianchi         &     5 & 83.478 \\
\multicolumn{3}{c}{\ldots} \\
UMUTeam         &    23 & \textbf{75.350}   \\
\multicolumn{3}{c}{\ldots} \\
Baseline        &    - & 74.606  \\
\bottomrule
\end{tabular}
\end{table}

In order to perform an error analysis and to observe the behavior of our model in predicting different classes of texts, we have extracted the confusion matrix of our model on the test set, as shown in Figure \ref{fig:cm}. We can see that our model predicts the texts generated by Bloomz, Dolly, ChatGPT and Davinci very well, with accuracies above 90\%. However, when predicting human-generated texts, our model tends to confuse them with those generated by the Dolly model by 27.73\%. It is important to note that when predicting Cohere-generated texts, our model tends to classify them as Davinci-generated texts by 70\%, which leads to a decrease in accuracy.


\begin{figure}[htb]
\centering
\includegraphics[width=0.5\textwidth]{confusion-matrix.png}
\caption{The confusion matrix of our RoBERTa-based system in the test set.}
\label{fig:cm}
\end{figure}


\section{Conclusion}
\label{sec:conclusion}
This paper describes the participation of the UMUTeam in the Common Task 8 of SemEval 2024. The aim of this task was to develop models for the identification of automatic systems for the recognition of machine-generated text in order to mitigate its potential misuse. The task consisted of three subtasks: Subtask A, a binary classification task to determine whether a given full-text was written by a human or generated by a machine; Subtask B, a multi-class classification problem to determine, given a full-text, who generated it. It can be written by a human or generated by a specific language model; and Subtask C, mixed human-machine text recognition. In this joint task, we participated in Subtask 3, using a approach based on fine-tuning a RoBERTa pre-trained model with syntactic features of texts. In terms of results, our system achieved the 23rd position with a score of 75.350\%, outperforming the baseline.


\section*{Acknowledgements}
This work is part of the research projects LaTe4PoliticES (PID2022-138099OB-I00) funded by MCIN/AEI/10.13039/501100011033 and the European Regional Development Fund (FEDER)-a way to make Europe and LT-SWM (TED2021-131167B-I00) funded by MCIN/AEI/10.13039/ 501100011033 and by the European Union NextGenerationEU/PRTR. In addition, this work was funded by the Spanish Government, the Spanish Ministry of Economy and Digital Transformation through the Digital Transformation through the "Recovery, Transformation and Resilience Plan" and also funded by the European Union NextGenerationEU/PRTR through the research project 2021/C005/0015007.

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

\appendix


\end{document}
