% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.


\documentclass[11pt]{article}


% Remove the "review" option to generate the final version.
\usepackage[]{ACL2023}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out.
% However, it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

%DARF ICH DAS?
\usepackage{graphicx}

\usepackage{hyperref}

%Skip star in \thanks symbols.
\makeatletter
\def\@fnsymbol#1{\ensuremath{\ifcase#1\or \dagger\or \ddagger\or
\mathsection\or \mathparagraph\or \|\or **\or \dagger\dagger
\or \ddagger\ddagger \else\@ctrerr\fi}}
\makeatother


% If the title and author information does not fit in the area allocated, uncomment the following
%
\setlength\titlebox{6cm}
%
% and set <dim> to something 5cm or larger.

\title{Team art-nat-HHU at SemEval-2024 Task 8: Stylistically Informed Fusion Model for MGT-Detection}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
% \author{Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n}} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a seperate ``row'' of authors use \AND, as in
\author{Vittorio Ciccarelli\Thanks{Equal contribution.} \\ Heinrich-Heine-Universität \\ Düsseldorf (HHU) \\ vittorio.ciccarelli@hhu.de \And
        Cornelia Genz\footnotemark[1]\\ Heinrich-Heine-Universität \\ Düsseldorf (HHU) \\ cornelia.genz@hhu.de \And
        Nele Mastracchio\footnotemark[1] \\ Heinrich-Heine-Universität \\ Düsseldorf (HHU) \\ nele.mastracchio@hhu.de  
        \AND
        Wiebke Petersen\footnotemark[1] \\ Heinrich-Heine-Universität \\ Düsseldorf (HHU) \\ wiebke.petersen@hhu.de\And
        Anna Sophia Stein\footnotemark[1] \\ Heinrich-Heine-Universität \\ Düsseldorf (HHU)  \\ anna.stein@hhu.de \And
        Hanxin Xia\footnotemark[1] \\ Heinrich-Heine-Universität \\ Düsseldorf (HHU)  \\ hanxin.xia@hhu.de}

%> \paragraph=macro:
%->\@startsection {paragraph}{4}{\z@ }{1.5ex plus 0.5ex minus .2ex}{-1em}{\normalsize \bf }.
%l.79 \show\paragraph
                    
\makeatletter
\renewcommand{\paragraph}{%
  \@startsection{paragraph}{4}%
  {\z@}{1ex plus 0.5ex minus .2ex}%
  {-1em}%
  {\normalsize \bf }%
}
\makeatother

\begin{document}
\maketitle
\begin{abstract}
This paper presents our solution for subtask A of shared task 8 of SemEval 2024 for classifying human- and machine-written texts in English across multiple domains. We propose a fusion model consisting of a RoBERTa-based pre-classifier and two MLPs that have been trained to correct the pre-classifier using linguistic features. Our model achieved an accuracy of 85\%.
\end{abstract}

\section{Introduction}
 % which task, semeval, why is it an important task (plagiarism detection, automatic created fakenews, ...)
% What is the task about and why is it important? Be sure to mention the language(s) covered and cite the task overview paper. ~1 paragraph
After rapid developments in large language models (LLMs) and generative AIs in the last years, the detection of machine-generated content has become one focus of study as deepfakes, machine-generated lawyer statements and even libel suits \cite{libel_suit} concerning language machines stress the importance of detecting texts not written by humans. The SemEval shared task 8 in 2024 aims at multi- and monolingual machine-generated text (MGT) detection from various domains by multiple models.

% -  fake news generation (Zellers et al., 2019; Brown et al., 2020; Uchendu et al., 2020)
% - spam/phishing . (Weiss, 2019)
% and the monolingual detection for english was chosen here.
% What is the main strategy your system uses? ~1 paragraph fusion something, improve on roberta predictions using specialized NN classifiers \& features (siehe system overview)
For the monolingual English data in subtask A \citep{semeval2024task8} we propose a fusion model built using pre-trained RoBERTa word embeddings specialized for AI-generated text detection and correction MLP classifiers, supported by the additional computation of linguistic, stylistic and probabilistic features selected based on their informational value.
% What did you discover by participating in this task? Key quantitative and qualitative results, such as how you ranked relative to other teams and what your system strKey algorithms and modeling decisions in your system; resources used beyond the provided training data; challenging aspects of the task and how your system addresses them. 
% struggles ~1 paragraph
With this system design, our model ranked at position 25 out of 124 with an 0.855 accuracy score on the task.
The only data used for training was the one provided by the organizers without further data augmentation. Because of the different distributions of the data in the development and test data sets several strategies were tested and a fusion model was chosen as the best strategy.
The fine-tuned RoBERTa Base OpenAI Detector alone performed well but developed a bias towards the machine class. To stabilize the model linguistic, probabilistic and stylistic features were added, which improved the overall F1 score of the fusion architecture.
% *Roberta performed well but not well enough
% * model overfitted for human data
% * adding the features stabilized the F1 score - fusion architecture good choice
% * wir haben keine neue Data aus den bestehenden erstellt, nur Roberta war schon pretrained
% * general problems: overfitting, different models/model distribution in the dev /test set

\section{Background}
%1 = Machine, 0 = human
% counts & percentages von df['columnname'].value_count()
% Over the course of the last years, numerous approaches have been proposed to tackle the task of MGT detection. There are models specialized at detecting text from certain generative AI models, such as DetectGPT by \citet{mitchell2023detectgpt} for filtering texts generated by LLMs in the GPT family, which performs very well on fake news detection. 
% Other detectors have been developed to classify texts from specific domains, such as \citet{shijaku2023tofel}'s approach for detecting machine-written TOFEL essays. 
 

Over the last years, numerous approaches have been proposed to tackle the task of MGT detection. Some models, such as DetectMGT \citep{mitchell2023detectgpt}, focus on detecting texts from a specific source, such as GPT-family LLMs, while other approaches are specialized in texts from a specific genre, such as \citet{shijaku2023tofel} for TOEFL essays. Other architectures, like ensemble models combining different classifiers \citep{classifictiondelCampo} have been successfully used for machine-generated text detection to improve out-of-distribution performance \citep{lai2024adaptive}.

\citet{guo2023zhencgpt} show that, overall, deep-learning approaches, and in particular a RoBERTa-based-detector, are one of the best individual models for MGT detection. The RoBERTa-based-detector was shown to be particularly robust against oov scenarios in both Chinese and English, compared to a machine learning model.  Moreover, \citet{wang2023m4} and \citet{he2024mgtbench} conducted large-scale bench-marking on existing approaches for MGT detection across multiple domains, models, and languages and concluded that the RoBERTa language model, especially the variants that have been optimized for AI detection tasks, consistently outperforms most other methods across evaluation metrics.



%Ensemble models for adding additional info correction filters
%https://link.springer.com/chapter/10.1007/978-3-540-74825-0_10
%https://arxiv.org/pdf/2403.13335.pdf.

%\citep{lai2024adaptive}, \citep{classifictiondelCampo}

% \begin{table*}[]
% \centering
% \begin{tabular}{llll}
% \hline
%         & train         & dev          & test          \\ \hline
% machine & 56,406 (53\%) & 2,500 (50\%) & 180,00 (53\%) \\
% human   & 63,351 (47\%) & 2,500 (50\%) & 16,272 (47\%) \\ \hline
% total   & 119,757       & 5,000        & 34,272        \\ \hline
% \end{tabular}
% \caption{\label{data-stats} Label distribution across train, dev and test set.}
% \end{table*}

% Data paper: https://arxiv.org/abs/2305.14902
% Distribution of datasets: 
% \begin{itemize}
%    \item test: 1= 180,00 (53\%), 0= 16,272 (47\%), total= 34,272, Model distribution: GPT4 3000, Cohere 3000, chatGPT 3000, BLOOMz 3000, dolly 3000, DaVinci 3000, Domain distribution: all outfox
%    \item dev:  1= 2,500 (50\%), 0= 2,500 (50\%), total= 5,000, Model distribution: all BLOOMz
%    \item train: 1= 56,406 (53\%), 0= 63,351 (47\%), total= 119,757, Model distribution (percentage out of model texts): DaVinci 14,343, chatGPT 14,339, dolly 14,046, Cohere 13,678, Source distribution: reddit 27500 (23\%), wikihow 27499 (23\%), arxiv 27497 (23\%), wikipedia 25530 (21\%), peerread 11731 (10\%)
% \end{itemize}

\section{System overview}

%Our system is based on the model RoBERTa Base OpenAI Detector\footnote{\url{https://huggingface.co/openai-community/roberta-base-openai-detector}} \citep{solaiman2019release}, a RoBERTa model fine-tuned for AI generated text detection. An overview of the system setup is given in Figure \ref{fig:model-architecture}.


Our system is based on a pre-classifier that is a RoBERTa model fine-tuned for AI generated text detection. In order to correct the predictions of the pre-classifier, two correction classifiers have been trained that are based on linguistic, stylistic and probabilistic features. An overview of the system setup is given in Figure \ref{fig:model-architecture}.


\begin{figure}
    \centering
    \includegraphics[width=0.75\linewidth]{plots/model_architetcure.png}
    \caption{Model architecture used to obtain submission results.}
    \label{fig:model-architecture}
\end{figure}

Our RoBERTa pre-classifier is based on the RoBERTa Base OpenAI Detector\footnote{\url{https://huggingface.co/openai-community/roberta-base-openai-detector}} \citep{solaiman2019release}, a RoBERTa model fine-tuned for AI generated text detection. This model has been further fine-tuned on 10\% of the training data. A prediction was then generated for each text in the train, dev and test set. In order to improve the predictions, two correction Multi-layer Perceptron (MLP) classifiers, one for each label, were trained on the training and development data of their respective label (see Figure \ref{fig:model-architecture}) as well as on a range of features outlined in Section \ref{sec:feature-extraction}. 
% In other words, each correction classifier was trained only on the texts in the training and development data that were labeled with its corresponding label, as well as the features of that text . 
To generate the final classification, all texts were classified again by the correction MLP that corresponded to the label predicted by the RoBERTa pre-classifier. This provided an opportunity for the more specialized classifier to correct the initial prediction.

%An earlier iteration of this architecture will be used to compare the results of the main architecture.  This alternative architecture was not involved in generating the final results that were submitted for the challenge. [outline architecture]

\section{Experimental setup} \label{sec:exp-setup}
The M4 dataset consists of both machine (label 1) and human-generated texts (label 0). The dataset features texts from six different LLM generators (Davinci, chatGPT, Dolly, Cohere, BLOOMz and GPT4) and five different genres (Reddit, WikiHow, ArXive, Wikipedia, and peerRead). Participants were provided first with a train and dev set and later with a test set with 119,757, 5,000, and 34,272 texts in total, respectively. 

 Roughly 53\% of the documents in the train set are machine generated (DaVinci: 14,343, chatGPT: 14,339, Dolly: 14,046, Cohere: 13,678), and 47\% are human-written. In the dev set, exactly half of the texts were machine-generated by the BLOOMz model, the other half was human-written. The test set contains 18,000 (53\%) machine-generated texts from Davinci, chatGPT, Dolly, Cohere, BLOOMz and GPT4 (3,000 texts each) and 16,272 (47\%) human-written texts. An overview of the data is provided in Table \ref{data-stats}. Since we did not include genre- or machine-specific information for our approach, this information is excluded from the table. 


\begin{table}[ht]
\centering
\begin{tabular}{lccc}
\hline
        & train         & dev          & test          \\ \hline
machine & 53\% & 50\% & 53\% \\
human   & 47\% & 50\% & 47\% \\ \hline
total texts   & 119,757       & 5,000        & 34,272        \\ \hline
\end{tabular}
\caption{\label{data-stats} Label distribution across train, dev and test set.}
\end{table}

 

% Since BLOOMz generated texts are not present in train set but in in the final test, we utilized both train and dev for training our detector, as we believe a model trained on data from more models would be able to capture characteristics of texts generated by different machines on a more holistic level.

\subsection{RoBERTa pre-classifier}\label{sec:pre-classifier}

We used a fine-tuned RoBERTa Base OpenAI Detector as our pre-classifier. Because the OpenAI Detector had already been fine-tuned for human-machine classification, and to facilitate replication of the experiment, we used only 10\% of the training data to further fine-tune the model\footnote{The data for fine-tuning consisted of 2,000 texts of each author category (Davinci, Cohere, Dolly, chatGPT, and human).}.Training was done for 3 epochs with a learning rate of $2e^{-5}$ on Google Colab using a T4 run-time and took 45 minutes.

\subsection{Correction classifiers}
\subsubsection{Feature extraction}\label{sec:feature-extraction}

To capture characteristics of machine-generated and human-written texts, the data was analyzed for various linguistic features. Altogether 70 features, widely used in NLP and easy to compute, were extracted, 35 of which exhibited a high to medium correlation with the gold label (see Table \ref{tab:correlations-human-machines} in the Appendix). All features were computed on a 24GB RAM machine with a Ryzen 7 7730U, which took up to 6 hours for all texts depending on the feature. 
%All numeric features were scaled with the Scikit-learn StandardScaler \citep{scikit-learn}.

%\subsubsection{Linguistic features}

\paragraph{Count-based features.} The texts were split into words and punctuation using regular expressions to derive the following features: mean sentence length, ratio of punctuation to words, ratio of word types to tokens, ratio of vowels to words and mean word length. The NLTK stopword list was used to get the ratio of content words to other words. Additionally the number of hapax legomena per text and the number of negation words (manually compiled list) per text were computed.

%n_punctuation, n_vowels,  n_words, TTR, ratio_content_words, avg_word_length, n_unique_words, n-negations


\paragraph{Syntactic features.}
All texts were POS-tagged with the NLTK part of speech tagger to compute syntactic features: ratio of nouns, verbs, adjectives, adpositions, adverbs, conjunctions, numerals, pronouns and determiners to words alltogether, ratio of adjectives to nouns and ratio of verbs to nouns.

% ratio_ADJ_NOUN, ratio_ADJ_word_count, ratio_ADP_word_count, ratio_ADV_word_count, ratio_CONJ_word_count, ratio_DET_word_count, ratio_NOUN_word_count, ratio_NUM_word_count, ratio_PRON_word_count, ratio_VERB_NOUN, ratio_VERB_word_count

Using the dependency parser of Spacy \citep{spacy} we extracted the maximum depth of a dependency tree, mean depth of all dependency trees in a text, and number of passive constructions (determined by the number of \textit{nsubjpass} POS tags) per sentence.

% max_depth, mean_depth

%avg_passive_constructions!

\paragraph{Frequency features.} To capture whether the texts differ in word use, the logarithmic frequency of all content words in the human texts were computed. Additionally, lists of frequent words (frequency $\geq$ 12) and hapax legomena (frequency = 1) have been computed. From this the following features were extracted for all texts: mean log frequency of content words, ratio of frequent words to content words, ratio of hapax legomena to content words and number of hapax legomena. 

%mean_log_freq_content_words,  prop_freq_content_words, prop_unfreq_content_words, num_freq1_words, n_unique_words

Additionally, we used the Wiktionary frequency lists for English\footnote{\url{https://en.wiktionary.org/wiki/Wiktionary:Frequency_lists/English}} and extracted a list of high frequency words (top 10\%), mid-high frequency words (top 20\%) as well as field specific word lists, namely the most frequent words in fantasy texts and in Wikipedia articles. For each list and each text in the datasets we extracted the ratio of words belonging to the lists to the content words as a feature.

% ratio_top10_content_words, ratio_top20_content_words, ratio_fantasy_words, ratio_wikipedia_words


\paragraph{Word difficulty features.}
The CEFR-J\footnote{\url{https://www.cefr-j.org/}} project provides vocabulary lists for the different proficiency levels of the Common European Framework of Reference for Languages (CEFR)\footnote{\url{https://www.coe.int/en/web/common-european-framework-reference-languages}}. We used these lists\footnote{\url{https://github.com/openlanguageprofiles/olp-en-cefrj/tree/master}} to compute the following features: ratio of A1/A2/B1/B2/C1/C2-level words to content words. This was done twice: once on the basis of the stemmed and once on the lemmatized words. We used the Porter stemmer and the WordNet lemmatizer from NLTK.

%ratio_a1_lemma_word_count, ratio_a1_stem_word_count, ratio_a2_lemma_word_count, ratio_a2_stem_word_count
%ratio_b1_lemma_word_count, ratio_b1_stem_word_count, ratio_b2_lemma_word_count, ratio_b2_stem_word_count
%ratio_c1_lemma_word_count, ratio_c1_stem_word_count, ratio_c2_lemma_word_count, ratio_c2_stem_word_count
% ratio_cefrj_lemma_word_count, ratio_cefrj_stem_word_count





\paragraph{Stylistic and sentiment features.}
A number of features concerning text style and text sentiment were extracted. Using the same method as for the difficulty features above, we extracted the ratio of words in the list of negative opinion words compiled by \citet{NegOpWordList} as well as the readability score of the texts according to the Flesch reading-ease test\footnote{\url{https://github.com/textstat/textstat}} .
The other features in this subset have been extracted by using available fine-tuned classifiers. Emotion English DistilRoBERTa-base\footnote{\url{https://huggingface.co/j-hartmann/emotion-english-distilroberta-base}} is a classifier that predicts Ekman's six basic emotions, plus a neutral class \citep[cf.][]{hartmann2022emotionenglish}. The logit for each class provides one feature (anger,	disgust, fear, joy, neutral, sadness, surprise). As a standard sentiment analyzer we used the sentiment-analysis-pipeline from Hugging Face\footnote{\url{https://huggingface.co/}} and, using the logits, extracted two features (positive, negative). Analogously, the features `formal' and `informal'  were extracted using the formality ranker by \citet{FormalityRanker}, which is a RoBERTa model trained to predict to which register a sentence belongs. Finally, we used a toxicity classification model\footnote{\url{https://huggingface.co/s-nlp/roberta_toxicity_classifier}} that is a RoBERTa model fine-tuned to predict whether a text is toxic or not. 



% https://gist.github.com/mkulakowski2/4289441 for negative word list

%ratio_negative_lemma_word_count, ratio_negative_stem_word_count,
%readability,
%sadness,surprise,neutral,disgust,  fear,joy, anger
%POSITIVE, NEGATIVE, 
% toxic, ,non_toxic,   formal, informal, 





\paragraph{Features extracted from the pre-classifier.} In order to inform the correction classifiers on the basis of the decision of the pre-classifier, we extracted the logits and the last hidden state of our RoBERTa pre-classifier for each text. The last hidden states were reduced from 768 to 2 dimensions using PCA (principal component analysis) and UMAP (uniform manifold approximation and projection). For UMAP the hyperparameters \texttt{min\_dist}, \texttt{n-neighbors} and \texttt{metric} were tuned by a combination of random search and grid search and evaluated on the accuracy of a logistic regression classifier that predicts the label from the 2 dimensions. The extracted features included in our feature set are the logits, the 2-dimensional PCA representation of the last hidden state and the 2 UMAP dimensions gained by setting \texttt{min\_dist} to 0.01 and \texttt{n-neighbors} to 100. We kept two metrics, namely \texttt{cosine} and \texttt{jaccard}.


%X_pca_0, X_pca_1, X_umap_0, X_umap_1, X_umap_jac_0, X_umap_jac_1, logits_0_roberta, logits_1_roberta



\subsubsection{Feature selection}\label{sec:feature-selection}
To account for the variability in features, we initially scaled all 70 features using the Standard Scaler from scikit-learn \citep{scikit-learn}.
During the collection of the 70 features, no attention was paid to whether they contained quasi-duplications. Features which were highly correlated with other features (>0.9) were removed subsequently using a correlation matrix. After this removal, 51 features remained.
%For instance, in binary classification, the two logit values always sum up to 1, resulting in a correlation value of 1. 
% We found a correlation of 0.91 between the verb-noun-ratio and the verb-word-ratio. To address highly interrelated features, a correlation matrix was created, and features with a correlation value exceeding 0.9 with another feature were removed. This step ensures that highly correlated features do not introduce redundancy into the model. 

In the next step, only features with high or medium correlation with the gold label (Pearson correlation $\geq 0.1$ or $\leq -0.1$) were retained in order to choose the features most relevant for the classification task. Table \ref{tab:correlations-human-machines} in the Appendix shows all features (including those which are highly correlated to each other) that have at least a medium positive or negative correlation with the gold label. After both selection steps, 26 features remained (see Table \ref{tab:features-used}).  

\subsubsection{Model selection and training}
In a comparison of various classifiers from scikit-learn (i.e. Random Forest, Logistic regression), MLPs performed best in most settings: whether trained on all features, trained only on at least medium correlated features, or trained only on features that are not extracted from the pre-classifier. We therefore chose MLP as our correction classifiers. 

Before conducting training on the combined train and dev dataset, we separated the texts for which the RoBERTa pre-classifier had predicted the human label from those for which it had predicted the machine label, thus creating two splits. Then, we trained two separate MLPs on the two splits of the training data using the 26 features identified as relevant in the feature selection process (\ref{sec:feature-selection}). The idea behind this approach was that the models might learn in which cases the fine-tuned transformer classified the data incorrectly, and would thus have to be corrected. The test data was then prepared by calculating the 26 features, on which the pair of MLP correction classifiers made the final predictions. 

\section{Results}

Table \ref{tab:classification-reports-4models} shows the performance of the submitted fusion model, obtained using the \verb|classification_report| from scikit-learn. Overall, the fusion model achieves an accuracy of 85\%. The table additionally shows the performances of two other models on the test data in comparison: (i) the RoBERTa pre-classifier;  (ii) an MLP model that was trained with the same hyperparameters as used for the correction classifiers and the same features selected in the feature selection process (see Section \ref{sec:feature-selection}), except for the ones extracted from the pre-classifier (see Section \ref{sec:feature-extraction}). 

\begin{table}%\small
\begin{tabular}{@{}lllll@{}}
\hline
model & label & prec. & rec. & f1 \\ \hline\hline
\textbf{fusion model} & human & 0.85      & 0.85   & \textbf{0.85}     \\
             &machine & 0.86      & 0.86   & \textbf{0.86}   \\ 
accuracy: \textbf{0.85}           \\  \hline
pre-classifier & human & \textbf{0.99}      & 0.48   & 0.64     \\
& machine & 0.68      & \textbf{0.99}   & 0.81    \\
accuracy: 0.75\\ \hline

%RoBERTa  base & human & 0.57 & \textbf{0.98} & 0.72\\
%OpenAI detector & machine & \textbf{0.95} & 0.34 & 0.50\\
%accuracy: 0.64\\ \hline
MLP& human &  0.53   &   0.89  &    0.67   \\
& machine &  0.75    &  0.30   &   0.43 \\ 
accuracy: 0.58\\ \hline
\end{tabular}
\caption{Precision, recall, f1-score, and overall accuracy for the submitted fusion model and two models for comparison: the RoBERTa pre-classifier and an MLP model trained with the selected linguistic features. The support for the `human' class is 16,272 and for the `machine class 18,000.} \label{tab:classification-reports-4models}
\end{table}
 

Although the pre-classifier performed fairly well on the dev data (accuracy: 0.89, for more details see Table \ref{tab:performance-pre-classifier-dev} in the Appendix), we opted for a fusion model with a correction layer in order to improve robustness for data from new generators and domains. The implementation of the two correction MLPs corroborated the hypothethis. On the test data, the accuracy of the pre-classifier drops to 0.75, while the addition of the correction layer improved the accuracy to 0.85. 

A closer look at the recall and precision for the two classes `human' and `machine' reveals that the fusion model balances out the problems of the pre-classifier and the MLP. The high precision and low recall of the pre-classifier for `human' and vice versa for `machine' indicate that it is biased towards the `machine' class. Accordingly the relatively high precision and low recall of the MLP classifier for `machine' and vice versa for `human' indicate that it is biased towards the `human' class. In contrast, the fusion model shows equally high precision and recall for both classes.

As described in Section \ref{sec:pre-classifier}, the pre-classifier is obtained from the RoBERTa base OpenAI detector by further training. Comparing its performance to the original model (see Table \ref{tab:performance-roberta-base}, Appendix), finetuning has led from a bias towards the `human' class to a bias towards the `machine class'. This is likely due to the fact that the fine-tuning data had an imbalance towards machine-generated texts.



% \begin{itemize}
%     \item Precision:
%     \item Out of all texts that the model predicted would be human, 85\% were human.
%     \item Out of all texts that the model predicted would be machine-generated, 86\% were machine-generated.
%     \item Recall:
%     \item Out of all texts that are human, the model predicted this outcome correctly for 85\%.
%     \item Out of all texts that are human, the model predicted this outcome correctly for 86\%.
% \end{itemize}

%While other factors could also have caused the stabilization of precision and recall, it seems plausible that it was caused by the addition of the features and possibly feature scaling. Especially when compared to the performance of RoBERTa classifications alone, recall has significantly improved from 0.48 to 0.85 leading to less over-fitting and better overall performance (see \ref{tab:roberta-classification-report} in the appendix). 

% \begin{table}[h!]
% \begin{tabular}{lllll}
% \hline
%  label & precision & recall & f1-score & support \\ \hline
% human & 0.85      & 0.85   & 0.85     & 16272   \\
% machine & 0.86      & 0.86   & 0.86     & 18000   \\ \hline
% \end{tabular}
% \caption{Precision, recall, f1-score, and support for the final submission. Support indicates the number of texts for that label.}\label{tab:final-classification-report}
% \end{table}



% Quantitative analysis: Ablations or other comparisons of different design decisions to better understand what works best. Indicate which data split is used for the analyses (e.g. in table captions). If you modify your system subsequent to the official submission, clearly indicate which results are from the modified system.



\subsection{Error Analysis}
An error analysis was completed in three parts: We examined the influence of the different labels, the features, and the correctional classifiers on accuracy. The influence of the domain was not examined since there was only one domain present in the test data. 

When inspecting the label distribution for the misclassified texts, we can see an almost perfect 50\% split between human and machine-labeled texts. Between the models, the errors are not distributed as evenly, as shown in Figure \ref{fig:errors-model-ditr} in the Appendix. GPT4 and dolly texts were misclassified most often, followed by Cohere, DaVinci, and chatGPT, while BLOOMz texts were rarely classified incorrectly. Since GPT4 texts were not seen in the train or dev data, it is not unsurprising that those texts were classified least accurately. A further reason could be that GPT4-generated texts are known to be very `human-like', hence harder to differentiate from human texts.
%Generally, it seems that the model can handle models that it has seen in training and development yet struggles with unseen models. It is unclear whether this is due to special characteristics of the texts from the GPT4 model or whether this is a general phenomenon for all unseen models.  

Figure \ref{fig:error-splits} shows the classification by the pre-classifier and whether it was modified by the correction classifier (the same data in numbers is given in Table \ref{tab:error-splits}).
%Table \ref{tab:error-splits} shows the classification by the pre-classifier and the correction classifier (h = human, m = machine). 
A text identified as human by the pre-classifier was typically classified accurately and only rarely adjusted by the correcting classifier. For the texts where the pre-classifier predicted a machine label, the prediction was corrected often. However, as shown in table  \ref{tab:error-splits}, 2,308 cases should have been corrected and were not. The machine label predictions by the pre-classifier have caused most errors, as that label was predicted so often. This is also reflected in the recall of the pre-classifier-only model in Table \ref{tab:classification-reports-4models}.

Finally, we correlated all features used in the fusion model with the labels that were predicted incorrectly. The strongest correlation was shown by the features 
%\verb|1st UMAP-dimension (Jaccard)|  (- 0.82), \verb|ratio of CEFR-B1 words (stem)| (- 0.71), \verb|ratio of CEFR-B2 words (stem)| (- 0.57), \verb|neutral sentiment score| (- 0.51), and \verb|ratio of pronouns to content words| (0.55). 
\textit{1st UMAP-dimension (Jaccard)} (- 0.82), \textit{ratio of CEFR-B1 words (stem)} (- 0.71), \textit{ratio of CEFR-B2 words (stem)} (- 0.57), \textit{neutral sentiment score} (- 0.51), and \textit{ratio of pronouns to content words} (0.55). 
Since the correlations are on the wrong predictions, a strong negative correlation indicates a correlation with an incorrectly predicted human label (label 0), while a strong positive correlation implies the opposite. It is possible that the low correlation threshold chosen for feature acceptance led to the inclusion of features initially weakly correlated with the labels in the training and development data, which may have adversely affected the correctional classifiers' decisions. Alternatively, the test set data might exhibit a different distribution for those features compared to the training and development data.

\begin{figure}
    \centering
    \includegraphics[width = 1\columnwidth]{plots/predictions_and_corrections.png}
    \caption{
    Left two bars: predictions by correction classifier correcting texts pre-classified as “human”, right two bars: predictions by correction classifier correcting texts pre-classified as “machine”.}
    \label{fig:error-splits}
\end{figure}

\begin{table}
    \begin{tabular}{@{}l|cccc@{}} \hline 
    pre-classifier&  \multicolumn{2}{c}{h} &   \multicolumn{2}{c}{m}\\
    correction classifer&  h &  m &  m & h\\ \hline
%    fusion model&  &  &  & \\
    correct& 7622& 86&15402& 6189\\
    wrong& 12& 153& 2308 &2500\\\hline
    \end{tabular}
    \caption{Classification errors split by prediction by the pre-classifier and correction classifier (h = human, m = machine).}
    \label{tab:error-splits}
\end{table}


% - hoher recall = sehr viel wird als maschine klassifiziert
% - bei allen Machinen bis auf BLOOMz verantwortlich für 100\% der Fehler


% Modelle:
% - Roberta AI pur  (Cornelia) (pre classifier no finetune)
% - Roberta AI fine-tuned (pre classifier fine-tuned)
% - Roberta AI features + linguistische features mit classifier (Anna und Wiebke am letzten Tag zusammen)
% - 2-stufig: erst Roberta fine-tuned dann auf Klassifikation linguistische features (Wiebke letzter ABschnitt im Notebook)
% - nur linguistische features (




% \begin{itemize}
%     \item Label distribution in errors: machine 2512 (49\%), human 2461 (51\%) 
%     \item Model distribution in errors: GPT4 757, dolly 647, Cohere 408, DaVinci 361, chatGPT 308, BLOOMz 31 (plot here \ref{fig:errors-model-ditr}. Good classification for Bloomz is unsurprising since dev was entirely BLOOMz, chatgpt4 war neu 
%     \item Wurde BLOOMz predictions mehr von den correntional classifiern corrected, weil nur die dev gesehen haben?
%     \item correlation > 0.5 (oder -0.5) with wrong prediction: \verb|X_umap_jac_0| (-0.82), \verb|ratio_b1_stem_word_count| (-0.71), \verb|ratio_b2_stem_word_count| (-0.57), \verb|neutral| (0.-51), \verb|ratio_PRON_word_count| (0.55). 
%     \item Negative Werte = high correlation with false 0 aka human classification(?)
% \end{itemize}

% Nichts hiervon muss rein. Diente mehr dem VErständnis der Daten.
% Ich würde erwarten, dass ein hoher B1-Anteil eher bei Maschinen als bei Menschen zu erwarten sind (trainiert auf einfache Sprache). Wenn das der Fall wäre, dann hätten wir eine hohe positive Korrelation (hoher Anteil = hoher Labelwert). Der hohe negative Wert für falsch klassifizierten, den wir sehen bedeutet, dass dort wo hoher Anteil der 0-Label (MEnsch) richtig wäre unser Modell aber fälschlich Maschine sagt.

% Sollten die Korrelationen mit den predicted fusion Labeln berechnet worden sein, dann heißt negative Korr. dass viele B1-Wörter mit falscher mensch-prediction korrelieren.  

% für B1 mit allen Testdatn 0.2 => hoher Anteil eher bei Maschinen. Wir sehen aber negativen Wert. 

\section{Conclusion and Limitations}
Overall, this study has highlighted the benefit of using a fusion architecture consisting of a pre-classifier and linguistically informed correctional classifiers. By adding syntactic, stylistic, sentiment, frequency- and word difficulty-based features, we were able to improve the performance of a fine-tuned pre-trained RoBERTa model for AI generated text detection and adjust the bias towards the machine label. 
Because our fusion model uses a pre-trained RoBERTa model, all computations for this paper can be run locally or, in the case of the RoBERTa fine-tuning, using a free Google Colab account. This means that our model can be easily expanded and leaves a smaller environmental footprint. 

Future studies could expand our fusion model by incorporating more semantic-level or complex features such as contextual predictability, as well as fine-tuning the pre-classifier using more, balanced data. Our code is available on GitHub\footnote{\url{https://github.com/ansost/art-nat-HHU-semeval2024}}.

%\newpage


% The analysis of the linguistic features allows us to draw some conclusions about the characteristics of machine-generated texts and human-written texts. As can be seen from table \ref{tab:co rrelations-machine} in the Appendix, MGTs tend to be more neutral, more formal, and generally have a more positive sentiment. On the other hand, HWTs are more informal and more often include words conveying negative emotions, as shown by the correlations in table \ref{tab:correlations-human}. In addition to this, MWTs use more simple and more common words, while HWTs use a wider range of words (high TTR) and more infrequent words. It is also interesting to note that human and AI texts have higher proportions of certain POS: AI texts use more determiners, human texts use more adverbs and pronouns.

% \begin{itemize}
%     % \item future development of the model could put more emphasis on the analysis of the domain of the text to be classified. Haben wir ja schon irgendwie abegdeckt mit den features. Vtl 
%     \item mehr/other features, da es ja eine range an features gibt, die zwar attested ist to work well, die wir aber nicht verwendet haben: mehr stylistic features, argument structures, incorporate spontaneity of human speech (word surprisal and entropy / perplexity), more refined contextual information and frequency measures, contextual predictability of words etc.
%     \item data document type too restricted (only relatively formal and written long texts)
% \end{itemize}

\bibliography{anthology,custom}
\bibliographystyle{acl_natbib}

\newpage
\appendix
\label{sec:appendix}




\section{Features with medium or high correlation}
\begin{table}[h]
\small
\begin{tabular}{@{}lr@{}}\hline
feature   & corr.  \\
\hline
1st UMAP-dimension (jaccard)                 &  $0.94$\\ 
logits for label 1 from pre-classifier              &  $0.92$\\ 
roBERTa prediction                 &  $0.92$\\ 
positive sentiment score                       &  $0.28$\\ 
ratio of determiners to content words          &  $0.28$\\ 
ratio of pronouns to content words        &  $0.25$\\ 
score for formal                  &  $0.22$\\ 
ratio of CEFR-B1 words (stem)      &  $0.20$\\ 
ratio CEFR (all levels) words (stem)    &  $0.17$\\ 
ratio of CEFR-B2 words (stem)       &  $0.17$\\ 
ratio of CEFR-A2 words (stem)      &  $0.13$\\ 
ratio of CEFR-B1 words (lemma)     &  $0.13$\\ 
ratio of conjunctions to words         &  $0.13$\\ 
ratio of CEFR-A2 words (lemma)      &  $0.12$\\ 
score for joy                           &  $0.11$\\ 
ratio of fantasy words            &  $0.10$\\ 
score for neutral                      &  $0.10$\\ 
ratio of Wikipedia words          &  $0.10$\\ 
word ratio of top 10\% freq. Wiktionary words      &  $0.10$\\ 
word ratio of top 20\% freq. Wiktionary words      &  $0.10$\\
\hline
$\cdots$  \\
\hline
score for fear                            &   $-0.10$ \\ 
1st UMAP-dimension (cosine)                         &   $-0.10$ \\ 
score for anger                            &   $-0.11$ \\ 
ratio of pronouns to words            &   $-0.15$ \\ 
number of hapaxes                 &   $-0.17$ \\ 
score for informal                        &   $-0.22$ \\ 
ratio of adverbs to words             &   $-0.22$ \\ 
prop. of unfreq. words to content words        &   $-0.25$ \\ 
TTR                              &   $-0.27$ \\ 
number of unique words                   &   $-0.27$ \\ 
negative sentiment score                         &   $-0.28$ \\ 
mean depth of dep. tree for sentences                    &   $-0.40$ \\ 
max depth of dependency tree                        &   $-0.45$ \\ 
2nd UMAP-dimension (jaccard)                     &   $-0.59$ \\ 
logits for label 0 from pre-classifier                 &   $-0.92$\\
\hline 
\end{tabular}
\caption{Features with medium or strong positive or negative correlation ($-0.1 \leq$ corr $\leq 0.1$) with label 1 (machine) in train data}
\label{tab:correlations-human-machines}
\end{table}

\newpage

\section{Fusion model features}
\begin{table}[h]
\begin{tabular}{@{}l@{}}
\hline
feature name\\ \hline
type-to-token ratio (TTR)\\
ratio of adverbs to content words \\
ratio of pronouns to content words \\
ratio of determiners to content words\\
ratio of conjunctions to content words\\
ratio CEFR (all levels) words\\
ratio of CEFR-A2 words \\
ratio of CEFR-B1 words \\
ratio of CEFR-B2 words \\
number of hapaxes \\
ratio of frequent words to content words\\
ratio of hapaxes to content words\\
1st UMAP-dimension (cosine) \\
negative sentiment score\\
positive sentiment score \\
score for anger \\
score for fear \\
score for neutral \\
score for joy \\
score for formal \\
score for informal \\
max depth of dependency tree \\
mean depth of dependency tree for sentences\\
word ratio of top 10\% freq. Wiktionary words\\
1st UMAP-dimension (jaccard)\\
logits for label 0 from RoBERTa pre-classifier\\
\hline
\end{tabular}
\caption{Features used to train the fusion model.}
\label{tab:features-used}
\end{table}

\section{RoBERTa pre-classifier performance on dev}
\begin{table}[h]
\begin{tabular}{llll}
\hline
label & precision & recall & f1-score  \\ \hline
human &      0.91   &   0.86  &  0.88  \\     
machine &       0.87  &    0.91  &    0.89     \\ \hline
\end{tabular}
\caption{Precision, recall, f1-score, and support for the RoBERTa pre-classifier on the dev data. Accuracy is 0.89}\label{tab:performance-pre-classifier-dev} 
\end{table}


% \section{Features with medium or high correlation with label 1 in train}
% \begin{table}[h]
% \begin{tabular}{@{}ll@{}}
% \hline
% feature & corr. \\
% \hline
% 1st UMAP-dimension (jaccard)                 &  $0.94$\\ 
% logits for label 1 from pre-classifier              &  $0.92$\\ 
% roBERTa prediction                 &  $0.92$\\ 
% positive sentiment score                       &  $0.28$\\ 
% ratio of determiners to content words          &  $0.28$\\ 
% ratio of pronouns to content words        &  $0.25$\\ 
% score for formal                  &  $0.22$\\ 
% ratio of CEFR-B1 words (stem)      &  $0.20$\\ 
% ratio CEFR (all levels) words (stem)    &  $0.17$\\ 
% ratio of CEFR-B2 words (stem)       &  $0.17$\\ 
% ratio of CEFR-A2 words (stem)      &  $0.13$\\ 
% ratio of CEFR-B1 words (lemma)     &  $0.13$\\ 
% ratio of conjunctions to words         &  $0.13$\\ 
% ratio of CEFR-A2 words (lemma)      &  $0.12$\\ 
% score for joy                           &  $0.11$\\ 
% ratio of fantasy words            &  $0.10$\\ 
% score for neutral                      &  $0.10$\\ 
% ratio of Wikipedia words          &  $0.10$\\ 
% word ratio of top 10\% freq. Wiktionary words      &  $0.10$\\ 
% word ratio of top 20\% freq. Wiktionary words      &  $0.10$\\
% \hline
% \end{tabular}
% \caption{Features with high or medium positive correlation (corr.) with label 1 in train data (machine)}
% \label{tab:correlations-machine}
% \end{table}


% \section{Features with medium or high correlation with label 0 in train}

% \begin{table}[h]
% \begin{tabular}{@{}ll@{}}\hline
% feature   & corr.  \\
% \hline
% score for fear                            &   $-0.10$ \\ 
% 1st UMAP-dimension (cosine)                         &   $-0.10$ \\ 
% score for anger                            &   $-0.11$ \\ 
% ratio of pronouns to words            &   $-0.15$ \\ 
% number of hapaxes                 &   $-0.17$ \\ 
% score for informal                        &   $-0.22$ \\ 
% ratio of adverbs to words             &   $-0.22$ \\ 
% prop. of unfreq. words to content words        &   $-0.25$ \\ 
% TTR                              &   $-0.27$ \\ 
% number of unique words                   &   $-0.27$ \\ 
% negative sentiment score                         &   $-0.28$ \\ 
% mean depth of dep. tree for sentences                    &   $-0.40$ \\ 
% max depth of dependency tree                        &   $-0.45$ \\ 
% 2nd UMAP-dimension (jaccard)                     &   $-0.59$ \\ 
% logits for label 0 from pre-classifier                 &   $-0.92$\\
% \hline 
% \end{tabular}
% \caption{Features with high or medium negative correlation (corr.) with label 0 in train data  (machine)}
% \label{tab:correlations-human}
% \end{table}


% \section{Features with medium or high correlation with label 1 in train}
% \begin{table}[h]
% \begin{tabular}{@{}ll@{}}
% \hline
% feature & corr. \\
% \hline
% 1st UMAP-dimension (jaccard)                 &  $0.94$\\ 
% logits for label 1 from pre-classifier              &  $0.92$\\ 
% roBERTa prediction                 &  $0.92$\\ 
% positive sentiment score                       &  $0.28$\\ 
% ratio of determiners to content words          &  $0.28$\\ 
% ratio of pronouns to content words        &  $0.25$\\ 
% score for formal                  &  $0.22$\\ 
% ratio of CEFR-B1 words (stem)      &  $0.20$\\ 
% ratio CEFR (all levels) words (stem)    &  $0.17$\\ 
% ratio of CEFR-B2 words (stem)       &  $0.17$\\ 
% ratio of CEFR-A2 words (stem)      &  $0.13$\\ 
% ratio of CEFR-B1 words (lemma)     &  $0.13$\\ 
% ratio of conjunctions to words         &  $0.13$\\ 
% ratio of CEFR-A2 words (lemma)      &  $0.12$\\ 
% score for joy                           &  $0.11$\\ 
% ratio of fantasy words            &  $0.10$\\ 
% score for neutral                      &  $0.10$\\ 
% ratio of Wikipedia words          &  $0.10$\\ 
% word ratio of top 10\% freq. Wiktionary words      &  $0.10$\\ 
% word ratio of top 20\% freq. Wiktionary words      &  $0.10$\\
% \hline
% \end{tabular}
% \caption{Features with high or medium positive correlation (corr.) with label 1 in train data (machine)}
% \label{tab:correlations-machine}
% \end{table}


\newpage

\section{RoBERTa base OpenAI detector performance on test}
\begin{table}[h]
\begin{tabular}{llll}
\hline
 label & precision & recall & f1-score  \\ \hline
human & 0.57 & \textbf{0.98} & 0.72\\
machine & \textbf{0.95} & 0.34 & 0.50\\ \hline
\end{tabular}
\caption{Precision, recall, f1-score, and support for the RoBERTa base OpenAI detector on the test data. Accuracy is 0.64}\label{tab:performance-roberta-base}
\end{table}


\section{Distribution of errors}
\begin{figure}[h]
    \centering
    \includegraphics[width=1\linewidth]{plots/model-distr-errors.png}
    \caption{Distribution of models in false predictions.}\label{fig:errors-model-ditr}
\end{figure}





% \section{Classification errors and corrections}
% \begin{table}[h]
%     \begin{tabular}{@{}l|cccc@{}} \hline 
%     pre-classifier&  \multicolumn{2}{c}{h} &   \multicolumn{2}{c}{m}\\
%     correction classifer&  h &  m &  m & h\\ \hline
% %    fusion model&  &  &  & \\
%     correct& 7622& 86&15402& 6189\\
%     wrong& 12& 153& 2308 &2500\\\hline
%     \end{tabular}
%     \caption{Classification errors split by prediction by the pre-classifier and correction classifier (h = human, m = machine).}
%     \label{tab:error-splits}
% \end{table}


\end{document}



% % (n words \~ n sentences, n punctuation, n unique words, n vowels \~ n words, n content words \~ n stop words, n content words \~ top 10\% frequent words, n content words \~ top 20\% frequent words, n fantasy words \~ n content words, n wikipedia words \~ n content words, n verbs \~ n nouns, n negation words* \~ n words XX, n 'nsubjpass' \~ n sentences)
% A variety of word proportions in the text was extracted for each text\footnote{The tilde ($\sim$) will be used to mean 'as a proportion of'} First, the texts were split into words and punctuation to derive the following features: \textit{number of words $\sim$ number of sentences, number of punctuation $\sim$ ???, number of unique words, number of vowels $\sim$ number of words}. In addition, the nltk stopword package (citation) and spacy (citation) was used to derive the features \textit{number of content words $\sim$ number of stop words, number of verbs $\sim$ number of nouns, number of negation words $\sim$ number of (content??) words} and \textit{number of 'nsubjpass' $\sim$  number of sentences}. 
% % For each document, the surface level features such as token, type, word and punctuation per sentence counts are collected. Moreover, we also calculated some ratio-based measures based on word count statistics including type-token and punctuation-word ratio.

% \textbf{Frequency features} Using wikipedia frequency lists, we used three different frequency lists in order to derive the ratios of high frequency words from the fields of fantasy, wikipedia articles and the English language in general. From this we derived the following features: \textit{number of high frequency words (top 10\%) percent $\sim$  number of content words, number of mid-high frequency words (top 20\%)}, as well as the \textit{number of words in the wikipedia and fantasy word list $\sim$  content words}. 
% [add wiebkes frequency features]

% % We sourced a large varieties of word lists as our targets for counting amount of words in certain domains. The target dictionaries include negative stems, top 10 and 20 content words in the English language, Wikipedia common words, common words used in fantasy literature, CEFR (Common European Framework of Reference for Languages) vocabulary per level etc. Most of the features are saved as numeric ratios in respect to the whole document length. 

% \subsubsection{RoBERTa fine-tuned features} In order to obtain word embeddings particularly suited for the MGT detection task, RoBERTa Base OpenAI Detector\footnote{\href{openai-community/roberta-base-openai-detector}{https://huggingface.co/openai-community/roberta-base-openai-detector}}, a transformer model specifically designed for AI generated texts detection \citep{solaiman2019release}, was fine-tuned on 10 percent of the training data for 3 epochs with a learning rate of 2e-5. Then, the last hidden state was extracted. Using only 10 percent of the training data might prevent over-specialization on the training data and provide more flexibility when the model comes across texts generated by unknown models or from unknown domains. The predictions from the fine-tuned model were also used as a feature.

% To determine the features most relevant for the classification task, only features with a high correlation with the labels were kept (Pearson  correlation coefficient < 0.1). A list of feature can be found in table \ref{fig:features-kept} in the Appendix.

% \subsubsection{Sentiment features} [wiebke]
% ...

% + readability + dependency tree measures using spacy



% \subsection{Correction classifiers}

% Before conducting training on the combined train and dev dataset, we divided it into two parts, one with the datapoints for which RoBERTa pre-classifier had predicted label 0, and one with the datapoints for which RoBERTa had predicted 1. Then, we trained two separate MLPs on the two parts of our training data. The idea behind this approach was that the models might learn in what cases the fine-tuned transformer classified the data incorrectly, and would thus be able to  correct it. In a comparison of various classifiers, MLPs had produced better results than random forest models or logistic regression. The test data was then prepared by calculating the features identified as relevant during the creation of our model. Then, we used the pair of MLP classifiers to make the final predictions. 


% \subsection{Model selection}

% \subsection{Training cost}
% }


preclassifier             |   h    ||   m     
correction classier      | h |  m || h | m  
--------------------------------------------
fusion model
correct predictions      |121 | 2323 || 23 | 43343
wrong predictions        |121 | 2323 || 23 | 43343



#############################################################################################################

Begin python code:
#############################################################################################################

#     correct& 7622& 86& 6189&15402\\
#   wrong& 12& 153& 2500&2308\\\hline
chh = 7622
chm = 86
cmh = 6189
cmm = 15402
whh = 12
whm = 153
wmh = 2500
wmm = 2308

import matplotlib.pyplot as plt
import numpy as np


# plot as barplots. height of bars depending on values. 1st bar for chh and whh, 2nd for chm and whm, 3rd for cmh and wmh, 4th for cmm and wmm
# use stacked bars for correct and wrong

col_correct = 'darkgreen'
col_wrong = 'orangered'

fig, ax = plt.subplots()
ax.bar(1, chh, label='correct', color=col_correct)
ax.bar(1, whh, bottom=chh, label='wrong', color=col_wrong)
ax.bar(2, chm,  color=col_correct)
ax.bar(2, whm, bottom=chm,  color=col_wrong)
ax.bar(3, cmm,  color=col_correct)
ax.bar(3, wmm, bottom=cmm, color=col_wrong)
ax.bar(4, cmh,  color=col_correct)
ax.bar(4, wmh, bottom=cmh,  color=col_wrong)


# label for x-axis: for chh and whh "human kept", for chm and whm "human modified", for cmh and wmh "machine kept", for cmm and wmm "machine modified"
ax.set_xticks([1, 2, 3, 4])
ax.set_xticklabels(["human kept", "human modified", "machine kept", "machine modified"])



ax.set_ylabel('number of texts')
#ax.set_title
ax.legend()

# save plot
plt.savefig(datapath_out + 'predictions_and_corrections.png')
