% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Remove the "review" option to generate the final version.
% \usepackage[review]{acl}
\usepackage{acl}


% Standard package includes
\usepackage{times}
\usepackage{latexsym}
\usepackage{url}
\usepackage{graphicx}
\usepackage{tabularray}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Transformers at SemEval-2024 Task 5: Legal Argument Reasoning Task in Civil Procedure using RoBERTa}

\author{Kriti Singhal\textsuperscript{1}, Jatin Bedi\textsuperscript{2} \\
  Computer Science and Engineering Department, \\
  Thapar Institute of Engineering and Technology, India \\
  \texttt{\textsuperscript{1}kritisinghal711@gmail.com, \textsuperscript{2}jatin.bedi@thapar.edu} \\}


\begin{document}
\maketitle
\begin{abstract}
Legal argument reasoning task in civil procedure is a new NLP task utilizing a dataset from the domain of the U.S. civil procedure. The task aims at identifying whether the solution to a question in the legal domain is correct or not. This paper describes the team "Transformers" submission to the Legal Argument Reasoning Task in Civil Procedure shared task at SemEval-2024 Task 5. We use a BERT-based architecture for the shared task. The highest F1-score score and accuracy achieved was 0.6172 and 0.6531 respectively. We secured the 13\textsuperscript{th} rank in the Legal Argument Reasoning Task in Civil Procedure shared task.
\end{abstract}

\section{Introduction}

Mastering the art of arguing a legal case is essential for lawyers. This necessitates deep knowledge of the particular area of law along with advanced reasoning capabilities, including drawing similarities and differences. Researchers have made significant efforts towards setting the benchmark models for the new Natural Language Processing (NLP) problems in the domain of legal language understanding \cite{chalkidis-etal-2022-lexglue}.

The task, Legal Argument Reasoning Task in Civil Procedure\footnote{https://github.com/trusthlt/semeval24} \cite{held-habernal-2024-legalreasoning}, organized at SemEval-2024 aimed at classifying the solution to a given problem as right or wrong.

Classifying an answer to a given question as correct or incorrect is a new NLP task. In particular, in the legal domain limited number of publicly available corpora exist. This contributes to added difficulty of this task \citep{fawei-etal-2016-passing}.

Recent advances in the field of NLP have addressed various issues, such as long texts and under-resourced domains. These include Long Short Term Memory \cite{hochreiter1997long}, and Gated Recurrent Units \cite{chung2014empirical}. But transformers \cite{vaswani2017attention} have taken the performance to new heights which were not possible earlier.

In the past, various efforts have been made to perform domain-specific adaption of different existing techniques and models. Some of these adaptions include SciBERT which was pre-trained for scientific texts, specifically in the bio-medical domain \cite{beltagy-etal-2019-scibert}. Similarly, BioBERT was created with special emphasis on the bio-medical area \cite{Lee_2019}.

In this paper, we discuss our use of a transformer-based model, RoBERTa, in the shared task of Legal Argument Reasoning Task in Civil Procedure at SemEval-2024.





\begin{figure*}
\centering
\includegraphics[width=15cm]{training.png}
\caption{Proposed Methodology}
\label{training}
\end{figure*}


\begin{figure*}
\centering
\includegraphics[width=12cm]{testing.png}
\caption{Label Generation for Unseen Data}
\label{testing}
\end{figure*}






\section{Related Work}
Researchers have used and explored various techniques in the past. In the work done by \citet{beltagy-etal-2019-scibert,Lee_2019}, it was found that BERT-based architectures did not perform very well on problems that required specialized domain knowledge. Two possible solutions were found to address this issue. The first was to further pre-train BERT on domain-specific corpora, and the second possible solution was to pre-train BERT from scratch on domain-specific corpora \cite{chalkidis-etal-2020-legal}.

\citet{Lee_2019} performed domain-adaption of BERT in the bio-medical domain. The experiment explored the effect of further pre-training BERT base for 470,000 steps on biomedical articles. The performance of the resulting model, BioBERT, was evaluated on biomedical datasets. This led to an improvement in performance when compared to BERT base.

\citet{beltagy-etal-2019-scibert} proposed a family of BERT-based models, SciBERT, for scientific texts with a special focus on the bio-medical domain. Two approaches were followed for SciBERT, the first was further pre-training BERT base, and the second approach was to pre-train BERT base on domain-specific corpora from scratch. In the second approach, random initialization of the model was performed, and a fresh new vocabulary was created. An improvement in performance was observed in the downstream tasks for both the approaches.

In the work carried out by \citet{chalkidis-etal-2020-legal}, BERT domain adaption was performed for the legal domain. A systematic analysis was performed for the three available techniques. The first technique was to use BERT out of box, the second technique was to perform additional pre-training on BERT using domain-specific corpora, and the third approach was to perform pre-training from the start using domain-specific corpora.


\section{Dataset Description}
The dataset provided by the organizers was selected from the domain of the U.S. civil procedure and is based on a book aimed at law students.

In the training set, there are 666 instances, out of which 505 are labeled as 0 and 161 are labeled as 1. For each instance in the training data, there is a general introduction to a case, a question from that case, a possible argument solution along with a detailed analysis of why the argument is valid for that case. The test set, on the other hand, contains a question, answer and an explanation on the basis of which a label needs to be assigned to each instance. The assigned label will indicate whether or not the answer to the question is right or not.


\section{Methodology} 

It was observed that in the dataset provided by the organizers, the number of instances in class 0 was 505, while the number of instances labeled as 1 was 161. Hence, in order to address the data imbalance, minority sampling was performed by randomly picking 161 instances from those labeled as class 0. This ensured that no bias existed in the trained model. 

For identifying whether the answer to a given problem was correct or not, the RoBERTa Large model was employed. The RoBERTa model was designed by Facebook AI in 2019 \cite{liu2019roberta}. RoBERTa is a pre-trained transformer model which was trained in a self-supervised manner, i.e. only raw texts were used to train it without the involvement of human labeling.

While training the model, all the fields present in the training data, namely, question, answer, and analysis, were used to predict the provided label. The weighted Adam optimizer along with cross-entropy loss was used as the optimizer and the loss function respectively. The learning of the optimizer was set at 1e-5. The RoBERTa model was trained for 100 epochs with the aforementioned parameters with a batch size of 8.  

The training procedure has been summarised in Figure \ref{training}. The fine-tuned transformer was used to then predict the label for the unseen data as shown in Figure \ref{testing}.


\begin{table}[]
\caption{RoBERTa Performance Comparison}
\label{roberta_comparision}
\centering
\begin{tabular}{|c|c|c|}
\hline
\textbf{Model} & \textbf{F1-Score} & \textbf{Accuracy} \\ \hline
RoBERTa Base   & 0.5511            & 0.6020            \\ \hline
RoBERTa Large  & 0.6172            & 0.6531            \\ \hline
\end{tabular}
\end{table}


\section{Results and Discussion}
A BERT-based transformer, RoBERTa was discussed to perform categorization of an answer as right or wrong given a case, question, and a possible answer.

The data imbalance was handled by performing under sampling on the majority class instances in a random fashion. This was followed by fine-tuning the RoBERTa Large model for 100 epochs. After fine-tuning, the model achieved an F1 score of 0.5511 and an accuracy of 0.6020. 

As shown in Table \ref{roberta_comparision}, the RoBERTa Base model performed better than RoBERTa Large, when fine-tuned for 100 epochs using the same methodology and hyper parameters. And it achieved an F1 score of 0.6172 and an accuracy of 0.6531.

Overall, we achieved the 13\textsuperscript{th} rank in the Legal Argument Reasoning Task in Civil Procedure shared task at SemEval-2024 out of the 21 participating teams.


\section{Conclusion and Future Work}
Legal argument reasoning is a new NLP task, aimed at classifying a candidate answer as correct or incorrect given an introduction to the topic, a question and a candidate answer.

In this work, we describe our use of a BERT-based architecture, RoBERTa in the Legal Argument Reasoning Task in Civil Procedure shared task at SemEval-2024.

Ensembling techniques have shown promising results on various NLP tasks in different domains. Using an ensemble approach of different transformers may hence improve the performance. Transformers trained specifically with a focus on legal transformation such as Legal-BERT \cite{chalkidis-etal-2020-legal} can improve the performance further.

% Bibliography entries for the entire Anthology, followed by custom entries
%\bibliography{anthology,custom}
% Custom bibliography entries only
\bibliography{custom}

\appendix

\end{document}
