% Use this file for citations not found in the ACL Anthology (contained in "anthology.bib").

@inproceedings{bongard-etal-2022-legal,
	title        = {The Legal Argument Reasoning Task in Civil Procedure},
	author       = {Bongard, Leonard  and Held, Lena  and Habernal, Ivan},
	year         = 2022,
	month        = dec,
	booktitle    = {Proceedings of the Natural Legal Language Processing Workshop 2022},
	publisher    = {Association for Computational Linguistics},
	address      = {Abu Dhabi, United Arab Emirates (Hybrid)},
	pages        = {194--207},
	doi          = {10.18653/v1/2022.nllp-1.17},
	url          = {https://aclanthology.org/2022.nllp-1.17},
	editor       = {Aletras, Nikolaos  and Chalkidis, Ilias  and Barrett, Leslie  and Goan{\textcommabelow{t}}{\u{a}}, C{\u{a}}t{\u{a}}lina  and Preo{\textcommabelow{t}}iuc-Pietro, Daniel}
}

@book{Glannon.2018,
  title={The {G}lannon Guide To Civil Procedure: {L}earning Civil Procedure Through Multiple-Choice Questions and Analysis},
  author={Glannon, Joseph W},
  year={2018},
  edition = {4th},
  publisher={Aspen Publishing},
}

@inproceedings{Fawei.2016,
    title = {Passing a {USA} National Bar Exam: a First Corpus for Experimentation},
    author = "Fawei, Biralatei  and
      Wyner, Adam  and
      Pan, Jeff",
    booktitle = "Proceedings of the Tenth International Conference on Language Resources and Evaluation ({LREC}'16)",
    month = may,
    year = "2016",
    address = "Portoro{\v{z}}, Slovenia",
    publisher = "European Language Resources Association (ELRA)",
    url = "https://aclanthology.org/L16-1538",
    pages = "3373--3378",
}

@inproceedings{Holzenberger.2020,
 author = {Holzenberger, Nils and Blair-Stanek, Andrew and {van Durme}, Benjamin},
 title = {A Dataset for Statutory Reasoning in Tax Law Entailment and Question Answering},
 editor = {{Nikolaos Aletras} and {Ion Androutsopoulos} and {Leslie Barrett} and {Adam Meyers} and {Daniel Preoţiuc-Pietro}},
 booktitle = {{Proceedings of the Natural Legal Language Processing Workshop 2020 co-located with the 26th ACM SIGKDD International Conference on Knowledge Discovery \& Data Mining (KDD 2020)}},
 year = {2020},
 month = {Aug.},
 url = {http://ceur-ws.org/Vol-2645/paper5.pdf}
}

@inproceedings{Kano.2019,
 author = {Kano, Yoshinobu and Kim, Mi-Young and Yoshioka, Masaharu and Lu, Yao and Rabelo, Juliano and Kiyota, Naoki and Goebel, Randy and Satoh, Ken},
 title = {{COLIEE-2018}: {E}valuation of the Competition on Legal Information Extraction and Entailment},
 keywords = {COLIEE},
 pages = {177--192},
 volume = {11717},
 publisher = {{Springer International Publishing}},
 isbn = {978-3-030-31604-4},
 series = {{Lecture Notes in Computer Science}},
 editor = {Kojima, Kazuhiro and Sakamoto, Maki and Mineshima, Koji and Satoh, Ken},
 booktitle = {{New Frontiers in Artificial Intelligence}},
 year = {2019},
 address = {Cham},
 doi = {10.1007/978-3-030-31605-1_14}
}

@article{Rabelo.2022,
 author = {Rabelo, Juliano and Goebel, Randy and Kim, Mi-Young and Kano, Yoshinobu and Yoshioka, Masaharu and Satoh, Ken},
 year = {2022},
 title = {Overview and Discussion of the Competition on Legal Information Extraction/Entailment {(COLIEE)} 2021},
 pages = {111--133},
 volume = {16},
 number = {1},
 issn = {2523-3173},
 journal = {{The Review of Socionetwork Strategies}},
 doi = {10.1007/s12626-022-00105-z}
}

@misc{katzGPT4PassesBar2023a,
  type = {{{SSRN Scholarly Paper}}},
  title = {{GPT-4} Passes the {Bar Exam}},
  author = {Katz, Daniel Martin and Bommarito, Michael James and Gao, Shang and Arredondo, Pablo},
  year = {2023},
  month = mar,
  number = {4389233},
  address = {{Rochester, NY}},
  doi = {10.2139/ssrn.4389233}
}

@misc{guha2023legalbench,
      title={LegalBench: {A} Collaboratively Built Benchmark for Measuring Legal Reasoning in Large Language Models}, 
      author={Neel Guha and Julian Nyarko and Daniel E. Ho and Christopher Ré and Adam Chilton and Aditya Narayana and Alex Chohlas-Wood and Austin Peters and Brandon Waldon and Daniel N. Rockmore and Diego Zambrano and Dmitry Talisman and Enam Hoque and Faiz Surani and Frank Fagan and Galit Sarfaty and Gregory M. Dickinson and Haggai Porat and Jason Hegland and Jessica Wu and Joe Nudell and Joel Niklaus and John Nay and Jonathan H. Choi and Kevin Tobia and Margaret Hagan and Megan Ma and Michael Livermore and Nikon Rasumov-Rahe and Nils Holzenberger and Noam Kolt and Peter Henderson and Sean Rehaag and Sharad Goel and Shang Gao and Spencer Williams and Sunny Gandhi and Tom Zur and Varun Iyer and Zehua Li},
      year={2023},
      eprint={2308.11462},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{gpt4,
      title={{GPT-4} Technical Report}, 
      author={OpenAI},
      year={2023},
      eprint={2303.08774},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{llama,
      title={{LLaMA}: {O}pen and Efficient Foundation Language Models}, 
      author={Hugo Touvron and Thibaut Lavril and Gautier Izacard and Xavier Martinet and Marie-Anne Lachaux and Timothée Lacroix and Baptiste Rozière and Naman Goyal and Eric Hambro and Faisal Azhar and Aurelien Rodriguez and Armand Joulin and Edouard Grave and Guillaume Lample},
      year={2023},
      eprint={2302.13971},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{llama2,
      title={{Llama 2}: {O}pen Foundation and Fine-Tuned Chat Models}, 
      author={Hugo Touvron and Louis Martin and Kevin Stone and Peter Albert and Amjad Almahairi and Yasmine Babaei and Nikolay Bashlykov and Soumya Batra and Prajjwal Bhargava and Shruti Bhosale and Dan Bikel and Lukas Blecher and Cristian Canton Ferrer and Moya Chen and Guillem Cucurull and David Esiobu and Jude Fernandes and Jeremy Fu and Wenyin Fu and Brian Fuller and Cynthia Gao and Vedanuj Goswami and Naman Goyal and Anthony Hartshorn and Saghar Hosseini and Rui Hou and Hakan Inan and Marcin Kardas and Viktor Kerkez and Madian Khabsa and Isabel Kloumann and Artem Korenev and Punit Singh Koura and Marie-Anne Lachaux and Thibaut Lavril and Jenya Lee and Diana Liskovich and Yinghai Lu and Yuning Mao and Xavier Martinet and Todor Mihaylov and Pushkar Mishra and Igor Molybog and Yixin Nie and Andrew Poulton and Jeremy Reizenstein and Rashi Rungta and Kalyan Saladi and Alan Schelten and Ruan Silva and Eric Michael Smith and Ranjan Subramanian and Xiaoqing Ellen Tan and Binh Tang and Ross Taylor and Adina Williams and Jian Xiang Kuan and Puxin Xu and Zheng Yan and Iliyan Zarov and Yuchen Zhang and Angela Fan and Melanie Kambadur and Sharan Narang and Aurelien Rodriguez and Robert Stojnic and Sergey Edunov and Thomas Scialom},
      year={2023},
      eprint={2307.09288},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{jiang2023mistral,
      title={{Mistral 7B}}, 
      author={Albert Q. Jiang and Alexandre Sablayrolles and Arthur Mensch and Chris Bamford and Devendra Singh Chaplot and Diego de las Casas and Florian Bressand and Gianna Lengyel and Guillaume Lample and Lucile Saulnier and Lélio Renard Lavaud and Marie-Anne Lachaux and Pierre Stock and Teven Le Scao and Thibaut Lavril and Thomas Wang and Timothée Lacroix and William El Sayed},
      year={2023},
      eprint={2310.06825},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{jiang2024mixtral,
      title={{Mixtral of Experts}}, 
      author={Albert Q. Jiang and Alexandre Sablayrolles and Antoine Roux and Arthur Mensch and Blanche Savary and Chris Bamford and Devendra Singh Chaplot and Diego de las Casas and Emma Bou Hanna and Florian Bressand and Gianna Lengyel and Guillaume Bour and Guillaume Lample and Lélio Renard Lavaud and Lucile Saulnier and Marie-Anne Lachaux and Pierre Stock and Sandeep Subramanian and Sophia Yang and Szymon Antoniak and Teven Le Scao and Théophile Gervet and Thibaut Lavril and Thomas Wang and Timothée Lacroix and William El Sayed},
      year={2024},
      eprint={2401.04088},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@inproceedings{legalbert,
    title = "{{LEGAL}-{BERT}: {T}he Muppets straight out of Law School}",
    author = "Chalkidis, Ilias  and
      Fergadiotis, Manos  and
      Malakasiotis, Prodromos  and
      Aletras, Nikolaos  and
      Androutsopoulos, Ion",
    editor = "Cohn, Trevor  and
      He, Yulan  and
      Liu, Yang",
    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2020",
    month = nov,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.findings-emnlp.261",
    doi = "10.18653/v1/2020.findings-emnlp.261",
    pages = "2898--2904",
}

@misc{zephyr,
      title={{Zephyr}: {D}irect Distillation of LM Alignment}, 
      author={Lewis Tunstall and Edward Beeching and Nathan Lambert and Nazneen Rajani and Kashif Rasul and Younes Belkada and Shengyi Huang and Leandro von Werra and Clémentine Fourrier and Nathan Habib and Nathan Sarrazin and Omar Sanseviero and Alexander M. Rush and Thomas Wolf},
      year={2023},
      eprint={2310.16944},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@article{Beltagy2020Longformer,
  title={{Longformer}: {T}he Long-Document Transformer},
  author={Iz Beltagy and Matthew E. Peters and Arman Cohan},
  journal={arXiv:2004.05150},
  year={2020},
}

@misc{flant5,
      title={The {Flan} Collection: {D}esigning Data and Methods for Effective Instruction Tuning}, 
      author={Shayne Longpre and Le Hou and Tu Vu and Albert Webson and Hyung Won Chung and Yi Tay and Denny Zhou and Quoc V. Le and Barret Zoph and Jason Wei and Adam Roberts},
      year={2023},
      eprint={2301.13688},
      archivePrefix={arXiv},
      primaryClass={cs.AI}
}

@inproceedings{bigbird,
 author = {Zaheer, Manzil and Guruganesh, Guru and Dubey, Kumar Avinava and Ainslie, Joshua and Alberti, Chris and Ontanon, Santiago and Pham, Philip and Ravula, Anirudh and Wang, Qifan and Yang, Li and Ahmed, Amr},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M.F. Balcan and H. Lin},
 pages = {17283--17297},
 publisher = {Curran Associates, Inc.},
 title = {{Big Bird}: {T}ransformers for Longer Sequences},
 url = {https://proceedings.neurips.cc/paper_files/paper/2020/file/c8512d142a2d849725f31a9a7a361ab9-Paper.pdf},
 volume = {33},
 year = {2020}
}

@misc{roberta,
      title={{RoBERTa}: {A} Robustly Optimized BERT Pretraining Approach}, 
      author={Yinhan Liu and Myle Ott and Naman Goyal and Jingfei Du and Mandar Joshi and Danqi Chen and Omer Levy and Mike Lewis and Luke Zettlemoyer and Veselin Stoyanov},
      year={2019},
      eprint={1907.11692},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@inproceedings{devlin2018bert,
    title = {{BERT}: {P}re-training of Deep Bidirectional Transformers for Language Understanding},
    author = "Devlin, Jacob  and
      Chang, Ming-Wei  and
      Lee, Kenton  and
      Toutanova, Kristina",
    booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",
    month = jun,
    year = "2019",
    address = "Minneapolis, Minnesota",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/N19-1423",
    doi = "10.18653/v1/N19-1423",
    pages = "4171--4186",
}

@inproceedings{Wei-CoT,
	title        = {{Chain-of-Thought} Prompting Elicits Reasoning in Large Language Models},
	author       = {Wei, Jason and Wang, Xuezhi and Schuurmans, Dale and Bosma, Maarten and ichter, brian and Xia, Fei and Chi, Ed and Le, Quoc V and Zhou, Denny},
	year         = 2022,
	booktitle    = {Advances in Neural Information Processing Systems},
	publisher    = {Curran Associates, Inc.},
	volume       = 35,
	pages        = {24824--24837},
	url          = {https://proceedings.neurips.cc/paper_files/paper/2022/file/9d5609613524ecf4f15af0f7b31abca4-Paper-Conference.pdf},
	editor       = {S. Koyejo and S. Mohamed and A. Agarwal and D. Belgrave and K. Cho and A. Oh}
}

@InProceedings{r1,
  author    = {Zhao, Xiaofeng  and  Qiao, Xiaosong  and  Ou, Kaiwen  and  Zhang, Min  and  Chang, Su  and  Piao, Mengyao  and  Li, Yuang  and  Li, Yinglu  and  Zhu, Ming  and  Liu, Yilun  and  Yao, Feiyu  and  tao, shimin  and  Yang, Hao  and  Jiang, Yanfei},
  title     = {{HW-TSC} at {SemEval-2024} Task 5: {S}elf-Eval? A Confident LLM System for Auto Prediction and Evaluation for the Legal Argument Reasoning Task},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {1817--1821},
  abstract  = {In this article, we present an effective system for semeval-2024 task 5. The task involves assessing the feasibility of a given solution in civil litigation cases based on relevant legal provisions, issues, solutions, and analysis. This task demands a high level of proficiency in U.S. law and natural language reasoning. In this task, we designed a self-eval LLM system that simultaneously performs reasoning and self-assessment tasks. We created a confidence interval and a prompt instructing the LLM to output the answer to a question along with its confidence level. We designed a series of experiments to prove the effectiveness of the self-eval mechanism. In order to avoid the randomness of the results, the final result is obtained by voting on three results generated by the GPT-4. Our submission was conducted under zero-resource setting, and we achieved first place in the task with an F1-score of 0.8231 and an accuracy of 0.8673.},
  url       = {https://aclanthology.org/2024.semeval2024-1.253}
}
@InProceedings{r2,
  author    = {Benedetto, Irene  and  Koudounas, Alkis  and  Vaiani, Lorenzo  and  Pastor, Eliana  and  Cagliero, Luca  and  Tarasconi, Francesco},
  title     = {{MAINDZ} at {SemEval-2024} Task 5: {CLUEDO} - {C}hoosing {L}egal o{U}tcome by {E}xplaining {D}ecision through {O}versight},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {986--994},
  abstract  = {Large language models (LLMs) have recently obtained strong performance on complex reasoning tasks. However, their capabilities in specialized domains like law remain relatively unexplored. We present CLUEDO, a system to tackle a novel legal reasoning task that involves determining if a provided answer correctly addresses a legal question derived from U.S. civil procedure cases. CLUEDO utilizes multiple collaborator models that are trained using multiple-choice prompting to choose the right label and generate explanations. These collaborators are overseen by a final "detective" model that identifies the most accurate answer in a zero-shot manner. Our approach achieves an F1 macro score of 0.74 on the development set and 0.76 on the test set, outperforming individual models. Unlike the powerful GPT-4, CLUEDO provides more stable predictions thanks to the ensemble approach. Our results showcase the promise of tailored frameworks to enhance legal reasoning capabilities in LLMs.},
  url       = {https://aclanthology.org/2024.semeval2024-1.142}
}
@InProceedings{r3,
  author    = {Krumov, Kristiyan  and  Boytcheva, Svetla  and  Koytchev, Ivan},
  title     = {{SU-FMI} at {SemEval-2024} Task 5: {F}rom {BERT} Fine-Tuning to LLM Prompt Engineering - Approaches in Legal Argument Reasoning},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {1662--1668},
  abstract  = {This paper presents our approach and findings for SemEval-2024 Task 5, focusing on legal argument reasoning. We explored the effectiveness of fine-tuning pre-trained BERT models and the innovative application of large language models (LLMs) through prompt engineering in the context of legal texts. Our methodology involved a combination of techniques to address the challenges posed by legal language processing, including handling long texts and optimizing natural language understanding (NLU) capabilities for the legal domain. Our contributions were validated by achieving a third-place ranking on the SemEval 2024 Task 5 Leaderboard. The results underscore the potential of LLMs and prompt engineering in enhancing legal reasoning tasks, offering insights into the evolving landscape of NLU technologies within the legal field.},
  url       = {https://aclanthology.org/2024.semeval2024-1.233}
}
@InProceedings{r5,
  author    = {Schumacher, Dan  and  Rios, Anthony},
  title     = {{Team UTSA-NLP} at {SemEval 2024} Task 5: {P}rompt Ensembling for Argument Reasoning in Civil Procedures with {GPT4}},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {1283--1290},
  abstract  = {In this paper, we present our system for the SemEval Task 5, The Legal Argument Reasoning Task in Civil Procedure Challenge. Legal argument reasoning is an essential skill that all law students must master. Moreover, it is important to develop natural language processing solutions that can reason about a question given terse domain-specific contextual information. Our system explores a prompt-based solution using GPT4 to reason over legal arguments. We also evaluate an ensemble of prompting strategies, including chain-of-thought reasoning and in-context learning. Overall, our system results in a Macro F1 of .8095 on the validation dataset and .7315 (5th out of 21 teams) on the final test set. Code for this project is available at https://github.com/danschumac1/CivilPromptReasoningGPT4.},
  url       = {https://aclanthology.org/2024.semeval2024-1.185}
}
@InProceedings{r7,
  author    = {Pahilajani, Anish  and  Jain, Samyak Rajesh  and  Trivedi, Devasha},
  title     = {{NLP at UC Santa Cruz} at {SemEval-2024} Task 5: {L}egal Answer Validation using Few-Shot Multi-Choice {QA}},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {1298--1303},
  abstract  = {This paper presents our submission to the SemEval 2024 Task 5: The Legal Argument Reasoning Task in Civil Procedure. We present two approaches to solving the task of legal answer validation, given an introduction to the case, a question and an answer candidate. Firstly, we fine-tuned pre-trained BERT-based models and found that models trained on domain knowledge perform better. Secondly, we performed few-shot prompting on GPT models and found that reformulating the answer validation task to be a multiple-choice QA task remarkably improves the performance of the model. Our best submission is a BERT-based model that achieved the 7th place out of 20.},
  url       = {https://aclanthology.org/2024.semeval2024-1.187}
}
@InProceedings{r9,
  author    = {Lu, Yu-An  and  Kao, Hung-Yu},
  title     = {{0x.Yuan} at {SemEval-2024} Task 5: {E}nhancing Legal Argument Reasoning with Structured Prompts},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {385--390},
  abstract  = {The intersection of legal reasoning and Natural Language Processing (NLP) technologies, particularly Large Language Models (LLMs), offers groundbreaking potential for augmenting human capabilities in the legal domain. This paper presents our approach and findings from participating in SemEval-2024 Task 5, focusing on the effect of argument reasoning in civil procedures using legal reasoning prompts. We investigated the impact of structured legal reasoning methodologies, including TREACC, IRAC, IRAAC, and MIRAC, on guiding LLMs to analyze and evaluate legal arguments systematically. Our experimental setup involved crafting specific prompts based on these methodologies to instruct the LLM to dissect and scrutinize legal cases, aiming to discern the cogency of argumentative solutions within a zero-shot learning framework. The performance of our approach, as measured by F1 score and accuracy, demonstrated the efficacy of integrating structured legal reasoning into LLMs for legal analysis. The findings underscore the promise of LLMs, when equipped with legal reasoning prompts, in enhancing their ability to process and reason through complex legal texts, thus contributing to the broader application of AI in legal studies and practice.},
  url       = {https://aclanthology.org/2024.semeval2024-1.60}
}

@InProceedings{r10,
  author    = {Shi, Peng  and  Wang, Jin  and  Zhang, Xuejie},
  title     = {{YNU-HPCC} at {SemEval-2024} Task 5: {R}egularized {Legal-BERT} for Legal Argument Reasoning Task in Civil Procedure},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {743--748},
  abstract  = {This paper describes the submission of team YNU-HPCC to SemEval-2024 for Task 5: The Legal Argument Reasoning Task in Civil Procedure. The task asks candidates the topic, questions, and answers, classifying whether a given candidate's answer is correct (True) or incorrect (False). To make a sound judgment, we propose a system. This system is based on fine-tuning the Legal-BERT model that specializes in solving legal problems. Meanwhile,Regularized Dropout (R-Drop) and focal Loss were used in the model. R-Drop is used for data augmentation, and focal loss addresses data imbalances. Our system achieved relatively good results on the competition's official leaderboard. The code of this paper is available at https://github.com/YNU-PengShi/SemEval-2024-Task5.},
  url       = {https://aclanthology.org/2024.semeval2024-1.106}
}
@InProceedings{r11,
  author    = {Siino, Marco},
  title     = {{Mistral} at {SemEval-2024} Task 5: {{M}istral 7B} for argument reasoning in Civil Procedure},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {155--162},
  abstract  = {At the SemEval-2024 Task 5, the organizers introduce a novel natural language processing (NLP) challenge and dataset within the realm of the United States civil procedure. Each datum within the dataset comprises a comprehensive overview of a legal case, a specific inquiry associated with it, and a potential argument in support of a solution, supplemented with an in-depth rationale elucidating the applicability of the argument within the given context. Derived from a text designed for legal education purposes, this dataset presents a multifaceted benchmarking task for contemporary legal language models. Our manuscript delineates the approach we adopted for participation in this competition. Specifically, we detail the use of a Mistral 7B model to answer the question provided. Our only and best submission reach an F1-score equal to 0.5597 and an Accuracy of 0.5714, outperforming the baseline provided for the task.},
  url       = {https://aclanthology.org/2024.semeval2024-1.24}
}
@InProceedings{r13,
  author    = {Singhal, Kriti  and  Bedi, Jatin},
  title     = {{Transformers} at {SemEval-2024} Task 5: {L}egal Argument Reasoning Task in Civil Procedure using {RoBERTa}},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {956--959},
  abstract  = {Legal argument reasoning task in civil procedure is a new NLP task utilizing a dataset from the domain of the U.S. civil procedure. The task aims at identifying whether the solution to a question in the legal domain is correct or not. This paper describes the team "Transformers" submission to the Legal Argument Reasoning Task in Civil Procedure shared task at SemEval-2024 Task 5. We use a BERT-based architecture for the shared task. The highest F1-score score and accuracy achieved was 0.6172 and 0.6531 respectively. We secured the 13th rank in the Legal Argument Reasoning Task in Civil Procedure shared task.},
  url       = {https://aclanthology.org/2024.semeval2024-1.138}
}
@InProceedings{r14,
  author    = {Sun, Binjie  and  Zhou, Xiaobing},
  title     = {{ignore} at {SemEval-2024} Task 5: {A} Legal Classification Model with Summary Generation and Contrastive Learning},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {517--522},
  abstract  = {This paper describes our work for SemEval-2024 Task 5: The Legal Argument Reasoning Task in Civil Procedure. After analyzing the task requirements and the training dataset, we used data augmentation, adopted the large model GPT for summary generation, and added supervised contrastive learning to the basic BERT model. Our system achieved an F1 score of 0.551, ranking 14th in the competition leaderboard. Our system achieves an F1 score improvement of 0.1241 over the official baseline model.},
  url       = {https://aclanthology.org/2024.semeval2024-1.78}
}


@InProceedings{r15,
  author    = {Chlapanis, Odysseas  and  Androutsopoulos, Ion  and  Galanis, Dimitrios},
  title     = {{Archimedes-AUEB} at {SemEval-2024} Task 5: {LLM} explains Civil Procedure},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {1617--1632},
  abstract  = {The SemEval task on Argument Reasoning in Civil Procedure is challenging in that it requires understanding legal concepts and inferring complex arguments. Currently, most Large Language Models (LLM) excelling in the legal realm are principally purposed for classification tasks, hence their reasoning rationale is subject to contention. The approach we advocate involves using a powerful teacher-LLM (ChatGPT) to extend the training dataset with explanations and generate synthetic data. The resulting data are then leveraged to fine-tune a small student-LLM. Contrary to previous work, our explanations are not directly derived from the teacher's internal knowledge. Instead they are grounded in authentic human analyses, therefore delivering a superior reasoning signal. Additionally, a new `mutation' method generates artificial data instances inspired from existing ones. We are publicly releasing the explanations as an extension to the original dataset, along with the synthetic dataset and the prompts that were used to generate both. Our system ranked 15th in the SemEval competition. It outperforms its own teacher and can produce explanations aligned with the original human analyses, as verified by legal experts.},
  url       = {https://aclanthology.org/2024.semeval2024-1.227}
}

@InProceedings{r16,
  author    = {Prabhu, M Manvith  and  Srinivasa, Haricharana  and  M, Anand Kumar},
  title     = {{SCaLAR NITK} at {SemEval-2024} Task 5: {T}owards Unsupervised Question Answering system with Multi-level Summarization for Legal Text},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {193--199},
  abstract  = {This paper summarizes Team SCaLAR's work on SemEval-2024 Task 5: Legal Argument Reasoning in Civil Procedure. To address this Binary Classification task, which was daunting due to the complexity of the Legal Texts involved, we propose a simple yet novel similarity and distance-based unsupervised approach to generate labels. Further, we explore the Multi-level fusion of Legal-Bert embeddings using ensemble features, including CNN, GRU, and LSTM. To address the lengthy nature of Legal explanation in the dataset, we introduce T5-based segment-wise summarization, which successfully retained crucial information, enhancing the model's performance. Our unsupervised system witnessed a 20-point increase in macro F1-score on the development set and a 10-point increase on the test set, which is promising given its uncomplicated architecture.},
  url       = {https://aclanthology.org/2024.semeval2024-1.30}
}

@InProceedings{r17,
  author    = {Sabzevari, Hoorieh  and  Rostamkhani, Mohammadmostafa  and  Eetemadi, Sauleh},
  title     = {{eagerlearners} at {SemEval2024} Task 5: {T}he Legal Argument Reasoning Task in Civil Procedure},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {909--913},
  abstract  = {This study investigates the performance of the zero-shot method in classifying data using three large language models, alongside two models with large input token sizes and the two pre-trained models on legal data. Our main dataset comes from the domain of U.S. civil procedure. It includes summaries of legal cases, specific questions, potential answers, and detailed explanations for why each solution is relevant, all sourced from a book aimed at law students. By comparing different methods, we aimed to understand how effectively they handle the complexities found in legal datasets. Our findings show how well the zero-shot method of large language models can understand complicated data. We achieved our highest F1 score of 64\% in these experiments.},
  url       = {https://aclanthology.org/2024.semeval2024-1.131}
}
@InProceedings{r18,
  author    = {Maslaris, Ioannis  and  Arampatzis, Avi},
  title     = {{DUTh} at {SemEval 2024} Task 5: {A} multi-task learning approach for the Legal Argument Reasoning Task in Civil Procedure},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {1042--1046},
  abstract  = {Text-generative models have proven to be good reasoners. Although reasoning abilities are mostly observed in larger language models, a number of strategies try to transfer this skill to smaller language models. This paper presents our approach to SemEval 2024 Task-5: The Legal Argument Reasoning Task in Civil Procedure. This shared task aims to develop a system that efficiently handles a multiple-choice question-answering task in the context of the US civil procedure domain. The dataset provides a human-generated rationale for each answer. Given the complexity of legal issues, this task certainly challenges the reasoning abilities of LLMs and AI systems in general. Our work explores fine-tuning an LLM as a correct/incorrect answer classifier. In this context, we are making use of multi-task learning to incorporate the rationales into the fine-tuning process.},
  url       = {https://aclanthology.org/2024.semeval2024-1.150}
}
