
@inproceedings{jullien-etal-2023-semeval,
    title = "{S}em{E}val-2023 Task 7: Multi-Evidence Natural Language Inference for Clinical Trial Data",
    author = {Jullien, Ma{\"e}l  and
      Valentino, Marco  and
      Frost, Hannah  and
      O{'}regan, Paul  and
      Landers, Donal  and
      Freitas, Andr{\'e}},
    editor = {Ojha, Atul Kr.  and
      Do{\u{g}}ru{\"o}z, A. Seza  and
      Da San Martino, Giovanni  and
      Tayyar Madabushi, Harish  and
      Kumar, Ritesh  and
      Sartori, Elisa},
    booktitle = "Proceedings of the 17th International Workshop on Semantic Evaluation (SemEval-2023)",
    month = jul,
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.semeval-1.307",
    doi = "10.18653/v1/2023.semeval-1.307",
    pages = "2216--2226",
    abstract = "This paper describes the results of SemEval 2023 task 7 {--} Multi-Evidence Natural Language Inference for Clinical Trial Data (NLI4CT) {--} consisting of 2 tasks, a Natural Language Inference (NLI) task, and an evidence selection task on clinical trial data. The proposed challenges require multi-hop biomedical and numerical reasoning, which are of significant importance to the development of systems capable of large-scale interpretation and retrieval of medical evidence, to provide personalized evidence-based care. Task 1, the entailment task, received 643 submissions from 40 participants, and Task 2, the evidence selection task, received 364 submissions from 23 participants. The tasks are challenging, with the majority of submitted systems failing to significantly outperform the majority class baseline on the entailment task, and we observe significantly better performance on the evidence selection task than on the entailment task. Increasing the number of model parameters leads to a direct increase in performance, far more significant than the effect of biomedical pre-training. Future works could explore the limitations of large models for generalization and numerical inference, and investigate methods to augment clinical datasets to allow for more rigorous testing and to facilitate fine-tuning. We envisage that the dataset, models, and results of this task will be useful to the biomedical NLI and evidence retrieval communities. The dataset, competition leaderboard, and website are publicly available.",
}

@inproceedings{jullien-etal-2023-nli4ct,
    title = "{NLI}4{CT}: Multi-Evidence Natural Language Inference for Clinical Trial Reports",
    author = "Jullien, Mael  and
      Valentino, Marco  and
      Frost, Hannah  and
      O{'}Regan, Paul  and
      Landers, D{\'o}nal  and
      Freitas, Andre",
    editor = "Bouamor, Houda  and
      Pino, Juan  and
      Bali, Kalika",
    booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing",
    month = dec,
    year = "2023",
    address = "Singapore",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.emnlp-main.1041",
    doi = "10.18653/v1/2023.emnlp-main.1041",
    pages = "16745--16764",
    abstract = "How can we interpret and retrieve medical evidence to support clinical decisions? Clinical trial reports (CTR) amassed over the years contain indispensable information for the development of personalized medicine. However, it is practically infeasible to manually inspect over 400,000+ clinical trial reports in order to find the best evidence for experimental treatments. Natural Language Inference (NLI) offers a potential solution to this problem, by allowing the scalable computation of textual entailment. However, existing NLI models perform poorly on biomedical corpora, and previously published datasets fail to capture the full complexity of inference over CTRs. In this work, we present a novel resource to advance research on NLI for reasoning on CTRs. The resource includes two main tasks. Firstly, to determine the inference relation between a natural language statement, and a CTR. Secondly, to retrieve supporting facts to justify the predicted relation. We provide NLI4CT, a corpus of 2400 statements and CTRs, annotated for these tasks. Baselines on this corpus expose the limitations of existing NLI approaches, with 6 state-of-the-art NLI models achieving a maximum F1 score of 0.627. To the best of our knowledge, we are the first to design a task that covers the interpretation of full CTRs. To encourage further work on this challenging dataset, we make the corpus, competition leaderboard, and website, available on CodaLab, and code to replicate the baseline experiments on GitHub.",
}
@article{percha2022natural,
  title={Natural language inference for curation of structured clinical registries from unstructured text},
  author={Percha, Bethany and Pisapati, Kereeti and Gao, Cynthia and Schmidt, Hank},
  journal={Journal of the American Medical Informatics Association},
  volume={29},
  number={1},
  pages={97--108},
  year={2022},
  publisher={Oxford University Press}
}
@article{percha2021natural,
  title={Natural language inference for clinical registry curation},
  author={Percha, Bethany and Pisapati, Kereeti and Gao, Cynthia and Schmidt, Hank},
  journal={medRxiv},
  pages={2021--06},
  year={2021},
  publisher={Cold Spring Harbor Laboratory Press}
}
@article{mustafa2024enhancing,
  title={Enhancing Robustness in Biomedical NLI Models: A Probing Approach for Clinical Trials},
  author={Mustafa, Ata},
  journal={arXiv preprint arXiv:2402.02558},
  year={2024}
}
@inproceedings{jullien-etal-2024-semeval,
    title = "{S}em{E}val-2024 Task 2: Safe Biomedical Natural Language Inference for Clinical Trials",
    author = {Jullien, Ma{\"e}l  and
      Valentino, Marco  and
      Freitas, Andr{\'e}},
    booktitle = "Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)",
    year = "2024",
    publisher = "Association for Computational Linguistics",
}
@article{zhou2023thifly,
  title={THiFLY Research at SemEval-2023 Task 7: A Multi-granularity System for CTR-based Textual Entailment and Evidence Retrieval},
  author={Zhou, Yuxuan and Jin, Ziyu and Li, Meiwei and Li, Miao and Liu, Xien and You, Xinxin and Wu, Ji},
  journal={arXiv preprint arXiv:2306.01245},
  year={2023}
}
@inproceedings{kanakarajan,
  title={Saama AI Research at SemEval-2023 Task 7: Exploring the Capabilities of Flan-T5 for Multi-evidence Natural Language Inference in Clinical Trial Data},
  author={Kanakarajan, Kamal Raj and Sankarasubbu, Malaikannan},
  booktitle={Proceedings of the 17th International Workshop on Semantic Evaluation (SemEval-2023)},
  pages={995--1003},
  year={2023}
}
@article{vladika2023sebis,
  title={Sebis at SemEval-2023 Task 7: A Joint System for Natural Language Inference and Evidence Retrieval from Clinical Trial Reports},
  author={Vladika, Juraj and Matthes, Florian},
  journal={arXiv preprint arXiv:2304.13180},
  year={2023}
}
@inproceedings{chen2023ncuee,
  title={NCUEE-NLP at SemEval-2023 Task 7: Ensemble Biomedical LinkBERT Transformers in Multi-evidence Natural Language Inference for Clinical Trial Data},
  author={Chen, Chao-Yi and Tien, Kao-Yuan and Cheng, Yuan-Hao and Lee, Lung-Hao},
  booktitle={Proceedings of the 17th International Workshop on Semantic Evaluation (SemEval-2023)},
  pages={776--781},
  year={2023}
}
@inproceedings{wang2023knowcomp,
  title={Knowcomp at semeval-2023 task 7: Fine-tuning pre-trained language models for clinical trial entailment identification},
  author={Wang, Weiqi and Xu, Baixuan and Fang, Tianqing and Zhang, Lirong and Song, Yangqiu},
  booktitle={Proceedings of the 17th International Workshop on Semantic Evaluation (SemEval-2023)},
  pages={1--9},
  year={2023}
}
@inproceedings{pahwa2023bphigh,
  title={BpHigh at SemEval-2023 Task 7: Can Fine-tuned Cross-encoders Outperform GPT-3.5 in NLI Tasks on Clinical Trial Data?},
  author={Pahwa, Bhavish and Pahwa, Bhavika},
  booktitle={Proceedings of the 17th International Workshop on Semantic Evaluation (SemEval-2023)},
  pages={1936--1944},
  year={2023}
}
@inproceedings{dias2023team,
  title={Team INF-UFRGS at SemEval-2023 Task 7: Supervised Contrastive Learning for Pair-level Sentence Classification and Evidence Retrieval},
  author={Dias, Abel Corr{\^e}a and Dias, Filipe and Moreira, Higor and Moreira, Viviane and Comba, Jo{\~a}o Luiz},
  booktitle={Proceedings of the 17th International Workshop on Semantic Evaluation (SemEval-2023)},
  pages={700--706},
  year={2023}
}
@inproceedings{vassileva2023fmi,
  title={FMI-SU at SemEval-2023 Task 7: Two-level Entailment Classification of Clinical Trials Enhanced by Contextual Data Augmentation},
  author={Vassileva, Sylvia and Grazhdanski, Georgi and Boytcheva, Svetla and Koychev, Ivan},
  booktitle={Proceedings of the 17th International Workshop on Semantic Evaluation (SemEval-2023)},
  pages={1454--1462},
  year={2023}
}
@inproceedings{volosincu2023fii,
  title={FII SMART at SemEval 2023 Task7: Multi-evidence Natural Language Inference for Clinical Trial Data},
  author={Volosincu, Mihai and Lupu, Cosmin and Trandabat, Diana and Gifu, Daniela},
  booktitle={Proceedings of the 17th International Workshop on Semantic Evaluation (SemEval-2023)},
  pages={212--220},
  year={2023}
}
@inproceedings{wu2019wtmed,
  title={WTMED at MEDIQA 2019: a hybrid approach to biomedical natural language inference},
  author={Wu, Zhaofeng and Song, Yan and Huang, Sicong and Tian, Yuanhe and Xia, Fei},
  booktitle={Proceedings of the 18th BioNLP Workshop and Shared Task},
  pages={415--426},
  year={2019}
}
@article{hu2021lora,
  title={Lora: Low-rank adaptation of large language models},
  author={Hu, Edward J and Shen, Yelong and Wallis, Phillip and Allen-Zhu, Zeyuan and Li, Yuanzhi and Wang, Shean and Wang, Lu and Chen, Weizhu},
  journal={arXiv preprint arXiv:2106.09685},
  year={2021}
}

@article{2020t5,
  author  = {Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu},
  title   = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer},
  journal = {Journal of Machine Learning Research},
  year    = {2020},
  volume  = {21},
  number  = {140},
  pages   = {1-67},
  url     = {http://jmlr.org/papers/v21/20-074.html}
}