% Use this file for citations not found in the ACL Anthology (contained in "anthology.bib").

@book{Aho:72,
    author  = {Alfred V. Aho and Jeffrey D. Ullman},
    title   = {The Theory of Parsing, Translation and Compiling},
    year    = "1972",
    volume  = "1",
    publisher = {Prentice-Hall},
    address = {Englewood Cliffs, NJ}
}

@book{APA:83,
    author  = {{American Psychological Association}},
    title   = {Publications Manual},
    year    = "1983",
    publisher = {American Psychological Association},
    address = {Washington, DC}
}

@article{Chandra:81,
	author = {Ashok K. Chandra and Dexter C. Kozen and Larry J. Stockmeyer},
	year = "1981",
	title = {Alternation},
	journal = {Journal of the Association for Computing Machinery},
	volume = "28",
	number = "1",
	pages = "114--133",
	doi = "10.1145/322234.322243",
}

@inproceedings{andrew2007scalable,
  title={Scalable training of {L1}-regularized log-linear models},
  author={Andrew, Galen and Gao, Jianfeng},
  booktitle={Proceedings of the 24th International Conference on Machine Learning},
  pages={33--40},
  year={2007},
}

@book{Gusfield:97,
    author  = {Dan Gusfield},
    title   = {Algorithms on Strings, Trees and Sequences},
    year    = "1997",
    publisher = {Cambridge University Press},
    address = {Cambridge, UK}
}

@article{rasooli-tetrault-2015,
    author    = {Mohammad Sadegh Rasooli and Joel R. Tetreault},
    title     = {Yara Parser: {A} Fast and Accurate Dependency Parser},
    journal   = {Computing Research Repository},
    volume    = {arXiv:1503.06733},
    year      = {2015},
    url       = {http://arxiv.org/abs/1503.06733},
    note    = {version 2}
}

@article{Ando2005,
	Acmid = {1194905},
	Author = {Ando, Rie Kubota and Zhang, Tong},
	Issn = {1532-4435},
	Issue_Date = {12/1/2005},
	Journal = {Journal of Machine Learning Research},
	Month = dec,
	Numpages = {37},
	Pages = {1817--1853},
	Publisher = {JMLR.org},
	Title = {A Framework for Learning Predictive Structures from Multiple Tasks and Unlabeled Data},
	Volume = {6},
	Year = {2005}
}

@misc{gu2025surveyllmasajudge,
      title={A Survey on LLM-as-a-Judge}, 
      author={Jiawei Gu and Xuhui Jiang and Zhichao Shi and Hexiang Tan and Xuehao Zhai and Chengjin Xu and Wei Li and Yinghan Shen and Shengjie Ma and Honghao Liu and Saizhuo Wang and Kun Zhang and Yuanzhuo Wang and Wen Gao and Lionel Ni and Jian Guo},
      year={2025},
      eprint={2411.15594},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2411.15594}, 
}

@misc{bavaresco2024llmsinsteadhumanjudges,
      title={LLMs instead of Human Judges? A Large Scale Empirical Study across 20 NLP Evaluation Tasks}, 
      author={Anna Bavaresco and Raffaella Bernardi and Leonardo Bertolazzi and Desmond Elliott and Raquel Fernández and Albert Gatt and Esam Ghaleb and Mario Giulianelli and Michael Hanna and Alexander Koller and André F. T. Martins and Philipp Mondorf and Vera Neplenbroek and Sandro Pezzelle and Barbara Plank and David Schlangen and Alessandro Suglia and Aditya K Surikuchi and Ece Takmaz and Alberto Testoni},
      year={2024},
      eprint={2406.18403},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2406.18403}, 
}

@misc{hosseini2024benchmarklongformmedicalquestion,
      title={A Benchmark for Long-Form Medical Question Answering}, 
      author={Pedram Hosseini and Jessica M. Sin and Bing Ren and Bryceton G. Thomas and Elnaz Nouri and Ali Farahanchi and Saeed Hassanpour},
      year={2024},
      eprint={2411.09834},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2411.09834}, 
}

@misc{szymanski2024limitationsllmasajudgeapproachevaluating,
      title={Limitations of the LLM-as-a-Judge Approach for Evaluating LLM Outputs in Expert Knowledge Tasks}, 
      author={Annalisa Szymanski and Noah Ziems and Heather A. Eicher-Miller and Toby Jia-Jun Li and Meng Jiang and Ronald A. Metoyer},
      year={2024},
      eprint={2410.20266},
      archivePrefix={arXiv},
      primaryClass={cs.HC},
      url={https://arxiv.org/abs/2410.20266}, 
}

@misc{han2024medsafetybenchevaluatingimprovingmedical,
      title={MedSafetyBench: Evaluating and Improving the Medical Safety of Large Language Models}, 
      author={Tessa Han and Aounon Kumar and Chirag Agarwal and Himabindu Lakkaraju},
      year={2024},
      eprint={2403.03744},
      archivePrefix={arXiv},
      primaryClass={cs.AI},
      url={https://arxiv.org/abs/2403.03744}, 
}

@misc{kanithi2024mediccomprehensiveframeworkevaluating,
      title={MEDIC: Towards a Comprehensive Framework for Evaluating LLMs in Clinical Applications}, 
      author={Praveen K Kanithi and Clément Christophe and Marco AF Pimentel and Tathagata Raha and Nada Saadi and Hamza Javed and Svetlana Maslenkova and Nasir Hayat and Ronnie Rajan and Shadab Khan},
      year={2024},
      eprint={2409.07314},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2409.07314}, 
}

@misc{krolik2024leveraginglargelanguagemodels,
      title={Towards Leveraging Large Language Models for Automated Medical Q\&A Evaluation}, 
      author={Jack Krolik and Herprit Mahal and Feroz Ahmad and Gaurav Trivedi and Bahador Saket},
      year={2024},
      eprint={2409.01941},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2409.01941}, 
}

@misc{zheng2023judgingllmasajudgemtbenchchatbot,
      title={Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena}, 
      author={Lianmin Zheng and Wei-Lin Chiang and Ying Sheng and Siyuan Zhuang and Zhanghao Wu and Yonghao Zhuang and Zi Lin and Zhuohan Li and Dacheng Li and Eric P. Xing and Hao Zhang and Joseph E. Gonzalez and Ion Stoica},
      year={2023},
      eprint={2306.05685},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2306.05685}, 
}

@article{guyon-elisseeff-03,
  title   = "An Introduction to Variable and Feature Selection",
  author  = "I. Guyon and A. Elisseeff",
  journal = "JMLR",
  volume  = "3",
  month   = MAR,
  pages   = "1157-1182",
  year    = 2003
}

@article{van2017effect,
  title={The effect of Dr Google on doctor--patient encounters in primary care: a quantitative, observational, cross-sectional study},
  author={Van Riel, Noor and Auwerx, Koen and Debbaut, Pieterjan and Van Hees, Sanne and Schoenmakers, Birgitte},
  journal={BJGP open},
  volume={1},
  number={2},
  year={2017},
  publisher={Royal College of General Practitioners}
}

@article{lucas2024reasoning,
  title={Reasoning with large language models for medical question answering},
  author={Lucas, Mary M and Yang, Justin and Pomeroy, Jon K and Yang, Christopher C},
  journal={Journal of the American Medical Informatics Association},
  volume={31},
  number={9},
  pages={1964--1975},
  year={2024},
  publisher={Oxford University Press}
}

@article{cocco2018dr,
  title={Dr Google in the ED: searching for online health information by adult emergency department patients},
  author={Cocco, Anthony M and Zordan, Rachel and Taylor, David McD and Weiland, Tracey J and Dilley, Stuart J and Kant, Joyce and Dombagolla, Mahesha and Hendarto, Andreas and Lai, Fiona and Hutton, Jennie},
  journal={Medical Journal of Australia},
  volume={209},
  number={8},
  pages={342--347},
  year={2018},
  publisher={Wiley Online Library}
}

@techreport{guyon2007causalreport,
  author      = {I. Guyon and C. Aliferis and A. Elisseeff},
  title       = {Causal Feature Selection},
  institution = {Clopinet},
  year        = 2007,
  type        = {Technical Report },
  source      = {\url{http://clopinet.com/isabelle/Papers/causalFS.pdf}}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@article{tan2024proposed,
  title={A Proposed SCORE Evaluation Framework for Large Language Models: Safety, Consensus, Objectivity, Reproducibility and Explainability},
  author={Tan, Ting Fang and Elangovan, Kabilan and Ong, Jasmine and Shah, Nigam and Sung, Joseph and Wong, Tien Yin and Xue, Lan and Liu, Nan and Wang, Haibo and Kuo, Chang Fu and others},
  journal={arXiv preprint arXiv:2407.07666},
  year={2024}
}


@inproceedings{lin2004rouge,
  title={Rouge: A package for automatic evaluation of summaries},
  author={Lin, Chin-Yew},
  booktitle={Text summarization branches out},
  pages={74--81},
  year={2004}
}

@inproceedings{papineni2002bleu,
  title={Bleu: a method for automatic evaluation of machine translation},
  author={Papineni, Kishore and Roukos, Salim and Ward, Todd and Zhu, Wei-Jing},
  booktitle={Proceedings of the 40th annual meeting of the Association for Computational Linguistics},
  pages={311--318},
  year={2002}
}

@inproceedings{zhang2020bertscore,
  title={BERTScore: Evaluating Text Generation with BERT},
  author={Tianyi Zhang and Varsha Kishore and Felix Wu and Kilian Q. Weinberger and Yoav Artzi},
  booktitle={International Conference on Learning Representations},
  year={2020},
  url={https://openreview.net/forum?id=SkeHuCVFDr}
}

@inproceedings{hao2024llmsyn,
  title={LLMSYN: Generating Synthetic Electronic Health Records Without Patient-Level Data},
  author={Hao, Yijie and He, Huan and Ho, Joyce C},
  booktitle = {Proceedings of the 9th Machine Learning for Healthcare Conference},
  year={2024},
  organization={PMLR}
}

@inproceedings{shao2023synthetic,
  title={Synthetic prompting: Generating chain-of-thought demonstrations for large language models},
  author={Shao, Zhihong and Gong, Yeyun and Shen, Yelong and Huang, Minlie and Duan, Nan and Chen, Weizhu},
  booktitle={International Conference on Machine Learning},
  pages={30706--30775},
  year={2023},
  organization={PMLR}
}

@inproceedings{moon2024generative,
  title={Generative Interpretation: Toward Human-Like Evaluation for Educational Question-Answer Pair Generation},
  author={Moon, Hyeonseok and Lee, Jaewook and Eo, Sugyeong and Park, Chanjun and Seo, Jaehyung and Lim, Heui-Seok},
  booktitle={Findings of the Association for Computational Linguistics: EACL 2024},
  pages={2185--2196},
  year={2024}
}

@inproceedings{ji2023towards,
  title={Towards mitigating LLM hallucination via self reflection},
  author={Ji, Ziwei and Yu, Tiezheng and Xu, Yan and Lee, Nayeon and Ishii, Etsuko and Fung, Pascale},
  booktitle={Findings of the Association for Computational Linguistics: EMNLP 2023},
  pages={1827--1843},
  year={2023}
}

@article{zakka2024almanac,
  title={Almanac—retrieval-augmented language models for clinical medicine},
       author = {{Zakka}, Cyril and {Chaurasia}, Akash and {Shad}, Rohan and {Dalal}, Alex R. and {Kim}, Jennifer L. and {Moor}, Michael and {Alexander}, Kevin and {Ashley}, Euan and {Boyd}, Jack and {Boyd}, Kathleen and {Hirsch}, Karen and {Langlotz}, Curt and {Nelson}, Joanna and {Hiesinger}, William},
  journal={NEJM AI},
  volume={1},
  number={2},
  pages={AIoa2300068},
  year={2024},
  publisher={Massachusetts Medical Society}
}

@article{singhal2023large,
  title={Large language models encode clinical knowledge},
  author={Singhal, Karan and Azizi, Shekoofeh and Tu, Tao and Mahdavi, S Sara and Wei, Jason and Chung, Hyung Won and Scales, Nathan and Tanwani, Ajay and Cole-Lewis, Heather and Pfohl, Stephen and others},
  journal={Nature},
  volume={620},
  number={7972},
  pages={172--180},
  year={2023},
  publisher={Nature Publishing Group}
}


@inproceedings{nguyenMedRedQAMedicalConsumer2023,
  title = {{{MedRedQA}} for {{Medical Consumer Question Answering}}: {{Dataset}}, {{Tasks}}, and {{Neural Baselines}}},
  shorttitle = {{{MedRedQA}} for {{Medical Consumer Question Answering}}},
  booktitle = {Proceedings of the 13th {{International Joint Conference}} on {{Natural Language Processing}} and the 3rd {{Conference}} of the {{Asia-Pacific Chapter}} of the {{Association}} for {{Computational Linguistics}} ({{Volume}} 1: {{Long Papers}})},
  author = {Nguyen, Vincent and Karimi, Sarvnaz and Rybinski, Maciej and Xing, Zhenchang},
  editor = {Park, Jong C. and Arase, Yuki and Hu, Baotian and Lu, Wei and Wijaya, Derry and Purwarianti, Ayu and Krisnadhi, Adila Alfa},
  year = {2023},
  month = nov,
  pages = {629--648},
  publisher = {Association for Computational Linguistics},
  address = {Nusa Dua, Bali},
  doi = {10.18653/v1/2023.ijcnlp-main.42},
  urldate = {2024-07-24}
}

@misc{reginMedicalQuestionAnswer2017,
  author = {Regin, Lasse},
  title = {Medical Question Answer Data},
  year = {2017},
  url = {https://github.com/LasseRegin/medical-question-answer-data},
  urldate = {2023-05-15},
  note = {Accessed: May 15, 2023}
}

@inproceedings{LiveMedQA2017,
  author    = {Asma {Ben Abacha} and Eugene Agichtein and Yuval Pinter and Dina Demner{-}Fushman}, 
  title     = {Overview of the Medical Question Answering Task at TREC 2017 LiveQA}, 
  booktitle = {TREC 2017},
  year      = {2017}
} 

@article{MedQUaD_dataset,
  title = {A Question-Entailment Approach to Question Answering},
  author = {Ben Abacha, Asma and {Demner-Fushman}, Dina},
  year = {2019},
  month = oct,
  journal = {BMC Bioinformatics},
  volume = {20},
  number = {1},
  pages = {511},
  issn = {1471-2105},
  doi = {10.1186/s12859-019-3119-4},
  urldate = {2024-09-02},
  keywords = {Consumer Health Questions,Deep Learning,Information Retrieval,Machine Learning,Medical Question-Answer Dataset,Question Answering,Recognizing Question Entailment}
}



@misc{jin2020diseasedoespatienthave,
      title={What Disease does this Patient Have? A Large-scale Open Domain Question Answering Dataset from Medical Exams}, 
      author={Di Jin and Eileen Pan and Nassim Oufattole and Wei-Hung Weng and Hanyi Fang and Peter Szolovits},
      year={2020},
      eprint={2009.13081},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2009.13081}, 
}

@inproceedings{wadhwa-etal-2023-redhot,
    title = "{R}ed{HOT}: A Corpus of Annotated Medical Questions, Experiences, and Claims on Social Media",
    author = "Wadhwa, Somin  and
      Khetan, Vivek  and
      Amir, Silvio  and
      Wallace, Byron",
    editor = "Vlachos, Andreas  and
      Augenstein, Isabelle",
    booktitle = "Findings of the Association for Computational Linguistics: EACL 2023",
    month = may,
    year = "2023",
    address = "Dubrovnik, Croatia",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.findings-eacl.61",
    doi = "10.18653/v1/2023.findings-eacl.61",
    pages = "809--827",
    abstract = "We present Reddit Health Online Talk (RedHOT), a corpus of 22,000 richly annotated social media posts from Reddit spanning 24 health conditions. Annotations include demarcations of spans corresponding to medical claims, personal experiences, and questions. We collect additional granular annotations on identified claims. Specifically, we mark snippets that describe patient Populations, Interventions, and Outcomes (PIO elements) within these. Using this corpus, we introduce the task of retrieving trustworthy evidence relevant to a given claim made on social media. We propose a new method to automatically derive (noisy) supervision for this task which we use to train a dense retrieval model; this outperforms baseline models. Manual evaluation of retrieval results performed by medical doctors indicate that while our system performance is promising, there is considerable room for improvement. We release all annotations collected (and scripts to assemble the dataset), and all code necessary to reproduce the results in this paper at: \url{https://sominw.com/redhot}.",
}

@misc{schulhoff2024promptreportsystematicsurvey,
      title={The Prompt Report: A Systematic Survey of Prompting Techniques}, 
      author={Sander Schulhoff and Michael Ilie and Nishant Balepur and Konstantine Kahadze and Amanda Liu and Chenglei Si and Yinheng Li and Aayush Gupta and HyoJung Han and Sevien Schulhoff and Pranav Sandeep Dulepet and Saurav Vidyadhara and Dayeon Ki and Sweta Agrawal and Chau Pham and Gerson Kroiz and Feileen Li and Hudson Tao and Ashay Srivastava and Hevander Da Costa and Saloni Gupta and Megan L. Rogers and Inna Goncearenco and Giuseppe Sarli and Igor Galynker and Denis Peskoff and Marine Carpuat and Jules White and Shyamal Anadkat and Alexander Hoyle and Philip Resnik},
      year={2024},
      eprint={2406.06608},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2406.06608}, 
}

@misc{kweon2024ehrnoteqallmbenchmarkrealworld,
      title={EHRNoteQA: An LLM Benchmark for Real-World Clinical Practice Using Discharge Summaries}, 
      author={Sunjun Kweon and Jiyoun Kim and Heeyoung Kwak and Dongchul Cha and Hangyul Yoon and Kwanghyun Kim and Jeewon Yang and Seunghyun Won and Edward Choi},
      year={2024},
      eprint={2402.16040},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2402.16040}, 
}

@misc{yagnik2024medlmexploringlanguagemodels,
      title={MedLM: Exploring Language Models for Medical Question Answering Systems}, 
      author={Niraj Yagnik and Jay Jhaveri and Vivek Sharma and Gabriel Pila},
      year={2024},
      eprint={2401.11389},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2401.11389}, 
}

@misc{malik2024tarzantolkiencontrollinglanguage,
      title={From Tarzan to Tolkien: Controlling the Language Proficiency Level of LLMs for Content Generation}, 
      author={Ali Malik and Stephen Mayhew and Chris Piech and Klinton Bicknell},
      year={2024},
      eprint={2406.03030},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2406.03030}, 
}

@misc{chen2023meditron70bscalingmedicalpretraining,
      title={MEDITRON-70B: Scaling Medical Pretraining for Large Language Models}, 
      author={Zeming Chen and Alejandro Hernández Cano and Angelika Romanou and Antoine Bonnet and Kyle Matoba and Francesco Salvi and Matteo Pagliardini and Simin Fan and Andreas Köpf and Amirkeivan Mohtashami and Alexandre Sallinen and Alireza Sakhaeirad and Vinitra Swamy and Igor Krawczuk and Deniz Bayazit and Axel Marmet and Syrielle Montariol and Mary-Anne Hartley and Martin Jaggi and Antoine Bosselut},
      year={2023},
      eprint={2311.16079},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2311.16079}, 
}

@misc{wu2023pmcllamabuildingopensourcelanguage,
      title={PMC-LLaMA: Towards Building Open-source Language Models for Medicine}, 
      author={Chaoyi Wu and Weixiong Lin and Xiaoman Zhang and Ya Zhang and Yanfeng Wang and Weidi Xie},
      year={2023},
      eprint={2304.14454},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2304.14454}, 
}

@misc{han2023medalpacaopensourcecollection,
      title={MedAlpaca -- An Open-Source Collection of Medical Conversational AI Models and Training Data}, 
      author={Tianyu Han and Lisa C. Adams and Jens-Michalis Papaioannou and Paul Grundmann and Tom Oberhauser and Alexander Löser and Daniel Truhn and Keno K. Bressem},
      year={2023},
      eprint={2304.08247},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2304.08247}, 
}

@misc{llama3modelcard,
title={Llama 3 Model Card},
author={AI@Meta},
year={2024},
url = {https://github.com/meta-llama/llama3/blob/main/MODEL_CARD.md}

}

@inproceedings{10.3115/1073083.1073135,
author = {Papineni, Kishore and Roukos, Salim and Ward, Todd and Zhu, Wei-Jing},
title = {BLEU: a method for automatic evaluation of machine translation},
year = {2002},
publisher = {Association for Computational Linguistics},
address = {USA},
url = {https://doi.org/10.3115/1073083.1073135},
doi = {10.3115/1073083.1073135},
abstract = {Human evaluations of machine translation are extensive but expensive. Human evaluations can take months to finish and involve human labor that can not be reused. We propose a method of automatic machine translation evaluation that is quick, inexpensive, and language-independent, that correlates highly with human evaluation, and that has little marginal cost per run. We present this method as an automated understudy to skilled human judges which substitutes for them when there is need for quick or frequent evaluations.},
booktitle = {Proceedings of the 40th Annual Meeting on Association for Computational Linguistics},
pages = {311–318},
numpages = {8},
location = {Philadelphia, Pennsylvania},
series = {ACL '02}
}

@inproceedings{rouge,
author = {Lin, Chin-Yew},
year = {2004},
month = {01},
pages = {10},
title = {ROUGE: A Package for Automatic Evaluation of summaries},
journal = {Proceedings of the ACL Workshop: Text Summarization Braches Out 2004}
}


@article{jiBEAVERTAILSImprovedSafety,
  title = {{{BEAVERTAILS}}: {{Towards Improved Safety Alignment}} of {{LLM}} via a {{Human-Preference Dataset}}},
  author = {Ji, Jiaming and Liu, Mickel and Dai, Juntao and Pan, Xuehai and Zhang, Chi and Bian, Ce and Chen, Boyuan and Sun, Ruiyang and Wang, Yizhou and Yang, Yaodong},
  langid = {english}
}

@inproceedings{finch_UnifiedDialogueSystem2020,
  title = {Towards {{Unified Dialogue System Evaluation}}: {{A Comprehensive Analysis}} of {{Current Evaluation Protocols}}},
  shorttitle = {Towards {{Unified Dialogue System Evaluation}}},
  booktitle = {Proceedings of the 21th {{Annual Meeting}} of the {{Special Interest Group}} on {{Discourse}} and {{Dialogue}}},
  author = {Finch, Sarah E. and Choi, Jinho D.},
  editor = {Pietquin, Olivier and Muresan, Smaranda and Chen, Vivian and Kennington, Casey and Vandyke, David and Dethlefs, Nina and Inoue, Koji and Ekstedt, Erik and Ultes, Stefan},
  year = {2020},
  month = jul,
  pages = {236--245},
  publisher = {Association for Computational Linguistics},
  address = {1st virtual meeting},
  doi = {10.18653/v1/2020.sigdial-1.29},
  urldate = {2024-08-30}
}





@article{RUP_bench_perturbations,
  title = {{{RUPBench}}: {{Benchmarking Reasoning Under Perturbations}} for {{Robustness Evaluation}} in {{Large Language Models}}},
  shorttitle = {{{RUPBench}}},
  author = {Wang, Yuqing and Zhao, Yun},
  journal={arXiv preprint arXiv:2406.11020},
  year={2024}
}

@article{brown2024enhancing,
  title={Enhancing Trust in LLMs: Algorithms for Comparing and Interpreting LLMs},
  author={Brown, Nik Bear},
  journal={arXiv preprint arXiv:2406.01943},
  year={2024}
}


@misc{MLMs_evaluating_robustness_perturbations,
 title = "Evaluating the Robustness of Neural Language Models to Input Perturbations",
    author = "Moradi, Milad  and
      Samwald, Matthias",
    editor = "Moens, Marie-Francine  and
      Huang, Xuanjing  and
      Specia, Lucia  and
      Yih, Scott Wen-tau",
    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
    month = nov,
    year = "2021",
    address = "Online and Punta Cana, Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.emnlp-main.117",
    doi = "10.18653/v1/2021.emnlp-main.117",
    pages = "1558--1570",
    abstract = "High-performance neural language models have obtained state-of-the-art results on a wide range of Natural Language Processing (NLP) tasks. However, results for common benchmark datasets often do not reflect model reliability and robustness when applied to noisy, real-world data. In this study, we design and implement various types of character-level and word-level perturbation methods to simulate realistic scenarios in which input texts may be slightly noisy or different from the data distribution on which NLP systems were trained. Conducting comprehensive experiments on different NLP tasks, we investigate the ability of high-performance language models such as BERT, XLNet, RoBERTa, and ELMo in handling different types of input perturbations. The results suggest that language models are sensitive to input perturbations and their performance can decrease even when small changes are introduced. We highlight that models need to be further improved and that current benchmarks are not reflecting model robustness well. We argue that evaluations on perturbed inputs should routinely complement widely-used benchmarks in order to yield a more realistic understanding of NLP systems{'} robustness.",
}


@inproceedings{noisyExamplars_Robustness,
  title = {Noisy {{Exemplars Make Large Language Models More Robust}}: {{A Domain-Agnostic Behavioral Analysis}}},
  shorttitle = {Noisy {{Exemplars Make Large Language Models More Robust}}},
  booktitle = {Proceedings of the 2023 {{Conference}} on {{Empirical Methods}} in {{Natural Language Processing}}},
  author = {Zheng, Hongyi and Saparov, Abulhair},
  editor = {Bouamor, Houda and Pino, Juan and Bali, Kalika},
  year = {2023},
  month = dec,
  pages = {4560--4568},
  publisher = {Association for Computational Linguistics},
  address = {Singapore},
  doi = {10.18653/v1/2023.emnlp-main.277},
  urldate = {2024-09-03},
  file = {/Users/chasefensore/Zotero/storage/I5IFS7HN/Zheng and Saparov - 2023 - Noisy Exemplars Make Large Language Models More Ro.pdf}
}

@misc{physionet_MeLLaMA,
  title = {Me-{{LLaMA}}: {{Foundation Large Language Models}} for {{Medical Applications}}},
  shorttitle = {Me-{{LLaMA}}},
  author = {Xie, Qianqian and Chen, Qingyu and Chen, Aokun and Peng, Cheng and Hu, Yan and Lin, Fongci and Peng, Xueqing and Huang, Jimin and Zhang, Jeffrey and Keloth, Vipina and Zhou, Xinyu and He, Huan and {Ohno-Machado}, Lucila and Wu, Yonghui and Xu, Hua and Bian, Jiang},
  publisher = {PhysioNet},
  doi = {10.13026/WWFD-2T39},
  year = {2024},
  urldate = {2024-09-05}
}

@article{review_AI,
author = {Park, Seong and Han, Kyunghwa and Lee, June-Goo},
year = {2024},
month = {09},
pages = {},
title = {Conceptual review of outcome metrics and measures used in clinical evaluation of artificial intelligence in radiology},
journal = {La Radiologia medica},
doi = {10.1007/s11547-024-01886-9}
}

@article{Generative_Transformer-4,
author = {Hasani, Amir and Singh, Shiva and Zahergivar, Aryan and Farhadi, Faraz and Malayeri, Ashkan and Ryan, Beth and Nethala, Daniel and Bravomontenegro, Gabriela and Mendhiratta, Neil and Ball, Mark},
year = {2023},
month = {11},
pages = {1-9},
title = {Evaluating the performance of Generative Pre-trained Transformer-4 (GPT-4) in standardizing radiology reports},
volume = {34},
journal = {European Radiology},
doi = {10.1007/s00330-023-10384-x}
}

@InProceedings{pmlr-v259-diekmann25a,
  title = 	 {Evaluating Safety of Large Language Models for Patient-facing Medical Question Answering},
  author =       {Diekmann, Yella and Fensore, Chase M and Carrillo-Larco, Rodrigo M and Pradhan, Nishant and Appana, Bhavya and Ho, Joyce C},
  booktitle = 	 {Proceedings of the 4th Machine Learning for Health Symposium},
  pages = 	 {267--290},
  year = 	 {2025},
  editor = 	 {Hegselmann, Stefan and Zhou, Helen and Healey, Elizabeth and Chang, Trenton and Ellington, Caleb and Mhasawade, Vishwali and Tonekaboni, Sana and Argaw, Peniel and Zhang, Haoran},
  volume = 	 {259},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {15--16 Dec},
  publisher =    {PMLR},
  pdf = 	 {https://raw.githubusercontent.com/mlresearch/v259/main/assets/diekmann25a/diekmann25a.pdf},
  url = 	 {https://proceedings.mlr.press/v259/diekmann25a.html},
  abstract = 	 {Large language models (LLMs) have revolutionized the question answering (QA) domain by achieving near-human performance across a broad range of tasks. Recent studies have suggested LLMs are capable of answering clinical questions and providing medical advice. Although LLMs’ answers must be reliable and safe, existing evaluations of medical QA systems often only focus on the accuracy of the content. However, a critical, underexplored aspect is whether variations in patient inquiries - rephrasing the same question - lead to inconsistent or unsafe LLM responses. We propose a new evaluation methodology leveraging synthetic question generation to rigorously assess the safety of LLMs in patient-facing medical QA. In benchmarking 8 LLMs, we observe a weak correlation between standard automated quality metrics and human evaluations, underscoring the need for enhanced sensitivity analysis in evaluating patient medical QA safety.}
}

@article{scikit-learn,
  title={Scikit-learn: Machine Learning in {P}ython},
  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
          and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
          and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
          Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
  journal={Journal of Machine Learning Research},
  volume={12},
  pages={2825--2830},
  year={2011}
}

@misc{OpenBioLLMs,
  author = {Ankit Pal, Malaikannan Sankarasubbu},
  title = {OpenBioLLMs: Advancing Open-Source Large Language Models for Healthcare and Life Sciences},
  year = {2024},
  publisher = {Hugging Face},
  journal = {Hugging Face repository},
  howpublished = {\url{https://huggingface.co/aaditya/OpenBioLLM-Llama3-70B}}
}

@misc{kim2024prometheus,
    title={Prometheus 2: An Open Source Language Model Specialized in Evaluating Other Language Models},
    author={Seungone Kim and Juyoung Suk and Shayne Longpre and Bill Yuchen Lin and Jamin Shin and Sean Welleck and Graham Neubig and Moontae Lee and Kyungjae Lee and Minjoon Seo},
    year={2024},
    eprint={2405.01535},
    archivePrefix={arXiv},
    primaryClass={cs.CL}
}

@misc{med42v2,
Author = {Cl{\'e}ment Christophe and Praveen K Kanithi and Tathagata Raha and Shadab Khan and Marco AF Pimentel},
Title = {Med42-v2: A Suite of Clinical LLMs},
Year = {2024},
Eprint = {arXiv:2408.06142},
url={https://arxiv.org/abs/2408.06142}, 
}

@misc{jiang2024mixtralexperts,
      title={Mixtral of Experts}, 
      author={Albert Q. Jiang and Alexandre Sablayrolles and Antoine Roux and Arthur Mensch and Blanche Savary and Chris Bamford and Devendra Singh Chaplot and Diego de las Casas and Emma Bou Hanna and Florian Bressand and Gianna Lengyel and Guillaume Bour and Guillaume Lample and Lélio Renard Lavaud and Lucile Saulnier and Marie-Anne Lachaux and Pierre Stock and Sandeep Subramanian and Sophia Yang and Szymon Antoniak and Teven Le Scao and Théophile Gervet and Thibaut Lavril and Thomas Wang and Timothée Lacroix and William El Sayed},
      year={2024},
      eprint={2401.04088},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2401.04088}, 
}

@misc{ness2024medfuzzexploringrobustnesslarge,
      title={MedFuzz: Exploring the Robustness of Large Language Models in Medical Question Answering}, 
      author={Robert Osazuwa Ness and Katie Matton and Hayden Helm and Sheng Zhang and Junaid Bajwa and Carey E. Priebe and Eric Horvitz},
      year={2024},
      eprint={2406.06573},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2406.06573}, 
}

@misc{xu2023llmfoolitselfpromptbased,
      title={An LLM can Fool Itself: A Prompt-Based Adversarial Attack}, 
      author={Xilie Xu and Keyi Kong and Ning Liu and Lizhen Cui and Di Wang and Jingfeng Zhang and Mohan Kankanhalli},
      year={2023},
      eprint={2310.13345},
      archivePrefix={arXiv},
      primaryClass={cs.CR},
      url={https://arxiv.org/abs/2310.13345}, 
}

@misc{perez2022redteaminglanguagemodels,
      title={Red Teaming Language Models with Language Models}, 
      author={Ethan Perez and Saffron Huang and Francis Song and Trevor Cai and Roman Ring and John Aslanides and Amelia Glaese and Nat McAleese and Geoffrey Irving},
      year={2022},
      eprint={2202.03286},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2202.03286}, 
}

@misc{lin2022truthfulqameasuringmodelsmimic,
      title={TruthfulQA: Measuring How Models Mimic Human Falsehoods}, 
      author={Stephanie Lin and Jacob Hilton and Owain Evans},
      year={2022},
      eprint={2109.07958},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2109.07958}, 
}

@INPROCEEDINGS{10487363,
  author={Jones, Rebet and Omar, Marwan and Mohammed, Derek},
  booktitle={2023 Congress in Computer Science, Computer Engineering, \& Applied Computing (CSCE)}, 
  title={Harnessing the Power of the GPT Model to Generate Adversarial Examples}, 
  year={2023},
  volume={},
  number={},
  pages={1699-1702},
  keywords={Sentiment analysis;Analytical models;Computational modeling;Perturbation methods;Benchmark testing;Iterative methods;Task analysis;GPT;Adversarial examples;Learning algorithm;Yelp;MR;IMDB;Accuracy},
  doi={10.1109/CSCE60160.2023.00279}}

@misc{castro-2017-fast-krippendorff,
  author = {Santiago Castro},
  title = {Fast {K}rippendorff: Fast computation of {K}rippendorff's alpha agreement measure},
  year = {2017},
  publisher = {GitHub},
  journal = {GitHub repository},
  howpublished = {\url{https://github.com/pln-fing-udelar/fast-krippendorff}}
}

@article{sebastianski2022disease,
  title={Disease outbreaks linked to pasteurized and unpasteurized dairy products in Canada and the United States: a systematic review},
  author={Sebastianski, Meghan and Bridger, Natalie A and Featherstone, Robin M and Robinson, Joan L},
  journal={Canadian Journal of Public Health},
  volume={113},
  number={4},
  pages={569--578},
  year={2022},
  publisher={Springer}
}