% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.

\documentclass[11pt]{article}

% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage[final]{acl}

% Standard package includes
\usepackage{times}
\usepackage{latexsym}

% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets

% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}

% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}

% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}

%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}

%ADDED
\usepackage[table]{xcolor}
\usepackage{booktabs}
\usepackage{tabularx}
\usepackage{array}
\usepackage{listings}
\usepackage{hyperref}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{caption}
\hypersetup{
    colorlinks=true,
    linkcolor=blue,
    filecolor=magenta,
    urlcolor=cyan,
    pdftitle={Your Paper Title},
    pdfpagemode=FullScreen,
}

% \captionsetup[table]{skip=10pt} % 테이블 제목과 표 사이 간격 조정
\setlength{\abovecaptionskip}{2pt}  % 캡션 위 여백 (기본값: 10pt~12pt)
\setlength{\belowcaptionskip}{2pt}  % 캡션 아래 여백 (기본값: 0pt~3pt)

% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.

\title{Questioning Our Questions: How Well Do Medical QA Benchmarks Evaluate Clinical Capabilities of Language Models?}

% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
%         Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
%         Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \And  ... \And
%         Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\  ... \\ Address line
%         \AND
%         Author 2 \\ Address line \\ ... \\ Address line \And
%         Author 3 \\ Address line \\ ... \\ Address line}

\author{
  Siun Kim \\
  Seoul National University Hospital \\
  Seoul, South Korea \\
  \texttt{shiuhn95@snu.ac.kr} \\\And
  Hyung-Jin Yoon \\
  Seoul National University Hospital \\
  Seoul, South Korea \\
  \texttt{hjyoon@snu.ac.kr} \\
}

%\author{
%  \textbf{First Author\textsuperscript{1}},
%  \textbf{Second Author\textsuperscript{1,2}},
%  \textbf{Third T. Author\textsuperscript{1}},
%  \textbf{Fourth Author\textsuperscript{1}},
%\\
%  \textbf{Fifth Author\textsuperscript{1,2}},
%  \textbf{Sixth Author\textsuperscript{1}},
%  \textbf{Seventh Author\textsuperscript{1}},
%  \textbf{Eighth Author \textsuperscript{1,2,3,4}},
%\\
%  \textbf{Ninth Author\textsuperscript{1}},
%  \textbf{Tenth Author\textsuperscript{1}},
%  \textbf{Eleventh E. Author\textsuperscript{1,2,3,4,5}},
%  \textbf{Twelfth Author\textsuperscript{1}},
%\\
%  \textbf{Thirteenth Author\textsuperscript{3}},
%  \textbf{Fourteenth F. Author\textsuperscript{2,4}},
%  \textbf{Fifteenth Author\textsuperscript{1}},
%  \textbf{Sixteenth Author\textsuperscript{1}},
%\\
%  \textbf{Seventeenth S. Author\textsuperscript{4,5}},
%  \textbf{Eighteenth Author\textsuperscript{3,4}},
%  \textbf{Nineteenth N. Author\textsuperscript{2,5}},
%  \textbf{Twentieth Author\textsuperscript{1}}
%\\
%\\
%  \textsuperscript{1}Affiliation 1,
%  \textsuperscript{2}Affiliation 2,
%  \textsuperscript{3}Affiliation 3,
%  \textsuperscript{4}Affiliation 4,
%  \textsuperscript{5}Affiliation 5
%\\
%  \small{
%    \textbf{Correspondence:} \href{mailto:email@domain}{email@domain}
%  }
%}

\begin{document}
\maketitle
\begin{abstract}
Recent advances in large language models (LLMs) have led to impressive  performance on medical question-answering (QA) benchmarks. However, the extent to which these benchmarks reflect real-world clinical capabilities remains uncertain. To address this gap, we systematically analyzed the correlation between LLM performance on major medical QA benchmarks (e.g., MedQA, MedMCQA, PubMedQA, and MMLU medicine subjects) and clinical performance in real-world settings. Our dataset included 702 clinical evaluations of 85 LLMs from 168 studies. Benchmark scores demonsrated a moderate correlation with clinical performance (Spearman's $\rho$ = 0.59), albeit substantially lower than inter-benchmark correlations. Among them, MedQA was the most predictive but failed to capture essential competencies such as patient communication, longitudinal care, and clinical information extraction. Using Bayesian hierarchical modeling, we estimated representative clinical performance and identified GPT-4 and GPT-4o as consistently top-performing models, often matching or exceeding human physicians. Despite longstanding concerns about the clinical validity of medical QA benchmarks, this study offers the first quantitative analysis of their alignment with real-world clinical performance.\footnote{The dataset and code are available at: \href{https://github.com/SiunKim/questioning-medqa}{https://github.com/SiunKim/questioning-medqa}.}
\end{abstract}

\section{Introduction}

\label{sec:intro}
The rapid advancement of large language models (LLMs), accelerated by the release of ChatGPT, has continued  into 2025. Open-source models such as Llama 3.3, Phi-4, and DeepSeek-R1 are rapidly narrowing the performance gap with proprietary models (\citealp[]{grattafiori2024llama, abdin2024phi, guo2025deepseek}). This progress is especially consequential in healthcare, where strigent privacy and security requirments frequently necessitate on-premise deployment (\citealp[]{faray2025does, gupta2025llms}).

\begin{figure}[htbp]
    \centering
    \includegraphics[width=\columnwidth]{Figure1.png}
    \caption{Overview of our study assessing the alignment between medical QA benchmarks and real-world clinical performance.}
    \label{fig:study_overview}
\end{figure}

As LLMs attains expert-level performance on both general and medical QA benchmarks, the limitations of such benchmarks have become increasingly apparent. For instance, OpenAI's o1-preview achieved 96\% on MedQA and 99\% on MMLU Medical Genetics, outperforming human experts (\citealp[]{nori2024medprompt, lievin2024can}). However, such benchmarks are thought to focus predominantly on static knowledge and structured reasoning, which may not fully reflect core competencies essential for clinical practice \citep{nori2023capabilities, singhal2023large}, such as decision-making under uncertainty (\citealp[]{han2011varieties}), patient communication (\citealp[]{barry2012shared}), and ethical reasoning (\citealp[]{kaldjian2005clinician}).

Although concerns over the limited clinical validity of existing medical QA benchmarks have been raised, there remains a lack of systematic evidence. In this study, we address this gap through a comprehensive meta-analysis evaluating how effectively conventional medical QA benchmarks reflect the real-world clinical performance of LLMs (Figure~\ref{fig:study_overview}).

Our key contributions are as follows:
\begin{itemize}
    \item \textbf{Quantitative assessment of benchmark-clinical alignment}: We demonstrate a moderate correlation (Spearman’s $\rho$ = 0.59) between medical QA benchmarks and real-world clinical performance, highlighting significant limitations in the current evaluation practices.
    \item \textbf{Identification of clinical gaps in MedQA}: MedQA demonstrates strong alignment with core competencies such as treatment, clinical knowledge, and diagnosis. However, it fails to adequately assess essential aspects of real-world clinical practice, including patient communication, longitudinal care, and clinical information extraction.
    \item \textbf{Bayesian modeling of representative clinical performance}: Using hierarchical Bayesian models, we estimate the generalized clinical capabilities of LLMs, suggesting that models like GPT-4 and GPT-4o match or exceed human physician-level performance in real-world clinical settings.
\end{itemize}

\section{Related Works}
MedQA—based on the USMLE Step 1 and 2 exams—has emerged as a de facto benchmark in the medical domain, owing to its high-quality multiple-choice questions (MCQs) and comprehensive topical coverage \citep{jin2021disease}. As a representative benchmark, improvements in MedQA performance have frequently been interpreted as a proxy for progress in medical LLMs \citep{singhal2025toward, saab2024capabilities}.

MedMCQA, derived from Indian medical entrance exams (AIIMS and NEET PG), complements MedQA by offering broader topical diversity and varied question types \citep{pal2022medmcqa}. In contrast, PubMedQA focused on biomedical literature comprehension by requiring models to infer answers from PubMed abstracts \citep{jin2019pubmedqa}.

Despite their widespread use, these traditional medical benchmarks primarily assess factual recall and structured reasoning. They have been criticized for failing to evaluate essential aspects of practical clinical competence \citep{tang2023medagents, kim2025limitations, liu2024large}.

In response, recent datasets aim to capture the complexity of real-world clinical practice. Datasets like Medbullet (\citealp[]{chen2024benchmarking}), MedExQA (\citealp[]{kim2024medexqa}), and MedXpertQA (\citealp[]{zuo2025medxpertqa}) introduce open-ended questions, expert-written explanations, and multimodal data to facilitate more comprehensive evaluations. Furthermore, integrated evaluation frameworks like MedAgentBench (\citealp[]{tang2025medagentsbench}) and MEDIC (\citealp[]{kanithi2024medic}) encompass multiple clinical tasks and explicitly address ethical and safety concerns.

In parallel, agent-based evaluations have emerged to assess interactive and dynamic reasoning. For instance, MedQA-CS adopts OSCE-style clinical scenarios \citep{yao2024medqa}, while AgentClinic \citep{schmidgall2024agentclinic} evaluates LLMs during simulated physician-patient dialogues.

Building on these developments, our study systematically examines the alignment between conventional medical QA benchmarks and real-world clinical evaluations. By identifying existing gaps, we aim to inform the design of future benchmarks that more accurately reflect practical clinical competencies.



\section{Methods}
To evaluate the extent to which existing medical QA benchmarks reflect real-world clinical performance, we analyzed 168 published studies that assessed at least three distinct language models in clinical settings. Benchmark scores on both medical QA and general-purpose benchmarks were collected and standardized to ensure comparability. To address the missing benchmark scores, multiple imputation was applied. Correlations between benchmark scores and clinical performance were calculated using rank-based methods weighted by sample size. Finally, we employed Bayesian hierarchical modeling to estimate each model’s representative clinical capability.

\subsection{Literature Review for Collecting Clinical Performance Data}
We conducted a multi-stage literature review to identify studies evaluating LLM performance in real-world clinical settings (Figure \ref{fig:flowchart_selecting_articles}). Using the Semantic Scholar API, we first retrieved articles published between January 1, 2023, and January 10, 2025, based on search queries designed to encompass a wide range of clinical scenarios (Appendix \ref{apd:search_query_construction}). Title-based filtering retained studies explicitly mentioning LLM-related terms, followed by DOI-based deduplication. Abstract and full-texts were retrieved via publisher and open-access APIs.

\begin{figure}[htbp]
    \centering
    \includegraphics[width=0.75\columnwidth]{literature_review_flowchart_selectable.pdf}
    \caption{Flowchart of literature review for collecting LLM performances in real-world clinical settings.}
    \label{fig:flowchart_selecting_articles}
\end{figure}

Phi-4 model-assisted screening of abstracts and full-texts identified studies that reported performance for at least three distinct LLMs, enabling correlation analyses (Appendix\ref{apd:screening_by_llm}). Manual review was conducted to extract structured data, normalized model names (Appendix \ref{apd:mode_name_norm}) and classified evaluation settings (Appendix \ref{apd:classif_eval_settings}).


\subsection{LLM Performance Collection in Real-World Clinical Settings}
To address overrepresentation issues caused by redundant evaluations of similar model abilities within a single study, we extracted one representative performance score for each task-model combination. Preference was given to the simplest inference setting (e.g., zero-shot without CoT). If multiple measures existed for the same therapeutic area and capabilities, we selected the most frequently used metric, or averaged scores, if no dominant measure was evident. Evaluations spanning multiple therapeutic areas or distinct capabilities were treated as a separate task. 

Studies relying on readability metrics, inter-model correlation analyses, or with fewer than 20 evaluation samples were excluded. Encoder-based language models (e.g., BERT, RoBERTa) were also excluded because the study focused solely on autoregressive LLMs.

Performance scores were normalized to a 0–100 scale using min-max scaling. Metrics indicating better performance through lower values (e.g., proportion of biased answers) were inverted by subtracting from 100.

\subsection{Benchmark Performance Collection}
Benchmarks were divided into medical QA benchmarks and general benchmarks. Medical QA benchmarks included MedQA, MedMCQA, PubMedQA, and six MMLU medical subsets (Anatomy, Clinical Knowledge, College Biology, College Medicine, Medical Genetics, and Professional Medicine). General benchmarks consisted of MMLU, MMLU Pro, BBH, HumanEval, GSM8K, and MATH.

Performance data were extracted from published articles, technical reports, and model cards. Additional web searches supplemented version-specific scores for widely used proprietary models (e.g., GPT, Claude, and Gemini).

We standardized benchmark performances by focusing on zero-shot without CoT. If multiple results were available, averages were used. If zero-shot data were unavailable, performance was estimated through linear regression using reported results under different inference settings, by considering few-shot examples and CoT usage as covariates.

\subsection{Benchmark Performance Imputation}
\label{sec:method_imputation_on_bench}
While complete benchmark data are ideal for reliable correlation analyses, missing values were inevitable as performance scores were compiled  through literature review rather than direct evaluation. To systematically address missing values, we employed Multiple Imputation by Chained Equations (MICE), which leverages observed interdependencies within available data to estimate absent benchmark performances.

Before imputation, we confirmed the Missing at Random (MAR) assumption, a necessary condition for reducing bias in estimation. Two imputation techniques were tested: Random Forest (RF-MICE) for capturing non-linear dependencies and Bayesian Ridge (BR-MICE) for small datasets with collinearity. Imputation was performed separately for each benchmark category.

We validated imputation accuracy through masking test, randomly removing and subsequently estimating 10\% of the observed values. To incorporate uncertainty, multiple imputations were conducted, and within- and between-imputation variance were estimated (Appendix \ref{apd:imputation_on_bench_method}). Based on validation results, we selected a final version of the imputed dataset for downstream analysis.

\subsection{Correlation Measurement}
\label{sec:methods_correlation_measure}
We evaluated correlations in two ways: benchmark-to-benchmark and benchmark-to-clinical performance.

Benchmark-to-benchmark correlation were calculated based on the performance scores of models that were evaluated on both. This analysis allowed us to identify redundant benchmarks, assess the quality of benchmark datasets, and set a correlation baseline for subsequent benchmark-to-clinical correlation analyses.

Benchmark-to-clinical correlations were computed at the evaluation task level, weighted logarithmically by evaluation sample size to reflect varying reliability across studies. Analyses utilized imputed benchmark scores primarily, with non-imputed data serving as sensitivity checks.

Although we measured rank-based (Spearman's rank correlation coefficient and Kendall's tau) and linear-based (Pearson's correlation coefficient and Lin's concordance correlation coefficient) metrics, primary analyses used Spearman's rank and Kendall's tau due to their suitability for handling diverse evaluation scoring scales without assuming linear relationships. Linear correlations were calculated but used only as reference points.

\begin{table}[t]
    \centering
    \caption{Summary of clinical performance dataset and evaluation settings.}
    \resizebox{.97\linewidth}{!}{%
    \begin{tabular}{l c}
        \toprule
        \textbf{Category} & \textbf{Count (\%)} \\
        \midrule
        \textbf{Total samples} & 702 (100.0) \\
        \midrule
        \textbf{Task type} & \\
        \hspace{3mm} Diagnosis & 183 (26.1) \\
        \hspace{3mm} Clinical Knowledge & 182 (25.9) \\
        \hspace{3mm} Overall Management & 111 (15.8) \\
        \hspace{3mm} Answering to Patients & 83 (11.8) \\
        \hspace{3mm} Information Extraction & 61 (8.7) \\
        \hspace{3mm} Treatment & 48 (6.8) \\
        \hspace{3mm} Other & 34 (4.8) \\ 
        \midrule
        \textbf{Data source} & \\
        \hspace{3mm} Clinical Vignettes & 271 (38.6) \\
        \hspace{3mm} Quizzes & 160 (22.8) \\
        \hspace{3mm} Board Examination & 114 (16.2) \\
        \hspace{3mm} FAQs & 74 (10.5) \\
        \hspace{3mm} Other & 83 (11.8) \\
        \midrule
        \textbf{Therapeutic area} & \\
        \hspace{3mm} General Medicine & 154 (21.9) \\
        \hspace{3mm} Oncology & 77 (11.0) \\
        \hspace{3mm} Ophthalmology & 60 (8.5) \\
        \hspace{3mm} Orthopedics \& Musculoskeletal & 58 (8.3) \\
        \hspace{3mm} Emergency Medicine & 53 (7.5) \\
        \hspace{3mm} Neuropsychiatric & 53 (7.5) \\
        \hspace{3mm} Others & 247 (35.2) \\
        \midrule
        \textbf{Evaluation type} & \\
        \hspace{3mm} MCQs & 463 (66.0) \\
        \hspace{3mm} Human Rating & 239 (34.0) \\
        \bottomrule
    \end{tabular}}
    \label{tab:summary_clinical_performance}
\end{table}

\subsection{Bayesian Modeling}
\label{sec:method_bayesian_modeling}
To estimate representative clinical performance for each language model independent of task-specific biases, we employed Bayesian hierarchical modeling. Given the limited number of model evaluated per task (average 3.6 models), individual task effects could not be directly estimated. Instead, task-related variations were approximated using metadata attributes including task type, data source, and evaluation methods. Therapeutic areas were excluded due to inconsistent categorization and unclear impact on performance (Appendix \ref{apd:model_structure_bayesian}). Furthermore, models for which performance data were available for fewer than three distinct tasks were excluded to enhance the reliability of model-specific performance estimates, which served as proxies for general clinical competence.

To further assess the robustness of the model-specific estimates, connectivity measures were calculated. Higher connectivity indicates stronger support from direct and indirect comparisons across models, thereby resulting in more stable and accurate performance estimates (Appendix \ref{apd:centrality_measure}).


\begin{table*}[htbp]
    \centering
    \tiny
    \caption{Average correlation coefficients of medical QA benchmarks with other benchmarks. 
    The highest score in each column is \textbf{bold}, and the second highest is \underline{underlined}.}
    \label{tab:bench_to_bench_corr_medical_qa}
    \footnotesize
    \resizebox{0.95\textwidth}{!}{%
    \begin{tabular}{l cc cc}
        \hline
        \textbf{Medical QA Benchmarks} & \multicolumn{2}{c}{\textbf{Spearman}} & \multicolumn{2}{c}{\textbf{Kendall}} \\
        \cline{2-3} \cline{4-5}
        & \textbf{Medical QA} & \textbf{General} & \textbf{Medical QA} & \textbf{General} \\
        \hline
        MedQA               & 0.809  & \textbf{0.867} & 0.664  & \textbf{0.703} \\
        MedMCQA             & 0.808  & \underline{0.855} & 0.651  & \underline{0.693} \\
        MMLU Medical Genetics & 0.835  & 0.748  & 0.684  & 0.607 \\
        MMLU Clinical Knowledge & \textbf{0.851}  & 0.820  & \textbf{0.714}  & 0.664 \\
        MMLU College Medicine & 0.822  & 0.784  & 0.683  & 0.618 \\
        MMLU Professional Medicine & \underline{0.849}  & 0.789  & \underline{0.705}  & 0.632 \\
        MMLU College Biology & 0.819  & 0.666  & 0.672  & 0.522 \\
        MMLU Anatomy        & 0.703  & 0.558  & 0.571  & 0.449 \\
        PubMedQA            & 0.484  & 0.441  & 0.333  & 0.318 \\
        \hline
        \textbf{Average}    & \textbf{0.787}  & \textbf{0.725}  & \textbf{0.675}  & \textbf{0.576} \\
        \hline
    \end{tabular}%
    }
\end{table*}

\section{Results and Discussion}
\subsection{Clinical Performance Dataset}
Our dataset comprised 702 clinical performance evaluations from 168 studies covering 195 distinct clinical tasks. Evaluations involved 85 LLMs, predominantly from GPT (51.7\%), LLaMA (10.3\%), and Gemini (8.8\%) families. Task types included diagnosis (26.1\%), clinical knowledge assessment (25.9\%), and overall patient management (15.8\%). Data sources were primarily clinical vignettes (38.6\%) and quizzes (22.8\%), with evaluations conducted through MCQs (66.0\%) and expert human ratings (34.0\%) (Table \ref{tab:summary_clinical_performance}).

\subsection{Benchmark Performance Imputation}
\label{sec:results_imputation_on_bench}
The benchmark dataset contained a notable proportion of missing values: 42.4\% for medical QA benchmarks (9 benchmarks, 138 models, 715 scores) and 40.6\% for general benchmarks (6 benchmarks, 126 models, 449 scores).

Imputation accuracy, assessed through masking tests, indicated RF-MICE outperformed BR-MICE. Specifically, RF-MICE achieved lower mean absolute error (MAE=2.04) and higher R² (0.98) on medical QA benchmarks (Table \ref{tab:imputatoin_masking_test}). Variance analysis of multiple imputations further supported RF-MICE due to lower total variance and improved stability (Table \ref{tab:multiple_imputation}). Consequently, RF-MICE was utilized to generate the final imputed dataset.

\subsection{Benchmark-to-Benchmark Correlation}
Medical QA benchmarks showed strong internal correlations overall, with MMLU Clinical Knowledge and MMLU Professional Medicine exhibiting particularly high correlations with other medical QA benchmarks (Table \ref{tab:bench_to_bench_corr_medical_qa}). This is likely due to their broad content coverage, encompassing topics found in other MMLU medical subjects, thereby forming a high-correlation block (Figure \ref{fig:medical_correlation_panel}).

In contrast, PubMedQA and MMLU Anatomy showed weaker correlations with other medical QA benchmarks. PubMedQA's lower correlations may stem from its distinct task formulation, which is more aligned with biomedical summarization rather than clinical reasoning (\citealp[]{jin2019pubmedqa}). Similarly, MMLU Anatomy's lower correlations likely reflect its narrower content scope compared to other benchmarks.

MedQA and MedMCQA demonstrated the highest correlations with general benchmarks among medical QA benchmarks (Table \ref{tab:bench_to_bench_corr_medical_qa}, Figure \ref{fig:medical_correlation_panel}). This suggests that these two datasets not only assess domain-specific knowledge but also required a broad set of reasoning skills, many of which overlapped with general benchmarks. 

Within general benchmarks, BBH (Spearman's 0.891) and MMLU (0.853) exhibited the strongest correlations with medical QA benchmarks (Table \ref{tab:bench_to_bench_corr_general}). This result also indicates logical reasoning capabilities and broad domain knowledge are closely linked to solve medical problems. In contrast, mathematics-focused benchmarks (i.e., GSM8K and MATH) displayed weaker correlations, highlighting the distinct types of reasoning involved in medical contexts.

\subsection{Benchmark-to-Clinical Performance Correlation}
MedQA showed the strongest correlation with real-world clinical performance, outperforming general benchmarks in capturing actual clinical competency (Spearman’s 0.588, Kendall’s 0.520; Figure \ref{fig:bench_to_clinical_corr}A). However, correlation strength was notably lower than inter-benchmark correlations (0.675–0.787; Table \ref{tab:bench_to_bench_corr_medical_qa}). These results suggest MedQA remains the most representative current benchmark for clinical tasks, although its ability to predict comprehensive clinical performance remains  limited.

\begin{figure*}[t]
    \centering
    \includegraphics[width=\textwidth]{correlation_plot_imputated_weighted_two_measures.pdf}
    \caption{Comparison of correlation coefficients between benchmark and clinical performance.}
    \label{fig:bench_to_clinical_corr}
\end{figure*}

Further analysis across evaluation settings highlighted MedQA’s strengths and limitations (Figure \ref{fig:medqa_to_clinical_corr}). MedQA performed well predicting clinical competency in tasks involving treatment, clinical knowledge, and diagnosis (Figure \ref{fig:medqa_to_clinical_corr}A). In contrast, it showed significantly weaker correlations in patient communication, overall patient management, and information extraction. 

Similarly, while MedQA strongly correlated with performance derived from board examination-style datasets, correlations with evaluations based on clinical vignette or FAQ, which closely resemble real-world clinical practice, were considerably lower (Figure \ref{fig:medqa_to_clinical_corr}B). These findings suggest that while MedQA reliably evaluates core medical knowledge and reasoning skills, it does not adequately reflect the broader competencies required in real-world clinical practice.

MMLU Medical Genetics, College Medicine, Professional Medicine, Clinical Knowledge, and MedMCQA displayed moderately high correlations with clinical performance, outperforming general benchmarks (Figure \ref{fig:bench_to_clinical_corr}A). Conversely, PubMedQA and MMLU Anatomy consistently underperformed, indicating their limited suitability as representative clinical evaluation tools (Figures \ref{fig:correlation_plot_w_imput_w_weighting}, \ref{fig:correlation_plot_wo_imput_w_weighting}).

\begin{figure*}[t]
    \centering
    \includegraphics[width=\textwidth]{medqa_correlation_panels_imputated_weighted_two_measures.pdf}
    \caption{Comparison of correlations between MedQA performance and individual outcomes measured in real-world clinical settings across different task types, therapeutic areas, data sources, and evaluation methods.}
    \label{fig:medqa_to_clinical_corr}
\end{figure*}

\subsection{Representative Clinical Performances Estimated through Bayesian Modeling}
Representative clinical performances of 59 language models were robustly estimated using Bayesian hierarchical modeling across 717 performance samples. Model convergence was strong, indicated by effective sample sizes (ESS) above 300 and R-hat values below 1.02.

Among evaluated models, GPT-4 and GPT-4o consistently demonstrated the highest clinical performance, often exceeding the average performance of medical professionals (labeled as 'human - doctor') and substantially outperforming both smaller open-source models and other proprietary models (Figure \ref{fig:repr_clinical_perf}). The strong and consistent performance of the GPT family is further supported by newly developed medical benchmark studies (\citealp{olatunji2024afrimed, yao2024medqa, zuo2025medxpertqa}), which similarly highlight their superior clinical reasoning capabilities.

Proprietary models (purple) generally outperformed open-source models (orange, Figure \ref{fig:repr_clinical_perf}), suggesting that commercially optimized systems remain more reliable in clinical settings—though this conclusion may shift with the rapid progress of open-source LLMs in 2025.

Within the open-source category, Llama-3.1-8B-instruct was the only model to surpass the minimum threshold set for human-level performance (labeled as 'human - cut-off'). Notably, however, its lower connectivity implies that this performance estimate should be interpreted with caution due to high uncertainty.

Notably, language models fine-tuned for the medical domain  (marked with a star, $\bigstar$) did not show substantial improvements over general-purpose models like Llama, despite having comparable model sizes (Figure \ref{fig:repr_clinical_perf}). This may be due to overfitting to the specific characteristics of their training datasets—typically composed of structured medical QA corpora or textbook-style materials—which could limit their generalizability in practical clinical contexts (\citealp[]{olatunji2024afrimed}). These findings are consistent with previous results showing that biomedical models often underperform on newer, more complex benchmarks, and support concerns regarding their sensitivity to dataset-specific biases and limitations (\citealp[]{olatunji2024afrimed, yao2024medqa}).

\begin{figure*}[htbp]
    \centering
    \includegraphics[width=\textwidth]{bayesian_vs_clinical_model_performance_comparison.pdf}
    \caption{Representative clinical performance estimated via Bayesian modeling. Proprietary models appear in purple text while open-source models are shown in orange. Medical domain fine-tuned models are marked with a star ($\bigstar$) and small language models with 13B parameters or fewer display a circle prefix ($\odot$).}
    \label{fig:repr_clinical_perf}
\end{figure*}


\section{Conclusion}
This study demonstrates that existing medical QA benchmarks posses only a  moderate capacity to predict real-world clinical performance. Among them, MedQA showed the strongest correlation with clinical performance but was still insufficient for evaluating practical clinical competencies such as patient interaction, longitudinal patient management, and clinical information extraction. Bayesian hierarchical modeling further revealed that proprietary models—particularly GPT-4 and GPT-4o—consistently outperformed open-source counterparts and, across many versions, matched or exceeded the performance of human experts in real-world clinical settings. Notably, despite longstanding concerns regarding the validity of medical QA benchmarks, this study provides the first systematic and quantitative evidence evaluating the alignment between medical QA benchmarks and actual clinical performance.

\section*{Limitations}
This study has several limitations. First, our analysis is based on published studies, which inevitably lag behind ongoing LLM advancements due to publication delays. Consequently, it does not account for recent developments in LLMs, such as the emergence of reasoning-based LLMs (\citealp[]{guo2025deepseek}).

Second, although several medical benchmarks have been introduced to better assess multifaceted capabilities (\citealp[]{kim2024medexqa, yao2024medqa, zuo2025medxpertqa}), we could not obtain sufficient model performance results on these datasets to conduct correlation analyses. To support future research, we make our clinical performance dataset available and encourage its use in validating how well these newly proposed medical benchmarks reflect the complexity of real-world medical tasks.

Lastly, despite employing statistical methods to address missing data and selection biases, our findings are inherently constrained by the incompleteness and potential biases of literature-derived data.

\appendix

\section*{Acknowledgments}
This work was supported by the Institute of Information & Communications Technology Planning & Evaluation (IITP)-Innovative Human Resource Development for Local Intellecutallization program grant funded by the Korea goverment (MSIT) (IITP-2024-RS-2024-00441407). This research was supported by a grant of the Korea Health Technology R&D Project through the Korea Health Industry Development Institute (KHIDI), funded by the Ministry of Health & Welfare, Republic of Korea (grant number: RS-2023-KH136520).

\bibliography{custom}

\appendix

\section{Supplementary Methods}
\label{apd:method}

\subsection{Search Query for Literature Review}
\label{apd:search_query_construction}
Our literature search followed a systematic approach to identify studies at the intersection of large language models (LLMs) and medical applications. The search queries were structured using three essential components: LLM-related terms, medical terms, and evaluation terms (Table~\ref{tab:query_structure}).

Each query was formulated as: \begin{quote} \textbf{[LLM Term] AND [Medical Term] AND [Evaluation Term]} \end{quote} where the medical terms were drawn from either MedQA-related categories (e.g., "medical question answering", "clinical reasoning") or clinical application categories (e.g., medical specialties, clinical documents, diseases, and procedures). The search was restricted to publications from 2022 to 2025 to ensure coverage of recent developments.

This combinatorial approach balanced coverage and precision, ensuring that retrieved papers addressed all three aspects of our research focus (Table~\ref{tab:query_examples}).

\begin{table}[htbp]
\centering
\caption{Search Query Components for LLM Applications in Medical Research.}
\label{tab:query_structure}
\begin{tabular}{p{0.22\linewidth}p{0.72\linewidth}}
\toprule
\textbf{Query Component} & \textbf{Terms} \\
\midrule
\textbf{LLM Terms} & ``large language model'', ``language model'', ``GPT-4'', ``ChatGPT'' \\
\midrule
\textbf{MedQA Terms} & ``medical question answering'', ``USMLE'', ``MedQA'', ``medical benchmark'', ``clinical reasoning'' \\
\midrule
\textbf{Clinical Application Terms} & 
\textit{Medical Specialties (31 terms):} ``internal medicine'', ``surgery'', ``pediatrics'', ``obstetrics'', ``gynecology'',  \ldots \\
& \textit{Surgery Settings (10 terms):} ``surgery'', ``pediatric surgery'', ``breast surgery'', ``colorectal surgery'', ``neurosurgery'',  \ldots \\
& \textit{Clinical Settings (6 terms):} ``emergency department'', ``icu'', ``operating room'', ``outpatient'', ``primary care'', ``trauma center'' \\
& \textit{Clinical Documents (11 terms):} ``electronic health record'', ``clinical notes'', ``discharge summary'', ``medical history'', ``radiology report'', \ldots \\
& \textit{Common Diseases (34 terms):} ``breast cancer'', ``lung cancer'', ``colorectal cancer'', ``prostate cancer'', ``leukemia'', ``lymphoma'',  \ldots \\
& \textit{Clinical Procedures (6 terms):} ``chemotherapy'', ``radiation therapy'', ``transplantation'', ``dialysis'', ``ventilation'', ``ecmo'' \\
& \textit{Age-Specific Care (7 terms):} ``newborn care'', ``child development'', ``growth disorders'', ``birth defects'', ``falls prevention'', ``memory disorders'', ``polypharmacy management'' \\
& \textit{Special Populations (6 terms):} ``maternal health'', ``prenatal care'', ``postpartum care'', ``women's health'', ``social determinants'', ``medical ethics'' \\
\midrule
\textbf{Evaluation Terms} & ``evaluation'', ``accuracy'', ``benchmark'', ``validation'', ``application'' \\
\bottomrule
\end{tabular}
\end{table}

\begin{table}[htbp]
\centering
\caption{Query Construction Pattern and Examples.}
\label{tab:query_examples}
\begin{tabular}{p{0.9\linewidth}}
\toprule
\textbf{Query Pattern:} [LLM Term] AND [Medical Term] AND [Evaluation Term] \\
\midrule
\textbf{Medical Term Selection:} \\
Either [MedQA Terms] OR [Clinical Application Terms] based on research focus \\
\midrule
\textbf{Example Queries:} \\
\textit{With MedQA Terms:}\\
1. large language model'' AND medical question answering'' AND evaluation'' \\
2. ChatGPT'' AND clinical reasoning'' AND benchmark'' \\
\textit{With Clinical Application Terms:}\\
3. GPT-4'' AND electronic health record'' AND validation'' \\
4. language model'' AND internal medicine'' AND application'' \\
5. GPT-4'' AND breast cancer'' AND ``accuracy'' \\
\bottomrule
\end{tabular}
\end{table} 


\subsection{Screening Process for Collecting Performance data of LLM in Real-World Clinical Settings}
\label{apd:screening_by_llm}
We conducted abstract screening and full-text review based on LLM to refine selection process and alleviated burden of manual curation. The LLM utilized for this process was Phi-4 (14.7B), Q4\_K\_M quantized, based on the Ollama framework (as of March 18, 2025). The model was deployed locally on a single RTX 4080 GPU.

\begin{table*}[htbp]
\centering
\renewcommand{\arraystretch}{1.2}
\setlength{\arrayrulewidth}{0.8pt}
\caption{Abstract Screening Prompt Template.}
\label{tab:abstract-screening-prompt}
\begin{tabular}{|p{0.95\textwidth}|}
\hline
\textbf{Prompt for abstract screening} \\
\hline
\small
Please analyze the following research paper's title and abstract to extract information about LLM performance evaluation in clinical settings. Present your analysis in the following structured format, maintaining exact quotes where possible. Start your response with ``ANALYSIS\_START'' \textbf{and end with} ``ANALYSIS\_END''. 
\textbf{INPUT REQUIRED:}
\begin{itemize}
    \item Title: [\textit{paper title}]
    \item Abstract: [\textit{paper abstract}]
\end{itemize}

\textbf{TASK:}
Analyze the title and abstract to extract the following information:

\vspace{0.3em}
1. \texttt{PAPER\_TYPE}: Classify the paper as one of the following:
   \begin{itemize}
       \item \textbf{``Clinical LLM Performance Evaluation - Original''}: Paper that conducts new experiments to evaluate LLM performance in clinical tasks and reports original performance metrics/results.
       \item \textbf{``Clinical LLM Performance Review''}: Paper that summarizes or analyzes existing LLM clinical performance evaluations without conducting new experiments or reporting new performance data.
       \item \textbf{``Non-Clinical LLM Evaluation''}: Paper not related to clinical LLM performance evaluation.
   \end{itemize}

2. \texttt{MODELS}: Extract all LLM models mentioned in the abstract. \\
   Format: [``model1'', ``model2'', ...] \\
   Return empty list if no specific models are mentioned.

3. \texttt{MULTIPLE\_MODELS\_USAGE}: For papers classified as ``Clinical LLM Performance Evaluation - Original'' only. \\
   Format: true/false/NA
   \begin{itemize}
       \item \textbf{true}: Paper clearly evaluates multiple LLMs.
       \item \textbf{false}: Paper clearly focuses on single LLM evaluation.
       \item \textbf{NA}: For non-original clinical LLM evaluation papers.
   \end{itemize}

4. \texttt{HUMAN\_GROUPS}: Extract all medical professional groups that underwent the same evaluation tasks as the models for direct performance comparison. \\
   Format: [``group1'', ``group2'', ...] \\
   Return empty list if no human groups underwent direct performance comparison.

5. \texttt{EVALUATION\_TASKS}: Extract all clinical evaluation tasks. \\
   Format:
   \begin{verbatim}
   [{"task_name_extractive": "exact task name",
     "task_name_abstractive": "standardized name",
     "task_description": "exact quote",
     "metrics_extractive": ["metric1", "metric2"],
     "metrics_abstractive": ["std_metric1", "std_metric2"]}, ...] \end{verbatim}

6. \texttt{PERFORMANCE\_RESULTS}: Extract all reported performance metrics. \\
   Format:
   \begin{verbatim}
   [{"value": "exact value with units",
     "metric": "exact metric name",
     "subject": "model/human group name"}, ...]
   \end{verbatim} \\
\hline
\end{tabular}
\end{table*}

\begin{table*}[htbp]
\centering
\renewcommand{\arraystretch}{1.2}
\setlength{\arrayrulewidth}{0.8pt} % 테두리 두께 조정
\caption{Abstract Screening Prompt Template (continued).}
\label{tab:abstract-screening-prompt}
\begin{tabular}{|p{0.95\textwidth}|}
\hline
\textbf{Prompt for abstract screening (continued)} \\
\hline
\small
\textbf{INPUT EXAMPLE:}
\begin{verbatim}
Title: A Comparison of LLMs in Clinical Triage: Brief Study
Abstract: We evaluated ChatGPT and GEMINI for triaging complex maxillofacial trauma cases at 
a referral center. Using 10 standardized cases, we compared LLM recommendations against
center guidelines. Results showed ChatGPT achieved 70% accuracy in examinations while
GEMINI reached 50%. Additional metrics included diagnosis accuracy scores (GEMINI: 3.30, 
ChatGPT: 2.30) and recommendation relevance (GEMINI: 2.90, ChatGPT: 3.50).
\end{verbatim}

\textbf{EXAMPLE OUTPUT:}
\begin{verbatim}
ANALYSIS_START

<PAPER_TYPE>Clinical LLM Performance Evaluation - Original</PAPER_TYPE>

<MODELS>["ChatGPT", "GEMINI"]</MODELS>

<MULTIPLE_MODELS_USAGE>true</MULTIPLE_MODELS_USAGE>

<HUMAN_GROUPS>[]</HUMAN_GROUPS>

<EVALUATION_TASKS>
[{"task_name_extractive": "triaging complex maxillofacial 
trauma cases",
  "task_name_abstractive": "clinical trauma triage assessment",
  "task_description": "triaging complex maxillofacial trauma cases at a referral center",
  "metrics_extractive": ["accuracy in examinations", "diagnosis accuracy scores", 
  "recommendation relevance"],
  "metrics_abstractive": ["examination accuracy", "diagnostic performance", 
  "recommendation quality"]}]
</EVALUATION_TASKS>

<PERFORMANCE_RESULTS>
[{"value": "70%", "metric": "accuracy in examinations", "subject": "ChatGPT"},
 {"value": "50%", "metric": "accuracy in examinations", "subject": "GEMINI"},
 {"value": "2.30", "metric": "diagnosis accuracy scores", "subject": "ChatGPT"},
 {"value": "3.30", "metric": "diagnosis accuracy scores", "subject": "GEMINI"},
 {"value": "3.50", "metric": "recommendation relevance", "subject": "ChatGPT"},
 {"value": "2.90", "metric": "recommendation relevance", "subject": "GEMINI"}]
</PERFORMANCE_RESULTS>

ANALYSIS_END
\end{verbatim}

\vspace{0.3em}
Now analyzing the following paper:
\begin{itemize}
    \item Title: [\textit{paper title}]
    \item Abstract: [\textit{paper abstract}]
\end{itemize}
\\
\hline
\end{tabular}
\addtocounter{table}{-1}
\end{table*}

\begin{table*}[htbp]
\centering
\renewcommand{\arraystretch}{1.2}
\setlength{\arrayrulewidth}{0.8pt} % 테두리 두께 조정
\begin{tabular}{|p{0.95\textwidth}|}
\hline
\textbf{Prompt for full-text review} \\
\hline
\small
Please analyze the research paper to extract information about LLM performance evaluation in clinical settings. Present your analysis in the following structured format, maintaining exact quotes where possible. Start your response with ``ANALYSIS\_START'' \textbf{and end with} ``ANALYSIS\_END''.

\textbf{REQUIRED:}
\begin{itemize}
    \item Title: [\textit{paper title}]
    \item Full Text: [\textit{full paper text}]
\end{itemize}

\textbf{TASK:}
Extract the following structured information from the paper:

\vspace{0.3em}
1. \texttt{PAPER\_TYPE}: Classify the paper as one of the following:
   \begin{itemize}
       \item \textbf{``Clinical LLM Performance Evaluation - Original''}: Paper that conducts new experiments to evaluate LLM performance in clinical tasks and reports original performance metrics/results.
       \item \textbf{``Clinical LLM Performance Review''}: Paper that summarizes or analyzes existing LLM clinical performance evaluations without conducting new experiments or reporting new performance data.
       \item \textbf{``Non-Clinical LLM Evaluation''}: Paper not related to clinical LLM performance evaluation.
   \end{itemize}

   \textbf{Note:} If the paper is not classified as ``Clinical LLM Performance Evaluation - Original'', return empty values for all subsequent sections.

2. \texttt{BIBLIOGRAPHIC\_DATES}: Extract the paper’s submission and publication dates. \\
   Format:
   \begin{verbatim}
   {"received_date": "YYYY-MM-DD",
    "accepted_date": "YYYY-MM-DD",
    "published_date": "YYYY-MM-DD"}
   \end{verbatim}

3. \texttt{CLINICAL\_DOMAIN}: Extract the clinical specialty and context information. \\
   Format:
   \begin{verbatim}
   {"specialty": "primary clinical specialty field",
    "disease_treatment": "specific diseases or treatments in focus",
    "mesh_terms": ["relevant MeSH term 1", "relevant MeSH term 2"]}
   \end{verbatim}

4. \texttt{MODELS}: Extract all LLM models mentioned in the paper. \\
   Format:
   \begin{verbatim}
   [{"common_name": "most frequently used name in paper",
     "full_name": "complete name including version",
     "base_model": "base model name if fine-tuned, NA if not applicable"}]
   \end{verbatim}

5. \texttt{EXPERIMENTAL\_SETTINGS}: Extract LLM inference settings. \\
   Format:
   \begin{verbatim}
   {"llm_inference_temperature": "0.x",
    "llm_inference_few_shot": "n-shot",
    "llm_inference_CoT": true/false}
   \end{verbatim}

\\
\hline
\end{tabular}
\caption{Full-Text Review Prompt Template.}
\label{tab:full-text-review-prompt}
\end{table*}

\begin{table*}[htbp]
\centering
\renewcommand{\arraystretch}{1.2}
\setlength{\arrayrulewidth}{0.8pt} % 테두리 두께 조정
\caption{Full-Text Review Prompt Template (continued).}
\label{tab:full-text-review-prompt}
\begin{tabular}{|p{0.95\textwidth}|}
\hline
\textbf{Prompt for full-text review (continued)} \\
\hline
\small
6. \texttt{HUMAN\_GROUPS}: Extract all medical professional groups that underwent the same evaluation tasks as the models. \\
   Format:
   \begin{verbatim}
   ["group1", "group2"]
   \end{verbatim}

7. \texttt{EVALUATION\_TASKS}: Extract all clinical evaluation tasks. \\
   Format:
   \begin{verbatim}
   [{"task_name_extractive": "exact task name",
     "task_name_abstractive": "standardized task name",
     "reference_sentence": "exact quote describing the task",
     "metrics": [{"metric_name_extractive": "exact metric name from text",
                  "metric_name_abstractive": "standardized metric name",
                  "value_range": [min, max],
                  "higher_better": true/false,
                  "reference_sentence": "exact quote describing the metric"}],
     "sample_size": integer,
     "sample_size_reference_sentence": "exact quote mentioning sample size",
     "data_source_extractive": "exact quote of data source",
     "data_source_abstractive": "standardized description of data source"}]
   \end{verbatim}

8. \texttt{PERFORMANCE\_RESULTS}: Extract all reported performance metrics. \\
   Format:
   \begin{verbatim}
   [{"value": "exact performance value with units/confidence intervals",
     "metric": "exact metric name from EVALUATION_TASKS metrics_extractive",
     "subject": "model name or human group name",
     "reference_sentence": "exact quote reporting this result"}]
   \end{verbatim}

\\
\hline
\end{tabular}
\addtocounter{table}{-1}  % 테이블 번호 유지
\end{table*}

\subsection{Model Name Normalization}
\label{apd:mode_name_norm}
We normalized language model names by categorizing them into proprietary (Table \ref{tab:model_release_timeline_prop}) and open-source models (Table \ref{tab:model_release_timeline_open}). For proprietary models, specific model names were often unspecified in papers, as they were accessed via APIs. In such cases, we assumed the most recent model available at the research time: three months before the received date for journal papers and six months before publication for conference papers. For open-source models, normalization was based on explicitly stated model names, versions, and parameter sizes in billions (Table \ref{tab:model_release_timeline_open}). If these details were insufficient, we excluded the model from analysis.

\begin{table*}[!ht]
\centering
\footnotesize
\caption{Proprietary Language Models Release Timeline.}
\label{tab:model_release_timeline_prop}
\begin{tabular}{p{2.5cm}p{4.5cm}p{2.5cm}p{5cm}}
\toprule
\textbf{Company} & \textbf{Model Name} & \textbf{Release Date} & \textbf{Normalized Name} \\
\midrule
\multirow{}{}{\textbf{OpenAI}} & \textbf{ChatGPT/GPT-3.5} & 2022-12-30 & gpt-3.5-turbo \\
& \hspace{2mm}gpt-3.5-0301 & 2023-03-01 & gpt-3.5-0301 \\
& \hspace{2mm}gpt-3.5-turbo-0613 & 2023-06-13 & gpt-3.5-turbo-0613 \\
& \hspace{2mm}gpt-3.5-turbo-1106 & 2023-11-06 & gpt-3.5-turbo-1106 \\
& \hspace{2mm}gpt-3.5-turbo-0125 & 2024-01-25 & gpt-3.5-turbo-0125 \\
\cmidrule{2-4}
& \textbf{GPT-4} &  &  \\
& \hspace{2mm}gpt-4-0314 & 2023-03-14 & gpt-4-0314 \\
& \hspace{2mm}gpt-4-0613 & 2023-06-13 & gpt-4-0613 \\
& \hspace{2mm}gpt-4-1106-preview & 2023-11-06 & gpt-4-1106-preview \\
& \hspace{2mm}gpt-4-0125-preview & 2024-01-25 & gpt-4-0125-preview \\
& \hspace{2mm}gpt-4-turbo-2024-04-09 & 2024-04-09 & gpt-4-turbo-2024-04-09 \\
\cmidrule{2-4}
& \textbf{GPT-4o} &  &  \\
& \hspace{2mm}gpt-4o updates & 2024-05-13 & gpt-4o-2024-05-13 \\
& \hspace{2mm}gpt-4o updates & 2024-08-06 & gpt-4o-2024-08-06 \\
& \hspace{2mm}gpt-4o updates & 2024-11-20 & gpt-4o-2024-11-20 \\
& \hspace{2mm}GPT-4o Mini & 2024-07-18 & gpt-4o-mini-2024-07-18 \\
\midrule
\multirow{}{}{\textbf{Microsoft}} & \textbf{Bing Chat} & 2023-02-07 &  \\
& \hspace{2mm}Rebranded as Copilot & 2023-09-21 & Based on latest GPT models\\
& \hspace{2mm}Bing Chat integration & 2023-11-15 & Based on latest GPT models\\
& \hspace{2mm}Copilot upgrade & 2024-05-20 & Based on latest GPT models\\
\midrule
\multirow{}{}{\textbf{Claude}} & \textbf{Claude 1} & &  \\
& \hspace{2mm}Claude 1.0/Claude 1.1 & 2023-03-14 & claude-1.0/claude-1.1 \\
& \hspace{2mm}Claude 1.2 & 2023-08-09 & claude-1.2 \\
& \hspace{2mm}Claude 1.3 & 2023-04-18 & claude-1.3 \\
\cmidrule{2-4}
& \textbf{Claude 2} &  &  \\
& \hspace{2mm}Claude 2.0 & 2023-17-11 & claude-2.0 \\
& \hspace{2mm}Claude 2.1 & 2023-11-21 & claude-2.1 \\
\cmidrule{2-4}
& \textbf{Claude 3} &  &  \\
& \hspace{2mm}Claude 3 Haiku & 2024-03-07 & claude-3-haiku-20240307 \\
& \hspace{2mm}Claude 3 Sonnet & 2024-02-29 & claude-3-sonnet-20240229 \\
& \hspace{2mm}Claude 3 Opus & 2024-02-29 & claude-3-opus-20240229 \\
\cmidrule{2-4}
& \textbf{Claude 3.5} &  &  \\
& \hspace{2mm}Claude 3.5 Sonnet & 2024-06-20 & claude-3-5-sonnet-20240622 \\
& \hspace{2mm}Claude 3.5 Haiku & 2024-10-22 & claude-3-5-haiku-20241022 \\
& \hspace{2mm}Claude 3.5 Opus & 2024-10-22 & claude-3-5-opus-20241022 \\
\cmidrule{2-4}
& \textbf{Claude 3.5 update} & 2024-12-03 & claude-3.5-sonnet-20241203 \\
\midrule
\multirow{}{}{\textbf{Google}} & \textbf{Bard} & 2023-03-21 & lamda \\
& \hspace{2mm}Bard upgrade & 2023-05-10 & palm-2 \\
\cmidrule{2-4}
& \textbf{Gemini 1.0} &  &  \\
& \hspace{2mm}Gemini 1.0 Nano & 2023-12-06 & gemini-1.0-nano \\
& \hspace{2mm}Gemini 1.0 Pro & 2023-12-06 & gemini-1.0-pro \\
& \hspace{2mm}Gemini 1.0 Ultra (Advanced) & 2023-12-06 & gemini-1.0-ultra \\
\cmidrule{2-4}
& \textbf{Gemini 1.5} &  & \\
& \hspace{2mm}Gemini 1.5 Flash (Basic) & 2024-02-15 & gemini-1.5-flash \\
& \hspace{2mm}Gemini 1.5 Pro & 2024-05-23 & gemini-1.5-pro-001 \\
& \hspace{2mm}Gemini 1.5 Pro update & 2024-09-24 & gemini-1.5-pro-002 \\
\cmidrule{2-4}
& \textbf{Gemini 2.0} & &  \\
& \hspace{2mm}Gemini 2.0 & 2025-01-22 & gemini-2.0-flash-001 \\
& \hspace{2mm}Gemini 2.0 Flash + Thinking & 2025-01-22 & Not used \\
\midrule
\multirow{}{}{\textbf{Cohere}} & \textbf{Command} & 2024-02-07 & command \\
& \hspace{2mm}Command R & 2024-06-04 & command-r \\
& \hspace{2mm}Command R+ & 2024-04-30 & command-rplus \\
& \hspace{2mm}Command R-08-2024 & 2024-08-28 & command-r-2408 \\
& \hspace{2mm}Command R+ 08-2024 & 2024-08-29 & command-rplus-2408 \\
\bottomrule
\end{tabular}
\label{tab:model_release_timeline_prop}
\end{table*}

\begin{table*}[!ht]
\centering
\footnotesize
\caption{Open-Source Language Models Release Timeline.}
\label{tab:model_release_timeline_open}
\begin{tabular}{p{4.3cm}p{3.5cm}p{2.0cm}p{5cm}}
\toprule
\textbf{Model Brand} & \textbf{Model Name} & \textbf{Release Date} & \textbf{Normalized Name} \\
\midrule
\multirow{}{}{\textbf{LLaMA (Meta)}} & \textbf{LLaMA 1} & 2023-03 & llama-1-7B, llama-1-13B, llama-1-30B, llama-1-65B \\
\cmidrule{2-4}
& \textbf{LLaMA 2} & 2023-07 & llama-2-7B, llama-2-13B, llama-2-70, llama-2-7B-chat, llama-2-13B-chat \\
\cmidrule{2-4}
& \textbf{LLaMA 3} & 2024-04-18 & llama-3-8B, llama-3-70B \\
\cmidrule{2-4}
& \textbf{LLaMA 3.1} & 2024-07-23 & llama-3.1-8B, llama-3.1-70B, llama-3.1-405B \\
\cmidrule{2-4}
& \textbf{LLaMA 3.2} & 2024-10 & llama-3.2-1B, llama-3.2-3B, llama-3.2-11B, llama-3.2-90B \\
\cmidrule{2-4}
& \textbf{LLaMA 3.3} & 2024-12 & llama-3.2-70B, llama-3.2-405B \\
\midrule
\multirow{}{}{\textbf{Phi (Microsoft)}} & \textbf{Phi-1} & 2023-06 & phi-1-1.3B \\
& \textbf{Phi-1.5} & 2023-11 & phi-1.5-1.3B \\
& \textbf{Phi-2} & 2024-02 & phi-2-2.7B \\
\cmidrule{2-4}
& \textbf{Phi-3} &  &  \\
& \hspace{2mm}Phi-3 Mini & NA & phi-3-mini-3B \\
& \hspace{2mm}Phi-3 Small & NA & phi-3-small-7B \\
& \hspace{2mm}Phi-3 Medium & NA & phi-3-medium-14B \\
\cmidrule{2-4}
& \textbf{Phi-3.5} & 2024-09 & phi-3.5-3.8B \\
\cmidrule{2-4}
& \textbf{Phi-4} & 2025-01-20 & Not used \\
\midrule
\multirow{}{}{\textbf{Gemma (DeepMind)}} & \textbf{Gemma 2B} & 2024-02-21 & gemma-1-2B \\
& \textbf{Gemma 7B} & 2024-02-21 & gemma-1-7B \\
& \textbf{Gemma 1.1} & 2024-04-05 & gemma-1.1 \\
\cmidrule{2-4}
& \textbf{Gemma 2 (9B, 27B)} & 2024-06-27 & gemma-2-9B, gemma-2-27B \\
& \textbf{Gemma 2 (2B)} & 2024-07-31 & gemma-2-2B \\
\midrule
\multirow{}{}{\textbf{Qwen (Alibaba)}} & \textbf{Qwen-7B} & 2023-08-03 & qwen-7B \\
& \textbf{Qwen-14B} & 2023-09-25 & qwen-14B \\
& \textbf{Qwen-72B} & 2023-11-30 & qwen-72B \\
& \textbf{Qwen-2-7B-instruct} & 2024-05-16 & qwen-2-7B-instruct \\
& \textbf{Qwen-2-72B-instruct} & 2024-10-18 & qwen-2-72B-instruct \\
& \textbf{Qwen Max} & 2025-01-29 & qwen-max \\
\midrule
\multirow{}{}{\textbf{Mistral}} & \textbf{Mistral 7B} & & \\
& \hspace{2mm}mistral-7B-instruct-v0.1 & 2023-09-27  & mistral-7B-instruct-v0.1 \\
& \hspace{2mm}mistral-7B-instruct-v0.2 & 2023-10 & mistral-7B-instruct-v0.2 \\
& \hspace{2mm}mistral-7B-instruct-v0.3 & 2023-11 & mistral-7B-instruct-v0.3 \\
\cmidrule{2-4}
& \textbf{Mistral Medium} & 2023-12 & mistral-medium-2312 \\
& \textbf{Mixtral 8x7B} & 2023-12-09 & mixtral-8x7B \\
& \textbf{Mixtral 8x22B} & 2024-04-10 & mixtral-8x22B \\
\cmidrule{2-4}
& \textbf{Mistral Large} & 2024-02-26 & mistral-large-2402 \\
& \textbf{Mistral Small} & NA & mistral-small-2402 \\
\cmidrule{2-4}
& \textbf{Mistral Large 24.07} & 2024-07-24 & mistral-large-2707 \\
\midrule\\
\multirow{}{}{\textbf{Medical Domain Fine-tuned}} & \textbf{ClinicalCamel-1-70B} & NA & clinicalcamel-1-70B \\
& \textbf{Med42-70B} & NA & med42-70B \\
& \textbf{BioMistral-7B} & NA & biomistral-7B \\
& \textbf{Meditron} & NA & meditron-7B \\
& \textbf{MedLlama} & NA & medllama-1-2 \\
\bottomrule
\end{tabular}
\end{table*}

\subsection{Classification of Evaluation Settings}
\label{apd:classif_eval_settings}
We categorized the evaluation settings for language models in clinical contexts based on four key criteria: (1) task type, (2) therapeutic area, (3) data source, and (4) evaluation method.

\paragraph{Task Type}  
Task types represent the core capabilities being assessed, classified into the following six categories:
\begin{itemize}
    \item \textbf{Clinical Knowledge:} General assessment of fundamental clinical knowledge within a given specialty, without a specific focus on diagnosis, treatment, or prevention.
    \item \textbf{Treatment:} Evaluation of the model’s ability to recommend and assess treatment plans based on a given clinical scenario.
    \item \textbf{Diagnosis:} Determining the correct diagnosis based on the provided patient information.
    \item \textbf{Answering to Patients:} Providing responses to common patient inquiries or explaining clinical conditions in plain language understandable by non-experts.
    \item \textbf{Overall Management:} Beyond diagnosis and treatment, evaluating long-term patient management and decision-making.
    \item \textbf{Information Extraction:} Extracting specific clinical information from given texts.
\end{itemize}

\paragraph{Therapeutic Area}  
For normalization and analysis purposes, we predefined 22 therapeutic areas, including cardiology, oncology, dentistry, and emergency medicine. Depending on the analytical objectives, these areas were further grouped into broader categories.

\paragraph{Data Source}  
The source of evaluation data was classified into four types:
\begin{itemize}
    \item \textbf{Board Examinations:} Questions derived from professional board certification exams used to assess medical expertise.
    \item \textbf{Quizzes:} Clinical questions sourced from textbooks, medical societies, or online educational platforms, excluding board exams.
    \item \textbf{Frequently Asked Questions:} Questions reflecting common patient inquiries in clinical settings.
    \item \textbf{Clinical Vignettes:} Case-based questions developed using real patient data, publicly available case reports, or LLM-generated simulated patient scenarios.
\end{itemize}

\paragraph{Evaluation Method}  
Evaluation methods were divided into two main categories:
\begin{itemize}
    \item \textbf{Multiple-Choice Questions (MCQs):} Assessing correctness based on predefined answer choices.
    \item \textbf{Human Rating:} Clinical experts rating model-generated responses according to structured evaluation guidelines. This includes both closed-ended rating systems with predefined criteria and open-ended assessments.
\end{itemize}

\subsection{Benchmark Performance Imputation}
\label{apd:imputation_on_bench_method}
The MICE framework was configured with optimized settings to ensure stable imputation. Missing values were initially imputed using the median of observed values, followed by a maximum of 50 iterative updates with a convergence tolerance of \( 1 \times 10^{-6} \)

For BR-MICE, posterior sampling was enabled, and each missing variable was modeled using all available predictors. The base random seed was set to 42, with independent seeds assigned for multiple imputations.

For RF-MICE, 100 trees were used with bootstrap sampling enabled. The maximum tree depth was set to 15, and feature selection per split followed the square root of the total number of features. BR-MICE regularization parameters were optimized iteratively, with convergence determined via evidence maximization.

To estimate the variance of imputed values, we computed both the within-imputation variance (\( W \)) and between-imputation variance (\( B \)) across \( m \) independent imputations. The total variance (\( T \)) was calculated using Rubin’s rules:

\[
W = \frac{1}{m} \sum_{j=1}^{m} S_j^2
\]

\[
B = \frac{1}{m-1} \sum_{j=1}^{m} (\bar{Q}_j - \bar{Q})^2
\]

\[
T = W + \left(1 + \frac{1}{m} \right) B
\]

where \( S_j^2 \) is the variance of the \( j \)-th imputed dataset, \( \bar{Q}_j \) is the mean of the \( j \)-th imputation, and \( \bar{Q} \) is the overall mean of all imputations. The final imputed values were obtained by taking the median of all imputations to ensure robustness against extreme values.

\subsection{Bayesian Modeling}
\label{apd:bayesian_modeling}
We implemented our hierarchical Bayesian model usingNumPyro (v0.17.0) with a JAX (v0.5.0) backend. Posterior inference was conducted via the No-U-Turn Sampler (NUTS), utilizing 1,000 warmup iterations and 2,000 sampling iterations across 8 parallel chains. We assessed convergence using the Gelman-Rubin diagnostic ($\hat{R}$) and effective sample size. The model specification is as follows:

\subsubsection{Model Structure}
\label{apd:model_structure_bayesian}
We formulate our hierarchical Bayesian model as follows:
We begin by specifying half-normal hyperpriors for the standard deviations that govern the variability of different components in our model:
\begin{align*}
\sigma_{\text{model}} &\sim \text{HalfNormal}(1) \\
\sigma_{\text{obs}} &\sim \text{HalfNormal}(1) \\
\sigma_{\text{type}} &\sim \text{HalfNormal}(1) \\
\sigma_{\text{source}} &\sim \text{HalfNormal}(1) \\
\sigma_{\text{eval}} &\sim \text{HalfNormal}(1) 
\end{align*}
These hyperpriors control the variation in model effects, observation noise, task type effects, data source effects, and evaluation method effects, respectively.

The model effects component captures the inherent performance capabilities of each language model:
\begin{align*}
\mu_{\text{model}} &\sim \text{Normal}(0, 1) \\
\beta_{\text{model}, j} &\sim \text{Normal}(\mu_{\text{model}}, \sigma_{\text{model}})
\end{align*}
where $j = 1, 2, \ldots, n_{\text{models}}$, and $\beta_{\text{model}, j}$ represents the effect of the $j$-th model. The parameter $\mu_{\text{model}}$ serves as a global mean for model effects.

We model three task-related components:
\begin{align*}
\beta_{\text{type}, k} &\sim \text{Normal}(0, \sigma_{\text{type}})\\
\beta_{\text{source}, l} &\sim \text{Normal}(0, \sigma_{\text{source}})\\
\beta_{\text{eval}, m} &\sim \text{Normal}(0, \sigma_{\text{eval}})
\end{align*}
where:
\begin{itemize}
\item $k = 1, 2, \ldots, n_{\text{task\_types}}$, with $\beta_{\text{type}, k}$ representing the effect of the $k$-th task type
\item $l = 1, 2, \ldots, n_{\text{data\_sources}}$, with $\beta_{\text{source}, l}$ representing the effect of the $l$-th data source
\item $m = 1, 2, \ldots, n_{\text{evaluation\_methods}}$, with $\beta_{\text{eval}, m}$ representing the effect of the $m$-th evaluation method
\end{itemize}
Each task-related effect is centered at zero, reflecting our assumption that these effects represent deviations from an average difficulty level.

The predicted performance for each data point $i$ is given by:
\begin{align*}
\mu_i = \beta_{\text{model}, \text{model}[i]} + \beta_{\text{type}, \text{type}[i]} + \beta_{\text{source}, \text{source}[i]} \\
+ \beta_{\text{eval}, \text{eval}[i]} + \epsilon_{\mu}
\end{align*}
where $\text{model}[i]$, $\text{type}[i]$, $\text{source}[i]$, and $\text{eval}[i]$ are the indices for model, task type, data source, and evaluation method for data point $i$, respectively, and $\epsilon_{\mu} \sim \text{Normal}(0, 0.1)$ represents additional noise in the prediction process.

Finally, we model the observed performance metrics using a normal likelihood:
\begin{align*}
y_i \sim \text{Normal}(\mu_i, \sigma_{\text{obs}})
\end{align*}
where $y_i$ is the observed performance metric for data point $i$ on the normalized scale.

\subsubsection{Centrality Measurement}
\label{apd:centrality_measure}
Quantifying the connectivity of models within the evaluation network is essential for understanding their role and influence. Some models exhibit weak connections to other major models, meaning they contribute useful information to Bayesian modeling but have limited relevance for downstream analysis. By computing centrality scores, we classified models based on their connectivity and excluded the lower 50\% from downstream evaluations.

The evaluation network was represented as a bipartite graph $G = (V, E)$, where the vertex set $V$ consisted of two disjoint subsets: models and tasks. An edge $(m, t) \in E$ was formed if and only if model $m$ was evaluated on task $t$. This structure provided a basis for analyzing connectivity patterns and assessing the relative importance of models within the evaluation framework.

Model connectivity was quantified using three centrality measures. Degree centrality ($C_D$) captured the number of direct connections a model had, normalized by the maximum possible connections:

\begin{equation*}
    C_D(v) = \frac{deg(v)}{|V| - 1}
\end{equation*}
where $deg(v)$ represents the number of edges incident to node $v$. Between-ness centrality ($C_B$) measured how often a model served as a bridge along the shortest paths between other nodes:

\begin{equation*}
    C_B(v) = \sum_{s \neq v \neq t} \frac{\sigma_{st}(v)}{\sigma_{st}}
\end{equation*}
where $\sigma_{st}$ is the total number of shortest paths from node $s$ to node $t$, and $\sigma_{st}(v)$ is the number of those paths passing through node $v$. Closeness centrality ($C_C$) assessed how close a model was to all other nodes in the network:

\begin{equation*}
    C_C(v) = \frac{|V| - 1}{\sum_{u \neq v} d(v, u)}
\end{equation*}
where $d(v, u)$ is the shortest-path distance between nodes $v$ and $u$.
To integrate these measures into a single ranking, a combined connectivity score was computed by summing the three normalized centrality values:

\begin{equation*}
    \text{Combined Score}(m) = C_D(m) + C_B(m) + C_C(m)
\end{equation*}

Models were then ranked based on their combined scores, and those below the $P_{th}$ percentile were classified as low-connectivity models:

\begin{equation*}
    \text{Low Connectivity}(m) = 
    \begin{cases}
        \text{True} & \text{if Combined-} \\ & \text{  Score}(m) < P_{th} \\
        \text{False} & \text{otherwise}
    \end{cases}
\end{equation*}
where $P_{th}$ was set at the 50th percentile, identifying the bottom 50\% of models as low-connectivity.
For downstream analysis, only high-connectivity models were retained. This ensured that subsequent evaluations focused on models with strong integration within the network while still utilizing all available information in Bayesian modeling.


\subsection{Correlation Measurement}
To evaluate the relationship between LLM performance on different benchmarks and in clinical settings, we computed four correlation measures: Pearson’s correlation coefficient, Spearman’s rank correlation coefficient, Kendall’s tau, and Lin’s concordance correlation coefficient (CCC). Among these, Spearman’s and Kendall’s correlations were used as the primary measures, as they better capture rank-based relationships given the diversity of evaluation methodologies.

Pearson’s correlation coefficient ($r$) measures the strength of the linear relationship between two continuous variables. It is computed as:

\begin{equation*}
r = \frac{\sum (x_i - \bar{x})(y_i - \bar{y})}{\sqrt{\sum (x_i - \bar{x})^2} \sqrt{\sum (y_i - \bar{y})^2}}
\end{equation*}

where $x_i$ and $y_i$ are individual data points, and $\bar{x}$ and $\bar{y}$ are their respective means.

Spearman’s rank correlation coefficient ($\rho$) assesses the monotonic relationship between two variables by comparing their rank orders rather than raw values. It is given by:

\begin{equation*}
\rho = 1 - \frac{6 \sum d_i^2}{n(n^2 - 1)}
\end{equation*}

where $d_i$ is the rank difference for each pair of observations, and $n$ is the number of observations.

Kendall’s tau ($\tau$) quantifies the ordinal association between two variables based on concordant and discordant pairs:

\begin{equation*}
\tau = \frac{C - D}{\frac{1}{2} n (n-1)}
\end{equation*}

where $C$ is the number of concordant pairs, and $D$ is the number of discordant pairs.

Lin’s CCC ($\rho_c$) evaluates both correlation and agreement between two variables by incorporating measures of precision and accuracy:

\begin{equation*}
\rho_c = \frac{2 r \sigma_x \sigma_y}{\sigma_x^2 + \sigma_y^2 + (\mu_x - \mu_y)^2}
\end{equation*}

where $r$ is Pearson’s correlation coefficient, $\sigma_x$ and $\sigma_y$ are standard deviations, and $\mu_x$ and $\mu_y$ are means of the two variables.



\section{Supplementary Results}
\label{apd:supple_results}

\subsection{Benchmark Performance Imputation}
Table \ref{tab:imputatoin_masking_test} presents the imputation accuracy of benchmark models evaluated through a masking test across both general and medical QA domains.

\begin{table}[htbp]
    \centering
    \caption{Imputation accuracy on masking test for benchmark performances.}
    \label{tab:imputatoin_masking_test}
    \begin{tabularx}{\linewidth}{l>{\centering\arraybackslash}X>{\centering\arraybackslash}X>{\centering\arraybackslash}X}  
        \hline
        \textbf{Model} & \textbf{MAE} & \textbf{RMSE} & \textbf{R$^2$} \\
        \hline
        \multicolumn{4}{l}{\textbf{General}} \\
        RandomForest   & 4.21 & 8.17 & 0.89 \\
        BayesianRidge  & 5.63 & 8.14 & 0.89 \\
        \hline
        \multicolumn{4}{l}{\textbf{Medical QA}} \\
        RandomForest   & 2.04 & 3.18 & 0.98 \\
        BayesianRidge  & 4.14 & 6.81 & 0.90 \\
        \hline
    \end{tabularx}
\end{table}


\begin{table}[htbp]
    \centering
    \caption{Within- and between- variance results from multiple imputation.}
    \label{tab:multiple_imputation}
    \begin{tabular}{lccc}
        \hline
        \textbf{Category} & \textbf{Within} & \textbf{Between} & \textbf{Total} \\
        \hline
        \multicolumn{4}{l}{\textbf{General Benchmarks (Overall Variance: 481.1)}} \\
        RandomForest   & 1.1  & 204.5  & 215.7  \\
        \quad (\% of Overall) & 0.2\%  & 42.5\%  & 44.8\%  \\
        BayesianRidge  & 100.6  & 189.4  & 299.5  \\
        \quad (\% of Overall) & 20.9\%  & 39.4\%  & 62.3\%  \\
        \hline
        \multicolumn{4}{l}{\textbf{Medical Benchmarks (Overall Variance: 390.5)}} \\
        RandomForest   & 2.1  & 43.2  & 47.5  \\
        \quad (\% of Overall) & 0.5\%  & 11.1\%  & 12.2\%  \\
        BayesianRidge  & 67.5  & 40.0  & 109.5  \\
        \quad (\% of Overall) & 17.3\%  & 10.2\%  & 28.0\%  \\
        \hline
    \end{tabular}
\end{table}


\subsection{Benchmark-to-Benchmark Correlation}
\label{apd:bench_to_bench_corr}
Table \ref{tab:bench_to_bench_corr_general} summarizes the average correlation coefficients between general benchmarks and other benchmarks, providing a comparative view across domains and correlation metrics. 

Figures \ref{fig:medical_correlation_panel}, \ref{fig:general_correlation_panel}, and \ref{fig:cross_correlation_panel} further illustrate the internal correlations within each domain and the cross-domain relationships.

\begin{table*}[htbp]
    \centering
    \tiny
    \caption{Average correlation coefficients of general benchmarks with other benchmarks. 
    The highest score in each column is \textbf{bold}, and the second highest is \underline{underlined}.}
    \label{tab:bench_to_bench_corr_general}
    \footnotesize
    \resizebox{\textwidth}{!}{%
    \begin{tabular}{l cc cc}
        \hline
        \textbf{General Benchmark} & \multicolumn{2}{c}{\textbf{Spearman}} & \multicolumn{2}{c}{\textbf{Kendall}} \\
        \cline{2-3} \cline{4-5}
        & \textbf{Medical QA} & \textbf{General} & \textbf{Medical QA} & \textbf{General} \\
        \hline
        MMLU                & \underline{0.853}  & 0.715  & \underline{0.690}  & 0.576 \\
        MMLU Pro            & 0.851  & \textbf{0.773}  & 0.679  & \textbf{0.628} \\
        BBH                 & \textbf{0.891}  & 0.725  & \textbf{0.736}  & 0.573 \\
        HumanEval           & 0.838  & \underline{0.764}  & 0.671  & \underline{0.618} \\
        GSM8K               & 0.816  & 0.756  & 0.645  & 0.599 \\
        MATH                & 0.785  & 0.625  & 0.618  & 0.502 \\
        \hline
        \textbf{Average}    & \textbf{0.839}  & \textbf{0.726}  & \textbf{0.673}  & \textbf{0.583} \\
        \hline
    \end{tabular}%
    }
\end{table*}

\begin{figure*}[htbp]
    \centering
    \includegraphics[width=\textwidth]{medical_correlation_panel.pdf}
    \caption{Correlations within medical QA benchmarks.}
    \label{fig:medical_correlation_panel}
\end{figure*}

\begin{figure*}[htbp]
    \centering
    \includegraphics[width=\textwidth]{general_correlation_panel.pdf}
    \caption{Correlations within general benchmarks.}
    \label{fig:general_correlation_panel}
\end{figure*}

\begin{figure*}[htbp]
    \centering
    \includegraphics[width=\textwidth]{cross_correlation_panel.pdf}
    \caption{Correlations between general and medical QA benchmarks.}
    \label{fig:cross_correlation_panel}
\end{figure*}


\subsection{Benchmark-to-Clinical Performance Correlation}
\label{apd:bench_to_clinical_corr}

Figures \ref{fig:correlation_plot_w_imput_w_weighting} and \ref{fig:correlation_plot_wo_imput_w_weighting} present the correlations between benchmark performance and language model performance in real-world clinical settings, with and without imputed benchmark scores. The results are reported using four correlation measures—Pearson correlation coefficient, Spearman rank correlation coefficient, Kendall's tau, and Lin’s CCC—to ensure robustness from multiple statistical perspectives.

\begin{figure*}[htbp]
    \centering
    \includegraphics[width=\textwidth]{correlation_plot_imputated_weighted.pdf}
    \caption{Correlations between benchmark performance and language model performance in real-world clinical settings with imputed benchmark scores.}
    \label{fig:correlation_plot_w_imput_w_weighting}
\end{figure*}

\begin{figure*}[htbp]
    \centering
    \includegraphics[width=\textwidth]{correlation_plot_weighted.pdf}
    \caption{Correlations between benchmark performance and language model performance in real-world clinical settings with non-imputed benchmark scores.}
    \label{fig:correlation_plot_wo_imput_w_weighting}
\end{figure*}

\end{document}
