@inproceedings{guo-etal-2023-chatgpt,
title = "Is {C}hat{GPT} a Financial Expert? Evaluating Language Models on Financial Natural Language Processing",
author = "Guo, Yue and
Xu, Zian and
Yang, Yi",
editor = "Bouamor, Houda and
Pino, Juan and
Bali, Kalika",
booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2023",
month = dec,
year = "2023",
address = "Singapore",
publisher = "Association for Computational Linguistics",
url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2023.findings-emnlp.58/",
doi = "10.18653/v1/2023.findings-emnlp.58",
pages = "815--821",
abstract = "The emergence of Large Language Models (LLMs), such as ChatGPT, has revolutionized general natural language preprocessing (NLP) tasks. However, their expertise in the financial domain lacks a comprehensive evaluation. To assess the ability of LLMs to solve financial NLP tasks, we present FinLMEval, a framework for Financial Language Model Evaluation, comprising nine datasets designed to evaluate the performance of language models. This study compares the performance of fine-tuned auto-encoding language models (BERT, RoBERTa, FinBERT) and the LLM ChatGPT. Our findings reveal that while ChatGPT demonstrates notable performance across most financial tasks, it generally lags behind the fine-tuned expert models, especially when dealing with proprietary datasets. We hope this study builds foundation evaluation benchmarks for continuing efforts to build more advanced LLMs in the financial domain."
}
Markdown (Informal)
[Is ChatGPT a Financial Expert? Evaluating Language Models on Financial Natural Language Processing](https://preview.aclanthology.org/jlcl-multiple-ingestion/2023.findings-emnlp.58/) (Guo et al., Findings 2023)
ACL