@inproceedings{ackerman-etal-2024-novel,
    title = "A Novel Metric for Measuring the Robustness of Large Language Models in Non-adversarial Scenarios",
    author = "Ackerman, Samuel  and
      Rabinovich, Ella  and
      Farchi, Eitan  and
      Anaby Tavor, Ateret",
    editor = "Al-Onaizan, Yaser  and
      Bansal, Mohit  and
      Chen, Yun-Nung",
    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2024",
    month = nov,
    year = "2024",
    address = "Miami, Florida, USA",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-sig-urls/2024.findings-emnlp.158/",
    doi = "10.18653/v1/2024.findings-emnlp.158",
    pages = "2794--2802",
    abstract = "We evaluate the robustness of several large language models on multiple datasets. Robustness here refers to the relative insensitivity of the model{'}s answers to meaning-preserving variants of their input. Benchmark datasets are constructed by introducing naturally-occurring, non-malicious perturbations, or by generating semantically equivalent paraphrases of input questions or statements. We further propose a novel metric for assessing a model robustness, and demonstrate its benefits in the non-adversarial scenario by empirical evaluation of several models on the created datasets."
}