@inproceedings{dolev-etal-2024-whisper,
    title = "Does {W}hisper Understand {S}wiss {G}erman? An Automatic, Qualitative, and Human Evaluation",
    author = {Dolev, Eyal  and
      Lutz, Clemens  and
      Aepli, No{\"e}mi},
    editor = {Scherrer, Yves  and
      Jauhiainen, Tommi  and
      Ljube{\v{s}}i{\'c}, Nikola  and
      Zampieri, Marcos  and
      Nakov, Preslav  and
      Tiedemann, J{\"o}rg},
    booktitle = "Proceedings of the Eleventh Workshop on NLP for Similar Languages, Varieties, and Dialects (VarDial 2024)",
    month = jun,
    year = "2024",
    address = "Mexico City, Mexico",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/add-emnlp-2024-awards/2024.vardial-1.3/",
    doi = "10.18653/v1/2024.vardial-1.3",
    pages = "28--40",
    abstract = {Whisper is a state-of-the-art automatic speech recognition (ASR) model (Radford et al., 2022). Although Swiss German dialects are allegedly not part of Whisper`s training data, preliminary experiments showed Whisper can transcribe Swiss German quite well, with the output being a speech translation into Standard German. To gain a better understanding of Whisper`s performance on Swiss German, we systematically evaluate it using automatic, qualitative, and human evaluation. We test its performance on three existing test sets: SwissDial (Dogan-Sch{\"o}nberger et al., 2021), STT4SG-350 (Pl{\"u}ss et al., 2023), and Swiss Parliaments Corpus (Pl{\"u}ss et al., 2021). In addition, we create a new test set for this study based on short mock clinical interviews. To automatically evaluate performance, we used word error rate (WER) and BLEU. We also conducted a qualitative analysis of Whisper`s performance, discussing its strengths and weaknesses. Finally, 28 people participated in a survey evaluating Whisper`s performance. All of our evaluations showed that Whisper is a viable ASR system for Swiss German, so long as the Standard German output is desired.}
}