@inproceedings{wilken-etal-2022-suber,
    title = "{S}ub{ER} - A Metric for Automatic Evaluation of Subtitle Quality",
    author = "Wilken, Patrick  and
      Georgakopoulou, Panayota  and
      Matusov, Evgeny",
    editor = "Salesky, Elizabeth  and
      Federico, Marcello  and
      Costa-juss{\`a}, Marta",
    booktitle = "Proceedings of the 19th International Conference on Spoken Language Translation (IWSLT 2022)",
    month = may,
    year = "2022",
    address = "Dublin, Ireland (in-person and online)",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.iwslt-1.1",
    doi = "10.18653/v1/2022.iwslt-1.1",
    pages = "1--10",
    abstract = "This paper addresses the problem of evaluating the quality of automatically generated subtitles, which includes not only the quality of the machine-transcribed or translated speech, but also the quality of line segmentation and subtitle timing. We propose SubER - a single novel metric based on edit distance with shifts that takes all of these subtitle properties into account. We compare it to existing metrics for evaluating transcription, translation, and subtitle quality. A careful human evaluation in a post-editing scenario shows that the new metric has a high correlation with the post-editing effort and direct human assessment scores, outperforming baseline metrics considering only the subtitle text, such as WER and BLEU, and existing methods to integrate segmentation and timing features.",
}