@inproceedings{zufle-niehues-2025-contrastive,
    title = "Contrastive Learning for Task-Independent {S}peech{LLM}-Pretraining",
    author = {Z{\"u}fle, Maike  and
      Niehues, Jan},
    editor = "Che, Wanxiang  and
      Nabende, Joyce  and
      Shutova, Ekaterina  and
      Pilehvar, Mohammad Taher",
    booktitle = "Findings of the Association for Computational Linguistics: ACL 2025",
    month = jul,
    year = "2025",
    address = "Vienna, Austria",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/mtsummit-25-ingestion/2025.findings-acl.445/",
    doi = "10.18653/v1/2025.findings-acl.445",
    pages = "8469--8490",
    ISBN = "979-8-89176-256-5",
    abstract = "Large language models (LLMs) excel in natural language processing but adapting these LLMs to speech processing tasks efficiently is not straightforward. Direct task-specific fine-tuning is limited by overfitting risks, data requirements, and computational costs. To address these challenges, we propose a scalable, two-stage training approach: (1) A task-independent speech pretraining stage using contrastive learning to align text and speech representations over all layers, followed by (2) a task-specific fine-tuning stage requiring minimal data. This approach outperforms traditional ASR pretraining and enables the model to surpass models specialized on speech translation and question answering while being trained on only 10{\%} of the task-specific data."
}