@inproceedings{chung-etal-2024-stable,
    title = "Stable Language Model Pre-training by Reducing Embedding Variability",
    author = "Chung, Woojin  and
      Hong, Jiwoo  and
      An, Na Min  and
      Thorne, James  and
      Yun, Se-Young",
    editor = "Al-Onaizan, Yaser  and
      Bansal, Mohit  and
      Chen, Yun-Nung",
    booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing",
    month = nov,
    year = "2024",
    address = "Miami, Florida, USA",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-sig-urls/2024.emnlp-main.606/",
    doi = "10.18653/v1/2024.emnlp-main.606",
    pages = "10852--10863",
    abstract = "Stable pre-training is essential for achieving better-performing language models. However, tracking pre-training stability is impractical due to high computational costs. We study Token Embedding Variability as a simple proxy to estimate pre-training stability. We theoretically and empirically demonstrate that Multi-head Low-Rank Attention acts as a fundamental approach to reducing instability. This is supported by empirical findings on variants on GPT-2, demonstrating improved stability and lower perplexities, even at deeper layer counts."
}