@inproceedings{kishino-etal-2026-establishing,
    title = "Establishing a Scale for {K}ullback-{L}eibler Divergence in Language Models Across Various Settings",
    author = "Kishino, Ryo  and
      Takase, Yusuke  and
      Oyama, Momose  and
      Yamagiwa, Hiroaki  and
      Shimodaira, Hidetoshi",
    editor = "Liakata, Maria  and
      Moreira, Viviane P.  and
      Zhang, Jiajun  and
      Jurgens, David",
    booktitle = "Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2026",
    month = jul,
    year = "2026",
    address = "San Diego, California, United States",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/ingest-acl/2026.findings-acl.1163/",
    pages = "23223--23248",
    ISBN = "979-8-89176-395-1",
    abstract = "Log-likelihood vectors define a common space for comparing language models as probability distributions, enabling unified comparisons across heterogeneous settings. We extend this framework to training checkpoints and intermediate layers, and establish a consistent scale for KL divergence across pretraining, model size, random seeds, quantization, fine-tuning, and layers. Analysis of Pythia pretraining trajectories further shows that changes in log-likelihood space, as measured by the scaling behavior of KL divergence, are much smaller than in weight space, resulting in subdiffusive learning trajectories and early stabilization of language-model behavior despite weight drift."
}