@inproceedings{horn-elliott-2025-tracking,
    title = "Tracking Universal Features Through Fine-Tuning and Model Merging",
    author = "Horn, Niels Nielsen  and
      Elliott, Desmond",
    editor = "Adlakha, Vaibhav  and
      Chronopoulou, Alexandra  and
      Li, Xiang Lorraine  and
      Majumder, Bodhisattwa Prasad  and
      Shi, Freda  and
      Vernikos, Giorgos",
    booktitle = "Proceedings of the 10th Workshop on Representation Learning for NLP (RepL4NLP-2025)",
    month = may,
    year = "2025",
    address = "Albuquerque, NM",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/moar-dois/2025.repl4nlp-1.2/",
    doi = "10.18653/v1/2025.repl4nlp-1.2",
    pages = "26--37",
    ISBN = "979-8-89176-245-9",
    abstract = "We study how features emerge, disappear, and persist across models fine-tuned on different domains of text. More specifically, we start from a base one-layer Transformer language model that is trained on a combination of the BabyLM corpus, and a collection of Python code from The Stack. This base model is adapted to two new domains of text: TinyStories, and the Lua programming language, respectively; and then these two models are merged using these two models using spherical linear interpolation. Our exploration aims to provide deeper insights into the stability and transformation of features across typical transfer-learning scenarios using small-scale models and sparse auto-encoders."
}