@inproceedings{torstensson-holmstrom-2025-grammar,
    title = "A Grammar-Based Method for Instilling Empirical Dependency Structure in {LLM}s",
    author = {Torstensson, Olle  and
      Holmstr{\"o}m, Oskar},
    editor = "Trosterud, Trond  and
      Wiechetek, Linda  and
      Pirinen, Flammie",
    booktitle = "Proceedings of the 9th Workshop on Constraint Grammar and Finite State NLP",
    month = mar,
    year = "2025",
    address = "Tallinn, Estonia",
    publisher = "University of Tartu Library",
    url = "https://preview.aclanthology.org/moar-dois/2025.cgmta-1.7/",
    pages = "45--49",
    ISBN = "978-9908-53-113-7",
    abstract = "We investigate whether synthetic pretraining data generated from a formal grammar modeling syntactic dependencies can improve English language models. Building upon the structured pretraining data approach of Papadimitriou and Jurafsky (2023), we develop a grammar that more closely mirrors empirical dependency structures. Our results are negative {--} this type of pretraining significantly degrades model performance, with both our and their pretraining approach performing worse than no pretraining at all. We analyze potential explanations for these findings and discuss implications for future work on structured-data pretraining."
}