@inproceedings{huang-etal-2023-inducing,
title = "Inducing Character-level Structure in Subword-based Language Models with Type-level Interchange Intervention Training",
author = "Huang, Jing and
Wu, Zhengxuan and
Mahowald, Kyle and
Potts, Christopher",
editor = "Rogers, Anna and
Boyd-Graber, Jordan and
Okazaki, Naoaki",
booktitle = "Findings of the Association for Computational Linguistics: ACL 2023",
month = jul,
year = "2023",
address = "Toronto, Canada",
publisher = "Association for Computational Linguistics",
url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2023.findings-acl.770/",
doi = "10.18653/v1/2023.findings-acl.770",
pages = "12163--12180",
abstract = "Language tasks involving character-level manipulations (e.g., spelling corrections, arithmetic operations, word games) are challenging for models operating on subword units. To address this, we develop a causal intervention framework to learn robust and interpretable character representations inside subword-based language models. Our method treats each character as a typed variable in a causal model and learns such causal structures by adapting the interchange intervention training method of Geiger et al. (2021). We additionally introduce a suite of character-level tasks that systematically vary in their dependence on meaning and sequence-level context. While character-level models still perform best on purely form-based tasks like string reversal, our method outperforms character-level models on more complex tasks that blend form, meaning, and context, such as spelling correction in context and word search games. Compared with standard subword-based models, our approach also significantly improves robustness on unseen token sequences and leads to human-interpretable internal representations of characters."
}
Markdown (Informal)
[Inducing Character-level Structure in Subword-based Language Models with Type-level Interchange Intervention Training](https://preview.aclanthology.org/jlcl-multiple-ingestion/2023.findings-acl.770/) (Huang et al., Findings 2023)
ACL