@inproceedings{malon-etal-2022-fast,
    title = "Fast Few-shot Debugging for {NLU} Test Suites",
    author = "Malon, Christopher  and
      Li, Kai  and
      Kruus, Erik",
    editor = "Agirre, Eneko  and
      Apidianaki, Marianna  and
      Vuli{\'c}, Ivan",
    booktitle = "Proceedings of Deep Learning Inside Out (DeeLIO 2022): The 3rd Workshop on Knowledge Extraction and Integration for Deep Learning Architectures",
    month = may,
    year = "2022",
    address = "Dublin, Ireland and Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-sig-urls/2022.deelio-1.8/",
    doi = "10.18653/v1/2022.deelio-1.8",
    pages = "79--86",
    abstract = "We study few-shot debugging of transformer based natural language understanding models, using recently popularized test suites to not just diagnose but correct a problem. Given a few debugging examples of a certain phenomenon, and a held-out test set of the same phenomenon, we aim to maximize accuracy on the phenomenon at a minimal cost of accuracy on the original test set. We examine several methods that are faster than full epoch retraining. We introduce a new fast method, which samples a few in-danger examples from the original training set. Compared to fast methods using parameter distance constraints or Kullback-Leibler divergence, we achieve superior original accuracy for comparable debugging accuracy."
}