@inproceedings{senel-etal-2022-coda21,
    title = "{C}o{DA}21: Evaluating Language Understanding Capabilities of {NLP} Models With Context-Definition Alignment",
    author = {Senel, L{\"u}tfi Kerem  and
      Schick, Timo  and
      Schuetze, Hinrich},
    editor = "Muresan, Smaranda  and
      Nakov, Preslav  and
      Villavicencio, Aline",
    booktitle = "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-sig-urls/2022.acl-short.92/",
    doi = "10.18653/v1/2022.acl-short.92",
    pages = "815--824",
    abstract = "Pretrained language models (PLMs) have achieved superhuman performance on many benchmarks, creating a need for harder tasks. We introduce CoDA21 (Context Definition Alignment), a challenging benchmark that measures natural language understanding (NLU) capabilities of PLMs: Given a definition and a context each for k words, but not the words themselves, the task is to align the k definitions with the k contexts. CoDA21 requires a deep understanding of contexts and definitions, including complex inference and world knowledge. We find that there is a large gap between human and PLM performance, suggesting that CoDA21 measures an aspect of NLU that is not sufficiently covered in existing benchmarks."
}