@inproceedings{jia-etal-2022-question,
title = "Question Answering Infused Pre-training of General-Purpose Contextualized Representations",
author = "Jia, Robin and
Lewis, Mike and
Zettlemoyer, Luke",
editor = "Muresan, Smaranda and
Nakov, Preslav and
Villavicencio, Aline",
booktitle = "Findings of the Association for Computational Linguistics: ACL 2022",
month = may,
year = "2022",
address = "Dublin, Ireland",
publisher = "Association for Computational Linguistics",
url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2022.findings-acl.59/",
doi = "10.18653/v1/2022.findings-acl.59",
pages = "711--728",
abstract = "We propose a pre-training objective based on question answering (QA) for learning general-purpose contextual representations, motivated by the intuition that the representation of a phrase in a passage should encode all questions that the phrase can answer in context. To this end, we train a bi-encoder QA model, which independently encodes passages and questions, to match the predictions of a more accurate cross-encoder model on 80 million synthesized QA pairs. By encoding QA-relevant information, the bi-encoder`s token-level representations are useful for non-QA downstream tasks without extensive (or in some cases, any) fine-tuning. We show large improvements over both RoBERTa-large and previous state-of-the-art results on zero-shot and few-shot paraphrase detection on four datasets, few-shot named entity recognition on two datasets, and zero-shot sentiment analysis on three datasets."
}
Markdown (Informal)
[Question Answering Infused Pre-training of General-Purpose Contextualized Representations](https://preview.aclanthology.org/jlcl-multiple-ingestion/2022.findings-acl.59/) (Jia et al., Findings 2022)
ACL