@inproceedings{li-choi-2020-transformers,
    title = "Transformers to Learn Hierarchical Contexts in Multiparty Dialogue for Span-based Question Answering",
    author = "Li, Changmao  and
      Choi, Jinho D.",
    editor = "Jurafsky, Dan  and
      Chai, Joyce  and
      Schluter, Natalie  and
      Tetreault, Joel",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/fix-sig-urls/2020.acl-main.505/",
    doi = "10.18653/v1/2020.acl-main.505",
    pages = "5709--5714",
    abstract = "We introduce a novel approach to transformers that learns hierarchical representations in multiparty dialogue. First, three language modeling tasks are used to pre-train the transformers, token- and utterance-level language modeling and utterance order prediction, that learn both token and utterance embeddings for better understanding in dialogue contexts. Then, multi-task learning between the utterance prediction and the token span prediction is applied to fine-tune for span-based question answering (QA). Our approach is evaluated on the FriendsQA dataset and shows improvements of 3.8{\%} and 1.4{\%} over the two state-of-the-art transformer models, BERT and RoBERTa, respectively."
}