@inproceedings{shin-etal-2020-fast,
title = "Fast and Accurate Deep Bidirectional Language Representations for Unsupervised Learning",
author = "Shin, Joongbo and
Lee, Yoonhyung and
Yoon, Seunghyun and
Jung, Kyomin",
editor = "Jurafsky, Dan and
Chai, Joyce and
Schluter, Natalie and
Tetreault, Joel",
booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
month = jul,
year = "2020",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://preview.aclanthology.org/fix-sig-urls/2020.acl-main.76/",
doi = "10.18653/v1/2020.acl-main.76",
pages = "823--835",
abstract = "Even though BERT has achieved successful performance improvements in various supervised learning tasks, BERT is still limited by repetitive inferences on unsupervised tasks for the computation of contextual language representations. To resolve this limitation, we propose a novel deep bidirectional language model called a Transformer-based Text Autoencoder (T-TA). The T-TA computes contextual language representations without repetition and displays the benefits of a deep bidirectional architecture, such as that of BERT. In computation time experiments in a CPU environment, the proposed T-TA performs over six times faster than the BERT-like model on a reranking task and twelve times faster on a semantic similarity task. Furthermore, the T-TA shows competitive or even better accuracies than those of BERT on the above tasks. Code is available at \url{https://github.com/joongbo/tta}."
}
Markdown (Informal)
[Fast and Accurate Deep Bidirectional Language Representations for Unsupervised Learning](https://preview.aclanthology.org/fix-sig-urls/2020.acl-main.76/) (Shin et al., ACL 2020)
ACL