@inproceedings{salton-etal-2017-attentive,
    title = "Attentive Language Models",
    author = "Salton, Giancarlo  and
      Ross, Robert  and
      Kelleher, John",
    editor = "Kondrak, Greg  and
      Watanabe, Taro",
    booktitle = "Proceedings of the Eighth International Joint Conference on Natural Language Processing (Volume 1: Long Papers)",
    month = nov,
    year = "2017",
    address = "Taipei, Taiwan",
    publisher = "Asian Federation of Natural Language Processing",
    url = "https://preview.aclanthology.org/landing_page/I17-1045/",
    pages = "441--450",
    abstract = "In this paper, we extend Recurrent Neural Network Language Models (RNN-LMs) with an attention mechanism. We show that an ``attentive'' RNN-LM (with 11M parameters) achieves a better perplexity than larger RNN-LMs (with 66M parameters) and achieves performance comparable to an ensemble of 10 similar sized RNN-LMs. We also show that an ``attentive'' RNN-LM needs less contextual information to achieve similar results to the state-of-the-art on the wikitext2 dataset."
}