@inproceedings{salton-kelleher-2019-persistence,
    title = "Persistence pays off: Paying Attention to What the {LSTM} Gating Mechanism Persists",
    author = "Salton, Giancarlo  and
      Kelleher, John",
    editor = "Mitkov, Ruslan  and
      Angelova, Galia",
    booktitle = "Proceedings of the International Conference on Recent Advances in Natural Language Processing (RANLP 2019)",
    month = sep,
    year = "2019",
    address = "Varna, Bulgaria",
    publisher = "INCOMA Ltd.",
    url = "https://preview.aclanthology.org/ingest-emnlp/R19-1121/",
    doi = "10.26615/978-954-452-056-4_121",
    pages = "1052--1059",
    abstract = "Recurrent Neural Network Language Models composed of LSTM units, especially those augmented with an external memory, have achieved state-of-the-art results in Language Modeling. However, these models still struggle to process long sequences which are more likely to contain long-distance dependencies because of information fading. In this paper we demonstrate an effective mechanism for retrieving information in a memory augmented LSTM LM based on attending to information in memory in proportion to the number of timesteps the LSTM gating mechanism persisted the information."
}