@article{hochreiter1997long,
  title={Long short-term memory},
  author={Hochreiter, Sepp and Schmidhuber, J{\"u}rgen},
  journal={Neural computation},
  volume={9},
  number={8},
  pages={1735--1780},
  year={1997},
  publisher={MIT Press}
}

@article{graves2005framewise,
  title={Framewise phoneme classification with bidirectional LSTM and other neural network architectures},
  author={Graves, Alex and Schmidhuber, J{\"u}rgen},
  journal={Neural Networks},
  volume={18},
  number={5-6},
  pages={602--610},
  year={2005},
  publisher={Elsevier}
}


@article{devlin2018bert,
  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={arXiv preprint arXiv:1810.04805},
  year={2018}
}


@article{WangPruksachatkunEtAl_2019_SuperGLUE_Stickier_Benchmark_for_General-Purpose_Language_Understanding_Systems,
  archivePrefix = {arXiv},
  eprinttype = {arxiv},
  eprint = {1905.00537},
  primaryClass = {cs},
  title = {{{SuperGLUE}}: {{A Stickier Benchmark}} for {{General}}-{{Purpose Language Understanding Systems}}},
  shorttitle = {{{SuperGLUE}}},
  abstract = {In the last year, new models and methods for pretraining and transfer learning have driven striking performance improvements across a range of language understanding tasks. The GLUE benchmark, introduced one year ago, offers a single-number metric that summarizes progress on a diverse set of such tasks, but performance on the benchmark has recently come close to the level of non-expert humans, suggesting limited headroom for further research. This paper recaps lessons learned from the GLUE benchmark and presents SuperGLUE, a new benchmark styled after GLUE with a new set of more difficult language understanding tasks, improved resources, and a new public leaderboard. SuperGLUE will be available soon at super.gluebenchmark.com.},
  journal = {arXiv:1905.00537 [cs]},
  url = {http://arxiv.org/abs/1905.00537},
  author = {Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},
  month = may,
  year = {2019},
  file = {WangPruksachatkunEtAl_2019_SuperGLUE_Stickier_Benchmark_for_General-Purpose_Language_Understanding_Systems.pdf:/mnt/Data/Cloud/Dropbox/Zot/WangPruksachatkunEtAl_2019_SuperGLUE_Stickier_Benchmark_for_General-Purpose_Language_Understanding_Systems.pdf:application/pdf}
}




@book{Schutze_1996_Empirical_Base_of_Linguistics_Grammaticality_Judgments_and_Linguistic_Methodology,
  title = {The {{Empirical Base}} of {{Linguistics}}: {{Grammaticality Judgments}} and {{Linguistic Methodology}}},
  isbn = {978-0-226-74154-3},
  shorttitle = {The {{Empirical Base}} of {{Linguistics}}},
  abstract = {Throughout much of the history of linguistics, grammaticality judgments - intuitions about the well-formedness of sentences - have constituted most of the empirical base against which theoretical hypotheses have been tested. Although such judgments often rest on subtle intuitions, there is no systematic methodology for eliciting them, and their apparent instability and unreliability have led many to conclude that they should be abandoned as a source of data. Carson T. Schutze presents here a detailed critical overview of the vast literature on the nature of utility of grammaticality judgments and other linguistic intuitions, and the ways they have been used in linguistic research. He shows how variation in the judgment process can arise from factors such as biological, cognitive, and social differences among subjects, the particular elicitation method used, and extraneous features of the materials being judged. He then assesses the status of judgments as reliable indicators of a speaker's grammar. Integrating substantive and methodological findings, Schutze proposes a model in which grammaticality judgments result from interactions of linguistic competence with general cognitive processes. He argues that this model provides the underpinning for empirical arguments to show that once extragrammatical variance is factored out, universal grammar succumbs to a simpler, more elegant analysis than judgment data initially lead us to expect. Finally, Schutze offers numerous practical suggestions on how to collect better and more useful data. The result is a work of vital importance that will be required reading for linguists, cognitive psychologists, and philosophers of language alike.},
  language = {en},
  publisher = {{University of Chicago Press}},
  author = {Schutze, Carson T.},
  year = {1996},
  keywords = {Language Arts \& Disciplines / Grammar \& Punctuation,Social Science / Anthropology / Physical,Social Science / General},
  file = {Schutze_1996_Empirical_Base_of_Linguistics_Grammaticality_Judgments_and_Linguistic_Methodology.pdf:/home/anna/Dropbox/Zot/Schutze_1996_Empirical_Base_of_Linguistics_Grammaticality_Judgments_and_Linguistic_Methodology.pdf:application/pdf}
}




@article{rajpurkar2016squad,
  title={Squad: 100,000+ questions for machine comprehension of text},
  author={Rajpurkar, Pranav and Zhang, Jian and Lopyrev, Konstantin and Liang, Percy},
  journal={arXiv preprint arXiv:1606.05250},
  year={2016}
}

@inproceedings{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  booktitle={Advances in neural information processing systems},
  pages={5998--6008},
  year={2017}
}

@article{liu2019linguistic,
  title={Linguistic Knowledge and Transferability of Contextual Representations},
  author={Liu, Nelson F and Gardner, Matt and Belinkov, Yonatan and Peters, Matthew and Smith, Noah A},
  journal={arXiv preprint arXiv:1903.08855},
  year={2019}
}

@techreport{radford2018improving,
  title={Improving language understanding with unsupervised learning},
  author={Radford, Alec and Narasimhan, Karthik and Salimans, Time and Sutskever, Ilya},
  year={2018},
  institution={Technical report, OpenAI}
}

@article{wang2018glue,
  title={Glue: A multi-task benchmark and analysis platform for natural language understanding},
  author={Wang, Alex and Singh, Amapreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1804.07461},
  year={2018}
}

@inproceedings{baker1998berkeley,
  title={The berkeley framenet project},
  author={Baker, Collin F and Fillmore, Charles J and Lowe, John B},
  booktitle={Proceedings of the 17th international conference on Computational linguistics-Volume 1},
  pages={86--90},
  year={1998},
  organization={Association for Computational Linguistics}
}

@inproceedings{pradhan2012conll,
  title={CoNLL-2012 shared task: Modeling multilingual unrestricted coreference in OntoNotes},
  author={Pradhan, Sameer and Moschitti, Alessandro and Xue, Nianwen and Uryupina, Olga and Zhang, Yuchen},
  booktitle={Joint Conference on EMNLP and CoNLL-Shared Task},
  pages={1--40},
  year={2012},
  organization={Association for Computational Linguistics}
}

@article{goldberg2019assessing,
  title={Assessing BERT's Syntactic Abilities},
  author={Goldberg, Yoav},
  journal={arXiv preprint arXiv:1901.05287},
  year={2019}
}

@article{frankle2018lottery,
  title={The lottery ticket hypothesis: Finding sparse, trainable neural networks},
  author={Frankle, Jonathan and Carbin, Michael},
  journal={arXiv preprint arXiv:1803.03635},
  year={2018}
}

@article{tang2018self,
  title={Why self-attention? a targeted evaluation of neural machine translation architectures},
  author={Tang, Gongbo and M{\"u}ller, Mathias and Rios, Annette and Sennrich, Rico},
  journal={arXiv preprint arXiv:1808.08946},
  year={2018}
}

@article{tran2018importance,
  title={The importance of being recurrent for modeling hierarchical structure},
  author={Tran, Ke and Bisazza, Arianna and Monz, Christof},
  journal={arXiv preprint arXiv:1803.03585},
  year={2018}
}

@inproceedings{adhikari2019rethinking,
  title={Rethinking Complex Neural Network Architectures for Document Classification},
  author={Adhikari, Ashutosh and Ram, Achyudh, and Tang, Raphael and Lin, Jimmy },
  booktitle={Proceedings of NAACL 2019: Conference of the North American Chapter of the Association for Computational Linguistics},
  year={2019}
}

@inproceedings{jawahar:hal-02131630,
  TITLE = {{What does BERT learn about the structure of language?}},
  AUTHOR = {Jawahar, Ganesh and Sagot, Beno{\^i}t and Seddah, Djam{\'e}},
  URL = {https://hal.inria.fr/hal-02131630},
  BOOKTITLE = {{57th Annual Meeting of the Association for Computational Linguistics (ACL)}},
  ADDRESS = {Florence, Italy},
  YEAR = {2019},
  MONTH = Jul,
  HAL_ID = {hal-02131630},
  HAL_VERSION = {v1},
}

@article{wu2019pay,
  title={Pay Less Attention with Lightweight and Dynamic Convolutions},
  author={Wu, Felix and Fan, Angela and Baevski, Alexei and Dauphin, Yann N and Auli, Michael},
  journal={arXiv preprint arXiv:1901.10430},
  year={2019}
}

@inproceedings{dolan2005automatically,
  title={Automatically constructing a corpus of sentential paraphrases},
  author={Dolan, William B and Brockett, Chris},
  booktitle={Proceedings of the Third International Workshop on Paraphrasing (IWP2005)},
  year={2005}
}
@inproceedings{cer2017semeval,
  title={SemEval-2017 Task 1: Semantic Textual Similarity Multilingual and Crosslingual Focused Evaluation},
  author={Cer, Daniel and Diab, Mona and Agirre, Eneko and Lopez-Gazpio, Inigo and Specia, Lucia},
  booktitle={Proceedings of the 11th International Workshop on Semantic Evaluation (SemEval-2017)},
  pages={1--14},
  year={2017}
}
@inproceedings{williams2018broad,
  title={A Broad-Coverage Challenge Corpus for Sentence Understanding through Inference},
  author={Williams, Adina and Nangia, Nikita and Bowman, Samuel},
  booktitle={Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)},
  pages={1112--1122},
  year={2018}
}

@inproceedings{socher2013recursive,
  title={Recursive deep models for semantic compositionality over a sentiment treebank},
  author={Socher, Richard and Perelygin, Alex and Wu, Jean and Chuang, Jason and Manning, Christopher D and Ng, Andrew and Potts, Christopher},
  booktitle={Proceedings of the 2013 conference on empirical methods in natural language processing},
  pages={1631--1642},
  year={2013}
}
@inproceedings{yosinski2014transferable,
  title={How transferable are features in deep neural networks?},
  author={Yosinski, Jason and Clune, Jeff and Bengio, Yoshua and Lipson, Hod},
  booktitle={Advances in neural information processing systems},
  pages={3320--3328},
  year={2014}
}
@inproceedings{romanov2018lessons,
  title={Lessons from Natural Language Inference in the Clinical Domain},
  author={Romanov, Alexey and Shivade, Chaitanya},
  booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
  pages={1586--1596},
  year={2018}
}

@article{voita2019analyzing,
  title={Analyzing Multi-Head Self-Attention: Specialized Heads Do the Heavy Lifting, the Rest Can Be Pruned},
  author={Voita, Elena and Talbot, David and Moiseev, Fedor and Sennrich, Rico and Titov, Ivan},
  journal={arXiv preprint arXiv:1905.09418},
  year={2019}
}

@article{michel2019sixteen,
  title={Are Sixteen Heads Really Better than One?},
  author={Michel, Paul and Levy, Omer and Neubig, Graham},
  journal={arXiv preprint arXiv:1905.10650},
  year={2019}
}