@inproceedings{gao2018representation,
  title={Representation Degeneration Problem in Training Natural Language Generation Models},
  author={Gao, Jun and He, Di and Tan, Xu and Qin, Tao and Wang, Liwei and Liu, Tieyan},
  booktitle={International Conference on Learning Representations},
  year={2018}
}

@inproceedings{lauscher2020specializing,
  title={Specializing Unsupervised Pretraining Models for Word-Level Semantic Similarity},
  author={Lauscher, Anne and Vuli{\'c}, Ivan and Ponti, Edoardo Maria and Korhonen, Anna and Glava{\v{s}}, Goran},
  booktitle={Proceedings of the 28th International Conference on Computational Linguistics},
  pages={1371--1383},
  year={2020}
}

@inproceedings{zhang-etal-2019-ernie,
    title = "{ERNIE}: Enhanced Language Representation with Informative Entities",
    author = "Zhang, Zhengyan  and
      Han, Xu  and
      Liu, Zhiyuan  and
      Jiang, Xin  and
      Sun, Maosong  and
      Liu, Qun",
    booktitle = "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2019",
    address = "Florence, Italy",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/P19-1139",
    doi = "10.18653/v1/P19-1139",
    pages = "1441--1451",
    abstract = "Neural language representation models such as BERT pre-trained on large-scale corpora can well capture rich semantic patterns from plain text, and be fine-tuned to consistently improve the performance of various NLP tasks. However, the existing pre-trained language models rarely consider incorporating knowledge graphs (KGs), which can provide rich structured knowledge facts for better language understanding. We argue that informative entities in KGs can enhance language representation with external knowledge. In this paper, we utilize both large-scale textual corpora and KGs to train an enhanced language representation model (ERNIE), which can take full advantage of lexical, syntactic, and knowledge information simultaneously. The experimental results have demonstrated that ERNIE achieves significant improvements on various knowledge-driven tasks, and meanwhile is comparable with the state-of-the-art model BERT on other common NLP tasks. The code and datasets will be available in the future.",
}
@inproceedings{peters-etal-2019-knowledge,
    title = "Knowledge Enhanced Contextual Word Representations",
    author = "Peters, Matthew E.  and
      Neumann, Mark  and
      Logan, Robert  and
      Schwartz, Roy  and
      Joshi, Vidur  and
      Singh, Sameer  and
      Smith, Noah A.",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)",
    month = nov,
    year = "2019",
    address = "Hong Kong, China",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/D19-1005",
    doi = "10.18653/v1/D19-1005",
    pages = "43--54",
    abstract = "Contextual word representations, typically trained on unstructured, unlabeled text, do not contain any explicit grounding to real world entities and are often unable to remember facts about those entities. We propose a general method to embed multiple knowledge bases (KBs) into large scale models, and thereby enhance their representations with structured, human-curated knowledge. For each KB, we first use an integrated entity linker to retrieve relevant entity embeddings, then update contextual word representations via a form of word-to-entity attention. In contrast to previous approaches, the entity linkers and self-supervised language modeling objective are jointly trained end-to-end in a multitask setting that combines a small amount of entity linking supervision with a large amount of raw text. After integrating WordNet and a subset of Wikipedia into BERT, the knowledge enhanced BERT (KnowBert) demonstrates improved perplexity, ability to recall facts as measured in a probing task and downstream performance on relationship extraction, entity typing, and word sense disambiguation. KnowBert{'}s runtime is comparable to BERT{'}s and it scales to large KBs.",
}

@article{wang2020k,
  title={K-adapter: Infusing knowledge into pre-trained models with adapters},
  author={Wang, Ruize and Tang, Duyu and Duan, Nan and Wei, Zhongyu and Huang, Xuanjing and Cao, Cuihong and Jiang, Daxin and Zhou, Ming and others},
  journal={arXiv preprint arXiv:2002.01808},
  year={2020}
}

@inproceedings{lauscher-etal-2020-specializing,
    title = "Specializing Unsupervised Pretraining Models for Word-Level Semantic Similarity",
    author = "Lauscher, Anne  and
      Vuli{\'c}, Ivan  and
      Ponti, Edoardo Maria  and
      Korhonen, Anna  and
      Glava{\v{s}}, Goran",
    booktitle = "Proceedings of the 28th International Conference on Computational Linguistics",
    month = dec,
    year = "2020",
    address = "Barcelona, Spain (Online)",
    publisher = "International Committee on Computational Linguistics",
    url = "https://www.aclweb.org/anthology/2020.coling-main.118",
    doi = "10.18653/v1/2020.coling-main.118",
    pages = "1371--1383",
    abstract = "Unsupervised pretraining models have been shown to facilitate a wide range of downstream NLP applications. These models, however, retain some of the limitations of traditional static word embeddings. In particular, they encode only the distributional knowledge available in raw text corpora, incorporated through language modeling objectives. In this work, we complement such distributional knowledge with external lexical knowledge, that is, we integrate the discrete knowledge on word-level semantic similarity into pretraining. To this end, we generalize the standard BERT model to a multi-task learning setting where we couple BERT{'}s masked language modeling and next sentence prediction objectives with an auxiliary task of binary word relation classification. Our experiments suggest that our {``}Lexically Informed{''} BERT (LIBERT), specialized for the word-level semantic similarity, yields better performance than the lexically blind {``}vanilla{''} BERT on several language understanding tasks. Concretely, LIBERT outperforms BERT in 9 out of 10 tasks of the GLUE benchmark and is on a par with BERT in the remaining one. Moreover, we show consistent gains on 3 benchmarks for lexical simplification, a task where knowledge about word-level semantic similarity is paramount, as well as large gains on lexical reasoning probes.",
}

@article{arora2016simple,
  title={A simple but tough-to-beat baseline for sentence embeddings},
  author={Arora, Sanjeev and Liang, Yingyu and Ma, Tengyu},
  year={2016}
}

@misc{jurafsky2019speech,
  title={Speech and Language Processing (3rd draft ed.)},
  author={Jurafsky, Dan and Martin, James H},
  year={2019},
  publisher={Stanford Univ}
}

@inproceedings{jo2018extrofitting,
  title={Extrofitting: Enriching Word Representation and its Vector Space with Semantic Lexicons},
  author={Jo, Hwiyeol and Choi, Stanley Jungkyu},
  booktitle={Proceedings of The Third Workshop on Representation Learning for NLP},
  pages={24--29},
  year={2018}
}

@inproceedings{sap-etal-2020-commonsense,
    title = "Commonsense Reasoning for Natural Language Processing",
    author = "Sap, Maarten  and
      Shwartz, Vered  and
      Bosselut, Antoine  and
      Choi, Yejin  and
      Roth, Dan",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics: Tutorial Abstracts",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/2020.acl-tutorials.7",
    doi = "10.18653/v1/2020.acl-tutorials.7",
    pages = "27--33",
    abstract = "Commonsense knowledge, such as knowing that {``}bumping into people annoys them{''} or {``}rain makes the road slippery{''}, helps humans navigate everyday situations seamlessly. Yet, endowing machines with such human-like commonsense reasoning capabilities has remained an elusive goal of artificial intelligence research for decades. In recent years, commonsense knowledge and reasoning have received renewed attention from the natural language processing (NLP) community, yielding exploratory studies in automated commonsense understanding. We organize this tutorial to provide researchers with the critical foundations and recent advances in commonsense representation and reasoning, in the hopes of casting a brighter light on this promising area of future research. In our tutorial, we will (1) outline the various types of commonsense (e.g., physical, social), and (2) discuss techniques to gather and represent commonsense knowledge, while highlighting the challenges specific to this type of knowledge (e.g., reporting bias). We will then (3) discuss the types of commonsense knowledge captured by modern NLP systems (e.g., large pretrained language models), and (4) present ways to measure systems{'} commonsense reasoning abilities. We will finish with (5) a discussion of various ways in which commonsense reasoning can be used to improve performance on NLP tasks, exemplified by an (6) interactive session on integrating commonsense into a downstream task.",
}

@article{da2019cracking,
  title={Cracking the Contextual Commonsense Code: Understanding Commonsense Reasoning Aptitude of Deep Contextual Representations},
  author={Da, Jeff and Kasai, Jungo},
  journal={EMNLP 2019},
  pages={1},
  year={2019}
}
@inproceedings{clark2019electra,
  title={ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators},
  author={Clark, Kevin and Luong, Minh-Thang and Le, Quoc V and Manning, Christopher D},
  booktitle={International Conference on Learning Representations},
  year={2019}
}
@inproceedings{miaschi2020contextual,
  title={Contextual and Non-Contextual Word Embeddings: an in-depth Linguistic Investigation},
  author={Miaschi, Alessio and Dell’Orletta, Felice},
  booktitle={Proceedings of the 5th Workshop on Representation Learning for NLP},
  pages={110--119},
  year={2020}
}

@article{peters2018deep,
  title={Deep contextualized word representations},
  author={Peters, Matthew E and Neumann, Mark and Iyyer, Mohit and Gardner, Matt and Clark, Christopher and Lee, Kenton and Zettlemoyer, Luke},
  journal={arXiv preprint arXiv:1802.05365},
  year={2018}
}

@article{bojanowski2017enriching,
  title={Enriching word vectors with subword information},
  author={Bojanowski, Piotr and Grave, Edouard and Joulin, Armand and Mikolov, Tomas},
  journal={Transactions of the Association for Computational Linguistics},
  volume={5},
  pages={135--146},
  year={2017},
  publisher={MIT Press}
}
@article{mikolov2013efficient,
  title={Efficient estimation of word representations in vector space},
  author={Mikolov, Tomas and Chen, Kai and Corrado, Greg and Dean, Jeffrey},
  journal={arXiv preprint arXiv:1301.3781},
  year={2013}
}

@book{miller1998wordnet,
  title={WordNet: An electronic lexical database},
  author={Miller, George A},
  year={1998},
  publisher={MIT press}
}

@article{devlin2018bert,
  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={arXiv preprint arXiv:1810.04805},
  year={2018}
}

@article{radford2019language,
  title={Language models are unsupervised multitask learners},
  author={Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya},
  journal={OpenAI blog},
  volume={1},
  number={8},
  pages={9},
  year={2019}
}

@inproceedings{yang2019xlnet,
  title={Xlnet: Generalized autoregressive pretraining for language understanding},
  author={Yang, Zhilin and Dai, Zihang and Yang, Yiming and Carbonell, Jaime and Salakhutdinov, Russ R and Le, Quoc V},
  booktitle={Advances in neural information processing systems},
  pages={5753--5763},
  year={2019}
}
@inproceedings{nair2020contextualized,
  title={Contextualized Word Embeddings Encode Aspects of Human-Like Word Sense Knowledge},
  author={Nair, Sathvik and Srinivasan, Mahesh and Meylan, Stephan},
  booktitle={Proceedings of the Workshop on the Cognitive Aspects of the Lexicon},
  pages={129--141},
  year={2020}
}
@inproceedings{NEURIPS2019_159c1ffe,
 author = {Reif, Emily and Yuan, Ann and Wattenberg, Martin and Viegas, Fernanda B and Coenen, Andy and Pearce, Adam and Kim, Been},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Visualizing and Measuring the Geometry of BERT},
 url = {https://proceedings.neurips.cc/paper/2019/file/159c1ffe5b61b41b3c4d8f4c2150f6c4-Paper.pdf},
 volume = {32},
 year = {2019}
}

@inproceedings{mimno2017strange,
  title={The strange geometry of skip-gram with negative sampling},
  author={Mimno, David and Thompson, Laure},
  booktitle={Empirical Methods in Natural Language Processing},
  year={2017}
}

@inproceedings{vial2018ufsac,
  title={UFSAC: Unification of sense annotated corpora and tools},
  author={Vial, Lo{\"\i}c and Lecouteux, Benjamin and Schwab, Didier},
  booktitle={Language Resources and Evaluation Conference (LREC)},
  year={2018}
}

@inproceedings{navigli2007semeval,
  title={Semeval-2007 task 07: Coarse-grained english all-words task},
  author={Navigli, Roberto and Litkowski, Kenneth C and Hargraves, Orin},
  booktitle={Proceedings of the Fourth International Workshop on Semantic Evaluations (SemEval-2007)},
  pages={30--35},
  year={2007}
}

@inproceedings{moro2015semeval,
  title={Semeval-2015 task 13: Multilingual all-words sense disambiguation and entity linking},
  author={Moro, Andrea and Navigli, Roberto},
  booktitle={Proceedings of the 9th international workshop on semantic evaluation (SemEval 2015)},
  pages={288--297},
  year={2015}
}

@inproceedings{navigli2013semeval,
  title={Semeval-2013 task 12: Multilingual word sense disambiguation},
  author={Navigli, Roberto and Jurgens, David and Vannella, Daniele},
  booktitle={Second Joint Conference on Lexical and Computational Semantics (* SEM), Volume 2: Proceedings of the Seventh International Workshop on Semantic Evaluation (SemEval 2013)},
  pages={222--231},
  year={2013}
}

@inproceedings{edmonds2001senseval,
  title={Senseval-2: overview},
  author={Edmonds, Philip and Cotton, Scott},
  booktitle={Proceedings of SENSEVAL-2 Second International Workshop on Evaluating Word Sense Disambiguation Systems},
  pages={1--5},
  year={2001}
}

@inproceedings{snyder2004english,
  title={The English all-words task},
  author={Snyder, Benjamin and Palmer, Martha},
  booktitle={Proceedings of SENSEVAL-3, the Third International Workshop on the Evaluation of Systems for the Semantic Analysis of Text},
  pages={41--43},
  year={2004}
}

@article{ethayarajh2019contextual,
  title={How contextual are contextualized word representations? Comparing the geometry of BERT, ELMo, and GPT-2 embeddings},
  author={Ethayarajh, Kawin},
  journal={arXiv preprint arXiv:1909.00512},
  year={2019}
}

@inproceedings{schnabel2015evaluation,
  title={Evaluation methods for unsupervised word embeddings},
  author={Schnabel, Tobias and Labutov, Igor and Mimno, David and Joachims, Thorsten},
  booktitle={Proceedings of the 2015 conference on empirical methods in natural language processing},
  pages={298--307},
  year={2015}
}

@inproceedings{vulic2020probing,
  title={Probing Pretrained Language Models for Lexical Semantics},
  author={Vuli{\'c}, Ivan and Ponti, Edoardo Maria and Litschko, Robert and Glava{\v{s}}, Goran and Korhonen, Anna},
  booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
  pages={7222--7240},
  year={2020}
}

@inproceedings{reif2019visualizing,
  title={Visualizing and Measuring the Geometry of BERT},
  author={Reif, Emily and Yuan, Ann and Wattenberg, Martin and Viegas, Fernanda B and Coenen, Andy and Pearce, Adam and Kim, Been},
  booktitle={Advances in Neural Information Processing Systems},
  pages={8594--8603},
  year={2019}
}

@inproceedings{zhang2020revisiting,
  title={Revisiting Representation Degeneration Problem in Language Modeling},
  author={Zhang, Zhong and Gao, Chongming and Xu, Cong and Miao, Rui and Yang, Qinli and Shao, Junming},
  booktitle={Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: Findings},
  pages={518--527},
  year={2020}
}

@inproceedings{mu2018all,
  title={All-but-the-top: Simple and effective post-processing for word representations},
  author={Mu, Jiaqi and Viswanath, Pramod},
  booktitle={6th International Conference on Learning Representations, ICLR 2018},
  year={2018}
}

@inproceedings{raunak2019effective,
  title={Effective dimensionality reduction for word embeddings},
  author={Raunak, Vikas and Gupta, Vivek and Metze, Florian},
  booktitle={Proceedings of the 4th Workshop on Representation Learning for NLP (RepL4NLP-2019)},
  pages={235--243},
  year={2019}
}

@article{wang2019evaluating,
  title={Evaluating word embedding models: Methods and experimental results},
  author={Wang, Bin and Wang, Angela and Chen, Fenxiao and Wang, Yuncheng and Kuo, C-C Jay},
  journal={APSIPA transactions on signal and information processing},
  volume={8},
  year={2019},
  publisher={Cambridge University Press}
}

@inproceedings{faruqui2015retrofitting,
  title={Retrofitting Word Vectors to Semantic Lexicons},
  author={Faruqui, Manaal and Dodge, Jesse and Jauhar, Sujay Kumar and Dyer, Chris and Hovy, Eduard and Smith, Noah A},
  booktitle={Proceedings of the 2015 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
  pages={1606--1615},
  year={2015}
}

@inproceedings{vulic2018specialising,
  title={Specialising Word Vectors for Lexical Entailment},
  author={Vuli{\'c}, Ivan and Mrk{\v{s}}i{\'c}, Nikola},
  booktitle={Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)},
  pages={1134--1145},
  year={2018}
}

@inproceedings{vulic2018injecting,
  title={Injecting lexical contrast into word vectors by guiding vector space specialisation},
  author={Vuli{\'c}, Ivan},
  booktitle={Proceedings of The Third Workshop on Representation Learning for NLP},
  pages={137--143},
  year={2018}
}

@article{landes1998building,
  title={Building semantic concordances},
  author={Landes, Shari and Leacock, Claudia and Tengi, Randee I},
  journal={WordNet: an electronic lexical database},
  volume={199},
  number={216},
  pages={199--216},
  year={1998},
  publisher={MIT Press, Cambridge, MA}
}

@article{miller1995wordnet,
  title={WordNet: a lexical database for English},
  author={Miller, George A},
  journal={Communications of the ACM},
  volume={38},
  number={11},
  pages={39--41},
  year={1995},
  publisher={ACM New York, NY, USA}
}