
@inproceedings{Devlin2019bert,
  author    = {Jacob Devlin and
               Ming{-}Wei Chang and
               Kenton Lee and
               Kristina Toutanova},
  title     = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language
               Understanding},
  booktitle = {Proceedings of NAACL 2019 },
  year      = {2019}, 
  url       = {https://doi.org/10.18653/v1/n19-1423},
  doi       = {10.18653/v1/n19-1423},
  timestamp = {Tue, 28 Jan 2020 10:30:29 +0100},
  biburl    = {https://dblp.org/rec/conf/naacl/DevlinCLT19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{Lample2019xlm,
  author    = {Alexis Conneau and
               Guillaume Lample},
  title     = {Cross-lingual Language Model Pretraining},
  booktitle = {Advances in  NeurIPS 2019},
  year      = {2019}, 
  url       = {http://papers.nips.cc/paper/8928-cross-lingual-language-model-pretraining},
  timestamp = {Fri, 06 Mar 2020 16:59:27 +0100},
  biburl    = {https://dblp.org/rec/conf/nips/ConneauL19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{Cao:2020iclr,
  author    = {Steven Cao and
               Nikita Kitaev and
               Dan Klein},
  title     = {{Multilingual Alignment of Contextual Word Representations}},
booktitle = {Proceedings of ICLR  2020},
  year      = {2020},
  url       = {https://arxiv.org/abs/2002.03518},
  archivePrefix = {arXiv},
  eprint    = {2002.03518},
  timestamp = {Wed, 12 Feb 2020 16:38:55 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-2002-03518.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}




@inproceedings{Wu2020emerging,
  author    = {Shijie Wu and
               Alexis Conneau and
               Haoran Li and
               Luke Zettlemoyer and
               Veselin Stoyanov},
  title     = {Emerging Cross-lingual Structure in Pretrained Language Models},
booktitle = {Proceedings of ACL 2020},
  year      = {2020},
  url       = {http://arxiv.org/abs/1911.01464},
}

@article{Santoro:2016arxiv,
  author    = {Adam Santoro and
               Sergey Bartunov and
               Matthew Botvinick and
               Daan Wierstra and
               Timothy P. Lillicrap},
  title     = {One-shot Learning with Memory-Augmented Neural Networks},
  journal   = {arXiv preprint},
  year      = {2016},
  url       = {http://arxiv.org/abs/1605.06065},
  archivePrefix = {arXiv},
  eprint    = {1605.06065},
  timestamp = {Mon, 13 Aug 2018 16:47:16 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/SantoroBBWL16.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{liu2019roberta,
  title={Roberta: A robustly optimized bert pretraining approach},
  author={Liu, Yinhan and Ott, Myle and Goyal, Naman and Du, Jingfei and Joshi, Mandar and Chen, Danqi and Levy, Omer and Lewis, Mike and Zettlemoyer, Luke and Stoyanov, Veselin},
  journal={arXiv preprint},
  year={2019},
  url={https://arxiv.org/abs/1907.11692}
}

@incollection{McCloskey:1989,
  title={Catastrophic interference in connectionist networks: {T}he sequential learning problem},
  author={McCloskey, Michael and Cohen, Neal J.},
  booktitle={Psychology of Learning and Motivation},
  volume={24},
  pages={109--165},
  year={1989},
}

@inproceedings{Glavas:2019acl,
  author    = {Goran Glava\v{s} and
               Robert Litschko and
               Sebastian Ruder and
               Ivan Vuli\'{c}},
  title     = {How to (Properly) Evaluate Cross-Lingual Word Embeddings: On Strong
               Baselines, Comparative Analyses, and Some Misconceptions},
  booktitle = {Proceedings of ACL 2019},
  pages     = {710--721},
  year      = {2019}, 
  url       = {https://doi.org/10.18653/v1/p19-1070},
  doi       = {10.18653/v1/p19-1070},
  timestamp = {Tue, 28 Jan 2020 10:27:55 +0100},
  biburl    = {https://dblp.org/rec/conf/acl/GlavasLRV19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{Artetxe2020cross-lingual,
  author    = {Mikel Artetxe and
               Sebastian Ruder and
               Dani Yogatama},
  title     = {On the Cross-lingual Transferability of Monolingual Representations},
booktitle = {Proceedings of ACL 2020},
  year      = {2020},
  url       = {http://arxiv.org/abs/1910.11856},
  timestamp = {Thu, 31 Oct 2019 14:02:26 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1910-11856.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@article{Wenzek:2019arxiv,
  author    = {Guillaume Wenzek and
               Marie{-}Anne Lachaux and
               Alexis Conneau and
               Vishrav Chaudhary and
               Francisco Guzm{\'{a}}n and
               Armand Joulin and
               Edouard Grave},
  title     = {{CCNet: E}xtracting High Quality Monolingual Datasets from {W}eb Crawl
               Data},
journal   = {arXiv preprint},
  year      = {2019},
  url       = {http://arxiv.org/abs/1911.00359},
  archivePrefix = {arXiv},
  eprint    = {1911.00359},
  timestamp = {Mon, 11 Nov 2019 18:38:09 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1911-00359.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{Wang:2020iclr,
author = {Wang, Zirui and Xie, Jiateng and Xu, Ruochen and Yang, Yiming and Neubig, Graham and Carbonell, Jaime},
booktitle = {Proceedings of ICLR  2020},
title = {{Cross-lingual Alignment vs Joint Training: A Comparative Study and A Simple Unified Framework}},
year = {2020},
  url       = {http://arxiv.org/abs/1910.04708},
}




@inproceedings{Chidambaram:2019repl,
  author    = {Muthuraman Chidambaram and
               Yinfei Yang and
               Daniel Cer and
               Steve Yuan and
               Yun{-}Hsuan Sung and
               Brian Strope and
               Ray Kurzweil},
  title     = {Learning Cross-Lingual Sentence Representations via a Multi-task Dual-Encoder
               Model},
  booktitle = {Proceedings of RepL4NLP@ACL 2019},
  pages     = {250--259},
  year      = {2019}, 
  url       = {https://doi.org/10.18653/v1/w19-4330},
  doi       = {10.18653/v1/w19-4330},
  timestamp = {Tue, 28 Jan 2020 10:30:34 +0100},
  biburl    = {https://dblp.org/rec/conf/rep4nlp/ChidambaramYCYS19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{Dinh2014NICE,
  author    = {Laurent Dinh and
               David Krueger and
               Yoshua Bengio},
  title     = {{NICE:} Non-linear Independent Components Estimation},
  booktitle = {Workshop Track Proceedings ICLR 2015},
  year      = {2015},
  url       = {http://arxiv.org/abs/1410.8516},
  timestamp = {Wed, 17 Jul 2019 10:40:54 +0200},
}


@article{Virtanen:2019arxiv,
  author    = {Antti Virtanen and
               Jenna Kanerva and
               Rami Ilo and
               Jouni Luoma and
               Juhani Luotolahti and
               Tapio Salakoski and
               Filip Ginter and
               Sampo Pyysalo},
  title     = {Multilingual is not enough: {BERT} for {F}innish},
journal   = {arXiv preprint},
  year      = {2019},
  url       = {http://arxiv.org/abs/1912.07076},
  archivePrefix = {arXiv},
  eprint    = {1912.07076},
  timestamp = {Fri, 03 Jan 2020 16:10:45 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1912-07076.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@book{glottolog,
  address   = {Jena},
  editor    = {Hammarström, Harald and Forkel, Robert and Haspelmath, Martin and Bank, Sebastian},
  publisher = {Max Planck Institute for the Science of Human History},
  title     = {Glottolog 2.7},
  url       = {http://glottolog.org},
  year      = {2016}
}

@article{Nozza:2020arxiv,
  author    = {Debora Nozza and
               Federico Bianchi and
               Dirk Hovy},
  title     = {What the [MASK]? Making Sense of Language-Specific {BERT} Models},
journal   = {arXiv preprint},
  year      = {2020},
  url       = {https://arxiv.org/abs/2003.02912},
  archivePrefix = {arXiv},
  eprint    = {2003.02912},
  timestamp = {Tue, 10 Mar 2020 13:33:48 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-2003-02912.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{Conneau2020xlm-r,
  author    = {Alexis Conneau and
               Kartikay Khandelwal and
               Naman Goyal and
               Vishrav Chaudhary and
               Guillaume Wenzek and
               Francisco Guzm{\'{a}}n and
               Edouard Grave and
               Myle Ott and
               Luke Zettlemoyer and
               Veselin Stoyanov},
  title     = {Unsupervised Cross-lingual Representation Learning at Scale},
booktitle = {Proceedings of ACL 2020},

  year      = {2020},
  url       = {http://arxiv.org/abs/1911.02116},
}

@online{WinNT,
  author = {Iyer, Shankar and Dandekar, Nikhil and Csernai, Kornel},
  title = {First Quora Dataset Release: Question Pairs},
  year = 2017,
  url = {https://data.quora.com/First-Quora-Dataset-Release-Question-Pairs},
  urldate = {2019-04-03}
}



@inproceedings{cer2017semeval,
    title = "{S}em{E}val-2017 Task 1: Semantic Textual Similarity Multilingual and Crosslingual Focused Evaluation",
    author = "Cer, Daniel  and
      Diab, Mona  and
      Agirre, Eneko  and
      Lopez-Gazpio, I{\~n}igo  and
      Specia, Lucia",
    booktitle = "Proceedings of {S}em{E}val-2017",
    year = "2017",
    url = "https://www.aclweb.org/anthology/S17-2001",
    doi = "10.18653/v1/S17-2001",
}


@inproceedings{dolan2005automatically,
  title={Automatically constructing a corpus of sentential paraphrases},
  author={Dolan, William B and Brockett, Chris},
  booktitle={Proceedings of IWP@IJCNLP 2005},
  year={2005},
    url       = {https://www.aclweb.org/anthology/I05-5002/},
}


@inproceedings{dagan2005pascal,
  title={The PASCAL recognising textual entailment challenge},
  author={Dagan, Ido and Glickman, Oren and Magnini, Bernardo},
  booktitle={{PASCAL} {MLCW} 2005},
  year={2005},

  url       = {https://doi.org/10.1007/11736790\_9},
}
@inproceedings{bar2006second,
  title={The second pascal recognising textual entailment challenge},
  author={Bar-Haim, Roy and Dagan, Ido and Dolan, Bill and Ferro, Lisa and Giampiccolo, Danilo and Magnini, Bernardo and Szpektor, Idan},
  booktitle={Proceedings of the PASCAL@ACL 2006},
  year={2006},
}
@inproceedings{giampiccolo2007third,
  title={The third pascal recognizing textual entailment challenge},
  author={Giampiccolo, Danilo and Magnini, Bernardo and Dagan, Ido and Dolan, Bill},
  booktitle={Proceedings of the PASCAL@ACL 2007},
  year={2007},
}
@inproceedings{bentivogli2009fifth,
  title={The Fifth PASCAL Recognizing Textual Entailment Challenge.},
  author={Bentivogli, Luisa and Clark, Peter and Dagan, Ido and Giampiccolo, Danilo},
  booktitle={TAC},
  year={2009}
}

@article{warstadt2018neural,
  title={Neural network acceptability judgments},
  author={Warstadt, Alex and Singh, Amanpreet and Bowman, Samuel R},
  journal={Transactions of the ACL 2019},
    url={https://www.aclweb.org/anthology/N19-5004.pdf},
  year={2019}
}


@inproceedings{Williams18MNLI,
  author    = {Adina Williams and
               Nikita Nangia and
               Samuel R. Bowman},
  title     = {A Broad-Coverage Challenge Corpus for Sentence Understanding through
               Inference},
  booktitle = {Proceedings NAACL-HLT 2018},
  year      = {2018}, 
  url       = {https://doi.org/10.18653/v1/n18-1101},
  doi       = {10.18653/v1/n18-1101},
  timestamp = {Tue, 28 Jan 2020 10:30:19 +0100},
  biburl    = {https://dblp.org/rec/conf/naacl/WilliamsNB18.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{Lauscher:2019libert,
  author    = {Anne Lauscher and
               Ivan Vuli\'{c} and
               Edoardo Maria Ponti and
               Anna Korhonen and
               Goran Glava\v{s}},
  title     = {Specializing Unsupervised Pretraining Models for Word-Level Semantic Similarity},
  journal   = {arXiv preprint},
  year      = {2019},
  url       = {http://arxiv.org/abs/1909.02339},
}

@article{Brown:2020gpt3,
  author    = {Tom B. Brown and
               Benjamin Mann and
               Nick Ryder and
               Melanie Subbiah and
               Jared Kaplan and
               Prafulla Dhariwal and
               Arvind Neelakantan and
               Pranav Shyam and
               Girish Sastry and
               Amanda Askell and
               Sandhini Agarwal and
               Ariel Herbert{-}Voss and
               Gretchen Krueger and
               Tom Henighan and
               Rewon Child and
               Aditya Ramesh and
               Daniel M. Ziegler and
               Jeffrey Wu and
               Clemens Winter and
               Christopher Hesse and
               Mark Chen and
               Eric Sigler and
               Mateusz Litwin and
               Scott Gray and
               Benjamin Chess and
               Jack Clark and
               Christopher Berner and
               Sam McCandlish and
               Alec Radford and
               Ilya Sutskever and
               Dario Amodei},
  title     = {Language Models are Few-Shot Learners},
  journal   = {arXiv preprint},
  year      = {2020},
  url       = {https://arxiv.org/abs/2005.14165},
}

@article{rueckle2020adapterdrop,
author = {R{\"{u}}ckl{\'{e}}, Andreas and Glockner, Max and Geigle, Gregor and Beck, Tilman and  Pfeiffer, Jonas  and Reimers, Nils and Gurevych, Iryna},
journal = {arXiv preprint},
title = {{AdapterDrop: On the Efficiency of Adapters in Transformers}},
year = {2020}
}

@inproceedings{pfeiffer20madx,
  title={{MAD-X}: An {A}dapter-based {F}ramework for {M}ulti-task {C}ross-lingual {T}ransfer},
  author={Pfeiffer, Jonas and Vuli\'{c}, Ivan and Gurevych, Iryna and Ruder, Sebastian},
  booktitle={Proceedings of EMNLP 2020 },
  year={2020},
  url       = {https://arxiv.org/pdf/2005.00052.pdf},
}

@inproceedings{Eisenschlos2019multifit,
  author    = {Julian Eisenschlos and
               Sebastian Ruder and
               Piotr Czapla and
               Marcin Kardas and
               Sylvain Gugger and
               Jeremy Howard},
  title     = {MultiFiT: Efficient Multi-lingual Language Model Fine-tuning},
  booktitle = {Proceedings of the EMNLP-IJCNLP 2019},
  pages     = {5701--5706},
  year      = {2019}, 
  url       = {https://doi.org/10.18653/v1/D19-1572},
  doi       = {10.18653/v1/D19-1572},
  timestamp = {Mon, 16 Mar 2020 15:48:25 +0100},
  biburl    = {https://dblp.org/rec/conf/emnlp/EisenschlosRCKG19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{Rebuffi2017adapters,
  author    = {Sylvestre{-}Alvise Rebuffi and
               Hakan Bilen and
               Andrea Vedaldi},
  title     = {Learning multiple visual domains with residual adapters},
  booktitle = {Advances in  NeurIPS 2017},
  year      = {2017}, 
  url       = {http://papers.nips.cc/paper/6654-learning-multiple-visual-domains-with-residual-adapters},
  timestamp = {Fri, 06 Mar 2020 16:59:36 +0100},
  biburl    = {https://dblp.org/rec/conf/nips/RebuffiBV17.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{rebuffi2017learning,
  author    = {Sylvestre{-}Alvise Rebuffi and
               Hakan Bilen and
               Andrea Vedaldi},
  title     = {Learning multiple visual domains with residual adapters},
  booktitle = {Advances in  NeurIPS 2017},
  year      = {2017}, 
  url       = {http://papers.nips.cc/paper/6654-learning-multiple-visual-domains-with-residual-adapters},
  timestamp = {Fri, 06 Mar 2020 16:59:36 +0100},
  biburl    = {https://dblp.org/rec/conf/nips/RebuffiBV17.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{peters2019tune,
  author    = {Matthew E. Peters and
               Sebastian Ruder and
               Noah A. Smith},
  title     = {To Tune or Not to Tune? Adapting Pretrained Representations to Diverse
               Tasks},
  booktitle = {Proceedings of the RepL4NLP@ACL 2019},

  year      = {2019}, 
  url       = {https://doi.org/10.18653/v1/w19-4302},
  doi       = {10.18653/v1/w19-4302},
  timestamp = {Tue, 28 Jan 2020 10:30:32 +0100},
  biburl    = {https://dblp.org/rec/conf/rep4nlp/PetersRS19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{Wang20kadapters,
  author    = {Ruize Wang and
               Duyu Tang and
               Nan Duan and
               Zhongyu Wei and
               Xuanjing Huang and
               Jianshu Ji and
               Guihong Cao and
               Daxin Jiang and
               Ming Zhou},
  title     = {K-Adapter: Infusing Knowledge into Pre-Trained Models with Adapters},
  journal   = {arXiv preprint},
  year      = {2020},
  url       = {https://arxiv.org/abs/2002.01808},
  archivePrefix = {arXiv},
  eprint    = {2002.01808},
  timestamp = {Wed, 12 Feb 2020 13:04:16 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-2002-01808.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}
@inproceedings{StrubellGM19,
  author    = {Emma Strubell and
               Ananya Ganesh and
               Andrew McCallum},
  title     = {Energy and Policy Considerations for Deep Learning in {NLP}},
  booktitle = {Proceedings of ACL 2019},
  year      = {2019},
  url       = {https://doi.org/10.18653/v1/p19-1355},
  doi       = {10.18653/v1/p19-1355},
  timestamp = {Tue, 28 Jan 2020 10:27:59 +0100},
  biburl    = {https://dblp.org/rec/conf/acl/StrubellGM19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}
@inproceedings{socher2013-sst,
    title = "Recursive Deep Models for Semantic Compositionality Over a Sentiment Treebank",
    author = "Socher, Richard  and
      Perelygin, Alex  and
      Wu, Jean  and
      Chuang, Jason  and
      Manning, Christopher D.  and
      Ng, Andrew  and
      Potts, Christopher",
    booktitle = "Proceedings of EMNLP 2013",
    year = "2013",
    url = "https://www.aclweb.org/anthology/D13-1170",
}
@inproceedings{wang18glue,
  author    = {Alex Wang and
               Amanpreet Singh and
               Julian Michael and
               Felix Hill and
               Omer Levy and
               Samuel R. Bowman},
  title     = {{GLUE:} {A} Multi-Task Benchmark and Analysis Platform for Natural
               Language Understanding},
  booktitle = {Proceedings of BlackboxNLP@EMNLP 2018},
  year      = {2018}, 
  url       = {https://doi.org/10.18653/v1/w18-5446},
  doi       = {10.18653/v1/w18-5446},
  timestamp = {Tue, 28 Jan 2020 10:28:12 +0100},
  biburl    = {https://dblp.org/rec/conf/emnlp/WangSMHLB18.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@InProceedings{pmlr-v97-stickland19a,
  title = 	 {{BERT} and {PAL}s: Projected Attention Layers for Efficient Adaptation in Multi-Task Learning},
  author = 	 {Stickland, Asa Cooper and Murray, Iain},
  booktitle = 	 {Proceedings of ICML 2019},
  year = 	 {2019},
  editor = 	 {Chaudhuri, Kamalika and Salakhutdinov, Ruslan},

  pdf = 	 {http://proceedings.mlr.press/v97/stickland19a/stickland19a.pdf},
  url = 	 {http://proceedings.mlr.press/v97/stickland19a.html},
  abstract = 	 {Multi-task learning shares information between related tasks, sometimes reducing the number of parameters required. State-of-the-art results across multiple natural language understanding tasks in the GLUE benchmark have previously used transfer from a single large task: unsupervised pre-training with BERT, where a separate BERT model was fine-tuned for each task. We explore multi-task approaches that share a \hbox{single} BERT model with a small number of additional task-specific parameters. Using new adaptation modules, PALs or ‘projected attention layers’, we match the performance of separately fine-tuned models on the GLUE benchmark with $\approx$7 times fewer parameters, and obtain state-of-the-art results on the Recognizing Textual Entailment dataset.}
}

@article{Ba16_LayerNorm,
  author    = {Lei Jimmy Ba and
               Jamie Ryan Kiros and
               Geoffrey E. Hinton},
  title     = {Layer Normalization},
  journal   = {arXiv preprint},
  year      = {2016},
  url       = {http://arxiv.org/abs/1607.06450},
  archivePrefix = {arXiv},
  eprint    = {1607.06450},
  timestamp = {Tue, 23 Jul 2019 17:33:23 +0200},
  biburl    = {https://dblp.org/rec/bib/journals/corr/BaKH16},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}
@inproceedings{Houlsby2019adapters,
  author    = {Neil Houlsby and
               Andrei Giurgiu and
               Stanis{l}aw Jastrz{k{e}}bski and
               Bruna Morrone and
               Quentin de Laroussilhe and
               Andrea Gesmundo and
               Mona Attariyan and
               Sylvain Gelly},
  title     = {Parameter-Efficient Transfer Learning for {NLP}},
  booktitle = {Proceedings of ICML 2019},
  year      = {2019}, 
  url       = {http://proceedings.mlr.press/v97/houlsby19a.html},
  timestamp = {Tue, 11 Jun 2019 15:37:38 +0200},
  biburl    = {https://dblp.org/rec/conf/icml/HoulsbyGJMLGAG19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{Bapna2019adapters,
  author    = {Ankur Bapna and
               Orhan Firat},
  title     = {Simple, Scalable Adaptation for Neural Machine Translation},
  booktitle = {Proceedings of EMNLP-IJCNLP 2019},
  year      = {2019}, 
  url       = {https://doi.org/10.18653/v1/D19-1165},
  doi       = {10.18653/v1/D19-1165},
  timestamp = {Thu, 12 Dec 2019 13:23:44 +0100},
  biburl    = {https://dblp.org/rec/conf/emnlp/BapnaF19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@article{Ponti2020factorization,
  author    = {Edoardo Maria Ponti and
               Ivan Vuli\'{c} and
               Ryan Cotterell and
               Marinela Parovic and
               Roi Reichart and
               Anna Korhonen},
  title     = {Parameter Space Factorization for Zero-Shot Learning across Tasks
               and Languages},
journal   = {arXiv preprint},
  year      = {2020},
  url       = {https://arxiv.org/abs/2001.11453},
  archivePrefix = {arXiv},
  eprint    = {2001.11453},
  timestamp = {Mon, 03 Feb 2020 11:21:05 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-2001-11453.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{Hu2020xtreme,
  author    = {Junjie Hu and
               Sebastian Ruder and
               Aditya Siddhant and
               Graham Neubig and
               Orhan Firat and
               Melvin Johnson},
  title     = {{XTREME:} {A} Massively Multilingual Multi-task Benchmark for Evaluating
               Cross-lingual Generalization},
  booktitle = {Proceedings of ICML 2020},
  year      = {2020},
  url       = {https://arxiv.org/abs/2003.11080},
  archivePrefix = {arXiv},
  eprint    = {2003.11080},
  timestamp = {Wed, 01 Apr 2020 17:39:11 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/abs-2003-11080.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{Pan2017wikiann,
  author    = {Xiaoman Pan and
               Boliang Zhang and
               Jonathan May and
               Joel Nothman and
               Kevin Knight and
               Heng Ji},
  title     = {Cross-lingual Name Tagging and Linking for 282 Languages},
  booktitle = {Proceedings of ACL 2017},
  pages     = {1946--1958},
  year      = {2017}, 
  url       = {https://doi.org/10.18653/v1/P17-1178},
  doi       = {10.18653/v1/P17-1178},
  timestamp = {Sat, 19 Oct 2019 19:58:43 +0200},
  biburl    = {https://dblp.org/rec/conf/acl/PanZMNKJ17.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}




@inproceedings{Rahimi2019massively,
  author    = {Afshin Rahimi and
               Yuan Li and
               Trevor Cohn},
  title     = {Massively Multilingual Transfer for {NER}},
  booktitle = {Proceedings of ACL 2019},
  pages     = {151--164},
  year      = {2019}, 
  url       = {https://doi.org/10.18653/v1/p19-1015},
  doi       = {10.18653/v1/p19-1015},
  timestamp = {Tue, 28 Jan 2020 10:27:34 +0100},
  biburl    = {https://dblp.org/rec/conf/acl/RahimiLC19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@article{Ruder2019survey,
  author    = {Sebastian Ruder and
               Ivan Vuli\'{c} and
               Anders S{\o}gaard},
  title     = {A Survey of Cross-lingual Word Embedding Models},
  journal   = {Journal of Artificial Intelligence Research},
  volume    = {65},
  pages     = {569--631},
  year      = {2019},
  url       = {https://doi.org/10.1613/jair.1.11640},
  doi       = {10.1613/jair.1.11640},
  timestamp = {Wed, 21 Aug 2019 17:13:47 +0200},
  biburl    = {https://dblp.org/rec/journals/jair/RuderVS19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@article{mikolov2013exploiting,
  author    = {Tomas Mikolov and
               Quoc V. Le and
               Ilya Sutskever},
  title     = {Exploiting Similarities among Languages for Machine Translation},
journal   = {arXiv preprint},
  year      = {2013},
  url       = {http://arxiv.org/abs/1309.4168},
  archivePrefix = {arXiv},
  eprint    = {1309.4168},
  timestamp = {Mon, 13 Aug 2018 16:48:55 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/MikolovLS13.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{gouws2015bilbowa,
  author    = {Stephan Gouws and
               Yoshua Bengio and
               Greg Corrado},
  title     = {BilBOWA: Fast Bilingual Distributed Representations without Word Alignments},
  booktitle = {Proceedings of ICML 2015},
  year      = {2015}, 
  url       = {http://proceedings.mlr.press/v37/gouws15.html},
  timestamp = {Wed, 29 May 2019 08:41:45 +0200},
  biburl    = {https://dblp.org/rec/conf/icml/GouwsBC15.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{Wu2019beto,
  author    = {Shijie Wu and
               Mark Dredze},
  title     = {Beto, Bentz, Becas: The Surprising Cross-Lingual Effectiveness of
               {BERT}},
  booktitle = {Proceedings of EMNLP-IJCNLP 2019},
  pages     = {833--844},
  year      = {2019}, 
  url       = {https://doi.org/10.18653/v1/D19-1077},
  doi       = {10.18653/v1/D19-1077},
  timestamp = {Thu, 12 Dec 2019 13:23:45 +0100},
  biburl    = {https://dblp.org/rec/conf/emnlp/WuD19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

 
@inproceedings{Pires2019,
  author    = {Telmo Pires and
               Eva Schlinger and
               Dan Garrette},
  title     = {How Multilingual is Multilingual BERT?},
  booktitle = {Proceedings of ACL 2019},
  year      = {2019}, 
  url       = {https://doi.org/10.18653/v1/p19-1493},
  doi       = {10.18653/v1/p19-1493},
  timestamp = {Tue, 28 Jan 2020 10:27:16 +0100},
  biburl    = {https://dblp.org/rec/conf/acl/PiresSG19.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

 

@inproceedings{Rebuffi2018,
  author    = {Sylvestre{-}Alvise Rebuffi and
               Hakan Bilen and
               Andrea Vedaldi},
  title     = {Efficient Parametrization of Multi-Domain Deep Neural Networks},
  booktitle = {Proceedings of CVPR 2018},
  year      = {2018}, 
  url       = {http://openaccess.thecvf.com/content\_cvpr\_2018/html/Rebuffi\_Efficient\_Parametrization\_of\_CVPR\_2018\_paper.html},
  doi       = {10.1109/CVPR.2018.00847},
  timestamp = {Wed, 16 Oct 2019 14:14:50 +0200},
  biburl    = {https://dblp.org/rec/conf/cvpr/RebuffiBV18.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}




@inproceedings{Howard2018ulmfit,
  author    = {Jeremy Howard and
               Sebastian Ruder},
  title     = {{Universal Language Model Fine-tuning for Text Classification}},
  booktitle = {Proceedings of ACL 2018},
  year      = {2018}, 
  url       = {https://www.aclweb.org/anthology/P18-1031/},
  doi       = {10.18653/v1/P18-1031},
  timestamp = {Mon, 16 Sep 2019 13:46:41 +0200},
  biburl    = {https://dblp.org/rec/conf/acl/RuderH18.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{Wolf2019transformers,
  author    = {Thomas Wolf and
               Lysandre Debut and
               Victor Sanh and
               Julien Chaumond and
               Clement Delangue and
               Anthony Moi andArt
               Pierric Cistac and
               Tim Rault and
               R{\'{e}}mi Louf and
               Morgan Funtowicz and
               Jamie Brew},
  title     = {{HuggingFace's Transformers: State-of-the-art Natural Language Processing}},
booktitle   = {Proceedings of EMNLP 2020: Systems Demonstrations},
  year      = {2019},
  url       = {http://arxiv.org/abs/1910.03771},
  archivePrefix = {arXiv},
  eprint    = {1910.03771},
  timestamp = {Wed, 16 Oct 2019 16:25:53 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1910-03771.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@article{Pfeiffer2020adapterfusion,
author = {Pfeiffer, Jonas and Kamath, Aishwarya and R{\"{u}}ckl{\'{e}}, Andreas and Cho, Kyunghyun and Gurevych, Iryna},
journal = {arXiv preprint},
title = {{AdapterFusion}:  Non-Destructive Task Composition for Transfer Learning},
 url       = {https://arxiv.org/pdf/2005.00247.pdf},
year = {2020}
}



@inproceedings{Cooper2019adapters,
  author    = {Asa Cooper Stickland and
               Iain Murray},
  title     = {{BERT and PALs: Projected Attention Layers for Efficient Adaptation in Multi-Task Learning}},
  booktitle = {Proceedings of ICML 2019},
  year      = {2019}, 
  url       = {http://proceedings.mlr.press/v97/stickland19a.html},
  timestamp = {Tue, 11 Jun 2019 15:37:38 +0200},
  biburl    = {https://dblp.org/rec/conf/icml/Stickland019.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{Wiese2017,
  author    = {Georg Wiese and
               Dirk Weissenborn and
               Mariana L. Neves},
  title     = {{Neural Domain Adaptation for Biomedical Question Answering}},
  booktitle = {Proceedings of CoNLL 2017},
  pages     = {281--289},
  year      = {2017}, 
  url       = {https://doi.org/10.18653/v1/K17-1029},
  doi       = {10.18653/v1/K17-1029},
  timestamp = {Mon, 16 Sep 2019 17:08:53 +0200},
  biburl    = {https://dblp.org/rec/conf/conll/WieseWN17.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{Vaswani2017transformer,
  author    = {Ashish Vaswani and
               Noam Shazeer and
               Niki Parmar and
               Jakob Uszkoreit and
               Llion Jones and
               Aidan N. Gomez and
               Lukasz Kaiser and
               Illia Polosukhin},
  title     = {{Attention Is All You Need}},
  booktitle = {Advances in NeurIPS 2017},
  year      = {2017}, 
  url       = {http://papers.nips.cc/paper/7181-attention-is-all-you-need},
  timestamp = {Fri, 06 Mar 2020 17:00:11 +0100},
  biburl    = {https://dblp.org/rec/conf/nips/VaswaniSPUJGKP17.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}




@inproceedings{Rajpurkar2016squad,
  author    = {Pranav Rajpurkar and
               Jian Zhang and
               Konstantin Lopyrev and
               Percy Liang},
  title     = {{SQuAD: 100,000+ Questions for Machine Comprehension of Text}},
  booktitle = {Proceedings of EMNLP 2016},
  year      = {2016}, 
  url       = {https://doi.org/10.18653/v1/d16-1264},
  doi       = {10.18653/v1/d16-1264},
  timestamp = {Tue, 28 Jan 2020 10:28:11 +0100},
  biburl    = {https://dblp.org/rec/conf/emnlp/RajpurkarZLL16.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}




@inproceedings{roemmele2011choice,
  author    = {Melissa Roemmele and
               Cosmin Adrian Bejan and
               Andrew S. Gordon},
  title     = {Choice of Plausible Alternatives: An Evaluation of Commonsense Causal
               Reasoning},
  booktitle = {Logical Formalizations of Commonsense Reasoning, Papers from the 2011
               {AAAI} Spring Symposium, Technical Report},
  year      = {2011},
  url       = {http://www.aaai.org/ocs/index.php/SSS/SSS11/paper/view/2418},
  timestamp = {Mon, 13 Feb 2012 17:11:42 +0100},
  biburl    = {https://dblp.org/rec/conf/aaaiss/RoemmeleBG11.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}



@article{Ponti2020xcopa,
author = {Ponti, Edoardo Maria and Glava{\v{s}}, Goran and Majewska, Olga and Liu, Qianchu and Vuli{\'{c}}, Ivan and Korhonen, Anna},
journal = {arXiv preprint arXiv:2005.00333},
title = {{XCOPA: A Multilingual Dataset for Causal Commonsense Reasoning}},
url = {https://arxiv.org/pdf/2005.00333.pdf},
year = {2020}
}

@article{sap2019socialiqa,
  title={Socialiqa: Commonsense reasoning about social interactions},
  author={Sap, Maarten and Rashkin, Hannah and Chen, Derek and LeBras, Ronan and Choi, Yejin},
  journal={arXiv preprint arXiv:1904.09728},
  url = {https://arxiv.org/pdf/1904.09728.pdf},
  year={2019}
}

@article{Kaplan2020scalinglaws,
author = {Kaplan, Jared and McCandlish, Sam and Henighan, Tom and Brown, Tom B. and Chess, Benjamin and Child, Rewon and Gray, Scott and Radford, Alec and Wu, Jeffrey and Amodei, Dario},
journal = {arXiv preprint},
title = {{Scaling Laws for Neural Language Models}},
year = {2020},
url       = {https://arxiv.org/abs/2001.08361},
}

@article{Raffel2019t3,
author = {Raffel, Colin and Shazeer, Noam and Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and Li, Wei and Liu, Peter J.},
eprint = {1910.10683},
file = {:Users/sebastian/Library/Application Support/Mendeley Desktop/Downloaded/Raffel et al. - 2019 - Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer.pdf:pdf},
journal = {arXiv preprint},
title = {{Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer}},
url = {http://arxiv.org/abs/1910.10683},
year = {2019}
}

@inproceedings{Andreas2016learningtocompose,
author = {Andreas, Jacob and Rohrbach, Marcus and Darrell, Trevor and Klein, Dan},
booktitle = {Proceedings of NAACL 2016},
eprint = {arXiv:1601.01705v1},
file = {:Users/sebastian/Library/Application Support/Mendeley Desktop/Downloaded/Andreas et al. - 2016 - Learning to Compose Neural Networks for Question Answering.pdf:pdf},
mendeley-groups = {Deep Learning,Deep Learning/QA},
title = {{Learning to Compose Neural Networks for Question Answering}},
year = {2016}
}

@inproceedings{Shazeer2017mixtureofexperts,
author = {Shazeer, Noam and Mirhoseini, Azalia and Maziarz, Krzysztof and Davis, Andy and Le, Quoc and Dean, Jeff},
booktitle = {Proceedings of ICLR 2017},
file = {:Users/sebastian/Library/Application Support/Mendeley Desktop/Downloaded/Shazeer et al. - 2017 - Outrageously Large Neural Networks The Sparsely-Gated Mixture-of-Experts Layer.pdf:pdf},
mendeley-groups = {Deep Learning/LM,Papers/Semi-supervised data selection},
title = {{Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer}},
url       = {https://openreview.net/forum?id=B1ckMDqlg},
year = {2017}
}

@article{Wang2020k-adapter,
author = {Wang, Ruize and Tang, Duyu and Duan, Nan and Wei, Zhongyu and Huang, Xuanjing and Cao, Guihong and Jiang, Daxin and Zhou, Ming},
eprint = {arXiv:2002.01808v3},
file = {:Users/sebastian/Library/Application Support/Mendeley Desktop/Downloaded/Wang et al. - 2019 - Infusing Knowledge into Pre-Trained Models with Adapters.pdf:pdf},
journal = {arXiv preprint},
mendeley-groups = {Deep Learning/Embeddings/Entity/relation,Adaptation/Transfer Learning},
title = {{K-Adapter: Infusing Knowledge into Pre-Trained Models with Adapters}},
year = {2020}
}

@inproceedings{Pruksachatkun2020,
author = {Pruksachatkun, Yada and Phang, Jason and Liu, Haokun and Htut, Phu Mon and Zhang, Xiaoyi and Pang, Richard Yuanzhe and Vania, Clara and Kann, Katharina and Bowman, Samuel R.},
booktitle = {Proceedings of ACL 2020},
title = {{Intermediate-Task Transfer Learning with Pretrained Models for Natural Language Understanding: When and Why Does It Work?}},
url = {http://arxiv.org/abs/2005.00628},
year = {2020}
}

@inproceedings{ruder2019transfer,
  title={Transfer learning in natural language processing},
  author={Ruder, Sebastian and Peters, Matthew E and Swayamdipta, Swabha and Wolf, Thomas},
  booktitle={Proceedings of NAACL 2019: Tutorials},
  year={2019},
  url={https://www.aclweb.org/anthology/N19-5004.pdf}
}

@article{Ruder2017,
author = {Ruder, Sebastian},
journal = {arXiv preprint},
eprint = {arXiv:1706.05098v1},
file = {:Users/sebastian/Library/Application Support/Mendeley Desktop/Downloaded/Ruder - 2017 - An Overview of Multi-Task Learning in Deep Neural Networks.pdf:pdf},
title = {{An Overview of Multi-Task Learning in Deep Neural Networks}},
year = {2017}
}

@inproceedings{Masson2019episodicmemory,
author = {Masson, Cyprien De and Ruder, Sebastian and Kong, Lingpeng and Yogatama, Dani},
booktitle = {Advances in  NeurIPS 2019},
title = {{Episodic Memory in Lifelong Language Learning}},
year = {2019},
  url       = {http://papers.nips.cc/paper/9471-episodic-memory-in-lifelong-language-learning},
}

@inproceedings{Hashimoto2017ajointmanytask,
author = {Hashimoto, Kazuma and Xiong, Caiming and Tsuruoka, Yoshimasa and Socher, Richard},
booktitle = {Proceedings of EMNLP 2017},
eprint = {1611.01587},
file = {:Users/sebastian/Library/Application Support/Mendeley Desktop/Downloaded/Hashimoto et al. - 2017 - A Joint Many-Task Model Growing a Neural Network for Multiple NLP Tasks.pdf:pdf},
isbn = {9781467365611},
mendeley-groups = {Papers/Mini viva,Adaptation/Multitask Learning},
title = {{A Joint Many-Task Model: Growing a Neural Network for Multiple NLP Tasks}},
url = {http://arxiv.org/abs/1611.01587},
year = {2017}
}

@inproceedings{Sanh2019hierarchical,
author = {Sanh, Victor and Wolf, Thomas and Ruder, Sebastian},
booktitle = {Proceedings of AAAI 2019},
eprint = {arXiv:1811.06031v1},
file = {:Users/sebastian/Library/Application Support/Mendeley Desktop/Downloaded/Sanh, Wolf, Ruder - 2019 - A Hierarchical Multi-task Approach for Learning Embeddings from Semantic Tasks.pdf:pdf},
mendeley-groups = {Adaptation/Multitask Learning},
title = {{A Hierarchical Multi-task Approach for Learning Embeddings from Semantic Tasks}},
year = {2019}
}

@article{phang2018sentence,
  title={Sentence encoders on stilts: Supplementary training on intermediate labeled-data tasks},
  author={Phang, Jason and F{\'e}vry, Thibault and Bowman, Samuel R},
  journal={arXiv preprint},
  url       = {http://arxiv.org/abs/1811.01088},
  year={2018}
}

@inproceedings{gururangan2020don,
  title={Don't Stop Pretraining: Adapt Language Models to Domains and Tasks},
  author={Gururangan, Suchin and Marasovi{\'c}, Ana and Swayamdipta, Swabha and Lo, Kyle and Beltagy, Iz and Downey, Doug and Smith, Noah A},
  booktitle={Proceedings of ACL 2020},
  year={2020},
  url={https://www.aclweb.org/anthology/2020.acl-main.740.pdf}
}