@inproceedings{ramrakhiyani-etal-2023-zero,
    title = "Zero-shot Probing of Pretrained Language Models for Geography Knowledge",
    author = "Ramrakhiyani, Nitin  and
      Varma, Vasudeva  and
      Palshikar, Girish  and
      Pawar, Sachin",
    editor = {Deutsch, Daniel  and
      Dror, Rotem  and
      Eger, Steffen  and
      Gao, Yang  and
      Leiter, Christoph  and
      Opitz, Juri  and
      R{\"u}ckl{\'e}, Andreas},
    booktitle = "Proceedings of the 4th Workshop on Evaluation and Comparison of NLP Systems",
    month = nov,
    year = "2023",
    address = "Bali, Indonesia",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2023.eval4nlp-1.5/",
    doi = "10.18653/v1/2023.eval4nlp-1.5",
    pages = "49--61",
    abstract = "Gauging the knowledge of Pretrained Language Models (PLMs) about facts in niche domains is an important step towards making them better in those domains. In this paper, we aim at evaluating multiple PLMs for their knowledge about world Geography. We contribute (i) a sufficiently sized dataset of masked Geography sentences to probe PLMs on masked token prediction and generation tasks, (ii) benchmark the performance of multiple PLMs on the dataset. We also provide a detailed analysis of the performance of the PLMs on different Geography facts."
}