%%% ====================================================================
%%%  BibTeX-file{
%%%     author          = "David Rhead",
%%%     version         = "1.00",
%%%     date            = "17 Feb 1990",
%%%     time            = "17:00 GMT",
%%%     filename        = "test.bib",
%%%     address         = "Cripps Computing Centre,
%%%                        University of Nottingham,
%%%                        University Park,
%%%                        Nottingham,
%%%                        NG7 2RD,
%%%                        United Kingdom",
%%%     telephone       = "+44 602 484848 Ext 2670",
%%%     FAX             = "+44 602 588138",
%%%     checksum        = "05151 839 2908 25082",
%%%     email           = "David_Rhead at uk.ac.nott.vme (JANET)",
%%%     codetable       = "ISO/ASCII",
%%%     keywords        = "bibliography, citation, references",
%%%     supported       = "no",
%%%     docstring       = "This BibTeX database file contains entries
%%%                        designed for testing whether a BibTeX style
%%%                        file lays references out as recommended by
%%%                        certain authorities.  (Note, however, that
%%%                        the BS 1629 examples are from the 1976
%%%                        edition.  The file needs updating to use
%%%                        examples from the 1989 edition instead.)
%%%
%%%                        The checksum field above contains a CRC-16
%%%                        checksum as the first value, followed by the
%%%                        equivalent of the standard UNIX wc (word
%%%                        count) utility output of lines, words, and
%%%                        characters.  This is produced by Robert
%%%                        Solovay's checksum utility.",
%%%  }
%%% ====================================================================


@inproceedings{clark-manning-2016-improving,
    title = "Improving Coreference Resolution by Learning Entity-Level Distributed Representations",
    author = "Clark, Kevin  and
      Manning, Christopher D.",
    booktitle = "Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = aug,
    year = "2016",
    address = "Berlin, Germany",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/P16-1061",
    doi = "10.18653/v1/P16-1061",
    pages = "643--653",
}

@inproceedings{drozdov-etal-2022-inducing,
    title = "Inducing and Using Alignments for Transition-based {AMR} Parsing",
    author = "Drozdov, Andrew  and
      Zhou, Jiawei  and
      Florian, Radu  and
      McCallum, Andrew  and
      Naseem, Tahira  and
      Kim, Yoon  and
      Astudillo, Ram{\'o}n",
    booktitle = "Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",
    month = jul,
    year = "2022",
    address = "Seattle, United States",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.naacl-main.80",
    doi = "10.18653/v1/2022.naacl-main.80",
    pages = "1086--1098",
    abstract = "Transition-based parsers for Abstract Meaning Representation (AMR) rely on node-to-word alignments. These alignments are learned separately from parser training and require a complex pipeline of rule-based components, pre-processing, and post-processing to satisfy domain-specific constraints. Parsers also train on a point-estimate of the alignment pipeline, neglecting the uncertainty due to the inherent ambiguity of alignment. In this work we explore two avenues for overcoming these limitations. First, we propose a neural aligner for AMR that learns node-to-word alignments without relying on complex pipelines. We subsequently explore a tighter integration of aligner and parser training by considering a distribution over oracle action sequences arising from aligner uncertainty. Empirical results show this approach leads to more accurate alignments and generalization better from the AMR2.0 to AMR3.0 corpora. We attain a new state-of-the art for gold-only trained models, matching silver-trained performance without the need for beam search on AMR3.0.",
}

@inproceedings{cai-etal-2021-multilingual-amr,
    title = "Multilingual {AMR} Parsing with Noisy Knowledge Distillation",
    author = "Cai, Deng  and
      Li, Xin  and
      Ho, Jackie Chun-Sing  and
      Bing, Lidong  and
      Lam, Wai",
    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2021",
    month = nov,
    year = "2021",
    address = "Punta Cana, Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.findings-emnlp.237",
    doi = "10.18653/v1/2021.findings-emnlp.237",
    pages = "2778--2789",
    abstract = "We study multilingual AMR parsing from the perspective of knowledge distillation, where the aim is to learn and improve a multilingual AMR parser by using an existing English parser as its teacher. We constrain our exploration in a strict multilingual setting: there is but one model to parse all different languages including English. We identify that noisy input and precise output are the key to successful distillation. Together with extensive pre-training, we obtain an AMR parser whose performances surpass all previously published results on four different foreign languages, including German, Spanish, Italian, and Chinese, by large margins (up to 18.8 Smatch points on Chinese and on average 11.3 Smatch points). Our parser also achieves comparable performance on English to the latest state-of-the-art English-only parser.",
}

@inproceedings{kiela-etal-2021-dynabench,
    title = "Dynabench: Rethinking Benchmarking in {NLP}",
    author = "Kiela, Douwe  and
      Bartolo, Max  and
      Nie, Yixin  and
      Kaushik, Divyansh  and
      Geiger, Atticus  and
      Wu, Zhengxuan  and
      Vidgen, Bertie  and
      Prasad, Grusha  and
      Singh, Amanpreet  and
      Ringshia, Pratik  and
      Ma, Zhiyi  and
      Thrush, Tristan  and
      Riedel, Sebastian  and
      Waseem, Zeerak  and
      Stenetorp, Pontus  and
      Jia, Robin  and
      Bansal, Mohit  and
      Potts, Christopher  and
      Williams, Adina",
    booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",
    month = jun,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.naacl-main.324",
    doi = "10.18653/v1/2021.naacl-main.324",
    pages = "4110--4124",
    abstract = "We introduce Dynabench, an open-source platform for dynamic dataset creation and model benchmarking. Dynabench runs in a web browser and supports human-and-model-in-the-loop dataset creation: annotators seek to create examples that a target model will misclassify, but that another person will not. In this paper, we argue that Dynabench addresses a critical need in our community: contemporary models quickly achieve outstanding performance on benchmark tasks but nonetheless fail on simple challenge examples and falter in real-world scenarios. With Dynabench, dataset creation, model development, and model assessment can directly inform each other, leading to more robust and informative benchmarks. We report on four initial NLP tasks, illustrating these concepts and highlighting the promise of the platform, and address potential objections to dynamic benchmarking as a new standard for the field.",
}


@inproceedings{lin-2004-rouge,
    title = "{ROUGE}: A Package for Automatic Evaluation of Summaries",
    author = "Lin, Chin-Yew",
    booktitle = "Text Summarization Branches Out",
    month = jul,
    year = "2004",
    address = "Barcelona, Spain",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/W04-1013",
    pages = "74--81",
}

@inproceedings{banerjee-lavie-2005-meteor,
    title = "{METEOR}: An Automatic Metric for {MT} Evaluation with Improved Correlation with Human Judgments",
    author = "Banerjee, Satanjeev  and
      Lavie, Alon",
    booktitle = "Proceedings of the {Association for Computational Linguistics} Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and/or Summarization",
    month = jun,
    year = "2005",
    address = "Ann Arbor, Michigan",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/W05-0909",
    pages = "65--72",
}

@inproceedings{papineni-etal-2002-bleu,
    title = "{B}leu: a Method for Automatic Evaluation of Machine Translation",
    author = "Papineni, Kishore  and
      Roukos, Salim  and
      Ward, Todd  and
      Zhu, Wei-Jing",
    booktitle = "Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2002",
    address = "Philadelphia, Pennsylvania, USA",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/P02-1040",
    doi = "10.3115/1073083.1073135",
    pages = "311--318",
}

@inproceedings{chen-etal-2019-evaluating,
    title = "Evaluating Question Answering Evaluation",
    author = "Chen, Anthony  and
      Stanovsky, Gabriel  and
      Singh, Sameer  and
      Gardner, Matt",
    booktitle = "Proceedings of the 2nd Workshop on Machine Reading for Question Answering",
    month = nov,
    year = "2019",
    address = "Hong Kong, China",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/D19-5817",
    doi = "10.18653/v1/D19-5817",
    pages = "119--124",
    abstract = "As the complexity of question answering (QA) datasets evolve, moving away from restricted formats like span extraction and multiple-choice (MC) to free-form answer generation, it is imperative to understand how well current metrics perform in evaluating QA. This is especially important as existing metrics (BLEU, ROUGE, METEOR, and F1) are computed using n-gram similarity and have a number of well-known drawbacks. In this work, we study the suitability of existing metrics in QA. For generative QA, we show that while current metrics do well on existing datasets, converting multiple-choice datasets into free-response datasets is challenging for current metrics. We also look at span-based QA, where F1 is a reasonable metric. We show that F1 may not be suitable for all extractive QA tasks depending on the answer types. Our study suggests that while current metrics may be suitable for existing QA datasets, they limit the complexity of QA datasets that can be created. This is especially true in the context of free-form QA, where we would like our models to be able to generate more complex and abstractive answers, thus necessitating new metrics that go beyond n-gram based matching. As a step towards a better QA metric, we explore using BERTScore, a recently proposed metric for evaluating translation, for QA. We find that although it fails to provide stronger correlation with human judgements, future work focused on tailoring a BERT-based metric to QA evaluation may prove fruitful.",
}

@article{DBLP:journals/corr/SchulmanWDRK17,
  author       = {John Schulman and
                  Filip Wolski and
                  Prafulla Dhariwal and
                  Alec Radford and
                  Oleg Klimov},
  title        = {Proximal Policy Optimization Algorithms},
  journal      = {CoRR},
  volume       = {abs/1707.06347},
  year         = {2017},
  url          = {http://arxiv.org/abs/1707.06347},
  eprinttype    = {arXiv},
  eprint       = {1707.06347},
  timestamp    = {Mon, 13 Aug 2018 16:47:34 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/SchulmanWDRK17.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{siriwardhana-etal-2023-improving,
    title = "Improving the Domain Adaptation of Retrieval Augmented Generation ({RAG}) Models for Open Domain Question Answering",
    author = "Siriwardhana, Shamane  and
      Weerasekera, Rivindu  and
      Wen, Elliott  and
      Kaluarachchi, Tharindu  and
      Rana, Rajib  and
      Nanayakkara, Suranga",
    journal = "Transactions of the Association for Computational Linguistics",
    volume = "11",
    year = "2023",
    address = "Cambridge, MA",
    publisher = "MIT Press",
    url = "https://aclanthology.org/2023.tacl-1.1",
    doi = "10.1162/tacl_a_00530",
    pages = "1--17",
    abstract = "Retrieval Augment Generation (RAG) is a recent advancement in Open-Domain Question Answering (ODQA). RAG has only been trained and explored with a Wikipedia-based external knowledge base and is not optimized for use in other specialized domains such as healthcare and news. In this paper, we evaluate the impact of joint training of the retriever and generator components of RAG for the task of domain adaptation in ODQA. We propose RAG-end2end, an extension to RAG that can adapt to a domain-specific knowledge base by updating all components of the external knowledge base during training. In addition, we introduce an auxiliary training signal to inject more domain-specific knowledge. This auxiliary signal forces RAG-end2end to reconstruct a given sentence by accessing the relevant information from the external knowledge base. Our novel contribution is that, unlike RAG, RAG-end2end does joint training of the retriever and generator for the end QA task and domain adaptation. We evaluate our approach with datasets from three domains: COVID-19, News, and Conversations, and achieve significant performance improvements compared to the original RAG model. Our work has been open-sourced through the HuggingFace Transformers library, attesting to our work{'}s credibility and technical consistency.",
}

@inproceedings{DBLP:conf/nips/VaswaniSPUJGKP17,
  author       = {Ashish Vaswani and
                  Noam Shazeer and
                  Niki Parmar and
                  Jakob Uszkoreit and
                  Llion Jones and
                  Aidan N. Gomez and
                  Lukasz Kaiser and
                  Illia Polosukhin},
  editor       = {Isabelle Guyon and
                  Ulrike von Luxburg and
                  Samy Bengio and
                  Hanna M. Wallach and
                  Rob Fergus and
                  S. V. N. Vishwanathan and
                  Roman Garnett},
  title        = {Attention is All you Need},
  booktitle    = {Advances in Neural Information Processing Systems 30: Annual Conference
                  on Neural Information Processing Systems 2017, December 4-9, 2017,
                  Long Beach, CA, {USA}},
  pages        = {5998--6008},
  year         = {2017},
  url          = {https://proceedings.neurips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html},
  timestamp    = {Thu, 21 Jan 2021 15:15:21 +0100},
  biburl       = {https://dblp.org/rec/conf/nips/VaswaniSPUJGKP17.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@book{DBLP:series/synthesis/2021Hogan,
  author       = {Aidan Hogan and
                  Eva Blomqvist and
                  Michael Cochez and
                  Claudia d'Amato and
                  Gerard de Melo and
                  Claudio Gutierrez and
                  Sabrina Kirrane and
                  Jos{\'{e}} Emilio Labra Gayo and
                  Roberto Navigli and
                  Sebastian Neumaier and
                  Axel{-}Cyrille Ngonga Ngomo and
                  Axel Polleres and
                  Sabbir M. Rashid and
                  Anisa Rula and
                  Lukas Schmelzeisen and
                  Juan Sequeda and
                  Steffen Staab and
                  Antoine Zimmermann},
  title        = {Knowledge Graphs},
  series       = {Synthesis Lectures on Data, Semantics, and Knowledge},
  publisher    = {Morgan {\&} Claypool Publishers},
  year         = {2021},
  url          = {https://doi.org/10.2200/S01125ED1V01Y202109DSK022},
  doi          = {10.2200/S01125ED1V01Y202109DSK022},
  timestamp    = {Mon, 26 Jun 2023 08:47:45 +0200},
  biburl       = {https://dblp.org/rec/series/synthesis/2021Hogan.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{petroni-etal-2019-language,
    title = "Language Models as Knowledge Bases?",
    author = {Petroni, Fabio  and
      Rockt{\"a}schel, Tim  and
      Riedel, Sebastian  and
      Lewis, Patrick  and
      Bakhtin, Anton  and
      Wu, Yuxiang  and
      Miller, Alexander},
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)",
    month = nov,
    year = "2019",
    address = "Hong Kong, China",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/D19-1250",
    doi = "10.18653/v1/D19-1250",
    pages = "2463--2473",
    abstract = "Recent progress in pretraining language models on large textual corpora led to a surge of improvements for downstream NLP tasks. Whilst learning linguistic knowledge, these models may also be storing relational knowledge present in the training data, and may be able to answer queries structured as {``}fill-in-the-blank{''} cloze statements. Language models have many advantages over structured knowledge bases: they require no schema engineering, allow practitioners to query about an open class of relations, are easy to extend to more data, and require no human supervision to train. We present an in-depth analysis of the relational knowledge already present (without fine-tuning) in a wide range of state-of-the-art pretrained language models. We find that (i) without fine-tuning, BERT contains relational knowledge competitive with traditional NLP methods that have some access to oracle knowledge, (ii) BERT also does remarkably well on open-domain question answering against a supervised baseline, and (iii) certain types of factual knowledge are learned much more readily than others by standard language model pretraining approaches. The surprisingly strong ability of these models to recall factual knowledge without any fine-tuning demonstrates their potential as unsupervised open-domain QA systems. The code to reproduce our analysis is available at \url{https://github.com/facebookresearch/LAMA}.",
}

@inproceedings{DBLP:conf/fat/BenderGMS21,
  author       = {Emily M. Bender and
                  Timnit Gebru and
                  Angelina McMillan{-}Major and
                  Shmargaret Shmitchell},
  editor       = {Madeleine Clare Elish and
                  William Isaac and
                  Richard S. Zemel},
  title        = {On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?},
  booktitle    = {FAccT '21: 2021 {ACM} Conference on Fairness, Accountability, and
                  Transparency, Virtual Event / Toronto, Canada, March 3-10, 2021},
  pages        = {610--623},
  publisher    = {{Association for Computing Machinery}},
  year         = {2021},
  url          = {https://doi.org/10.1145/3442188.3445922},
  doi          = {10.1145/3442188.3445922},
  timestamp    = {Fri, 05 May 2023 15:54:55 +0200},
  biburl       = {https://dblp.org/rec/conf/fat/BenderGMS21.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{DBLP:conf/icml/GuuLTPC20,
  author       = {Kelvin Guu and
                  Kenton Lee and
                  Zora Tung and
                  Panupong Pasupat and
                  Ming{-}Wei Chang},
  title        = {Retrieval Augmented Language Model Pre-Training},
  booktitle    = {Proceedings of the 37th International Conference on Machine Learning,
                  {ICML} 2020, 13-18 July 2020, Virtual Event},
  series       = {Proceedings of Machine Learning Research},
  volume       = {119},
  pages        = {3929--3938},
  publisher    = {{PMLR}},
  year         = {2020},
  url          = {http://proceedings.mlr.press/v119/guu20a.html},
  timestamp    = {Tue, 15 Dec 2020 17:40:18 +0100},
  biburl       = {https://dblp.org/rec/conf/icml/GuuLTPC20.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{DBLP:conf/nips/LewisPPPKGKLYR020,
  author       = {Patrick S. H. Lewis and
                  Ethan Perez and
                  Aleksandra Piktus and
                  Fabio Petroni and
                  Vladimir Karpukhin and
                  Naman Goyal and
                  Heinrich K{\"{u}}ttler and
                  Mike Lewis and
                  Wen{-}tau Yih and
                  Tim Rockt{\"{a}}schel and
                  Sebastian Riedel and
                  Douwe Kiela},
  editor       = {Hugo Larochelle and
                  Marc'Aurelio Ranzato and
                  Raia Hadsell and
                  Maria{-}Florina Balcan and
                  Hsuan{-}Tien Lin},
  title        = {Retrieval-Augmented Generation for Knowledge-Intensive {NLP} Tasks},
  booktitle    = {Advances in Neural Information Processing Systems 33: Annual Conference
                  on Neural Information Processing Systems 2020, NeurIPS 2020, December
                  6-12, 2020, virtual},
  year         = {2020},
  url          = {https://proceedings.neurips.cc/paper/2020/hash/6b493230205f780e1bc26945df7481e5-Abstract.html},
  timestamp    = {Tue, 19 Jan 2021 15:57:07 +0100},
  biburl       = {https://dblp.org/rec/conf/nips/LewisPPPKGKLYR020.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/jmlr/RaffelSRLNMZLL20,
  author       = {Colin Raffel and
                  Noam Shazeer and
                  Adam Roberts and
                  Katherine Lee and
                  Sharan Narang and
                  Michael Matena and
                  Yanqi Zhou and
                  Wei Li and
                  Peter J. Liu},
  title        = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text
                  Transformer},
  journal      = {J. Mach. Learn. Res.},
  volume       = {21},
  pages        = {140:1--140:67},
  year         = {2020},
  url          = {http://jmlr.org/papers/v21/20-074.html},
  timestamp    = {Fri, 05 Feb 2021 15:43:41 +0100},
  biburl       = {https://dblp.org/rec/journals/jmlr/RaffelSRLNMZLL20.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{laskar-etal-2023-systematic,
    title = "A Systematic Study and Comprehensive Evaluation of {C}hat{GPT} on Benchmark Datasets",
    author = "Laskar, Md Tahmid Rahman  and
      Bari, M Saiful  and
      Rahman, Mizanur  and
      Bhuiyan, Md Amran Hossen  and
      Joty, Shafiq  and
      Huang, Jimmy",
    booktitle = "Findings of the Association for Computational Linguistics: Association for Computational Linguistics 2023",
    month = jul,
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.findings-acl.29",
    doi = "10.18653/v1/2023.findings-acl.29",
    pages = "431--469",
    abstract = "The development of large language models (LLMs) such as ChatGPT has brought a lot of attention recently. However, their evaluation in the benchmark academic datasets remains under-explored due to the difficulty of evaluating the generative outputs produced by this model against the ground truth. In this paper, we aim to present a thorough evaluation of ChatGPT{'}s performance on diverse academic datasets, covering tasks like question-answering, text summarization, code generation, commonsense reasoning, mathematical problem-solving, machine translation, bias detection, and ethical considerations. Specifically, we evaluate ChatGPT across 140 tasks and analyze 255K responses it generates in these datasets. This makes our work the largest evaluation of ChatGPT in NLP benchmarks. In short, our study aims to validate the strengths and weaknesses of ChatGPT in various tasks and provide insights for future research using LLMs. We also report a new emergent ability to follow multi-query instructions that we mostly found in ChatGPT and other instruction-tuned models. Our extensive evaluation shows that even though ChatGPT is capable of performing a wide variety of tasks, and may obtain impressive performance in several benchmark datasets, it is still far from achieving the ability to reliably solve many challenging tasks. By providing a thorough assessment of ChatGPT{'}s performance across diverse NLP tasks, this paper sets the stage for a targeted deployment of ChatGPT-like LLMs in real-world applications.",
}

@inproceedings{DBLP:conf/nips/Ouyang0JAWMZASR22,
  author       = {Long Ouyang and
                  Jeffrey Wu and
                  Xu Jiang and
                  Diogo Almeida and
                  Carroll L. Wainwright and
                  Pamela Mishkin and
                  Chong Zhang and
                  Sandhini Agarwal and
                  Katarina Slama and
                  Alex Ray and
                  John Schulman and
                  Jacob Hilton and
                  Fraser Kelton and
                  Luke Miller and
                  Maddie Simens and
                  Amanda Askell and
                  Peter Welinder and
                  Paul F. Christiano and
                  Jan Leike and
                  Ryan Lowe},
  title        = {{Training language models to follow instructions with human feedback}},
  booktitle    = {NeurIPS},
  year         = {2022},
  url          = {http://papers.nips.cc/paper\_files/paper/2022/hash/b1efde53be364a73914f58805a001731-Abstract-Conference.html},
  timestamp    = {Thu, 11 May 2023 17:08:21 +0200},
  biburl       = {https://dblp.org/rec/conf/nips/Ouyang0JAWMZASR22.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{yuan-etal-2022-biobart,
    title = "{B}io{BART}: Pretraining and Evaluation of A Biomedical Generative Language Model",
    author = "Yuan, Hongyi  and
      Yuan, Zheng  and
      Gan, Ruyi  and
      Zhang, Jiaxing  and
      Xie, Yutao  and
      Yu, Sheng",
    booktitle = "Proceedings of the 21st Workshop on Biomedical Language Processing",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.bionlp-1.9",
    doi = "10.18653/v1/2022.bionlp-1.9",
    pages = "97--109",
    abstract = "Pretrained language models have served as important backbones for natural language processing. Recently, in-domain pretraining has been shown to benefit various domain-specific downstream tasks. In the biomedical domain, natural language generation (NLG) tasks are of critical importance, while understudied. Approaching natural language understanding (NLU) tasks as NLG achieves satisfying performance in the general domain through constrained language generation or language prompting. We emphasize the lack of in-domain generative language models and the unsystematic generative downstream benchmarks in the biomedical domain, hindering the development of the research community. In this work, we introduce the generative language model BioBART that adapts BART to the biomedical domain. We collate various biomedical language generation tasks including dialogue, summarization, entity linking, and named entity recognition. BioBART pretrained on PubMed abstracts has enhanced performance compared to BART and set strong baselines on several tasks. Furthermore, we conduct ablation studies on the pretraining tasks for BioBART and find that sentence permutation has negative effects on downstream tasks.",
}

@article{DBLP:journals/bib/LuoSXQZPL22,
  author       = {Renqian Luo and
                  Liai Sun and
                  Yingce Xia and
                  Tao Qin and
                  Sheng Zhang and
                  Hoifung Poon and
                  Tie{-}Yan Liu},
  title        = {{BioGPT: generative pre-trained transformer for biomedical text generation
                  and mining}},
  journal      = {Briefings Bioinform.},
  volume       = {23},
  number       = {6},
  year         = {2022},
  url          = {https://doi.org/10.1093/bib/bbac409},
  doi          = {10.1093/bib/bbac409},
  timestamp    = {Mon, 28 Aug 2023 21:31:13 +0200},
  biburl       = {https://dblp.org/rec/journals/bib/LuoSXQZPL22.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/corr/abs-2212-14548,
  author       = {Bowen Zhang and
                  Daijun Ding and
                  Liwen Jing},
  title        = {{How would Stance Detection Techniques Evolve after the Launch of ChatGPT?}},
  journal      = {CoRR},
  volume       = {abs/2212.14548},
  year         = {2022},
  url          = {https://doi.org/10.48550/arXiv.2212.14548},
  doi          = {10.48550/arXiv.2212.14548},
  eprinttype    = {arXiv},
  eprint       = {2212.14548},
  timestamp    = {Sun, 08 Jan 2023 14:16:55 +0100},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2212-14548.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{DBLP:conf/emnlp/DodgeSMAIGM021,
  author       = {Jesse Dodge and
                  Maarten Sap and
                  Ana Marasovic and
                  William Agnew and
                  Gabriel Ilharco and
                  Dirk Groeneveld and
                  Margaret Mitchell and
                  Matt Gardner},
  editor       = {Marie{-}Francine Moens and
                  Xuanjing Huang and
                  Lucia Specia and
                  Scott Wen{-}tau Yih},
  title        = {Documenting Large Webtext Corpora: {A} Case Study on the Colossal
                  Clean Crawled Corpus},
  booktitle    = {Proceedings of the 2021 Conference on Empirical Methods in Natural
                  Language Processing, {EMNLP} 2021, Virtual Event / Punta Cana, Dominican
                  Republic, 7-11 November, 2021},
  pages        = {1286--1305},
  publisher    = {Association for Computational Linguistics},
  year         = {2021},
  url          = {https://doi.org/10.18653/v1/2021.emnlp-main.98},
  doi          = {10.18653/v1/2021.emnlp-main.98},
  timestamp    = {Thu, 20 Jan 2022 10:02:08 +0100},
  biburl       = {https://dblp.org/rec/conf/emnlp/DodgeSMAIGM021.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{DBLP:conf/uss/CarliniTWJHLRBS21,
  author       = {Nicholas Carlini and
                  Florian Tram{\`{e}}r and
                  Eric Wallace and
                  Matthew Jagielski and
                  Ariel Herbert{-}Voss and
                  Katherine Lee and
                  Adam Roberts and
                  Tom B. Brown and
                  Dawn Song and
                  {\'{U}}lfar Erlingsson and
                  Alina Oprea and
                  Colin Raffel},
  editor       = {Michael Bailey and
                  Rachel Greenstadt},
  title        = {{Extracting Training Data from Large Language Models}},
  booktitle    = {30th {USENIX} Security Symposium, {USENIX} Security 2021, August 11-13,
                  2021},
  pages        = {2633--2650},
  publisher    = {{USENIX} Association},
  year         = {2021},
  url          = {https://www.usenix.org/conference/usenixsecurity21/presentation/carlini-extracting},
  timestamp    = {Sun, 02 Oct 2022 16:16:46 +0200},
  biburl       = {https://dblp.org/rec/conf/uss/CarliniTWJHLRBS21.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@online{chatgpt,
  author = {OpenAI},
  title = {{Chatgpt: Optimizing language models for dialogue}},
  year = 2022,
  url = {https://archive.ph/4snnY}
}


@article{DBLP:journals/csur/JiLFYSXIBMF23,
  author       = {Ziwei Ji and
                  Nayeon Lee and
                  Rita Frieske and
                  Tiezheng Yu and
                  Dan Su and
                  Yan Xu and
                  Etsuko Ishii and
                  Yejin Bang and
                  Andrea Madotto and
                  Pascale Fung},
  title        = {{Survey of Hallucination in Natural Language Generation}},
  journal      = {{ACM} Comput. Surv.},
  volume       = {55},
  number       = {12},
  pages        = {248:1--248:38},
  year         = {2023},
  url          = {https://doi.org/10.1145/3571730},
  doi          = {10.1145/3571730},
  timestamp    = {Mon, 28 Aug 2023 21:19:33 +0200},
  biburl       = {https://dblp.org/rec/journals/csur/JiLFYSXIBMF23.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{devlin-etal-2019-bert,
    title = "{BERT}: Pre-training of Deep Bidirectional Transformers for Language Understanding",
    author = "Devlin, Jacob  and
      Chang, Ming-Wei  and
      Lee, Kenton  and
      Toutanova, Kristina",
    booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",
    month = jun,
    year = "2019",
    address = "Minneapolis, Minnesota",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/N19-1423",
    doi = "10.18653/v1/N19-1423",
    pages = "4171--4186",
    abstract = "We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models (Peters et al., 2018a; Radford et al., 2018), BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications. BERT is conceptually simple and empirically powerful. It obtains new state-of-the-art results on eleven natural language processing tasks, including pushing the GLUE score to 80.5 (7.7 point absolute improvement), MultiNLI accuracy to 86.7{\%} (4.6{\%} absolute improvement), SQuAD v1.1 question answering Test F1 to 93.2 (1.5 point absolute improvement) and SQuAD v2.0 Test F1 to 83.1 (5.1 point absolute improvement).",
}

@inproceedings{DBLP:conf/nips/BrownMRSKDNSSAA20,
  author       = {Tom B. Brown and
                  Benjamin Mann and
                  Nick Ryder and
                  Melanie Subbiah and
                  Jared Kaplan and
                  Prafulla Dhariwal and
                  Arvind Neelakantan and
                  Pranav Shyam and
                  Girish Sastry and
                  Amanda Askell and
                  Sandhini Agarwal and
                  Ariel Herbert{-}Voss and
                  Gretchen Krueger and
                  Tom Henighan and
                  Rewon Child and
                  Aditya Ramesh and
                  Daniel M. Ziegler and
                  Jeffrey Wu and
                  Clemens Winter and
                  Christopher Hesse and
                  Mark Chen and
                  Eric Sigler and
                  Mateusz Litwin and
                  Scott Gray and
                  Benjamin Chess and
                  Jack Clark and
                  Christopher Berner and
                  Sam McCandlish and
                  Alec Radford and
                  Ilya Sutskever and
                  Dario Amodei},
  editor       = {Hugo Larochelle and
                  Marc'Aurelio Ranzato and
                  Raia Hadsell and
                  Maria{-}Florina Balcan and
                  Hsuan{-}Tien Lin},
  title        = {{Language Models are Few-Shot Learners}},
  booktitle    = {Advances in Neural Information Processing Systems 33: Annual Conference
                  on Neural Information Processing Systems 2020, NeurIPS 2020, December
                  6-12, 2020, virtual},
  year         = {2020},
  url          = {https://proceedings.neurips.cc/paper/2020/hash/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html},
  timestamp    = {Thu, 25 May 2023 10:38:31 +0200},
  biburl       = {https://dblp.org/rec/conf/nips/BrownMRSKDNSSAA20.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{DBLP:journals/csur/LiuYFJHN23,
  author       = {Pengfei Liu and
                  Weizhe Yuan and
                  Jinlan Fu and
                  Zhengbao Jiang and
                  Hiroaki Hayashi and
                  Graham Neubig},
  title        = {{Pre-train, Prompt, and Predict: {A} Systematic Survey of Prompting
                  Methods in Natural Language Processing}},
  journal      = {{ACM} Comput. Surv.},
  volume       = {55},
  number       = {9},
  pages        = {195:1--195:35},
  year         = {2023},
  url          = {https://doi.org/10.1145/3560815},
  doi          = {10.1145/3560815},
  timestamp    = {Mon, 28 Aug 2023 21:19:32 +0200},
  biburl       = {https://dblp.org/rec/journals/csur/LiuYFJHN23.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{jahan-etal-2023-evaluation,
    title = "Evaluation of {C}hat{GPT} on Biomedical Tasks: A Zero-Shot Comparison with Fine-Tuned Generative Transformers",
    author = "Jahan, Israt  and
      Laskar, Md Tahmid Rahman  and
      Peng, Chun  and
      Huang, Jimmy",
    booktitle = "The 22nd Workshop on Biomedical Natural Language Processing and BioNLP Shared Tasks",
    month = jul,
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.bionlp-1.30",
    doi = "10.18653/v1/2023.bionlp-1.30",
    pages = "326--336",
    abstract = "ChatGPT is a large language model developed by OpenAI. Despite its impressive performance across various tasks, no prior work has investigated its capability in the biomedical domain yet. To this end, this paper aims to evaluate the performance of ChatGPT on various benchmark biomedical tasks, such as relation extraction, document classification, question answering, and summarization. To the best of our knowledge, this is the first work that conducts an extensive evaluation of ChatGPT in the biomedical domain. Interestingly, we find based on our evaluation that in biomedical datasets that have smaller training sets, zero-shot ChatGPT even outperforms the state-of-the-art fine-tuned generative transformer models, such as BioGPT and BioBART. This suggests that ChatGPT{'}s pre-training on large text corpora makes it quite specialized even in the biomedical domain. Our findings demonstrate that ChatGPT has the potential to be a valuable tool for various tasks in the biomedical domain that lack large annotated data.",
}

@inproceedings{aiyappa-etal-2023-trust,
    title = "Can we trust the evaluation on {C}hat{GPT}?",
    author = "Aiyappa, Rachith  and
      An, Jisun  and
      Kwak, Haewoon  and
      Ahn, Yong-yeol",
    booktitle = "Proceedings of the 3rd Workshop on Trustworthy Natural Language Processing (TrustNLP 2023)",
    month = jul,
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.trustnlp-1.5",
    doi = "10.18653/v1/2023.trustnlp-1.5",
    pages = "47--54",
    abstract = "ChatGPT, the first large language model with mass adoption, has demonstrated remarkableperformance in numerous natural language tasks. Despite its evident usefulness, evaluatingChatGPT{'}s performance in diverse problem domains remains challenging due to the closednature of the model and its continuous updates via Reinforcement Learning from HumanFeedback (RLHF). We highlight the issue of data contamination in ChatGPT evaluations, with a case study in stance detection. We discuss the challenge of preventing data contamination and ensuring fair model evaluation in the age of closed and continuously trained models.",
}

@article{Bevilacqua2021,
title={{One SPRING to Rule Them Both: Symmetric AMR Semantic Parsing and Generation without a Complex Pipeline}},
volume={35},
url={https://ojs.aaai.org/index.php/AAAI/article/view/17489},
abstractNote={In Text-to-AMR parsing, current state-of-the-art semantic parsers use cumbersome pipelines integrating several different modules or components, and exploit graph recategorization, i.e., a set of content-specific heuristics that are developed on the basis of the training set. However, the generalizability of graph recategorization in an out-of-distribution setting is unclear. In contrast, state-of-the-art AMR-to-Text generation, which can be seen as the inverse to parsing, is based on simpler seq2seq. In this paper, we cast Text-to-AMR and AMR-to-Text as a symmetric transduction task and show that by devising a careful graph linearization and extending a pretrained encoder-decoder model, it is possible to obtain state-of-the-art performances in both tasks using the very same seq2seq approach, i.e., SPRING (Symmetric PaRsIng aNd Generation). Our model does not require complex pipelines, nor heuristics built on heavy assumptions. In fact, we drop the need for graph recategorization, showing that this technique is actually harmful outside of the standard benchmark. Finally, we outperform the previous state of the art on the English AMR 2.0 dataset by a large margin: on Text-to-AMR we obtain an improvement of 3.6 Smatch points, while on AMR-to-Text we outperform the state of the art by 11.2 BLEU points. We release the software at github.com/SapienzaNLP/spring.},
number={14},
journal={Proceedings of the AAAI Conference on Artificial Intelligence}, author={Bevilacqua, Michele and Blloshmi, Rexhina and Navigli, Roberto}, year={2021},
month={May},
pages={12564--12573}
}

@inproceedings{wu-etal-2020-scalable,
    title = {{Scalable Zero-shot Entity Linking with Dense Entity Retrieval}},
    author = {Wu, Ledell  and
      Petroni, Fabio  and
      Josifoski, Martin  and
      Riedel, Sebastian  and
      Zettlemoyer, Luke},
    editor = {Webber, Bonnie  and
      Cohn, Trevor  and
      He, Yulan  and
      Liu, Yang},
    booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    month = {November},
    year = {2020},
    address = {Online},
    publisher = {Association for Computational Linguistics},
    url = {https://aclanthology.org/2020.emnlp-main.519},
    doi = {10.18653/v1/2020.emnlp-main.519},
    pages = {6397--6407},
    abstract = {This paper introduces a conceptually simple, scalable, and highly effective BERT-based entity linking model, along with an extensive evaluation of its accuracy-speed trade-off. We present a two-stage zero-shot linking algorithm, where each entity is defined only by a short textual description. The first stage does retrieval in a dense space defined by a bi-encoder that independently embeds the mention context and the entity descriptions. Each candidate is then re-ranked with a cross-encoder, that concatenates the mention and entity text. Experiments demonstrate that this approach is state of the art on recent zero-shot benchmarks (6 point absolute gains) and also on more established non-zero-shot evaluations (e.g. TACKBP-2010), despite its relative simplicity (e.g. no explicit entity embeddings or manually engineered mention tables). We also show that bi-encoder linking is very fast with nearest neighbor search (e.g. linking with 5.9 million candidates in 2 milliseconds), and that much of the accuracy gain from the more expensive cross-encoder can be transferred to the bi-encoder via knowledge distillation. Our code and models are available at https://github.com/facebookresearch/BLINK.},
}

@inproceedings{Banarescu2013,
    title = {{A}bstract {M}eaning {R}epresentation for Sembanking"},
    author = {Laura Banarescu  and
      Claire Bonial  and
      Shu Cai  and
      Madalina Georgescu  and
      Kira Griffitt  and
      Ulf Hermjakob  and
      Kevin Knight and
      Philipp Koehn  and
      Martha Palmer  and
      Nathan Schneider},
    booktitle = {{Proceedings of the 7th Linguistic Annotation Workshop and Interoperability with Discourse}},
    month = {August},
    year = {2013},
    address = {Sofia, Bulgaria},
    publisher = {Association for Computational Linguistics},
    url = {https://aclanthology.org/W13-2322},
    pages = {178--186},
}

@inproceedings{cheng-etal-2023-marked,
    title = "Marked Personas: Using Natural Language Prompts to Measure Stereotypes in Language Models",
    author = "Cheng, Myra  and
      Durmus, Esin  and
      Jurafsky, Dan",
    booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = {July},
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.acl-long.84",
    doi = "10.18653/v1/2023.acl-long.84",
    pages = "1504--1532",
    abstract = "To recognize and mitigate harms from large language models (LLMs), we need to understand the prevalence and nuances of stereotypes in LLM outputs. Toward this end, we present Marked Personas, a prompt-based method to measure stereotypes in LLMs for intersectional demographic groups without any lexicon or data labeling.Grounded in the sociolinguistic concept of markedness (which characterizes explicitly linguistically marked categories versus unmarked defaults), our proposed method is twofold: 1) prompting an LLM to generate personas, i.e., natural language descriptions, of the target demographic group alongside personas of unmarked, default groups; 2) identifying the words that significantly distinguish personas of the target group from corresponding unmarked ones.We find that the portrayals generated by GPT-3.5 and GPT-4 contain higher rates of racial stereotypes than human-written portrayals using the same prompts. The words distinguishing personas of marked (non-white, non-male) groups reflect patterns of othering and exoticizing these demographics. An intersectional lens further reveals tropes that dominate portrayals of marginalized groups, such as tropicalism and the hypersexualization of minoritized women. These representational harms have concerning implications for downstream applications like story generation.",
}

@inproceedings{stranisci-etal-2023-wikibio,
    title = "{W}iki{B}io: a Semantic Resource for the Intersectional Analysis of Biographical Events",
    author = "Stranisci, Marco Antonio  and
      Damiano, Rossana  and
      Mensa, Enrico  and
      Patti, Viviana  and
      Radicioni, Daniele  and
      Caselli, Tommaso",
    booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = "July",
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.acl-long.691",
    doi = "10.18653/v1/2023.acl-long.691",
    pages = "12370--12384",
    abstract = "Biographical event detection is a relevant task that allows for the exploration and comparison of the ways in which people{'}s lives are told and represented. This may support several real-life applications in digital humanities and in works aimed at exploring bias about minoritized groups. Despite that, there are no corpora and models specifically designed for this task. In this paper we fill this gap by presenting a new corpus annotated for biographical event detection. The corpus, which includes 20 Wikipedia biographies, was aligned with 5 existing corpora in order to train a model for the biographical event detection task. The model was able to detect all mentions of the target-entity in a biography with an F-score of 0.808 and the entity-related events with an F-score of 0.859. Finally, the model was used for performing an analysis of biases about women and non-Western people in Wikipedia biographies.",
}

@InProceedings{pmlr-v202-kandpal23a,
  title = 	 {{Large Language Models Struggle to Learn Long-Tail Knowledge}},
  author =       {Kandpal, Nikhil and Deng, Haikang and Roberts, Adam and Wallace, Eric and Raffel, Colin},
  booktitle = 	 {Proceedings of the 40th International Conference on Machine Learning},
  pages = 	 {15696--15707},
  year = 	 {2023},
  editor = 	 {Krause, Andreas and Brunskill, Emma and Cho, Kyunghyun and Engelhardt, Barbara and Sabato, Sivan and Scarlett, Jonathan},
  volume = 	 {202},
  series = 	 {Proceedings of Machine Learning Research},
  month = 	 {July},
  publisher =    {PMLR},
  pdf = 	 {https://proceedings.mlr.press/v202/kandpal23a/kandpal23a.pdf},
  url = 	 {https://proceedings.mlr.press/v202/kandpal23a.html},
  abstract = 	 {The Internet contains a wealth of knowledge—from the birthdays of historical figures to tutorials on how to code—all of which may be learned by language models. However, while certain pieces of information are ubiquitous on the web, others appear extremely rarely. In this paper, we study the relationship between the knowledge memorized by large language models and the information in pre-training datasets scraped from the web. In particular, we show that a language model’s ability to answer a fact-based question relates to how many documents associated with that question were seen during pre-training. We identify these relevant documents by entity linking pre-training datasets and counting documents that contain the same entities as a given question-answer pair. Our results demonstrate strong correlational and causal relationships between accuracy and relevant document count for numerous question answering datasets (e.g., TriviaQA), pre-training corpora (e.g., ROOTS), and model sizes (e.g., 176B parameters). Moreover, while larger models are better at learning long-tail knowledge, we estimate that today’s models must be scaled by many orders of magnitude to reach competitive QA performance on questions with little support in the pre-training data. Finally, we show that retrieval-augmentation can reduce the dependence on relevant pre-training information, presenting a promising approach for capturing the long-tail.}
}

@inproceedings{chen-etal-2021-evaluating,
    title = "Evaluating Entity Disambiguation and the Role of Popularity in Retrieval-Based {NLP}",
    author = "Chen, Anthony  and
      Gudipati, Pallavi  and
      Longpre, Shayne  and
      Ling, Xiao  and
      Singh, Sameer",
    booktitle = "Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)",
    month =  {August},
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.acl-long.345",
    doi = "10.18653/v1/2021.acl-long.345",
    pages = "4472--4485",
    abstract = "Retrieval is a core component for open-domain NLP tasks. In open-domain tasks, multiple entities can share a name, making disambiguation an inherent yet under-explored problem. We propose an evaluation benchmark for assessing the entity disambiguation capabilities of these retrievers, which we call Ambiguous Entity Retrieval (AmbER) sets. We define an AmbER set as a collection of entities that share a name along with queries about those entities. By covering the set of entities for polysemous names, AmbER sets act as a challenging test of entity disambiguation. We create AmbER sets for three popular open-domain tasks: fact checking, slot filling, and question answering, and evaluate a diverse set of retrievers. We find that the retrievers exhibit popularity bias, significantly under-performing on rarer entities that share a name, e.g., they are twice as likely to retrieve erroneous documents on queries for the less popular entity under the same name. These experiments on AmbER sets show their utility as an evaluation tool and highlight the weaknesses of popular retrieval systems.",
}

@inproceedings{zhou-etal-2021-structure,
    title = "Structure-aware Fine-tuning of Sequence-to-sequence Transformers for Transition-based {AMR} Parsing",
    author = "Zhou, Jiawei  and
      Naseem, Tahira  and
      Fernandez Astudillo, Ram{\'o}n  and
      Lee, Young-Suk  and
      Florian, Radu  and
      Roukos, Salim",
    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
    month = nov,
    year = "2021",
    address = "Online and Punta Cana, Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.emnlp-main.507",
    doi = "10.18653/v1/2021.emnlp-main.507",
    pages = "6279--6290",
    abstract = "Predicting linearized Abstract Meaning Representation (AMR) graphs using pre-trained sequence-to-sequence Transformer models has recently led to large improvements on AMR parsing benchmarks. These parsers are simple and avoid explicit modeling of structure but lack desirable properties such as graph well-formedness guarantees or built-in graph-sentence alignments. In this work we explore the integration of general pre-trained sequence-to-sequence language models and a structure-aware transition-based approach. We depart from a pointer-based transition system and propose a simplified transition set, designed to better exploit pre-trained language models for structured fine-tuning. We also explore modeling the parser state within the pre-trained encoder-decoder architecture and different vocabulary strategies for the same purpose. We provide a detailed comparison with recent progress in AMR parsing and show that the proposed parser retains the desirable properties of previous transition-based approaches, while being simpler and reaching the new parsing state of the art for AMR 2.0, without the need for graph re-categorization.",
}


@inproceedings{Rao2017,
    title = {{Biomedical Event Extraction using {A}bstract {M}eaning {R}epresentation}},
    author = {Rao, Sudha  and
      Marcu, Daniel  and
      Knight, Kevin  and
      Daum{\'e} III, Hal},
    editor = {Cohen, Kevin Bretonnel  and
      Demner-Fushman, Dina  and
      Ananiadou, Sophia  and
      Tsujii, Junichi},
    booktitle = {{B}io{NLP} 2017},
    month = {August},
    year = {2017},
    address = {Vancouver, Canada},
    publisher = {Association for Computational Linguistics},
    url = {https://aclanthology.org/W17-2315},
    doi = {10.18653/v1/W17-2315},
    pages = {126--135},
    abstract = {We propose a novel, Abstract Meaning Representation (AMR) based approach to identifying molecular events/interactions in biomedical text. Our key contributions are: (1) an empirical validation of our hypothesis that an event is a subgraph of the AMR graph, (2) a neural network-based model that identifies such an event subgraph given an AMR, and (3) a distant supervision based approach to gather additional training data. We evaluate our approach on the 2013 Genia Event Extraction dataset and show promising results.},
}


@inproceedings{Zhang2021Biomedical,
    title = {{Fine-grained Information Extraction from Biomedical Literature based on Knowledge-enriched {A}bstract {M}eaning {R}epresentation}},
    author = {Zhang, Zixuan  and
      Parulian, Nikolaus  and
      Ji, Heng  and
      Elsayed, Ahmed  and
      Myers, Skatje  and
      Palmer, Martha},
    booktitle = {{Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)}},
    month = aug,
    year = {2021},
    address = {Online},
    publisher = {Association for Computational Linguistics},
    url = {https://aclanthology.org/2021.acl-long.489},
    doi = {10.18653/v1/2021.acl-long.489},
    pages = {6261--6270},
    abstract = {Biomedical Information Extraction from scientific literature presents two unique and non-trivial challenges. First, compared with general natural language texts, sentences from scientific papers usually possess wider contexts between knowledge elements. Moreover, comprehending the fine-grained scientific entities and events urgently requires domain-specific background knowledge. In this paper, we propose a novel biomedical Information Extraction (IE) model to tackle these two challenges and extract scientific entities and events from English research papers. We perform Abstract Meaning Representation (AMR) to compress the wide context to uncover a clear semantic structure for each complex sentence. Besides, we construct the sentence-level knowledge graph from an external knowledge base and use it to enrich the AMR graph to improve the model{'}s understanding of complex scientific concepts. We use an edge-conditioned graph attention network to encode the knowledge-enriched AMR graph for biomedical IE tasks. Experiments on the GENIA 2011 dataset show that the AMR and external knowledge have contributed 1.8{\%} and 3.0{\%} absolute F-score gains respectively. In order to evaluate the impact of our approach on real-world problems that involve topic-specific fine-grained knowledge elements, we have also created a new ontology and annotated corpus for entity and event extraction for the COVID-19 scientific literature, which can serve as a new benchmark for the biomedical IE community.},
}


@article{Velickovic2018GraphAN,
  title={{Graph Attention Networks}},
  author={Petar Velickovic and Guillem Cucurull and Arantxa Casanova and Adriana Romero and Pietro Lio’ and Yoshua Bengio},
  journal={ArXiv},
  year={2018},
  volume={abs/1710.10903}
}


@inproceedings{Kapanipathi2021,
    title = {{Leveraging {A}bstract {M}eaning {R}epresentation for Knowledge Base Question Answering}},
    author = {Kapanipathi, Pavan  and
      Abdelaziz, Ibrahim  and
      Ravishankar, Srinivas  and
      Roukos, Salim  and
      Gray, Alexander  and
      Fernandez Astudillo, Ram{\'o}n  and
      Chang, Maria  and
      Cornelio, Cristina  and
      Dana, Saswati  and
      Fokoue, Achille  and
      Garg, Dinesh  and
      Gliozzo, Alfio  and
      Gurajada, Sairam  and
      Karanam, Hima  and
      Khan, Naweed  and
      Khandelwal, Dinesh  and
      Lee, Young-Suk  and
      Li, Yunyao  and
      Luus, Francois  and
      Makondo, Ndivhuwo  and
      Mihindukulasooriya, Nandana  and
      Naseem, Tahira  and
      Neelam, Sumit  and
      Popa, Lucian  and
      Gangi Reddy, Revanth  and
      Riegel, Ryan  and
      Rossiello, Gaetano  and
      Sharma, Udit  and
      Bhargav, G P Shrivatsa  and
      Yu, Mo},
    editor = {Zong, Chengqing  and
      Xia, Fei  and
      Li, Wenjie  and
      Navigli, Roberto},
    booktitle = {{Findings of the Association for Computational Linguistics: Association for Computational Linguistics-IJCNLP 2021}},
    month = {August},
    year = {2021},
    address = {Online},
    publisher = {Association for Computational Linguistics},
    url = {https://aclanthology.org/2021.findings-acl.339},
    doi = {10.18653/v1/2021.findings-acl.339},
    pages = {3884--3894},
}

@inproceedings{blloshmi-etal-2020-xl,
    title = {{XL-AMR: Enabling Cross-Lingual AMR Parsing with Transfer Learning Techniques}},
    author = {Blloshmi, Rexhina  and
      Tripodi, Rocco  and
      Navigli, Roberto},
    booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
    month = {November},
    year = {2020},
    address = {Online},
    publisher = {Association for Computational Linguistics},
    url = {https://aclanthology.org/2020.emnlp-main.195},
    doi = {10.18653/v1/2020.emnlp-main.195},
    pages = {2487--2500},
    abstract = {Abstract Meaning Representation (AMR) is a popular formalism of natural language that represents the meaning of a sentence as a semantic graph. It is agnostic about how to derive meanings from strings and for this reason it lends itself well to the encoding of semantics across languages. However, cross-lingual AMR parsing is a hard task, because training data are scarce in languages other than English and the existing English AMR parsers are not directly suited to being used in a cross-lingual setting. In this work we tackle these two problems so as to enable cross-lingual AMR parsing: we explore different transfer learning techniques for producing automatic AMR annotations across languages and develop a cross-lingual AMR parser, XL-AMR. This can be trained on the produced data and does not rely on AMR aligners or source-copy mechanisms as is commonly the case in English AMR parsing. The results of XL-AMR significantly surpass those previously reported in Chinese, German, Italian and Spanish. Finally we provide a qualitative analysis which sheds light on the suitability of AMR across languages. We release XL-AMR at github.com/SapienzaNLP/xl-amr.},
}



@inproceedings{Tripodi2021,
    title = {{SGL: Speaking the Graph Languages of Semantic Parsing via Multilingual Translation}},
    author = {Procopio, Luigi  and
      Tripodi, Rocco  and
      Navigli, Roberto},
   editor = {Toutanova, Kristina  and
      Rumshisky, Anna  and
      Zettlemoyer, Luke  and
      Hakkani-Tur, Dilek  and
      Beltagy, Iz  and
      Bethard, Steven  and
      Cotterell, Ryan  and
      Chakraborty, Tanmoy  and
      Zhou, Yichao},  
    booktitle = {Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
    month = {June},
    year = {2021},
    address = {Online},
    publisher = {Association for Computational Linguistics},
    url = {https://aclanthology.org/2021.naacl-main.30},
    doi = {10.18653/v1/2021.naacl-main.30},
    pages = {325--337}
}


@inproceedings{Zhang2021,
    title = {{A}bstract {M}eaning {R}epresentation Guided Graph Encoding and Decoding for Joint Information Extraction},
    author = {Zhang, Zixuan  and
      Ji, Heng},
    booktitle = {{Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies}},
    month = jun,
    year = {2021},
    address = {Online},
    publisher = {Association for Computational Linguistics},
    url = {https://aclanthology.org/2021.naacl-main.4},
    doi = {10.18653/v1/2021.naacl-main.4},
    pages = {39--49},
    abstract = {The tasks of Rich Semantic Parsing, such as Abstract Meaning Representation (AMR), share similar goals with Information Extraction (IE) to convert natural language texts into structured semantic representations. To take advantage of such similarity, we propose a novel AMR-guided framework for joint information extraction to discover entities, relations, and events with the help of a pre-trained AMR parser. Our framework consists of two novel components: 1) an AMR based semantic graph aggregator to let the candidate entity and event trigger nodes collect neighborhood information from AMR graph for passing message among related knowledge elements; 2) an AMR guided graph decoder to extract knowledge elements based on the order decided by the hierarchical structures in AMR. Experiments on multiple datasets have shown that the AMR graph encoder and decoder have provided significant gains and our approach has achieved new state-of-the-art performance on all IE subtasks.},
}


@inproceedings{Jungwoo2020,
    title = {{I Know What You Asked: Graph Path Learning using AMR for Commonsense Reasoning}},
    author = {Lim, Jungwoo  and
      Oh, Dongsuk  and
      Jang, Yoonna  and
      Yang, Kisu  and
      Lim, Heuiseok},
    editor = {Scott, Donia  and
      Bel, Nuria  and
      Zong, Chengqing},
    booktitle = {{Proceedings of the 28th International Conference on Computational Linguistics}},
    month = {December},
    year = {2020},
    address = {Barcelona, Spain (Online)},
    publisher = {International Committee on Computational Linguistics},
    url = {https://aclanthology.org/2020.coling-main.222},
    doi = {10.18653/v1/2020.coling-main.222},
    pages = {2459--2471},
    abstract = {CommonsenseQA is a task in which a correct answer is predicted through commonsense reasoning with pre-defined knowledge. Most previous works have aimed to improve the performance with distributed representation without considering the process of predicting the answer from the semantic representation of the question. To shed light upon the semantic interpretation of the question, we propose an AMR-ConceptNet-Pruned (ACP) graph. The ACP graph is pruned from a full integrated graph encompassing Abstract Meaning Representation (AMR) graph generated from input questions and an external commonsense knowledge graph, ConceptNet (CN). Then the ACP graph is exploited to interpret the reasoning path as well as to predict the correct answer on the CommonsenseQA task. This paper presents the manner in which the commonsense reasoning process can be interpreted with the relations and concepts provided by the ACP graph. Moreover, ACP-based models are shown to outperform the baselines.}
}


@article{bornea2021learning,
  author    = {Mihaela Bornea and
               Ram{\'{o}}n Fernandez Astudillo and
               Tahira Naseem and
               Nandana Mihindukulasooriya and
               Ibrahim Abdelaziz and
               Pavan Kapanipathi and
               Radu Florian and
               Salim Roukos},
  title     = {Learning to Transpile {AMR} into {SPARQL}},
  journal   = {Computing Research Repository (CoRR)},
  volume    = {abs/2112.07877},
  year      = {2021},
  url       = {https://arxiv.org/abs/2112.07877},
  eprinttype = {arXiv},
  eprint    = {2112.07877},
  timestamp = {Mon, 03 Jan 2022 15:45:35 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-2112-07877.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@book{Penman1991,
   title =     {{Text Generation and Systemic-Functional Linguistics: Experiences from English and Japanese}},
   author =    {Christian M. I. M. Matthiessen and John A. Bateman},
   publisher = {Pinter Publishers},
   year =      {1991}
    }
}

@inproceedings{ijcai2022p568,
  title     = {{Interpretable AMR-Based Question Decomposition for Multi-hop Question Answering}},
  author    = {Deng, Zhenyun and Zhu, Yonghua and Chen, Yang and Witbrock, Michael and Riddle, Patricia},
  booktitle = {Proceedings of the Thirty-First International Joint Conference on
               Artificial Intelligence, {IJCAI-22}},
  publisher = {International Joint Conferences on Artificial Intelligence Organization},
  editor    = {Lud De Raedt},
  pages     = {4093--4099},
  year      = {2022},
  month     = {7},
  note      = {Main Track},
  doi       = {10.24963/ijcai.2022/568},
  url       = {https://doi.org/10.24963/ijcai.2022/568},
}

@inproceedings{Gangemi2023,
 address = {Athens, Greece},
 author = {Gangemi, Aldo  and
Graciotti, Arianna  and
Meloni, Antonello  and
Nuzzolese, Andrea  and
Presutti, Valentina and Reforgiato Recupero, Diego and Russo, Alessandro and Tripodi, Rocco},
 booktitle = {22nd International Semantic Web Conference},
 month = {November},
 publisher = {CEUR Workshop Proceedings},
 title = {{Text2AMR2FRED, a tool for transforming text into RDF/OWL Knowledge Graphs via Abstract Meaning Representation}},
 year = {2023}
}

@inproceedings{bevilacqua-navigli-2020-breaking,
    title = "Breaking Through the 80{\%} Glass Ceiling: {R}aising the State of the Art in Word Sense Disambiguation by Incorporating Knowledge Graph Information",
    author = "Bevilacqua, Michele  and
      Navigli, Roberto",
    editor = "Jurafsky, Dan  and
      Chai, Joyce  and
      Schluter, Natalie  and
      Tetreault, Joel",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.acl-main.255",
    doi = "10.18653/v1/2020.acl-main.255",
    pages = "2854--2864",
    abstract = "Neural architectures are the current state of the art in Word Sense Disambiguation (WSD). However, they make limited use of the vast amount of relational information encoded in Lexical Knowledge Bases (LKB). We present Enhanced WSD Integrating Synset Embeddings and Relations (EWISER), a neural supervised architecture that is able to tap into this wealth of knowledge by embedding information from the LKB graph within the neural architecture, and to exploit pretrained synset embeddings, enabling the network to predict synsets that are not in the training set. As a result, we set a new state of the art on almost all the evaluation settings considered, also breaking through, for the first time, the 80{\%} ceiling on the concatenation of all the standard all-words English WSD evaluation benchmarks. On multilingual all-words WSD, we report state-of-the-art results by training on nothing but English.",
}

@inproceedings{Meloni2017,
 address = {Cham},
 author = {Meloni, Antonello and others},
 booktitle = {{The Semantic Web: ESWC 2017 Satellite Events}},
 doi = {10.1007/978-3-319-70407-4\_9},
 editor = {Blomqvist, Eva
and others},
 isbn = {978-3-319-70407-4},
 pages = {43--47},
 publisher = {Springer International Publishing},
 title = {{AMR2FRED, A Tool for Translating Abstract Meaning Representation to Motif-Based Linguistic Knowledge Graphs}},
 year = {2017}
}

@article{Gangemi2017,
  author    = {Aldo Gangemi and
               Valentina Presutti and
               Diego Reforgiato Recupero and
               Andrea Giovanni Nuzzolese and
               Francesco Draicchio and
               Misael Mongiov{\`{\i}}},
  title     = {{Semantic Web Machine Reading with FRED}},
  journal   = {Semantic Web},
  volume    = {8},
  number    = {6},
  pages     = {873--893},
  year      = {2017},
  url       = {https://doi.org/10.3233/SW-160240},
  doi       = {10.3233/SW-160240},
  timestamp = {Fri, 27 Mar 2020 08:39:51 +0100},
  biburl    = {https://dblp.org/rec/journals/semweb/GangemiPRNDM17.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{Framester2016,
 abstract = {Semantic web applications leveraging NLP can benefit from easy access to expressive lexical resources such as FrameNet. However, the usefulness of FrameNet is affected by its limited coverage and non-standard semantics. The access to existing linguistic resources is also limited because of poor connectivity among them. We present some strategies based on Linguistic Linked Data to broaden FrameNet coverage and formal linkage of lexical and factual resources. We created a novel resource, Framester, which acts as a hub between FrameNet, WordNet, VerbNet, BabelNet, DBpedia, Yago, DOLCE-Zero, as well as other resources. Framester is not only a strongly connected knowledge graph, but also applies a rigorous formal treatment for Fillmore's frame semantics, enabling full-fledged OWL querying and reasoning on a large frame-based knowledge graph. We also describe Word Frame Disambiguation, an application that reuses Framester data as a base in order to perform frame detection from text, with results comparable in precision to the state of the art, but with a much higher coverage.},
 address = {Bologna, Italy},
 author = {Gangemi, Aldo
and Alam, Mehwish
and Asprino, Luigi
and Presutti, Valentina
and Recupero, Diego Reforgiato},
 booktitle = {Knowledge Engineering and Knowledge Management},
 editor = {Blomqvist, Eva
and Ciancarini, Paolo
and Poggi, Francesco
and Vitali, Fabio},
 isbn = {978-3-319-49004-5},
 pages = {239--254},
 publisher = {Springer International Publishing},
 title = {{Framester: A Wide Coverage Linguistic Linked Data Hub}},
 year = {2016}
}


@inproceedings{roberts-etal-2020-much,
    title = "How Much Knowledge Can You Pack Into the Parameters of a Language Model?",
    author = "Roberts, Adam  and
      Raffel, Colin  and
      Shazeer, Noam",
    editor = "Webber, Bonnie  and
      Cohn, Trevor  and
      He, Yulan  and
      Liu, Yang",
    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
    month = nov,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.emnlp-main.437",
    doi = "10.18653/v1/2020.emnlp-main.437",
    pages = "5418--5426",
    abstract = "It has recently been observed that neural language models trained on unstructured text can implicitly store and retrieve knowledge using natural language queries. In this short paper, we measure the practical utility of this approach by fine-tuning pre-trained models to answer questions without access to any external context or knowledge. We show that this approach scales with model size and performs competitively with open-domain systems that explicitly retrieve answers from an external knowledge source when answering questions. To facilitate reproducibility and future work, we release our code and trained models.",
}

@inproceedings{ettinger-etal-2023-expert,
    title = "{``}You Are An Expert Linguistic Annotator{''}: Limits of {LLM}s as Analyzers of {A}bstract {M}eaning {R}epresentation",
    author = "Ettinger, Allyson  and
      Hwang, Jena  and
      Pyatkin, Valentina  and
      Bhagavatula, Chandra  and
      Choi, Yejin",
    editor = "Bouamor, Houda  and
      Pino, Juan  and
      Bali, Kalika",
    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2023",
    month = dec,
    year = "2023",
    address = "Singapore",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.findings-emnlp.553",
    doi = "10.18653/v1/2023.findings-emnlp.553",
    pages = "8250--8263",
    abstract = "Large language models (LLMs) demonstrate an amazing proficiency and fluency in the $\textit{use}$ of language. Does that mean that they have also acquired insightful linguistic knowledge $\textit{about}$ the language, to an extent that they can serve as an {``}expert linguistic annotator{'}? In this paper, we examine the successes and limitations of the GPT-3, ChatGPT, and GPT-4 models, focusing on the Abstract Meaning Representation (AMR) parsing formalism (Banarescu et al., 2013), which provides rich graphical representations of sentence meaning structure while abstracting away from surface forms. We compare models{'} analysis of this semantic structure across two settings: 1) direct production of AMR parses based on zero- and few-shot examples, and 2) indirect partial reconstruction of AMR via metalinguistic natural language queries (e.g., {``}Identify the primary event of this sentence, and the predicate corresponding to that event.{''}). Across these settings, we find that models can reliably reproduce the basic format of AMR, as well as some core event, argument, and modifier structure$-$however, model outputs are prone to frequent and major errors, and holistic analysis of parse acceptability shows that even with few-shot demonstrations, models have virtually 0{\%} success in producing fully accurate parses. Eliciting responses in natural language produces similar patterns of errors. Overall, our findings indicate that these models out-of-the-box can accurately identify some core aspects of semantic structure, but there remain key limitations in their ability to support fully accurate semantic analyses or parses.",
}

@inproceedings{mallen-etal-2023-trust,
    title = "When Not to Trust Language Models: Investigating Effectiveness of Parametric and Non-Parametric Memories",
    author = "Mallen, Alex  and
      Asai, Akari  and
      Zhong, Victor  and
      Das, Rajarshi  and
      Khashabi, Daniel  and
      Hajishirzi, Hannaneh",
    editor = "Rogers, Anna  and
      Boyd-Graber, Jordan  and
      Okazaki, Naoaki",
    booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = jul,
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.acl-long.546",
    doi = "10.18653/v1/2023.acl-long.546",
    pages = "9802--9822",
    abstract = "Despite their impressive performance on diverse tasks, large language models (LMs) still struggle with tasks requiring rich world knowledge, implying the difficulty of encoding a wealth of world knowledge in their parameters. This paper aims to understand LMs{'} strengths and limitations in memorizing factual knowledge, by conducting large-scale knowledge probing experiments on two open-domain entity-centric QA datasets: PopQA, our new dataset with 14k questions about long-tail entities, and EntityQuestions, a widely used open-domain QA dataset. We find that LMs struggle with less popular factual knowledge, and that retrieval augmentation helps significantly in these cases. Scaling, on the other hand, mainly improves memorization of popular knowledge, and fails to appreciably improve memorization of factual knowledge in the tail. Based on those findings, we devise a new method for retrieval-augmentation that improves performance and reduces inference costs by only retrieving non-parametric memories when necessary.",
}