@inproceedings{aiyappa-etal-2023-trust,
 abstract = {ChatGPT, the first large language model with mass adoption, has demonstrated remarkableperformance in numerous natural language tasks. Despite its evident usefulness, evaluatingChatGPT{'}s performance in diverse problem domains remains challenging due to the closednature of the model and its continuous updates via Reinforcement Learning from HumanFeedback (RLHF). We highlight the issue of data contamination in ChatGPT evaluations, with a case study in stance detection. We discuss the challenge of preventing data contamination and ensuring fair model evaluation in the age of closed and continuously trained models.},
 address = {Toronto, Canada},
 author = {Aiyappa, Rachith  and
An, Jisun  and
Kwak, Haewoon  and
Ahn, Yong-yeol},
 booktitle = {Proceedings of the 3rd Workshop on Trustworthy Natural Language Processing (TrustNLP 2023)},
 doi = {10.18653/v1/2023.trustnlp-1.5},
 month = {July},
 pages = {47--54},
 publisher = {Association for Computational Linguistics},
 title = {Can we trust the evaluation on {C}hat{GPT}?},
 url = {https://aclanthology.org/2023.trustnlp-1.5},
 year = {2023}
}

@inproceedings{Banarescu2013,
 address = {Sofia, Bulgaria},
 author = {Laura Banarescu  and
Claire Bonial  and
Shu Cai  and
Madalina Georgescu  and
Kira Griffitt  and
Ulf Hermjakob  and
Kevin Knight and
Philipp Koehn  and
Martha Palmer  and
Nathan Schneider},
 booktitle = {{Proceedings of the 7th Linguistic Annotation Workshop and Interoperability with Discourse}},
 month = {August},
 pages = {178--186},
 publisher = {Association for Computational Linguistics},
 title = {{A}bstract {M}eaning {R}epresentation for Sembanking"},
 url = {https://aclanthology.org/W13-2322},
 year = {2013}
}

@inproceedings{banerjee-lavie-2005-meteor,
 address = {Ann Arbor, Michigan},
 author = {Banerjee, Satanjeev  and
Lavie, Alon},
 booktitle = {Proceedings of the {Association for Computational Linguistics} Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and/or Summarization},
 month = {June},
 pages = {65--72},
 publisher = {Association for Computational Linguistics},
 title = {{METEOR}: An Automatic Metric for {MT} Evaluation with Improved Correlation with Human Judgments},
 url = {https://aclanthology.org/W05-0909},
 year = {2005}
}

@article{Bevilacqua2021,
 abstractnote = {In Text-to-AMR parsing, current state-of-the-art semantic parsers use cumbersome pipelines integrating several different modules or components, and exploit graph recategorization, i.e., a set of content-specific heuristics that are developed on the basis of the training set. However, the generalizability of graph recategorization in an out-of-distribution setting is unclear. In contrast, state-of-the-art AMR-to-Text generation, which can be seen as the inverse to parsing, is based on simpler seq2seq. In this paper, we cast Text-to-AMR and AMR-to-Text as a symmetric transduction task and show that by devising a careful graph linearization and extending a pretrained encoder-decoder model, it is possible to obtain state-of-the-art performances in both tasks using the very same seq2seq approach, i.e., SPRING (Symmetric PaRsIng aNd Generation). Our model does not require complex pipelines, nor heuristics built on heavy assumptions. In fact, we drop the need for graph recategorization, showing that this technique is actually harmful outside of the standard benchmark. Finally, we outperform the previous state of the art on the English AMR 2.0 dataset by a large margin: on Text-to-AMR we obtain an improvement of 3.6 Smatch points, while on AMR-to-Text we outperform the state of the art by 11.2 BLEU points. We release the software at github.com/SapienzaNLP/spring.},
 author = {Bevilacqua, Michele and Blloshmi, Rexhina and Navigli, Roberto},
 journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
 month = {May},
 number = {14},
 pages = {12564--12573},
 title = {{One SPRING to Rule Them Both: Symmetric AMR Semantic Parsing and Generation without a Complex Pipeline}},
 url = {https://ojs.aaai.org/index.php/AAAI/article/view/17489},
 volume = {35},
 year = {2021}
}


@inproceedings{bevilacqua-navigli-2020-breaking,
    title = "Breaking Through the 80{\%} Glass Ceiling: {R}aising the State of the Art in Word Sense Disambiguation by Incorporating Knowledge Graph Information",
    author = "Bevilacqua, Michele  and
      Navigli, Roberto",
    editor = "Jurafsky, Dan  and
      Chai, Joyce  and
      Schluter, Natalie  and
      Tetreault, Joel",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.acl-main.255",
    doi = "10.18653/v1/2020.acl-main.255",
    pages = "2854--2864",
    abstract = "Neural architectures are the current state of the art in Word Sense Disambiguation (WSD). However, they make limited use of the vast amount of relational information encoded in Lexical Knowledge Bases (LKB). We present Enhanced WSD Integrating Synset Embeddings and Relations (EWISER), a neural supervised architecture that is able to tap into this wealth of knowledge by embedding information from the LKB graph within the neural architecture, and to exploit pretrained synset embeddings, enabling the network to predict synsets that are not in the training set. As a result, we set a new state of the art on almost all the evaluation settings considered, also breaking through, for the first time, the 80{\%} ceiling on the concatenation of all the standard all-words English WSD evaluation benchmarks. On multilingual all-words WSD, we report state-of-the-art results by training on nothing but English.",
}

@inproceedings{blloshmi-etal-2020-xl,
 abstract = {Abstract Meaning Representation (AMR) is a popular formalism of natural language that represents the meaning of a sentence as a semantic graph. It is agnostic about how to derive meanings from strings and for this reason it lends itself well to the encoding of semantics across languages. However, cross-lingual AMR parsing is a hard task, because training data are scarce in languages other than English and the existing English AMR parsers are not directly suited to being used in a cross-lingual setting. In this work we tackle these two problems so as to enable cross-lingual AMR parsing: we explore different transfer learning techniques for producing automatic AMR annotations across languages and develop a cross-lingual AMR parser, XL-AMR. This can be trained on the produced data and does not rely on AMR aligners or source-copy mechanisms as is commonly the case in English AMR parsing. The results of XL-AMR significantly surpass those previously reported in Chinese, German, Italian and Spanish. Finally we provide a qualitative analysis which sheds light on the suitability of AMR across languages. We release XL-AMR at github.com/SapienzaNLP/xl-amr.},
 address = {Online},
 author = {Blloshmi, Rexhina  and
Tripodi, Rocco  and
Navigli, Roberto},
 booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
 doi = {10.18653/v1/2020.emnlp-main.195},
 month = {November},
 pages = {2487--2500},
 publisher = {Association for Computational Linguistics},
 title = {{XL-AMR: Enabling Cross-Lingual AMR Parsing with Transfer Learning Techniques}},
 url = {https://aclanthology.org/2020.emnlp-main.195},
 year = {2020}
}

@misc{bornea2021learning,
    author    = {Mihaela Bornea and Ram{\'{o}}n Fernandez Astudillo and Tahira Naseem and Nandana Mihindukulasooriya and Ibrahim Abdelaziz and Pavan Kapanipathi and Radu Florian and Salim Roukos},
    title     = {Learning to Transpile {AMR} into {SPARQL}},
    year      = {2021},
    eprint    = {2112.07877},
    archivePrefix = {arXiv},
    url       = {https://arxiv.org/abs/2112.07877},
    note      = {Available at arXiv:2112.07877}
}


@book{DBLP:series/synthesis/2021Hogan,
  author       = {Aidan Hogan and
                  Eva Blomqvist and
                  Michael Cochez and
                  Claudia d'Amato and
                  Gerard de Melo and
                  Claudio Gutierrez and
                  Sabrina Kirrane and
                  Jos{\'{e}} Emilio Labra Gayo and
                  Roberto Navigli and
                  Sebastian Neumaier and
                  Axel{-}Cyrille Ngonga Ngomo and
                  Axel Polleres and
                  Sabbir M. Rashid and
                  Anisa Rula and
                  Lukas Schmelzeisen and
                  Juan Sequeda and
                  Steffen Staab and
                  Antoine Zimmermann},
  title        = {Knowledge Graphs},
  series       = {Synthesis Lectures on Data, Semantics, and Knowledge},
  publisher    = {Morgan {\&} Claypool Publishers},
  year         = {2021},
  url          = {https://doi.org/10.2200/S01125ED1V01Y202109DSK022},
  doi          = {10.2200/S01125ED1V01Y202109DSK022},
  timestamp    = {Mon, 26 Jun 2023 08:47:45 +0200},
  biburl       = {https://dblp.org/rec/series/synthesis/2021Hogan.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{cai-etal-2021-multilingual-amr,
 abstract = {We study multilingual AMR parsing from the perspective of knowledge distillation, where the aim is to learn and improve a multilingual AMR parser by using an existing English parser as its teacher. We constrain our exploration in a strict multilingual setting: there is but one model to parse all different languages including English. We identify that noisy input and precise output are the key to successful distillation. Together with extensive pre-training, we obtain an AMR parser whose performances surpass all previously published results on four different foreign languages, including German, Spanish, Italian, and Chinese, by large margins (up to 18.8 Smatch points on Chinese and on average 11.3 Smatch points). Our parser also achieves comparable performance on English to the latest state-of-the-art English-only parser.},
 address = {Punta Cana, Dominican Republic},
 author = {Cai, Deng  and
Li, Xin  and
Ho, Jackie Chun-Sing  and
Bing, Lidong  and
Lam, Wai},
 booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2021},
 doi = {10.18653/v1/2021.findings-emnlp.237},
 month = {November},
 pages = {2778--2789},
 publisher = {Association for Computational Linguistics},
 title = {Multilingual {AMR} Parsing with Noisy Knowledge Distillation},
 url = {https://aclanthology.org/2021.findings-emnlp.237},
 year = {2021}
}


@online{chatgpt,
  author = {OpenAI},
  title = {{Chatgpt: Optimizing language models for dialogue}},
  year = 2022,
  url = {https://archive.ph/4snnY}
}



@inproceedings{chen-etal-2019-evaluating,
 abstract = {As the complexity of question answering (QA) datasets evolve, moving away from restricted formats like span extraction and multiple-choice (MC) to free-form answer generation, it is imperative to understand how well current metrics perform in evaluating QA. This is especially important as existing metrics (BLEU, ROUGE, METEOR, and F1) are computed using n-gram similarity and have a number of well-known drawbacks. In this work, we study the suitability of existing metrics in QA. For generative QA, we show that while current metrics do well on existing datasets, converting multiple-choice datasets into free-response datasets is challenging for current metrics. We also look at span-based QA, where F1 is a reasonable metric. We show that F1 may not be suitable for all extractive QA tasks depending on the answer types. Our study suggests that while current metrics may be suitable for existing QA datasets, they limit the complexity of QA datasets that can be created. This is especially true in the context of free-form QA, where we would like our models to be able to generate more complex and abstractive answers, thus necessitating new metrics that go beyond n-gram based matching. As a step towards a better QA metric, we explore using BERTScore, a recently proposed metric for evaluating translation, for QA. We find that although it fails to provide stronger correlation with human judgements, future work focused on tailoring a BERT-based metric to QA evaluation may prove fruitful.},
 address = {Hong Kong, China},
 author = {Chen, Anthony  and
Stanovsky, Gabriel  and
Singh, Sameer  and
Gardner, Matt},
 booktitle = {Proceedings of the 2nd Workshop on Machine Reading for Question Answering},
 doi = {10.18653/v1/D19-5817},
 month = {November},
 pages = {119--124},
 publisher = {Association for Computational Linguistics},
 title = {Evaluating Question Answering Evaluation},
 url = {https://aclanthology.org/D19-5817},
 year = {2019}
}

@inproceedings{chen-etal-2021-evaluating,
 abstract = {Retrieval is a core component for open-domain NLP tasks. In open-domain tasks, multiple entities can share a name, making disambiguation an inherent yet under-explored problem. We propose an evaluation benchmark for assessing the entity disambiguation capabilities of these retrievers, which we call Ambiguous Entity Retrieval (AmbER) sets. We define an AmbER set as a collection of entities that share a name along with queries about those entities. By covering the set of entities for polysemous names, AmbER sets act as a challenging test of entity disambiguation. We create AmbER sets for three popular open-domain tasks: fact checking, slot filling, and question answering, and evaluate a diverse set of retrievers. We find that the retrievers exhibit popularity bias, significantly under-performing on rarer entities that share a name, e.g., they are twice as likely to retrieve erroneous documents on queries for the less popular entity under the same name. These experiments on AmbER sets show their utility as an evaluation tool and highlight the weaknesses of popular retrieval systems.},
 address = {Online},
 author = {Chen, Anthony  and
Gudipati, Pallavi  and
Longpre, Shayne  and
Ling, Xiao  and
Singh, Sameer},
 booktitle = {Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},
 doi = {10.18653/v1/2021.acl-long.345},
 month = {August},
 pages = {4472--4485},
 publisher = {Association for Computational Linguistics},
 title = {Evaluating Entity Disambiguation and the Role of Popularity in Retrieval-Based {NLP}},
 url = {https://aclanthology.org/2021.acl-long.345},
 year = {2021}
}

@inproceedings{cheng-etal-2023-marked,
 abstract = {To recognize and mitigate harms from large language models (LLMs), we need to understand the prevalence and nuances of stereotypes in LLM outputs. Toward this end, we present Marked Personas, a prompt-based method to measure stereotypes in LLMs for intersectional demographic groups without any lexicon or data labeling.Grounded in the sociolinguistic concept of markedness (which characterizes explicitly linguistically marked categories versus unmarked defaults), our proposed method is twofold: 1) prompting an LLM to generate personas, i.e., natural language descriptions, of the target demographic group alongside personas of unmarked, default groups; 2) identifying the words that significantly distinguish personas of the target group from corresponding unmarked ones.We find that the portrayals generated by GPT-3.5 and GPT-4 contain higher rates of racial stereotypes than human-written portrayals using the same prompts. The words distinguishing personas of marked (non-white, non-male) groups reflect patterns of othering and exoticizing these demographics. An intersectional lens further reveals tropes that dominate portrayals of marginalized groups, such as tropicalism and the hypersexualization of minoritized women. These representational harms have concerning implications for downstream applications like story generation.},
 address = {Toronto, Canada},
 author = {Cheng, Myra  and
Durmus, Esin  and
Jurafsky, Dan},
 booktitle = {Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
 doi = {10.18653/v1/2023.acl-long.84},
 month = {July},
 pages = {1504--1532},
 publisher = {Association for Computational Linguistics},
 title = {Marked Personas: Using Natural Language Prompts to Measure Stereotypes in Language Models},
 url = {https://aclanthology.org/2023.acl-long.84},
 year = {2023}
}

@inproceedings{clark-manning-2016-improving,
 address = {Berlin, Germany},
 author = {Clark, Kevin  and
Manning, Christopher D.},
 booktitle = {Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
 doi = {10.18653/v1/P16-1061},
 month = {August},
 pages = {643--653},
 publisher = {Association for Computational Linguistics},
 title = {Improving Coreference Resolution by Learning Entity-Level Distributed Representations},
 url = {https://aclanthology.org/P16-1061},
 year = {2016}
}

@inproceedings{DBLP:conf/emnlp/DodgeSMAIGM021,
 author = {Jesse Dodge and
Maarten Sap and
Ana Marasovic and
William Agnew and
Gabriel Ilharco and
Dirk Groeneveld and
Margaret Mitchell and
Matt Gardner},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/emnlp/DodgeSMAIGM021.bib},
 booktitle = {Proceedings of the 2021 Conference on Empirical Methods in Natural
Language Processing, {EMNLP} 2021, Virtual Event / Punta Cana, Dominican
Republic, 7-11 November, 2021},
 doi = {10.18653/v1/2021.emnlp-main.98},
 editor = {Marie{-}Francine Moens and
Xuanjing Huang and
Lucia Specia and
Scott Wen{-}tau Yih},
 pages = {1286--1305},
 publisher = {Association for Computational Linguistics},
 timestamp = {Thu, 20 Jan 2022 10:02:08 +0100},
 title = {Documenting Large Webtext Corpora: {A} Case Study on the Colossal
Clean Crawled Corpus},
 url = {https://doi.org/10.18653/v1/2021.emnlp-main.98},
 year = {2021}
}

@inproceedings{DBLP:conf/fat/BenderGMS21,
 author = {Emily M. Bender and
Timnit Gebru and
Angelina McMillan{-}Major and
Shmargaret Shmitchell},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/fat/BenderGMS21.bib},
 booktitle = {FAccT '21: 2021 {ACM} Conference on Fairness, Accountability, and
Transparency, Virtual Event / Toronto, Canada, March 3-10, 2021},
 doi = {10.1145/3442188.3445922},
 editor = {Madeleine Clare Elish and
William Isaac and
Richard S. Zemel},
 pages = {610--623},
 publisher = {{Association for Computing Machinery}},
 timestamp = {Fri, 05 May 2023 15:54:55 +0200},
 title = {On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?},
 url = {https://doi.org/10.1145/3442188.3445922},
 year = {2021}
}

DBLP:series/synthesis/2021Hogan,
  author       = {Aidan Hogan and
                  Eva Blomqvist and
                  Michael Cochez and
                  Claudia d'Amato and
                  Gerard de Melo and
                  Claudio Gutierrez and
                  Sabrina Kirrane and
                  Jos{\'{e}} Emilio Labra Gayo and
                  Roberto Navigli and
                  Sebastian Neumaier and
                  Axel{-}Cyrille Ngonga Ngomo and
                  Axel Polleres and
                  Sabbir M. Rashid and
                  Anisa Rula and
                  Lukas Schmelzeisen and
                  Juan Sequeda and
                  Steffen Staab and
                  Antoine Zimmermann},
  title        = {Knowledge Graphs},
  series       = {Synthesis Lectures on Data, Semantics, and Knowledge},
  publisher    = {Morgan {\&} Claypool Publishers},
  year         = {2021},
  url          = {https://doi.org/10.2200/S01125ED1V01Y202109DSK022},
  doi          = {10.2200/S01125ED1V01Y202109DSK022},
  timestamp    = {Mon, 26 Jun 2023 08:47:45 +0200},
  biburl       = {https://dblp.org/rec/series/synthesis/2021Hogan.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{DBLP:conf/icml/GuuLTPC20,
 author = {Kelvin Guu and
Kenton Lee and
Zora Tung and
Panupong Pasupat and
Ming{-}Wei Chang},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/icml/GuuLTPC20.bib},
 booktitle = {Proceedings of the 37th International Conference on Machine Learning,
{ICML} 2020, 13-18 July 2020, Virtual Event},
 pages = {3929--3938},
 publisher = {{PMLR}},
 series = {Proceedings of Machine Learning Research},
 timestamp = {Tue, 15 Dec 2020 17:40:18 +0100},
 title = {Retrieval Augmented Language Model Pre-Training},
 url = {http://proceedings.mlr.press/v119/guu20a.html},
 volume = {119},
 year = {2020}
}

@inproceedings{DBLP:conf/nips/BrownMRSKDNSSAA20,
 author = {Tom B. Brown and
Benjamin Mann and
Nick Ryder and
Melanie Subbiah and
Jared Kaplan and
Prafulla Dhariwal and
Arvind Neelakantan and
Pranav Shyam and
Girish Sastry and
Amanda Askell and
Sandhini Agarwal and
Ariel Herbert{-}Voss and
Gretchen Krueger and
Tom Henighan and
Rewon Child and
Aditya Ramesh and
Daniel M. Ziegler and
Jeffrey Wu and
Clemens Winter and
Christopher Hesse and
Mark Chen and
Eric Sigler and
Mateusz Litwin and
Scott Gray and
Benjamin Chess and
Jack Clark and
Christopher Berner and
Sam McCandlish and
Alec Radford and
Ilya Sutskever and
Dario Amodei},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/BrownMRSKDNSSAA20.bib},
 booktitle = {Advances in Neural Information Processing Systems 33: Annual Conference
on Neural Information Processing Systems 2020, NeurIPS 2020, December
6-12, 2020, virtual},
 editor = {Hugo Larochelle and
Marc'Aurelio Ranzato and
Raia Hadsell and
Maria{-}Florina Balcan and
Hsuan{-}Tien Lin},
 timestamp = {Thu, 25 May 2023 10:38:31 +0200},
 title = {{Language Models are Few-Shot Learners}},
 url = {https://proceedings.neurips.cc/paper/2020/hash/1457c0d6bfcb4967418bfb8ac142f64a-Abstract.html},
 year = {2020}
}

@inproceedings{DBLP:conf/nips/LewisPPPKGKLYR020,
 author = {Patrick S. H. Lewis and
Ethan Perez and
Aleksandra Piktus and
Fabio Petroni and
Vladimir Karpukhin and
Naman Goyal and
Heinrich K{\"{u}}ttler and
Mike Lewis and
Wen{-}tau Yih and
Tim Rockt{\"{a}}schel and
Sebastian Riedel and
Douwe Kiela},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/LewisPPPKGKLYR020.bib},
 booktitle = {Advances in Neural Information Processing Systems 33: Annual Conference
on Neural Information Processing Systems 2020, NeurIPS 2020, December
6-12, 2020, virtual},
 editor = {Hugo Larochelle and
Marc'Aurelio Ranzato and
Raia Hadsell and
Maria{-}Florina Balcan and
Hsuan{-}Tien Lin},
 timestamp = {Tue, 19 Jan 2021 15:57:07 +0100},
 title = {Retrieval-Augmented Generation for Knowledge-Intensive {NLP} Tasks},
 url = {https://proceedings.neurips.cc/paper/2020/hash/6b493230205f780e1bc26945df7481e5-Abstract.html},
 year = {2020}
}

@inproceedings{DBLP:conf/nips/Ouyang0JAWMZASR22,
 author = {Long Ouyang and
Jeffrey Wu and
Xu Jiang and
Diogo Almeida and
Carroll L. Wainwright and
Pamela Mishkin and
Chong Zhang and
Sandhini Agarwal and
Katarina Slama and
Alex Ray and
John Schulman and
Jacob Hilton and
Fraser Kelton and
Luke Miller and
Maddie Simens and
Amanda Askell and
Peter Welinder and
Paul F. Christiano and
Jan Leike and
Ryan Lowe},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/Ouyang0JAWMZASR22.bib},
 booktitle = {NeurIPS},
 timestamp = {Thu, 11 May 2023 17:08:21 +0200},
 title = {{Training language models to follow instructions with human feedback}},
 url = {http://papers.nips.cc/paper\_files/paper/2022/hash/b1efde53be364a73914f58805a001731-Abstract-Conference.html},
 year = {2022}
}

@inproceedings{DBLP:conf/nips/VaswaniSPUJGKP17,
 author = {Ashish Vaswani and
Noam Shazeer and
Niki Parmar and
Jakob Uszkoreit and
Llion Jones and
Aidan N. Gomez and
Lukasz Kaiser and
Illia Polosukhin},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/nips/VaswaniSPUJGKP17.bib},
 booktitle = {Advances in Neural Information Processing Systems 30: Annual Conference
on Neural Information Processing Systems 2017, December 4-9, 2017,
Long Beach, CA, {USA}},
 editor = {Isabelle Guyon and
Ulrike von Luxburg and
Samy Bengio and
Hanna M. Wallach and
Rob Fergus and
S. V. N. Vishwanathan and
Roman Garnett},
 pages = {5998--6008},
 timestamp = {Thu, 21 Jan 2021 15:15:21 +0100},
 title = {Attention is All you Need},
 url = {https://proceedings.neurips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html},
 year = {2017}
}

@inproceedings{DBLP:conf/uss/CarliniTWJHLRBS21,
 author = {Nicholas Carlini and
Florian Tram{\`{e}}r and
Eric Wallace and
Matthew Jagielski and
Ariel Herbert{-}Voss and
Katherine Lee and
Adam Roberts and
Tom B. Brown and
Dawn Song and
{\'{U}}lfar Erlingsson and
Alina Oprea and
Colin Raffel},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/conf/uss/CarliniTWJHLRBS21.bib},
 booktitle = {30th {USENIX} Security Symposium, {USENIX} Security 2021, August 11-13,
2021},
 editor = {Michael Bailey and
Rachel Greenstadt},
 pages = {2633--2650},
 publisher = {{USENIX} Association},
 timestamp = {Sun, 02 Oct 2022 16:16:46 +0200},
 title = {{Extracting Training Data from Large Language Models}},
 url = {https://www.usenix.org/conference/usenixsecurity21/presentation/carlini-extracting},
 year = {2021}
}


@article{DBLP:journals/bib/LuoSXQZPL22,
    author = {Luo, Renqian and Sun, Liai and Xia, Yingce and Qin, Tao and Zhang, Sheng and Poon, Hoifung and Liu, Tie-Yan},
    title = "{BioGPT: generative pre-trained transformer for biomedical text generation and mining}",
    journal = {Briefings in Bioinformatics},
    volume = {23},
    number = {6},
    pages = {bbac409},
    year = {2022},
    month = {09},
    abstract = "{Pre-trained language models have attracted increasing attention in the biomedical domain, inspired by their great success in the general natural language domain. Among the two main branches of pre-trained language models in the general language domain, i.e. BERT (and its variants) and GPT (and its variants), the first one has been extensively studied in the biomedical domain, such as BioBERT and PubMedBERT. While they have achieved great success on a variety of discriminative downstream biomedical tasks, the lack of generation ability constrains their application scope. In this paper, we propose BioGPT, a domain-specific generative Transformer language model pre-trained on large-scale biomedical literature. We evaluate BioGPT on six biomedical natural language processing tasks and demonstrate that our model outperforms previous models on most tasks. Especially, we get 44.98\\%, 38.42\\% and 40.76\\% F1 score on BC5CDR, KD-DTI and DDI end-to-end relation extraction tasks, respectively, and 78.2\\% accuracy on PubMedQA, creating a new record. Our case study on text generation further demonstrates the advantage of BioGPT on biomedical literature to generate fluent descriptions for biomedical terms.}",
    issn = {1477-4054},
    doi = {10.1093/bib/bbac409},
    url = {https://doi.org/10.1093/bib/bbac409},
    eprint = {https://academic.oup.com/bib/article-pdf/23/6/bbac409/47144271/bbac409.pdf},
}


@misc{DBLP:journals/corr/abs-2212-14548,
    author = {Bowen Zhang and Daijun Ding and Liwen Jing},
    title = {How would Stance Detection Techniques Evolve after the Launch of ChatGPT?},
    year = {2022},
    eprint = {2212.14548},
    archivePrefix = {arXiv},
    doi = {10.48550/arXiv.2212.14548},
    url = {https://arxiv.org/abs/2212.14548},
    note = {arXiv preprint arXiv:2212.14548}
}


@misc{DBLP:journals/corr/SchulmanWDRK17,
    author    = {John Schulman and Filip Wolski and Prafulla Dhariwal and Alec Radford and Oleg Klimov},
    title     = {Proximal Policy Optimization Algorithms},
    year      = {2017},
    eprint    = {1707.06347},
    archivePrefix = {arXiv},
    url       = {https://arxiv.org/abs/1707.06347},
    note      = {arXiv preprint arXiv:1707.06347}
}


@article{Ji2023,
author = {Ji, Ziwei and Lee, Nayeon and Frieske, Rita and Yu, Tiezheng and Su, Dan and Xu, Yan and Ishii, Etsuko and Bang, Ye Jin and Madotto, Andrea and Fung, Pascale},
title = {Survey of Hallucination in Natural Language Generation},
year = {2023},
issue_date = {December 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {55},
number = {12},
issn = {0360-0300},
url = {https://doi.org/10.1145/3571730},
doi = {10.1145/3571730},
abstract = {Natural Language Generation (NLG) has improved exponentially in recent years thanks to the development of sequence-to-sequence deep learning technologies such as Transformer-based language models. This advancement has led to more fluent and coherent NLG, leading to improved development in downstream tasks such as abstractive summarization, dialogue generation, and data-to-text generation. However, it is also apparent that deep learning based generation is prone to hallucinate unintended text, which degrades the system performance and fails to meet user expectations in many real-world scenarios. To address this issue, many studies have been presented in measuring and mitigating hallucinated texts, but these have never been reviewed in a comprehensive manner before.In this survey, we thus provide a broad overview of the research progress and challenges in the hallucination problem in NLG. The survey is organized into two parts: (1) a general overview of metrics, mitigation methods, and future directions, and (2) an overview of task-specific research progress on hallucinations in the following downstream tasks, namely abstractive summarization, dialogue generation, generative question answering, data-to-text generation, and machine translation. This survey serves to facilitate collaborative efforts among researchers in tackling the challenge of hallucinated texts in NLG.},
journal = {ACM Computing Surveys},
month = {mar},
articleno = {248},
numpages = {38},
keywords = {consistency in NLG, factuality in NLG, faithfulness in NLG, extrinsic hallucination, intrinsic hallucination, Hallucination}
}

@article{DBLP:journals/csur/LiuYFJHN23,
 author = {Pengfei Liu and
Weizhe Yuan and
Jinlan Fu and
Zhengbao Jiang and
Hiroaki Hayashi and
Graham Neubig},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/journals/csur/LiuYFJHN23.bib},
 doi = {10.1145/3560815},
 journal = {{ACM} Computing Surveys},
 number = {9},
 pages = {195:1--195:35},
 timestamp = {Mon, 28 Aug 2023 21:19:32 +0200},
 title = {{Pre-train, Prompt, and Predict: {A} Systematic Survey of Prompting
Methods in Natural Language Processing}},
 url = {https://doi.org/10.1145/3560815},
 volume = {55},
 year = {2023}
}

@article{DBLP:journals/jmlr/RaffelSRLNMZLL20,
 author = {Colin Raffel and
Noam Shazeer and
Adam Roberts and
Katherine Lee and
Sharan Narang and
Michael Matena and
Yanqi Zhou and
Wei Li and
Peter J. Liu},
 bibsource = {dblp computer science bibliography, https://dblp.org},
 biburl = {https://dblp.org/rec/journals/jmlr/RaffelSRLNMZLL20.bib},
 journal = {Journal of Machine Learning Research},
 pages = {140:1--140:67},
 timestamp = {Fri, 05 Feb 2021 15:43:41 +0100},
 title = {Exploring the Limits of Transfer Learning with a Unified Text-to-Text
Transformer},
 url = {http://jmlr.org/papers/v21/20-074.html},
 volume = {21},
 year = {2020}
}


@inproceedings{devlin-etal-2019-bert,
 abstract = {We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models (Peters et al., 2018a; Radford et al., 2018), BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications. BERT is conceptually simple and empirically powerful. It obtains new state-of-the-art results on eleven natural language processing tasks, including pushing the GLUE score to 80.5 (7.7 point absolute improvement), MultiNLI accuracy to 86.7{\%} (4.6{\%} absolute improvement), SQuAD v1.1 question answering Test F1 to 93.2 (1.5 point absolute improvement) and SQuAD v2.0 Test F1 to 83.1 (5.1 point absolute improvement).},
 address = {Minneapolis, Minnesota},
 author = {Devlin, Jacob  and
Chang, Ming-Wei  and
Lee, Kenton  and
Toutanova, Kristina},
 booktitle = {Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)},
 doi = {10.18653/v1/N19-1423},
 month = {June},
 pages = {4171--4186},
 publisher = {Association for Computational Linguistics},
 title = {{BERT}: Pre-training of Deep Bidirectional Transformers for Language Understanding},
 url = {https://aclanthology.org/N19-1423},
 year = {2019}
}

@inproceedings{ettinger-etal-2023-expert,
 abstract = {Large language models (LLMs) demonstrate an amazing proficiency and fluency in the $\textit{use}$ of language. Does that mean that they have also acquired insightful linguistic knowledge $\textit{about}$ the language, to an extent that they can serve as an {``}expert linguistic annotator{'}? In this paper, we examine the successes and limitations of the GPT-3, ChatGPT, and GPT-4 models, focusing on the Abstract Meaning Representation (AMR) parsing formalism (Banarescu et al., 2013), which provides rich graphical representations of sentence meaning structure while abstracting away from surface forms. We compare models{'} analysis of this semantic structure across two settings: 1) direct production of AMR parses based on zero- and few-shot examples, and 2) indirect partial reconstruction of AMR via metalinguistic natural language queries (e.g., {``}Identify the primary event of this sentence, and the predicate corresponding to that event.{''}). Across these settings, we find that models can reliably reproduce the basic format of AMR, as well as some core event, argument, and modifier structure$-$however, model outputs are prone to frequent and major errors, and holistic analysis of parse acceptability shows that even with few-shot demonstrations, models have virtually 0{\%} success in producing fully accurate parses. Eliciting responses in natural language produces similar patterns of errors. Overall, our findings indicate that these models out-of-the-box can accurately identify some core aspects of semantic structure, but there remain key limitations in their ability to support fully accurate semantic analyses or parses.},
 address = {Singapore},
 author = {Ettinger, Allyson  and
Hwang, Jena  and
Pyatkin, Valentina  and
Bhagavatula, Chandra  and
Choi, Yejin},
 booktitle = {Findings of the Association for Computational Linguistics: EMNLP 2023},
 doi = {10.18653/v1/2023.findings-emnlp.553},
 editor = {Bouamor, Houda  and
Pino, Juan  and
Bali, Kalika},
 month = {December},
 pages = {8250--8263},
 publisher = {Association for Computational Linguistics},
 title = {{``}You Are An Expert Linguistic Annotator{''}: Limits of {LLM}s as Analyzers of {A}bstract {M}eaning {R}epresentation},
 url = {https://aclanthology.org/2023.findings-emnlp.553},
 year = {2023}
}

@inproceedings{Framester2016,
 abstract = {Semantic web applications leveraging NLP can benefit from easy access to expressive lexical resources such as FrameNet. However, the usefulness of FrameNet is affected by its limited coverage and non-standard semantics. The access to existing linguistic resources is also limited because of poor connectivity among them. We present some strategies based on Linguistic Linked Data to broaden FrameNet coverage and formal linkage of lexical and factual resources. We created a novel resource, Framester, which acts as a hub between FrameNet, WordNet, VerbNet, BabelNet, DBpedia, Yago, DOLCE-Zero, as well as other resources. Framester is not only a strongly connected knowledge graph, but also applies a rigorous formal treatment for Fillmore's frame semantics, enabling full-fledged OWL querying and reasoning on a large frame-based knowledge graph. We also describe Word Frame Disambiguation, an application that reuses Framester data as a base in order to perform frame detection from text, with results comparable in precision to the state of the art, but with a much higher coverage.},
 address = {Bologna, Italy},
 author = {Gangemi, Aldo
and Alam, Mehwish
and Asprino, Luigi
and Presutti, Valentina
and Recupero, Diego Reforgiato},
 booktitle = {Knowledge Engineering and Knowledge Management},
 editor = {Blomqvist, Eva
and Ciancarini, Paolo
and Poggi, Francesco
and Vitali, Fabio},
 isbn = {978-3-319-49004-5},
 pages = {239--254},
 publisher = {Springer International Publishing},
 title = {{Framester: A Wide Coverage Linguistic Linked Data Hub}},
 year = {2016}
}


@article{Gangemi2017,
  author    = {Aldo Gangemi and
               Valentina Presutti and
               Diego Reforgiato Recupero and
               Andrea Giovanni Nuzzolese and
               Francesco Draicchio and
               Misael Mongiov{\`{\i}}},
  title     = {{Semantic Web Machine Reading with FRED}},
  journal   = {Semantic Web},
  volume    = {8},
  number    = {6},
  pages     = {873--893},
  year      = {2017},
  url       = {https://doi.org/10.3233/SW-160240},
  doi       = {10.3233/SW-160240},
  timestamp = {Fri, 27 Mar 2020 08:39:51 +0100},
  biburl    = {https://dblp.org/rec/journals/semweb/GangemiPRNDM17.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{Gangemi2023,
 address = {Athens, Greece},
 author = {Gangemi, Aldo  and
Graciotti, Arianna  and
Meloni, Antonello  and
Nuzzolese, Andrea  and
Presutti, Valentina and Reforgiato Recupero, Diego and Russo, Alessandro and Tripodi, Rocco},
 booktitle = {22nd International Semantic Web Conference},
 month = {November},
 publisher = {CEUR Workshop Proceedings},
 title = {{Text2AMR2FRED, a tool for transforming text into RDF/OWL Knowledge Graphs via Abstract Meaning Representation}},
 year = {2023}
}


@inproceedings{ijcai2022p568,
 author = {Deng, Zhenyun and Zhu, Yonghua and Chen, Yang and Witbrock, Michael and Riddle, Patricia},
 booktitle = {Proceedings of the Thirty-First International Joint Conference on
Artificial Intelligence, {IJCAI-22}},
 doi = {10.24963/ijcai.2022/568},
 editor = {Lud De Raedt},
 month = {7},
 note = {Main Track},
 pages = {4093--4099},
 publisher = {International Joint Conferences on Artificial Intelligence Organization},
 title = {{Interpretable AMR-Based Question Decomposition for Multi-hop Question Answering}},
 url = {https://doi.org/10.24963/ijcai.2022/568},
 year = {2022}
}

@inproceedings{jahan-etal-2023-evaluation,
 abstract = {ChatGPT is a large language model developed by OpenAI. Despite its impressive performance across various tasks, no prior work has investigated its capability in the biomedical domain yet. To this end, this paper aims to evaluate the performance of ChatGPT on various benchmark biomedical tasks, such as relation extraction, document classification, question answering, and summarization. To the best of our knowledge, this is the first work that conducts an extensive evaluation of ChatGPT in the biomedical domain. Interestingly, we find based on our evaluation that in biomedical datasets that have smaller training sets, zero-shot ChatGPT even outperforms the state-of-the-art fine-tuned generative transformer models, such as BioGPT and BioBART. This suggests that ChatGPT{'}s pre-training on large text corpora makes it quite specialized even in the biomedical domain. Our findings demonstrate that ChatGPT has the potential to be a valuable tool for various tasks in the biomedical domain that lack large annotated data.},
 address = {Toronto, Canada},
 author = {Jahan, Israt  and
Laskar, Md Tahmid Rahman  and
Peng, Chun  and
Huang, Jimmy},
 booktitle = {The 22nd Workshop on Biomedical Natural Language Processing and BioNLP Shared Tasks},
 doi = {10.18653/v1/2023.bionlp-1.30},
 month = {July},
 pages = {326--336},
 publisher = {Association for Computational Linguistics},
 title = {Evaluation of {C}hat{GPT} on Biomedical Tasks: A Zero-Shot Comparison with Fine-Tuned Generative Transformers},
 url = {https://aclanthology.org/2023.bionlp-1.30},
 year = {2023}
}

@inproceedings{Jungwoo2020,
 abstract = {CommonsenseQA is a task in which a correct answer is predicted through commonsense reasoning with pre-defined knowledge. Most previous works have aimed to improve the performance with distributed representation without considering the process of predicting the answer from the semantic representation of the question. To shed light upon the semantic interpretation of the question, we propose an AMR-ConceptNet-Pruned (ACP) graph. The ACP graph is pruned from a full integrated graph encompassing Abstract Meaning Representation (AMR) graph generated from input questions and an external commonsense knowledge graph, ConceptNet (CN). Then the ACP graph is exploited to interpret the reasoning path as well as to predict the correct answer on the CommonsenseQA task. This paper presents the manner in which the commonsense reasoning process can be interpreted with the relations and concepts provided by the ACP graph. Moreover, ACP-based models are shown to outperform the baselines.},
 address = {Barcelona, Spain (Online)},
 author = {Lim, Jungwoo  and
Oh, Dongsuk  and
Jang, Yoonna  and
Yang, Kisu  and
Lim, Heuiseok},
 booktitle = {{Proceedings of the 28th International Conference on Computational Linguistics}},
 doi = {10.18653/v1/2020.coling-main.222},
 editor = {Scott, Donia  and
Bel, Nuria  and
Zong, Chengqing},
 month = {December},
 pages = {2459--2471},
 publisher = {International Committee on Computational Linguistics},
 title = {{I Know What You Asked: Graph Path Learning using AMR for Commonsense Reasoning}},
 url = {https://aclanthology.org/2020.coling-main.222},
 year = {2020}
}

@inproceedings{Kapanipathi2021,
 address = {Online},
 author = {Kapanipathi, Pavan  and
Abdelaziz, Ibrahim  and
Ravishankar, Srinivas  and
Roukos, Salim  and
Gray, Alexander  and
Fernandez Astudillo, Ram{\'o}n  and
Chang, Maria  and
Cornelio, Cristina  and
Dana, Saswati  and
Fokoue, Achille  and
Garg, Dinesh  and
Gliozzo, Alfio  and
Gurajada, Sairam  and
Karanam, Hima  and
Khan, Naweed  and
Khandelwal, Dinesh  and
Lee, Young-Suk  and
Li, Yunyao  and
Luus, Francois  and
Makondo, Ndivhuwo  and
Mihindukulasooriya, Nandana  and
Naseem, Tahira  and
Neelam, Sumit  and
Popa, Lucian  and
Gangi Reddy, Revanth  and
Riegel, Ryan  and
Rossiello, Gaetano  and
Sharma, Udit  and
Bhargav, G P Shrivatsa  and
Yu, Mo},
 booktitle = {{Findings of the Association for Computational Linguistics: Association for Computational Linguistics-IJCNLP 2021}},
 doi = {10.18653/v1/2021.findings-acl.339},
 editor = {Zong, Chengqing  and
Xia, Fei  and
Li, Wenjie  and
Navigli, Roberto},
 month = {August},
 pages = {3884--3894},
 publisher = {Association for Computational Linguistics},
 title = {{Leveraging {A}bstract {M}eaning {R}epresentation for Knowledge Base Question Answering}},
 url = {https://aclanthology.org/2021.findings-acl.339},
 year = {2021}
}

@inproceedings{kiela-etal-2021-dynabench,
 abstract = {We introduce Dynabench, an open-source platform for dynamic dataset creation and model benchmarking. Dynabench runs in a web browser and supports human-and-model-in-the-loop dataset creation: annotators seek to create examples that a target model will misclassify, but that another person will not. In this paper, we argue that Dynabench addresses a critical need in our community: contemporary models quickly achieve outstanding performance on benchmark tasks but nonetheless fail on simple challenge examples and falter in real-world scenarios. With Dynabench, dataset creation, model development, and model assessment can directly inform each other, leading to more robust and informative benchmarks. We report on four initial NLP tasks, illustrating these concepts and highlighting the promise of the platform, and address potential objections to dynamic benchmarking as a new standard for the field.},
 address = {Online},
 author = {Kiela, Douwe  and
Bartolo, Max  and
Nie, Yixin  and
Kaushik, Divyansh  and
Geiger, Atticus  and
Wu, Zhengxuan  and
Vidgen, Bertie  and
Prasad, Grusha  and
Singh, Amanpreet  and
Ringshia, Pratik  and
Ma, Zhiyi  and
Thrush, Tristan  and
Riedel, Sebastian  and
Waseem, Zeerak  and
Stenetorp, Pontus  and
Jia, Robin  and
Bansal, Mohit  and
Potts, Christopher  and
Williams, Adina},
 booktitle = {Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
 doi = {10.18653/v1/2021.naacl-main.324},
 month = {June},
 pages = {4110--4124},
 publisher = {Association for Computational Linguistics},
 title = {Dynabench: Rethinking Benchmarking in {NLP}},
 url = {https://aclanthology.org/2021.naacl-main.324},
 year = {2021}
}

@inproceedings{laskar-etal-2023-systematic,
 abstract = {The development of large language models (LLMs) such as ChatGPT has brought a lot of attention recently. However, their evaluation in the benchmark academic datasets remains under-explored due to the difficulty of evaluating the generative outputs produced by this model against the ground truth. In this paper, we aim to present a thorough evaluation of ChatGPT{'}s performance on diverse academic datasets, covering tasks like question-answering, text summarization, code generation, commonsense reasoning, mathematical problem-solving, machine translation, bias detection, and ethical considerations. Specifically, we evaluate ChatGPT across 140 tasks and analyze 255K responses it generates in these datasets. This makes our work the largest evaluation of ChatGPT in NLP benchmarks. In short, our study aims to validate the strengths and weaknesses of ChatGPT in various tasks and provide insights for future research using LLMs. We also report a new emergent ability to follow multi-query instructions that we mostly found in ChatGPT and other instruction-tuned models. Our extensive evaluation shows that even though ChatGPT is capable of performing a wide variety of tasks, and may obtain impressive performance in several benchmark datasets, it is still far from achieving the ability to reliably solve many challenging tasks. By providing a thorough assessment of ChatGPT{'}s performance across diverse NLP tasks, this paper sets the stage for a targeted deployment of ChatGPT-like LLMs in real-world applications.},
 address = {Toronto, Canada},
 author = {Laskar, Md Tahmid Rahman  and
Bari, M Saiful  and
Rahman, Mizanur  and
Bhuiyan, Md Amran Hossen  and
Joty, Shafiq  and
Huang, Jimmy},
 booktitle = {Findings of the Association for Computational Linguistics: Association for Computational Linguistics 2023},
 doi = {10.18653/v1/2023.findings-acl.29},
 month = {July},
 pages = {431--469},
 publisher = {Association for Computational Linguistics},
 title = {A Systematic Study and Comprehensive Evaluation of {C}hat{GPT} on Benchmark Datasets},
 url = {https://aclanthology.org/2023.findings-acl.29},
 year = {2023}
}

@inproceedings{lin-2004-rouge,
 address = {Barcelona, Spain},
 author = {Lin, Chin-Yew},
 booktitle = {Text Summarization Branches Out},
 month = {July},
 pages = {74--81},
 publisher = {Association for Computational Linguistics},
 title = {{ROUGE}: A Package for Automatic Evaluation of Summaries},
 url = {https://aclanthology.org/W04-1013},
 year = {2004}
}

@inproceedings{mallen-etal-2023-trust,
 abstract = {Despite their impressive performance on diverse tasks, large language models (LMs) still struggle with tasks requiring rich world knowledge, implying the difficulty of encoding a wealth of world knowledge in their parameters. This paper aims to understand LMs{'} strengths and limitations in memorizing factual knowledge, by conducting large-scale knowledge probing experiments on two open-domain entity-centric QA datasets: PopQA, our new dataset with 14k questions about long-tail entities, and EntityQuestions, a widely used open-domain QA dataset. We find that LMs struggle with less popular factual knowledge, and that retrieval augmentation helps significantly in these cases. Scaling, on the other hand, mainly improves memorization of popular knowledge, and fails to appreciably improve memorization of factual knowledge in the tail. Based on those findings, we devise a new method for retrieval-augmentation that improves performance and reduces inference costs by only retrieving non-parametric memories when necessary.},
 address = {Toronto, Canada},
 author = {Mallen, Alex  and
Asai, Akari  and
Zhong, Victor  and
Das, Rajarshi  and
Khashabi, Daniel  and
Hajishirzi, Hannaneh},
 booktitle = {Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
 doi = {10.18653/v1/2023.acl-long.546},
 editor = {Rogers, Anna  and
Boyd-Graber, Jordan  and
Okazaki, Naoaki},
 month = {July},
 pages = {9802--9822},
 publisher = {Association for Computational Linguistics},
 title = {When Not to Trust Language Models: Investigating Effectiveness of Parametric and Non-Parametric Memories},
 url = {https://aclanthology.org/2023.acl-long.546},
 year = {2023}
}


@inproceedings{Meloni2017,
 address = {Cham},
 author = {Meloni, Antonello and others},
 booktitle = {{The Semantic Web: ESWC 2017 Satellite Events}},
 doi = {10.1007/978-3-319-70407-4\_9},
 editor = {Blomqvist, Eva
and others},
 isbn = {978-3-319-70407-4},
 pages = {43--47},
 publisher = {Springer International Publishing},
 title = {{AMR2FRED, A Tool for Translating Abstract Meaning Representation to Motif-Based Linguistic Knowledge Graphs}},
 year = {2017}
}

@inproceedings{papineni-etal-2002-bleu,
 address = {Philadelphia, Pennsylvania, USA},
 author = {Papineni, Kishore  and
Roukos, Salim  and
Ward, Todd  and
Zhu, Wei-Jing},
 booktitle = {Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics},
 doi = {10.3115/1073083.1073135},
 month = {July},
 pages = {311--318},
 publisher = {Association for Computational Linguistics},
 title = {{B}leu: a Method for Automatic Evaluation of Machine Translation},
 url = {https://aclanthology.org/P02-1040},
 year = {2002}
}

@book{Penman1991,
 author = {Christian M. I. M. Matthiessen and John A. Bateman},
 publisher = {Pinter Publishers},
 title = {{Text Generation and Systemic-Functional Linguistics: Experiences from English and Japanese}},
 year = {1991}
}

@inproceedings{petroni-etal-2019-language,
 abstract = {Recent progress in pretraining language models on large textual corpora led to a surge of improvements for downstream NLP tasks. Whilst learning linguistic knowledge, these models may also be storing relational knowledge present in the training data, and may be able to answer queries structured as {``}fill-in-the-blank{''} cloze statements. Language models have many advantages over structured knowledge bases: they require no schema engineering, allow practitioners to query about an open class of relations, are easy to extend to more data, and require no human supervision to train. We present an in-depth analysis of the relational knowledge already present (without fine-tuning) in a wide range of state-of-the-art pretrained language models. We find that (i) without fine-tuning, BERT contains relational knowledge competitive with traditional NLP methods that have some access to oracle knowledge, (ii) BERT also does remarkably well on open-domain question answering against a supervised baseline, and (iii) certain types of factual knowledge are learned much more readily than others by standard language model pretraining approaches. The surprisingly strong ability of these models to recall factual knowledge without any fine-tuning demonstrates their potential as unsupervised open-domain QA systems. The code to reproduce our analysis is available at \url{https://github.com/facebookresearch/LAMA}.},
 address = {Hong Kong, China},
 author = {Petroni, Fabio  and
Rockt{\"a}schel, Tim  and
Riedel, Sebastian  and
Lewis, Patrick  and
Bakhtin, Anton  and
Wu, Yuxiang  and
Miller, Alexander},
 booktitle = {Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)},
 doi = {10.18653/v1/D19-1250},
 month = {November},
 pages = {2463--2473},
 publisher = {Association for Computational Linguistics},
 title = {Language Models as Knowledge Bases?},
 url = {https://aclanthology.org/D19-1250},
 year = {2019}
}

@inproceedings{pmlr-v202-kandpal23a,
 abstract = {The Internet contains a wealth of knowledge—from the birthdays of historical figures to tutorials on how to code—all of which may be learned by language models. However, while certain pieces of information are ubiquitous on the web, others appear extremely rarely. In this paper, we study the relationship between the knowledge memorized by large language models and the information in pre-training datasets scraped from the web. In particular, we show that a language model’s ability to answer a fact-based question relates to how many documents associated with that question were seen during pre-training. We identify these relevant documents by entity linking pre-training datasets and counting documents that contain the same entities as a given question-answer pair. Our results demonstrate strong correlational and causal relationships between accuracy and relevant document count for numerous question answering datasets (e.g., TriviaQA), pre-training corpora (e.g., ROOTS), and model sizes (e.g., 176B parameters). Moreover, while larger models are better at learning long-tail knowledge, we estimate that today’s models must be scaled by many orders of magnitude to reach competitive QA performance on questions with little support in the pre-training data. Finally, we show that retrieval-augmentation can reduce the dependence on relevant pre-training information, presenting a promising approach for capturing the long-tail.},
 author = {Kandpal, Nikhil and Deng, Haikang and Roberts, Adam and Wallace, Eric and Raffel, Colin},
 booktitle = {Proceedings of the 40th International Conference on Machine Learning},
 editor = {Krause, Andreas and Brunskill, Emma and Cho, Kyunghyun and Engelhardt, Barbara and Sabato, Sivan and Scarlett, Jonathan},
 month = {July},
 pages = {15696--15707},
 pdf = {https://proceedings.mlr.press/v202/kandpal23a/kandpal23a.pdf},
 publisher = {PMLR},
 series = {Proceedings of Machine Learning Research},
 title = {{Large Language Models Struggle to Learn Long-Tail Knowledge}},
 url = {https://proceedings.mlr.press/v202/kandpal23a.html},
 volume = {202},
 year = {2023}
}

@inproceedings{Rao2017,
 abstract = {We propose a novel, Abstract Meaning Representation (AMR) based approach to identifying molecular events/interactions in biomedical text. Our key contributions are: (1) an empirical validation of our hypothesis that an event is a subgraph of the AMR graph, (2) a neural network-based model that identifies such an event subgraph given an AMR, and (3) a distant supervision based approach to gather additional training data. We evaluate our approach on the 2013 Genia Event Extraction dataset and show promising results.},
 address = {Vancouver, Canada},
 author = {Rao, Sudha  and
Marcu, Daniel  and
Knight, Kevin  and
Daum{\'e} III, Hal},
 booktitle = {{B}io{NLP} 2017},
 doi = {10.18653/v1/W17-2315},
 editor = {Cohen, Kevin Bretonnel  and
Demner-Fushman, Dina  and
Ananiadou, Sophia  and
Tsujii, Junichi},
 month = {August},
 pages = {126--135},
 publisher = {Association for Computational Linguistics},
 title = {{Biomedical Event Extraction using {A}bstract {M}eaning {R}epresentation}},
 url = {https://aclanthology.org/W17-2315},
 year = {2017}
}

@inproceedings{roberts-etal-2020-much,
 abstract = {It has recently been observed that neural language models trained on unstructured text can implicitly store and retrieve knowledge using natural language queries. In this short paper, we measure the practical utility of this approach by fine-tuning pre-trained models to answer questions without access to any external context or knowledge. We show that this approach scales with model size and performs competitively with open-domain systems that explicitly retrieve answers from an external knowledge source when answering questions. To facilitate reproducibility and future work, we release our code and trained models.},
 address = {Online},
 author = {Roberts, Adam  and
Raffel, Colin  and
Shazeer, Noam},
 booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
 doi = {10.18653/v1/2020.emnlp-main.437},
 editor = {Webber, Bonnie  and
Cohn, Trevor  and
He, Yulan  and
Liu, Yang},
 month = {November},
 pages = {5418--5426},
 publisher = {Association for Computational Linguistics},
 title = {How Much Knowledge Can You Pack Into the Parameters of a Language Model?},
 url = {https://aclanthology.org/2020.emnlp-main.437},
 year = {2020}
}

@article{siriwardhana-etal-2023-improving,
 abstract = {Retrieval Augment Generation (RAG) is a recent advancement in Open-Domain Question Answering (ODQA). RAG has only been trained and explored with a Wikipedia-based external knowledge base and is not optimized for use in other specialized domains such as healthcare and news. In this paper, we evaluate the impact of joint training of the retriever and generator components of RAG for the task of domain adaptation in ODQA. We propose RAG-end2end, an extension to RAG that can adapt to a domain-specific knowledge base by updating all components of the external knowledge base during training. In addition, we introduce an auxiliary training signal to inject more domain-specific knowledge. This auxiliary signal forces RAG-end2end to reconstruct a given sentence by accessing the relevant information from the external knowledge base. Our novel contribution is that, unlike RAG, RAG-end2end does joint training of the retriever and generator for the end QA task and domain adaptation. We evaluate our approach with datasets from three domains: COVID-19, News, and Conversations, and achieve significant performance improvements compared to the original RAG model. Our work has been open-sourced through the HuggingFace Transformers library, attesting to our work{'}s credibility and technical consistency.},
 address = {Cambridge, MA},
 author = {Siriwardhana, Shamane  and
Weerasekera, Rivindu  and
Wen, Elliott  and
Kaluarachchi, Tharindu  and
Rana, Rajib  and
Nanayakkara, Suranga},
 doi = {10.1162/tacl_a_00530},
 journal = {Transactions of the Association for Computational Linguistics},
 pages = {1--17},
 publisher = {MIT Press},
 title = {Improving the Domain Adaptation of Retrieval Augmented Generation ({RAG}) Models for Open Domain Question Answering},
 url = {https://aclanthology.org/2023.tacl-1.1},
 volume = {11},
 year = {2023}
}

@inproceedings{stranisci-etal-2023-wikibio,
 abstract = {Biographical event detection is a relevant task that allows for the exploration and comparison of the ways in which people{'}s lives are told and represented. This may support several real-life applications in digital humanities and in works aimed at exploring bias about minoritized groups. Despite that, there are no corpora and models specifically designed for this task. In this paper we fill this gap by presenting a new corpus annotated for biographical event detection. The corpus, which includes 20 Wikipedia biographies, was aligned with 5 existing corpora in order to train a model for the biographical event detection task. The model was able to detect all mentions of the target-entity in a biography with an F-score of 0.808 and the entity-related events with an F-score of 0.859. Finally, the model was used for performing an analysis of biases about women and non-Western people in Wikipedia biographies.},
 address = {Toronto, Canada},
 author = {Stranisci, Marco Antonio  and
Damiano, Rossana  and
Mensa, Enrico  and
Patti, Viviana  and
Radicioni, Daniele  and
Caselli, Tommaso},
 booktitle = {Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
 doi = {10.18653/v1/2023.acl-long.691},
 month = {July},
 pages = {12370--12384},
 publisher = {Association for Computational Linguistics},
 title = {{W}iki{B}io: a Semantic Resource for the Intersectional Analysis of Biographical Events},
 url = {https://aclanthology.org/2023.acl-long.691},
 year = {2023}
}

@inproceedings{Tripodi2021,
 address = {Online},
 author = {Procopio, Luigi  and
Tripodi, Rocco  and
Navigli, Roberto},
 booktitle = {Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies},
 doi = {10.18653/v1/2021.naacl-main.30},
 editor = {Toutanova, Kristina  and
Rumshisky, Anna  and
Zettlemoyer, Luke  and
Hakkani-Tur, Dilek  and
Beltagy, Iz  and
Bethard, Steven  and
Cotterell, Ryan  and
Chakraborty, Tanmoy  and
Zhou, Yichao},
 month = {June},
 pages = {325--337},
 publisher = {Association for Computational Linguistics},
 title = {{SGL: Speaking the Graph Languages of Semantic Parsing via Multilingual Translation}},
 url = {https://aclanthology.org/2021.naacl-main.30},
 year = {2021}
}


@misc{Velickovic2018GraphAN,
    author    = {Petar Velickovic and Guillem Cucurull and Arantxa Casanova and Adriana Romero and Pietro Li{\`{o}} and Yoshua Bengio},
    title     = {Graph Attention Networks},
    year      = {2018},
    eprint    = {1710.10903},
    archivePrefix = {arXiv},
    url       = {https://arxiv.org/abs/1710.10903},
    note      = {arXiv preprint arXiv:1710.10903}
}


@inproceedings{wu-etal-2020-scalable,
 abstract = {This paper introduces a conceptually simple, scalable, and highly effective BERT-based entity linking model, along with an extensive evaluation of its accuracy-speed trade-off. We present a two-stage zero-shot linking algorithm, where each entity is defined only by a short textual description. The first stage does retrieval in a dense space defined by a bi-encoder that independently embeds the mention context and the entity descriptions. Each candidate is then re-ranked with a cross-encoder, that concatenates the mention and entity text. Experiments demonstrate that this approach is state of the art on recent zero-shot benchmarks (6 point absolute gains) and also on more established non-zero-shot evaluations (e.g. TACKBP-2010), despite its relative simplicity (e.g. no explicit entity embeddings or manually engineered mention tables). We also show that bi-encoder linking is very fast with nearest neighbor search (e.g. linking with 5.9 million candidates in 2 milliseconds), and that much of the accuracy gain from the more expensive cross-encoder can be transferred to the bi-encoder via knowledge distillation. Our code and models are available at https://github.com/facebookresearch/BLINK.},
 address = {Online},
 author = {Wu, Ledell  and
Petroni, Fabio  and
Josifoski, Martin  and
Riedel, Sebastian  and
Zettlemoyer, Luke},
 booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
 doi = {10.18653/v1/2020.emnlp-main.519},
 editor = {Webber, Bonnie  and
Cohn, Trevor  and
He, Yulan  and
Liu, Yang},
 month = {November},
 pages = {6397--6407},
 publisher = {Association for Computational Linguistics},
 title = {{Scalable Zero-shot Entity Linking with Dense Entity Retrieval}},
 url = {https://aclanthology.org/2020.emnlp-main.519},
 year = {2020}
}

@inproceedings{yuan-etal-2022-biobart,
 abstract = {Pretrained language models have served as important backbones for natural language processing. Recently, in-domain pretraining has been shown to benefit various domain-specific downstream tasks. In the biomedical domain, natural language generation (NLG) tasks are of critical importance, while understudied. Approaching natural language understanding (NLU) tasks as NLG achieves satisfying performance in the general domain through constrained language generation or language prompting. We emphasize the lack of in-domain generative language models and the unsystematic generative downstream benchmarks in the biomedical domain, hindering the development of the research community. In this work, we introduce the generative language model BioBART that adapts BART to the biomedical domain. We collate various biomedical language generation tasks including dialogue, summarization, entity linking, and named entity recognition. BioBART pretrained on PubMed abstracts has enhanced performance compared to BART and set strong baselines on several tasks. Furthermore, we conduct ablation studies on the pretraining tasks for BioBART and find that sentence permutation has negative effects on downstream tasks.},
 address = {Dublin, Ireland},
 author = {Yuan, Hongyi  and
Yuan, Zheng  and
Gan, Ruyi  and
Zhang, Jiaxing  and
Xie, Yutao  and
Yu, Sheng},
 booktitle = {Proceedings of the 21st Workshop on Biomedical Language Processing},
 doi = {10.18653/v1/2022.bionlp-1.9},
 month = {May},
 pages = {97--109},
 publisher = {Association for Computational Linguistics},
 title = {{B}io{BART}: Pretraining and Evaluation of A Biomedical Generative Language Model},
 url = {https://aclanthology.org/2022.bionlp-1.9},
 year = {2022}
}

@inproceedings{Zhang2021,
 abstract = {The tasks of Rich Semantic Parsing, such as Abstract Meaning Representation (AMR), share similar goals with Information Extraction (IE) to convert natural language texts into structured semantic representations. To take advantage of such similarity, we propose a novel AMR-guided framework for joint information extraction to discover entities, relations, and events with the help of a pre-trained AMR parser. Our framework consists of two novel components: 1) an AMR based semantic graph aggregator to let the candidate entity and event trigger nodes collect neighborhood information from AMR graph for passing message among related knowledge elements; 2) an AMR guided graph decoder to extract knowledge elements based on the order decided by the hierarchical structures in AMR. Experiments on multiple datasets have shown that the AMR graph encoder and decoder have provided significant gains and our approach has achieved new state-of-the-art performance on all IE subtasks.},
 address = {Online},
 author = {Zhang, Zixuan  and
Ji, Heng},
 booktitle = {{Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies}},
 doi = {10.18653/v1/2021.naacl-main.4},
 month = {June},
 pages = {39--49},
 publisher = {Association for Computational Linguistics},
 title = {{A}bstract {M}eaning {R}epresentation Guided Graph Encoding and Decoding for Joint Information Extraction},
 url = {https://aclanthology.org/2021.naacl-main.4},
 year = {2021}
}

@inproceedings{Zhang2021Biomedical,
 abstract = {Biomedical Information Extraction from scientific literature presents two unique and non-trivial challenges. First, compared with general natural language texts, sentences from scientific papers usually possess wider contexts between knowledge elements. Moreover, comprehending the fine-grained scientific entities and events urgently requires domain-specific background knowledge. In this paper, we propose a novel biomedical Information Extraction (IE) model to tackle these two challenges and extract scientific entities and events from English research papers. We perform Abstract Meaning Representation (AMR) to compress the wide context to uncover a clear semantic structure for each complex sentence. Besides, we construct the sentence-level knowledge graph from an external knowledge base and use it to enrich the AMR graph to improve the model{'}s understanding of complex scientific concepts. We use an edge-conditioned graph attention network to encode the knowledge-enriched AMR graph for biomedical IE tasks. Experiments on the GENIA 2011 dataset show that the AMR and external knowledge have contributed 1.8{\%} and 3.0{\%} absolute F-score gains respectively. In order to evaluate the impact of our approach on real-world problems that involve topic-specific fine-grained knowledge elements, we have also created a new ontology and annotated corpus for entity and event extraction for the COVID-19 scientific literature, which can serve as a new benchmark for the biomedical IE community.},
 address = {Online},
 author = {Zhang, Zixuan  and
Parulian, Nikolaus  and
Ji, Heng  and
Elsayed, Ahmed  and
Myers, Skatje  and
Palmer, Martha},
 booktitle = {{Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)}},
 doi = {10.18653/v1/2021.acl-long.489},
 month = {August},
 pages = {6261--6270},
 publisher = {Association for Computational Linguistics},
 title = {{Fine-grained Information Extraction from Biomedical Literature based on Knowledge-enriched {A}bstract {M}eaning {R}epresentation}},
 url = {https://aclanthology.org/2021.acl-long.489},
 year = {2021}
}

@inproceedings{zhou-etal-2021-structure,
 abstract = {Predicting linearized Abstract Meaning Representation (AMR) graphs using pre-trained sequence-to-sequence Transformer models has recently led to large improvements on AMR parsing benchmarks. These parsers are simple and avoid explicit modeling of structure but lack desirable properties such as graph well-formedness guarantees or built-in graph-sentence alignments. In this work we explore the integration of general pre-trained sequence-to-sequence language models and a structure-aware transition-based approach. We depart from a pointer-based transition system and propose a simplified transition set, designed to better exploit pre-trained language models for structured fine-tuning. We also explore modeling the parser state within the pre-trained encoder-decoder architecture and different vocabulary strategies for the same purpose. We provide a detailed comparison with recent progress in AMR parsing and show that the proposed parser retains the desirable properties of previous transition-based approaches, while being simpler and reaching the new parsing state of the art for AMR 2.0, without the need for graph re-categorization.},
 address = {Online and Punta Cana, Dominican Republic},
 author = {Zhou, Jiawei  and
Naseem, Tahira  and
Fernandez Astudillo, Ram{\'o}n  and
Lee, Young-Suk  and
Florian, Radu  and
Roukos, Salim},
 booktitle = {Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},
 doi = {10.18653/v1/2021.emnlp-main.507},
 month = {November},
 pages = {6279--6290},
 publisher = {Association for Computational Linguistics},
 title = {Structure-aware Fine-tuning of Sequence-to-sequence Transformers for Transition-based {AMR} Parsing},
 url = {https://aclanthology.org/2021.emnlp-main.507},
 year = {2021}
}
