% Use this file for citations not found in the ACL Anthology (contained in "anthology.bib").

@book{Aho:72,
    author  = {Alfred V. Aho and Jeffrey D. Ullman},
    title   = {The Theory of Parsing, Translation and Compiling},
    year    = "1972",
    volume  = "1",
    publisher = {Prentice-Hall},
    address = {Englewood Cliffs, NJ}
}

@book{APA:83,
    author  = {{American Psychological Association}},
    title   = {Publications Manual},
    year    = "1983",
    publisher = {American Psychological Association},
    address = {Washington, DC}
}

@article{Chandra:81,
	author = {Ashok K. Chandra and Dexter C. Kozen and Larry J. Stockmeyer},
	year = "1981",
	title = {Alternation},
	journal = {Journal of the Association for Computing Machinery},
	volume = "28",
	number = "1",
	pages = "114--133",
	doi = "10.1145/322234.322243",
}

@inproceedings{andrew2007scalable,
  title={Scalable training of {L1}-regularized log-linear models},
  author={Andrew, Galen and Gao, Jianfeng},
  booktitle={Proceedings of the 24th International Conference on Machine Learning},
  pages={33--40},
  year={2007},
}

@book{Gusfield:97,
    author  = {Dan Gusfield},
    title   = {Algorithms on Strings, Trees and Sequences},
    year    = "1997",
    publisher = {Cambridge University Press},
    address = {Cambridge, UK}
}

@article{rasooli-tetrault-2015,
    author    = {Mohammad Sadegh Rasooli and Joel R. Tetreault},
    title     = {Yara Parser: {A} Fast and Accurate Dependency Parser},
    journal   = {Computing Research Repository},
    volume    = {arXiv:1503.06733},
    year      = {2015},
    url       = {http://arxiv.org/abs/1503.06733},
    note    = {version 2}
}

@article{Ando2005,
	Acmid = {1194905},
	Author = {Ando, Rie Kubota and Zhang, Tong},
	Issn = {1532-4435},
	Issue_Date = {12/1/2005},
	Journal = {Journal of Machine Learning Research},
	Month = dec,
	Numpages = {37},
	Pages = {1817--1853},
	Publisher = {JMLR.org},
	Title = {A Framework for Learning Predictive Structures from Multiple Tasks and Unlabeled Data},
	Volume = {6},
	Year = {2005}
}


@inproceedings{babakov-etal-2022-large,
    title = "A large-scale computational study of content preservation measures for text style transfer and paraphrase generation",
    author = "Babakov, Nikolay  and
      Dale, David  and
      Logacheva, Varvara  and
      Panchenko, Alexander",
    editor = "Louvan, Samuel  and
      Madotto, Andrea  and
      Madureira, Brielen",
    booktitle = "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics: Student Research Workshop",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.acl-srw.23",
    doi = "10.18653/v1/2022.acl-srw.23",
    pages = "300--321",
    abstract = "Text style transfer and paraphrasing of texts are actively growing areas of NLP, dozens of methods for solving these tasks have been recently introduced. In both tasks, the system is supposed to generate a text which should be semantically similar to the input text. Therefore, these tasks are dependent on methods of measuring textual semantic similarity. However, it is still unclear which measures are the best to automatically evaluate content preservation between original and generated text. According to our observations, many researchers still use BLEU-like measures, while there exist more advanced measures including neural-based that significantly outperform classic approaches. The current problem is the lack of a thorough evaluation of the available measures. We close this gap by conducting a large-scale computational study by comparing 57 measures based on different principles on 19 annotated datasets. We show that measures based on cross-encoder models outperform alternative approaches in almost all cases. We also introduce the Mutual Implication Score (MIS), a measure that uses the idea of paraphrasing as a bidirectional entailment and outperforms all other measures on the paraphrase detection task and performs on par with the best measures in the text style transfer task.",
}
@inproceedings{gekhman-etal-2023-trueteacher,
    title = "{T}rue{T}eacher: Learning Factual Consistency Evaluation with Large Language Models",
    author = "Gekhman, Zorik  and
      Herzig, Jonathan  and
      Aharoni, Roee  and
      Elkind, Chen  and
      Szpektor, Idan",
    editor = "Bouamor, Houda  and
      Pino, Juan  and
      Bali, Kalika",
    booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing",
    month = dec,
    year = "2023",
    address = "Singapore",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.emnlp-main.127",
    doi = "10.18653/v1/2023.emnlp-main.127",
    pages = "2053--2070",
    abstract = "Factual consistency evaluation is often conducted using Natural Language Inference (NLI) models, yet these models exhibit limited success in evaluating summaries. Previous work improved such models with synthetic training data. However, the data is typically based on perturbed human-written summaries, which often differ in their characteristics from real model-generated summaries and have limited coverage of possible factual errors. Alternatively, large language models (LLMs) have recently shown promising results in directly evaluating generative tasks, but are too computationally expensive for practical use. Motivated by these limitations, we introduce TrueTeacher, a method for generating synthetic data by annotating diverse model-generated summaries using a LLM. Unlike prior work, TrueTeacher does not rely on human-written summaries, and is multilingual by nature. Experiments on the TRUE benchmark show that a student model trained using our data, substantially outperforms both the state-of-the-art model with similar capacity, and the LLM teacher. In a systematic study, we compare TrueTeacher to existing synthetic data generation methods and demonstrate its superiority and robustness to domain-shift. We also show that our method generalizes to multilingual scenarios. Lastly, we release our large scale synthetic dataset (1.4M examples), generated using TrueTeacher, and a checkpoint trained on this data.",
}

@article{Sharma2019NaturalLU,
  title={Natural Language Understanding with the Quora Question Pairs Dataset},
  author={Lakshay Sharma and Laura Graesser and Nikita Nangia and Utku Evci},
  journal={ArXiv},
  year={2019},
  volume={abs/1907.01041},
  url={https://api.semanticscholar.org/CorpusID:195776066}
}

@inproceedings{zhang-etal-2019-paws,
    title = "{PAWS}: Paraphrase Adversaries from Word Scrambling",
    author = "Zhang, Yuan  and
      Baldridge, Jason  and
      He, Luheng",
    editor = "Burstein, Jill  and
      Doran, Christy  and
      Solorio, Thamar",
    booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",
    month = jun,
    year = "2019",
    address = "Minneapolis, Minnesota",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/N19-1131",
    doi = "10.18653/v1/N19-1131",
    pages = "1298--1308",
    abstract = "Existing paraphrase identification datasets lack sentence pairs that have high lexical overlap without being paraphrases. Models trained on such data fail to distinguish pairs like flights from New York to Florida and flights from Florida to New York. This paper introduces PAWS (Paraphrase Adversaries from Word Scrambling), a new dataset with 108,463 well-formed paraphrase and non-paraphrase pairs with high lexical overlap. Challenging pairs are generated by controlled word swapping and back translation, followed by fluency and paraphrase judgments by human raters. State-of-the-art models trained on existing datasets have dismal performance on PAWS ({\textless}40{\%} accuracy); however, including PAWS training data for these models improves their accuracy to 85{\%} while maintaining performance on existing tasks. In contrast, models that do not capture non-local contextual information fail even with PAWS training examples. As such, PAWS provides an effective instrument for driving further progress on models that better exploit structure, context, and pairwise comparisons.",
}

@inproceedings{nie-etal-2020-adversarial,
  author       = {Yixin Nie and
                  Adina Williams and
                  Emily Dinan and
                  Mohit Bansal and
                  Jason Weston and
                  Douwe Kiela},
  editor       = {Dan Jurafsky and
                  Joyce Chai and
                  Natalie Schluter and
                  Joel R. Tetreault},
  title        = {Adversarial {NLI:} {A} New Benchmark for Natural Language Understanding},
  booktitle    = {Proceedings of the 58th Annual Meeting of the Association for Computational
                  Linguistics, {ACL} 2020, Online, July 5-10, 2020},
  pages        = {4885--4901},
  publisher    = {Association for Computational Linguistics},
  year         = {2020},
  url          = {https://doi.org/10.18653/v1/2020.acl-main.441},
  doi          = {10.18653/V1/2020.ACL-MAIN.441},
  timestamp    = {Thu, 14 Oct 2021 09:46:09 +0200},
  biburl       = {https://dblp.org/rec/conf/acl/NieWDBWK20.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}



@inproceedings{feng2020languageagnostic,
    title = "Language-agnostic {BERT} Sentence Embedding",
    author = "Feng, Fangxiaoyu  and
      Yang, Yinfei  and
      Cer, Daniel  and
      Arivazhagan, Naveen  and
      Wang, Wei",
    editor = "Muresan, Smaranda  and
      Nakov, Preslav  and
      Villavicencio, Aline",
    booktitle = "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.acl-long.62",
    doi = "10.18653/v1/2022.acl-long.62",
    pages = "878--891",
    abstract = "While BERT is an effective method for learning monolingual sentence embeddings for semantic similarity and embedding based transfer learning BERT based cross-lingual sentence embeddings have yet to be explored. We systematically investigate methods for learning multilingual sentence embeddings by combining the best methods for learning monolingual and cross-lingual representations including: masked language modeling (MLM), translation language modeling (TLM), dual encoder translation ranking, and additive margin softmax. We show that introducing a pre-trained multilingual language model dramatically reduces the amount of parallel training data required to achieve good performance by 80{\%}. Composing the best of these methods produces a model that achieves 83.7{\%} bi-text retrieval accuracy over 112 languages on Tatoeba, well above the 65.5{\%} achieved by LASER, while still performing competitively on monolingual transfer learning benchmarks. Parallel data mined from CommonCrawl using our best model is shown to train competitive NMT models for en-zh and en-de. We publicly release our best multilingual sentence embedding model for 109+ languages at \url{https://tfhub.dev/google/LaBSE}.",
}

@article{wang2022text,
  author       = {Liang Wang and
                  Nan Yang and
                  Xiaolong Huang and
                  Binxing Jiao and
                  Linjun Yang and
                  Daxin Jiang and
                  Rangan Majumder and
                  Furu Wei},
  title        = {Text Embeddings by Weakly-Supervised Contrastive Pre-training},
  journal      = {CoRR},
  volume       = {abs/2212.03533},
  year         = {2022},
  url          = {https://doi.org/10.48550/arXiv.2212.03533},
  doi          = {10.48550/ARXIV.2212.03533},
  eprinttype    = {arXiv},
  eprint       = {2212.03533},
  timestamp    = {Mon, 01 May 2023 13:01:50 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2212-03533.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{muennighoff2022mteb,
  author       = {Niklas Muennighoff and
                  Nouamane Tazi and
                  Lo{\"{\i}}c Magne and
                  Nils Reimers},
  editor       = {Andreas Vlachos and
                  Isabelle Augenstein},
  title        = {{MTEB:} Massive Text Embedding Benchmark},
  booktitle    = {Proceedings of the 17th Conference of the European Chapter of the
                  Association for Computational Linguistics, {EACL} 2023, Dubrovnik,
                  Croatia, May 2-6, 2023},
  pages        = {2006--2029},
  publisher    = {Association for Computational Linguistics},
  year         = {2023},
  url          = {https://doi.org/10.18653/v1/2023.eacl-main.148},
  doi          = {10.18653/V1/2023.EACL-MAIN.148},
  timestamp    = {Thu, 05 Oct 2023 18:05:03 +0200},
  biburl       = {https://dblp.org/rec/conf/eacl/MuennighoffTMR23.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{wang2023improving,
  author       = {Liang Wang and
                  Nan Yang and
                  Xiaolong Huang and
                  Linjun Yang and
                  Rangan Majumder and
                  Furu Wei},
  title        = {Improving Text Embeddings with Large Language Models},
  journal      = {CoRR},
  volume       = {abs/2401.00368},
  year         = {2024},
  url          = {https://doi.org/10.48550/arXiv.2401.00368},
  doi          = {10.48550/ARXIV.2401.00368},
  eprinttype    = {arXiv},
  eprint       = {2401.00368},
  timestamp    = {Fri, 23 Feb 2024 17:02:25 +0100},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2401-00368.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{honovich-etal-2022-true-evaluating,
    title = "{TRUE}: Re-evaluating Factual Consistency Evaluation",
    author = "Honovich, Or  and
      Aharoni, Roee  and
      Herzig, Jonathan  and
      Taitelbaum, Hagai  and
      Kukliansy, Doron  and
      Cohen, Vered  and
      Scialom, Thomas  and
      Szpektor, Idan  and
      Hassidim, Avinatan  and
      Matias, Yossi",
    booktitle = "Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",
    month = jul,
    year = "2022",
    address = "Seattle, United States",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.naacl-main.287",
    doi = "10.18653/v1/2022.naacl-main.287",
    pages = "3905--3920",
}

@inproceedings{
hu2022lora,
title={Lo{RA}: Low-Rank Adaptation of Large Language Models},
author={Edward J Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen},
booktitle={International Conference on Learning Representations},
year={2022},
url={https://openreview.net/forum?id=nZeVKeeFYf9}
}

@article{jiang2023mistral,
  author       = {Albert Q. Jiang and
                  Alexandre Sablayrolles and
                  Arthur Mensch and
                  Chris Bamford and
                  Devendra Singh Chaplot and
                  Diego de Las Casas and
                  Florian Bressand and
                  Gianna Lengyel and
                  Guillaume Lample and
                  Lucile Saulnier and
                  L{\'{e}}lio Renard Lavaud and
                  Marie{-}Anne Lachaux and
                  Pierre Stock and
                  Teven Le Scao and
                  Thibaut Lavril and
                  Thomas Wang and
                  Timoth{\'{e}}e Lacroix and
                  William El Sayed},
  title        = {Mistral 7B},
  journal      = {CoRR},
  volume       = {abs/2310.06825},
  year         = {2023},
  url          = {https://doi.org/10.48550/arXiv.2310.06825},
  doi          = {10.48550/ARXIV.2310.06825},
  eprinttype    = {arXiv},
  eprint       = {2310.06825},
  timestamp    = {Thu, 26 Oct 2023 16:46:26 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2310-06825.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{touvron2023llama,
  author       = {Hugo Touvron and
                  Louis Martin and
                  Kevin Stone and
                  Peter Albert and
                  Amjad Almahairi and
                  Yasmine Babaei and
                  Nikolay Bashlykov and
                  Soumya Batra and
                  Prajjwal Bhargava and
                  Shruti Bhosale and
                  Dan Bikel and
                  Lukas Blecher and
                  Cristian Canton{-}Ferrer and
                  Moya Chen and
                  Guillem Cucurull and
                  David Esiobu and
                  Jude Fernandes and
                  Jeremy Fu and
                  Wenyin Fu and
                  Brian Fuller and
                  Cynthia Gao and
                  Vedanuj Goswami and
                  Naman Goyal and
                  Anthony Hartshorn and
                  Saghar Hosseini and
                  Rui Hou and
                  Hakan Inan and
                  Marcin Kardas and
                  Viktor Kerkez and
                  Madian Khabsa and
                  Isabel Kloumann and
                  Artem Korenev and
                  Punit Singh Koura and
                  Marie{-}Anne Lachaux and
                  Thibaut Lavril and
                  Jenya Lee and
                  Diana Liskovich and
                  Yinghai Lu and
                  Yuning Mao and
                  Xavier Martinet and
                  Todor Mihaylov and
                  Pushkar Mishra and
                  Igor Molybog and
                  Yixin Nie and
                  Andrew Poulton and
                  Jeremy Reizenstein and
                  Rashi Rungta and
                  Kalyan Saladi and
                  Alan Schelten and
                  Ruan Silva and
                  Eric Michael Smith and
                  Ranjan Subramanian and
                  Xiaoqing Ellen Tan and
                  Binh Tang and
                  Ross Taylor and
                  Adina Williams and
                  Jian Xiang Kuan and
                  Puxin Xu and
                  Zheng Yan and
                  Iliyan Zarov and
                  Yuchen Zhang and
                  Angela Fan and
                  Melanie Kambadur and
                  Sharan Narang and
                  Aur{\'{e}}lien Rodriguez and
                  Robert Stojnic and
                  Sergey Edunov and
                  Thomas Scialom},
  title        = {Llama 2: Open Foundation and Fine-Tuned Chat Models},
  journal      = {CoRR},
  volume       = {abs/2307.09288},
  year         = {2023},
  url          = {https://doi.org/10.48550/arXiv.2307.09288},
  doi          = {10.48550/ARXIV.2307.09288},
  eprinttype    = {arXiv},
  eprint       = {2307.09288},
  timestamp    = {Mon, 28 Aug 2023 21:26:22 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2307-09288.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}


@article{huang2023survey,
  author       = {Lei Huang and
                  Weijiang Yu and
                  Weitao Ma and
                  Weihong Zhong and
                  Zhangyin Feng and
                  Haotian Wang and
                  Qianglong Chen and
                  Weihua Peng and
                  Xiaocheng Feng and
                  Bing Qin and
                  Ting Liu},
  title        = {A Survey on Hallucination in Large Language Models: Principles, Taxonomy,
                  Challenges, and Open Questions},
  journal      = {CoRR},
  volume       = {abs/2311.05232},
  year         = {2023},
  url          = {https://doi.org/10.48550/arXiv.2311.05232},
  doi          = {10.48550/ARXIV.2311.05232},
  eprinttype    = {arXiv},
  eprint       = {2311.05232},
  timestamp    = {Wed, 15 Nov 2023 08:37:27 +0100},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2311-05232.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{OpenAI_GPT4_2023,
  author       = {OpenAI},
  title        = {{GPT-4} Technical Report},
  journal      = {CoRR},
  volume       = {abs/2303.08774},
  year         = {2023},
  url          = {https://doi.org/10.48550/arXiv.2303.08774},
  doi          = {10.48550/ARXIV.2303.08774},
  eprinttype    = {arXiv},
  eprint       = {2303.08774},
  timestamp    = {Mon, 28 Aug 2023 21:26:19 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-2303-08774.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}


@inproceedings{gao2021simcse,
    title = "{S}im{CSE}: Simple Contrastive Learning of Sentence Embeddings",
    author = "Gao, Tianyu  and
      Yao, Xingcheng  and
      Chen, Danqi",
    editor = "Moens, Marie-Francine  and
      Huang, Xuanjing  and
      Specia, Lucia  and
      Yih, Scott Wen-tau",
    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing",
    month = nov,
    year = "2021",
    address = "Online and Punta Cana, Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2021.emnlp-main.552",
    doi = "10.18653/v1/2021.emnlp-main.552",
    pages = "6894--6910",
    abstract = "This paper presents SimCSE, a simple contrastive learning framework that greatly advances the state-of-the-art sentence embeddings. We first describe an unsupervised approach, which takes an input sentence and predicts itself in a contrastive objective, with only standard dropout used as noise. This simple method works surprisingly well, performing on par with previous supervised counterparts. We find that dropout acts as minimal data augmentation and removing it leads to a representation collapse. Then, we propose a supervised approach, which incorporates annotated pairs from natural language inference datasets into our contrastive learning framework, by using {``}entailment{''} pairs as positives and {``}contradiction{''} pairs as hard negatives. We evaluate SimCSE on standard semantic textual similarity (STS) tasks, and our unsupervised and supervised models using BERT base achieve an average of 76.3{\%} and 81.6{\%} Spearman{'}s correlation respectively, a 4.2{\%} and 2.2{\%} improvement compared to previous best results. We also show{---}both theoretically and empirically{---}that contrastive learning objective regularizes pre-trained embeddings{'} anisotropic space to be more uniform, and it better aligns positive pairs when supervised signals are available.",
}

@inproceedings{papineni2002bleu,
    title = "{B}leu: a Method for Automatic Evaluation of Machine Translation",
    author = "Papineni, Kishore  and
      Roukos, Salim  and
      Ward, Todd  and
      Zhu, Wei-Jing",
    editor = "Isabelle, Pierre  and
      Charniak, Eugene  and
      Lin, Dekang",
    booktitle = "Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2002",
    address = "Philadelphia, Pennsylvania, USA",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/P02-1040",
    doi = "10.3115/1073083.1073135",
    pages = "311--318",
}

@inproceedings{popovic2015chrf,
    title = "chr{F}: character n-gram {F}-score for automatic {MT} evaluation",
    author = "Popovi{\'c}, Maja",
    editor = "Bojar, Ond{\v{r}}ej  and
      Chatterjee, Rajan  and
      Federmann, Christian  and
      Haddow, Barry  and
      Hokamp, Chris  and
      Huck, Matthias  and
      Logacheva, Varvara  and
      Pecina, Pavel",
    booktitle = "Proceedings of the Tenth Workshop on Statistical Machine Translation",
    month = sep,
    year = "2015",
    address = "Lisbon, Portugal",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/W15-3049",
    doi = "10.18653/v1/W15-3049",
    pages = "392--395",
}

@inproceedings{banerjee2005meteor,
  title={METEOR: An automatic metric for MT evaluation with improved correlation with human judgments},
  author={Banerjee, Satanjeev and Lavie, Alon},
  booktitle={Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and/or summarization},
  pages={65--72},
  year={2005}
}


@inproceedings{sellam2020bleurt,
    title = "{BLEURT}: Learning Robust Metrics for Text Generation",
    author = "Sellam, Thibault  and
      Das, Dipanjan  and
      Parikh, Ankur",
    editor = "Jurafsky, Dan  and
      Chai, Joyce  and
      Schluter, Natalie  and
      Tetreault, Joel",
    booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics",
    month = jul,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.acl-main.704",
    doi = "10.18653/v1/2020.acl-main.704",
    pages = "7881--7892",
    abstract = "Text generation has made significant advances in the last few years. Yet, evaluation metrics have lagged behind, as the most popular choices (e.g., BLEU and ROUGE) may correlate poorly with human judgment. We propose BLEURT, a learned evaluation metric for English based on BERT. BLEURT can model human judgment with a few thousand possibly biased training examples. A key aspect of our approach is a novel pre-training scheme that uses millions of synthetic examples to help the model generalize. BLEURT provides state-of-the-art results on the last three years of the WMT Metrics shared task and the WebNLG data set. In contrast to a vanilla BERT-based approach, it yields superior results even when the training data is scarce and out-of-distribution.",
}

@InProceedings{mickus-etal-2024-semeval, 
author = {Mickus, Timothee and Zosa, Elaine and Vázquez, Raúl and Vahtola, Teemu and Tiedemann, Jörg and Segonne, Vincent and Raganato, Alessandro and Apidianaki, Marianna}, 
title = {SemEval-2024 Shared Task 6: SHROOM, a Shared-task on Hallucinations and Related Observable Overgeneration Mistakes}, 
booktitle = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)}, 
month = {June}, 
year = {2024}, 
address = {Mexico City, Mexico}, 
publisher = {Association for Computational Linguistics}, 
pages = {1980--1994}, 
abstract = {This paper presents the results of the SHROOM, a shared task focused on detecting hallucinations: outputs from natural language generation (NLG) systems that are fluent, yet inaccurate. Such cases of overgeneration put in jeopardy many NLG applications, where correctness is often mission-critical. The shared task was conducted with a newly constructed dataset of 4000 model outputs labeled by 5 annotators each, spanning 3 NLP tasks: machine translation, paraphrase generation and definition modeling. The shared task was tackled by a total of 58 different users grouped in 42 teams, out of which 27 elected to write a system description paper; collectively, they submitted over 300 prediction sets on both tracks of the shared task. We observe a number of key trends in how this approach was tackled—many participants rely on a handful of model, and ofte! n rely either on synthetic data for fine-tuning or zero-shot prompting strategies. While a majority of the teams did outperform our proposed baseline system, the performances of top-scoring systems are still consistent with a random handling of the more challenging items.}, 
url = {https://aclanthology.org/2024.semeval2024-1.270} 
}