
@misc{manakul_selfcheckgpt_2023,
	title = {{SelfCheckGPT}: {Zero}-{Resource} {Black}-{Box} {Hallucination} {Detection} for {Generative} {Large} {Language} {Models}},
	shorttitle = {{SelfCheckGPT}},
	url = {http://arxiv.org/abs/2303.08896},
	abstract = {Generative Large Language Models (LLMs) such as GPT-3 are capable of generating highly fluent responses to a wide variety of user prompts. However, LLMs are known to hallucinate facts and make non-factual statements which can undermine trust in their output. Existing fact-checking approaches either require access to the output probability distribution (which may not be available for systems such as ChatGPT) or external databases that are interfaced via separate, often complex, modules. In this work, we propose "SelfCheckGPT", a simple sampling-based approach that can be used to fact-check the responses of black-box models in a zero-resource fashion, i.e. without an external database. SelfCheckGPT leverages the simple idea that if an LLM has knowledge of a given concept, sampled responses are likely to be similar and contain consistent facts. However, for hallucinated facts, stochastically sampled responses are likely to diverge and contradict one another. We investigate this approach by using GPT-3 to generate passages about individuals from the WikiBio dataset, and manually annotate the factuality of the generated passages. We demonstrate that SelfCheckGPT can: i) detect non-factual and factual sentences; and ii) rank passages in terms of factuality. We compare our approach to several baselines and show that our approach has considerably higher AUC-PR scores in sentence-level hallucination detection and higher correlation scores in passage-level factuality assessment compared to grey-box methods.},
	urldate = {2024-02-16},
	publisher = {arXiv},
	author = {Manakul, Potsawee and Liusie, Adian and Gales, Mark J. F.},
	month = oct,
	year = {2023},
	note = {arXiv:2303.08896 [cs]},
	keywords = {Computer Science - Computation and Language},
}

@misc{noauthor_edinburghnlpawesome-hallucination-detection_2024,
	title = {{EdinburghNLP}/awesome-hallucination-detection},
	copyright = {Apache-2.0},
	url = {https://github.com/EdinburghNLP/awesome-hallucination-detection},
	abstract = {List of papers on hallucination detection in LLMs.},
	urldate = {2024-02-11},
	publisher = {Edinburgh NLP},
	month = feb,
	year = {2024},
	note = {original-date: 2023-09-15T18:23:21Z},
	keywords = {hallucinations, llms, nlp},
}

@inproceedings{guerreiro_looking_2023,
	address = {Dubrovnik, Croatia},
	title = {Looking for a {Needle} in a {Haystack}: {A} {Comprehensive} {Study} of {Hallucinations} in {Neural} {Machine} {Translation}},
	shorttitle = {Looking for a {Needle} in a {Haystack}},
	url = {https://aclanthology.org/2023.eacl-main.75},
	doi = {10.18653/v1/2023.eacl-main.75},
	abstract = {Although the problem of hallucinations in neural machine translation (NMT) has received some attention, research on this highly pathological phenomenon lacks solid ground. Previous work has been limited in several ways: it often resorts to artificial settings where the problem is amplified, it disregards some (common) types of hallucinations, and it does not validate adequacy of detection heuristics. In this paper, we set foundations for the study of NMT hallucinations. First, we work in a natural setting, i.e., in-domain data without artificial noise neither in training nor in inference. Next, we annotate a dataset of over 3.4k sentences indicating different kinds of critical errors and hallucinations. Then, we turn to detection methods and both revisit methods used previously and propose using glass-box uncertainty-based detectors. Overall, we show that for preventive settings, (i) previously used methods are largely inadequate, (ii) sequence log-probability works best and performs on par with reference-based methods. Finally, we propose DeHallucinator, a simple method for alleviating hallucinations at test time that significantly reduces the hallucinatory rate.},
	urldate = {2024-02-10},
	booktitle = {Proceedings of the 17th {Conference} of the {European} {Chapter} of the {Association} for {Computational} {Linguistics}},
	publisher = {Association for Computational Linguistics},
	author = {Guerreiro, Nuno M. and Voita, Elena and Martins, André},
	editor = {Vlachos, Andreas and Augenstein, Isabelle},
	month = may,
	year = {2023},
	pages = {1059--1075},
}

@misc{friel_chainpoll_2023,
	title = {Chainpoll: {A} high efficacy method for {LLM} hallucination detection},
	shorttitle = {Chainpoll},
	url = {http://arxiv.org/abs/2310.18344},
	abstract = {Large language models (LLMs) have experienced notable advancements in generating coherent and contextually relevant responses. However, hallucinations - incorrect or unfounded claims - are still prevalent, prompting the creation of automated metrics to detect these in LLM outputs. Our contributions include: introducing ChainPoll, an innovative hallucination detection method that excels compared to its counterparts, and unveiling RealHall, a refined collection of benchmark datasets to assess hallucination detection metrics from recent studies. While creating RealHall, we assessed tasks and datasets from previous hallucination detection studies and observed that many are not suitable for the potent LLMs currently in use. Overcoming this, we opted for four datasets challenging for modern LLMs and pertinent to real-world scenarios. Using RealHall, we conducted a comprehensive comparison of ChainPoll with numerous hallucination metrics from recent studies. Our findings indicate that ChainPoll outperforms in all RealHall benchmarks, achieving an overall AUROC of 0.781. This surpasses the next best theoretical method by 11\% and exceeds industry standards by over 23\%. Additionally, ChainPoll is cost-effective and offers greater transparency than other metrics. We introduce two novel metrics to assess LLM hallucinations: Adherence and Correctness. Adherence is relevant to Retrieval Augmented Generation workflows, evaluating an LLM's analytical capabilities within given documents and contexts. In contrast, Correctness identifies logical and reasoning errors.},
	urldate = {2024-02-10},
	publisher = {arXiv},
	author = {Friel, Robert and Sanyal, Atindriyo},
	month = oct,
	year = {2023},
	note = {arXiv:2310.18344 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language, Computer Science - Machine Learning},
}

@inproceedings{yang_hierarchical_2016,
	address = {San Diego, California},
	title = {Hierarchical {Attention} {Networks} for {Document} {Classification}},
	url = {http://aclweb.org/anthology/N16-1174},
	doi = {10.18653/v1/N16-1174},
	abstract = {We propose a hierarchical attention network for document classiﬁcation. Our model has two distinctive characteristics: (i) it has a hierarchical structure that mirrors the hierarchical structure of documents; (ii) it has two levels of attention mechanisms applied at the wordand sentence-level, enabling it to attend differentially to more and less important content when constructing the document representation. Experiments conducted on six large scale text classiﬁcation tasks demonstrate that the proposed architecture outperform previous methods by a substantial margin. Visualization of the attention layers illustrates that the model selects qualitatively informative words and sentences.},
	language = {en},
	urldate = {2024-02-09},
	booktitle = {Proceedings of the 2016 {Conference} of the {North} {American} {Chapter} of the {Association} for {Computational} {Linguistics}: {Human} {Language} {Technologies}},
	publisher = {Association for Computational Linguistics},
	author = {Yang, Zichao and Yang, Diyi and Dyer, Chris and He, Xiaodong and Smola, Alex and Hovy, Eduard},
	year = {2016},
	pages = {1480--1489},
}

@misc{chen_label-aware_2023,
	title = {Label-{Aware} {Hyperbolic} {Embeddings} for {Fine}-grained {Emotion} {Classification}},
	url = {http://arxiv.org/abs/2306.14822},
	abstract = {Fine-grained emotion classification (FEC) is a challenging task. Specifically, FEC needs to handle subtle nuance between labels, which can be complex and confusing. Most existing models only address text classification problem in the euclidean space, which we believe may not be the optimal solution as labels of close semantic (e.g., afraid and terrified) may not be differentiated in such space, which harms the performance. In this paper, we propose HypEmo, a novel framework that can integrate hyperbolic embeddings to improve the FEC task. First, we learn label embeddings in the hyperbolic space to better capture their hierarchical structure, and then our model projects contextualized representations to the hyperbolic space to compute the distance between samples and labels. Experimental results show that incorporating such distance to weight cross entropy loss substantially improves the performance with significantly higher efficiency. We evaluate our proposed model on two benchmark datasets and found 4.8\% relative improvement compared to the previous state of the art with 43.2\% fewer parameters and 76.9\% less training time. Code is available at https: //github.com/dinobby/HypEmo.},
	urldate = {2024-02-09},
	publisher = {arXiv},
	author = {Chen, Chih-Yao and Hung, Tun-Min and Hsu, Yi-Li and Ku, Lun-Wei},
	month = jun,
	year = {2023},
	note = {arXiv:2306.14822 [cs]},
	keywords = {Computer Science - Artificial Intelligence, Computer Science - Computation and Language},
}

@inproceedings{assylbekov_hyperbolic_2022,
	address = {Dublin, Ireland},
	title = {From {Hyperbolic} {Geometry} {Back} to {Word} {Embeddings}},
	url = {https://aclanthology.org/2022.repl4nlp-1.5},
	doi = {10.18653/v1/2022.repl4nlp-1.5},
	abstract = {We choose random points in the hyperbolic disc and claim that these points are already word representations. However, it is yet to be uncovered which point corresponds to which word of the human language of interest. This correspondence can be approximately established using a pointwise mutual information between words and recent alignment techniques.},
	urldate = {2024-02-09},
	booktitle = {Proceedings of the 7th {Workshop} on {Representation} {Learning} for {NLP}},
	publisher = {Association for Computational Linguistics},
	author = {Assylbekov, Zhenisbek and Nurmukhamedov, Sultan and Sheverdin, Arsen and Mach, Thomas},
	editor = {Gella, Spandana and He, He and Majumder, Bodhisattwa Prasad and Can, Burcu and Giunchiglia, Eleonora and Cahyawijaya, Samuel and Min, Sewon and Mozes, Maximilian and Li, Xiang Lorraine and Augenstein, Isabelle and Rogers, Anna and Cho, Kyunghyun and Grefenstette, Edward and Rimell, Laura and Dyer, Chris},
	month = may,
	year = {2022},
	pages = {39--45},
}

@misc{ganea_hyperbolic_2018,
	title = {Hyperbolic {Neural} {Networks}},
	url = {http://arxiv.org/abs/1805.09112},
	abstract = {Hyperbolic spaces have recently gained momentum in the context of machine learning due to their high capacity and tree-likeliness properties. However, the representational power of hyperbolic geometry is not yet on par with Euclidean geometry, mostly because of the absence of corresponding hyperbolic neural network layers. This makes it hard to use hyperbolic embeddings in downstream tasks. Here, we bridge this gap in a principled manner by combining the formalism of M{\textbackslash}"obius gyrovector spaces with the Riemannian geometry of the Poincar{\textbackslash}'e model of hyperbolic spaces. As a result, we derive hyperbolic versions of important deep learning tools: multinomial logistic regression, feed-forward and recurrent neural networks such as gated recurrent units. This allows to embed sequential data and perform classification in the hyperbolic space. Empirically, we show that, even if hyperbolic optimization tools are limited, hyperbolic sentence embeddings either outperform or are on par with their Euclidean variants on textual entailment and noisy-prefix recognition tasks.},
	urldate = {2024-02-09},
	publisher = {arXiv},
	author = {Ganea, Octavian-Eugen and Bécigneul, Gary and Hofmann, Thomas},
	month = jun,
	year = {2018},
	note = {arXiv:1805.09112 [cs, stat]},
	keywords = {Computer Science - Machine Learning, Statistics - Machine Learning},
}

@misc{peng_hyperbolic_2021,
	title = {Hyperbolic {Deep} {Neural} {Networks}: {A} {Survey}},
	shorttitle = {Hyperbolic {Deep} {Neural} {Networks}},
	url = {http://arxiv.org/abs/2101.04562},
	abstract = {Recently, there has been a rising surge of momentum for deep representation learning in hyperbolic spaces due to theirhigh capacity of modeling data like knowledge graphs or synonym hierarchies, possessing hierarchical structure. We refer to the model as hyperbolic deep neural network in this paper. Such a hyperbolic neural architecture potentially leads to drastically compact model withmuch more physical interpretability than its counterpart in Euclidean space. To stimulate future research, this paper presents acoherent and comprehensive review of the literature around the neural components in the construction of hyperbolic deep neuralnetworks, as well as the generalization of the leading deep approaches to the Hyperbolic space. It also presents current applicationsaround various machine learning tasks on several publicly available datasets, together with insightful observations and identifying openquestions and promising future directions.},
	urldate = {2024-02-09},
	publisher = {arXiv},
	author = {Peng, Wei and Varanka, Tuomas and Mostafa, Abdelrahman and Shi, Henglin and Zhao, Guoying},
	month = feb,
	year = {2021},
	note = {arXiv:2101.04562 [cs]},
	keywords = {Computer Science - Computer Vision and Pattern Recognition, Computer Science - Machine Learning},
}

@inproceedings{chen_fully_2022,
	address = {Dublin, Ireland},
	title = {Fully {Hyperbolic} {Neural} {Networks}},
	url = {https://aclanthology.org/2022.acl-long.389},
	doi = {10.18653/v1/2022.acl-long.389},
	abstract = {Hyperbolic neural networks have shown great potential for modeling complex data. However, existing hyperbolic networks are not completely hyperbolic, as they encode features in the hyperbolic space yet formalize most of their operations in the tangent space (a Euclidean subspace) at the origin of the hyperbolic model. This hybrid method greatly limits the modeling ability of networks. In this paper, we propose a fully hyperbolic framework to build hyperbolic networks based on the Lorentz model by adapting the Lorentz transformations (including boost and rotation) to formalize essential operations of neural networks. Moreover, we also prove that linear transformation in tangent spaces used by existing hyperbolic networks is a relaxation of the Lorentz rotation and does not include the boost, implicitly limiting the capabilities of existing hyperbolic networks. The experimental results on four NLP tasks show that our method has better performance for building both shallow and deep networks. Our code will be released to facilitate follow-up research.},
	urldate = {2024-02-09},
	booktitle = {Proceedings of the 60th {Annual} {Meeting} of the {Association} for {Computational} {Linguistics} ({Volume} 1: {Long} {Papers})},
	publisher = {Association for Computational Linguistics},
	author = {Chen, Weize and Han, Xu and Lin, Yankai and Zhao, Hexu and Liu, Zhiyuan and Li, Peng and Sun, Maosong and Zhou, Jie},
	editor = {Muresan, Smaranda and Nakov, Preslav and Villavicencio, Aline},
	month = may,
	year = {2022},
	pages = {5672--5686},
}

@misc{tifrea_poincare_2018,
	title = {Poincar{\textbackslash}'e {GloVe}: {Hyperbolic} {Word} {Embeddings}},
	shorttitle = {Poincar{\textbackslash}'e {GloVe}},
	url = {http://arxiv.org/abs/1810.06546},
	abstract = {Words are not created equal. In fact, they form an aristocratic graph with a latent hierarchical structure that the next generation of unsupervised learned word embeddings should reveal. In this paper, justified by the notion of delta-hyperbolicity or tree-likeliness of a space, we propose to embed words in a Cartesian product of hyperbolic spaces which we theoretically connect to the Gaussian word embeddings and their Fisher geometry. This connection allows us to introduce a novel principled hypernymy score for word embeddings. Moreover, we adapt the well-known Glove algorithm to learn unsupervised word embeddings in this type of Riemannian manifolds. We further explain how to solve the analogy task using the Riemannian parallel transport that generalizes vector arithmetics to this new type of geometry. Empirically, based on extensive experiments, we prove that our embeddings, trained unsupervised, are the first to simultaneously outperform strong and popular baselines on the tasks of similarity, analogy and hypernymy detection. In particular, for word hypernymy, we obtain new state-of-the-art on fully unsupervised WBLESS classification accuracy.},
	urldate = {2024-02-08},
	publisher = {arXiv},
	author = {Tifrea, Alexandru and Bécigneul, Gary and Ganea, Octavian-Eugen},
	month = nov,
	year = {2018},
	note = {arXiv:1810.06546 [cs]},
	keywords = {Computer Science - Computation and Language},
}

@inproceedings{ulcar_multilingual_2020,
	address = {Marseille, France},
	title = {Multilingual {Culture}-{Independent} {Word} {Analogy} {Datasets}},
	isbn = {979-10-95546-34-4},
	url = {https://aclanthology.org/2020.lrec-1.501},
	abstract = {In text processing, deep neural networks mostly use word embeddings as an input. Embeddings have to ensure that relations between words are reflected through distances in a high-dimensional numeric space. To compare the quality of different text embeddings, typically, we use benchmark datasets. We present a collection of such datasets for the word analogy task in nine languages: Croatian, English, Estonian, Finnish, Latvian, Lithuanian, Russian, Slovenian, and Swedish. We designed the monolingual analogy task to be much more culturally independent and also constructed cross-lingual analogy datasets for the involved languages. We present basic statistics of the created datasets and their initial evaluation using fastText embeddings.},
	language = {English},
	urldate = {2024-02-08},
	booktitle = {Proceedings of the {Twelfth} {Language} {Resources} and {Evaluation} {Conference}},
	publisher = {European Language Resources Association},
	author = {Ulčar, Matej and Vaik, Kristiina and Lindström, Jessica and Dailidėnaitė, Milda and Robnik-Šikonja, Marko},
	editor = {Calzolari, Nicoletta and Béchet, Frédéric and Blache, Philippe and Choukri, Khalid and Cieri, Christopher and Declerck, Thierry and Goggi, Sara and Isahara, Hitoshi and Maegaard, Bente and Mariani, Joseph and Mazo, Hélène and Moreno, Asuncion and Odijk, Jan and Piperidis, Stelios},
	month = may,
	year = {2020},
	pages = {4074--4080},
}

@misc{noauthor_processing_2020,
	title = {Processing {Wikipedia} in a few hours on a single {PC}},
	url = {https://jamesthorne.com/blog/processing-wikipedia-in-a-couple-of-hours/},
	abstract = {Wikipedia is a valuable resource to build Natural Language Processing systems
for tasks such as Question Answering [https://github.com/facebookresearch/DrQA] 
and Fact Verification [http://fever.ai]. However, the sheer size of the resource
can become an obstacle for new starters or those who think they haven't got the},
	language = {en},
	urldate = {2024-02-06},
	journal = {James Thorne},
	month = jan,
	year = {2020},
}

@misc{noauthor_into_2019,
	title = {Into the {Wild}: {Machine} {Learning} {In} {Non}-{Euclidean} {Spaces} · {Stanford} {DAWN}},
	shorttitle = {Into the {Wild}},
	url = {https://dawn.cs.stanford.edu/2019/10/10/noneuclidean/},
	language = {en},
	urldate = {2024-02-06},
	month = oct,
	year = {2019},
}

@misc{noauthor_cut_2023,
	title = {Cut the {Bull}…. {Detecting} {Hallucinations} in {Large} {Language} {Models}},
	url = {https://vectara.com/blog/cut-the-bull-detecting-hallucinations-in-large-language-models/},
	abstract = {Vectara launches the open source Hughes Hallucination Evaluation Model (H2EM) and uses it to compare hallucination rates across top LLMs including OpenAI, Cohere, PaLM, Anthropic’s Claude 2 and more.},
	language = {en-US},
	urldate = {2024-02-06},
	journal = {Vectara},
	month = nov,
	year = {2023},
}

@article{gu_learning_2019,
	title = {{LEARNING} {MIXED}-{CURVATURE} {REPRESENTATIONS} {IN} {PRODUCTS} {OF} {MODEL} {SPACES}},
	abstract = {The quality of the representations achieved by embeddings is determined by how well the geometry of the embedding space matches the structure of the data. Euclidean space has been the workhorse for embeddings; recently hyperbolic and spherical spaces have gained popularity due to their ability to better embed new types of structured data—such as hierarchical data—but most data is not structured so uniformly. We address this problem by proposing learning embeddings in a product manifold combining multiple copies of these model spaces (spherical, hyperbolic, Euclidean), providing a space of heterogeneous curvature suitable for a wide variety of structures. We introduce a heuristic to estimate the sectional curvature of graph data and directly determine an appropriate signature—the number of component spaces and their dimensions—of the product manifold. Empirically, we jointly learn the curvature and the embedding in the product space via Riemannian optimization. We discuss how to deﬁne and compute intrinsic quantities such as means—a challenging notion for product manifolds—and provably learnable optimization functions. On a range of datasets and reconstruction tasks, our product space embeddings outperform single Euclidean or hyperbolic spaces used in previous works, reducing distortion by 32.55\% on a Facebook social network dataset. We learn word embeddings and ﬁnd that a product of hyperbolic spaces in 50 dimensions consistently improves on baseline Euclidean and hyperbolic embeddings, by 2.6 points in Spearman rank correlation on similarity tasks and 3.4 points on analogy accuracy.},
	language = {en},
	author = {Gu, Albert and Sala, Frederic and Gunel, Beliz and Re, Christopher},
	year = {2019},
}

@inproceedings{sala_representation_2018,
	title = {Representation {Tradeoffs} for {Hyperbolic} {Embeddings}},
	url = {https://proceedings.mlr.press/v80/sala18a.html},
	abstract = {Hyperbolic embeddings offer excellent quality with few dimensions when embedding hierarchical data structures. We give a combinatorial construction that embeds trees into hyperbolic space with arbitrarily low distortion without optimization. On WordNet, this algorithm obtains a mean-average-precision of 0.989 with only two dimensions, outperforming existing work by 0.11 points. We provide bounds characterizing the precision-dimensionality tradeoff inherent in any hyperbolic embedding. To embed general metric spaces, we propose a hyperbolic generalization of multidimensional scaling (h-MDS). We show how to perform exact recovery of hyperbolic points from distances, provide a perturbation analysis, and give a recovery result that enables us to reduce dimensionality. Finally, we extract lessons from the algorithms and theory above to design a scalable PyTorch-based implementation that can handle incomplete information.},
	language = {en},
	urldate = {2024-02-06},
	booktitle = {Proceedings of the 35th {International} {Conference} on {Machine} {Learning}},
	publisher = {PMLR},
	author = {Sala, Frederic and Sa, Chris De and Gu, Albert and Re, Christopher},
	month = jul,
	year = {2018},
	note = {ISSN: 2640-3498},
	pages = {4460--4469},
}

@inproceedings{zhu_hypertext_2020,
	address = {Online},
	title = {{HyperText}: {Endowing} {FastText} with {Hyperbolic} {Geometry}},
	shorttitle = {{HyperText}},
	url = {https://aclanthology.org/2020.findings-emnlp.104},
	doi = {10.18653/v1/2020.findings-emnlp.104},
	abstract = {Natural language data exhibit tree-like hierarchical structures such as the hypernym-hyponym hierarchy in WordNet. FastText, as the state-of-the-art text classifier based on shallow neural network in Euclidean space, may not represent such hierarchies precisely with limited representation capacity. Considering that hyperbolic space is naturally suitable for modelling tree-like hierarchical data, we propose a new model named HyperText for efficient text classification by endowing FastText with hyperbolic geometry. Empirically, we show that HyperText outperforms FastText on a range of text classification tasks with much reduced parameters.},
	urldate = {2024-02-06},
	booktitle = {Findings of the {Association} for {Computational} {Linguistics}: {EMNLP} 2020},
	publisher = {Association for Computational Linguistics},
	author = {Zhu, Yudong and Zhou, Di and Xiao, Jinghui and Jiang, Xin and Chen, Xiao and Liu, Qun},
	editor = {Cohn, Trevor and He, Yulan and Liu, Yang},
	month = nov,
	year = {2020},
	pages = {1166--1171},
}

@misc{geshkovski_mathematical_2023,
	title = {A mathematical perspective on {Transformers}},
	url = {http://arxiv.org/abs/2312.10794},
	abstract = {Transformers play a central role in the inner workings of large language models. We develop a mathematical framework for analyzing Transformers based on their interpretation as interacting particle systems, which reveals that clusters emerge in long time. Our study explores the underlying theory and offers new perspectives for mathematicians as well as computer scientists.},
	urldate = {2024-01-27},
	publisher = {arXiv},
	author = {Geshkovski, Borjan and Letrouit, Cyril and Polyanskiy, Yury and Rigollet, Philippe},
	month = dec,
	year = {2023},
	note = {arXiv:2312.10794 [cs, math]},
	keywords = {Computer Science - Machine Learning, Mathematics - Analysis of PDEs, Mathematics - Dynamical Systems},
}

@misc{zhang_bertscore_2020,
	title = {{BERTScore}: {Evaluating} {Text} {Generation} with {BERT}},
	shorttitle = {{BERTScore}},
	url = {http://arxiv.org/abs/1904.09675},
	doi = {10.48550/arXiv.1904.09675},
	abstract = {We propose BERTScore, an automatic evaluation metric for text generation. Analogously to common metrics, BERTScore computes a similarity score for each token in the candidate sentence with each token in the reference sentence. However, instead of exact matches, we compute token similarity using contextual embeddings. We evaluate using the outputs of 363 machine translation and image captioning systems. BERTScore correlates better with human judgments and provides stronger model selection performance than existing metrics. Finally, we use an adversarial paraphrase detection task to show that BERTScore is more robust to challenging examples when compared to existing metrics.},
	urldate = {2024-01-18},
	publisher = {arXiv},
	author = {Zhang, Tianyi and Kishore, Varsha and Wu, Felix and Weinberger, Kilian Q. and Artzi, Yoav},
	month = feb,
	year = {2020},
	note = {arXiv:1904.09675 [cs]},
	keywords = {Computer Science - Computation and Language},
}

@inproceedings{yuan_bartscore_2021,
	title = {{BARTScore}: {Evaluating} {Generated} {Text} as {Text} {Generation}},
	shorttitle = {{BARTScore}},
	url = {https://openreview.net/forum?id=5Ya8PbvpZ9},
	abstract = {A wide variety of NLP applications, such as machine translation, summarization, and dialog, involve text generation. One major challenge for these applications is how to evaluate whether such generated texts are actually fluent, accurate, or effective. In this work, we conceptualize the evaluation of generated text as a text generation problem, modeled using pre-trained sequence-to-sequence models. The general idea is that models trained to convert the generated text to/from a reference output or the source text will achieve higher scores when the generated text is better. We operationalize this idea using BART, an encoder-decoder based pre-trained model, and propose a metric BARTScore with a number of variants that can be flexibly applied in an unsupervised fashion to evaluation of text from different perspectives (e.g. informativeness, fluency, or factuality). BARTScore is conceptually simple and empirically effective. It can outperform existing top-scoring metrics in 16 of 22 test settings, covering evaluation of 16 datasets (e.g., machine translation, text summarization) and 7 different perspectives (e.g., informativeness, factuality). Code to calculate BARTScore is available at https://github.com/neulab/BARTScore, and we have released an interactive leaderboard for meta-evaluation at http://explainaboard.nlpedia.ai/leaderboard/task-meval/ on the ExplainaBoard platform, which allows us to interactively understand the strengths, weaknesses, and complementarity of each metric.},
	language = {en},
	urldate = {2024-01-18},
	author = {Yuan, Weizhe and Neubig, Graham and Liu, Pengfei},
	month = nov,
	year = {2021},
}

@inproceedings{deng_compression_2021,
	address = {Online and Punta Cana, Dominican Republic},
	title = {Compression, {Transduction}, and {Creation}: {A} {Unified} {Framework} for {Evaluating} {Natural} {Language} {Generation}},
	shorttitle = {Compression, {Transduction}, and {Creation}},
	url = {https://aclanthology.org/2021.emnlp-main.599},
	doi = {10.18653/v1/2021.emnlp-main.599},
	abstract = {Natural language generation (NLG) spans a broad range of tasks, each of which serves for specific objectives and desires different properties of generated text. The complexity makes automatic evaluation of NLG particularly challenging. Previous work has typically focused on a single task and developed individual evaluation metrics based on specific intuitions. In this paper, we propose a unifying perspective based on the nature of information change in NLG tasks, including compression (e.g., summarization), transduction (e.g., text rewriting), and creation (e.g., dialog). \_Information alignment\_ between input, context, and output text plays a common central role in characterizing the generation. With automatic alignment prediction models, we develop a family of interpretable metrics that are suitable for evaluating key aspects of different NLG tasks, often without need of gold reference data. Experiments show the uniformly designed metrics achieve stronger or comparable correlations with human judgement compared to state-of-the-art metrics in each of diverse tasks, including text summarization, style transfer, and knowledge-grounded dialog.},
	urldate = {2024-01-18},
	booktitle = {Proceedings of the 2021 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing}},
	publisher = {Association for Computational Linguistics},
	author = {Deng, Mingkai and Tan, Bowen and Liu, Zhengzhong and Xing, Eric and Hu, Zhiting},
	editor = {Moens, Marie-Francine and Huang, Xuanjing and Specia, Lucia and Yih, Scott Wen-tau},
	month = nov,
	year = {2021},
	pages = {7580--7605},
}

@inproceedings{honovich_true_2022,
	address = {Seattle, United States},
	title = {{TRUE}: {Re}-evaluating {Factual} {Consistency} {Evaluation}},
	shorttitle = {{TRUE}},
	url = {https://aclanthology.org/2022.naacl-main.287},
	doi = {10.18653/v1/2022.naacl-main.287},
	abstract = {Grounded text generation systems often generate text that contains factual inconsistencies, hindering their real-world applicability. Automatic factual consistency evaluation may help alleviate this limitation by accelerating evaluation cycles, filtering inconsistent outputs and augmenting training data. While attracting increasing attention, such evaluation metrics are usually developed and evaluated in silo for a single task or dataset, slowing their adoption. Moreover, previous meta-evaluation protocols focused on system-level correlations with human annotations, which leave the example-level accuracy of such metrics unclear. In this work, we introduce TRUE: a comprehensive survey and assessment of factual consistency metrics on a standardized collection of existing texts from diverse tasks, manually annotated for factual consistency. Our standardization enables an example-level meta-evaluation protocol that is more actionable and interpretable than previously reported correlations, yielding clearer quality measures. Across diverse state-of-the-art metrics and 11 datasets we find that large-scale NLI and question generation-and-answering-based approaches achieve strong and complementary results. We recommend those methods as a starting point for model and metric developers, and hope TRUE will foster progress towards even better evaluation methods.},
	urldate = {2024-01-18},
	booktitle = {Proceedings of the 2022 {Conference} of the {North} {American} {Chapter} of the {Association} for {Computational} {Linguistics}: {Human} {Language} {Technologies}},
	publisher = {Association for Computational Linguistics},
	author = {Honovich, Or and Aharoni, Roee and Herzig, Jonathan and Taitelbaum, Hagai and Kukliansy, Doron and Cohen, Vered and Scialom, Thomas and Szpektor, Idan and Hassidim, Avinatan and Matias, Yossi},
	editor = {Carpuat, Marine and de Marneffe, Marie-Catherine and Meza Ruiz, Ivan Vladimir},
	month = jul,
	year = {2022},
	pages = {3905--3920},
}

@inproceedings{opitz_amr4nli_2023,
	address = {Nancy, France},
	title = {{AMR4NLI}: {Interpretable} and robust {NLI} measures from semantic graphs},
	shorttitle = {{AMR4NLI}},
	url = {https://aclanthology.org/2023.iwcs-1.29},
	abstract = {The task of natural language inference (NLI) asks whether a given premise (expressed in NL) entails a given NL hypothesis. NLI benchmarks contain human ratings of entailment, but the meaning relationships driving these ratings are not formalized. Can the underlying sentence pair relationships be made more explicit in an interpretable yet robust fashion? We compare semantic structures to represent premise and hypothesis, including sets of *contextualized embeddings* and *semantic graphs* (Abstract Meaning Representations), and measure whether the hypothesis is a semantic substructure of the premise, utilizing interpretable metrics. Our evaluation on three English benchmarks finds value in both contextualized embeddings and semantic graphs; moreover, they provide complementary signals, and can be leveraged together in a hybrid model.},
	urldate = {2024-01-12},
	booktitle = {Proceedings of the 15th {International} {Conference} on {Computational} {Semantics}},
	publisher = {Association for Computational Linguistics},
	author = {Opitz, Juri and Wein, Shira and Steen, Julius and Frank, Anette and Schneider, Nathan},
	editor = {Amblard, Maxime and Breitholtz, Ellen},
	month = jun,
	year = {2023},
	pages = {275--283},
}

@misc{nieradzik_implementing_2020,
	title = {Implementing {Poincaré} {Embeddings} in {PyTorch}},
	url = {https://lars76.github.io/2020/07/24/implementing-poincare-embedding.html},
	abstract = {After having introduced Riemannian SGD in the last blog post, here I will give a concrete application for this optimization method. Poincaré embeddings [1][2] are hierarchical word embeddings which map integer-encoded words to the hyperbolic space.},
	language = {en},
	urldate = {2024-01-12},
	journal = {lars76.github.io},
	author = {Nieradzik, Lars},
	month = jul,
	year = {2020},
}

@misc{noauthor_ieee_nodate,
	title = {{IEEE} {Xplore} {Full}-{Text} {PDF}:},
	url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9395439},
	urldate = {2024-01-12},
}

@inproceedings{ribeiro_enhancing_2019,
	address = {Hong Kong, China},
	title = {Enhancing {AMR}-to-{Text} {Generation} with {Dual} {Graph} {Representations}},
	url = {https://www.aclweb.org/anthology/D19-1314},
	doi = {10.18653/v1/D19-1314},
	abstract = {Generating text from graph-based data, such as Abstract Meaning Representation (AMR), is a challenging task due to the inherent difﬁculty in how to properly encode the structure of a graph with labeled edges. To address this difﬁculty, we propose a novel graph-to-sequence model that encodes different but complementary perspectives of the structural information contained in the AMR graph. The model learns parallel top-down and bottom-up representations of nodes capturing contrasting views of the graph. We also investigate the use of different node message passing strategies, employing different state-of-the-art graph encoders to compute node representations based on incoming and outgoing perspectives. In our experiments, we demonstrate that the dual graph representation leads to improvements in AMR-to-text generation, achieving state-ofthe-art results on two AMR datasets1.},
	language = {en},
	urldate = {2024-01-12},
	booktitle = {Proceedings of the 2019 {Conference} on {Empirical} {Methods} in {Natural} {Language} {Processing} and the 9th {International} {Joint} {Conference} on {Natural} {Language} {Processing} ({EMNLP}-{IJCNLP})},
	publisher = {Association for Computational Linguistics},
	author = {Ribeiro, Leonardo F. R. and Gardent, Claire and Gurevych, Iryna},
	year = {2019},
	pages = {3181--3192},
}

@inproceedings{beck_graph--sequence_2018,
	address = {Melbourne, Australia},
	title = {Graph-to-{Sequence} {Learning} using {Gated} {Graph} {Neural} {Networks}},
	url = {http://aclweb.org/anthology/P18-1026},
	doi = {10.18653/v1/P18-1026},
	abstract = {Many NLP applications can be framed as a graph-to-sequence learning problem. Previous work proposing neural architectures on this setting obtained promising results compared to grammar-based approaches but still rely on linearisation heuristics and/or standard recurrent networks to achieve the best performance. In this work, we propose a new model that encodes the full structural information contained in the graph. Our architecture couples the recently proposed Gated Graph Neural Networks with an input transformation that allows nodes and edges to have their own hidden representations, while tackling the parameter explosion problem present in previous work. Experimental results show that our model outperforms strong baselines in generation from AMR graphs and syntax-based neural machine translation.},
	language = {en},
	urldate = {2024-01-12},
	booktitle = {Proceedings of the 56th {Annual} {Meeting} of the {Association} for {Computational} {Linguistics} ({Volume} 1: {Long} {Papers})},
	publisher = {Association for Computational Linguistics},
	author = {Beck, Daniel and Haffari, Gholamreza and Cohn, Trevor},
	year = {2018},
	pages = {273--283},
}

@article{ganea_non-euclidean_nodate,
	title = {Non-{Euclidean} {Neural} {Representation} {Learning} of {Words}, {Entities} and {Hierarchies}},
	language = {en},
	author = {Ganea, Octavian-Eugen and Zürich, ETH},
}

@inproceedings{dhingra_embedding_2018,
	address = {New Orleans, Louisiana, USA},
	title = {Embedding {Text} in {Hyperbolic} {Spaces}},
	url = {https://aclanthology.org/W18-1708},
	doi = {10.18653/v1/W18-1708},
	abstract = {Natural language text exhibits hierarchical structure in a variety of respects. Ideally, we could incorporate our prior knowledge of this hierarchical structure into unsupervised learning algorithms that work on text data. Recent work by Nickel and Kiela (2017) proposed using hyperbolic instead of Euclidean embedding spaces to represent hierarchical data and demonstrated encouraging results when embedding graphs. In this work, we extend their method with a re-parameterization technique that allows us to learn hyperbolic embeddings of arbitrarily parameterized objects. We apply this framework to learn word and sentence embeddings in hyperbolic space in an unsupervised manner from text corpora. The resulting embeddings seem to encode certain intuitive notions of hierarchy, such as word-context frequency and phrase constituency. However, the implicit continuous hierarchy in the learned hyperbolic space makes interrogating the model's learned hierarchies more difficult than for models that learn explicit edges between items. The learned hyperbolic embeddings show improvements over Euclidean embeddings in some – but not all – downstream tasks, suggesting that hierarchical organization is more useful for some tasks than others.},
	urldate = {2024-01-12},
	booktitle = {Proceedings of the {Twelfth} {Workshop} on {Graph}-{Based} {Methods} for {Natural} {Language} {Processing} ({TextGraphs}-12)},
	publisher = {Association for Computational Linguistics},
	author = {Dhingra, Bhuwan and Shallue, Christopher and Norouzi, Mohammad and Dai, Andrew and Dahl, George},
	editor = {Glavaš, Goran and Somasundaran, Swapna and Riedl, Martin and Hovy, Eduard},
	month = jun,
	year = {2018},
	pages = {59--69},
}

@inproceedings{meng_spherical_2019,
	title = {Spherical {Text} {Embedding}},
	volume = {32},
	url = {https://proceedings.neurips.cc/paper_files/paper/2019/hash/043ab21fc5a1607b381ac3896176dac6-Abstract.html},
	abstract = {Unsupervised text embedding has shown great power in a wide range of NLP tasks. While text embeddings are typically learned in the Euclidean space, directional similarity is often more effective in tasks such as word similarity and document clustering, which creates a gap between the training stage and usage stage of text embedding. To close this gap, we propose a spherical generative model based on which unsupervised word and paragraph embeddings are jointly learned. To learn text embeddings in the spherical space, we develop an efficient optimization algorithm with convergence guarantee based on Riemannian optimization. Our model enjoys high efficiency and achieves state-of-the-art performances on various text embedding tasks including word similarity and document clustering.},
	urldate = {2024-01-12},
	booktitle = {Advances in {Neural} {Information} {Processing} {Systems}},
	publisher = {Curran Associates, Inc.},
	author = {Meng, Yu and Huang, Jiaxin and Wang, Guangyuan and Zhang, Chao and Zhuang, Honglei and Kaplan, Lance and Han, Jiawei},
	year = {2019},
}

@inproceedings{iwamoto_polar_2021,
	address = {Online},
	title = {Polar {Embedding}},
	url = {https://aclanthology.org/2021.conll-1.37},
	doi = {10.18653/v1/2021.conll-1.37},
	abstract = {Hierarchical relationships are invaluable information for many natural language processing (NLP) tasks. Distributional representation has become a fundamental approach for encoding word relationships, particularly embeddings in hyperbolic space showed great performance in representing hierarchies by taking advantage of their spatial properties. However, most machine learning systems do not suppose to use in such complex non-Euclidean geometries. To achieve hierarchy representations in commonly used Euclidean space, we propose Polar Embedding that learns word embeddings with the polar coordinate system. Utilizing characteristics of polar coordinates, the hierarchy of words is expressed with two independent variables: radius (generality) and angles (similarity), and their variables are optimized separately. Polar embedding shows word hierarchies explicitly and allows us to use beneficial resources such as word frequencies or word generality annotations for computing radiuses. We introduce an optimization method for learning angles in limited ranges of polar coordinates, which combining a loss function controlling gradient and distribution uniformization. Experimental results on hypernymy datasets indicate that our approach outperforms other embeddings in low-dimensional Euclidean space and competitively performs even with hyperbolic embeddings, which possess a geometric advantage.},
	urldate = {2024-01-12},
	booktitle = {Proceedings of the 25th {Conference} on {Computational} {Natural} {Language} {Learning}},
	publisher = {Association for Computational Linguistics},
	author = {Iwamoto, Ran and Kohita, Ryosuke and Wachi, Akifumi},
	editor = {Bisazza, Arianna and Abend, Omri},
	month = nov,
	year = {2021},
	pages = {470--480},
}

@article{opitz_weisfeiler-leman_2021,
	title = {Weisfeiler-{Leman} in the {Bamboo}: {Novel} {AMR} {Graph} {Metrics} and a {Benchmark} for {AMR} {Graph} {Similarity}},
	volume = {9},
	shorttitle = {Weisfeiler-{Leman} in the {Bamboo}},
	url = {https://aclanthology.org/2021.tacl-1.85},
	doi = {10.1162/tacl_a_00435},
	abstract = {Several metrics have been proposed for assessing the similarity of (abstract) meaning representations (AMRs), but little is known about how they relate to human similarity ratings. Moreover, the current metrics have complementary strengths and weaknesses: Some emphasize speed, while others make the alignment of graph structures explicit, at the price of a costly alignment step. In this work we propose new Weisfeiler-Leman AMR similarity metrics that unify the strengths of previous metrics, while mitigating their weaknesses. Specifically, our new metrics are able to match contextualized substructures and induce n:m alignments between their nodes. Furthermore, we introduce a Benchmark for AMR Metrics based on Overt Objectives (Bamboo), the first benchmark to support empirical assessment of graph-based MR similarity metrics. Bamboo maximizes the interpretability of results by defining multiple overt objectives that range from sentence similarity objectives to stress tests that probe a metric's robustness against meaning-altering and meaning- preserving graph transformations. We show the benefits of Bamboo by profiling previous metrics and our own metrics. Results indicate that our novel metrics may serve as a strong baseline for future work.},
	urldate = {2024-01-12},
	journal = {Transactions of the Association for Computational Linguistics},
	author = {Opitz, Juri and Daza, Angel and Frank, Anette},
	editor = {Roark, Brian and Nenkova, Ani},
	year = {2021},
	note = {Place: Cambridge, MA
Publisher: MIT Press},
	pages = {1425--1441},
}

@article{leimeister_skip-gram_nodate,
	title = {{SKIP}-{GRAM} {WORD} {EMBEDDINGS} {IN} {HYPERBOLIC}},
	abstract = {Recent work has demonstrated that embeddings of tree-like graphs in hyperbolic space surpass their Euclidean counterparts in performance by a large margin. Inspired by these results and scale-free structure in the word co-occurrence graph, we present an algorithm for learning word embeddings in hyperbolic space from free text. An objective function based on the hyperbolic distance is derived and included in the skip-gram negative-sampling architecture of word2vec. The hyperbolic word embeddings are then evaluated on word similarity and analogy benchmarks. The results demonstrate the potential of hyperbolic word embeddings, particularly in low dimensions, though without clear superiority over their Euclidean counterparts. We further discuss subtleties in the formulation of the analogy task in curved spaces.},
	language = {en},
	author = {Leimeister, Matthias and Wilson, Benjamin J},
}

@misc{noauthor_zotero_nodate,
	title = {Zotero {\textbar} {Downloads}},
	url = {https://www.zotero.org/download/},
	urldate = {2024-01-12},
}
