@inproceedings{doogan-buntine-2021-topic,
title = "Topic Model or Topic Twaddle? Re-evaluating Semantic Interpretability Measures",
author = "Doogan, Caitlin and
Buntine, Wray",
booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",
month = jun,
year = "2021",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2021.naacl-main.300",
doi = "10.18653/v1/2021.naacl-main.300",
pages = "3824--3848",
abstract = "When developing topic models, a critical question that should be asked is: How well will this model work in an applied setting? Because standard performance evaluation of topic interpretability uses automated measures modeled on human evaluation tests that are dissimilar to applied usage, these models{'} generalizability remains in question. In this paper, we probe the issue of validity in topic model evaluation and assess how informative coherence measures are for specialized collections used in an applied setting. Informed by the literature, we propose four understandings of interpretability. We evaluate these using a novel experimental framework reflective of varied applied settings, including human evaluations using open labeling, typical of applied research. These evaluations show that for some specialized collections, standard coherence measures may not inform the most appropriate topic model or the optimal number of topics, and current interpretability performance validation methods are challenged as a means to confirm model quality in the absence of ground truth data.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="doogan-buntine-2021-topic">
<titleInfo>
<title>Topic Model or Topic Twaddle? Re-evaluating Semantic Interpretability Measures</title>
</titleInfo>
<name type="personal">
<namePart type="given">Caitlin</namePart>
<namePart type="family">Doogan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Wray</namePart>
<namePart type="family">Buntine</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2021-jun</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies</title>
</titleInfo>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Online</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>When developing topic models, a critical question that should be asked is: How well will this model work in an applied setting? Because standard performance evaluation of topic interpretability uses automated measures modeled on human evaluation tests that are dissimilar to applied usage, these models’ generalizability remains in question. In this paper, we probe the issue of validity in topic model evaluation and assess how informative coherence measures are for specialized collections used in an applied setting. Informed by the literature, we propose four understandings of interpretability. We evaluate these using a novel experimental framework reflective of varied applied settings, including human evaluations using open labeling, typical of applied research. These evaluations show that for some specialized collections, standard coherence measures may not inform the most appropriate topic model or the optimal number of topics, and current interpretability performance validation methods are challenged as a means to confirm model quality in the absence of ground truth data.</abstract>
<identifier type="citekey">doogan-buntine-2021-topic</identifier>
<identifier type="doi">10.18653/v1/2021.naacl-main.300</identifier>
<location>
<url>https://aclanthology.org/2021.naacl-main.300</url>
</location>
<part>
<date>2021-jun</date>
<extent unit="page">
<start>3824</start>
<end>3848</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Topic Model or Topic Twaddle? Re-evaluating Semantic Interpretability Measures
%A Doogan, Caitlin
%A Buntine, Wray
%S Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies
%D 2021
%8 jun
%I Association for Computational Linguistics
%C Online
%F doogan-buntine-2021-topic
%X When developing topic models, a critical question that should be asked is: How well will this model work in an applied setting? Because standard performance evaluation of topic interpretability uses automated measures modeled on human evaluation tests that are dissimilar to applied usage, these models’ generalizability remains in question. In this paper, we probe the issue of validity in topic model evaluation and assess how informative coherence measures are for specialized collections used in an applied setting. Informed by the literature, we propose four understandings of interpretability. We evaluate these using a novel experimental framework reflective of varied applied settings, including human evaluations using open labeling, typical of applied research. These evaluations show that for some specialized collections, standard coherence measures may not inform the most appropriate topic model or the optimal number of topics, and current interpretability performance validation methods are challenged as a means to confirm model quality in the absence of ground truth data.
%R 10.18653/v1/2021.naacl-main.300
%U https://aclanthology.org/2021.naacl-main.300
%U https://doi.org/10.18653/v1/2021.naacl-main.300
%P 3824-3848
Markdown (Informal)
[Topic Model or Topic Twaddle? Re-evaluating Semantic Interpretability Measures](https://aclanthology.org/2021.naacl-main.300) (Doogan & Buntine, NAACL 2021)
ACL