@inproceedings{kuksenok-martyniv-2019-evaluation,
title = "Evaluation and Improvement of Chatbot Text Classification Data Quality Using Plausible Negative Examples",
author = "Kuksenok, Kit and
Martyniv, Andriy",
booktitle = "Proceedings of the First Workshop on NLP for Conversational AI",
month = aug,
year = "2019",
address = "Florence, Italy",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/W19-4110",
doi = "10.18653/v1/W19-4110",
pages = "87--95",
abstract = "We describe and validate a metric for estimating multi-class classifier performance based on cross-validation and adapted for improvement of small, unbalanced natural-language datasets used in chatbot design. Our experiences draw upon building recruitment chatbots that mediate communication between job-seekers and recruiters by exposing the ML/NLP dataset to the recruiting team. Evaluation approaches must be understandable to various stakeholders, and useful for improving chatbot performance. The metric, nex-cv, uses negative examples in the evaluation of text classification, and fulfils three requirements. First, it is actionable: it can be used by non-developer staff. Second, it is not overly optimistic compared to human ratings, making it a fast method for comparing classifiers. Third, it allows model-agnostic comparison, making it useful for comparing systems despite implementation differences. We validate the metric based on seven recruitment-domain datasets in English and German over the course of one year.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="kuksenok-martyniv-2019-evaluation">
<titleInfo>
<title>Evaluation and Improvement of Chatbot Text Classification Data Quality Using Plausible Negative Examples</title>
</titleInfo>
<name type="personal">
<namePart type="given">Kit</namePart>
<namePart type="family">Kuksenok</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Andriy</namePart>
<namePart type="family">Martyniv</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2019-aug</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Workshop on NLP for Conversational AI</title>
</titleInfo>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Florence, Italy</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>We describe and validate a metric for estimating multi-class classifier performance based on cross-validation and adapted for improvement of small, unbalanced natural-language datasets used in chatbot design. Our experiences draw upon building recruitment chatbots that mediate communication between job-seekers and recruiters by exposing the ML/NLP dataset to the recruiting team. Evaluation approaches must be understandable to various stakeholders, and useful for improving chatbot performance. The metric, nex-cv, uses negative examples in the evaluation of text classification, and fulfils three requirements. First, it is actionable: it can be used by non-developer staff. Second, it is not overly optimistic compared to human ratings, making it a fast method for comparing classifiers. Third, it allows model-agnostic comparison, making it useful for comparing systems despite implementation differences. We validate the metric based on seven recruitment-domain datasets in English and German over the course of one year.</abstract>
<identifier type="citekey">kuksenok-martyniv-2019-evaluation</identifier>
<identifier type="doi">10.18653/v1/W19-4110</identifier>
<location>
<url>https://aclanthology.org/W19-4110</url>
</location>
<part>
<date>2019-aug</date>
<extent unit="page">
<start>87</start>
<end>95</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluation and Improvement of Chatbot Text Classification Data Quality Using Plausible Negative Examples
%A Kuksenok, Kit
%A Martyniv, Andriy
%S Proceedings of the First Workshop on NLP for Conversational AI
%D 2019
%8 aug
%I Association for Computational Linguistics
%C Florence, Italy
%F kuksenok-martyniv-2019-evaluation
%X We describe and validate a metric for estimating multi-class classifier performance based on cross-validation and adapted for improvement of small, unbalanced natural-language datasets used in chatbot design. Our experiences draw upon building recruitment chatbots that mediate communication between job-seekers and recruiters by exposing the ML/NLP dataset to the recruiting team. Evaluation approaches must be understandable to various stakeholders, and useful for improving chatbot performance. The metric, nex-cv, uses negative examples in the evaluation of text classification, and fulfils three requirements. First, it is actionable: it can be used by non-developer staff. Second, it is not overly optimistic compared to human ratings, making it a fast method for comparing classifiers. Third, it allows model-agnostic comparison, making it useful for comparing systems despite implementation differences. We validate the metric based on seven recruitment-domain datasets in English and German over the course of one year.
%R 10.18653/v1/W19-4110
%U https://aclanthology.org/W19-4110
%U https://doi.org/10.18653/v1/W19-4110
%P 87-95
Markdown (Informal)
[Evaluation and Improvement of Chatbot Text Classification Data Quality Using Plausible Negative Examples](https://aclanthology.org/W19-4110) (Kuksenok & Martyniv, 2019)
ACL