@inproceedings{derczynski-etal-2020-maintaining,
title = "Maintaining Quality in {FEVER} Annotation",
author = "Derczynski, Leon and
Binau, Julie and
Schulte, Henri",
booktitle = "Proceedings of the Third Workshop on Fact Extraction and VERification (FEVER)",
month = jul,
year = "2020",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2020.fever-1.6",
doi = "10.18653/v1/2020.fever-1.6",
pages = "42--46",
abstract = "We propose two measures for measuring the quality of constructed claims in the FEVER task. Annotating data for this task involves the creation of supporting and refuting claims over a set of evidence. Automatic annotation processes often leave superficial patterns in data, which learning systems can detect instead of performing the underlying task. Humans also can leave these superficial patterns, either voluntarily or involuntarily (due to e.g. fatigue). The two measures introduced attempt to detect the impact of these superficial patterns. One is a new information-theoretic and distributionality based measure, \textit{DCI}; and the other an extension of neural probing work over the ARCT task, \textit{utility}. We demonstrate these measures over a recent major dataset, that from the English FEVER task in 2019.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="derczynski-etal-2020-maintaining">
<titleInfo>
<title>Maintaining Quality in FEVER Annotation</title>
</titleInfo>
<name type="personal">
<namePart type="given">Leon</namePart>
<namePart type="family">Derczynski</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Julie</namePart>
<namePart type="family">Binau</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Henri</namePart>
<namePart type="family">Schulte</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2020-jul</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the Third Workshop on Fact Extraction and VERification (FEVER)</title>
</titleInfo>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Online</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>We propose two measures for measuring the quality of constructed claims in the FEVER task. Annotating data for this task involves the creation of supporting and refuting claims over a set of evidence. Automatic annotation processes often leave superficial patterns in data, which learning systems can detect instead of performing the underlying task. Humans also can leave these superficial patterns, either voluntarily or involuntarily (due to e.g. fatigue). The two measures introduced attempt to detect the impact of these superficial patterns. One is a new information-theoretic and distributionality based measure, DCI; and the other an extension of neural probing work over the ARCT task, utility. We demonstrate these measures over a recent major dataset, that from the English FEVER task in 2019.</abstract>
<identifier type="citekey">derczynski-etal-2020-maintaining</identifier>
<identifier type="doi">10.18653/v1/2020.fever-1.6</identifier>
<location>
<url>https://aclanthology.org/2020.fever-1.6</url>
</location>
<part>
<date>2020-jul</date>
<extent unit="page">
<start>42</start>
<end>46</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Maintaining Quality in FEVER Annotation
%A Derczynski, Leon
%A Binau, Julie
%A Schulte, Henri
%S Proceedings of the Third Workshop on Fact Extraction and VERification (FEVER)
%D 2020
%8 jul
%I Association for Computational Linguistics
%C Online
%F derczynski-etal-2020-maintaining
%X We propose two measures for measuring the quality of constructed claims in the FEVER task. Annotating data for this task involves the creation of supporting and refuting claims over a set of evidence. Automatic annotation processes often leave superficial patterns in data, which learning systems can detect instead of performing the underlying task. Humans also can leave these superficial patterns, either voluntarily or involuntarily (due to e.g. fatigue). The two measures introduced attempt to detect the impact of these superficial patterns. One is a new information-theoretic and distributionality based measure, DCI; and the other an extension of neural probing work over the ARCT task, utility. We demonstrate these measures over a recent major dataset, that from the English FEVER task in 2019.
%R 10.18653/v1/2020.fever-1.6
%U https://aclanthology.org/2020.fever-1.6
%U https://doi.org/10.18653/v1/2020.fever-1.6
%P 42-46
Markdown (Informal)
[Maintaining Quality in FEVER Annotation](https://aclanthology.org/2020.fever-1.6) (Derczynski et al., FEVER 2020)
ACL
- Leon Derczynski, Julie Binau, and Henri Schulte. 2020. Maintaining Quality in FEVER Annotation. In Proceedings of the Third Workshop on Fact Extraction and VERification (FEVER), pages 42–46, Online. Association for Computational Linguistics.