@inproceedings{fort-etal-2012-analyzing,
    title = "Analyzing the Impact of Prevalence on the Evaluation of a Manual Annotation Campaign",
    author = {Fort, Kar{\"e}n  and
      Fran{\c{c}}ois, Claire  and
      Galibert, Olivier  and
      Ghribi, Maha},
    editor = "Calzolari, Nicoletta  and
      Choukri, Khalid  and
      Declerck, Thierry  and
      Do{\u{g}}an, Mehmet U{\u{g}}ur  and
      Maegaard, Bente  and
      Mariani, Joseph  and
      Moreno, Asuncion  and
      Odijk, Jan  and
      Piperidis, Stelios",
    booktitle = "Proceedings of the Eighth International Conference on Language Resources and Evaluation ({LREC}'12)",
    month = may,
    year = "2012",
    address = "Istanbul, Turkey",
    publisher = "European Language Resources Association (ELRA)",
    url = "https://preview.aclanthology.org/iwcs-25-ingestion/L12-1310/",
    pages = "1474--1480",
    abstract = {This article details work aiming at evaluating the quality of the manual annotation of gene renaming couples in scientific abstracts, which generates sparse annotations. To evaluate these annotations, we compare the results obtained using the commonly advocated inter-annotator agreement coefficients such as S, {\ensuremath{\kappa}} and {\"I}€, the less known R, the weighted coefficients {\ensuremath{\kappa}}{\"I}‰ and {\^I}{\ensuremath{\pm}} as well as the F-measure and the SER. We analyze to which extent they are relevant for our data. We then study the bias introduced by prevalence by changing the way the contingency table is built. We finally propose an original way to synthesize the results by computing distances between categories, based on the produced annotations.}
}