@article{krovetz-2026-test,
    title = "A Test Collection for Part-of-Speech Tagging and Word Sense Disambiguation",
    author = "Krovetz, Robert",
    editor = "Piperidis, Stelios  and
      Bel, N{\'u}ria  and
      van den Heuvel, Henk  and
      Ide, Nancy  and
      Krek, Simon  and
      Toral, Antonio",
    journal = "International Conference on Language Resources and Evaluation",
    volume = "main",
    month = may,
    year = "2026",
    address = "Palma de Mallorca, Spain",
    publisher = "ELRA Language Resource Association",
    url = "https://preview.aclanthology.org/ingest-lrec/2026.lrec-main.925/",
    pages = "11813--11821",
    abstract = "We evaluate a focused test collection at the intersection of part-of-speech tagging and word-sense disambiguation. The collection targets words such as train, novel, and lean, where part-of-speech contrasts align with clear meaning differences. We use it to detect regressions across tagger versions, track quantitative and qualitative progress over time, and test robustness to orthographic variation. Experiments with the Stanford and TnT taggers show 68{\%} accuracy, compared with 92{\%} for a recent spaCy transformer model. Earlier taggers erred mainly on noun{--}verb distinctions; spaCy{'}s errors more often involve noun{--}adjective distinctions. Uppercase text roughly doubles error rates for all taggers. We discuss common problems and propose directions for future testing."
}