@inproceedings{wysocki-etal-2021-semeval,
    title = "What is {S}em{E}val evaluating? A Systematic Analysis of Evaluation Campaigns in {NLP}",
    author = "Wysocki, Oskar  and
      Florea, Malina  and
      Landers, D{\'o}nal  and
      Freitas, Andr{\'e}",
    editor = "Gao, Yang  and
      Eger, Steffen  and
      Zhao, Wei  and
      Lertvittayakumjorn, Piyawat  and
      Fomicheva, Marina",
    booktitle = "Proceedings of the 2nd Workshop on Evaluation and Comparison of NLP Systems",
    month = nov,
    year = "2021",
    address = "Punta Cana, Dominican Republic",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2021.eval4nlp-1.21/",
    doi = "10.18653/v1/2021.eval4nlp-1.21",
    pages = "209--229",
    abstract = "SemEval is the primary venue in the NLP community for the proposal of new challenges and for the systematic empirical evaluation of NLP systems. This paper provides a systematic quantitative analysis of SemEval aiming to evidence the patterns of the contributions behind SemEval. By understanding the distribution of task types, metrics, architectures, participation and citations over time we aim to answer the question on what is being evaluated by SemEval."
}