@inproceedings{bladsjo-munoz-sanchez-2025-introducing,
    title = "Introducing {MARB} {---} A Dataset for Studying the Social Dimensions of Reporting Bias in Language Models",
    author = {Bladsj{\"o}, Tom S{\"o}dahl  and
      Mu{\~n}oz S{\'a}nchez, Ricardo},
    editor = "Fale{\'n}ska, Agnieszka  and
      Basta, Christine  and
      Costa-juss{\`a}, Marta  and
      Sta{\'n}czak, Karolina  and
      Nozza, Debora",
    booktitle = "Proceedings of the 6th Workshop on Gender Bias in Natural Language Processing (GeBNLP)",
    month = aug,
    year = "2025",
    address = "Vienna, Austria",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/landing_page/2025.gebnlp-1.5/",
    doi = "10.18653/v1/2025.gebnlp-1.5",
    pages = "59--74",
    ISBN = "979-8-89176-277-0",
    abstract = "Reporting bias is the tendency for speakers to omit unnecessary or obvious information while mentioning things they consider relevant or surprising. In descriptions of people, reporting bias can manifest as a tendency to over report on attributes that deviate from the norm. While social bias in language models has garnered a lot of attention in recent years, a majority of the existing work equates ``bias'' with ``stereotypes''. We suggest reporting bias as an alternative lens through which to study how social attitudes manifest in language models. We present the MARB dataset, a diagnostic dataset for studying the interaction between social bias and reporting bias in language models. We use MARB to evaluate the off-the-shelf behavior of both masked and autoregressive language models and find signs of reporting bias with regards to marginalized identities, mirroring that which can be found in human text. This effect is particularly pronounced when taking gender into account, demonstrating the importance of considering intersectionality when studying social phenomena like biases."
}