@inproceedings{van-der-weide-etal-2026-crows,
    title = "{C}row{S}-Pairs-{NL}: A Benchmark to Evaluate {D}utch Stereotype Bias in {LLM}s",
    author = "van der Weide, Jens  and
      Nguyen, Dong  and
      Schaaphok, Marianne  and
      Bakker, Roos M.",
    editor = "Ma, Weicheng  and
      Vosoughi, Soroush  and
      Gillani, Nabeel  and
      Coto-Solano, Rolando",
    booktitle = "Proceedings of the 1st Workshop on Stereotypes Across Cultures in Language Technologies ({S}tere{AC}u{LT} 2026)",
    month = jul,
    year = "2026",
    address = "San Diego, California, United States",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/ingest-acl-workshops/2026.stereacult-1.1/",
    pages = "1--12",
    ISBN = "979-8-89176-408-8",
    abstract = "Bias benchmarks for LLMs largely focus on English, overlooking language- and culture-specific stereotypes. We introduce CrowS-Pairs-NL, a Dutch stereotype benchmark built by filtering, translating, and adapting the English CrowS-Pairs dataset to address known conceptual pitfalls, and extending it with newly crowdsourced Dutch sentence pairs. We evaluate six multilingual and Dutch-trained models using both a pseudo-log-likelihood metric adapted for autoregressive models and a prompt-based metric with three template variants. Models explicitly trained on Dutch data consistently exhibit higher stereotyping scores, suggesting that language-specific fine-tuning introduces language-specific bias. The two metrics broadly agree on model rankings but differ in sensitivity, with the prompt metric showing a narrower range of scores. Our benchmark and findings underscore the need for culturally grounded bias evaluation beyond English."
}