@inproceedings{umbet-etal-2025-kazbench,
    title = "{K}az{B}ench-{KK}: A Cultural-Knowledge Benchmark for {K}azakh",
    author = "Umbet, Sanzhar  and
      Murzakhmetov, Sanzhar  and
      Sagyndyk, Beksultan  and
      Yakunin, Kirill  and
      Akishev, Timur  and
      Zubitski, Pavel",
    editor = "Le Ferrand, {\'E}ric  and
      Klyachko, Elena  and
      Postnikova, Anna  and
      Shavrina, Tatiana  and
      Serikov, Oleg  and
      Voloshina, Ekaterina  and
      Vylomova, Ekaterina",
    booktitle = "Proceedings of the Fourth Workshop on NLP Applications to Field Linguistics",
    month = aug,
    year = "2025",
    address = "Vienna, Austria",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/transition-to-people-yaml/2025.fieldmatters-1.4/",
    pages = "38--57",
    ISBN = "979-8-89176-282-4",
    abstract = "We introduce KazBench-KK, a comprehensive 7,111-question multiple-choice benchmark designed to assess large language models' understanding of culturally grounded Kazakh knowledge. By combining expert-curated topics with LLM-assisted web mining, we create a diverse dataset spanning 17 culturally salient domains, including pastoral traditions, social hierarchies, and contemporary politics. Beyond evaluation, KazBench-KK serves as a practical tool for field linguists, enabling rapid lexical elicitation, glossing, and topic prioritization. Our benchmarking of various open-source LLMs reveals that reinforcement-tuned models outperform others, but smaller, domain-focused fine-tunes can rival larger models in specific cultural contexts."
}