@inproceedings{bahaj-ghogho-2026-mizanqa,
    title = "{M}izan{QA}: A Benchmark for Multi-Answer {M}oroccan Legal {QA}",
    author = "Bahaj, Adil  and
      Ghogho, Mounir",
    editor = {Matusevych, Yevgen  and
      Eryi{\u{g}}it, G{\"u}l{\c{s}}en  and
      Aletras, Nikolaos},
    booktitle = "Proceedings of the 19th Conference of the {E}uropean Chapter of the {A}ssociation for {C}omputational {L}inguistics (Volume 5: Industry Track)",
    month = mar,
    year = "2026",
    address = "Rabat, Morocco",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/ingest-eacl/2026.eacl-industry.10/",
    pages = "132--144",
    ISBN = "979-8-89176-384-5",
    abstract = "We present MizanQA, a benchmark for assessing LLMs on Moroccan legal MCQs, many with multiple correct answers. Covering 1,776 expert-verified questions in Modern Standard Arabic enriched with Moroccan idioms, the dataset reflects influences from Maliki jurisprudence, customary law, and French legal traditions. Unlike single-answer settings, MizanQA features variable option counts, creating added difficulty. We evaluate multilingual and Arabic-centric models in zero-shot, native-Arabic prompts, measuring accuracy, a precision-penalized F1-like score, and calibration errors. Results show large performance gaps and miscalibration, particularly under stricter penalties. By scoping this benchmark to parametric knowledge only, we provide a baseline for future retrieval-augmented and rationale-focused setups."
}