@inproceedings{alacam-etal-2024-wikiscenes,
    title = "{W}iki{S}cenes with Descriptions: Aligning Paragraphs and Sentences with Images in {W}ikipedia Articles",
    author = {Ala{\c{c}}am, {\"O}zge  and
      Utescher, Ronja  and
      Gr{\"o}nner, Hannes  and
      Sieker, Judith  and
      Zarrie{\ss}, Sina},
    editor = "Bollegala, Danushka  and
      Shwartz, Vered",
    booktitle = "Proceedings of the 13th Joint Conference on Lexical and Computational Semantics (*SEM 2024)",
    month = jun,
    year = "2024",
    address = "Mexico City, Mexico",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/ingest_wac_2008/2024.starsem-1.8/",
    doi = "10.18653/v1/2024.starsem-1.8",
    pages = "93--105",
    abstract = "Research in Language {\&} Vision rarely uses naturally occurring multimodal documents as Wikipedia articles, since they feature complex image-text relations and implicit image-text alignments. In this paper, we provide one of the first datasets that provides ground-truth annotations of image-text alignments in multi-paragraph multi-image articles. The dataset can be used to study phenomena of visual language grounding in longer documents and assess retrieval capabilities of language models trained on, e.g., captioning data. Our analyses show that there are systematic linguistic differences between the image captions and descriptive sentences from the article`s text and that intra-document retrieval is a challenging task for state-of-the-art models in L{\&}V (CLIP, VILT, MCSE)."
}