@inproceedings{ilinykh-dobnik-2022-decoding,
    title = "Do Decoding Algorithms Capture Discourse Structure in Multi-Modal Tasks? A Case Study of Image Paragraph Generation",
    author = "Ilinykh, Nikolai  and
      Dobnik, Simon",
    editor = "Bosselut, Antoine  and
      Chandu, Khyathi  and
      Dhole, Kaustubh  and
      Gangal, Varun  and
      Gehrmann, Sebastian  and
      Jernite, Yacine  and
      Novikova, Jekaterina  and
      Perez-Beltrachini, Laura",
    booktitle = "Proceedings of the 2nd Workshop on Natural Language Generation, Evaluation, and Metrics (GEM)",
    month = dec,
    year = "2022",
    address = "Abu Dhabi, United Arab Emirates (Hybrid)",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2022.gem-1.45/",
    doi = "10.18653/v1/2022.gem-1.45",
    pages = "480--493",
    abstract = "This paper describes insights into how different inference algorithms structure discourse in image paragraphs. We train a multi-modal transformer and compare 11 variations of decoding algorithms. We propose to evaluate image paragraphs not only with standard automatic metrics, but also with a more extensive, {\textquotedblleft}under the hood{\textquotedblright} analysis of the discourse formed by sentences. Our results show that while decoding algorithms can be unfaithful to the reference texts, they still generate grounded descriptions, but they also lack understanding of the discourse structure and differ from humans in terms of attentional structure over images."
}