% Use this file for citations not found in the ACL Anthology (contained in "anthology.bib").

@book{Aho:72,
    author  = {Alfred V. Aho and Jeffrey D. Ullman},
    title   = {The Theory of Parsing, Translation and Compiling},
    year    = "1972",
    volume  = "1",
    publisher = {Prentice-Hall},
    address = {Englewood Cliffs, NJ}
}

@book{APA:83,
    author  = {{American Psychological Association}},
    title   = {Publications Manual},
    year    = "1983",
    publisher = {American Psychological Association},
    address = {Washington, DC}
}

@article{Chandra:81,
	author = {Ashok K. Chandra and Dexter C. Kozen and Larry J. Stockmeyer},
	year = "1981",
	title = {Alternation},
	journal = {Journal of the Association for Computing Machinery},
	volume = "28",
	number = "1",
	pages = "114--133",
	doi = "10.1145/322234.322243",
}

@inproceedings{andrew2007scalable,
  title={Scalable training of {L1}-regularized log-linear models},
  author={Andrew, Galen and Gao, Jianfeng},
  booktitle={Proceedings of the 24th International Conference on Machine Learning},
  pages={33--40},
  year={2007},
}

@book{Gusfield:97,
    author  = {Dan Gusfield},
    title   = {Algorithms on Strings, Trees and Sequences},
    year    = "1997",
    publisher = {Cambridge University Press},
    address = {Cambridge, UK}
}

@article{rasooli-tetrault-2015,
    author    = {Mohammad Sadegh Rasooli and Joel R. Tetreault},
    title     = {Yara Parser: {A} Fast and Accurate Dependency Parser},
    journal   = {Computing Research Repository},
    volume    = {arXiv:1503.06733},
    year      = {2015},
    url       = {http://arxiv.org/abs/1503.06733},
    note    = {version 2}
}

@article{Ando2005,
	Acmid = {1194905},
	Author = {Ando, Rie Kubota and Zhang, Tong},
	Issn = {1532-4435},
	Issue_Date = {12/1/2005},
	Journal = {Journal of Machine Learning Research},
	Month = dec,
	Numpages = {37},
	Pages = {1817--1853},
	Publisher = {JMLR.org},
	Title = {A Framework for Learning Predictive Structures from Multiple Tasks and Unlabeled Data},
	Volume = {6},
	Year = {2005}
}


@article{saphra2021training,
  title={Training dynamics of neural language models},
  author={Saphra, Naomi},
  year={2021},
  publisher={The University of Edinburgh}
}

@article{belinkov2022probing,
  title={Probing classifiers: Promises, shortcomings, and advances},
  author={Belinkov, Yonatan},
  journal={Computational Linguistics},
  volume={48},
  number={1},
  pages={207--219},
  year={2022},
  publisher={MIT Press One Broadway, 12th Floor, Cambridge, Massachusetts 02142, USA~…}
}
@inproceedings{tan-bansal-2019-lxmert,
    title = "{LXMERT}: Learning Cross-Modality Encoder Representations from Transformers",
    author = "Tan, Hao  and
      Bansal, Mohit",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)",
    month = nov,
    year = "2019",
    address = "Hong Kong, China",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/D19-1514",
    doi = "10.18653/v1/D19-1514",
    pages = "5100--5111",
    abstract = "Vision-and-language reasoning requires an understanding of visual concepts, language semantics, and, most importantly, the alignment and relationships between these two modalities. We thus propose the LXMERT (Learning Cross-Modality Encoder Representations from Transformers) framework to learn these vision-and-language connections. In LXMERT, we build a large-scale Transformer model that consists of three encoders: an object relationship encoder, a language encoder, and a cross-modality encoder. Next, to endow our model with the capability of connecting vision and language semantics, we pre-train the model with large amounts of image-and-sentence pairs, via five diverse representative pre-training tasks: masked language modeling, masked object prediction (feature regression and label classification), cross-modality matching, and image question answering. These tasks help in learning both intra-modality and cross-modality relationships. After fine-tuning from our pre-trained parameters, our model achieves the state-of-the-art results on two visual question answering datasets (i.e., VQA and GQA). We also show the generalizability of our pre-trained cross-modality model by adapting it to a challenging visual-reasoning task, NLVR2, and improve the previous best result by 22{\%} absolute (54{\%} to 76{\%}). Lastly, we demonstrate detailed ablation studies to prove that both our novel model components and pre-training strategies significantly contribute to our strong results. Code and pre-trained models publicly available at: https://github.com/airsplay/lxmert",
}

@article{xudong2023,
    author = {Hong, Xudong and Sayeed, Asad and Mehra, Khushboo and Demberg, Vera and Schiele, Bernt},
    title = "{Visual Writing Prompts: Character-Grounded Story Generation with Curated Image Sequences}",
    journal = {Transactions of the Association for Computational Linguistics},
    volume = {11},
    pages = {565-581},
    year = {2023},
    month = {06},
    abstract = "{Current work on image-based story generation suffers from the fact that the existing image sequence collections do not have coherent plots behind them. We improve visual story generation by producing a new image-grounded dataset, Visual Writing Prompts (VWP). VWP contains almost 2K selected sequences of movie shots, each including 5-10 images. The image sequences are aligned with a total of 12K stories which were collected via crowdsourcing given the image sequences and a set of grounded characters from the corresponding image sequence. Our new image sequence collection and filtering process has allowed us to obtain stories that are more coherent, diverse, and visually grounded compared to previous work. We also propose a character-based story generation model driven by coherence as a strong baseline. Evaluations show that our generated stories are more coherent, visually grounded, and diverse than stories generated with the current state-of-the-art model. Our code, image features, annotations and collected stories are available at https://vwprompt.github.io/.}",
    issn = {2307-387X},
    doi = {10.1162/tacl_a_00553},
    url = {https://doi.org/10.1162/tacl\_a\_00553},
    eprint = {https://direct.mit.edu/tacl/article-pdf/doi/10.1162/tacl\_a\_00553/2134487/tacl\_a\_00553.pdf},
}





@inproceedings{agrawal2018,
  author       = {Aishwarya Agrawal and
                  Dhruv Batra and
                  Devi Parikh and
                  Aniruddha Kembhavi},
  title        = {Don't Just Assume; Look and Answer: Overcoming Priors for Visual Question
                  Answering},
  booktitle    = {2018 {IEEE} Conference on Computer Vision and Pattern Recognition,
                  {CVPR} 2018, Salt Lake City, UT, USA, June 18-22, 2018},
  pages        = {4971--4980},
  publisher    = {Computer Vision Foundation / {IEEE} Computer Society},
  year         = {2018},
  url          = {http://openaccess.thecvf.com/content\_cvpr\_2018/html/Agrawal\_Dont\_Just\_Assume\_CVPR\_2018\_paper.html},
  doi          = {10.1109/CVPR.2018.00522},
  timestamp    = {Fri, 24 Mar 2023 00:03:00 +0100},
  biburl       = {https://dblp.org/rec/conf/cvpr/AgrawalBPK18.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{thomason-etal-2019-shifting,
    title = "Shifting the Baseline: Single Modality Performance on Visual Navigation {\&} {QA}",
    author = "Thomason, Jesse  and
      Gordon, Daniel  and
      Bisk, Yonatan",
    booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",
    month = jun,
    year = "2019",
    address = "Minneapolis, Minnesota",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/N19-1197",
    doi = "10.18653/v1/N19-1197",
    pages = "1977--1983",
    abstract = "We demonstrate the surprising strength of unimodal baselines in multimodal domains, and make concrete recommendations for best practices in future research. Where existing work often compares against random or majority class baselines, we argue that unimodal approaches better capture and reflect dataset biases and therefore provide an important comparison when assessing the performance of multimodal techniques. We present unimodal ablations on three recent datasets in visual navigation and QA, seeing an up to 29{\%} absolute gain in performance over published baselines.",
}

@article{Ruis:2022aa,
	author = {Ruis, Laura and Khan, Akbir and Biderman, Stella and Hooker, Sara and Rockt{\"a}schel, Tim and Grefenstette, Edward},
	journal = {arXiv},
	pages = {1--45},
	title = {Large language models are not zero-shot communicators},
	volume = {arXiv:2210.14986 [cs.CL]},
	year = {2022},
	url = {https://arxiv.org/abs/2210.14986}}

@inproceedings{huang2016visual,
  title={Visual storytelling},
  author={Huang, Ting-Hao and Ferraro, Francis and Mostafazadeh, Nasrin and Misra, Ishan and Agrawal, Aishwarya and Devlin, Jacob and Girshick, Ross and He, Xiaodong and Kohli, Pushmeet and Batra, Dhruv and others},
  booktitle={Proceedings of the 2016 conference of the North American chapter of the association for computational linguistics: Human language technologies},
  pages={1233--1239},
  year={2016}
}

@inproceedings{ilinykh2019tell,
  title={Tell me more: {A} dataset of visual scene description sequences},
  author={Ilinykh, Nikolai and Zarrie{\ss}, Sina and Schlangen, David},
  booktitle={Proceedings of the 12th international conference on natural language generation},
  pages={152--157},
  year={2019}
}

@inproceedings{lin2014microsoft,
  title={Microsoft {COCO}: {Common} objects in context},
  author={Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\'a}r, Piotr and Zitnick, C Lawrence},
  booktitle={Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13},
  pages={740--755},
  year={2014},
  organization={Springer}
}

@article{devlin2018bert,
  title={{BERT}: {Pre-training} of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={arXiv preprint arXiv:1810.04805},
  year={2018}
}


@article{bugliarello2021multimodal,
  title={Multimodal pretraining unmasked: A meta-analysis and a unified framework of vision-and-language {BERTs}},
  author={Bugliarello, Emanuele and Cotterell, Ryan and Okazaki, Naoaki and Elliott, Desmond},
  journal={Transactions of the Association for Computational Linguistics},
  volume={9},
  pages={978--994},
  year={2021},
  publisher={MIT Press}
}


@article{iter2020pretraining,
  title={Pretraining with contrastive sentence objectives improves discourse performance of language models},
  author={Iter, Dan and Guu, Kelvin and Lansing, Larry and Jurafsky, Dan},
  journal={arXiv preprint arXiv:2005.10389},
  year={2020}
}

@article{de2021universal,
  title={Universal dependencies},
  author={De Marneffe, Marie-Catherine and Manning, Christopher D and Nivre, Joakim and Zeman, Daniel},
  journal={Computational linguistics},
  volume={47},
  number={2},
  pages={255--308},
  year={2021},
  publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…}
}

@article{dobnikhallucinate,
  title={Hallucinate or ground: how general or specific are object descriptions generated by a vision-and-language transformer?},
  author={Ilinykh, Nikolai and Dobnik, Simon},
  year={2022}
}

@article{conneau2018you,
  title={What you can cram into a single vector: Probing sentence embeddings for linguistic properties},
  author={Conneau, Alexis and Kruszewski, German and Lample, Guillaume and Barrault, Lo{\"\i}c and Baroni, Marco},
  journal={arXiv preprint arXiv:1805.01070},
  year={2018}
}

@inproceedings{bugliarello2022iglue,
  title={{IGLUE}: {A} benchmark for transfer learning across modalities, tasks, and languages},
  author={Bugliarello, Emanuele and Liu, Fangyu and Pfeiffer, Jonas and Reddy, Siva and Elliott, Desmond and Ponti, Edoardo Maria and Vuli{\'c}, Ivan},
  booktitle={International Conference on Machine Learning},
  pages={2370--2392},
  year={2022},
  organization={PMLR}
}

@article{belinkov2022probing,
  title={Probing classifiers: Promises, shortcomings, and advances},
  author={Belinkov, Yonatan},
  journal={Computational Linguistics},
  volume={48},
  number={1},
  pages={207--219},
  year={2022},
  publisher={MIT Press}
}

@inproceedings{sharma2018conceptual,
  title={Conceptual captions: A cleaned, hypernymed, image alt-text dataset for automatic image captioning},
  author={Sharma, Piyush and Ding, Nan and Goodman, Sebastian and Soricut, Radu},
  booktitle={Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
  pages={2556--2565},
  year={2018}
}


@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@article{pyatkin2023design,
  title={Design Choices for Crowdsourcing Implicit Discourse Relations: Revealing the Biases Introduced by Task Design},
  author={Pyatkin, Valentina and Yung, Frances and Scholman, Merel CJ and Tsarfaty, Reut and Dagan, Ido and Demberg, Vera},
  journal={arXiv preprint arXiv:2304.00815},
  year={2023}
}

@article{lu2019vilbert,
  title={{ViLBERT}: Pretraining task-agnostic visiolinguistic representations for vision-and-language tasks},
  author={Lu, Jiasen and Batra, Dhruv and Parikh, Devi and Lee, Stefan},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@ARTICLE{10.3389/frai.2023.1048874,
  
AUTHOR={Alikhani, Malihe and Khalid, Baber and Stone, Matthew},   
	 
TITLE={Image–text coherence and its implications for multimodal AI},      
	
JOURNAL={Frontiers in Artificial Intelligence},      
	
VOLUME={6},           
	
YEAR={2023},      
	  
URL={https://www.frontiersin.org/articles/10.3389/frai.2023.1048874},       
	
DOI={10.3389/frai.2023.1048874},      
	
ISSN={2624-8212},   
   
ABSTRACT={Human communication often combines imagery and text into integrated presentations, especially online. In this paper, we show how image–text coherence relations can be used to model the pragmatics of image–text presentations in AI systems. In contrast to alternative frameworks that characterize image–text presentations in terms of the priority, relevance, or overlap of information across modalities, coherence theory postulates that each unit of a discourse stands in specific pragmatic relations to other parts of the discourse, with each relation involving its own information goals and inferential connections. Text accompanying an image may, for example, characterize what's visible in the image, explain how the image was obtained, offer the author's appraisal of or reaction to the depicted situation, and so forth. The advantage of coherence theory is that it provides a simple, robust, and effective abstraction of communicative goals for practical applications. To argue this, we review case studies describing coherence in image–text data sets, predicting coherence from few-shot annotations, and coherence models of image–text tasks such as caption generation and caption evaluation.}
}

@inproceedings{alikhani-stone-2019-caption,
    title = "{``}Caption{''} as a Coherence Relation: Evidence and Implications",
    author = "Alikhani, Malihe  and
      Stone, Matthew",
    booktitle = "Proceedings of the Second Workshop on Shortcomings in Vision and Language",
    month = jun,
    year = "2019",
    address = "Minneapolis, Minnesota",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/W19-1806",
    doi = "10.18653/v1/W19-1806",
    pages = "58--67",
    abstract = "We study verbs in image{--}text corpora, contrasting \textit{caption} corpora, where texts are explicitly written to characterize image content, with \textit{depiction} corpora, where texts and images may stand in more general relations. Captions show a distinctively limited distribution of verbs, with strong preferences for specific tense, aspect, lexical aspect, and semantic field. These limitations, which appear in data elicited by a range of methods, restrict the utility of caption corpora to inform image retrieval, multimodal document generation, and perceptually-grounded semantic models. We suggest that these limitations reflect the discourse constraints in play when subjects write texts to accompany imagery, so we argue that future development of image{--}text corpora should work to increase the diversity of event descriptions, while looking explicitly at the different ways text and imagery can be coherently related.",
}

@article{clark2010computational,
  title={Computational learning theory and language acquisition},
  author={Clark, Alexander and Lappin, Shalom},
  journal={Philosophy of linguistics},
  pages={445--475},
  year={2010}
}

@article{takmaz2020refer,
  title={Refer, reuse, reduce: Generating subsequent references in visual and conversational contexts},
  author={Takmaz, Ece and Giulianelli, Mario and Pezzelle, Sandro and Sinclair, Arabella and Fern{\'a}ndez, Raquel},
  journal={arXiv preprint arXiv:2011.04554},
  year={2020}
}

@inproceedings{ilinykh2022attention,
  title={Attention as Grounding: Exploring Textual and Cross-Modal Attention on Entities and Relations in Language-and-Vision Transformer},
  author={Ilinykh, Nikolai and Dobnik, Simon},
  booktitle={Findings of the Association for Computational Linguistics: ACL 2022},
  pages={4062--4073},
  year={2022}
}

@article{DBLP:journals/corr/abs-1806-03822,
  author       = {Pranav Rajpurkar and
                  Robin Jia and
                  Percy Liang},
  title        = {Know What You Don't Know: Unanswerable Questions for SQuAD},
  journal      = {CoRR},
  volume       = {abs/1806.03822},
  year         = {2018},
  url          = {http://arxiv.org/abs/1806.03822},
  eprinttype    = {arXiv},
  eprint       = {1806.03822},
  timestamp    = {Mon, 13 Aug 2018 16:48:21 +0200},
  biburl       = {https://dblp.org/rec/journals/corr/abs-1806-03822.bib},
  bibsource    = {dblp computer science bibliography, https://dblp.org}
}

@article{wang2019superglue,
  title={{SuperGLUE}: A stickier benchmark for general-purpose language understanding systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel},
  journal={Advances in neural information processing systems},
  volume={32},
  year={2019}
}

@article{gehrmann2022gemv2,
  title={{GEMv2}: Multilingual {NLG} benchmarking in a single line of code},
  author={Gehrmann, Sebastian and Bhattacharjee, Abhik and Mahendiran, Abinaya and Wang, Alex and Papangelis, Alexandros and Madaan, Aman and McMillan-Major, Angelina and Shvets, Anna and Upadhyay, Ashish and Yao, Bingsheng and others},
  journal={arXiv preprint arXiv:2206.11249},
  year={2022}
}

@article{warstadt2020blimp,
  title={{BLiMP}: The benchmark of linguistic minimal pairs for English},
  author={Warstadt, Alex and Parrish, Alicia and Liu, Haokun and Mohananey, Anhad and Peng, Wei and Wang, Sheng-Fu and Bowman, Samuel R},
  journal={Transactions of the Association for Computational Linguistics},
  volume={8},
  pages={377--392},
  year={2020},
  publisher={MIT Press}
}

@article{taktasheva2021shaking,
  title={Shaking syntactic trees on the Sesame street: Multilingual probing with controllable perturbations},
  author={Taktasheva, Ekaterina and Mikhailov, Vladislav and Artemova, Ekaterina},
  journal={arXiv preprint arXiv:2109.14017},
  year={2021}
}

@article{stanczak2022same,
  title={Same neurons, different languages: Probing morphosyntax in multilingual pre-trained models},
  author={Sta{\'n}czak, Karolina and Ponti, Edoardo and Hennigen, Lucas Torroba and Cotterell, Ryan and Augenstein, Isabelle},
  journal={arXiv preprint arXiv:2205.02023},
  year={2022}
}

@article{maudslay2021syntactic,
  title={Do syntactic probes probe syntax? experiments with Jabberwocky probing},
  author={Maudslay, Rowan Hall and Cotterell, Ryan},
  journal={arXiv preprint arXiv:2106.02559},
  year={2021}
}

@article{lasri2022probing,
  title={Probing for the usage of grammatical number},
  author={Lasri, Karim and Pimentel, Tiago and Lenci, Alessandro and Poibeau, Thierry and Cotterell, Ryan},
  journal={arXiv preprint arXiv:2204.08831},
  year={2022}
}

@inproceedings{bender2020climbing,
  title={Climbing towards {NLU}: On meaning, form, and understanding in the age of data},
  author={Bender, Emily M and Koller, Alexander},
  booktitle={Proceedings of the 58th annual meeting of the association for computational linguistics},
  pages={5185--5198},
  year={2020}
}

@article{lake2017building,
  title={Building machines that learn and think like people},
  author={Lake, Brenden M and Ullman, Tomer D and Tenenbaum, Joshua B and Gershman, Samuel J},
  journal={Behavioral and brain sciences},
  volume={40},
  pages={e253},
  year={2017},
  publisher={Cambridge University Press}
}

@article{ettinger2020bert,
  title={What {BERT} is not: Lessons from a new suite of psycholinguistic diagnostics for language models},
  author={Ettinger, Allyson},
  journal={Transactions of the Association for Computational Linguistics},
  volume={8},
  pages={34--48},
  year={2020},
  publisher={MIT Press}
}

@article{fairclough1992discourse,
  title={Discourse and text: Linguistic and intertextual analysis within discourse analysis},
  author={Fairclough, Norman},
  journal={Discourse \& society},
  volume={3},
  number={2},
  pages={193--217},
  year={1992},
  publisher={Sage Publications}
}

@inproceedings{nie2019dissent,
  title={{DisSent}: Learning sentence representations from explicit discourse relations},
  author={Nie, Allen and Bennett, Erin and Goodman, Noah},
  booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
  pages={4497--4510},
  year={2019}
}

@article{chen2019evaluation,
  title={Evaluation benchmarks and learning criteria for discourse-aware sentence representations},
  author={Chen, Mingda and Chu, Zewei and Gimpel, Kevin},
  journal={arXiv preprint arXiv:1909.00142},
  year={2019}
}

@article{araujo2021augmenting,
  title={Augmenting {BERT-style} models with predictive coding to improve discourse-level representations},
  author={Araujo, Vladimir and Villa, Andr{\'e}s and Mendoza, Marcelo and Moens, Marie-Francine and Soto, Alvaro},
  journal={arXiv preprint arXiv:2109.04602},
  year={2021}
}

@inproceedings{prasad2008penn,
  title={The {Penn Discourse TreeBank 2.0.}},
  author={Prasad, Rashmi and Dinesh, Nikhil and Lee, Alan and Miltsakaki, Eleni and Robaldo, Livio and Joshi, Aravind K and Webber, Bonnie L},
  booktitle={LREC},
  year={2008}
}

@article{rogers2021primer,
  title={A primer in {BERTology}: What we know about how {BERT} works},
  author={Rogers, Anna and Kovaleva, Olga and Rumshisky, Anna},
  journal={Transactions of the Association for Computational Linguistics},
  volume={8},
  pages={842--866},
  year={2021},
  publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…}
}

@article{chandu2021grounding,
  title={Grounding 'Grounding' in {NLP}},
  author={Chandu, Khyathi Raghavi and Bisk, Yonatan and Black, Alan W},
  journal={arXiv preprint arXiv:2106.02192},
  year={2021}
}

@inproceedings{ilinykh-dobnik-2022-attention,
    title = "{Attention as Grounding: Exploring Textual and Cross-Modal Attention on Entities and Relations in Language-and-Vision Transformer}",
    author = "Ilinykh, Nikolai  and
      Dobnik, Simon",
    booktitle = "Findings of the Association for Computational Linguistics: ACL 2022",
    month = may,
    year = "2022",
    address = "Dublin, Ireland",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.findings-acl.320",
    doi = "10.18653/v1/2022.findings-acl.320",
    pages = "4062--4073",
    abstract = "We explore how a multi-modal transformer trained for generation of longer image descriptions learns syntactic and semantic representations about entities and relations grounded in objects at the level of masked self-attention (text generation) and cross-modal attention (information fusion). We observe that cross-attention learns the visual grounding of noun phrases into objects and high-level semantic information about spatial relations, while text-to-text attention captures low-level syntactic knowledge between words. This concludes that language models in a multi-modal task learn different semantic information about objects and relations cross-modally and uni-modally (text-only). Our code is available here: https://github.com/GU-CLASP/attention-as-grounding.",
}

@article{parcalabescu2021valse,
  title={VALSE: A task-independent benchmark for vision and language models centered on linguistic phenomena},
  author={Parcalabescu, Letitia and Cafagna, Michele and Muradjan, Lilitta and Frank, Anette and Calixto, Iacer and Gatt, Albert},
  journal={arXiv preprint arXiv:2112.07566},
  year={2021}
}

@article{wang2018glue,
  title={{GLUE}: A multi-task benchmark and analysis platform for natural language understanding},
  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1804.07461},
  year={2018}
}

@inproceedings{zhou2020unified,
  title={Unified vision-language pre-training for image captioning and {VQA}},
  author={Zhou, Luowei and Palangi, Hamid and Zhang, Lei and Hu, Houdong and Corso, Jason and Gao, Jianfeng},
  booktitle={Proceedings of the AAAI conference on artificial intelligence},
  volume={34},
  number={07},
  pages={13041--13049},
  year={2020}
}

@article{su2019vl,
  title={{VL-BERT}: Pre-training of generic visual-linguistic representations},
  author={Su, Weijie and Zhu, Xizhou and Cao, Yue and Li, Bin and Lu, Lewei and Wei, Furu and Dai, Jifeng},
  journal={arXiv preprint arXiv:1908.08530},
  year={2019}
}

@article{tan2019lxmert,
  title={{LXMERT}: Learning cross-modality encoder representations from transformers},
  author={Tan, Hao and Bansal, Mohit},
  journal={arXiv preprint arXiv:1908.07490},
  year={2019}
}

@inproceedings{ilinykh-dobnik-2022-decoding,
    title = "{Do Decoding Algorithms Capture Discourse Structure in Multi-Modal Tasks? A Case Study of Image Paragraph Generation}",
    author = "Ilinykh, Nikolai  and
      Dobnik, Simon",
    booktitle = "Proceedings of the 2nd Workshop on Natural Language Generation, Evaluation, and Metrics (GEM)",
    month = dec,
    year = "2022",
    address = "Abu Dhabi, United Arab Emirates (Hybrid)",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2022.gem-1.45",
    pages = "480--493",
    abstract = "This paper describes insights into how different inference algorithms structure discourse in image paragraphs. We train a multi-modal transformer and compare 11 variations of decoding algorithms. We propose to evaluate image paragraphs not only with standard automatic metrics, but also with a more extensive, {``}under the hood{''} analysis of the discourse formed by sentences. Our results show that while decoding algorithms can be unfaithful to the reference texts, they still generate grounded descriptions, but they also lack understanding of the discourse structure and differ from humans in terms of attentional structure over images.",
}


@article{frank2021vision,
  title={Vision-and-Language or Vision-for-Language? on cross-modal influence in multimodal transformers},
  author={Frank, Stella and Bugliarello, Emanuele and Elliott, Desmond},
  journal={arXiv preprint arXiv:2109.04448},
  year={2021}
}


@article{brown2020language,
  title={Language models are few-shot learners},
  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={1877--1901},
  year={2020}
}

@article{radford2019language,
  title={Language models are unsupervised multitask learners},
  author={Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya and others},
  journal={OpenAI blog},
  volume={1},
  number={8},
  pages={9},
  year={2019}
}

@article{grosz1995centering,
  title={Centering: A framework for modelling the local coherence of discourse},
  author={Grosz, Barbara J and Joshi, Aravind K and Weinstein, Scott},
  year={1995}
}

@book{clark1996using,
  title={Using language},
  author={Clark, Herbert H},
  year={1996},
  publisher={Cambridge university press}
}

@article{kenningtonsymbol,
  title={The Symbol Grounding Problem Re-framed as Concreteness-Abstractness Learned through Spoken Interaction},
  author={Kennington, Casey and Natouf, Osama}
}

@inproceedings{ilinykh2018task,
  title={The task matters: Comparing image captioning and task-based dialogical image description},
  author={Ilinykh, Nikolai and Zarrie{\ss}, Sina and Schlangen, David},
  booktitle={Proceedings of the 11th International Conference on Natural Language Generation},
  pages={397--402},
  year={2018}
}

@article{bernardi2021linguistic,
  title={Linguistic issues behind visual question answering},
  author={Bernardi, Raffaella and Pezzelle, Sandro},
  journal={Language and Linguistics Compass},
  volume={15},
  number={6},
  pages={elnc3--12417},
  year={2021},
  publisher={Wiley Online Library}
}

@article{liu2019linguistic,
  title={Linguistic knowledge and transferability of contextual representations},
  author={Liu, Nelson F and Gardner, Matt and Belinkov, Yonatan and Peters, Matthew E and Smith, Noah A},
  journal={arXiv preprint arXiv:1903.08855},
  year={2019}
}

@article{elazar2021amnesic,
  title={Amnesic probing: Behavioral explanation with amnesic counterfactuals},
  author={Elazar, Yanai and Ravfogel, Shauli and Jacovi, Alon and Goldberg, Yoav},
  journal={Transactions of the Association for Computational Linguistics},
  volume={9},
  pages={160--175},
  year={2021},
  publisher={MIT Press}
}

@article{clark1997bridging,
  title={Bridging},
  author={Clark, Herbert},
  journal={Thinking: Readings in Cognitive Science},
  year={1977}
}

@article{zhang2022visual,
  title={Visual commonsense in pretrained unimodal and multimodal models},
  author={Zhang, Chenyu and Van Durme, Benjamin and Li, Zhuowan and Stengel-Eskin, Elias},
  journal={arXiv preprint arXiv:2205.01850},
  year={2022}
}

@inproceedings{Anderson2017up-down,
  author = {Peter Anderson and Xiaodong He and Chris Buehler and Damien Teney and Mark Johnson and Stephen Gould and Lei Zhang},
  title = {Bottom-Up and Top-Down Attention for Image Captioning and Visual Question Answering},
  booktitle={CVPR},
  year = {2018}
}

@article{mikolov2013efficient,
  title={Efficient estimation of word representations in vector space},
  author={Mikolov, Tomas and Chen, Kai and Corrado, Greg and Dean, Jeffrey},
  journal={arXiv preprint arXiv:1301.3781},
  year={2013}
}

@inproceedings{pennington2014glove,
  title={Glove: Global vectors for word representation},
  author={Pennington, Jeffrey and Socher, Richard and Manning, Christopher D},
  booktitle={Proceedings of the 2014 conference on empirical methods in natural language processing (EMNLP)},
  pages={1532--1543},
  year={2014}
}

@inproceedings{he2016deep,
  title={Deep residual learning for image recognition},
  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
  pages={770--778},
  year={2016}
}

@article{alikhani2020clue,
  title={Clue: Cross-modal coherence modeling for caption generation},
  author={Alikhani, Malihe and Sharma, Piyush and Li, Shengjie and Soricut, Radu and Stone, Matthew},
  journal={arXiv preprint arXiv:2005.00908},
  year={2020}
}

@inproceedings{ilinykh2022look, title={Look and Answer the Question: On the Role of Vision in Embodied Question Answering}, author={Ilinykh, Nikolai and Emampoor, Yasmeen and Dobnik, Simon}, booktitle={Proceedings of the 15th International Conference on Natural Language Generation}, pages={236--245}, year={2022} }


@inproceedings{hong-etal-2020-diverse,
TITLE = {Diverse and Relevant Visual Storytelling with Scene Graph Embeddings},
AUTHOR = {Hong, Xudong and Shetty, Rakshith and Sayeed, Asad and Mehra, Khushboo and Demberg, Vera and Schiele, Bernt},
LANGUAGE = {eng},
ISBN = {978-1-952148-63-7},
DOI = {10.18653/v1/2020.conll-1.34},
PUBLISHER = {ACL},
YEAR = {2020},
MARGINALMARK = {$\bullet$},
BOOKTITLE = {Proceedings of the 24th Conference on Computational Natural Language Learning (CoNLL 2020)},
PAGES = {420--430},
ADDRESS = {Online},
}

@article{liang2022holistic,
  title={Holistic evaluation of language models},
  author={Liang, Percy and Bommasani, Rishi and Lee, Tony and Tsipras, Dimitris and Soylu, Dilara and Yasunaga, Michihiro and Zhang, Yian and Narayanan, Deepak and Wu, Yuhuai and Kumar, Ananya and others},
  journal={arXiv preprint arXiv:2211.09110},
  year={2022}
}

@article{yagcioglu2018recipeqa,
  title={Recipeqa: A challenge dataset for multimodal comprehension of cooking recipes},
  author={Yagcioglu, Semih and Erdem, Aykut and Erdem, Erkut and Ikizler-Cinbis, Nazli},
  journal={arXiv preprint arXiv:1809.00812},
  year={2018}
}

@article{hong2023visual,
  title={Visual Writing Prompts: Character-Grounded Story Generation with Curated Image Sequences},
  author={Hong, Xudong and Sayeed, Asad and Mehra, Khushboo and Demberg, Vera and Schiele, Bernt},
  journal={Transactions of the Association for Computational Linguistics},
  volume={11},
  pages={565--581},
  year={2023},
  publisher={MIT Press}
}