@inproceedings{zhuo-etal-2023-vilpact,
    title = "{V}i{LPA}ct: A Benchmark for Compositional Generalization on Multimodal Human Activities",
    author = "Zhuo, Terry Yue  and
      Liao, Yaqing  and
      Lei, Yuecheng  and
      Qu, Lizhen  and
      de Melo, Gerard  and
      Chang, Xiaojun  and
      Ren, Yazhou  and
      Xu, Zenglin",
    editor = "Vlachos, Andreas  and
      Augenstein, Isabelle",
    booktitle = "Findings of the Association for Computational Linguistics: EACL 2023",
    month = may,
    year = "2023",
    address = "Dubrovnik, Croatia",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2023.findings-eacl.164/",
    doi = "10.18653/v1/2023.findings-eacl.164",
    pages = "2192--2207",
    abstract = "We introduce $\texttt{ViLPAct}$, a novel vision-language benchmark for human activity planning. It is designed for a task where embodied AI agents can reason and forecast future actions of humans based on video clips about their initial activities and intents in text. The dataset consists of 2.9k videos from $\texttt{Charades}$ extended with intents via crowdsourcing, a multi-choice question test set, and four strong baselines. One of the baselines implements a neurosymbolic approach based on a multi-modal knowledge base (MKB), while the other ones are deep generative models adapted from recent state-of-the-art (SOTA) methods. According to our extensive experiments, the key challenges are compositional generalization and effective use of information from both modalities."
}