% Use this file for citations not found in the ACL Anthology (contained in "anthology.bib").

@inproceedings{jiang-etal-2023-brainteaser, title = "{BRAINTEASER}: Lateral Thinking Puzzles for Large Language Models", author = "Jiang, Yifan and Ilievski, Filip and Ma, Kaixin and Sourati, Zhivar", editor = "Bouamor, Houda and Pino, Juan and Bali, Kalika", booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing", month = dec, year = "2023", address = "Singapore", publisher = "Association for Computational Linguistics", url = "https://aclanthology.org/2023.emnlp-main.885", doi = "10.18653/v1/2023.emnlp-main.885", pages = "14317--14332", abstract = "The success of language models has inspired the NLP community to attend to tasks that require implicit and complex reasoning, relying on human-like commonsense mechanisms. While such vertical thinking tasks have been relatively popular, lateral thinking puzzles have received little attention. To bridge this gap, we devise BrainTeaser: a multiple-choice Question Answering task designed to test the model{'}s ability to exhibit lateral thinking and defy default commonsense associations. We design a three-step procedure for creating the first lateral thinking benchmark, consisting of data collection, distractor generation, and generation of adversarial examples, leading to 1,100 puzzles with high-quality annotations. To assess the consistency of lateral reasoning by models, we enrich BrainTeaser based on a semantic and contextual reconstruction of its questions. Our experiments with state-of-the-art instruction- and commonsense language models reveal a significant gap between human and model performance, which is further widened when consistency across adversarial formats is considered. We make all of our code and data available to stimulate work on developing and evaluating lateral thinking models."}

@InProceedings{jiang-ilievski-ma:2024:SemEval2024,
  author    = {Jiang, Yifan  and  Ilievski, Filip  and  Ma, Kaixin},
  title     = {SemEval-2024 Task 9: BRAINTEASER: A Novel Task Defying Common Sense},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {1996--2010},
  abstract  = {While vertical thinking relies on logical and commonsense reasoning, lateral thinking requires systems to defy commonsense associations and overwrite them through unconventional thinking. Lateral thinking has been shown to be challenging for current models but has received little attention. A recent benchmark, BRAINTEASER, aims to evaluate current models' lateral thinking ability in a zero-shot setting. In this paper, we split the original benchmark to also support fine-tuning setting and present SemEval Task 9, BRAINTEASER(S), the first task at this competition designed to test the system's reasoning and lateral thinking ability. As a popular task, BRAINTEASER(S)'s two subtasks receive 483 team submissions from 182 participants during the competition. This paper provides a fine-grained system analysis of the competition results, together with a reflection on what this means for the ability of the systems to reason laterally. We hope that the BRAINTEASER(S) subtasks and findings in this paper can stimulate future work on lateral thinking and robust reasoning by computational models},
  url       = {https://aclanthology.org/2024.semeval2024-1.271}
}


@misc{xlmroberta,
      title={Unsupervised Cross-lingual Representation Learning at Scale}, 
      author={Alexis Conneau and Kartikay Khandelwal and Naman Goyal and Vishrav Chaudhary and Guillaume Wenzek and Francisco Guzmán and Edouard Grave and Myle Ott and Luke Zettlemoyer and Veselin Stoyanov},
      year={2020},
      eprint={1911.02116},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}


@misc{mixtral,
      title={Mixtral of Experts}, 
      author={Albert Q. Jiang and Alexandre Sablayrolles and Antoine Roux and Arthur Mensch and Blanche Savary and Chris Bamford and Devendra Singh Chaplot and Diego de las Casas and Emma Bou Hanna and Florian Bressand and Gianna Lengyel and Guillaume Bour and Guillaume Lample and Lélio Renard Lavaud and Lucile Saulnier and Marie-Anne Lachaux and Pierre Stock and Sandeep Subramanian and Sophia Yang and Szymon Antoniak and Teven Le Scao and Théophile Gervet and Thibaut Lavril and Thomas Wang and Timothée Lacroix and William El Sayed},
      year={2024},
      eprint={2401.04088},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}


@misc{stepbystep,
      title={Large Language Models Are Human-Level Prompt Engineers}, 
      author={Yongchao Zhou and Andrei Ioan Muresanu and Ziwen Han and Keiran Paster and Silviu Pitis and Harris Chan and Jimmy Ba},
      year={2023},
      eprint={2211.01910},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}


@misc{birdqa,
      title={BiRdQA: A Bilingual Dataset for Question Answering on Tricky Riddles}, 
      author={Yunxiang Zhang and Xiaojun Wan},
      year={2022},
      eprint={2109.11087},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@InProceedings{riddlesense,
title={RiddleSense: Reasoning about Riddle Questions Featuring Linguistic Creativity and Commonsense Knowledge},
author={Lin, Bill Yuchen and Wu, Ziyi and Yang, Yichi and Lee, Dong-Ho and Ren, Xiang},
journal={Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics (ACL-IJCNLP 2021): Findings},
year={2021}
}

@misc{reconcile,
      title={ReConcile: Round-Table Conference Improves Reasoning via Consensus among Diverse LLMs}, 
      author={Justin Chih-Yao Chen and Swarnadeep Saha and Mohit Bansal},
      year={2023},
      eprint={2309.13007},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}