@InProceedings{jiang-ilievski-ma:2024:SemEval2024,
  author    = {Jiang, Yifan  and  Ilievski, Filip  and  Ma, Kaixin},
  title     = {SemEval-2024 Task 9: BRAINTEASER: A Novel Task Defying Common Sense},
  booktitle      = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
  month          = {June},
  year           = {2024},
  address        = {Mexico City, Mexico},
  publisher      = {Association for Computational Linguistics},
  pages     = {1996--2010},
  abstract  = {While vertical thinking relies on logical and commonsense reasoning, lateral thinking requires systems to defy commonsense associations and overwrite them through unconventional thinking. Lateral thinking has been shown to be challenging for current models but has received little attention. A recent benchmark, BRAINTEASER, aims to evaluate current models' lateral thinking ability in a zero-shot setting. In this paper, we split the original benchmark to also support fine-tuning setting and present SemEval Task 9, BRAINTEASER(S), the first task at this competition designed to test the system's reasoning and lateral thinking ability. As a popular task, BRAINTEASER(S)'s two subtasks receive 483 team submissions from 182 participants during the competition. This paper provides a fine-grained system analysis of the competition results, together with a reflection on what this means for the ability of the systems to reason laterally. We hope that the BRAINTEASER(S) subtasks and findings in this paper can stimulate future work on lateral thinking and robust reasoning by computational models},
  url       = {https://aclanthology.org/2024.semeval2024-1.271}
}

@misc{liu2023abstractive,
      title={Abstractive Summarization of Large Document Collections Using GPT}, 
      author={Sengjie Liu and Christopher G. Healey},
      year={2023},
      eprint={2310.05690},
      archivePrefix={arXiv},
      primaryClass={cs.AI}
}

@misc{ye2023comprehensive,
      title={A Comprehensive Capability Analysis of GPT-3 and GPT-3.5 Series Models}, 
      author={Junjie Ye and Xuanting Chen and Nuo Xu and Can Zu and Zekai Shao and Shichun Liu and Yuhan Cui and Zeyang Zhou and Chao Gong and Yang Shen and Jie Zhou and Siming Chen and Tao Gui and Qi Zhang and Xuanjing Huang},
      year={2023},
      eprint={2303.10420},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{bahak2023evaluating,
      title={Evaluating ChatGPT as a Question Answering System: A Comprehensive Analysis and Comparison with Existing Models}, 
      author={Hossein Bahak and Farzaneh Taheri and Zahra Zojaji and Arefeh Kazemi},
      year={2023},
      eprint={2312.07592},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@inproceedings{NIPS2017_3f5ee243,
 author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, \L ukasz and Polosukhin, Illia},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Attention is All you Need},
 url = {https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf},
 volume = {30},
 year = {2017}
}


@misc{gpt3.5,
      title={Language Models are Few-Shot Learners}, 
      author={Tom B. Brown and Benjamin Mann and Nick Ryder and Melanie Subbiah and Jared Kaplan and Prafulla Dhariwal and Arvind Neelakantan and Pranav Shyam and Girish Sastry and Amanda Askell and Sandhini Agarwal and Ariel Herbert-Voss and Gretchen Krueger and Tom Henighan and Rewon Child and Aditya Ramesh and Daniel M. Ziegler and Jeffrey Wu and Clemens Winter and Christopher Hesse and Mark Chen and Eric Sigler and Mateusz Litwin and Scott Gray and Benjamin Chess and Jack Clark and Christopher Berner and Sam McCandlish and Alec Radford and Ilya Sutskever and Dario Amodei},
      year={2020},
      eprint={2005.14165},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@inproceedings{wang-etal-2023-towards,
    title = "Towards Understanding Chain-of-Thought Prompting: An Empirical Study of What Matters",
    author = "Wang, Boshi  and
      Min, Sewon  and
      Deng, Xiang  and
      Shen, Jiaming  and
      Wu, You  and
      Zettlemoyer, Luke  and
      Sun, Huan",
    editor = "Rogers, Anna  and
      Boyd-Graber, Jordan  and
      Okazaki, Naoaki",
    booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    month = jul,
    year = "2023",
    address = "Toronto, Canada",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.acl-long.153",
    doi = "10.18653/v1/2023.acl-long.153",
    pages = "2717--2739",
    abstract = "Chain-of-Thought (CoT) prompting can dramatically improve the multi-step reasoning abilities of large language models (LLMs). CoT explicitly encourages the LLM to generate intermediate rationales for solving a problem, by providing a series of reasoning steps in the demonstrations. Despite its success, there is still little understanding of what makes CoT prompting effective and which aspects of the demonstrated reasoning steps contribute to its performance. In this paper, we show that CoT reasoning is possible even with invalid demonstrations - prompting with invalid reasoning steps can achieve over 80-90{\%} of the performance obtained using CoT under various metrics, while still generating coherent lines of reasoning during inference. Further experiments show that other aspects of the rationales, such as being relevant to the query and correctly ordering the reasoning steps, are much more important for effective CoT reasoning. Overall, these findings both deepen our understanding of CoT prompting, and open up new questions regarding LLMs{'} capability to learn to reason in context.",
}

@article{waks1997lateral,
  title={Lateral thinking and technology education},
  author={Waks, Shlomo},
  journal={Journal of Science Education and Technology},
  volume={6},
  pages={245--255},
  year={1997},
  publisher={Springer}
}

@article{devlin2018bert,
  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  journal={arXiv preprint arXiv:1810.04805},
  year={2018}
}

@misc{openai2022,
  author       = {OpenAI},
  title        = {ChatGPT: A Language Model by OpenAI},
  howpublished = {\url{https://www.openai.com}},
  year         = {2022},
}

@article{talmor2018commonsenseqa,
  title={Commonsenseqa: A question answering challenge targeting commonsense knowledge},
  author={Talmor, Alon and Herzig, Jonathan and Lourie, Nicholas and Berant, Jonathan},
  journal={arXiv preprint arXiv:1811.00937},
  year={2018}
}

@inproceedings{bisk2020piqa,
  title={Piqa: Reasoning about physical commonsense in natural language},
  author={Bisk, Yonatan and Zellers, Rowan and Gao, Jianfeng and Choi, Yejin and others},
  booktitle={Proceedings of the AAAI conference on artificial intelligence},
  volume={34},
  number={05},
  pages={7432--7439},
  year={2020}
}

@inproceedings{sap-etal-2019-social,
    title = "Social {IQ}a: Commonsense Reasoning about Social Interactions",
    author = "Sap, Maarten  and
      Rashkin, Hannah  and
      Chen, Derek  and
      Le Bras, Ronan  and
      Choi, Yejin",
    editor = "Inui, Kentaro  and
      Jiang, Jing  and
      Ng, Vincent  and
      Wan, Xiaojun",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)",
    month = nov,
    year = "2019",
    address = "Hong Kong, China",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/D19-1454",
    doi = "10.18653/v1/D19-1454",
    pages = "4463--4473",
    abstract = "We introduce Social IQa, the first large-scale benchmark for commonsense reasoning about social situations. Social IQa contains 38,000 multiple choice questions for probing emotional and social intelligence in a variety of everyday situations (e.g., Q: {``}Jordan wanted to tell Tracy a secret, so Jordan leaned towards Tracy. Why did Jordan do this?{''} A: {``}Make sure no one else could hear{''}). Through crowdsourcing, we collect commonsense questions along with correct and incorrect answers about social interactions, using a new framework that mitigates stylistic artifacts in incorrect answers by asking workers to provide the right answer to a different but related question. Empirical results show that our benchmark is challenging for existing question-answering models based on pretrained language models, compared to human performance ({\textgreater}20{\%} gap). Notably, we further establish Social IQa as a resource for transfer learning of commonsense knowledge, achieving state-of-the-art performance on multiple commonsense reasoning tasks (Winograd Schemas, COPA).",
}

@misc{sanh2022multitask,
      title={Multitask Prompted Training Enables Zero-Shot Task Generalization}, 
      author={Victor Sanh and Albert Webson and Colin Raffel and Stephen H. Bach and Lintang Sutawika and Zaid Alyafeai and Antoine Chaffin and Arnaud Stiegler and Teven Le Scao and Arun Raja and Manan Dey and M Saiful Bari and Canwen Xu and Urmish Thakker and Shanya Sharma Sharma and Eliza Szczechla and Taewoon Kim and Gunjan Chhablani and Nihal Nayak and Debajyoti Datta and Jonathan Chang and Mike Tian-Jian Jiang and Han Wang and Matteo Manica and Sheng Shen and Zheng Xin Yong and Harshit Pandey and Rachel Bawden and Thomas Wang and Trishala Neeraj and Jos Rozen and Abheesht Sharma and Andrea Santilli and Thibault Fevry and Jason Alan Fries and Ryan Teehan and Tali Bers and Stella Biderman and Leo Gao and Thomas Wolf and Alexander M. Rush},
      year={2022},
      eprint={2110.08207},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@misc{chung2022scaling,
      title={Scaling Instruction-Finetuned Language Models}, 
      author={Hyung Won Chung and Le Hou and Shayne Longpre and Barret Zoph and Yi Tay and William Fedus and Yunxuan Li and Xuezhi Wang and Mostafa Dehghani and Siddhartha Brahma and Albert Webson and Shixiang Shane Gu and Zhuyun Dai and Mirac Suzgun and Xinyun Chen and Aakanksha Chowdhery and Alex Castro-Ros and Marie Pellat and Kevin Robinson and Dasha Valter and Sharan Narang and Gaurav Mishra and Adams Yu and Vincent Zhao and Yanping Huang and Andrew Dai and Hongkun Yu and Slav Petrov and Ed H. Chi and Jeff Dean and Jacob Devlin and Adam Roberts and Denny Zhou and Quoc V. Le and Jason Wei},
      year={2022},
      eprint={2210.11416},
      archivePrefix={arXiv},
      primaryClass={cs.LG}
}

@inproceedings{jiang-etal-2023-brainteaser,
    title = "{BRAINTEASER}: Lateral Thinking Puzzles for Large Language Models",
    author = "Jiang, Yifan  and
      Ilievski, Filip  and
      Ma, Kaixin  and
      Sourati, Zhivar",
    editor = "Bouamor, Houda  and
      Pino, Juan  and
      Bali, Kalika",
    booktitle = "Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing",
    month = dec,
    year = "2023",
    address = "Singapore",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2023.emnlp-main.885",
    doi = "10.18653/v1/2023.emnlp-main.885",
    pages = "14317--14332",
    abstract = "The success of language models has inspired the NLP community to attend to tasks that require implicit and complex reasoning, relying on human-like commonsense mechanisms. While such vertical thinking tasks have been relatively popular, lateral thinking puzzles have received little attention. To bridge this gap, we devise BrainTeaser: a multiple-choice Question Answering task designed to test the model{'}s ability to exhibit lateral thinking and defy default commonsense associations. We design a three-step procedure for creating the first lateral thinking benchmark, consisting of data collection, distractor generation, and generation of adversarial examples, leading to 1,100 puzzles with high-quality annotations. To assess the consistency of lateral reasoning by models, we enrich BrainTeaser based on a semantic and contextual reconstruction of its questions. Our experiments with state-of-the-art instruction- and commonsense language models reveal a significant gap between human and model performance, which is further widened when consistency across adversarial formats is considered. We make all of our code and data available to stimulate work on developing and evaluating lateral thinking models.",
}

@article{liu2023pre,
  title={Pre-train, prompt, and predict: A systematic survey of prompting methods in natural language processing},
  author={Liu, Pengfei and Yuan, Weizhe and Fu, Jinlan and Jiang, Zhengbao and Hayashi, Hiroaki and Neubig, Graham},
  journal={ACM Computing Surveys},
  volume={55},
  number={9},
  pages={1--35},
  year={2023},
  publisher={ACM New York, NY}
}

@article{lu2021fantastically,
  title={Fantastically ordered prompts and where to find them: Overcoming few-shot prompt order sensitivity},
  author={Lu, Yao and Bartolo, Max and Moore, Alastair and Riedel, Sebastian and Stenetorp, Pontus},
  journal={arXiv preprint arXiv:2104.08786},
  year={2021}
}

@article{ma2024fairness,
  title={Fairness-guided few-shot prompting for large language models},
  author={Ma, Huan and Zhang, Changqing and Bian, Yatao and Liu, Lemao and Zhang, Zhirui and Zhao, Peilin and Zhang, Shu and Fu, Huazhu and Hu, Qinghua and Wu, Bingzhe},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2024}
}

@article{robinson2022leveraging,
  title={Leveraging large language models for multiple choice question answering},
  author={Robinson, Joshua and Rytting, Christopher Michael and Wingate, David},
  journal={arXiv preprint arXiv:2210.12353},
  year={2022}
}

@article{zheng2023large,
  title={On Large Language Models' Selection Bias in Multi-Choice Questions},
  author={Zheng, Chujie and Zhou, Hao and Meng, Fandong and Zhou, Jie and Huang, Minlie},
  journal={arXiv preprint arXiv:2309.03882},
  year={2023}
}