@inproceedings{kobayashi-etal-2022-diverse,
    title = "Diverse Lottery Tickets Boost Ensemble from a Single Pretrained Model",
    author = "Kobayashi, Sosuke  and
      Kiyono, Shun  and
      Suzuki, Jun  and
      Inui, Kentaro",
    editor = "Fan, Angela  and
      Ilic, Suzana  and
      Wolf, Thomas  and
      Gall{\'e}, Matthias",
    booktitle = "Proceedings of BigScience Episode {\#}5 -- Workshop on Challenges {\&} Perspectives in Creating Large Language Models",
    month = may,
    year = "2022",
    address = "virtual+Dublin",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/add-emnlp-2024-awards/2022.bigscience-1.4/",
    doi = "10.18653/v1/2022.bigscience-1.4",
    pages = "42--50",
    abstract = "Ensembling is a popular method used to improve performance as a last resort. However, ensembling multiple models finetuned from a single pretrained model has been not very effective; this could be due to the lack of diversity among ensemble members. This paper proposes Multi-Ticket Ensemble, which finetunes different subnetworks of a single pretrained model and ensembles them. We empirically demonstrated that winning-ticket subnetworks produced more diverse predictions than dense networks and their ensemble outperformed the standard ensemble in some tasks when accurate lottery tickets are found on the tasks."
}