@inproceedings{metel-etal-2024-draft,
    title = "Draft on the Fly: Adaptive Self-Speculative Decoding using Cosine Similarity",
    author = "Metel, Michael R.  and
      Lu, Peng  and
      Chen, Boxing  and
      Rezagholizadeh, Mehdi  and
      Kobyzev, Ivan",
    editor = "Al-Onaizan, Yaser  and
      Bansal, Mohit  and
      Chen, Yun-Nung",
    booktitle = "Findings of the Association for Computational Linguistics: EMNLP 2024",
    month = nov,
    year = "2024",
    address = "Miami, Florida, USA",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/jlcl-multiple-ingestion/2024.findings-emnlp.124/",
    doi = "10.18653/v1/2024.findings-emnlp.124",
    pages = "2267--2272",
    abstract = "We present a simple on the fly method for faster inference of large language models. Unlike other (self-)speculative decoding techniques, our method does not require fine-tuning or black-box optimization to generate a fixed draft model, relying instead on simple rules to generate varying draft models adapted to the input context. We show empirically that our light-weight algorithm is competitive with the current SOTA for self-speculative decoding, while being a truly plug-and-play method."
}