@inproceedings{acharya-aryal-2025-howard,
    title = "{H}oward {U}niversity-{AI}4{PC} at {S}em{E}val-2025 Task 4: Unlearning Sensitive Content From Large Language Models Using Finetuning and Distillation for Selective Knowledge Removal",
    author = "Acharya, Aayush  and
      Aryal, Saurav",
    editor = "Rosenthal, Sara  and
      Ros{\'a}, Aiala  and
      Ghosh, Debanjan  and
      Zampieri, Marcos",
    booktitle = "Proceedings of the 19th International Workshop on Semantic Evaluation (SemEval-2025)",
    month = jul,
    year = "2025",
    address = "Vienna, Austria",
    publisher = "Association for Computational Linguistics",
    url = "https://preview.aclanthology.org/corrections-2025-08/2025.semeval-1.233/",
    pages = "1772--1776",
    ISBN = "979-8-89176-273-2",
    abstract = "This paper presents our approach and submission to the SemEval 2025 task on ``Unlearning Sensitive Content from Large Language Models.'' The task focuses on making LLMs forget specific knowledge, such as copyrighted material and personally identifiable information (PII), without needing expensive retraining from scratch on the OLMo model. We propose a method to unlearn using fine-tuning and knowledge distillation. Our approach involves fine-tuning separate models on ``retain'' and ``forget'' datasets to preserve or suppress knowledge selectively. We then distill the model by suppressing logarithmic data from the fine-tuned model without learning using a combined loss of L2, KL divergence and cosine similarity while retaining knowledge from the fine-tuned model with retention using KL divergence loss."
}