@inproceedings{751643c733e447efb9682b7b2ca74450,
title = "Teacher-Free Knowledge Distillation Based on Non-Progressive Meta-Learned Simulated Annealing",
abstract = "Knowledge distillation (KD) is one of the effective methods in model compression. Recently, some teacher-free knowledge distillation (TFKD) methods have been proposed to overcome the problem that the teacher model is too huge in the traditional KD. In this paper, we proposed a TFKD framework based on a non-progressive method that the target distribution will change during training. We use simulated annealing (SA) to control the variety of free-form distribution. In addition, we also developed a meta-learned reinforcement learning (RL) method to control the energy calculation in SA. We conducted the experiments on CIFARIOO, CIFARIO, and SVHN with the VGG-8 model. The accuracy of the VGG-8 model can be improved to 72.88%, which is higher than the baseline model by 2.44%. Compared to another teacher-free method currently, the performance of our method is also the highest.",
keywords = "knowledge distillation, meta-learning, model compression, reinforcement learning, simulated annealing, Teacher-free knowledge distillation",
author = "Ho, {Pin Hsuan} and Jiang, {Bing Ru} and Lin, {Albert S.}",
note = "Publisher Copyright: {\textcopyright} 2024 IEEE.; 3rd IEEE International Conference on Computing and Machine Intelligence, ICMI 2024 ; Conference date: 13-04-2024 Through 14-04-2024",
year = "2024",
doi = "10.1109/ICMI60790.2024.10586018",
language = "English",
series = "2024 IEEE 3rd International Conference on Computing and Machine Intelligence, ICMI 2024 - Proceedings",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
editor = "Ahmed Abdelgawad and Akhtar Jamil and Hameed, {Alaa Ali}",
booktitle = "2024 IEEE 3rd International Conference on Computing and Machine Intelligence, ICMI 2024 - Proceedings",
address = "美國",
}