From c263e2cf130a3c6b5ef93ddb4969dcc8e532abf7 Mon Sep 17 00:00:00 2001 From: GassiGiuseppe Date: Wed, 8 Oct 2025 12:13:02 +0200 Subject: [PATCH] Custom learning rate sheduler from Attention is all you need --- .../Libs/Training/learning_rade_shedulers.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 Project_Model/Libs/Training/learning_rade_shedulers.py diff --git a/Project_Model/Libs/Training/learning_rade_shedulers.py b/Project_Model/Libs/Training/learning_rade_shedulers.py new file mode 100644 index 0000000..08bc319 --- /dev/null +++ b/Project_Model/Libs/Training/learning_rade_shedulers.py @@ -0,0 +1,39 @@ +import numpy as np +# custom LR from attention is all you need +class Custom_lr(): + def __init__(self, d_model: int, warmup_step:int) -> None: + + self.__d_model = d_model + self.__warmup_step = warmup_step + + + def get_lr(self,epoch) -> int: + return (self.__d_model ** -0.5) * min(epoch ** -0.5, + epoch * (self.__warmup_step ** -1.5)) + +# OTHER LR + +# Learning rate schedules (matching visualization parameters) +def step_lr(epoch, lr): + # StepLR: step_size=20, gamma=0.5 (from visualization) + return lr * 0.5 if epoch % 20 == 0 and epoch > 0 else lr + +def exp_lr(epoch, lr): + # ExponentialLR: gamma=0.95 (from visualization) + return lr * 0.95 + +def cosine_lr(epoch, lr): + # CosineAnnealingLR: lr_min=0.001, lr_max=0.1, max_epochs=100 (from visualization) + lr_min, lr_max = 0.001, 0.1 + max_epochs = 100 + return lr_min + 0.5 * (lr_max - lr_min) * (1 + np.cos(epoch * np.pi / max_epochs)) + +def cyclical_lr(epoch, lr): + # CyclicalLR: base_lr=0.001, max_lr=0.1, step_size=20 (from visualization) + base_lr = 0.001 + max_lr = 0.1 + step_size = 20 + + cycle = np.floor(1 + epoch / (2 * step_size)) + x = np.abs(epoch / step_size - 2 * cycle + 1) + return base_lr + (max_lr - base_lr) * max(0, (1 - x)) \ No newline at end of file