/
Handdas
/
FirstOrderAlgorythms
Обзор
Документация
Войти
/
Handdas
/
FirstOrderAlgorythms
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
CI/CD
Аналитика
Безопасность
master
blackbox_optimizer/algorithms/first_order.py
176 строк
6 KB
Your Name
initial commit
09 июн 2026, 10:13
09 июн 2026, 10:13
f88c506
Код
Авторство
О чём код?
"""First-order gradient optimizers.""" from __future__ import annotations import numpy as np from ..core.base import BaseOptimizer class SGDOptimizer(BaseOptimizer): def step(self, params: np.ndarray, grad: np.ndarray) -> np.ndarray: self.iteration += 1 g = self._regularize_grad(params, grad) return params - self.learning_rate * g class MomentumOptimizer(BaseOptimizer): def __init__(self, learning_rate: float, *, momentum: float = 0.9, **kwargs: float) -> None: super().__init__(learning_rate, **kwargs) self.momentum = momentum self.velocity: np.ndarray | None = None def reset_state(self) -> None: super().reset_state() self.velocity = None def step(self, params: np.ndarray, grad: np.ndarray) -> np.ndarray: self.iteration += 1 g = self._regularize_grad(params, grad) if self.velocity is None: self.velocity = np.zeros_like(params) self.velocity = self.momentum * self.velocity + g return params - self.learning_rate * self.velocity class NesterovOptimizer(BaseOptimizer): def __init__(self, learning_rate: float, *, momentum: float = 0.9, **kwargs: float) -> None: super().__init__(learning_rate, **kwargs) self.momentum = momentum self.velocity: np.ndarray | None = None def reset_state(self) -> None: super().reset_state() self.velocity = None def step(self, params: np.ndarray, grad: np.ndarray) -> np.ndarray: self.iteration += 1 g = self._regularize_grad(params, grad) if self.velocity is None: self.velocity = np.zeros_like(params) prev_velocity = self.velocity.copy() self.velocity = self.momentum * self.velocity + g lookahead = self.momentum * prev_velocity + g return params - self.learning_rate * lookahead class RMSpropOptimizer(BaseOptimizer): def __init__( self, learning_rate: float, *, beta2: float = 0.99, eps: float = 1e-8, **kwargs: float ) -> None: super().__init__(learning_rate, **kwargs) self.beta2 = beta2 self.eps = eps self.v: np.ndarray | None = None def reset_state(self) -> None: super().reset_state() self.v = None def step(self, params: np.ndarray, grad: np.ndarray) -> np.ndarray: self.iteration += 1 g = self._regularize_grad(params, grad) if self.v is None: self.v = np.zeros_like(params) self.v = self.beta2 * self.v + (1.0 - self.beta2) * (g * g) return params - self.learning_rate * g / (np.sqrt(self.v) + self.eps) class AdamWOptimizer(BaseOptimizer): def __init__( self, learning_rate: float, *, beta1: float = 0.9, beta2: float = 0.999, eps: float = 1e-8, weight_decay: float = 0.01, ) -> None: super().__init__(learning_rate, weight_decay=weight_decay) self.beta1 = beta1 self.beta2 = beta2 self.eps = eps self.m: np.ndarray | None = None self.v: np.ndarray | None = None def reset_state(self) -> None: super().reset_state() self.m = None self.v = None def step(self, params: np.ndarray, grad: np.ndarray) -> np.ndarray: self.iteration += 1 g: np.ndarray = grad.astype(float, copy=True) if self.m is None: self.m = np.zeros_like(params) self.v = np.zeros_like(params) assert self.v is not None self.m = self.beta1 * self.m + (1.0 - self.beta1) * g self.v = self.beta2 * self.v + (1.0 - self.beta2) * (g * g) m_hat = self.m / (1.0 - self.beta1**self.iteration) v_hat = self.v / (1.0 - self.beta2**self.iteration) decayed = params * (1.0 - self.learning_rate * self.weight_decay) return decayed - self.learning_rate * m_hat / (np.sqrt(v_hat) + self.eps) class LionOptimizer(BaseOptimizer): def __init__( self, learning_rate: float, *, beta1: float = 0.9, beta2: float = 0.99, **kwargs: float ) -> None: super().__init__(learning_rate, **kwargs) self.beta1 = beta1 self.beta2 = beta2 self.m: np.ndarray | None = None def reset_state(self) -> None: super().reset_state() self.m = None def step(self, params: np.ndarray, grad: np.ndarray) -> np.ndarray: self.iteration += 1 g = self._regularize_grad(params, grad) if self.m is None: self.m = np.zeros_like(params) update = self.beta1 * self.m + (1.0 - self.beta1) * g params = params - self.learning_rate * np.sign(update) self.m = self.beta2 * self.m + (1.0 - self.beta2) * g return params class SophiaOptimizer(BaseOptimizer): """Simplified Sophia variant using diagonal Hessian approximation.""" def __init__( self, learning_rate: float, *, beta1: float = 0.965, beta2: float = 0.99, eps: float = 1e-8, rho: float = 0.03, **kwargs: float, ) -> None: super().__init__(learning_rate, **kwargs) self.beta1 = beta1 self.beta2 = beta2 self.eps = eps self.rho = rho self.m: np.ndarray | None = None self.h: np.ndarray | None = None def reset_state(self) -> None: super().reset_state() self.m = None self.h = None def step(self, params: np.ndarray, grad: np.ndarray) -> np.ndarray: self.iteration += 1 g = self._regularize_grad(params, grad) if self.m is None: self.m = np.zeros_like(params) self.h = np.zeros_like(params) assert self.h is not None self.m = self.beta1 * self.m + (1.0 - self.beta1) * g self.h = self.beta2 * self.h + (1.0 - self.beta2) * (g * g) denom = np.maximum(np.sqrt(self.h) + self.eps, self.rho) return params - self.learning_rate * self.m / denom