Optim

toydl.core.optim.Momentum

Momentum(parameters: Sequence[Parameter], lr: float = 0.01, momentum: float = 0.9)

Bases: Optimizer

Momentum Optimizer

Init the Momentum optimizer

Parameters:

Name	Type	Description	Default
parameters	Sequence[Parameter]	the parameters that will be optimized	required
lr	float	learning rate	0.01
momentum	float	momentum coefficient	0.9

Source code in toydl/core/optim/_momentum.py

def __init__(
    self, parameters: Sequence[Parameter], lr: float = 0.01, momentum: float = 0.9
):
    """
    Init the Momentum optimizer

    :param parameters: the parameters that will be optimized
    :param lr: learning rate
    :param momentum: momentum coefficient
    """
    super().__init__(parameters)
    self.lr = lr
    self.momentum = momentum
    self.parameter_delta_map: dict[Parameter, float] = {}

step

step() -> None

Run a momentum step to update parameter value

Source code in toydl/core/optim/_momentum.py

def step(self) -> None:
    """
    Run a momentum step to update parameter value
    """
    for p in self.parameters:
        if hasattr(p.value, "derivative") and p.value.derivative is not None:
            delta = (
                -self.lr * p.value.derivative
                + self.momentum * self.parameter_delta_map.get(p, 0)
            )
            self.parameter_delta_map[p] = delta
            new_value = p.value + delta
            p.update(new_value)

zero_grad

zero_grad() -> None

Clear the grad/derivative value of parameter

Source code in toydl/core/optim/_momentum.py

def zero_grad(self) -> None:
    """
    Clear the grad/derivative value of parameter
    """
    for p in self.parameters:
        if hasattr(p.value, "derivative") and p.value.derivative is not None:
            p.value.derivative = None

toydl.core.optim.Optimizer

Optimizer(parameters: Sequence[Parameter])

The Optimizer base class

Source code in toydl/core/optim/_base.py

def __init__(self, parameters: Sequence[Parameter]):
    self.parameters = parameters

toydl.core.optim.SGD

SGD(parameters: Sequence[Parameter], lr: float = 1.0)

Bases: Optimizer

Stochastic Gradient Descent Optimizer

Init the SGD optimizer

Parameters:

Name	Type	Description	Default
parameters	Sequence[Parameter]	the parameters that will be optimized	required
lr	float	learning rate	1.0

Source code in toydl/core/optim/_sgd.py

def __init__(self, parameters: Sequence[Parameter], lr: float = 1.0):
    """
    Init the SGD optimizer

    :param parameters: the parameters that will be optimized
    :param lr: learning rate
    """
    super().__init__(parameters)
    self.lr = lr

step

step() -> None

Run a sgd step to update parameter value

Source code in toydl/core/optim/_sgd.py

def step(self) -> None:
    """
    Run a sgd step to update parameter value
    """
    for p in self.parameters:
        if hasattr(p.value, "derivative") and p.value.derivative is not None:
            new_value = p.value - self.lr * p.value.derivative
            p.update(new_value)

zero_grad

zero_grad() -> None

Clear the grad/derivative value of parameter

Source code in toydl/core/optim/_sgd.py

def zero_grad(self) -> None:
    """
    Clear the grad/derivative value of parameter
    """
    for p in self.parameters:
        if hasattr(p.value, "derivative") and p.value.derivative is not None:
            p.value.derivative = None