replace PiecewiseDecay, StepDecay, MultiStepDecay, LambdaDecay with 2.0 version (#53992)

longranger2 · web-flow · commit 63f242b6fb5a · 2023-06-28T11:49:42.000+08:00
* replace PiecewiseDecay(LearningRateDecay) with PiecewiseDecay(LRScheduler)

* fix bug

* fix bug

* replace the StepDecay,MultiStepDecay,LambdaDecay with 2.0 version
diff --git a/python/paddle/fluid/dygraph/learning_rate_scheduler.py b/python/paddle/fluid/dygraph/learning_rate_scheduler.py
@@ -23,13 +23,9 @@
 
 __all__ = [
     'NoamDecay',
-    'PiecewiseDecay',
     'PolynomialDecay',
     'LinearLrWarmup',
     'ReduceLROnPlateau',
-    'StepDecay',
-    'MultiStepDecay',
-    'LambdaDecay',
 ]
 
 
@@ -131,68 +127,6 @@ def step(self):
         raise NotImplementedError()
 
 
-class PiecewiseDecay(LearningRateDecay):
-    """
-    :api_attr: imperative
-
-    Piecewise decay scheduler.
-
-    The algorithm can be described as the code below.
-
-    .. code-block:: text
-
-        boundaries = [10000, 20000]
-        values = [1.0, 0.5, 0.1]
-        if global_step < 10000:
-            learning_rate = 1.0
-        elif 10000 <= global_step < 20000:
-            learning_rate = 0.5
-        else:
-            learning_rate = 0.1
-
-    Parameters:
-        boundaries(list): A list of steps numbers. The type of element in the list is python int.
-        values(list): A list of learning rate values that will be picked during
-            different step boundaries. The type of element in the list is python float.
-        begin(int): The begin step to initialize the global_step in the description above.
-        step(int, optional): The step size used to calculate the new global_step in the description above.
-            The default value is 1.
-        dtype(str, optional): The data type used to create the learning rate variable. The data type can be set as
-            'float32', 'float64'. The default value is 'float32'.
-
-    Returns:
-        None.
-
-    Examples:
-        .. code-block:: python
-
-          import paddle.fluid as fluid
-          import paddle
-          boundaries = [10000, 20000]
-          values = [1.0, 0.5, 0.1]
-          with fluid.dygraph.guard():
-              emb = paddle.nn.Embedding(10, 10)
-              optimizer = fluid.optimizer.SGD(
-                 learning_rate=fluid.dygraph.PiecewiseDecay(boundaries, values, 0),
-                 parameter_list = emb.parameters() )
-    """
-
-    def __init__(self, boundaries, values, begin, step=1, dtype='float32'):
-        super().__init__(begin, step, dtype)
-        self.boundaries = boundaries
-        self.values = values
-
-        self.vars = []
-        for value in values:
-            self.vars.append(value)
-
-    def step(self):
-        for i in range(len(self.boundaries)):
-            if self.step_num < self.boundaries[i]:
-                return self.vars[i]
-        return self.create_lr_var(self.vars[len(self.values) - 1])
-
-
 class PolynomialDecay(LearningRateDecay):
     r"""
     :api_attr: imperative
@@ -742,241 +676,3 @@ def epoch(self, epoch=None):
 
     def get_lr(self):
         raise NotImplementedError
-
-
-class StepDecay(_LearningRateEpochDecay):
-    """
-    :api_attr: imperative
-
-    Decays the learning rate of ``optimizer`` by ``decay_rate`` every ``step_size`` number of epoch.
-
-    The algorithm can be described as the code below.
-
-    .. code-block:: text
-
-        learning_rate = 0.5
-        step_size = 30
-        decay_rate = 0.1
-
-        learning_rate = 0.5     if epoch < 30
-        learning_rate = 0.05    if 30 <= epoch < 60
-        learning_rate = 0.005   if 60 <= epoch < 90
-        ...
-
-    Parameters:
-        learning_rate (float|int): The initial learning rate. It can be set to python float or int number.
-        step_size (int): Period of learning rate decay.
-        decay_rate (float, optional): The Ratio that the learning rate will be reduced. ``new_lr = origin_lr * decay_rate`` .
-            It should be less than 1.0. Default: 0.1.
-
-    Returns:
-        None.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle.fluid as fluid
-            import numpy as np
-            import paddle
-            with fluid.dygraph.guard():
-                x = np.random.uniform(-1, 1, [10, 10]).astype("float32")
-                linear = paddle.nn.Linear(10, 10)
-                input = fluid.dygraph.to_variable(x)
-                scheduler = fluid.dygraph.StepDecay(0.5, step_size=3)
-                adam = fluid.optimizer.Adam(learning_rate = scheduler, parameter_list = linear.parameters())
-
-                for epoch in range(9):
-                    for batch_id in range(5):
-                        out = linear(input)
-                        loss = paddle.mean(out)
-                        adam.minimize(loss)
-                    scheduler.epoch()
-
-                    print("epoch:{}, current lr is {}" .format(epoch, adam.current_step_lr()))
-                    # epoch:0, current lr is 0.5
-                    # epoch:1, current lr is 0.5
-                    # epoch:2, current lr is 0.5
-                    # epoch:3, current lr is 0.05
-                    # epoch:4, current lr is 0.05
-                    # epoch:5, current lr is 0.05
-                    # epoch:6, current lr is 0.005
-                    # epoch:7, current lr is 0.005
-                    # epoch:8, current lr is 0.005
-
-    """
-
-    def __init__(self, learning_rate, step_size, decay_rate=0.1):
-        if not isinstance(step_size, int):
-            raise TypeError(
-                "The type of 'step_size' must be 'int', but received %s."
-                % type(step_size)
-            )
-        if decay_rate >= 1.0:
-            raise ValueError('decay_rate should be < 1.0.')
-
-        self.step_size = step_size
-        self.decay_rate = decay_rate
-        super().__init__(learning_rate)
-
-    def get_lr(self):
-        decay_rate = self.create_lr_var(self.decay_rate)
-        i = self.epoch_num // self.step_size
-        return self.base_lr * (decay_rate**i)
-
-
-class MultiStepDecay(_LearningRateEpochDecay):
-    """
-    :api_attr: imperative
-
-    Decays the learning rate of ``optimizer`` by ``decay_rate`` once ``epoch`` reaches one of the milestones.
-
-    The algorithm can be described as the code below.
-
-    .. code-block:: text
-
-        learning_rate = 0.5
-        milestones = [30, 50]
-        decay_rate = 0.1
-        if epoch < 30:
-            learning_rate = 0.5
-        elif epoch < 50:
-            learning_rate = 0.05
-        else:
-            learning_rate = 0.005
-
-    Parameters:
-        learning_rate (float|int): The initial learning rate. It can be set to python float or int number.
-        milestones (tuple|list): List or tuple of each boundaries. Must be increasing.
-        decay_rate (float, optional): The Ratio that the learning rate will be reduced. ``new_lr = origin_lr * decay_rate`` .
-            It should be less than 1.0. Default: 0.1.
-
-    Returns:
-        None.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle.fluid as fluid
-            import numpy as np
-            import paddle
-            with fluid.dygraph.guard():
-                x = np.random.uniform(-1, 1, [10, 10]).astype("float32")
-                linear = paddle.nn.Linear(10, 10)
-                input = fluid.dygraph.to_variable(x)
-                scheduler = fluid.dygraph.MultiStepDecay(0.5, milestones=[3, 5])
-                adam = fluid.optimizer.Adam(learning_rate = scheduler, parameter_list = linear.parameters())
-
-                for epoch in range(6):
-                    for batch_id in range(5):
-                        out = linear(input)
-                        loss = paddle.mean(out)
-                        adam.minimize(loss)
-                    scheduler.epoch()
-
-                    print("epoch:{}, current lr is {}" .format(epoch, adam.current_step_lr()))
-                    # epoch:0, current lr is 0.5
-                    # epoch:1, current lr is 0.5
-                    # epoch:2, current lr is 0.5
-                    # epoch:3, current lr is 0.05
-                    # epoch:4, current lr is 0.05
-                    # epoch:5, current lr is 0.005
-
-    """
-
-    def __init__(self, learning_rate, milestones, decay_rate=0.1):
-        if not isinstance(milestones, (tuple, list)):
-            raise TypeError(
-                "The type of 'milestones' in 'MultiStepDecay' must be 'tuple, list', but received %s."
-                % type(milestones)
-            )
-
-        if not all(
-            [
-                milestones[i] < milestones[i + 1]
-                for i in range(len(milestones) - 1)
-            ]
-        ):
-            raise ValueError('The elements of milestones must be incremented')
-        if decay_rate >= 1.0:
-            raise ValueError('decay_rate should be < 1.0.')
-
-        self.milestones = milestones
-        self.decay_rate = decay_rate
-        super().__init__(learning_rate)
-
-    def get_lr(self):
-        decay_rate = self.create_lr_var(self.decay_rate)
-        for i in range(len(self.milestones)):
-            if self.epoch_num < self.milestones[i]:
-                return self.base_lr * (decay_rate**i)
-
-        return self.base_lr * (decay_rate ** len(self.milestones))
-
-
-class LambdaDecay(_LearningRateEpochDecay):
-    """
-    :api_attr: imperative
-
-    Sets the learning rate of ``optimizer`` to the initial lr times a multiplicative factor, and this multiplicative
-    factor is computed by function ``lr_lambda`` . ``lr_lambda`` is function which receives ``epoch`` .
-
-    The algorithm can be described as the code below.
-
-    .. code-block:: text
-
-        learning_rate = 0.5        # init learning_rate
-        lr_lambda = lambda epoch: 0.95 ** epoch
-
-        learning_rate = 0.5        # epoch 0
-        learning_rate = 0.475      # epoch 1
-        learning_rate = 0.45125    # epoch 2
-
-    Parameters:
-        learning_rate (float|int): The initial learning rate. It can be set to python float or int number.
-        lr_lambda (function): A function which computes a multiplicative factor given an integer parameter ``epoch`` , and
-            then multiply the initial learning rate by this multiplicative factor.
-
-    Returns:
-        None.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle.fluid as fluid
-            import numpy as np
-            import paddle
-            with fluid.dygraph.guard():
-                x = np.random.uniform(-1, 1, [10, 10]).astype("float32")
-                linear = paddle.nn.Linear(10, 10)
-                input = fluid.dygraph.to_variable(x)
-                scheduler = fluid.dygraph.LambdaDecay(0.5, lr_lambda=lambda x: 0.95**x)
-                adam = fluid.optimizer.Adam(learning_rate = scheduler, parameter_list = linear.parameters())
-
-                for epoch in range(6):
-                    for batch_id in range(5):
-                        out = linear(input)
-                        loss = paddle.mean(out)
-                        adam.minimize(loss)
-                    scheduler.epoch()
-
-                    print("epoch:%d, current lr is %f" .format(epoch, adam.current_step_lr()))
-                    # epoch:0, current lr is 0.5
-                    # epoch:1, current lr is 0.475
-                    # epoch:2, current lr is 0.45125
-
-    """
-
-    def __init__(self, learning_rate, lr_lambda):
-        if not callable(lr_lambda):
-            raise TypeError(
-                "The type of 'lr_lambda' in 'LambdaDecay' must be 'function', but received %s."
-                % type(lr_lambda)
-            )
-
-        self.lr_lambda = lr_lambda
-        super().__init__(learning_rate)
-
-    def get_lr(self):
-        base_lr = self.create_lr_var(self.base_lr)
-
-        return self.base_lr * self.lr_lambda(self.epoch_num)
diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py
@@ -410,10 +410,10 @@ def piecewise_decay(boundaries, values):
               paddle.enable_static()
               boundaries = [10000, 20000]
               values = [1.0, 0.5, 0.1]
-              optimizer = fluid.optimizer.Momentum(
+              optimizer = paddle.optimizer.Momentum(
                   momentum=0.9,
-                  learning_rate=fluid.layers.piecewise_decay(boundaries=boundaries, values=values),
-                  regularization=paddle.regularizer.L2Decay(1e-4))
+                  learning_rate=paddle.optimizer.lr.PiecewiseDecay(boundaries, values),
+                  weight_decay=paddle.regularizer.L2Decay(1e-4))
 
 
     """
@@ -422,7 +422,7 @@ def piecewise_decay(boundaries, values):
             raise ValueError("len(values) - len(boundaries) should be 1")
 
         if in_dygraph_mode():
-            decay = imperate_lr.PiecewiseDecay(boundaries, values, 0)
+            decay = paddle.optimizer.lr.PiecewiseDecay(boundaries, values)
             return decay
         else:
             global_step = _decay_step_counter()
diff --git a/python/paddle/optimizer/lr.py b/python/paddle/optimizer/lr.py
@@ -98,6 +98,8 @@ def __init__(self, learning_rate=0.1, last_epoch=-1, verbose=False):
                     type(learning_rate)
                 )
             )
+        if learning_rate < 0:
+            raise ValueError(f"Invalid learning rate: {learning_rate}")
         self.base_lr = float(learning_rate)
         self.last_lr = float(learning_rate)
         self.last_epoch = last_epoch
diff --git a/test/dygraph_to_static/test_basic_api_transformation.py b/test/dygraph_to_static/test_basic_api_transformation.py
@@ -376,9 +376,9 @@ def dyfunc_NoamDecay():
 def dyfunc_PiecewiseDecay():
     boundaries = [10000, 20000]
     values = [1.0, 0.5, 0.1]
-    pd = fluid.dygraph.PiecewiseDecay(boundaries, values, begin=0)
+    pd = paddle.optimizer.lr.PiecewiseDecay(boundaries, values)
     lr = pd()
-    return lr
+    return paddle.to_tensor(lr)
 
 
 def dyfunc_PolynomialDecay():
diff --git a/test/dygraph_to_static/test_yolov3.py b/test/dygraph_to_static/test_yolov3.py
@@ -94,11 +94,11 @@ def train(to_static):
         learning_rate = cfg.learning_rate
         values = [learning_rate * (gamma**i) for i in range(step_num + 1)]
 
-        lr = fluid.dygraph.PiecewiseDecay(
-            boundaries=boundaries, values=values, begin=0
+        lr = paddle.optimizer.lr.PiecewiseDecay(
+            boundaries=boundaries, values=values
         )
 
-        lr = fluid.layers.linear_lr_warmup(
+        lr = paddle.optimizer.lr.LinearWarmup(
             learning_rate=lr,
             warmup_steps=cfg.warm_up_iter,
             start_lr=0.0,
diff --git a/test/legacy_test/test_imperative_optimizer.py b/test/legacy_test/test_imperative_optimizer.py
diff --git a/test/legacy_test/test_learning_rate_scheduler.py b/test/legacy_test/test_learning_rate_scheduler.py

Original file line number	Diff line number	Diff line change
`@@ -98,6 +98,8 @@ def __init__(self, learning_rate=0.1, last_epoch=-1, verbose=False):`
`98`	`98`	`type(learning_rate)`
`99`	`99`	`)`
`100`	`100`	`)`
	`101`	`+ if learning_rate < 0:`
	`102`	`+ raise ValueError(f"Invalid learning rate: {learning_rate}")`
`101`	`103`	`self.base_lr = float(learning_rate)`
`102`	`104`	`self.last_lr = float(learning_rate)`
`103`	`105`	`self.last_epoch = last_epoch`