Optimizer API's attr weight_decay support int 易用性提升 (#67947)

NKNaN · web-flow · commit 48299898d279 · 2024-09-10T19:08:00.000+08:00
* update optimizers

* transfer

* delete dtype error unit test

* update test

* rerun ci
diff --git a/python/paddle/optimizer/adadelta.py b/python/paddle/optimizer/adadelta.py
@@ -72,8 +72,8 @@ class Adadelta(Optimizer):
             then the parameters are list of dict. Note that the learning_rate in paramter groups \
             represents the scale of base learning_rate. \
             The default value is None in static graph mode, at this time all parameters will be updated.
-        weight_decay (float|WeightDecayRegularizer|None, optional): The strategy of regularization. \
-            It canbe a float value as coeff of L2 regularization or \
+        weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization. \
+            It canbe a int or float value as coeff of L2 regularization or \
             :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
             If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already, \
             the regularization setting here in optimizer will be ignored for this parameter. \
diff --git a/python/paddle/optimizer/adagrad.py b/python/paddle/optimizer/adagrad.py
@@ -75,8 +75,8 @@ class Adagrad(Optimizer):
             then the parameters are list of dict. Note that the learning_rate in parameter groups
             represents the scale of base learning_rate.
             The default value is None in static graph mode, at this time all parameters will be updated.
-        weight_decay (float|WeightDecayRegularizer|None, optional): The strategy of regularization.
-            It canbe a float value as coeff of L2 regularization or
+        weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization.
+            It canbe a int or float value as coeff of L2 regularization or
             :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
             If a parameter has set regularizer using :ref:`api_paddle_base_param_attr_aramAttr` already,
             the regularization setting here in optimizer will be ignored for this parameter.
diff --git a/python/paddle/optimizer/adam.py b/python/paddle/optimizer/adam.py
@@ -97,8 +97,8 @@ class Adam(Optimizer):
             then the parameters are list of dict. Note that the learning_rate in parameter groups
             represents the scale of base learning_rate.
             The default value is None in static graph mode, at this time all parameters will be updated.
-        weight_decay (float|WeightDecayRegularizer|None, optional): The strategy of regularization.
-            It canbe a float value as coeff of L2 regularization or
+        weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization.
+            It canbe a int or float value as coeff of L2 regularization or
             :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
             If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already,
             the regularization setting here in optimizer will be ignored for this parameter.
diff --git a/python/paddle/optimizer/adamax.py b/python/paddle/optimizer/adamax.py
@@ -87,8 +87,8 @@ class Adamax(Optimizer):
             then the parameters are list of dict. Note that the learning_rate in parameter groups
             represents the scale of base learning_rate.
             The default value is None in static graph mode, at this time all parameters will be updated.
-        weight_decay (float|WeightDecayRegularizer|None, optional): The strategy of regularization.
-            It can be a float value as coeff of L2 regularization or
+        weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization.
+            It can be a int or float value as coeff of L2 regularization or
             :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
             If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already,
             the regularization setting here in optimizer will be ignored for this parameter.
diff --git a/python/paddle/optimizer/adamw.py b/python/paddle/optimizer/adamw.py
@@ -83,7 +83,7 @@ class AdamW(Optimizer):
             then the parameters are list of dict. Note that the learning_rate in parameter groups
             represents the scale of base learning_rate.
             The default value is None in static graph mode, at this time all parameters will be updated.
-        weight_decay (float|Tensor, optional): The weight decay coefficient, it can be float or Tensor. The default value is 0.01.
+        weight_decay (int|float|Tensor, optional): The weight decay coefficient, it can be int, float or Tensor. The default value is 0.01.
         lr_ratio (Callable|None, optional): If it is not None,
             the learning rate will be updated with layer-wise learning rate ratio.
             Otherwise, the learning rate is the original.
@@ -195,10 +195,10 @@ def __init__(
             raise ValueError("Invalid value of beta2, expect beta2 in [0,1).")
         if not isinstance(epsilon, Value) and not 0 <= epsilon:
             raise ValueError("Invalid value of epsilon, expect epsilon >= 0.")
-        if not isinstance(weight_decay, float) and not isinstance(
+        if not isinstance(weight_decay, (int, float)) and not isinstance(
             weight_decay, (framework.Variable, Value)
         ):
-            raise TypeError("weight_decay should be float or Tensor.")
+            raise TypeError("weight_decay should be int, float or Tensor.")
         if lr_ratio is not None:
             assert isinstance(lr_ratio, Callable)
             if (
@@ -275,7 +275,7 @@ def __init__(
         self._learning_rate = learning_rate
         self._params_name = set()
         self._apply_decay_param_fun = apply_decay_param_fun
-        self._weight_decay = weight_decay
+        self._weight_decay = float(weight_decay)
         self._grad_clip = grad_clip
         self._lr_ratio = lr_ratio
         self._beta1 = beta1
@@ -286,7 +286,7 @@ def __init__(
         self._master_weights = {}
 
         self._default_dict = {
-            'weight_decay': weight_decay,
+            'weight_decay': float(weight_decay),
             'beta1': beta1,
             'beta2': beta2,
             'epsilon': epsilon,
diff --git a/python/paddle/optimizer/asgd.py b/python/paddle/optimizer/asgd.py
@@ -67,8 +67,8 @@ class ASGD(Optimizer):
         parameters (list|tuple|None, optional): List/Tuple of ``Tensor`` to update to minimize ``loss``.
             This parameter is required in dygraph mode.
             The default value is None in static graph mode, at this time all parameters will be updated.
-        weight_decay (float|WeightDecayRegularizer|None, optional): The strategy of regularization.
-            It can be a float value as coeff of L2 regularization or :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
+        weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization.
+            It can be a int or float value as coeff of L2 regularization or :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
             If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already,
             the regularization setting here in optimizer will be ignored for this parameter.
             Otherwise, the regularization setting here in optimizer will take effect.
diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py
@@ -140,8 +140,8 @@ class Optimizer:
             then the parameters are list of dict. Note that the learning_rate in parameter groups \
             represents the scale of base learning_rate. \
             The default value is None in static graph mode, at this time all parameters will be updated.
-        weight_decay (float|WeightDecayRegularizer|None, optional): The strategy of regularization. \
-            It can be a float value as coeff of L2 regularization or \
+        weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization. \
+            It can be a int or float value as coeff of L2 regularization or \
             :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
             If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already, \
             the regularization setting here in optimizer will be ignored for this parameter. \
@@ -262,6 +262,8 @@ def __init__(
                 )
         if isinstance(weight_decay, float):
             self.regularization = L2Decay(weight_decay)
+        elif isinstance(weight_decay, int):
+            self.regularization = L2Decay(float(weight_decay))
         else:
             self.regularization = weight_decay
         self._grad_clip = grad_clip
diff --git a/test/legacy_test/test_adadelta_op.py b/test/legacy_test/test_adadelta_op.py
@@ -211,6 +211,24 @@ def test_raise_error(self):
         )
 
 
+class TestAdadeltaV2WeightDecay(unittest.TestCase):
+    def test_weight_decay_int(self):
+        paddle.disable_static(paddle.CPUPlace())
+        value = np.arange(26).reshape(2, 13).astype("float32")
+        a = paddle.to_tensor(value)
+        linear = paddle.nn.Linear(13, 5)
+        # This can be any optimizer supported by dygraph.
+        adam = paddle.optimizer.Adadelta(
+            learning_rate=0.01,
+            parameters=linear.parameters(),
+            weight_decay=1,
+        )
+        out = linear(a)
+        out.backward()
+        adam.step()
+        adam.clear_gradients()
+
+
 class TestAdadeltaV2Group(TestAdadeltaV2):
     def test_adadelta_dygraph(self):
         paddle.disable_static(paddle.CPUPlace())
diff --git a/test/legacy_test/test_adagrad_op_v2.py b/test/legacy_test/test_adagrad_op_v2.py
@@ -32,6 +32,21 @@ def test_v20_coverage(self):
         adagrad.clear_grad()
 
 
+class TestAdagradOpV2WeightDecay(unittest.TestCase):
+    def test_weight_decay_int(self):
+        paddle.disable_static()
+        inp = paddle.rand(shape=[10, 10])
+        linear = paddle.nn.Linear(10, 10)
+        out = linear(inp)
+        loss = paddle.mean(out)
+        adagrad = paddle.optimizer.Adagrad(
+            learning_rate=0.1, parameters=linear.parameters(), weight_decay=1
+        )
+        out.backward()
+        adagrad.step()
+        adagrad.clear_grad()
+
+
 class TestAdagradOpV2Group(TestAdagradOpV2):
     def test_v20_coverage(self):
         paddle.disable_static()
diff --git a/test/legacy_test/test_adam_op.py b/test/legacy_test/test_adam_op.py
@@ -808,6 +808,23 @@ def test_adam_op_with_sparse_input_and_weight_decay(self):
         paddle.enable_static()
 
 
+class TestAdamOpV2WeightDecay(unittest.TestCase):
+    def test_weight_decay_int(self):
+        paddle.disable_static()
+        value = np.arange(26).reshape(2, 13).astype("float32")
+        a = paddle.to_tensor(value)
+        linear = paddle.nn.Linear(13, 5)
+
+        adam = paddle.optimizer.Adam(
+            learning_rate=0.01, parameters=linear.parameters(), weight_decay=1
+        )
+        out = linear(a)
+        out.backward()
+        adam.step()
+        adam.clear_gradients()
+        paddle.enable_static()
+
+
 class TestAdamOpV2Group(TestAdamOpV2):
     def test_adam_op(self):
         paddle.disable_static()
diff --git a/test/legacy_test/test_adamax_api.py b/test/legacy_test/test_adamax_api.py
@@ -36,6 +36,23 @@ def test_adamax_api_dygraph(self):
         adam.clear_gradients()
 
 
+class TestAdamaxAPIWeightDecay(unittest.TestCase):
+    def test_weight_decay_int(self):
+        paddle.disable_static()
+        value = np.arange(26).reshape(2, 13).astype("float32")
+        a = paddle.to_tensor(value)
+        linear = paddle.nn.Linear(13, 5)
+        adam = paddle.optimizer.Adamax(
+            learning_rate=0.01,
+            parameters=linear.parameters(),
+            weight_decay=1,
+        )
+        out = linear(a)
+        out.backward()
+        adam.step()
+        adam.clear_gradients()
+
+
 class TestAdamaxAPIGroup(TestAdamaxAPI):
     def test_adamax_api_dygraph(self):
         paddle.disable_static()
diff --git a/test/legacy_test/test_adamw_op.py b/test/legacy_test/test_adamw_op.py
@@ -621,14 +621,6 @@ def test_main(self):
 
 class TestAdamWOpError(unittest.TestCase):
     def test_api_errors(self):
-        def test_weight_decay_dtype():
-            linear = paddle.nn.Linear(13, 5)
-            adam = paddle.optimizer.AdamW(
-                learning_rate=0.01,
-                parameters=linear.parameters(),
-                weight_decay=1,
-            )
-
         def test_parameters_dtype1():
             adam = paddle.optimizer.AdamW(
                 learning_rate=0.01,
@@ -674,7 +666,6 @@ def test_grad_clip_dtype():
                 grad_clip=0.1,
             )
 
-        self.assertRaises(TypeError, test_weight_decay_dtype)
         self.assertRaises(TypeError, test_parameters_dtype1)
         self.assertRaises(TypeError, test_parameters_dtype2)
         self.assertRaises(AttributeError, test_parameters_dtype3)
@@ -1282,6 +1273,128 @@ def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t):
 
             paddle.disable_static()
 
+    def test_weight_decay_int(self):
+        paddle.disable_static()
+        linear1 = paddle.nn.Linear(
+            13, 8, bias_attr=paddle.nn.initializer.Constant(value=1.0)
+        )
+        linear2 = paddle.nn.Linear(
+            8, 5, bias_attr=paddle.nn.initializer.Constant(value=1.0)
+        )
+
+        # fix the linear name, simple_lr_setting function will use the name
+        linear1.weight.name = "linear_1.w_0"
+        linear1.bias.name = "linear_1.b_0"
+        linear2.weight.name = "linear_2.w_0"
+        linear2.bias.name = "linear_2.b_0"
+
+        fc1_w = np.array(linear1.weight)
+        fc1_w_mon1 = np.zeros_like(fc1_w)
+        fc1_w_mon2 = np.zeros_like(fc1_w)
+        fc1_b = np.array(linear1.bias)
+        fc1_b_mon1 = np.zeros_like(fc1_b)
+        fc1_b_mon2 = np.zeros_like(fc1_b)
+
+        fc2_w = np.array(linear2.weight)
+        fc2_w_mon1 = np.zeros_like(fc2_w)
+        fc2_w_mon2 = np.zeros_like(fc2_w)
+        fc2_b = np.array(linear2.bias)
+        fc2_b_mon1 = np.zeros_like(fc2_b)
+        fc2_b_mon2 = np.zeros_like(fc2_b)
+
+        simple_lr_fun = partial(simple_lr_setting, decay_rate=0.8, n_layers=2)
+        learning_rate = 0.001
+        weight_decay = 0
+        beta1 = 0.9
+        beta2 = 0.999
+
+        opt = paddle.optimizer.AdamW(
+            learning_rate=learning_rate,
+            parameters=[
+                {'params': linear1.parameters()},
+                {
+                    'params': linear2.parameters(),
+                },
+            ],
+            apply_decay_param_fun=lambda name: True,
+            weight_decay=weight_decay,
+            lr_ratio=simple_lr_fun,
+        )
+
+        def get_numpy_output(param, grad, moment1, moment2, lr_ratio, t):
+            np_inputs = {
+                'Param': param,
+                'Grad': grad,
+                'Moment1': moment1,
+                'Moment2': moment2,
+                'LearningRate': np.array([learning_rate]).astype("float32"),
+                'Beta1Pow': np.array([beta1**t]).astype("float32"),
+                'Beta2Pow': np.array([beta2**t]).astype("float32"),
+            }
+
+            np_attrs = {
+                'epsilon': 1e-8,
+                'beta1': beta1,
+                'beta2': beta2,
+                "lr_ratio": lr_ratio,
+                "coeff": float(weight_decay),
+                "with_decay": True,
+            }
+            param_out, moment1_out, moment2_out = adamw_step(
+                np_inputs, np_attrs
+            )
+            return param_out, moment1_out, moment2_out
+
+        for i in range(5):
+            a = paddle.to_tensor(
+                np.random.uniform(-1, 1, (2, 13)).astype("float32")
+            )
+            a1 = linear1(a)
+            out = linear2(a1)
+            out = paddle.mean(out)
+            out.backward()
+
+            fc1_w, fc1_w_mon1, fc1_w_mon2 = get_numpy_output(
+                fc1_w,
+                np.array(linear1.weight.grad),
+                fc1_w_mon1,
+                fc1_w_mon2,
+                simple_lr_fun(linear1.weight),
+                i + 1,
+            )
+            fc1_b, fc1_b_mon1, fc1_b_mon2 = get_numpy_output(
+                fc1_b,
+                np.array(linear1.bias.grad),
+                fc1_b_mon1,
+                fc1_b_mon2,
+                simple_lr_fun(linear1.bias),
+                i + 1,
+            )
+            fc2_w, fc2_w_mon1, fc2_w_mon2 = get_numpy_output(
+                fc2_w,
+                np.array(linear2.weight.grad),
+                fc2_w_mon1,
+                fc2_w_mon2,
+                simple_lr_fun(linear2.weight),
+                i + 1,
+            )
+            fc2_b, fc2_b_mon1, fc2_b_mon2 = get_numpy_output(
+                fc2_b,
+                np.array(linear2.bias.grad),
+                fc2_b_mon1,
+                fc2_b_mon2,
+                simple_lr_fun(linear2.bias),
+                i + 1,
+            )
+
+            opt.step()
+            opt.clear_gradients()
+
+            np.testing.assert_allclose(linear1.weight.numpy(), fc1_w, rtol=1e-6)
+            np.testing.assert_allclose(linear1.bias.numpy(), fc1_b, rtol=1e-6)
+            np.testing.assert_allclose(linear2.weight.numpy(), fc2_w, rtol=1e-6)
+            np.testing.assert_allclose(linear2.bias.numpy(), fc2_b, rtol=1e-6)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/legacy_test/test_asgd_op.py b/test/legacy_test/test_asgd_op.py
@@ -220,6 +220,25 @@ def test_asgd_group_dygraph(self):
         asgd.clear_gradients()
 
 
+class TestASGDV2WeightDecay(unittest.TestCase):
+    def test_weight_decay_int(self):
+        paddle.disable_static()
+        value = np.arange(26).reshape(2, 13).astype("float32")
+        a = paddle.to_tensor(value)
+        linear = paddle.nn.Linear(13, 5)
+
+        asgd = paddle.optimizer.ASGD(
+            learning_rate=0.001,
+            batch_num=2,
+            parameters=linear.parameters(),
+            weight_decay=1,
+        )
+        out = linear(a)
+        out.backward()
+        asgd.step()
+        asgd.clear_gradients()
+
+
 class TestASGDMultiPrecision(unittest.TestCase):
     def dygraph_asgd_mp(self, mp):
         paddle.disable_static()
