Optimizer API's attr weight_decay support int 易用性提升 - part2 (#68033)

NKNaN · web-flow · commit e7f651e821e3 · 2024-09-14T16:21:48.000+08:00
diff --git a/python/paddle/optimizer/lbfgs.py b/python/paddle/optimizer/lbfgs.py
@@ -375,8 +375,8 @@ class LBFGS(Optimizer):
         line_search_fn (string|None, optional): either 'strong_wolfe' or None. The default value is strong_wolfe.
         parameters (list|tuple|None, optional): List/Tuple of ``Tensor`` names to update to minimize ``loss``. \
             This parameter is required in dygraph mode. The default value is None.
-        weight_decay (float|WeightDecayRegularizer|None, optional): The strategy of regularization. \
-            It canbe a float value as coeff of L2 regularization or \
+        weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization. \
+            It can be a int or float value as coeff of L2 regularization or \
             :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
             If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already, \
             the regularization setting here in optimizer will be ignored for this parameter. \
diff --git a/python/paddle/optimizer/momentum.py b/python/paddle/optimizer/momentum.py
@@ -81,8 +81,8 @@ class Momentum(Optimizer):
             represents the scale of base learning_rate. \
             The default value is None in static graph mode, at this time all parameters will be updated.
         use_nesterov(bool, optional): Enables Nesterov momentum. The default value is False.
-        weight_decay (float|WeightDecayRegularizer|None, optional): The strategy of regularization. \
-            It can be a float value as coeff of L2 regularization or \
+        weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization. \
+            It can be a int or float value as coeff of L2 regularization or \
             :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
             If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already, \
             the regularization setting here in optimizer will be ignored for this parameter. \
@@ -166,6 +166,8 @@ def __init__(
         if momentum is None:
             raise ValueError("momentum is not set")
 
+        if isinstance(weight_decay, int):
+            weight_decay = float(weight_decay)
         predicate = lambda regular: isinstance(regular, (L2Decay, float))
         if isinstance(parameters, list):
             if isinstance(parameters[0], dict):
diff --git a/python/paddle/optimizer/nadam.py b/python/paddle/optimizer/nadam.py
@@ -87,7 +87,7 @@ class NAdam(Optimizer):
             The default value is 0.999.
         epsilon (float, optional): A small float value for numerical stability.
             The default value is 1e-08.
-        weight_decay (float|Tensor|None, optional): The weight decay coefficient, it can be float or Tensor.
+        weight_decay (int|float|Tensor|None, optional): The weight decay coefficient, it can be int, float or Tensor.
             Default None, meaning there is no regularization.
         momentum_decay (float, optional): momentum momentum_decay. The default value is 0.004.
         grad_clip (GradientClipBase|None, optional): Gradient clipping strategy, it's an instance of
diff --git a/python/paddle/optimizer/radam.py b/python/paddle/optimizer/radam.py
@@ -91,7 +91,7 @@ class RAdam(Optimizer):
             The default value is 0.999.
         epsilon (float, optional): A small float value for numerical stability.
             The default value is 1e-08.
-        weight_decay (float|Tensor|WeightDecayRegularizer|None, optional): The weight decay coefficient, it can be float or Tensor.
+        weight_decay (int|float|Tensor|WeightDecayRegularizer|None, optional): The weight decay coefficient, it can be int, float or Tensor.
             Default None, meaning there is no regularization.
         grad_clip (GradientClipBase|None, optional): Gradient clipping strategy, it's an instance of
             some derived class of ``GradientClipBase`` . There are three clipping strategies
diff --git a/python/paddle/optimizer/rmsprop.py b/python/paddle/optimizer/rmsprop.py
@@ -113,8 +113,8 @@ class RMSProp(Optimizer):
           then the parameters are list of dict. Note that the learning_rate in parameter groups
           represents the scale of base learning_rate.
           The default value is None in static graph mode, at this time all parameters will be updated.
-        weight_decay (float|WeightDecayRegularizer|None, optional): The strategy of regularization.
-          It can be a float value as coeff of L2 regularization or \
+        weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization.
+          It can be a int or float value as coeff of L2 regularization or \
           :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
           If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already,
           the regularization setting here in optimizer will be ignored for this parameter.
diff --git a/python/paddle/optimizer/sgd.py b/python/paddle/optimizer/sgd.py
@@ -51,8 +51,8 @@ class SGD(Optimizer):
         parameters (list|tuple|None, optional): List/Tuple of ``Tensor`` to update to minimize ``loss``. \
             This parameter is required in dygraph mode. \
             The default value is None in static graph mode, at this time all parameters will be updated.
-        weight_decay (float|WeightDecayRegularizer|None, optional): The strategy of regularization. \
-            It can be a float value as coeff of L2 regularization or \
+        weight_decay (int|float|WeightDecayRegularizer|None, optional): The strategy of regularization. \
+            It can be a int or float value as coeff of L2 regularization or \
             :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
             If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already, \
             the regularization setting here in optimizer will be ignored for this parameter. \
diff --git a/test/legacy_test/test_momentum_op.py b/test/legacy_test/test_momentum_op.py
@@ -603,6 +603,23 @@ def test_raise_error(self):
         )
         self.assertRaises(ValueError, paddle.optimizer.Momentum, momentum=None)
 
+    def test_weight_decay_int(self):
+        paddle.disable_static()
+        value = np.arange(26).reshape(2, 13).astype("float32")
+        a = paddle.to_tensor(value)
+        linear = paddle.nn.Linear(13, 5)
+        # This can be any optimizer supported by dygraph.
+        adam = paddle.optimizer.Momentum(
+            learning_rate=0.01,
+            momentum=0.9,
+            parameters=linear.parameters(),
+            weight_decay=1,
+        )
+        out = linear(a)
+        out.backward()
+        adam.step()
+        adam.clear_gradients()
+
 
 class TestMomentumOpWithDecay(OpTest):
     def setUp(self):
diff --git a/test/legacy_test/test_nadam_op.py b/test/legacy_test/test_nadam_op.py
@@ -373,6 +373,25 @@ def test_nadam_invalid_input(self):
             )
 
 
+class TestNAdamAPIWeightDecay(unittest.TestCase):
+    def test_weight_decay_int(self):
+        paddle.disable_static()
+        value = np.arange(26).reshape(2, 13).astype("float32")
+        a = paddle.to_tensor(value)
+        linear = paddle.nn.Linear(13, 5)
+        nadam = paddle.optimizer.NAdam(
+            learning_rate=0.01,
+            parameters=linear.parameters(),
+            weight_decay=1,
+        )
+
+        for _ in range(2):
+            out = linear(a)
+            out.backward()
+            nadam.step()
+            nadam.clear_gradients()
+
+
 class TestNAdamAPIGroup(TestNAdamAPI):
     def test_nadam_dygraph(self):
         paddle.disable_static()
diff --git a/test/legacy_test/test_radam_op.py b/test/legacy_test/test_radam_op.py
@@ -384,6 +384,25 @@ def test_radam_invalid_input(self):
             )
 
 
+class TestRAdamAPIWeightDecay(unittest.TestCase):
+    def test_weight_decay_int(self):
+        paddle.disable_static()
+        value = np.arange(26).reshape(2, 13).astype("float32")
+        a = paddle.to_tensor(value)
+        linear = paddle.nn.Linear(13, 5)
+        radam = paddle.optimizer.RAdam(
+            learning_rate=0.01,
+            parameters=linear.parameters(),
+            weight_decay=1,
+        )
+
+        for _ in range(2):
+            out = linear(a)
+            out.backward()
+            radam.step()
+            radam.clear_gradients()
+
+
 class TestRAdamAPIGroup(TestRAdamAPI):
     def test_radam_dygraph(self):
         paddle.disable_static()
diff --git a/test/legacy_test/test_rmsprop_op.py b/test/legacy_test/test_rmsprop_op.py
@@ -345,6 +345,24 @@ def test_rmsprop_op_invalid_input(self):
             )
 
 
+class TestRMSPropV2WeightDecay(unittest.TestCase):
+    def test_weight_decay_int(self):
+        paddle.disable_static()
+        value = np.arange(26).reshape(2, 13).astype("float32")
+        a = paddle.to_tensor(value)
+        linear = paddle.nn.Linear(13, 5)
+        # This can be any optimizer supported by dygraph.
+        adam = paddle.optimizer.RMSProp(
+            learning_rate=0.01,
+            parameters=linear.parameters(),
+            weight_decay=1,
+        )
+        out = linear(a)
+        out.backward()
+        adam.step()
+        adam.clear_gradients()
+
+
 class TestRMSPropV2Group(TestRMSPropV2):
     def test_rmsprop_dygraph(self):
         paddle.disable_static()
diff --git a/test/legacy_test/test_sgd_op.py b/test/legacy_test/test_sgd_op.py
@@ -255,6 +255,22 @@ def test_sgd_group_dygraph(self):
         adam.step()
         adam.clear_gradients()
 
+    def test_weight_decay_int(self):
+        paddle.disable_static()
+        value = np.arange(26).reshape(2, 13).astype("float32")
+        a = paddle.to_tensor(value)
+        linear = paddle.nn.Linear(13, 5)
+        # This can be any optimizer supported by dygraph.
+        adam = paddle.optimizer.SGD(
+            learning_rate=0.01,
+            parameters=linear.parameters(),
+            weight_decay=1,
+        )
+        out = linear(a)
+        out.backward()
+        adam.step()
+        adam.clear_gradients()
+
 
 class TestSGDSimple(unittest.TestCase):
     def setUp(self) -> None:
