【易用性提升No.7】为transformerDecoderLayer新增layer_norm_eps参数 (#60084)

YibinLiu666 · web-flow · commit bfcf862ed928 · 2023-12-22T10:21:58.000+08:00
* add layer_norm_eps

* fix ci
diff --git a/python/paddle/nn/layer/transformer.py b/python/paddle/nn/layer/transformer.py
@@ -806,6 +806,7 @@ class TransformerDecoderLayer(Layer):
             corresponding layer would not have trainable bias parameter. See
             usage for details in :code:`ParamAttr` . Default: None,which means
             the default bias parameter property is used.
+        layer_norm_eps: the eps value in layer normalization components. Default=1e-5.
 
     Examples:
 
@@ -843,6 +844,7 @@ def __init__(
         normalize_before=False,
         weight_attr=None,
         bias_attr=None,
+        layer_norm_eps=1e-5,
     ):
         self._config = locals()
         self._config.pop("self")
@@ -889,9 +891,9 @@ def __init__(
         self.linear2 = Linear(
             dim_feedforward, d_model, weight_attrs[2], bias_attr=bias_attrs[2]
         )
-        self.norm1 = LayerNorm(d_model)
-        self.norm2 = LayerNorm(d_model)
-        self.norm3 = LayerNorm(d_model)
+        self.norm1 = LayerNorm(d_model, layer_norm_eps)
+        self.norm2 = LayerNorm(d_model, layer_norm_eps)
+        self.norm3 = LayerNorm(d_model, layer_norm_eps)
         self.dropout1 = Dropout(dropout, mode="upscale_in_train")
         self.dropout2 = Dropout(dropout, mode="upscale_in_train")
         self.dropout3 = Dropout(dropout, mode="upscale_in_train")
