【Hackathon 6th No.8】NO.8 为 Paddle 新增 FeatureAlphaDropout API (#64881)

megemini · web-flow · commit fb3154f4a8ec · 2024-06-28T12:04:39.000+08:00
* [Add] feature_alpha_dropout * [Add] FeatureAlphaDropout * [Change] float | int to float * [Change] feature alpha dropout unittest file * [Update] unittest ndim errors * [Update] type annotations * Revert "[Update] type annotations" This reverts commit b79f264. * [Update] type annotation * [Update] type annotation * [Update] merge develop
diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py
@@ -57,6 +57,7 @@
     Dropout2D,
     Dropout3D,
     Embedding,
+    FeatureAlphaDropout,
     Flatten,
     Fold,
     Identity,
@@ -196,6 +197,7 @@
     'Dropout3D',
     'Bilinear',
     'AlphaDropout',
+    'FeatureAlphaDropout',
     'Unfold',
     'Fold',
     'RNNCellBase',
diff --git a/python/paddle/nn/functional/__init__.py b/python/paddle/nn/functional/__init__.py
@@ -61,6 +61,7 @@
     dropout,
     dropout2d,
     dropout3d,
+    feature_alpha_dropout,
     fold,
     interpolate,
     label_smooth,
@@ -210,6 +211,7 @@
     'dropout2d',
     'dropout3d',
     'alpha_dropout',
+    'feature_alpha_dropout',
     'label_smooth',
     'linear',
     'pad',
diff --git a/python/paddle/nn/functional/common.py b/python/paddle/nn/functional/common.py
@@ -1537,41 +1537,13 @@ def dropout3d(
     )
 
 
-def alpha_dropout(
-    x: Tensor, p: float = 0.5, training: bool = True, name: str | None = None
+def _feature_alpha_dropout_impl(
+    x: Tensor,
+    feature_dropout: bool,
+    p: float,
+    training: bool = True,
+    name: str | None = None,
 ) -> Tensor:
-    """
-    Alpha Dropout is a type of Dropout that maintains the self-normalizing property.
-    For an input with zero mean and unit standard deviation, the output of Alpha Dropout
-    maintains the original mean and standard deviation of the input.
-    Alpha Dropout fits well to SELU activate function by randomly setting activations to the negative saturation value.
-
-    Args:
-        x (Tensor): The input tensor. The data type is float16, float32 or float64.
-        p (float | int): Probability of setting units to zero. Default 0.5.
-        training (bool): A flag indicating whether it is in train phrase or not. Default True.
-        name (str, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
-
-    Returns:
-        A Tensor representing the dropout, has same shape and data type as `x`.
-
-    Examples:
-        .. code-block:: python
-
-            >>> import paddle
-            >>> paddle.seed(1)
-            >>> x = paddle.to_tensor([[-1, 1], [-1, 1]]).astype(paddle.float32)
-            >>> y_train = paddle.nn.functional.alpha_dropout(x, 0.5)
-            >>> y_test = paddle.nn.functional.alpha_dropout(x, 0.5, training=False)
-            >>> print(y_train)
-            Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True,
-            [[-0.77919382,  1.66559887],
-            [-0.10721093, -0.77919382]])
-            >>> print(y_test)
-            Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True,
-            [[-1.,  1.],
-            [-1.,  1.]])
-    """
     if not isinstance(p, (float, int)):
         raise TypeError("p argument should be a float or int")
     if p < 0 or p > 1:
@@ -1593,7 +1565,14 @@ def alpha_dropout(
         b = -a * alpha_p * p
 
         dtype = x.dtype
-        input_shape = x.shape
+        if not feature_dropout:
+            input_shape = x.shape
+        else:
+            if x.ndim < 2:
+                raise ValueError(
+                    'Feature alpha dropout needs at least 2D input.'
+                )
+            input_shape = list(x.shape[:2]) + [1] * len(x.shape[2:])
 
         # get mask
         random_tensor = paddle.uniform(
@@ -1618,6 +1597,93 @@ def alpha_dropout(
         return x
 
 
+def alpha_dropout(
+    x: Tensor,
+    p: float = 0.5,
+    training: bool = True,
+    name: str | None = None,
+) -> Tensor:
+    """
+    Alpha Dropout is a type of Dropout that maintains the self-normalizing property.
+    For an input with zero mean and unit standard deviation, the output of Alpha Dropout
+    maintains the original mean and standard deviation of the input.
+    Alpha Dropout fits well to SELU activate function by randomly setting activations to the negative saturation value.
+
+    Args:
+        x (Tensor): The input tensor. The data type is bfloat16, float16, float32 or float64.
+        p (float | int, optional): Probability of setting units to zero. Default 0.5.
+        training (bool, optional): A flag indicating whether it is in train phrase or not. Default True.
+        name (str | None, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        Tensor: A Tensor representing the dropout, has same shape and data type as `x`.
+
+    Examples:
+        .. code-block:: python
+
+            >>> import paddle
+            >>> paddle.seed(1)
+            >>> x = paddle.to_tensor([[-1, 1], [-1, 1]]).astype(paddle.float32)
+            >>> y_train = paddle.nn.functional.alpha_dropout(x, 0.5)
+            >>> y_test = paddle.nn.functional.alpha_dropout(x, 0.5, training=False)
+            >>> print(y_train)
+            Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True,
+            [[-0.77919382,  1.66559887],
+            [-0.10721093, -0.77919382]])
+            >>> print(y_test)
+            Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True,
+            [[-1.,  1.],
+            [-1.,  1.]])
+    """
+    return _feature_alpha_dropout_impl(
+        x, feature_dropout=False, p=p, training=training, name=name
+    )
+
+
+def feature_alpha_dropout(
+    x: Tensor,
+    p: float = 0.5,
+    training: bool = True,
+    name: str | None = None,
+) -> Tensor:
+    """
+    A channel is a feature map, Feature Alpha Dropout randomly masks out entire channels.
+    Alpha Dropout is a type of Dropout that maintains the self-normalizing property.
+    For an input with zero mean and unit standard deviation, the output of Alpha Dropout
+    maintains the original mean and standard deviation of the input.
+    Alpha Dropout fits well to SELU activate function by randomly setting activations to the negative saturation value.
+
+    Args:
+        x (Tensor): The input tensor. The data type is bfloat16, float16, float32 or float64.
+        p (float | int, optional): Probability of setting units to zero. Default 0.5.
+        training (bool, optional): A flag indicating whether it is in train phrase or not. Default True.
+        name (str | None, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
+
+    Returns:
+        Tensor: A Tensor representing the dropout, has same shape and data type as `x`.
+
+    Examples:
+        .. code-block:: python
+
+            >>> import paddle
+            >>> paddle.seed(1)
+            >>> x = paddle.to_tensor([[-1, 1], [-1, 1]]).astype(paddle.float32)
+            >>> y_train = paddle.nn.functional.feature_alpha_dropout(x, 0.5)
+            >>> y_test = paddle.nn.functional.feature_alpha_dropout(x, 0.5, training=False)
+            >>> print(y_train)
+            Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True,
+            [[-0.77919382,  1.66559887],
+            [-0.10721093, -0.77919382]])
+            >>> print(y_test)
+            Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True,
+            [[-1.,  1.],
+            [-1.,  1.]])
+    """
+    return _feature_alpha_dropout_impl(
+        x, feature_dropout=True, p=p, training=training, name=name
+    )
+
+
 def pad(
     x: Tensor,
     pad: ShapeLike,
diff --git a/python/paddle/nn/layer/__init__.py b/python/paddle/nn/layer/__init__.py
@@ -35,6 +35,7 @@
     Dropout2D,
     Dropout3D,
     Embedding,
+    FeatureAlphaDropout,
     Flatten,
     Fold,
     Identity,
diff --git a/python/paddle/nn/layer/common.py b/python/paddle/nn/layer/common.py
@@ -1111,6 +1111,68 @@ def extra_repr(self) -> str:
         return f'p={self.p}{name_str}'
 
 
+class FeatureAlphaDropout(Layer):
+    """
+    A channel is a feature map, Feature Alpha Dropout randomly masks out entire channels.
+    Alpha Dropout is a type of Dropout that maintains the self-normalizing property. For an input with
+    zero mean and unit standard deviation, the output of Alpha Dropout maintains the original mean and
+    standard deviation of the input. Alpha Dropout fits well to SELU activate function by randomly setting
+    activations to the negative saturation value.
+
+    For more information, please refer to:
+    `Self-Normalizing Neural Networks <https://arxiv.org/abs/1706.02515>`_
+
+    In dygraph mode, please use ``eval()`` to switch to evaluation mode, where dropout is disabled.
+
+    Parameters:
+        p (float | int, optional): Probability of setting units to zero. Default: 0.5
+        name (str | None, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`.
+
+    Shape:
+        - input: N-D tensor.
+        - output: N-D tensor, the same shape as input.
+
+    Examples:
+        .. code-block:: python
+
+            >>> import paddle
+            >>> paddle.seed(2023)
+
+            >>> x = paddle.to_tensor([[-1, 1], [-1, 1]], dtype="float32")
+            >>> m = paddle.nn.FeatureAlphaDropout(p=0.5)
+            >>> y_train = m(x)
+            >>> print(y_train)
+            Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True,
+            [[-0.10721093,  1.66559887],
+             [-0.77919382,  1.66559887]])
+
+            >>> m.eval()  # switch the model to test phase
+            >>> y_test = m(x)
+            >>> print(y_test)
+            Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True,
+            [[-1.,  1.],
+             [-1.,  1.]])
+    """
+
+    p: float
+    name: str | None
+
+    def __init__(self, p: float = 0.5, name: str | None = None) -> None:
+        super().__init__()
+        self.p = p
+        self.name = name
+
+    def forward(self, input: Tensor) -> Tensor:
+        out = F.feature_alpha_dropout(
+            input, p=self.p, training=self.training, name=self.name
+        )
+        return out
+
+    def extra_repr(self) -> str:
+        name_str = f', name={self.name}' if self.name else ''
+        return f'p={self.p}{name_str}'
+
+
 class Pad1D(Layer):
     """
     This interface is used to construct a callable object of the ``Pad1D`` class.
diff --git a/test/legacy_test/test_alpha_dropout.py b/test/legacy_test/test_alpha_dropout.py
