PaddlePaddle
diff --git a/‎paddle/phi/api/yaml/backward.yaml
Copy file name to clipboard
+11Lines changed: 11 additions & 0 deletions b/‎paddle/phi/api/yaml/backward.yaml
Copy file name to clipboard
+11Lines changed: 11 additions & 0 deletions
diff --git a/‎paddle/phi/api/yaml/ops.yaml
Copy file name to clipboard
+10Lines changed: 10 additions & 0 deletions b/‎paddle/phi/api/yaml/ops.yaml
Copy file name to clipboard
+10Lines changed: 10 additions & 0 deletions
diff --git a/‎paddle/phi/infermeta/backward.cc
Copy file name to clipboard
+12Lines changed: 12 additions & 0 deletions b/‎paddle/phi/infermeta/backward.cc
Copy file name to clipboard
+12Lines changed: 12 additions & 0 deletions
diff --git a/‎paddle/phi/infermeta/backward.h
Copy file name to clipboard
+5Lines changed: 5 additions & 0 deletions b/‎paddle/phi/infermeta/backward.h
Copy file name to clipboard
+5Lines changed: 5 additions & 0 deletions
diff --git a/‎paddle/phi/infermeta/binary.cc
Copy file name to clipboard
+25Lines changed: 25 additions & 0 deletions b/‎paddle/phi/infermeta/binary.cc
Copy file name to clipboard
+25Lines changed: 25 additions & 0 deletions
diff --git a/‎paddle/phi/infermeta/binary.h
Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions b/‎paddle/phi/infermeta/binary.h
Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions
diff --git a/‎paddle/phi/kernels/cpu/swiglu_grad_kernel.cc
Copy file name to clipboard
+62Lines changed: 62 additions & 0 deletions b/‎paddle/phi/kernels/cpu/swiglu_grad_kernel.cc
Copy file name to clipboard
+62Lines changed: 62 additions & 0 deletions
diff --git a/‎paddle/phi/kernels/cpu/swiglu_kernel.cc
Copy file name to clipboard
+43Lines changed: 43 additions & 0 deletions b/‎paddle/phi/kernels/cpu/swiglu_kernel.cc
Copy file name to clipboard
+43Lines changed: 43 additions & 0 deletions
diff --git a/‎paddle/phi/kernels/funcs/activation_functor.h
Copy file name to clipboard
+34Lines changed: 34 additions & 0 deletions b/‎paddle/phi/kernels/funcs/activation_functor.h
Copy file name to clipboard
+34Lines changed: 34 additions & 0 deletions
@@ -2386,6 +2386,17 @@
     func : svd_grad
   optional: u_grad, vh_grad, s_grad
 
+- backward_op : swiglu_grad
+  forward : swiglu (Tensor x, Tensor y) -> Tensor(out)
+  args: (Tensor x, Tensor y, Tensor out_grad)
+  output : Tensor(x_grad), Tensor(y_grad)
+  infer_meta:
+    func: SwiGLUGradInferMeta
+    param: [x, y]
+  kernel:
+    func: swiglu_grad
+  optional: y
+
 - backward_op : take_along_axis_grad
   forward : take_along_axis (Tensor arr, Tensor indices, int axis) -> Tensor(out)
   args : (Tensor arr, Tensor indices, Tensor out_grad, int axis)
 
@@ -2693,6 +2693,16 @@
     func : svd
   backward : svd_grad
 
+- op : swiglu
+  args : (Tensor x, Tensor y)
+  output : Tensor(out)
+  infer_meta:
+     func: SwiGLUInferMeta
+  kernel:
+     func : swiglu
+  optional : y
+  backward: swiglu_grad
+
 - op : take_along_axis
   args : (Tensor arr, Tensor indices, int axis)
   output : Tensor
 
@@ -1158,6 +1158,18 @@ void StackGradInferMeta(const MetaTensor& out_grad,
   }
 }
 
+void SwiGLUGradInferMeta(const MetaTensor& x,
+                         const MetaTensor& y,
+                         MetaTensor* x_grad,
+                         MetaTensor* y_grad) {
+  if (x_grad) {
+    x_grad->share_meta(x);
+  }
+  if (y && y_grad) {
+    y_grad->share_meta(y);
+  }
+}
+
 void TransposeGradInferMeta(const MetaTensor& x,
                             const std::vector<int>& axis,
                             MetaTensor* out) {
 
@@ -463,6 +463,11 @@ void StackGradInferMeta(const MetaTensor& out_grad,
                         int axis,
                         std::vector<MetaTensor*> x_grad);
 
+void SwiGLUGradInferMeta(const MetaTensor& x,
+                         const MetaTensor& y,
+                         MetaTensor* x_grad,
+                         MetaTensor* y_grad);
+
 void TransposeInferMeta(const MetaTensor& x,
                         const std::vector<int>& axis,
                         MetaTensor* out);
 
@@ -3384,6 +3384,31 @@ void SolveInferMeta(const MetaTensor& x, const MetaTensor& y, MetaTensor* out) {
   out->share_lod(x);
 }
 
+void SwiGLUInferMeta(const MetaTensor& x,
+                     const MetaTensor& y,
+                     MetaTensor* out) {
+  if (y) {
+    PADDLE_ENFORCE_EQ(
+        x.dims(),
+        y.dims(),
+        phi::errors::InvalidArgument(
+            "The shape of Input(X) should be equal of the shape of Input(Y)."));
+    out->share_meta(x);
+  } else {
+    auto dims = x.dims();
+    PADDLE_ENFORCE_EQ(
+        dims[dims.size() - 1] % 2,
+        0,
+        phi::errors::InvalidArgument(
+            "The last dim of Input(X) should be exactly divided by 2."));
+    dims[dims.size() - 1] /= 2;
+    out->set_dims(dims);
+    out->set_dtype(x.dtype());
+    out->set_layout(x.layout());
+    out->share_lod(x);
+  }
+}
+
 void UnpoolInferMeta(const MetaTensor& x,
                      const MetaTensor& indices,
                      const std::vector<int>& ksize,
 
@@ -545,6 +545,8 @@ void ValueCompareInferMeta(const MetaTensor& x,
 
 void SolveInferMeta(const MetaTensor& x, const MetaTensor& y, MetaTensor* out);
 
+void SwiGLUInferMeta(const MetaTensor& x, const MetaTensor& y, MetaTensor* out);
+
 void UnpoolInferMeta(const MetaTensor& x,
                      const MetaTensor& indices,
                      const std::vector<int>& ksize,
 
@@ -0,0 +1,62 @@
+// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/swiglu_grad_kernel.h"
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/activation_functor.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void SwiGLUGradKernelImpl(const Context &ctx,
+                          const T *x,
+                          const T *y,
+                          const T *dz,
+                          T *dx,
+                          T *dy,
+                          int64_t m,
+                          int64_t n) {
+  funcs::SwiGLUGradFunctor<T> functor;
+  int64_t stride;
+  if (y) {
+    stride = n;
+  } else {
+    stride = 2 * n;
+    y = x + n;
+    dy = dx + n;
+  }
+
+  for (int64_t i = 0; i < m; ++i) {
+    for (int64_t j = 0; j < n; ++j) {
+      T dx_tmp, dy_tmp;
+      functor(x[i * stride + j],
+              y[i * stride + j],
+              dz[i * n + j],
+              &dx_tmp,
+              &dy_tmp);
+      if (dx) {
+        dx[i * stride + j] = dx_tmp;
+      }
+      if (dy) {
+        dy[i * stride + j] = dy_tmp;
+      }
+    }
+  }
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(
+    swiglu_grad, CPU, ALL_LAYOUT, phi::SwiGLUGradKernel, float, double) {}
@@ -0,0 +1,43 @@
+// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/phi/kernels/swiglu_kernel.h"
+#include "paddle/phi/backends/cpu/cpu_context.h"
+#include "paddle/phi/core/kernel_registry.h"
+#include "paddle/phi/kernels/funcs/activation_functor.h"
+
+namespace phi {
+
+template <typename T, typename Context>
+void SwiGLUKernelImpl(
+    const Context &ctx, const T *x, const T *y, T *z, int64_t m, int64_t n) {
+  funcs::SwiGLUFunctor<T> functor;
+  int64_t stride;
+  if (y) {
+    stride = n;
+  } else {
+    stride = 2 * n;
+    y = x + n;
+  }
+
+  for (int64_t i = 0; i < m; ++i) {
+    for (int64_t j = 0; j < n; ++j) {
+      z[i * n + j] = functor(x[i * stride + j], y[i * stride + j]);
+    }
+  }
+}
+
+}  // namespace phi
+
+PD_REGISTER_KERNEL(swiglu, CPU, ALL_LAYOUT, phi::SwiGLUKernel, float, double) {}
@@ -5007,5 +5007,39 @@ struct CudaCELUGradFunctor : public BaseActivationFunctor<T> {
 
 #endif
 
+template <typename T>
+struct SwiGLUFunctor {
+  using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
+
+  HOSTDEVICE T operator()(T x, T y) const {
+    MPType mp_x = static_cast<MPType>(x);
+    MPType mp_y = static_cast<MPType>(y);
+    MPType one = static_cast<MPType>(1);
+    return static_cast<T>(mp_y * mp_x / (one + exp(-mp_x)));
+  }
+};
+
+template <typename T, bool HasDX = true, bool HasDY = true>
+struct SwiGLUGradFunctor {
+  using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
+
+  HOSTDEVICE void operator()(T x, T y, T dz, T* dx, T* dy) const {
+    MPType one = static_cast<MPType>(1);
+
+    MPType mp_x = static_cast<MPType>(x);
+    MPType mp_dz = static_cast<MPType>(dz);
+
+    MPType sigmoid = one / (one + exp(-mp_x));
+    MPType tmp = mp_x * sigmoid;
+    if (HasDX) {
+      MPType mp_y = static_cast<MPType>(y);
+      *dx = static_cast<T>(mp_dz * mp_y * sigmoid * (one + mp_x - tmp));
+    }
+    if (HasDY) {
+      *dy = static_cast<T>(mp_dz * tmp);
+    }
+  }
+};
+
 }  // namespace funcs
 }  // namespace phi