PaddlePaddle
diff --git a/‎paddle/fluid/inference/api/analysis_predictor.cc
Copy file name to clipboard
+1Lines changed: 1 addition & 0 deletions b/‎paddle/fluid/inference/api/analysis_predictor.cc
Copy file name to clipboard
+1Lines changed: 1 addition & 0 deletions
diff --git a/‎paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
Copy file name to clipboard
+89Lines changed: 89 additions & 0 deletions b/‎paddle/fluid/inference/tensorrt/convert/generic_and_custom_plugin_creater.cc
Copy file name to clipboard
+89Lines changed: 89 additions & 0 deletions
diff --git a/‎paddle/fluid/inference/tensorrt/convert/op_converter.h
Copy file name to clipboard
+7Lines changed: 7 additions & 0 deletions b/‎paddle/fluid/inference/tensorrt/convert/op_converter.h
Copy file name to clipboard
+7Lines changed: 7 additions & 0 deletions
diff --git a/‎paddle/fluid/inference/tensorrt/op_teller.cc
Copy file name to clipboard
+44Lines changed: 44 additions & 0 deletions b/‎paddle/fluid/inference/tensorrt/op_teller.cc
Copy file name to clipboard
+44Lines changed: 44 additions & 0 deletions
diff --git a/‎paddle/fluid/inference/tensorrt/op_teller.h
Copy file name to clipboard
+6-1Lines changed: 6 additions & 1 deletion b/‎paddle/fluid/inference/tensorrt/op_teller.h
Copy file name to clipboard
+6-1Lines changed: 6 additions & 1 deletion
diff --git a/‎paddle/fluid/inference/tensorrt/plugin/CMakeLists.txt
Copy file name to clipboard
+1Lines changed: 1 addition & 0 deletions b/‎paddle/fluid/inference/tensorrt/plugin/CMakeLists.txt
Copy file name to clipboard
+1Lines changed: 1 addition & 0 deletions
@@ -3074,6 +3074,7 @@ USE_TRT_CONVERTER(trans_layernorm)
 USE_TRT_CONVERTER(skip_merge_layernorm)
 USE_TRT_CONVERTER(generic_plugin_creater)
 USE_TRT_CONVERTER(custom_plugin_creater)
+USE_TRT_CONVERTER(custom_generic_plugin_creater)
 USE_TRT_CONVERTER(fuse_eleadd_transpose)
 USE_TRT_CONVERTER(tanh_shrink)
 USE_TRT_CONVERTER(logsigmoid)
 
@@ -15,6 +15,7 @@ limitations under the License. */
 #include "paddle/common/errors.h"
 #include "paddle/fluid/inference/tensorrt/convert/op_converter.h"
 #include "paddle/fluid/inference/tensorrt/helper.h"
+#include "paddle/fluid/inference/tensorrt/plugin/custom_generic_plugin.h"
 #include "paddle/fluid/inference/tensorrt/plugin/generic_plugin.h"
 #include "paddle/fluid/inference/tensorrt/plugin_arg_mapping_context.h"
 #include "paddle/phi/api/ext/op_meta_info.h"
@@ -251,9 +252,97 @@ class GenericPluginCreater : public OpConverter {
   }
 };
 
+class CustomGenericPluginCreater : public OpConverter {
+ public:
+  void operator()(const framework::proto::OpDesc &op,
+                  const framework::Scope &scope,
+                  bool test_mode) override {
+    framework::OpDesc op_desc(op, nullptr);
+    VLOG(3) << "convert " << op_desc.Type()
+            << " op to custom generic pluign layer";
+
+    nvinfer1::ILayer *layer = nullptr;
+    std::vector<nvinfer1::ITensor *> inputs;
+
+    CHECK(block_);
+    const framework::BlockDesc block_desc(
+        nullptr, const_cast<framework::proto::BlockDesc *>(block_));
+
+    plugin::CustomGenericPlugin::InputOutPutVarInfo in_out_info;
+    using paddle::inference::tensorrt::plugin::
+        ProtoTypeToGenerateCustomGenericPluginDataType;
+
+    bool with_fp16 = engine_->WithFp16() && !engine_->disable_trt_plugin_fp16();
+
+    auto &op_meta_info_map = OpMetaInfoMap::Instance();
+    const auto &meta_info_map = op_meta_info_map.GetMap();
+    auto &op_info = meta_info_map.at(op_desc.Type()).front();
+
+    // set inputs
+    auto &op_input_names = OpMetaInfoHelper::GetInputs(op_info);
+    paddle::small_vector<const char *> input_names;
+    for (auto &input_name : op_input_names) {
+      input_names.emplace_back(input_name.c_str());
+    }
+    for (auto &param_name : input_names) {
+      for (auto &arg_name : op_desc.Input(param_name)) {
+        inputs.push_back(engine_->GetITensor(arg_name));
+        auto *var = block_desc.FindVar(arg_name);
+        PADDLE_ENFORCE_NOT_NULL(
+            var,
+            platform::errors::NotFound(
+                "There is no variable called %s in block.", arg_name.c_str()));
+        PADDLE_ENFORCE_EQ(
+            var->GetType(),
+            FluidDT::VarType_Type_LOD_TENSOR,
+            platform::errors::InvalidArgument("TensorRT engine only takes "
+                                              "LoDTensor as input"));
+        in_out_info.inputs_data_type.push_back(
+            ProtoTypeToGenerateCustomGenericPluginDataType(var->GetDataType()));
+      }
+    }
+
+    // set outputs
+    auto &op_output_names = OpMetaInfoHelper::GetOutputs(op_info);
+    paddle::small_vector<const char *> output_names;
+    for (auto &output_name : op_output_names) {
+      output_names.emplace_back(output_name.c_str());
+    }
+    std::vector<std::string> outputs;
+    for (auto &param_name : output_names) {
+      for (auto &arg_name : op_desc.Output(param_name)) {
+        outputs.push_back(arg_name);
+        auto *var = block_desc.FindVar(arg_name);
+        PADDLE_ENFORCE_NOT_NULL(
+            var,
+            platform::errors::NotFound(
+                "There is no variable called %s in block.", arg_name.c_str()));
+        PADDLE_ENFORCE_EQ(
+            var->GetType(),
+            FluidDT::VarType_Type_LOD_TENSOR,
+            platform::errors::InvalidArgument("TensorRT engine only takes "
+                                              "LoDTensor as input"));
+        in_out_info.outputs_data_type.push_back(
+            ProtoTypeToGenerateCustomGenericPluginDataType(var->GetDataType()));
+      }
+    }
+
+    auto *plugin = new plugin::CustomGenericPlugin(op, in_out_info, with_fp16);
+    CHECK(plugin);
+
+    layer = engine_->AddDynamicPlugin(
+        inputs.data(), inputs.size(), (plugin::DynamicPluginTensorRT *)plugin);
+    CHECK(layer);
+
+    RreplenishLayerAndOutput(layer, op_desc.Type(), outputs, test_mode);
+  }
+};
+
 }  // namespace tensorrt
 }  // namespace inference
 }  // namespace paddle
 
 REGISTER_TRT_OP_CONVERTER(custom_plugin_creater, CustomPluginCreater);
 REGISTER_TRT_OP_CONVERTER(generic_plugin_creater, GenericPluginCreater);
+REGISTER_TRT_OP_CONVERTER(custom_generic_plugin_creater,
+                          CustomGenericPluginCreater);
@@ -157,6 +157,13 @@ class OpConverter {
         it = Registry<OpConverter>::Global().Lookup("custom_plugin_creater");
         break;
 
+      case OpConverterType::CustomGenericPluginCreater:
+        LOG(INFO) << "There is no OpConverter for type " << op_desc.Type()
+                  << ", now use custom_generic_plugin_creater!";
+        it = Registry<OpConverter>::Global().Lookup(
+            "custom_generic_plugin_creater");
+        break;
+
       default:
         CHECK(false) << "no OpConverter for optype " << op_desc.Type();
     }
 
@@ -20,6 +20,7 @@
 #include "paddle/fluid/framework/data_layout.h"
 #include "paddle/fluid/framework/phi_utils.h"
 #include "paddle/fluid/inference/tensorrt/dynamic_shape_infermeta_factory.h"
+#include "paddle/phi/api/ext/op_meta_info.h"
 #include "paddle/phi/core/compat/op_utils.h"
 #include "paddle/phi/core/kernel_factory.h"
 
@@ -3237,6 +3238,40 @@ struct CustomPluginTeller : public Teller {
   }
 };
 
+struct CustomGenericPluginTeller : public Teller {
+  CustomGenericPluginTeller() = default;
+  bool operator()(const framework::OpDesc& desc,
+                  bool use_no_calib_int8 = false,
+                  bool with_dynamic_shape = false,
+                  bool use_explicit_quantization = false) override {
+    const std::string op_type = desc.Type();
+    auto& op_meta_info_map = OpMetaInfoMap::Instance();
+    const auto& meta_info_map = op_meta_info_map.GetMap();
+    if (meta_info_map.count(op_type) > 0) {
+      auto& op_info = meta_info_map.at(op_type).front();
+      auto& trt_infer_shape_fn = OpMetaInfoHelper::GetTrtInferShapeFn(op_info);
+      if (trt_infer_shape_fn == nullptr) {
+        VLOG(3) << op_type
+                << " has no trt getOutputDimensions function. Please set by "
+                   "SetTrtInferShapeFn.";
+        return false;
+      }
+      auto& trt_supports_formate_config =
+          OpMetaInfoHelper::GetTrtSupportsFormatConfig(op_info);
+      if (trt_supports_formate_config.empty()) {
+        VLOG(3)
+            << op_type
+            << " has no trt supportsFormatCombination config. Please set by "
+               "SetTrtSupportsFormatConfig.";
+        return false;
+      }
+      return true;
+    }
+    VLOG(3) << op_type << " has no meta info";
+    return false;
+  }
+};
+
 bool OpTeller::Tell(const framework::ir::Node* node,
                     bool use_no_calib_int8,
                     bool with_dynamic_shape,
@@ -3273,13 +3308,22 @@ bool OpTeller::Tell(const framework::ir::Node* node,
     SetOpConverterType(node->Op(), OpConverterType::CustomPluginCreater);
     return true;
   }
+  auto& custom_generic_plugin_teller = GetCustomGenericPluginTeller();
+  if ((*custom_generic_plugin_teller)(desc,
+                                      use_no_calib_int8,
+                                      with_dynamic_shape,
+                                      use_explicit_quantization)) {
+    SetOpConverterType(node->Op(), OpConverterType::CustomGenericPluginCreater);
+    return true;
+  }
   return false;
 }
 
 OpTeller::OpTeller() {  // NOLINT
   tellers_.emplace_back(new tensorrt::SimpleOpTypeSetTeller);
   tellers_.emplace_back(new tensorrt::GenericPluginTeller);
   tellers_.emplace_back(new tensorrt::CustomPluginTeller);
+  tellers_.emplace_back(new tensorrt::CustomGenericPluginTeller);
 }
 
 }  // namespace tensorrt
 
@@ -59,7 +59,8 @@ struct Teller {
 enum class OpConverterType {
   Default = 0,
   GenericPluginCreater,
-  CustomPluginCreater
+  CustomPluginCreater,
+  CustomGenericPluginCreater
 };
 /*
  * class OpTeller helps to tell whether a fluid
@@ -84,6 +85,10 @@ class OpTeller {
 
   std::unique_ptr<Teller>& GetCustomPluginTeller() { return tellers_.at(2); }
 
+  std::unique_ptr<Teller>& GetCustomGenericPluginTeller() {
+    return tellers_.at(3);
+  }
+
   void SetOpConverterType(framework::OpDesc* op_desc, OpConverterType type) {
     op_desc->SetAttr("converter_type", static_cast<int>(type));
   }
 
@@ -39,6 +39,7 @@ list(
   preln_groupnorm_act_op_plugin.cu
   elementwiseadd_transpose_op_plugin.cu
   generic_plugin.cu
+  custom_generic_plugin.cu
   many_emb_layernorm_plugin.cu
   many_emb_layernorm_kernel.cu
   prompt_tuning_emb_layernorm_varseqlen_kernel_hface.cu