PaddlePaddle
diff --git a/‎paddle/fluid/framework/new_executor/instruction/tensorrt_engine_instruction.cc
Copy file name to clipboard
+3Lines changed: 3 additions & 0 deletions b/‎paddle/fluid/framework/new_executor/instruction/tensorrt_engine_instruction.cc
Copy file name to clipboard
+3Lines changed: 3 additions & 0 deletions
diff --git a/‎paddle/fluid/pir/dialect/operator/ir/tensorrt_op.cc
Copy file name to clipboard
+11-15Lines changed: 11 additions & 15 deletions b/‎paddle/fluid/pir/dialect/operator/ir/tensorrt_op.cc
Copy file name to clipboard
+11-15Lines changed: 11 additions & 15 deletions
diff --git a/‎paddle/fluid/pir/transforms/tensorrt/trt_op_marker_pass.cc
Copy file name to clipboard
+3-2Lines changed: 3 additions & 2 deletions b/‎paddle/fluid/pir/transforms/tensorrt/trt_op_marker_pass.cc
Copy file name to clipboard
+3-2Lines changed: 3 additions & 2 deletions
diff --git a/‎paddle/fluid/pybind/manual_static_op_function.h
Copy file name to clipboard
+1-1Lines changed: 1 addition & 1 deletion b/‎paddle/fluid/pybind/manual_static_op_function.h
Copy file name to clipboard
+1-1Lines changed: 1 addition & 1 deletion
diff --git a/‎python/paddle/tensorrt/converter.py
Copy file name to clipboard
+15-5Lines changed: 15 additions & 5 deletions b/‎python/paddle/tensorrt/converter.py
Copy file name to clipboard
+15-5Lines changed: 15 additions & 5 deletions
diff --git a/‎python/paddle/tensorrt/impls/core.py
Copy file name to clipboard
+1-1Lines changed: 1 addition & 1 deletion b/‎python/paddle/tensorrt/impls/core.py
Copy file name to clipboard
+1-1Lines changed: 1 addition & 1 deletion
diff --git a/‎python/paddle/tensorrt/register.py
Copy file name to clipboard
+2-2Lines changed: 2 additions & 2 deletions b/‎python/paddle/tensorrt/register.py
Copy file name to clipboard
+2-2Lines changed: 2 additions & 2 deletions
diff --git a/‎python/paddle/tensorrt/util.py
Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions b/‎python/paddle/tensorrt/util.py
Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions
diff --git a/‎test/tensorrt/CMakeLists.txt
Copy file name to clipboard
+3Lines changed: 3 additions & 0 deletions b/‎test/tensorrt/CMakeLists.txt
Copy file name to clipboard
+3Lines changed: 3 additions & 0 deletions
diff --git a/‎test/tensorrt/tensorrt_test_base.py
Copy file name to clipboard
+211Lines changed: 211 additions & 0 deletions b/‎test/tensorrt/tensorrt_test_base.py
Copy file name to clipboard
+211Lines changed: 211 additions & 0 deletions
@@ -592,6 +592,9 @@ void TensorRTEngineInstruction::BindOutputTensor(
     int output_index,
     std::vector<void *> &buffers,
     int *runtime_batch) {
+  if (output_name == "") {
+    return;
+  }
   int binding_offset = 0;
   const int num_bindings = trt_engine_->GetNbBindings();
   nvinfer1::IExecutionContext *trt_context = nullptr;
 
@@ -185,13 +185,17 @@ void TensorRTEngineOp::Build(pir::Builder &builder,             // NOLINT
   std::vector<pir::Type> argument_outputs;
   std::vector<pir::Type> out_types;
   for (size_t i = 0; i < static_cast<size_t>(outputs_shape.size()); i++) {
-    out_types.push_back(pir::DenseTensorType::get(
-        pir::IrContext::Instance(),
-        TransToIrDataType(outputs_dtype[i]),
-        phi::DDim(outputs_shape[i].data(), outputs_shape[i].size()),
-        phi::DataLayout::ALL_LAYOUT,
-        phi::LoD(),
-        0));
+    if (outputs_dtype[i] == phi::DataType::UNDEFINED) {
+      out_types.push_back(pir::Type());
+    } else {
+      out_types.push_back(pir::DenseTensorType::get(
+          pir::IrContext::Instance(),
+          TransToIrDataType(outputs_dtype[i]),
+          phi::DDim(outputs_shape[i].data(), outputs_shape[i].size()),
+          phi::DataLayout::ALL_LAYOUT,
+          phi::LoD(),
+          0));
+    }
   }
   pir::Type out_vector_type =
       pir::VectorType::get(pir::IrContext::Instance(), out_types);
@@ -259,14 +263,6 @@ void TensorRTEngineOp::VerifySig() {
                       true,
                       common::errors::InvalidArgument(
                           "Type validation failed for the 0th output."));
-    if (auto vec_type = output_type.dyn_cast<pir::VectorType>()) {
-      for (size_t i = 0; i < vec_type.size(); i++) {
-        PADDLE_ENFORCE_EQ(vec_type[i].isa<pir::DenseTensorType>(),
-                          true,
-                          common::errors::InvalidArgument(
-                              "Type validation failed for the 0th output."));
-      }
-    }
   }
   VLOG(4) << "End Verifying for: TensorRTEngineOp.";
 }
 
@@ -1110,7 +1110,6 @@ class TrtOpMarkerPass : public pir::PatternRewritePass {
     ADD_PATTERN(Full)
     ADD_PATTERN(Fused_gemm_epilogue)
     ADD_PATTERN(Add)
-    ADD_PATTERN(Layer_norm)
     ADD_PATTERN(Silu)
     ADD_PATTERN(Conv2d)
     ADD_PATTERN(FusedConv2dAddAct)
@@ -1120,7 +1119,9 @@ class TrtOpMarkerPass : public pir::PatternRewritePass {
     ADD_PATTERN(Shape)
     ADD_PATTERN(Expand)
     ADD_PATTERN(Sigmoid)
-
+#if IS_TRT_VERSION_GE(8600)
+    ADD_PATTERN(Layer_norm)
+#endif
 #undef ADD_PATTERN
     ps.Add(std::make_unique<Pool2dOpPattern>(context));
     ps.Add(std::make_unique<Conv2dTransposeOpPattern>(context));
 
@@ -1054,7 +1054,7 @@ static PyObject *static_api_tensorrt_engine(PyObject *self,
 
     // Get Value from args
     PyObject *x_obj = PyTuple_GET_ITEM(args, 0);
-    auto x = CastPyArg2VectorOfValue(x_obj, "tensorrt_engine", 0);
+    auto x = CastPyArg2VectorOfValue(x_obj, "tensorrt_engine", 0, true);
 
     PyObject *param_obj = PyTuple_GET_ITEM(args, 1);
     if (!PyObject_TypeCheck(param_obj, g_tensorrt_engine_params_pytype)) {
 
@@ -40,7 +40,6 @@ def get_cache_path():
     return cache_path
 
 
-paddle.framework.set_flags({"FLAGS_enable_collect_shape": True})
 _logger = get_logger(
     __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s'
 )
@@ -103,7 +102,7 @@ def convert_subgraph_to_trt(self, program, group_op):
         _logger.info(f"start process {group_op}")
         operations = next(iter(group_op.blocks())).ops
         input_values, output_values = self.find_graph_inputs_outputs(group_op)
-        builder = trt.Builder(trt.Logger(trt.Logger.VERBOSE))
+        builder = trt.Builder(trt.Logger(trt.Logger.ERROR))
         network = builder.create_network(
             1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)
         )
@@ -212,12 +211,21 @@ def convert_subgraph_to_trt(self, program, group_op):
         out_shapes = []
         out_names = []
         out_types = []
-        for result_value in output_values:
+        for out_index in range(len(output_values)):
+            result_value = output_values[out_index]
             output_tensor = value_to_trt_tensor[result_value.id]
+            if output_tensor is None:
+                out_names.append("")
+                out_shapes.append([])
+                out_types.append(None)
+                continue
             network.mark_output(output_tensor)
             out_names.append(output_tensor.name)
             out_shapes.append(result_value.shape)
             out_types.append(result_value.dtype)
+            if group_op.result(out_index).use_empty():
+                # if result value is not used, it doesn't need get shape, continue
+                continue
             min_shape = get_value_shape_range_info(
                 result_value, False, paddle.base.core.ShapeMode.kMIN
             )
@@ -235,10 +243,9 @@ def convert_subgraph_to_trt(self, program, group_op):
 
         config = builder.create_builder_config()
         config.add_optimization_profile(profile)
-
         if version_list[0] > 8 or (
             version_list[0] == 8 and version_list[1] >= 6
-        ):
+        ):  # trt version >= 8.6
             config.builder_optimization_level = 5
         config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 30)
         trt_engine = builder.build_engine(network, config)
@@ -273,6 +280,9 @@ def convert_subgraph_to_trt(self, program, group_op):
             )
 
             for out_index in range(len(out)):
+                if group_op.result(out_index).use_empty():
+                    # if result value is not been used, it doesn't need get shape, continue
+                    continue
                 ori_value = output_values[out_index]
                 current_value = out[out_index]
                 orin_min_shape = self.shape_map[ori_value.id]["min_shape"]
 
@@ -194,7 +194,7 @@ def softmax_converter(network, paddle_op, inputs):
 
 
 @converter_registry.register(
-    "pd_op.layer_norm", trt_version="trt_version_gt=8.6"
+    "pd_op.layer_norm", trt_version="trt_version_ge=8.6"
 )
 def layernorm_converter(network, paddle_op, inputs):
     input_a, scale, bias = inputs
 
@@ -39,10 +39,10 @@ def _version_match(self, trt_version, version_range):
             return True
 
         trt_major, trt_minor = map(int, trt_version.split('.')[:2])
-        if version_range.startswith('trt_version_gt='):
+        if version_range.startswith('trt_version_ge='):
             min_version = float(version_range.split('=')[1])
             return float(trt_major) + trt_minor / 10 >= min_version
-        elif version_range.startswith('trt_version_lt='):
+        elif version_range.startswith('trt_version_le='):
             max_version = float(version_range.split('=')[1])
             return float(trt_major) + trt_minor / 10 <= max_version
         elif 'x' in version_range:
 
@@ -95,6 +95,7 @@ def predict_program(program, feed_data, fetch_var_list):
 
 
 def warmup_shape_infer(program, min_shape_feed, max_shape_feed):
+    paddle.framework.set_flags({"FLAGS_enable_collect_shape": True})
     with paddle.pir_utils.IrGuard():
         with paddle.static.program_guard(program):
             executor = paddle.static.Executor()
@@ -110,3 +111,4 @@ def warmup_shape_infer(program, min_shape_feed, max_shape_feed):
                 executor.run(
                     program, feed=max_shape_feed, fetch_list=[output_var]
                 )
+    paddle.framework.set_flags({"FLAGS_enable_collect_shape": False})
@@ -11,4 +11,7 @@ if(NOT WIN32 AND TENSORRT_FOUND)
   set_tests_properties(test_converter_model_bert PROPERTIES TIMEOUT "500")
   set_tests_properties(test_converter_model_dummy PROPERTIES TIMEOUT "500")
   set_tests_properties(test_converter_model_resnet50 PROPERTIES TIMEOUT "500")
+  set_tests_properties(test_converter_conv2d PROPERTIES TIMEOUT "100")
+  set_tests_properties(test_converter_add PROPERTIES TIMEOUT "100")
+  set_tests_properties(test_converter_batch_norm PROPERTIES TIMEOUT "100")
 endif()
@@ -0,0 +1,211 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import unittest
+
+import numpy as np
+
+import paddle
+from paddle.base import core
+from paddle.tensorrt.converter import PaddleToTensorRTConverter
+from paddle.tensorrt.util import (
+    run_pir_pass,
+    warmup_shape_infer,
+)
+
+
+class TensorRTBaseTest(unittest.TestCase):
+    def setUp(self):
+        self.python_api = None
+        self.api_args = None
+        self.program_config = None
+        self.min_shape = None
+        self.max_shape = None
+
+    def create_fake_program(self):
+        if self.python_api is None:
+            raise ValueError(
+                "The unittest must specify a python api that will be used for building pir program."
+            )
+        main_program = paddle.static.Program()
+        startup_program = paddle.static.Program()
+        with paddle.static.program_guard(main_program, startup_program):
+            api_args = copy.deepcopy(self.api_args)
+            for feed_name in self.program_config["feed_list"]:
+                if isinstance(self.api_args[feed_name], dict):
+                    new_list_args = []
+                    for sub_arg_name, sub_arg_value in self.api_args[
+                        feed_name
+                    ].items():
+                        input_shape_without_dynamic_dim = sub_arg_value.shape[
+                            1:
+                        ]
+                        input_dynamic_shape = [-1]
+                        input_dynamic_shape.extend(
+                            input_shape_without_dynamic_dim
+                        )
+                        input_dtype = sub_arg_value.dtype
+                        input_data = paddle.static.data(
+                            name=sub_arg_name,
+                            shape=input_dynamic_shape,
+                            dtype=input_dtype,
+                        )
+                        new_list_args.append(input_data)
+                    api_args[feed_name] = new_list_args
+                else:
+                    input_shape_without_dynamic_dim = self.api_args[
+                        feed_name
+                    ].shape[1:]
+                    input_dynamic_shape = [-1]
+                    input_dynamic_shape.extend(input_shape_without_dynamic_dim)
+                    input_dtype = self.api_args[feed_name].dtype
+                    input_data = paddle.static.data(
+                        name=feed_name,
+                        shape=input_dynamic_shape,
+                        dtype=input_dtype,
+                    )
+                    api_args[feed_name] = input_data
+            actual_args = []
+            for name, value in api_args.items():
+                actual_args.append(value)
+            output = self.python_api(*actual_args)
+            fetch_list = []
+            if isinstance(output, tuple):
+                fetch_list = list(output)
+            else:
+                fetch_list.append(output)
+        return main_program, startup_program, fetch_list
+
+    def run_program(self, main_program, fetch_list):
+        place = (
+            paddle.CUDAPlace(0)
+            if core.is_compiled_with_cuda()
+            else paddle.CPUPlace()
+        )
+        exe = paddle.static.Executor(place)
+        feed_data = dict()  # noqa: C408
+        for feed_name in self.program_config["feed_list"]:
+            if isinstance(self.api_args[feed_name], dict):
+                for sub_arg_name, sub_arg_value in self.api_args[
+                    feed_name
+                ].items():
+                    feed_data[sub_arg_name] = sub_arg_value
+            else:
+                feed_data[feed_name] = self.api_args[feed_name]
+        ret = exe.run(main_program, feed=feed_data, fetch_list=fetch_list)
+        return ret
+
+    def prepare_feed(self):
+        for arg_name, arg_value in self.api_args.items():
+            # deal with condition that input is a list tensor
+            if (
+                isinstance(self.api_args[arg_name], list)
+                and arg_name in self.program_config["feed_list"]
+            ):
+                new_list_args = dict()  # noqa: C408
+                for i in range(len(self.api_args[arg_name])):
+                    sub_arg_name = arg_name + str(i)
+                    new_list_args[sub_arg_name] = self.api_args[arg_name][i]
+                self.api_args[arg_name] = new_list_args
+
+    def check_trt_result(self):
+        paddle.framework.set_flags({"FLAGS_trt_min_group_size": 1})
+        with paddle.pir_utils.IrGuard():
+            self.prepare_feed()
+            main_program, startup_program, fetch_list = (
+                self.create_fake_program()
+            )
+            place = (
+                paddle.CUDAPlace(0)
+                if core.is_compiled_with_cuda()
+                else paddle.CPUPlace()
+            )
+            exe = paddle.static.Executor(place)
+            # init all parameter
+            exe.run(startup_program)
+            fetch_num = len(fetch_list)
+            fetch_index = [v.index() for v in fetch_list]
+            output_expected = self.run_program(main_program, fetch_list)
+
+            min_shape_data = dict()  # noqa: C408
+            max_shape_data = dict()  # noqa: C408
+            for feed_name in self.program_config["feed_list"]:
+                if isinstance(self.api_args[feed_name], dict):
+                    for i in range(len(self.min_shape[feed_name])):
+                        sub_feed_name = feed_name + str(i)
+                        min_shape_data[sub_feed_name] = np.random.randn(
+                            *self.min_shape[feed_name][i]
+                        ).astype(self.api_args[feed_name][sub_feed_name].dtype)
+                        max_shape_data[sub_feed_name] = np.random.randn(
+                            *self.max_shape[feed_name][i]
+                        ).astype(self.api_args[feed_name][sub_feed_name].dtype)
+                else:
+                    min_shape_data[feed_name] = np.random.randn(
+                        *self.min_shape[feed_name]
+                    ).astype(self.api_args[feed_name].dtype)
+                    max_shape_data[feed_name] = np.random.randn(
+                        *self.max_shape[feed_name]
+                    ).astype(self.api_args[feed_name].dtype)
+
+            warmup_shape_infer(
+                main_program,
+                min_shape_feed=min_shape_data,
+                max_shape_feed=max_shape_data,
+            )
+
+            for op in main_program.global_block().ops[::-1]:
+                # Remove all invalid fetch op
+                if op.name() == "pd_op.fetch":
+                    main_program.global_block().remove_op(op)
+
+            # run pir pass(including some fusion pass and trt_op_marker_pass)
+            main_program = run_pir_pass(main_program, partition_mode=False)
+
+            # run trt_sub_graph_extract_pass()
+            program_with_trt = run_pir_pass(main_program, partition_mode=True)
+
+            # run TRTConverter(would lower group_op into tensorrt_engine_op)
+            scope = paddle.static.global_scope()
+            converter = PaddleToTensorRTConverter(program_with_trt, scope)
+            converter.convert_program_to_trt()
+
+            # check whether has trt op
+            has_trt_op = False
+            for op in program_with_trt.global_block().ops:
+                if op.name() == "pd_op.tensorrt_engine":
+                    has_trt_op = True
+            self.assertEqual(has_trt_op, True)
+
+            trt_fetch_list = []
+            split_op = program_with_trt.global_block().ops[-1]
+            if split_op.name() == "builtin.split":
+                trt_fetch_list = [
+                    split_op.result(index) for index in fetch_index
+                ]
+            else:
+                raise ValueError(
+                    "The last op of convert pir Program in test must be split op that is the next op of pd_op.engine."
+                )
+
+            output_trt = self.run_program(program_with_trt, trt_fetch_list)
+
+        # Check that the results are close to each other within a tolerance of 1e-3
+        for i in range(fetch_num):
+            np.testing.assert_allclose(
+                output_expected[i],
+                output_trt[i],
+                rtol=1e-3,
+                atol=1e-3,
+            )
Original file line number	Diff line number	Diff line change
`@@ -194,7 +194,7 @@ def softmax_converter(network, paddle_op, inputs):`
`194`	`194`
`195`	`195`
`196`	`196`	`@converter_registry.register(`
`197`		`- "pd_op.layer_norm", trt_version="trt_version_gt=8.6"`
	`197`	`+ "pd_op.layer_norm", trt_version="trt_version_ge=8.6"`
`198`	`198`	`)`
`199`	`199`	`def layernorm_converter(network, paddle_op, inputs):`
`200`	`200`	`input_a, scale, bias = inputs`