[PIR] move pir, hlir code generating from make stage to cmake stage (#59129)

MarioLulab · web-flow · commit 20063ce7df9c · 2023-11-21T18:42:43.000+08:00
* initial modifying

* fix error for copy_if_different

* add add gitignore

* modify
paddle/cinn/hlir/dialect/operator/ir/CMakeLists.txt

* modify gitignore

* fix

* polish

* remove
include_directories(${CMAKE_CURRENT_BINARY_DIR})

* revert
delete include_directories(${CMAKE_CURRENT_BINARY_DIR})
diff --git a/.gitignore b/.gitignore
@@ -100,3 +100,9 @@ paddle/fluid/ir_adaptor/translator/op_compat_info.cc
 paddle/phi/kernels/fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen/*
 paddle/fluid/pybind/static_op_function.*
 paddle/fluid/pybind/ops_api.cc
+paddle/fluid/pir/dialect/operator/ir/pd_api.*
+paddle/fluid/pir/dialect/operator/ir/op_decomp.cc
+paddle/fluid/pir/dialect/operator/ir/pd_op_vjp.cc
+paddle/fluid/pir/dialect/operator/ir/pd_op.*
+paddle/cinn/hlir/dialect/generated/ops.parsed.yaml
+paddle/cinn/hlir/dialect/operator/ir/cinn_op.*
diff --git a/cmake/generic.cmake b/cmake/generic.cmake
@@ -90,7 +90,6 @@
 #
 #   paddle_test(example SRCS example_test.cc)
 #
-
 # including binary directory for generated headers.
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 # including io directory for inference lib paddle_api.h
diff --git a/paddle/cinn/hlir/dialect/operator/ir/CMakeLists.txt b/paddle/cinn/hlir/dialect/operator/ir/CMakeLists.txt
@@ -1,8 +1,8 @@
 # TODO(Aurelius84): pir_compiler depends on pd_op_dialect and could
 # not found under CINN_ONLY mode
 if(NOT CINN_ONLY)
-  set(CINN_DIALECT_BINARY_DIR
-      "${PADDLE_BINARY_DIR}/paddle/cinn/hlir/dialect/operator/ir")
+  set(CINN_DIALECT_SOURCE_DIR
+      "${PADDLE_SOURCE_DIR}/paddle/cinn/hlir/dialect/operator/ir")
 
   # Generate cinn_op_dialect files defining op using op_gen_file
   set(cinn_op_gen_parsed_yaml_file
@@ -25,34 +25,36 @@ if(NOT CINN_ONLY)
 
   set(cinn_op_namespace cinn,dialect)
   set(cinn_op_dialect_name cinn_op)
-  set(cinn_op_header_file ${CINN_DIALECT_BINARY_DIR}/cinn_op.h)
-  set(cinn_op_source_file ${CINN_DIALECT_BINARY_DIR}/cinn_op.cc)
+  set(cinn_op_header_file ${CINN_DIALECT_SOURCE_DIR}/cinn_op.h)
+  set(cinn_op_source_file ${CINN_DIALECT_SOURCE_DIR}/cinn_op.cc)
   set(cinn_op_header_file_tmp ${cinn_op_header_file}.tmp)
   set(cinn_op_source_file_tmp ${cinn_op_source_file}.tmp)
 
-  add_custom_command(
-    OUTPUT ${cinn_op_parsed_yaml_file}
+  execute_process(
     COMMAND ${CMAKE_COMMAND} -E make_directory ${parsed_op_dir}
     COMMAND ${PYTHON_EXECUTABLE} ${cinn_op_gen_parsed_yaml_file} --op_yaml_path
-            ${cinn_op_yaml_file} --output_path ${cinn_op_parsed_yaml_file}
-    DEPENDS ${cinn_op_gen_parsed_yaml_file} ${cinn_op_yaml_file}
-    VERBATIM)
+            ${cinn_op_yaml_file} --output_path ${cinn_op_parsed_yaml_file})
 
-  add_custom_command(
-    OUTPUT ${cinn_op_header_file} ${cinn_op_source_file}
+  execute_process(
     COMMAND
       ${PYTHON_EXECUTABLE} ${cinn_op_gen_file} --op_yaml_files
       ${cinn_op_parsed_yaml_files} --op_compat_yaml_file
       ${cinn_op_compat_yaml_file} --namespaces ${cinn_op_namespace}
       --dialect_name ${cinn_op_dialect_name} --op_def_h_file
-      ${cinn_op_header_file_tmp} --op_def_cc_file ${cinn_op_source_file_tmp}
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${cinn_op_header_file_tmp}
-            ${cinn_op_header_file}
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${cinn_op_source_file_tmp}
-            ${cinn_op_source_file}
-    DEPENDS ${cinn_op_gen_file} ${cinn_op_parsed_yaml_file}
-            ${cinn_op_compat_yaml_file}
-    VERBATIM)
+      ${cinn_op_header_file_tmp} --op_def_cc_file ${cinn_op_source_file_tmp})
+
+  set(generated_files_cinn_op "${cinn_op_header_file}" "${cinn_op_source_file}")
+  foreach(generated_file ${generated_files_cinn_op})
+    if(EXISTS "${generated_file}.tmp" AND EXISTS "${generated_file}")
+      execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                              "${generated_file}.tmp" "${generated_file}")
+      message("copy if different ${generated_file}.tmp ${generated_file}")
+    elseif(EXISTS "${generated_file}.tmp")
+      execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${generated_file}.tmp"
+                              "${generated_file}")
+      message("copy ${generated_file}.tmp ${generated_file}")
+    endif()
+  endforeach()
 
   cinn_cc_library(
     cinn_op_dialect
@@ -64,5 +66,5 @@ if(NOT CINN_ONLY)
     DEPS
     op_dialect_vjp)
 
-  target_include_directories(cinn_op_dialect PRIVATE ${CINN_DIALECT_BINARY_DIR})
+  target_include_directories(cinn_op_dialect PRIVATE ${CINN_DIALECT_SOURCE_DIR})
 endif()
diff --git a/paddle/fluid/pir/dialect/CMakeLists.txt b/paddle/fluid/pir/dialect/CMakeLists.txt
@@ -1,5 +1,5 @@
-set(PD_DIALECT_BINARY_DIR
-    "${PADDLE_BINARY_DIR}/paddle/fluid/pir/dialect/operator/ir")
+set(PD_DIALECT_SOURCE_DIR
+    "${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/operator/ir")
 
 # Generate pd_op_dialect files defining op using op_gen_file
 set(op_gen_parsed_yaml_file
@@ -48,13 +48,13 @@ set(op_yaml_files
 )
 set(op_namespace paddle,dialect)
 set(dialect_name pd_op)
-set(op_header_file ${PD_DIALECT_BINARY_DIR}/pd_op.h)
-set(op_source_file ${PD_DIALECT_BINARY_DIR}/pd_op.cc)
+set(op_header_file ${PD_DIALECT_SOURCE_DIR}/pd_op.h)
+set(op_source_file ${PD_DIALECT_SOURCE_DIR}/pd_op.cc)
 set(op_header_file_tmp ${op_header_file}.tmp)
 set(op_source_file_tmp ${op_source_file}.tmp)
 
-set(op_vjp_source_file ${PD_DIALECT_BINARY_DIR}/pd_op_vjp.cc)
-set(op_decomp_source_file ${PD_DIALECT_BINARY_DIR}/op_decomp.cc)
+set(op_vjp_source_file ${PD_DIALECT_SOURCE_DIR}/pd_op_vjp.cc)
+set(op_decomp_source_file ${PD_DIALECT_SOURCE_DIR}/op_decomp.cc)
 set(op_vjp_source_file_tmp ${op_vjp_source_file}.tmp)
 
 execute_process(
@@ -66,70 +66,36 @@ execute_process(
   COMMAND ${PYTHON_EXECUTABLE} ${op_gen_parsed_yaml_file} --op_yaml_path
           ${pd_op_backward_yaml_file} --output_path ${op_yaml_file4} --backward)
 
-add_custom_command(
-  OUTPUT ${op_header_file} ${op_source_file} ${op_vjp_source_file}
+execute_process(
   COMMAND
     ${PYTHON_EXECUTABLE} ${op_gen_file} --op_yaml_files ${op_yaml_files}
     --op_compat_yaml_file ${op_compat_yaml_file} --namespaces ${op_namespace}
     --dialect_name ${dialect_name} --op_def_h_file ${op_header_file_tmp}
     --op_def_cc_file ${op_source_file_tmp} --op_vjp_cc_file
-    ${op_vjp_source_file_tmp}
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${op_header_file_tmp}
-          ${op_header_file}
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${op_source_file_tmp}
-          ${op_source_file}
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${op_vjp_source_file_tmp}
-          ${op_vjp_source_file}
-  COMMENT
-    "copy_if_different ${op_header_file} ${op_source_file} ${op_vjp_source_file}"
-  DEPENDS ${op_gen_file}
-          ${op_forward_yaml_file1}
-          ${op_forward_yaml_file2}
-          ${op_backward_yaml_file1}
-          ${op_backward_yaml_file2}
-          ${op_compat_yaml_file}
-          ${op_yaml_file3}
-          ${op_yaml_file4}
-          ${op_yaml_file5}
-  VERBATIM)
-add_custom_target(
-  op_header_and_source_gen ALL DEPENDS ${op_header_file} ${op_source_file}
-                                       ${op_vjp_source_file})
+    ${op_vjp_source_file_tmp})
+
+set(generated_files_pd_op "${op_header_file}" "${op_source_file}"
+                          "${op_vjp_source_file}")
+
 set(api_gen_yaml_files
     ${op_forward_yaml_file1},${op_forward_yaml_file2},${op_backward_yaml_file1},${op_backward_yaml_file2},${op_yaml_file3},${op_yaml_file4},${op_yaml_file5}
 )
 set(api_gen_file
     ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/op_generator/api_gen.py)
-set(api_header_file ${PD_DIALECT_BINARY_DIR}/pd_api.h)
-set(api_source_file ${PD_DIALECT_BINARY_DIR}/pd_api.cc)
+set(api_header_file ${PD_DIALECT_SOURCE_DIR}/pd_api.h)
+set(api_source_file ${PD_DIALECT_SOURCE_DIR}/pd_api.cc)
 set(api_header_file_tmp ${api_header_file}.tmp)
 set(api_source_file_tmp ${api_source_file}.tmp)
 
-add_custom_command(
-  OUTPUT ${api_header_file} ${api_source_file}
+execute_process(
   COMMAND
     ${PYTHON_EXECUTABLE} ${api_gen_file} --op_yaml_files ${op_yaml_files}
     --op_compat_yaml_file ${op_compat_yaml_file} --namespaces ${op_namespace}
     --api_def_h_file ${api_header_file_tmp} --api_def_cc_file
-    ${api_source_file_tmp}
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${api_header_file_tmp}
-          ${api_header_file}
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${api_source_file_tmp}
-          ${api_source_file}
-  COMMENT "copy_if_different ${api_header_file} ${api_source_file}"
-  DEPENDS ${api_gen_file}
-          ${op_forward_yaml_file1}
-          ${op_forward_yaml_file2}
-          ${op_backward_yaml_file1}
-          ${op_backward_yaml_file2}
-          ${op_compat_yaml_file}
-          ${op_yaml_file3}
-          ${op_yaml_file4}
-          ${op_yaml_file5}
-  VERBATIM)
+    ${api_source_file_tmp})
+
+set(generated_files_pd_api "${api_header_file}" "${api_source_file}")
 
-add_custom_target(api_header_and_source_gen ALL DEPENDS ${api_header_file}
-                                                        ${api_source_file})
 set(python_c_gen_file
     ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/op_generator/python_c_gen.py)
 set(python_c_header_file
@@ -139,55 +105,51 @@ set(python_c_source_file
 set(python_c_header_file_tmp ${python_c_header_file}.tmp)
 set(python_c_source_file_tmp ${python_c_source_file}.tmp)
 
-add_custom_command(
-  OUTPUT ${python_c_header_file} ${python_c_source_file}
+execute_process(
   COMMAND
     ${PYTHON_EXECUTABLE} ${python_c_gen_file} --op_yaml_files ${op_yaml_files}
     --op_compat_yaml_file ${op_compat_yaml_file} --namespaces "paddle,pybind"
     --python_c_def_h_file ${python_c_header_file_tmp} --python_c_def_cc_file
-    ${python_c_source_file_tmp}
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${python_c_header_file_tmp}
-          ${python_c_header_file}
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${python_c_source_file_tmp}
-          ${python_c_source_file}
-  COMMENT "copy_if_different ${python_c_header_file} ${python_c_source_file}"
-  DEPENDS ${python_c_gen_file}
-          ${op_forward_yaml_file1}
-          ${op_forward_yaml_file2}
-          ${op_backward_yaml_file1}
-          ${op_backward_yaml_file2}
-          ${op_compat_yaml_file}
-          ${op_yaml_file3}
-          ${op_yaml_file4}
-          ${op_yaml_file5}
-  VERBATIM)
+    ${python_c_source_file_tmp})
 
-add_custom_target(static_op_function_gen ALL DEPENDS ${python_c_header_file}
-                                                     ${python_c_source_file})
+set(generated_files_python_c "${python_c_header_file}"
+                             "${python_c_source_file}")
 
 set(ops_api_gen_file
     ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py)
 set(ops_api_source_file ${PADDLE_SOURCE_DIR}/paddle/fluid/pybind/ops_api.cc)
 set(ops_api_source_file_tmp ${ops_api_source_file}.tmp)
 
-add_custom_command(
-  OUTPUT ${ops_api_source_file}
+execute_process(
   COMMAND
     ${PYTHON_EXECUTABLE} ${ops_api_gen_file} --op_yaml_files ${op_yaml_files}
     --op_compat_yaml_file ${op_compat_yaml_file} --namespaces "paddle,pybind"
-    --ops_api_file ${ops_api_source_file_tmp}
-  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ops_api_source_file_tmp}
-          ${ops_api_source_file}
-  COMMENT "copy_if_different ${ops_api_source_file}"
-  DEPENDS ${ops_api_gen_file}
-          ${op_forward_yaml_file1}
-          ${op_forward_yaml_file2}
-          ${op_backward_yaml_file1}
-          ${op_backward_yaml_file2}
-          ${op_compat_yaml_file}
-          ${python_c_header_file}
-          ${python_c_source_file}
-  VERBATIM)
+    --ops_api_file ${ops_api_source_file_tmp})
+
+set(generated_files_ops_api "${ops_api_source_file}")
+
+set(generated_files_pir ${generated_files_pd_op} ${generated_files_pd_api}
+                        ${generated_files_python_c} ${generated_files_ops_api})
+foreach(generated_file ${generated_files_pir})
+  if(EXISTS "${generated_file}.tmp" AND EXISTS "${generated_file}")
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                            "${generated_file}.tmp" "${generated_file}")
+    message("copy if different ${generated_file}.tmp ${generated_file}")
+  elseif(EXISTS "${generated_file}.tmp")
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${generated_file}.tmp"
+                            "${generated_file}")
+    message("copy ${generated_file}.tmp ${generated_file}")
+  endif()
+endforeach()
+
+add_custom_target(
+  op_header_and_source_gen ALL DEPENDS ${op_header_file} ${op_source_file}
+                                       ${op_vjp_source_file})
+add_custom_target(api_header_and_source_gen ALL DEPENDS ${api_header_file}
+                                                        ${api_source_file})
+
+add_custom_target(static_op_function_gen ALL DEPENDS ${python_c_header_file}
+                                                     ${python_c_source_file})
 
 add_custom_target(ops_api_gen ALL DEPENDS ${ops_api_source_file})
 
diff --git a/paddle/fluid/primitive/codegen/CMakeLists.txt b/paddle/fluid/primitive/codegen/CMakeLists.txt
@@ -46,7 +46,7 @@ execute_process(
     ${fwd_path} --fwd_legacy_path ${fwd_legacy_path} --fwd_pd_op_path
     ${fwd_pd_op_path} --templates_dir ${templates_dir} --compat_path
     ${compat_path} --destination_dir
-    ${PADDLE_BINARY_DIR}/paddle/fluid/pir/dialect/operator/ir/op_decomp.cc
+    ${PADDLE_SOURCE_DIR}/paddle/fluid/pir/dialect/operator/ir/op_decomp.cc
   RESULT_VARIABLE _result)
 if(${_result})
   message(

Original file line number	Diff line number	Diff line change
`@@ -90,7 +90,6 @@`
`90`	`90`	`#`
`91`	`91`	`# paddle_test(example SRCS example_test.cc)`
`92`	`92`	`#`
`93`		`-`
`94`	`93`	`# including binary directory for generated headers.`
`95`	`94`	`include_directories(${CMAKE_CURRENT_BINARY_DIR})`
`96`	`95`	`# including io directory for inference lib paddle_api.h`