[fluid_ops] lod_tensor.h (#69775)

co63oc · web-flow · commit 341fce9e3ad2 · 2024-12-02T11:18:07.000+08:00
diff --git a/paddle/fluid/distributed/fleet_executor/compute_interceptor.cc b/paddle/fluid/distributed/fleet_executor/compute_interceptor.cc
@@ -64,7 +64,7 @@ void ComputeInterceptor::DecodeMsgVars(const InterceptorMessage& msg) {
     std::istringstream ss(var_iter.stensor());
     auto* var = scope->Var(name);
     auto* tensor = var->GetMutable<phi::DenseTensor>();
-    framework::DeserializeFromStream(ss, tensor, dev_ctx);
+    phi::DeserializeFromStream(ss, tensor, dev_ctx);
 
     VLOG(3) << "Set vars " << name << " with value in scope " << scope_id
             << " with dims " << tensor->dims() << " with dtype "
@@ -98,7 +98,7 @@ InterceptorMessage ComputeInterceptor::PrepareVarsMsg() {
         common::errors::NotFound(
             "Variable %s not exists in scope %ld", var_name, cur_scope_id_));
     const auto& tensor = var->Get<phi::DenseTensor>();
-    framework::SerializeToStream(ss, tensor, dev_ctx);
+    phi::SerializeToStream(ss, tensor, dev_ctx);
     vars->set_stensor(ss.str());
     VLOG(3) << "Prepare vars msg " << var_name << " with dimension "
             << tensor.dims() << " dtype " << tensor.dtype();
diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_client.cc b/paddle/fluid/distributed/ps/service/brpc_ps_client.cc
@@ -1520,7 +1520,7 @@ int32_t BrpcPsClient::RecvAndSaveTable(const uint64_t table_id,
                     common::errors::Unavailable(
                         "Cannot open %s to save variables.", file_name));
 
-  framework::SerializeToStream(fout, *var_tensor, dev_ctx);
+  phi::SerializeToStream(fout, *var_tensor, dev_ctx);
   fout.close();
 
   return 0;
diff --git a/paddle/fluid/framework/details/nan_inf_utils_detail.h b/paddle/fluid/framework/details/nan_inf_utils_detail.h
@@ -86,7 +86,8 @@ void tensor_check(const std::string& op_type,
                   const phi::DenseTensor& tensor,
                   const phi::Place& place) {
   TensorCheckerVisitor<Context> vistor(op_type, var_name, tensor, place);
-  VisitDataType(framework::TransToProtoVarType(tensor.dtype()), vistor);
+  framework::VisitDataType(framework::TransToProtoVarType(tensor.dtype()),
+                           vistor);
 }
 
 void InitWhiteListFormEnv();
diff --git a/paddle/fluid/framework/io/save_load_tensor.cc b/paddle/fluid/framework/io/save_load_tensor.cc
@@ -34,7 +34,7 @@ void SaveTensor(const phi::DenseTensor& x,
                     true,
                     common::errors::Unavailable(
                         "Cannot open %s to save variables.", new_path));
-  framework::SerializeToStream(fout, x);
+  phi::SerializeToStream(fout, x);
 
   fout.close();
 }
@@ -51,6 +51,6 @@ void LoadTensor(const std::string& file_path, phi::DenseTensor* out) {
                           common::errors::InvalidArgument(
                               "The variable to be loaded cannot be found."));
 
-  framework::DeserializeFromStream(fin, out);
+  phi::DeserializeFromStream(fin, out);
 }
 }  // namespace paddle::framework
diff --git a/paddle/fluid/framework/lod_tensor.cc b/paddle/fluid/framework/lod_tensor.cc
@@ -75,117 +75,6 @@ bool CheckLoD(const LoD &in, int tensor_height) {
   return true;
 }
 
-void SerializeToStream(std::ostream &os,
-                       const phi::DenseTensor &tensor,
-                       const phi::DeviceContext &dev_ctx) {
-  {  // the 1st field, uint32_t version for DenseTensor
-    os.write(
-        reinterpret_cast<const char *>(&paddle::framework::kCurTensorVersion),
-        sizeof(paddle::framework::kCurTensorVersion));
-  }
-  {
-    // the 2st field, LoD information
-    // uint64_t lod_level
-    // uint64_t lod_level_1 size in byte.
-    // int*     lod_level_1 data
-    // ...
-    auto lod = tensor.lod();
-    uint64_t size = lod.size();
-    os.write(reinterpret_cast<const char *>(&size), sizeof(size));
-
-    for (auto &each : lod) {
-      size = each.size() * sizeof(phi::LoD::value_type::value_type);
-      os.write(reinterpret_cast<const char *>(&size), sizeof(size));
-      os.write(reinterpret_cast<const char *>(each.data()),
-               static_cast<std::streamsize>(size));
-    }
-  }
-  // the 3st field, Tensor
-  paddle::framework::TensorToStream(
-      os, static_cast<phi::DenseTensor>(tensor), dev_ctx);
-}
-
-void SerializeToStream(std::ostream &os, const phi::DenseTensor &tensor) {
-  phi::DeviceContextPool &pool = phi::DeviceContextPool::Instance();
-  const phi::DeviceContext *dev_ctx = nullptr;
-  auto place = tensor.place();
-  dev_ctx = pool.Get(place);
-  SerializeToStream(os, tensor, *dev_ctx);
-}
-
-void DeserializeFromStream(std::istream &os, phi::DenseTensor *tensor) {
-  phi::DeviceContextPool &pool = phi::DeviceContextPool::Instance();
-  const phi::DeviceContext *dev_ctx = nullptr;
-  dev_ctx = pool.Get(phi::CPUPlace());
-  DeserializeFromStream(os, tensor, *dev_ctx);
-}
-
-void DeserializeFromStream(std::istream &is,
-                           phi::DenseTensor *tensor,
-                           const phi::DeviceContext &dev_ctx,
-                           const size_t &seek,
-                           const std::vector<int64_t> &shape) {
-  {
-    // the 1st field, unit32_t version for DenseTensor
-    uint32_t version = 0;
-    is.read(reinterpret_cast<char *>(&version), sizeof(version));
-
-    PADDLE_ENFORCE_EQ(
-        version,
-        0U,
-        common::errors::InvalidArgument(
-            "Deserialize to tensor failed, maybe the loaded file is "
-            "not a paddle model(expected file format: 0, but %u found).",
-            version));
-  }
-  {
-    // the 2st field, LoD information
-    uint64_t lod_level = 0;
-    is.read(reinterpret_cast<char *>(&lod_level), sizeof(lod_level));
-    auto &lod = *tensor->mutable_lod();
-    lod.resize(lod_level);
-  }
-  // the 3st filed, Tensor
-  paddle::framework::TensorFromStream(
-      is, static_cast<phi::DenseTensor *>(tensor), dev_ctx, seek, shape);
-}
-
-void DeserializeFromStream(std::istream &is,
-                           phi::DenseTensor *tensor,
-                           const phi::DeviceContext &dev_ctx) {
-  {
-    // the 1st field, unit32_t version for DenseTensor
-    uint32_t version = 0;
-    is.read(reinterpret_cast<char *>(&version), sizeof(version));
-
-    PADDLE_ENFORCE_EQ(
-        version,
-        0U,
-        common::errors::InvalidArgument(
-            "Deserialize to tensor failed, maybe the loaded file is "
-            "not a paddle model(expected file format: 0, but %u found).",
-            version));
-  }
-  {
-    // the 2st field, LoD information
-    uint64_t lod_level = 0;
-    is.read(reinterpret_cast<char *>(&lod_level), sizeof(lod_level));
-    auto &lod = *tensor->mutable_lod();
-    lod.resize(lod_level);
-    for (uint64_t i = 0; i < lod_level; ++i) {
-      uint64_t size = 0;
-      is.read(reinterpret_cast<char *>(&size), sizeof(size));
-      std::vector<size_t> tmp(size / sizeof(size_t));
-      is.read(reinterpret_cast<char *>(tmp.data()),
-              static_cast<std::streamsize>(size));
-      lod[i] = tmp;
-    }
-  }
-  // the 3st filed, Tensor
-  paddle::framework::TensorFromStream(
-      is, static_cast<phi::DenseTensor *>(tensor), dev_ctx);
-}
-
 LoD ConvertToOffsetBasedLoD(const LoD &length_lod) {
   LoD offset_lod;
   offset_lod.reserve(length_lod.size());
diff --git a/paddle/fluid/framework/lod_tensor.h b/paddle/fluid/framework/lod_tensor.h
@@ -26,6 +26,7 @@ limitations under the License. */
 #include "paddle/fluid/platform/enforce.h"
 #include "paddle/phi/common/place.h"
 #include "paddle/phi/core/dense_tensor.h"
+#include "paddle/phi/core/framework/dense_tensor_serialize.h"
 #include "paddle/phi/core/mixed_vector.h"
 #include "paddle/utils/test_macros.h"
 
@@ -73,26 +74,5 @@ TEST_API bool CheckLoD(const LoD& in, int tensor_height = -1);
 
 TEST_API LoD ConvertToOffsetBasedLoD(const LoD& length_lod);
 
-/*
- * Serialize/Deserialize phi::DenseTensor to std::ostream
- * You can pass ofstream or ostringstream to serialize to file
- * or to a in memory string. GPU tensor will be copied to CPU.
- */
-void SerializeToStream(std::ostream& os,
-                       const phi::DenseTensor& tensor,
-                       const phi::DeviceContext& dev_ctx);
-void DeserializeFromStream(std::istream& is,
-                           phi::DenseTensor* tensor,
-                           const phi::DeviceContext& dev_ctx);
-void DeserializeFromStream(std::istream& is,
-                           phi::DenseTensor* tensor,
-                           const phi::DeviceContext& dev_ctx,
-                           const size_t& seek,
-                           const std::vector<int64_t>& shape);
-
-void SerializeToStream(std::ostream& os, const phi::DenseTensor& tensor);
-
-void DeserializeFromStream(std::istream& os, phi::DenseTensor* tensor);
-
 }  // namespace framework
 }  // namespace paddle
diff --git a/paddle/fluid/framework/selected_rows_utils.cc b/paddle/fluid/framework/selected_rows_utils.cc
@@ -14,84 +14,4 @@ limitations under the License. */
 
 #include "paddle/fluid/framework/selected_rows_utils.h"
 
-namespace paddle::framework {
-
-void SerializeToStream(std::ostream& os,
-                       const phi::SelectedRows& selected_rows,
-                       const phi::DeviceContext& dev_ctx) {
-  {  // the 1st field, uint32_t version
-    constexpr uint32_t version = 0;
-    os.write(reinterpret_cast<const char*>(&version), sizeof(version));
-  }
-  {
-    // the 2st field, rows information
-    auto& rows = selected_rows.rows();
-    uint64_t size = rows.size();
-    os.write(reinterpret_cast<const char*>(&size), sizeof(size));
-    for (uint64_t i = 0; i < size; ++i) {
-      os.write(reinterpret_cast<const char*>(&rows[i]), sizeof(rows[i]));
-    }
-  }
-  {
-    // the 3st field, the height of SelectedRows
-    int64_t height = selected_rows.height();
-    os.write(reinterpret_cast<const char*>(&height), sizeof(height));
-  }
-  // the 4st field, Tensor data
-  paddle::framework::TensorToStream(os, selected_rows.value(), dev_ctx);
-}
-
-void SerializeToStream(std::ostream& os,
-                       const phi::SelectedRows& selected_rows) {
-  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
-  const phi::DeviceContext* dev_ctx = nullptr;
-  auto place = selected_rows.place();
-  dev_ctx = pool.Get(place);
-  SerializeToStream(os, selected_rows, *dev_ctx);
-}
-
-void DeserializeFromStream(std::istream& is, phi::SelectedRows* selected_rows) {
-  phi::DeviceContextPool& pool = phi::DeviceContextPool::Instance();
-  const phi::DeviceContext* dev_ctx = nullptr;
-  dev_ctx = pool.Get(phi::CPUPlace());
-  DeserializeFromStream(is, selected_rows, *dev_ctx);
-}
-
-void DeserializeFromStream(std::istream& is,
-                           phi::SelectedRows* selected_rows,
-                           const phi::DeviceContext& dev_ctx) {
-  {
-    // the 1st field, unit32_t version for SelectedRows
-    uint32_t version = 0;
-    is.read(reinterpret_cast<char*>(&version), sizeof(version));
-    PADDLE_ENFORCE_EQ(version,
-                      0U,
-                      common::errors::InvalidArgument(
-                          "Only version 0 SelectedRows is supported."));
-  }
-  {
-    // the 2st field, rows information
-    uint64_t size = 0;
-    is.read(reinterpret_cast<char*>(&size), sizeof(size));
-    PADDLE_ENFORCE_EQ(
-        is.good(),
-        true,
-        common::errors::Unavailable("Cannot read the number of rows."));
-    auto& rows = *selected_rows->mutable_rows();
-    rows.resize(size);
-    for (uint64_t i = 0; i < size; ++i) {
-      is.read(reinterpret_cast<char*>(&rows[i]), sizeof(int64_t));
-    }
-  }
-  {
-    // the 3st field, the height of the SelectedRows
-    int64_t height = 0;
-    is.read(reinterpret_cast<char*>(&height), sizeof(int64_t));
-    selected_rows->set_height(height);
-  }
-  // the 4st field, tensor which contains the data
-  paddle::framework::TensorFromStream(
-      is, selected_rows->mutable_value(), dev_ctx);
-}
-
-}  // namespace paddle::framework
+namespace paddle::framework {}  // namespace paddle::framework
diff --git a/paddle/fluid/framework/selected_rows_utils.h b/paddle/fluid/framework/selected_rows_utils.h
@@ -22,27 +22,10 @@ limitations under the License. */
 #include <vector>
 
 #include "paddle/fluid/framework/tensor_util.h"
+#include "paddle/phi/core/framework/selected_rows_serialize.h"
 #include "paddle/phi/core/platform/device_context.h"
 #include "paddle/phi/core/selected_rows.h"
 
 namespace paddle {
-namespace framework {
-/*
- * Serialize/Deserialize SelectedRows to std::ostream
- * You can pass ofstream or ostringstream to serialize to file
- * or to a in memory string. GPU tensor will be copied to CPU.
- */
-void SerializeToStream(std::ostream& os,
-                       const phi::SelectedRows& selected_rows,
-                       const phi::DeviceContext& dev_ctx);
-void DeserializeFromStream(std::istream& is,
-                           phi::SelectedRows* selected_rows,
-                           const phi::DeviceContext& dev_ctx);
-
-void SerializeToStream(std::ostream& os,
-                       const phi::SelectedRows& selected_rows);
-
-void DeserializeFromStream(std::istream& is, phi::SelectedRows* selected_rows);
-
-}  // namespace framework
+namespace framework {}  // namespace framework
 }  // namespace paddle
diff --git a/paddle/fluid/jit/serializer.cc b/paddle/fluid/jit/serializer.cc
@@ -136,7 +136,7 @@ void Deserializer::ReadTensorData(
     Variable v;
     // TODO(dev): Support framework::Vocab
     DenseTensor* dense_tensor = v.GetMutable<DenseTensor>();
-    framework::DeserializeFromStream(fin, dense_tensor, dev_ctx);
+    phi::DeserializeFromStream(fin, dense_tensor, dev_ctx);
     (*params_dict)[item] = std::make_shared<Variable>(v);
   }
 }
diff --git a/paddle/fluid/operators/load_combine_op.h b/paddle/fluid/operators/load_combine_op.h
@@ -114,7 +114,7 @@ class LoadCombineOpKernel : public framework::OpKernel<T> {
         auto *tensor = out_vars[i]->GetMutable<phi::DenseTensor>();
 
         // Get data from fin to tensor
-        paddle::framework::DeserializeFromStream(*buffer, tensor, dev_ctx);
+        phi::DeserializeFromStream(*buffer, tensor, dev_ctx);
 
         auto in_dtype = tensor->dtype();
         auto out_dtype = load_as_fp16 ? phi::DataType::FLOAT16 : in_dtype;
diff --git a/paddle/fluid/operators/save_combine_op.h b/paddle/fluid/operators/save_combine_op.h
@@ -110,9 +110,9 @@ void SaveCombineTensorKernel(const Context& dev_ctx,
       framework::TransDataType(in_kernel_type, out_kernel_type, tensor, &out);
       // copy LoD info to the new tensor
       out.set_lod(tensor.lod());
-      framework::SerializeToStream(ss, out, dev_ctx);
+      phi::SerializeToStream(ss, out, dev_ctx);
     } else {
-      framework::SerializeToStream(ss, tensor, dev_ctx);
+      phi::SerializeToStream(ss, tensor, dev_ctx);
     }
   }
 
diff --git a/paddle/fluid/pir/serialize_deserialize/src/save_load_parameters.cc b/paddle/fluid/pir/serialize_deserialize/src/save_load_parameters.cc
@@ -91,9 +91,9 @@ void SaveFunction(const phi::DenseTensor& x,
   const phi::DeviceContext* dev_ctx = GetDeviceContext(x);
   if (in_dtype != out_dtype) {
     auto out = CastTensorType(dev_ctx, x, out_dtype);
-    paddle::framework::SerializeToStream(fout, out, *dev_ctx);
+    phi::SerializeToStream(fout, out, *dev_ctx);
   } else {
-    paddle::framework::SerializeToStream(fout, x, *dev_ctx);
+    phi::SerializeToStream(fout, x, *dev_ctx);
   }
   fout.close();
   VLOG(6) << "save func done ";
@@ -138,9 +138,9 @@ void SaveCombineFunction(const std::vector<const phi::DenseTensor*>& x,
     auto out_dtype = save_as_fp16 ? phi::DataType::FLOAT16 : in_dtype;
     if (in_dtype != out_dtype) {
       auto out = CastTensorType(dev_ctx, tensor, out_dtype);
-      paddle::framework::SerializeToStream(fout, out, *dev_ctx);
+      phi::SerializeToStream(fout, out, *dev_ctx);
     } else {
-      paddle::framework::SerializeToStream(fout, tensor, *dev_ctx);
+      phi::SerializeToStream(fout, tensor, *dev_ctx);
     }
   }
   fout.close();
@@ -170,9 +170,9 @@ void LoadFunction(const std::string& file_path,
                       0,
                       common::errors::InvalidArgument(
                           "seek with tensor must great than or equal to 0"));
-    paddle::framework::DeserializeFromStream(fin, out, *dev_ctx, seek, shape);
+    phi::DeserializeFromStream(fin, out, *dev_ctx, seek, shape);
   } else {
-    paddle::framework::DeserializeFromStream(fin, out, *dev_ctx);
+    phi::DeserializeFromStream(fin, out, *dev_ctx);
   }
 
   auto in_dtype = out->dtype();
@@ -205,7 +205,7 @@ void LoadCombineFunction(const std::string& file_path,
   const phi::DeviceContext* dev_ctx = GetDeviceContext(*(out->at(0)), place);
   for (size_t i = 0; i < names.size(); i++) {
     auto tensor = out->at(i);
-    paddle::framework::DeserializeFromStream(fin, tensor, *dev_ctx);
+    phi::DeserializeFromStream(fin, tensor, *dev_ctx);
 
     auto in_dtype = tensor->dtype();
     auto out_dtype = load_as_fp16 ? phi::DataType::FLOAT16 : in_dtype;
diff --git a/paddle/fluid/pybind/io.cc b/paddle/fluid/pybind/io.cc
diff --git a/paddle/phi/core/framework/data_type_transform.cc b/paddle/phi/core/framework/data_type_transform.cc
diff --git a/paddle/phi/core/framework/var_type_helper.cc b/paddle/phi/core/framework/var_type_helper.cc
diff --git a/paddle/phi/core/framework/var_type_helper.h b/paddle/phi/core/framework/var_type_helper.h
diff --git a/test/cpp/fluid/framework/selected_rows_utils_test.cc b/test/cpp/fluid/framework/selected_rows_utils_test.cc
diff --git a/test/cpp/fluid/framework/tensor_util_test.cc b/test/cpp/fluid/framework/tensor_util_test.cc

Original file line number	Diff line number	Diff line change
`@@ -86,7 +86,8 @@ void tensor_check(const std::string& op_type,`
`86`	`86`	`const phi::DenseTensor& tensor,`
`87`	`87`	`const phi::Place& place) {`
`88`	`88`	`TensorCheckerVisitor<Context> vistor(op_type, var_name, tensor, place);`
`89`		`- VisitDataType(framework::TransToProtoVarType(tensor.dtype()), vistor);`
	`89`	`+ framework::VisitDataType(framework::TransToProtoVarType(tensor.dtype()),`
	`90`	`+ vistor);`
`90`	`91`	`}`
`91`	`92`
`92`	`93`	`void InitWhiteListFormEnv();`
Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ void SaveTensor(const phi::DenseTensor& x,`
`34`	`34`	`true,`
`35`	`35`	`common::errors::Unavailable(`
`36`	`36`	`"Cannot open %s to save variables.", new_path));`
`37`		`- framework::SerializeToStream(fout, x);`
	`37`	`+ phi::SerializeToStream(fout, x);`
`38`	`38`
`39`	`39`	`fout.close();`
`40`	`40`	`}`
`@@ -51,6 +51,6 @@ void LoadTensor(const std::string& file_path, phi::DenseTensor* out) {`
`51`	`51`	`common::errors::InvalidArgument(`
`52`	`52`	`"The variable to be loaded cannot be found."));`
`53`	`53`
`54`		`- framework::DeserializeFromStream(fin, out);`
	`54`	`+ phi::DeserializeFromStream(fin, out);`
`55`	`55`	`}`
`56`	`56`	`} // namespace paddle::framework`
Original file line number	Diff line number	Diff line change
`@@ -136,7 +136,7 @@ void Deserializer::ReadTensorData(`
`136`	`136`	`Variable v;`
`137`	`137`	`// TODO(dev): Support framework::Vocab`
`138`	`138`	`DenseTensor* dense_tensor = v.GetMutable<DenseTensor>();`
`139`		`- framework::DeserializeFromStream(fin, dense_tensor, dev_ctx);`
	`139`	`+ phi::DeserializeFromStream(fin, dense_tensor, dev_ctx);`
`140`	`140`	`(*params_dict)[item] = std::make_shared<Variable>(v);`
`141`	`141`	`}`
`142`	`142`	`}`