Replace paddle::framework::Strings [fluid_ops] (#68719)

co63oc · web-flow · commit 439ceec5d957 · 2024-10-16T11:52:32.000+08:00
diff --git a/paddle/fluid/eager/eager_tensor.h b/paddle/fluid/eager/eager_tensor.h
@@ -212,12 +212,12 @@ class EagerVariable final {
         ConstructVariableFromTensor<phi::SelectedRows>(tensor);
       } else if (IsVariableCompatTensor(tensor) &&
                  static_cast<const VariableCompatTensor*>(tensor.impl().get())
-                     ->IsType<paddle::framework::Vocab>()) {
-        ConstructVariableFromCompatTensor<paddle::framework::Vocab>(tensor);
+                     ->IsType<phi::Vocab>()) {
+        ConstructVariableFromCompatTensor<phi::Vocab>(tensor);
       } else if (IsVariableCompatTensor(tensor) &&
                  static_cast<const VariableCompatTensor*>(tensor.impl().get())
-                     ->IsType<paddle::framework::Strings>()) {
-        ConstructVariableFromCompatTensor<paddle::framework::Strings>(tensor);
+                     ->IsType<phi::Strings>()) {
+        ConstructVariableFromCompatTensor<phi::Strings>(tensor);
       } else {
         PADDLE_THROW(common::errors::Fatal(
             "Unrecognized egr::EagerVariable type, only "
diff --git a/paddle/fluid/framework/new_executor/interpreter/static_build.cc b/paddle/fluid/framework/new_executor/interpreter/static_build.cc
@@ -187,8 +187,7 @@ bool BlockCanBeStaticBuilt(const framework::BlockDesc& block) {
 
 inline bool IsExtendedTensor(const phi::TensorBase& tensor) {
   return framework::RawTensor::classof(&tensor) ||
-         framework::Strings::classof(&tensor) ||
-         framework::Vocab::classof(&tensor);
+         phi::Strings::classof(&tensor) || phi::Vocab::classof(&tensor);
 }
 
 bool TensorShouldBeFakeInitialized(const OperatorBase& op,
@@ -280,8 +279,8 @@ phi::TensorBase* GetTensorFormVar(framework::Variable* var) {
       return var->template GetMutable<phi::SparseCooTensor>();
     } else if (var->template IsType<phi::TensorArray>()) {
       return var->template GetMutable<phi::TensorArray>();
-    } else if (var->template IsType<framework::Strings>()) {
-      return var->template GetMutable<framework::Strings>();
+    } else if (var->template IsType<phi::Strings>()) {
+      return var->template GetMutable<phi::Strings>();
     } else if (var->template IsType<paddle::framework::RawTensor>() ||
                !var->IsInitialized()) {
       return var->template GetMutable<paddle::framework::RawTensor>();
diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc
@@ -3254,11 +3254,11 @@ void OperatorWithKernel::BuildPhiKernelContext(
         need_prepare_phi_data_ = true;
         tensor_in = &(var->Get<phi::TensorArray>());
         phi_kernel_context->EmplaceBackInputWithoutSetRange(tensor_in);
-      } else if (var->IsType<framework::Vocab>()) {
-        tensor_in = &(var->Get<framework::Vocab>());
+      } else if (var->IsType<phi::Vocab>()) {
+        tensor_in = &(var->Get<phi::Vocab>());
         phi_kernel_context->EmplaceBackInputWithoutSetRange(tensor_in);
-      } else if (var->IsType<framework::Strings>()) {
-        tensor_in = &(var->Get<framework::Strings>());
+      } else if (var->IsType<phi::Strings>()) {
+        tensor_in = &(var->Get<phi::Strings>());
         phi_kernel_context->EmplaceBackInputWithoutSetRange(tensor_in);
       } else if (var->IsType<framework::FeedList>()) {
         tensor_in = &(var->Get<framework::FeedList>());
@@ -3311,8 +3311,8 @@ void OperatorWithKernel::BuildPhiKernelContext(
           // Note: If the input phi::TensorArray size is 0, the output
           // phi::TensorArray is also 0
           phi_kernel_context->EmplaceBackOutputWithoutSetRange(tensor_out);
-        } else if (var->template IsType<framework::Strings>()) {
-          tensor_out = var->template GetMutable<framework::Strings>();
+        } else if (var->template IsType<phi::Strings>()) {
+          tensor_out = var->template GetMutable<phi::Strings>();
           phi_kernel_context->EmplaceBackOutputWithoutSetRange(tensor_out);
         } else if (var->template IsType<paddle::framework::RawTensor>() ||
                    !var->IsInitialized()) {
diff --git a/paddle/fluid/imperative/var_helper.cc b/paddle/fluid/imperative/var_helper.cc
@@ -60,9 +60,9 @@ void InitializeVariable(paddle::framework::Variable *var,
   } else if (var_type == paddle::framework::proto::VarType::LOD_TENSOR_ARRAY) {
     var->GetMutable<phi::TensorArray>();
   } else if (var_type == paddle::framework::proto::VarType::STRINGS) {
-    var->GetMutable<paddle::framework::Strings>();
+    var->GetMutable<phi::Strings>();
   } else if (var_type == paddle::framework::proto::VarType::VOCAB) {
-    var->GetMutable<paddle::framework::Vocab>();
+    var->GetMutable<phi::Vocab>();
   } else if (var_type == paddle::framework::proto::VarType::PLACE_LIST) {
     var->GetMutable<phi::PlaceList>();
   } else if (var_type == paddle::framework::proto::VarType::READER) {
diff --git a/paddle/fluid/imperative/variable_wrapper.h b/paddle/fluid/imperative/variable_wrapper.h
@@ -157,8 +157,8 @@ class VariableWrapper {
       } else if (type_ == framework::proto::VarType::SELECTED_ROWS) {
         tensor = &(var_.Get<phi::SelectedRows>().value());
       } else if (type_ == framework::proto::VarType::VOCAB) {
-        const framework::Vocab* data = nullptr;
-        data = &(var_.Get<framework::Vocab>());
+        const phi::Vocab* data = nullptr;
+        data = &(var_.Get<phi::Vocab>());
         if (data && data->size() != 0) {
           VLOG(6) << "The tensor of variable " << name_
                   << " is not initialized";
diff --git a/paddle/fluid/inference/api/details/zero_copy_tensor.cc b/paddle/fluid/inference/api/details/zero_copy_tensor.cc
@@ -79,8 +79,7 @@ void Tensor::ReshapeStrings(const size_t &shape) {
       var,
       common::errors::PreconditionNotMet(
           "No tensor called [%s] in the runtime scope", name_));
-  paddle::framework::Strings *tensor =
-      var->GetMutable<paddle::framework::Strings>();
+  phi::Strings *tensor = var->GetMutable<phi::Strings>();
   tensor->resize(shape);
 }
 
@@ -368,7 +367,7 @@ void Tensor::ShareExternalData(const T *data,
 }
 
 void Tensor::CopyStringsFromCpu(const paddle_infer::Strings *data) {
-  EAGER_GET_TENSOR(paddle::framework::Strings);
+  EAGER_GET_TENSOR(phi::Strings);
   PADDLE_ENFORCE_GE(tensor->size(),
                     0,
                     common::errors::PreconditionNotMet(
diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc
@@ -2246,7 +2246,7 @@ static PyObject* tensor_method_set_vocab(TensorObject* self,
                                          PyObject* args,
                                          PyObject* kwargs) {
   EAGER_TRY
-  using Vocab = paddle::framework::Vocab;
+  using Vocab = phi::Vocab;
   auto vocab = CastPyArg2Vocab(PyTuple_GET_ITEM(args, 0), 0);
   auto var_tensor = std::make_shared<egr::VariableCompatTensor>();
   *var_tensor->GetMutable<Vocab>() = vocab;
@@ -2259,7 +2259,7 @@ static PyObject* tensor_method_set_string_list(TensorObject* self,
                                                PyObject* args,
                                                PyObject* kwargs) {
   EAGER_TRY
-  using Strings = paddle::framework::Strings;
+  using Strings = phi::Strings;
   auto strings = CastPyArg2VectorOfString(PyTuple_GET_ITEM(args, 0), 0);
   auto var_tensor = std::make_shared<egr::VariableCompatTensor>();
   *var_tensor->GetMutable<Strings>() = strings;
@@ -2277,7 +2277,7 @@ static PyObject* tensor_method_get_map_tensor(TensorObject* self,
       true,
       common::errors::Fatal(
           "this method is only effective for VariableCompatTensor"));
-  using Vocab = paddle::framework::Vocab;
+  using Vocab = phi::Vocab;
   auto* var_tensor =
       static_cast<const egr::VariableCompatTensor*>(self->tensor.impl().get());
   return ToPyObject(var_tensor->Get<Vocab>());
diff --git a/paddle/fluid/pybind/eager_properties.cc b/paddle/fluid/pybind/eager_properties.cc
@@ -581,12 +581,12 @@ PyObject* tensor_properties_get_shape(TensorObject* self, void* closure) {
   if (egr::IsVariableCompatTensor(self->tensor)) {
     auto* var_tensor = static_cast<const egr::VariableCompatTensor*>(
         self->tensor.impl().get());
-    if (var_tensor->IsType<paddle::framework::Vocab>()) {
-      value.emplace_back(static_cast<int64_t>(
-          var_tensor->Get<paddle::framework::Vocab>().size()));
-    } else if (var_tensor->IsType<paddle::framework::Strings>()) {
-      value.emplace_back(static_cast<int64_t>(
-          var_tensor->Get<paddle::framework::Strings>().size()));
+    if (var_tensor->IsType<phi::Vocab>()) {
+      value.emplace_back(
+          static_cast<int64_t>(var_tensor->Get<phi::Vocab>().size()));
+    } else if (var_tensor->IsType<phi::Strings>()) {
+      value.emplace_back(
+          static_cast<int64_t>(var_tensor->Get<phi::Strings>().size()));
     } else {
       PADDLE_THROW(common::errors::Unavailable(
           "VariableCompatTensor only support get shape from Vocab or "
@@ -856,9 +856,9 @@ PyObject* tensor_properties_get_dtype(TensorObject* self, void* closure) {
     if (egr::IsVariableCompatTensor(self->tensor)) {
       auto* var_tensor = static_cast<const egr::VariableCompatTensor*>(
           self->tensor.impl().get());
-      if (var_tensor->IsType<paddle::framework::Vocab>()) {
+      if (var_tensor->IsType<phi::Vocab>()) {
         return ToPyObject(phi::DataType::UNDEFINED);
-      } else if (var_tensor->IsType<paddle::framework::Strings>()) {
+      } else if (var_tensor->IsType<phi::Strings>()) {
         return ToPyObject(phi::DataType::PSTRING);
       } else {
         PADDLE_THROW(common::errors::Unavailable(
@@ -876,9 +876,9 @@ PyObject* tensor_properties_get_dtype(TensorObject* self, void* closure) {
     if (egr::IsVariableCompatTensor(self->tensor)) {
       auto* var_tensor = static_cast<const egr::VariableCompatTensor*>(
           self->tensor.impl().get());
-      if (var_tensor->IsType<paddle::framework::Vocab>()) {
+      if (var_tensor->IsType<phi::Vocab>()) {
         return ToPyObject(framework::proto::VarType::RAW);
-      } else if (var_tensor->IsType<paddle::framework::Strings>()) {
+      } else if (var_tensor->IsType<phi::Strings>()) {
         return ToPyObject(framework::proto::VarType::STRING);
       } else {
         PADDLE_THROW(common::errors::Unavailable(
diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc
@@ -884,9 +884,9 @@ paddle::DataType CastPyArg2DataTypeDirectly(PyObject* obj,
   return dtype;
 }
 
-paddle::framework::Vocab CastPyArg2Vocab(PyObject* obj, ssize_t arg_pos) {
+phi::Vocab CastPyArg2Vocab(PyObject* obj, ssize_t arg_pos) {
   if (PyDict_Check(obj)) {
-    paddle::framework::Vocab vocab;
+    phi::Vocab vocab;
     vocab = ::pybind11::handle(obj)
                 .cast<std::unordered_map<std::wstring, std::int32_t>>();
     return vocab;
@@ -1260,7 +1260,7 @@ PyObject* ToPyObject(
   return dict;
 }
 
-PyObject* ToPyObject(const paddle::framework::Vocab& value) {
+PyObject* ToPyObject(const phi::Vocab& value) {
   PyObject* dict = PyDict_New();
   for (const auto& map_iter : value) {
     // Convert Key
diff --git a/paddle/fluid/pybind/eager_utils.h b/paddle/fluid/pybind/eager_utils.h
@@ -112,7 +112,7 @@ std::vector<std::vector<size_t>> CastPyArg2VectorOfVectorOfSize_t(
     PyObject* obj, size_t arg_pos);
 framework::proto::VarType::Type CastPyArg2ProtoType(PyObject* obj,
                                                     ssize_t arg_pos);
-paddle::framework::Vocab CastPyArg2Vocab(PyObject* obj, ssize_t arg_pos);
+phi::Vocab CastPyArg2Vocab(PyObject* obj, ssize_t arg_pos);
 std::vector<std::string> CastPyArg2VectorOfString(PyObject* obj,
                                                   ssize_t arg_pos);
 std::shared_ptr<jit::Function> CastPyArg2JitFunction(PyObject* obj,
@@ -158,7 +158,7 @@ PyObject* ToPyObject(const void* value);
 PyObject* ToPyObject(const std::unordered_map<int, int>& value);
 PyObject* ToPyObject(
     const std::unordered_map<std::string, std::vector<std::string>>& value);
-PyObject* ToPyObject(const paddle::framework::Vocab& value);
+PyObject* ToPyObject(const phi::Vocab& value);
 
 PyObject* ToPyObject(std::shared_ptr<egr::GradNodeBase> grad_node);
 PyObject* ToPyObject(const pir::Value& value);
diff --git a/paddle/phi/kernels/cpu/faster_tokenizer_kernel.cc b/paddle/phi/kernels/cpu/faster_tokenizer_kernel.cc
@@ -36,7 +36,6 @@ using std::unordered_set;
 using std::vector;
 using std::wcout;
 using std::wstring;
-using Strings = paddle::framework::Strings;
 
 inline bool IsControl(const wchar_t& ch);
 inline bool IsChineseChar(const wchar_t& ch);
@@ -57,21 +56,21 @@ class BasicTokenizer {
 
 class WordPieceTokenizer {
  public:
-  explicit WordPieceTokenizer(const paddle::framework::Vocab* vocab,
+  explicit WordPieceTokenizer(const phi::Vocab* vocab,
                               const wstring& unk_token = L"[UNK]",
                               const size_t max_input_chars_per_word = 100);
   void Tokenize(const wstring& text, vector<int64_t>* output) const;
 
  private:
-  const paddle::framework::Vocab* vocab_;
+  const phi::Vocab* vocab_;
   wstring unk_token_{L"[UNK]"};
   int64_t unk_token_id_;
   size_t max_input_chars_per_word_;
 };
 
 class BertTokenizer {
  public:
-  explicit BertTokenizer(const paddle::framework::Vocab* vocab,
+  explicit BertTokenizer(const phi::Vocab* vocab,
                          bool do_lower_case = false,
                          const wstring& unk_token = L"[UNK]",
                          const wstring& pad_token = L"[PAD]",
@@ -114,7 +113,7 @@ class BertTokenizer {
   bool do_lower_case_;
   wstring unk_token_, pad_token_, cls_token_, mask_token_, sep_token_;
   string padding_site_;
-  const paddle::framework::Vocab* vocab_;
+  const phi::Vocab* vocab_;
   BasicTokenizer basic_tokenizer_;
   WordPieceTokenizer word_piece_tokenizer_;
   int64_t unk_token_id_, cls_token_id_, mask_token_id_, pad_token_id_,
@@ -204,7 +203,7 @@ void BasicTokenizer::Tokenize(const string& text, vector<wstring>* res) const {
 }
 
 WordPieceTokenizer::WordPieceTokenizer(
-    const paddle::framework::Vocab* vocab,
+    const phi::Vocab* vocab,
     const wstring& unk_token /* = L"[UNK]"*/,
     const size_t max_input_chars_per_word /* = 100 */)
     : vocab_(vocab),
@@ -260,7 +259,7 @@ void WordPieceTokenizer::Tokenize(const wstring& text,
   }
 }
 
-BertTokenizer::BertTokenizer(const paddle::framework::Vocab* vocab,
+BertTokenizer::BertTokenizer(const phi::Vocab* vocab,
                              bool do_lower_case /* = false */,
                              const wstring& unk_token /* = L"[UNK]" */,
                              const wstring& pad_token /* = L"[PAD]" */,
@@ -542,8 +541,7 @@ void FasterTokenizerKernel(const Context& dev_ctx,
                            bool pad_to_max_seq_len,
                            DenseTensor* input_ids,
                            DenseTensor* segment_ids) {
-  const auto* vocab =
-      reinterpret_cast<const paddle::framework::Vocab*>(&vocab_in);
+  const auto* vocab = reinterpret_cast<const phi::Vocab*>(&vocab_in);
   const auto* text = reinterpret_cast<const Strings*>(&text_in);
   const auto* text_pair =
       reinterpret_cast<const Strings*>(text_pair_in.get_ptr());
diff --git a/test/cpp/eager/data_structure_tests/eager_tensor_test.cc b/test/cpp/eager/data_structure_tests/eager_tensor_test.cc
@@ -356,20 +356,20 @@ TEST(EagerVariable, DataLayout) {
 TEST(VariableCompatTensor, MemberFunction) {
   egr::VariableCompatTensor var_tensor;
   // test GetMutable and Get
-  var_tensor.GetMutable<paddle::framework::Vocab>();
-  auto& vocab = var_tensor.Get<paddle::framework::Vocab>();
+  var_tensor.GetMutable<phi::Vocab>();
+  auto& vocab = var_tensor.Get<phi::Vocab>();
   EXPECT_EQ(vocab.size(), 0UL);
   bool caught_exception = false;
   try {
-    var_tensor.GetMutable<paddle::framework::Strings>();
+    var_tensor.GetMutable<phi::Strings>();
   } catch (paddle::platform::EnforceNotMet& error) {
     caught_exception = true;
     std::string ex_msg = error.what();
     EXPECT_TRUE(ex_msg.find("The Variable type must be") != std::string::npos);
   }
   EXPECT_TRUE(caught_exception);
   // test Type and IsType
-  EXPECT_TRUE(var_tensor.IsType<paddle::framework::Vocab>());
+  EXPECT_TRUE(var_tensor.IsType<phi::Vocab>());
   EXPECT_EQ(var_tensor.Type(),
             static_cast<int>(paddle::framework::proto::VarType::VOCAB));
   // test valid and initialized
