Skip to content

Commit

Permalink
resolve conflit with yunfei
Browse files Browse the repository at this point in the history
  • Loading branch information
@chenwhql
chenwhql committed Oct 20, 2021
2 parents beec280 + 466ce03 commit a49fd44
Show file tree
Hide file tree
Showing 132 changed files with 788 additions and 796 deletions.
22 changes: 11 additions & 11 deletions cmake/generic.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,19 +116,19 @@ function(find_fluid_modules TARGET_NAME)
endif()
endfunction(find_fluid_modules)

set_property(GLOBAL PROPERTY TCMPT_MODULES "")
# find all tcmpt modules is used for paddle static library
set_property(GLOBAL PROPERTY PTEN_MODULES "")
# find all pten modules is used for paddle static library
# for building inference libs
function(find_tcmpt_modules TARGET_NAME)
function(find_pten_modules TARGET_NAME)
get_filename_component(__target_path ${TARGET_NAME} ABSOLUTE)
string(REGEX REPLACE "^${PADDLE_SOURCE_DIR}/" "" __target_path ${__target_path})
string(FIND "${__target_path}" "tcmpt" pos)
string(FIND "${__target_path}" "pten" pos)
if(pos GREATER 1)
get_property(tcmpt_modules GLOBAL PROPERTY TCMPT_MODULES)
set(tcmpt_modules ${tcmpt_modules} ${TARGET_NAME})
set_property(GLOBAL PROPERTY TCMPT_MODULES "${tcmpt_modules}")
get_property(pten_modules GLOBAL PROPERTY PTEN_MODULES)
set(pten_modules ${pten_modules} ${TARGET_NAME})
set_property(GLOBAL PROPERTY PTEN_MODULES "${pten_modules}")
endif()
endfunction(find_tcmpt_modules)
endfunction(find_pten_modules)

function(common_link TARGET_NAME)
if (WITH_PROFILER)
Expand Down Expand Up @@ -324,7 +324,7 @@ function(cc_library TARGET_NAME)
else()
add_library(${TARGET_NAME} STATIC ${cc_library_SRCS})
find_fluid_modules(${TARGET_NAME})
find_tcmpt_modules(${TARGET_NAME})
find_pten_modules(${TARGET_NAME})
endif()
if(cc_library_DEPS)
# Don't need link libwarpctc.so
Expand Down Expand Up @@ -497,7 +497,7 @@ function(nv_library TARGET_NAME)
else()
add_library(${TARGET_NAME} STATIC ${nv_library_SRCS})
find_fluid_modules(${TARGET_NAME})
find_tcmpt_modules(${TARGET_NAME})
find_pten_modules(${TARGET_NAME})
endif()
if (nv_library_DEPS)
add_dependencies(${TARGET_NAME} ${nv_library_DEPS})
Expand Down Expand Up @@ -588,7 +588,7 @@ function(hip_library TARGET_NAME)
else()
hip_add_library(${TARGET_NAME} STATIC ${hip_library_SRCS})
find_fluid_modules(${TARGET_NAME})
find_tcmpt_modules(${TARGET_NAME})
find_pten_modules(${TARGET_NAME})
endif()
if (hip_library_DEPS)
add_dependencies(${TARGET_NAME} ${hip_library_DEPS})
Expand Down
10 changes: 5 additions & 5 deletions cmake/tcmpt.cmake → cmake/pten.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,13 @@ function(kernel_instantiate TARGET)
string(REGEX MATCH "[A-Z][A-Za-z0-9]+\\(" func_name ${signature})
string(REPLACE "(" "" func_name ${func_name})
# message(STATUS "FUNC NAME: ${func_name}")
string(REGEX REPLACE "${func_name}" "pt::${func_name}<${dtype}>" inst_signature ${signature})
string(REGEX REPLACE "${func_name}" "pten::${func_name}<${dtype}>" inst_signature ${signature})
# append namespace
string(REPLACE "CPUContext" "pt::CPUContext" inst_signature ${inst_signature})
string(REPLACE "CUDAContext" "pt::CUDAContext" inst_signature ${inst_signature})
string(REPLACE "DenseTensor" "pt::DenseTensor" inst_signature ${inst_signature})
string(REPLACE "CPUContext" "pten::CPUContext" inst_signature ${inst_signature})
string(REPLACE "CUDAContext" "pten::CUDAContext" inst_signature ${inst_signature})
string(REPLACE "DenseTensor" "pten::DenseTensor" inst_signature ${inst_signature})
# TODO(chenweihang): adapt SelectedRows after adding it
# string(REPLACE "SelectedRowsTensor" "pt::SelectedRowsTensor" inst_signature ${inst_signature})
# string(REPLACE "SelectedRowsTensor" "pten::SelectedRowsTensor" inst_signature ${inst_signature})
# message(STATUS "INST FUNC: ${inst_signature}")
string(APPEND instantiate_context "template ${inst_signature};\n")
endforeach()
Expand Down
2 changes: 1 addition & 1 deletion paddle/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
add_subdirectory(scripts)
add_subdirectory(testing)
set(PYTHON_TESTS_DIR ${PADDLE_BINARY_DIR}/python/paddle/fluid/tests CACHE INTERNAL "python tests directory")
add_subdirectory(tcmpt)
add_subdirectory(pten)
add_subdirectory(fluid)
8 changes: 4 additions & 4 deletions paddle/fluid/framework/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,10 +193,10 @@ cc_library(unused_var_check SRCS unused_var_check.cc DEPS glog no_need_buffer_va

IF(WITH_XPU)
cc_library(operator SRCS operator.cc DEPS xpu_op_list op_info device_context tensor scope glog trainer_desc_proto data_feed_proto
shape_inference data_transform lod_tensor profiler transfer_scope_cache op_kernel_type op_call_stack unused_var_check nan_inf_utils tcmpt tcmpt_utils)
shape_inference data_transform lod_tensor profiler transfer_scope_cache op_kernel_type op_call_stack unused_var_check nan_inf_utils pten pten_utils)
ELSE()
cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope glog trainer_desc_proto data_feed_proto
shape_inference data_transform lod_tensor profiler transfer_scope_cache op_kernel_type op_call_stack unused_var_check nan_inf_utils tcmpt tcmpt_utils)
shape_inference data_transform lod_tensor profiler transfer_scope_cache op_kernel_type op_call_stack unused_var_check nan_inf_utils pten pten_utils)
ENDIF()

cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry device_context)
Expand Down Expand Up @@ -390,7 +390,7 @@ cc_library(save_load_util SRCS save_load_util.cc DEPS tensor scope layer)
cc_test(save_load_util_test SRCS save_load_util_test.cc DEPS save_load_util tensor scope layer)
cc_library(generator SRCS generator.cc DEPS enforce place)

cc_library(tcmpt_utils SRCS tcmpt_utils.cc DEPS lod_tensor selected_rows place tcmpt var_type_traits)
cc_library(pten_utils SRCS pten_utils.cc DEPS lod_tensor selected_rows place pten var_type_traits)

# Get the current working branch
execute_process(
Expand Down Expand Up @@ -454,4 +454,4 @@ if(WITH_TESTING AND TEST selected_rows_test)
endif()

cc_test(scope_guard_test SRCS scope_guard_test.cc)
cc_test(tcmpt_utils_test SRCS tcmpt_utils_test.cc DEPS tcmpt_utils)
cc_test(pten_utils_test SRCS pten_utils_test.cc DEPS pten_utils)
24 changes: 13 additions & 11 deletions paddle/fluid/framework/operator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ limitations under the License. */
#include "paddle/fluid/framework/data_type_transform.h"
#include "paddle/fluid/framework/details/nan_inf_utils.h"
#include "paddle/fluid/framework/op_call_stack.h"
#include "paddle/fluid/framework/pten_utils.h"
#include "paddle/fluid/framework/shape_inference.h"
#include "paddle/fluid/framework/tcmpt_utils.h"
#include "paddle/fluid/framework/transfer_scope_cache.h"
#include "paddle/fluid/framework/unused_var_check.h"
#include "paddle/fluid/framework/var_type.h"
Expand Down Expand Up @@ -1140,7 +1140,7 @@ void OperatorWithKernel::RunImpl(const Scope& scope,
// and RCOM backend, the XPU, NPU and MKLDNN will be supported in the second
// phase
if (FLAGS_run_pt_kernel &&
pt::KernelFactory::Instance().ContainsKernel(type_.c_str())) {
pten::KernelFactory::Instance().ContainsKernel(type_.c_str())) {
if (pt_kernel_signature_.get() == nullptr || pt_kernel_.get() == nullptr) {
ChoosePtKernel(exe_ctx);
}
Expand Down Expand Up @@ -1286,10 +1286,11 @@ void OperatorWithKernel::ChoosePtKernel(const ExecutionContext& ctx) const {

kernel_type_.reset(new OpKernelType(InnerGetExpectedKernelType(ctx)));

auto pt_kernel_name = pt::KernelName(pt_kernel_signature_->first);
auto pt_kernel_name = pten::KernelName(pt_kernel_signature_->first);
auto pt_kernel_key = TransOpKernelTypeToPtKernelKey(*kernel_type_.get());
pt_kernel_.reset(new pt::Kernel(pt::KernelFactory::Instance().SelectKernel(
pt_kernel_name, pt_kernel_key)));
pt_kernel_.reset(
new pten::Kernel(pten::KernelFactory::Instance().SelectKernel(
pt_kernel_name, pt_kernel_key)));

if (pt_kernel_->IsValid()) {
VLOG(1) << "Static mode ChoosePtKernel - kernel name: " << pt_kernel_name
Expand Down Expand Up @@ -1781,7 +1782,7 @@ KernelSignature OperatorWithKernel::GetExpectedPtKernelArgs(
}
}

pt::KernelContext OperatorWithKernel::BuildPtKernelContext(
pten::KernelContext OperatorWithKernel::BuildPtKernelContext(
const RuntimeContext& ctx, const platform::DeviceContext& dev_ctx) const {
VLOG(1) << RuntimeContextDebugString(ctx);

Expand All @@ -1792,7 +1793,7 @@ pt::KernelContext OperatorWithKernel::BuildPtKernelContext(
// 3. needless attributes remove
// 4. use pt Tensor directly
// 5. kernel input is not DenseTensor
pt::KernelContext op_kernel_ctx(dev_ctx);
pten::KernelContext op_kernel_ctx(dev_ctx);

auto& input_names = std::get<0>(pt_kernel_signature_->second);
auto& attr_names = std::get<1>(pt_kernel_signature_->second);
Expand Down Expand Up @@ -1826,7 +1827,7 @@ pt::KernelContext OperatorWithKernel::BuildPtKernelContext(
<< in_def.layout;

auto ins_vector = ctx.inputs.at(input_names[i]);
std::vector<std::shared_ptr<tcmpt::TensorBase>> tmp_inputs;
std::vector<std::shared_ptr<pten::TensorBase>> tmp_inputs;

for (auto var : ins_vector) {
auto pt_in = framework::InputVariableToPtTensor(*var, in_def);
Expand All @@ -1839,7 +1840,7 @@ pt::KernelContext OperatorWithKernel::BuildPtKernelContext(
auto out_def = output_defs.at(i);
auto outs_vector = ctx.outputs.at(output_names[i]);

std::vector<std::shared_ptr<tcmpt::TensorBase>> tmp_outputs;
std::vector<std::shared_ptr<pten::TensorBase>> tmp_outputs;
for (auto var : outs_vector) {
auto pt_out = framework::OutputVariableToPtTensor(var, out_def);
tmp_outputs.emplace_back(pt_out);
Expand All @@ -1849,12 +1850,13 @@ pt::KernelContext OperatorWithKernel::BuildPtKernelContext(

for (size_t i = 0; i < attr_names.size(); ++i) {
auto& attr = Attrs().at(attr_names[i]);
if (attr_defs[i].type_index == std::type_index(typeid(pt::Scalar))) {
if (attr_defs[i].type_index == std::type_index(typeid(pten::Scalar))) {
// TODO(chenweihang): support other attrs later
// TODO(zhangyunfei): Scalar should hold scaler type, and we should check
// attribtue type by attr_defs
if (std::type_index(attr.type()) == std::type_index(typeid(float))) {
op_kernel_ctx.EmplaceBackAttr(pt::Scalar(BOOST_GET_CONST(float, attr)));
op_kernel_ctx.EmplaceBackAttr(
pten::Scalar(BOOST_GET_CONST(float, attr)));
} else {
PADDLE_THROW(platform::errors::Unimplemented(
"unsupported cast op attribute `%s` to Scalar when construct "
Expand Down
12 changes: 6 additions & 6 deletions paddle/fluid/framework/operator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ limitations under the License. */
#include "paddle/fluid/platform/variant.h"
#include "paddle/utils/flat_hash_map.h"

#include "paddle/tcmpt/api/include/core.h"
#include "paddle/pten/api/include/core.h"

namespace paddle {
namespace framework {
Expand Down Expand Up @@ -531,7 +531,7 @@ class OperatorWithKernel : public OperatorBase {
return kernel_type_->place_;
}

/* member functions for adapting to tcmpt lib */
/* member functions for adapting to pten lib */
/** In the Tensor calculation library, the new Kernel adopts a clearer and
* more streamlined design. The arguments of the Kernel and the input and
* output arguments registered in the original OpMaker do not match in some
Expand Down Expand Up @@ -582,10 +582,10 @@ class OperatorWithKernel : public OperatorBase {
Tensor* GetTensorFormInputSafely(const ExecutionContext& ctx,
const std::string& name) const;

/* member functions for adapting to tcmpt lib */
/* member functions for adapting to pten lib */
void ChoosePtKernel(const ExecutionContext& ctx) const;

pt::KernelContext BuildPtKernelContext(
pten::KernelContext BuildPtKernelContext(
const RuntimeContext& ctx, const platform::DeviceContext& dev_ctx) const;

protected:
Expand All @@ -599,11 +599,11 @@ class OperatorWithKernel : public OperatorBase {
mutable std::mutex cache_update_mutex_;
mutable bool enable_cache_transfer_scope_ = false;
// NOTE(chenweihang): Similar op members are used to adapt to
// new tcmpt kernel, if there is a better design in the future,
// new pten kernel, if there is a better design in the future,
// we may polish the implementation here
mutable bool run_pt_kernel_ = false;
mutable std::unique_ptr<KernelSignature> pt_kernel_signature_;
mutable std::unique_ptr<pt::Kernel> pt_kernel_;
mutable std::unique_ptr<pten::Kernel> pt_kernel_;
};

extern bool OpSupportGPU(const std::string& op_type);
Expand Down
Loading

1 comment on commit a49fd44

@paddle-bot-old
Copy link

@paddle-bot-old paddle-bot-old bot commented on a49fd44 Oct 20, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🕵️ CI failures summary

🔍 PR: #34425 Commit ID: a49fd44 contains failed CI.

🔹 Failed: PR-CI-APPROVAL

approve_failed
2021-10-20 13:59:14 正在保存至: “bk.txt”

2021-10-20 13:59:14      0K                                                       100% 2.26M=0s

2021-10-20 13:59:14 2021-10-20 13:59:14 (2.26 MB/s) - 已保存 “bk.txt” [5/5])

2021-10-20 13:59:22 ****************

2021-10-20 13:59:22 0. You must have one RD (lanxianghit (Recommend), phlrain or luotao1) approval for changing the FLAGS, which manages the environment variables.

2021-10-20 13:59:22 1. You must have Dianhai approval for change 20+ files or add than 1000+ lines of content.

2021-10-20 13:59:22 2. You must have one RD (XiaoguangHu01,chenwhql,zhiqiu,Xreki,luotao1) approval for paddle/fluid/framework/operator.h, which manages the underlying code for fluid.

2021-10-20 13:59:22 3. You must have one RD (zhiqiu (Recommend) , phlrain) approval for the changes of paddle/fluid/pybind/op_function_generator.cc, which manages the logic of automatic generating op functions for dygraph.

2021-10-20 13:59:22 4. You must have one RD (XiaoguangHu01,chenwhql,zhiqiu,Xreki,luotao1) approval for the usage of const_cast.

2021-10-20 13:59:22 5. You must have one RD (Avin0323(Recommend) or zhouwei25 or wanghuancoder or luotao1) approval for modifying unity_build_rule.cmake which the rules of Unity Build.

2021-10-20 13:59:22 There are 6 approved errors.

2021-10-20 13:59:22 ****************

2021-10-20 13:59:22 + EXCODE=6

2021-10-20 13:59:22 + echo 'EXCODE: 6'

2021-10-20 13:59:22 EXCODE: 6

2021-10-20 13:59:22 + echo 'ipipe_log_param_EXCODE: 6'

2021-10-20 13:59:22 ipipe_log_param_EXCODE: 6

2021-10-20 13:59:22 + exit 6

🔹 Failed: PR-CI-Kunlun

Unknown Failed
2021-10-20 14:55:56                                                                                              ~~^             ~~~~~~~~~~~~~~~~

2021-10-20 14:55:56                                                                                              %d

2021-10-20 14:56:43 [  4%] Performing install step for 'extern_openblas'

2021-10-20 14:56:44 -- extern_openblas install command succeeded.  See also /paddle/build/third_party/openblas/src/extern_openblas-stamp/extern_openblas-install-*.log

2021-10-20 14:56:44 [  4%] Completed 'extern_openblas'

2021-10-20 14:56:44 [  4%] Built target extern_openblas

2021-10-20 14:56:44 Makefile:140: recipe for target 'all' failed

2021-10-20 14:56:44 make: *** [all] Error 2

2021-10-20 14:56:44 + build_error=2

2021-10-20 14:56:44 + collect_ccache_hits

2021-10-20 14:56:44 ++ ccache -s

2021-10-20 14:56:44 ++ grep 'cache hit rate'

2021-10-20 14:56:44 ++ awk '{print $4}'

2021-10-20 14:56:44 + rate=0.00

2021-10-20 14:56:44 + echo 'ccache hit rate: 0.00%'

2021-10-20 14:56:44 + echo 'ipipe_log_param_Ccache_Hit_Rate: 0.00%'

2021-10-20 14:56:44 ccache hit rate: 0.00%

2021-10-20 14:56:44 + '[' 2 '!=' 0 ']'

2021-10-20 14:56:44 + exit 7

Please sign in to comment.