From f12e2e5a8b17fbf4d2169789325ea87007d56731 Mon Sep 17 00:00:00 2001 From: winter-wang <1030748926@qq.com> Date: Tue, 2 Jan 2024 12:04:51 +0000 Subject: [PATCH] [PIR] remove log simply name mechnism from phi to common. --- paddle/common/enforce.cc | 102 ++++++++++++++++++ paddle/common/enforce.h | 74 +++++-------- paddle/fluid/framework/CMakeLists.txt | 5 +- paddle/fluid/framework/type_defs.cc | 45 ++++++++ paddle/fluid/imperative/CMakeLists.txt | 2 +- paddle/fluid/imperative/type_defs.cc | 21 ++++ paddle/fluid/memory/stats.h | 6 +- paddle/fluid/platform/enforce.h | 1 + paddle/fluid/platform/init.cc | 4 +- paddle/phi/core/enforce.cc | 144 ++----------------------- paddle/phi/core/enforce.h | 23 ++-- paddle/utils/variant_test.cc | 2 +- 12 files changed, 220 insertions(+), 209 deletions(-) create mode 100644 paddle/common/enforce.cc create mode 100644 paddle/fluid/framework/type_defs.cc create mode 100644 paddle/fluid/imperative/type_defs.cc diff --git a/paddle/common/enforce.cc b/paddle/common/enforce.cc new file mode 100644 index 0000000000000..a97bcc856ae54 --- /dev/null +++ b/paddle/common/enforce.cc @@ -0,0 +1,102 @@ +/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/common/enforce.h" + +#include +#include +#include + +REGISTER_LOG_SIMPLY_STR(std::string); + +namespace { +class StrSizeCmp { + public: + bool operator()(const std::string& lhs, const std::string& rhs) const { + return lhs.size() > rhs.size(); + } +}; + +using LogSimplyStrMap = std::map; + +LogSimplyStrMap& GetLogStrSimplyMap() { + static LogSimplyStrMap str_simply_map; + return str_simply_map; +} + +std::string SimplifyDemangleStr(std::string str) { + auto& str_map = GetLogStrSimplyMap(); + for (auto& value : str_map) { + size_t start_pos = 0; + while ((start_pos = str.find(value.first, start_pos)) != + std::string::npos) { + str.replace(start_pos, value.first.length(), value.second); + start_pos += value.second.length(); + } + } + return str; +} +} // namespace + +namespace common { +namespace enforce { + +bool RegisterLogSimplyStr(const std::string& type_name, + const std::string& simply_name) { + return GetLogStrSimplyMap() + .emplace(std::make_pair(type_name, simply_name)) + .second; +} + +std::string GetCurrentTraceBackString(bool for_signal) { + std::ostringstream sout; + + if (!for_signal) { + sout << "\n\n--------------------------------------\n"; + sout << "C++ Traceback (most recent call last):"; + sout << "\n--------------------------------------\n"; + } +#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL) + static constexpr int TRACE_STACK_LIMIT = 100; + + std::array call_stack; + auto size = backtrace(call_stack.data(), TRACE_STACK_LIMIT); + auto symbols = backtrace_symbols(call_stack.data(), size); + Dl_info info; + int idx = 0; + // `for_signal` used to remove the stack trace introduced by + // obtaining the error stack trace when the signal error occurred, + // that is not related to the signal error self, remove it to + // avoid misleading users and developers + int end_idx = for_signal ? 2 : 0; + for (int i = size - 1; i >= end_idx; --i) { + if (dladdr(call_stack[i], &info) && info.dli_sname) { + auto demangled = common::demangle(info.dli_sname); + std::string path(info.dli_fname); + // C++ traceback info are from core.so + if (path.substr(path.length() - 3).compare(".so") == 0) { + sout << paddle::string::Sprintf( + "%-3d %s\n", idx++, SimplifyDemangleStr(demangled)); + } + } + } + free(symbols); // NOLINT +#else + sout << "Not support stack backtrace yet.\n"; +#endif + return sout.str(); +} + +} // namespace enforce +} // namespace common diff --git a/paddle/common/enforce.h b/paddle/common/enforce.h index 13e33c7e32a76..b734c90d0672b 100644 --- a/paddle/common/enforce.h +++ b/paddle/common/enforce.h @@ -31,6 +31,7 @@ #include "paddle/common/errors.h" #include "paddle/common/macros.h" +#include "paddle/utils/test_macros.h" #if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL) #include @@ -40,10 +41,20 @@ #define GLOG_NO_ABBREVIATED_SEVERITIES #include "paddle/utils/string/printf.h" #include "paddle/utils/string/to_string.h" -#include "paddle/utils/test_macros.h" #include "paddle/utils/variant.h" namespace common { +#ifdef __GNUC__ +inline std::string demangle(std::string name) { + int status = -4; // some arbitrary value to eliminate the compiler warning + std::unique_ptr res{ + abi::__cxa_demangle(name.c_str(), NULL, NULL, &status), std::free}; + return (status == 0) ? res.get() : name; +} +#else +inline std::string demangle(std::string name) { return name; } +#endif + class CommonNotMetException : public std::exception { public: explicit CommonNotMetException(const std::string& str) : err_str_(str) {} @@ -53,9 +64,7 @@ class CommonNotMetException : public std::exception { private: std::string err_str_; }; -} // namespace common -namespace common { namespace enforce { /** HELPER MACROS AND FUNCTIONS **/ @@ -161,6 +170,20 @@ using CommonType2 = typename std::add_lvalue_reference< #define COMMON_ENFORCE_LE(__VAL0, __VAL1, ...) \ __COMMON_BINARY_COMPARE(__VAL0, __VAL1, <=, >, __VA_ARGS__) +TEST_API bool RegisterLogSimplyStr(const std::string& type, + const std::string& simply); +TEST_API std::string GetCurrentTraceBackString(bool for_signal = false); +template +class LogSimplyStrRegistrar { + public: + static bool success; +}; + +#define REGISTER_LOG_SIMPLY_STR(Type) \ + template <> \ + bool ::common::enforce::LogSimplyStrRegistrar::success = \ + ::common::enforce::RegisterLogSimplyStr( \ + ::common::demangle(typeid(Type).name()), #Type); } // namespace enforce } // namespace common @@ -172,53 +195,10 @@ inline bool is_error(const T& stat) { } namespace pir { - -#ifdef __GNUC__ -inline std::string demangle(std::string name) { - int status = -4; // some arbitrary value to eliminate the compiler warning - std::unique_ptr res{ - abi::__cxa_demangle(name.c_str(), NULL, NULL, &status), std::free}; - return (status == 0) ? res.get() : name; -} -#else -inline std::string demangle(std::string name) { return name; } -#endif - -static std::string GetCurrentTraceBackString() { - std::ostringstream sout; - sout << "\n\n--------------------------------------\n"; - sout << "C++ Traceback (most recent call last):"; - sout << "\n--------------------------------------\n"; -#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL) - static constexpr int TRACE_STACK_LIMIT = 100; - - void* call_stack[TRACE_STACK_LIMIT]; - auto size = backtrace(call_stack, TRACE_STACK_LIMIT); - auto symbols = backtrace_symbols(call_stack, size); - Dl_info info; - int idx = 0; - int end_idx = 0; - for (int i = size - 1; i >= end_idx; --i) { - if (dladdr(call_stack[i], &info) && info.dli_sname) { - auto demangled = demangle(info.dli_sname); - std::string path(info.dli_fname); - // C++ traceback info are from core.so - if (path.substr(path.length() - 3).compare(".so") == 0) { - sout << idx++ << " " << demangled << "\n"; - } - } - } - free(symbols); -#else - sout << "Not support stack backtrace yet.\n"; -#endif - return sout.str(); -} - class IrNotMetException : public std::exception { public: explicit IrNotMetException(const std::string& str) - : err_str_(str + GetCurrentTraceBackString()) {} + : err_str_(str + ::common::enforce::GetCurrentTraceBackString()) {} const char* what() const noexcept override { return err_str_.c_str(); } diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index 338130c64d9a0..6e70167a853b6 100755 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -947,7 +947,10 @@ cc_library( imperative_flag layer) -cc_library(type_info SRCS type_info.cc) +cc_library( + type_info + SRCS type_info.cc type_defs.cc + DEPS common) target_link_libraries(type_info pir op_dialect) add_dependencies(type_info framework_proto auto_parallel_proto xxhash) if(WITH_MKLDNN) diff --git a/paddle/fluid/framework/type_defs.cc b/paddle/fluid/framework/type_defs.cc new file mode 100644 index 0000000000000..d8a6546ea718d --- /dev/null +++ b/paddle/fluid/framework/type_defs.cc @@ -0,0 +1,45 @@ +/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/framework/type_defs.h" + +#include "paddle/common/enforce.h" + +namespace paddle { + +using namespace framework; // NOLINT +template class variant, + std::vector, + std::vector, + bool, + std::vector, + BlockDesc*, + int64_t, + std::vector, + std::vector, + std::vector, + VarDesc*, + std::vector, + double, + paddle::experimental::Scalar, + std::vector, + ::pir::Block*, + std::vector<::pir::Value>>; +} // namespace paddle +REGISTER_LOG_SIMPLY_STR(paddle::framework::AttributeMap); +REGISTER_LOG_SIMPLY_STR(paddle::framework::Attribute); diff --git a/paddle/fluid/imperative/CMakeLists.txt b/paddle/fluid/imperative/CMakeLists.txt index 7a764f5302021..86688213ef186 100644 --- a/paddle/fluid/imperative/CMakeLists.txt +++ b/paddle/fluid/imperative/CMakeLists.txt @@ -4,7 +4,7 @@ cc_library( DEPS phi common) cc_library( var_helper - SRCS var_helper.cc + SRCS var_helper.cc type_defs.cc DEPS tensor phi common) if(WITH_XPU) cc_library( diff --git a/paddle/fluid/imperative/type_defs.cc b/paddle/fluid/imperative/type_defs.cc new file mode 100644 index 0000000000000..fa4a327f8a21e --- /dev/null +++ b/paddle/fluid/imperative/type_defs.cc @@ -0,0 +1,21 @@ +/* Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/imperative/type_defs.h" + +#include "paddle/common/enforce.h" + +REGISTER_LOG_SIMPLY_STR(paddle::imperative::NameTensorMap); +REGISTER_LOG_SIMPLY_STR(paddle::imperative::NameVarBaseMap); +REGISTER_LOG_SIMPLY_STR(paddle::imperative::NameVariableWrapperMap); diff --git a/paddle/fluid/memory/stats.h b/paddle/fluid/memory/stats.h index d2c8b04bc70ab..2cecfb16c3e01 100644 --- a/paddle/fluid/memory/stats.h +++ b/paddle/fluid/memory/stats.h @@ -77,8 +77,7 @@ class Stat : public StatBase { thread_local_stat->current += increment; VLOG(8) << string::split_string( - phi::enforce::demangle(typeid(*thread_local_stat).name()), - "::") + common::demangle(typeid(*thread_local_stat).name()), "::") .back() << ": Update current_value with " << increment << ", after update, current value = " << GetCurrentValue(); @@ -91,8 +90,7 @@ class Stat : public StatBase { !peak_value_.compare_exchange_weak(prev_value, current_value)) { } VLOG(8) << string::split_string( - phi::enforce::demangle(typeid(*thread_local_stat).name()), - "::") + common::demangle(typeid(*thread_local_stat).name()), "::") .back() << ": Update current_value with " << increment << ", after update, peak_value = " << peak_value_.load() diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h index 1a82b05f3bc3a..c7ee1707c4286 100644 --- a/paddle/fluid/platform/enforce.h +++ b/paddle/fluid/platform/enforce.h @@ -108,6 +108,7 @@ PHI_DECLARE_int32(call_stack_level); namespace paddle { namespace platform { using namespace ::phi::enforce; // NOLINT +using ::common::demangle; /** HELPER MACROS AND FUNCTIONS **/ diff --git a/paddle/fluid/platform/init.cc b/paddle/fluid/platform/init.cc index a3fff528f7903..62353a415617c 100644 --- a/paddle/fluid/platform/init.cc +++ b/paddle/fluid/platform/init.cc @@ -51,6 +51,7 @@ limitations under the License. */ #include "paddle/fluid/platform/device/ipu/ipu_info.h" #endif +#include "paddle/common/enforce.h" #include "paddle/fluid/memory/allocation/allocator_facade.h" #include "paddle/fluid/memory/memory.h" #include "paddle/fluid/platform/flags.h" @@ -310,7 +311,8 @@ void SignalHandle(const char *data, int size) { sout << "\n\n--------------------------------------\n"; sout << "C++ Traceback (most recent call last):"; sout << "\n--------------------------------------\n"; - auto traceback = platform::GetCurrentTraceBackString(/*for_signal=*/true); + auto traceback = + ::common::enforce::GetCurrentTraceBackString(/*for_signal=*/true); if (traceback.empty()) { sout << "No stack trace in paddle, may be caused by external reasons.\n"; diff --git a/paddle/phi/core/enforce.cc b/paddle/phi/core/enforce.cc index 5d4041738bc4f..979b147c6b3e1 100644 --- a/paddle/phi/core/enforce.cc +++ b/paddle/phi/core/enforce.cc @@ -21,6 +21,7 @@ limitations under the License. */ #include #include "glog/logging.h" +#include "paddle/common/enforce.h" #include "paddle/phi/common/scalar.h" #include "paddle/utils/blank.h" #include "paddle/utils/flags.h" @@ -30,146 +31,11 @@ limitations under the License. */ #endif // PADDLE_WITH_CUDA PD_DECLARE_int32(call_stack_level); - -namespace egr { -class EagerVariable; -} -namespace paddle { -namespace framework { -class VarDesc; -class BlockDesc; -using Attribute = paddle::variant, - std::vector, - std::vector, - bool, - std::vector, - BlockDesc*, - int64_t, - std::vector, - std::vector, - std::vector, - VarDesc*, - std::vector, - double, - paddle::experimental::Scalar, - std::vector>; -using AttributeMap = std::unordered_map; -} // namespace framework -namespace imperative { -class VariableWrapper; -class SavedVariableWrapperList; -class VarBase; - -namespace details { -template -struct NameVarMapTrait {}; - -template <> -struct NameVarMapTrait { - using Type = std::map>>; -}; - -template <> -struct NameVarMapTrait { - using Type = std::map; -}; - -template <> -struct NameVarMapTrait { - using Type = - std::map>>; -}; - -} // namespace details - -template -using NameVarMap = typename details::NameVarMapTrait::Type; - -using NameVarBaseMap = NameVarMap; -using NameVariableWrapperMap = NameVarMap; -using NameTensorMap = NameVarMap; - -} // namespace imperative -} // namespace paddle - namespace phi { namespace enforce { TEST_API int GetCallStackLevel() { return FLAGS_call_stack_level; } -template -static std::string ReplaceComplexTypeStr(std::string str, - const std::string& type_name) { - auto demangle_type_str = demangle(typeid(T).name()); - size_t start_pos = 0; - while ((start_pos = str.find(demangle_type_str, start_pos)) != - std::string::npos) { - str.replace(start_pos, demangle_type_str.length(), type_name); - start_pos += type_name.length(); - } - return str; -} - -#define __REPLACE_COMPLEX_TYPE_STR__(__TYPENAME, __STR) \ - do { \ - __STR = \ - phi::enforce::ReplaceComplexTypeStr<__TYPENAME>(__STR, #__TYPENAME); \ - } while (0) - -static std::string SimplifyDemangleStr(std::string str) { - // the older is important, you have to put complex types in front - __REPLACE_COMPLEX_TYPE_STR__(paddle::framework::AttributeMap, str); - __REPLACE_COMPLEX_TYPE_STR__(paddle::framework::Attribute, str); - __REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameVariableWrapperMap, str); - __REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameVarBaseMap, str); - __REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameTensorMap, str); - __REPLACE_COMPLEX_TYPE_STR__(std::string, str); - return str; -} - -TEST_API std::string GetCurrentTraceBackString(bool for_signal) { - std::ostringstream sout; - - if (!for_signal) { - sout << "\n\n--------------------------------------\n"; - sout << "C++ Traceback (most recent call last):"; - sout << "\n--------------------------------------\n"; - } -#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL) - static constexpr int TRACE_STACK_LIMIT = 100; - - std::array call_stack; - auto size = backtrace(call_stack.data(), TRACE_STACK_LIMIT); - auto symbols = backtrace_symbols(call_stack.data(), size); - Dl_info info; - int idx = 0; - // `for_signal` used to remove the stack trace introduced by - // obtaining the error stack trace when the signal error occurred, - // that is not related to the signal error self, remove it to - // avoid misleading users and developers - int end_idx = for_signal ? 2 : 0; - for (int i = size - 1; i >= end_idx; --i) { - if (dladdr(call_stack[i], &info) && info.dli_sname) { - auto demangled = demangle(info.dli_sname); - std::string path(info.dli_fname); - // C++ traceback info are from core.so - if (path.substr(path.length() - 3).compare(".so") == 0) { - sout << paddle::string::Sprintf( - "%-3d %s\n", idx++, SimplifyDemangleStr(demangled)); - } - } - } - free(symbols); // NOLINT -#else - sout << "Not support stack backtrace yet.\n"; -#endif - return sout.str(); -} - void ThrowWarnInternal(const std::string& msg) { LOG(WARNING) << "WARNING :" << msg; } @@ -276,7 +142,9 @@ std::string GetExternalErrorMsg(T status) { std::string search_path_3; #if !defined(_WIN32) Dl_info info; - if (dladdr(reinterpret_cast(GetCurrentTraceBackString), &info)) { + if (dladdr( + reinterpret_cast(common::enforce::GetCurrentTraceBackString), + &info)) { std::string phi_so_path(info.dli_fname); const size_t last_slash_idx = phi_so_path.find_last_of('/'); if (std::string::npos != last_slash_idx) { @@ -297,7 +165,9 @@ std::string GetExternalErrorMsg(T status) { char buf[512]; MEMORY_BASIC_INFORMATION mbi; HMODULE h_module = - (::VirtualQuery(GetCurrentTraceBackString, &mbi, sizeof(mbi)) != 0) + (::VirtualQuery(common::enforce::GetCurrentTraceBackString, + &mbi, + sizeof(mbi)) != 0) ? (HMODULE)mbi.AllocationBase : NULL; GetModuleFileName(h_module, buf, 512); diff --git a/paddle/phi/core/enforce.h b/paddle/phi/core/enforce.h index 61e502951f24e..feb2852a9dc67 100644 --- a/paddle/phi/core/enforce.h +++ b/paddle/phi/core/enforce.h @@ -79,17 +79,6 @@ limitations under the License. */ namespace phi { namespace enforce { -#ifdef __GNUC__ -inline std::string demangle(std::string name) { - int status = -4; // some arbitrary value to eliminate the compiler warning - std::unique_ptr res{ - abi::__cxa_demangle(name.c_str(), NULL, NULL, &status), std::free}; - return (status == 0) ? res.get() : name; -} -#else -inline std::string demangle(std::string name) { return name; } -#endif - namespace details { template inline constexpr bool IsArithmetic() { @@ -164,7 +153,6 @@ struct BinaryCompareMessageConverter { } // namespace details TEST_API int GetCallStackLevel(); -TEST_API std::string GetCurrentTraceBackString(bool for_signal = false); TEST_API std::string SimplifyErrorTypeFormat(const std::string& str); template @@ -192,7 +180,7 @@ std::string GetCompleteTraceBackString(StrType&& what, sout << paddle::string::Sprintf( "%s (at %s:%d)", std::forward(what), file, line) << std::endl; - return GetCurrentTraceBackString() + sout.str(); + return ::common::enforce::GetCurrentTraceBackString() + sout.str(); } template @@ -201,7 +189,8 @@ static std::string GetTraceBackString(StrType&& what, int line) { if (GetCallStackLevel() > 1) { // FLAGS_call_stack_level>1 means showing c++ call stack - return GetCurrentTraceBackString() + GetErrorSumaryString(what, file, line); + return ::common::enforce::GetCurrentTraceBackString() + + GetErrorSumaryString(what, file, line); } else { return GetErrorSumaryString(what, file, line); } @@ -453,7 +442,7 @@ struct EnforceNotMet : public std::exception { " 1. The %s is not the %s of operator %s;\n" \ " 2. The %s has no corresponding variable passed in;\n" \ " 3. The %s corresponding variable is not initialized.", \ - phi::demangle( \ + common::demangle( \ typeid(std::add_lvalue_reference::type) \ .name()), \ __ROLE, \ @@ -514,8 +503,8 @@ namespace details { "paddle::get failed, cannot get value " \ "(%s) by type %s, its type is %s.", \ expression, \ - phi::enforce::demangle(typeid(OutputType).name()), \ - phi::enforce::demangle(input.type().name())), \ + common::demangle(typeid(OutputType).name()), \ + common::demangle(input.type().name())), \ file, \ line); \ END_HANDLE_THE_ERROR \ diff --git a/paddle/utils/variant_test.cc b/paddle/utils/variant_test.cc index ef4a6cf8cd89c..854fe742e686d 100644 --- a/paddle/utils/variant_test.cc +++ b/paddle/utils/variant_test.cc @@ -18,7 +18,7 @@ #include "paddle/phi/core/enforce.h" TEST(interface_test, type) { - using phi::enforce::demangle; + using common::demangle; paddle::variant var;