Skip to content

Commit

Permalink
[libc++] Replace __compressed_pair with [[no_unique_address]]
Browse files Browse the repository at this point in the history
  • Loading branch information
@philnik777
philnik777 committed Jan 2, 2024
1 parent c92d3ce commit 8d3053c
Show file tree
Hide file tree
Showing 21 changed files with 822 additions and 568 deletions.
2 changes: 2 additions & 0 deletions libcxx/include/__config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,8 @@
// pointer from 16 to 8. This changes the output of std::string::max_size,
// which makes it ABI breaking
# define _LIBCPP_ABI_STRING_8_BYTE_ALIGNMENT
// Don't add padding from __compressed_pair
# define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
# elif _LIBCPP_ABI_VERSION == 1
# if !(defined(_LIBCPP_OBJECT_FORMAT_COFF) || defined(_LIBCPP_OBJECT_FORMAT_XCOFF))
// Enable compiling copies of now inline methods into the dylib to support
Expand Down
27 changes: 14 additions & 13 deletions libcxx/include/__functional/function.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,45 +138,46 @@ class __default_alloc_func;

template <class _Fp, class _Ap, class _Rp, class... _ArgTypes>
class __alloc_func<_Fp, _Ap, _Rp(_ArgTypes...)> {
__compressed_pair<_Fp, _Ap> __f_;
_LIBCPP_COMPRESSED_PAIR(_Fp, __func_, _Ap, __alloc_);

public:
typedef _LIBCPP_NODEBUG _Fp _Target;
typedef _LIBCPP_NODEBUG _Ap _Alloc;

_LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __f_.first(); }
_LIBCPP_HIDE_FROM_ABI const _Target& __target() const { return __func_; }

// WIN32 APIs may define __allocator, so use __get_allocator instead.
_LIBCPP_HIDE_FROM_ABI const _Alloc& __get_allocator() const { return __f_.second(); }
_LIBCPP_HIDE_FROM_ABI const _Alloc& __get_allocator() const { return __alloc_; }

_LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f)
: __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), std::forward_as_tuple()) {}
_LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f) : __func_(std::move(__f)), __alloc_() {}

_LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, const _Alloc& __a)
: __f_(piecewise_construct, std::forward_as_tuple(__f), std::forward_as_tuple(__a)) {}
_LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, const _Alloc& __a) : __func_(__f), __alloc_(__a) {}

_LIBCPP_HIDE_FROM_ABI explicit __alloc_func(const _Target& __f, _Alloc&& __a)
: __f_(piecewise_construct, std::forward_as_tuple(__f), std::forward_as_tuple(std::move(__a))) {}
: __func_(__f), __alloc_(std::move(__a)) {}

_LIBCPP_HIDE_FROM_ABI explicit __alloc_func(_Target&& __f, _Alloc&& __a)
: __f_(piecewise_construct, std::forward_as_tuple(std::move(__f)), std::forward_as_tuple(std::move(__a))) {}
: __func_(std::move(__f)), __alloc_(std::move(__a)) {}

_LIBCPP_HIDE_FROM_ABI _Rp operator()(_ArgTypes&&... __arg) {
typedef __invoke_void_return_wrapper<_Rp> _Invoker;
return _Invoker::__call(__f_.first(), std::forward<_ArgTypes>(__arg)...);
return _Invoker::__call(__func_, std::forward<_ArgTypes>(__arg)...);
}

_LIBCPP_HIDE_FROM_ABI __alloc_func* __clone() const {
typedef allocator_traits<_Alloc> __alloc_traits;
typedef __rebind_alloc<__alloc_traits, __alloc_func> _AA;
_AA __a(__f_.second());
_AA __a(__alloc_);
typedef __allocator_destructor<_AA> _Dp;
unique_ptr<__alloc_func, _Dp> __hold(__a.allocate(1), _Dp(__a, 1));
::new ((void*)__hold.get()) __alloc_func(__f_.first(), _Alloc(__a));
::new ((void*)__hold.get()) __alloc_func(__func_, _Alloc(__a));
return __hold.release();
}

_LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT { __f_.~__compressed_pair<_Target, _Alloc>(); }
_LIBCPP_HIDE_FROM_ABI void destroy() _NOEXCEPT {
__func_.~_Fp();
__alloc_.~_Alloc();
}

_LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__alloc_func* __f) {
typedef allocator_traits<_Alloc> __alloc_traits;
Expand Down
155 changes: 87 additions & 68 deletions libcxx/include/__hash_table
View file

Large diffs are not rendered by default.

162 changes: 20 additions & 142 deletions libcxx/include/__memory/compressed_pair.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,161 +11,39 @@
#define _LIBCPP___MEMORY_COMPRESSED_PAIR_H

#include <__config>
#include <__fwd/get.h>
#include <__fwd/tuple.h>
#include <__tuple/tuple_indices.h>
#include <__type_traits/decay.h>
#include <__type_traits/dependent_type.h>
#include <__type_traits/enable_if.h>
#include <__type_traits/is_default_constructible.h>
#include <__type_traits/datasizeof.h>
#include <__type_traits/is_empty.h>
#include <__type_traits/is_final.h>
#include <__type_traits/is_same.h>
#include <__type_traits/is_swappable.h>
#include <__utility/forward.h>
#include <__utility/move.h>
#include <__utility/piecewise_construct.h>
#include <cstddef>
#include <__type_traits/is_object.h>

#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
#endif

_LIBCPP_PUSH_MACROS
#include <__undef_macros>

_LIBCPP_BEGIN_NAMESPACE_STD

// Tag used to default initialize one or both of the pair's elements.
struct __default_init_tag {};
struct __value_init_tag {};

template <class _Tp, int _Idx, bool _CanBeEmptyBase = is_empty<_Tp>::value && !__libcpp_is_final<_Tp>::value>
struct __compressed_pair_elem {
using _ParamT = _Tp;
using reference = _Tp&;
using const_reference = const _Tp&;

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__default_init_tag) {}
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__value_init_tag) : __value_() {}

template <class _Up, class = __enable_if_t<!is_same<__compressed_pair_elem, __decay_t<_Up> >::value> >
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(_Up&& __u)
: __value_(std::forward<_Up>(__u)) {}

#ifndef _LIBCPP_CXX03_LANG
template <class... _Args, size_t... _Indices>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit __compressed_pair_elem(
piecewise_construct_t, tuple<_Args...> __args, __tuple_indices<_Indices...>)
: __value_(std::forward<_Args>(std::get<_Indices>(__args))...) {}
#endif

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference __get() _NOEXCEPT { return __value_; }
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference __get() const _NOEXCEPT { return __value_; }

private:
_Tp __value_;
};

template <class _Tp, int _Idx>
struct __compressed_pair_elem<_Tp, _Idx, true> : private _Tp {
using _ParamT = _Tp;
using reference = _Tp&;
using const_reference = const _Tp&;
using __value_type = _Tp;

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem() = default;
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__default_init_tag) {}
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(__value_init_tag) : __value_type() {}

template <class _Up, class = __enable_if_t<!is_same<__compressed_pair_elem, __decay_t<_Up> >::value> >
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair_elem(_Up&& __u)
: __value_type(std::forward<_Up>(__u)) {}

#ifndef _LIBCPP_CXX03_LANG
template <class... _Args, size_t... _Indices>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17
__compressed_pair_elem(piecewise_construct_t, tuple<_Args...> __args, __tuple_indices<_Indices...>)
: __value_type(std::forward<_Args>(std::get<_Indices>(__args))...) {}
#endif

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 reference __get() _NOEXCEPT { return *this; }
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const_reference __get() const _NOEXCEPT { return *this; }
};

template <class _T1, class _T2>
class __compressed_pair : private __compressed_pair_elem<_T1, 0>, private __compressed_pair_elem<_T2, 1> {
public:
// NOTE: This static assert should never fire because __compressed_pair
// is *almost never* used in a scenario where it's possible for T1 == T2.
// (The exception is std::function where it is possible that the function
// object and the allocator have the same type).
static_assert(
(!is_same<_T1, _T2>::value),
"__compressed_pair cannot be instantiated when T1 and T2 are the same type; "
"The current implementation is NOT ABI-compatible with the previous implementation for this configuration");

using _Base1 _LIBCPP_NODEBUG = __compressed_pair_elem<_T1, 0>;
using _Base2 _LIBCPP_NODEBUG = __compressed_pair_elem<_T2, 1>;

template <bool _Dummy = true,
class = __enable_if_t< __dependent_type<is_default_constructible<_T1>, _Dummy>::value &&
__dependent_type<is_default_constructible<_T2>, _Dummy>::value > >
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair()
: _Base1(__value_init_tag()), _Base2(__value_init_tag()) {}

template <class _U1, class _U2>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __compressed_pair(_U1&& __t1, _U2&& __t2)
: _Base1(std::forward<_U1>(__t1)), _Base2(std::forward<_U2>(__t2)) {}

#ifndef _LIBCPP_CXX03_LANG
template <class... _Args1, class... _Args2>
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 explicit __compressed_pair(
piecewise_construct_t __pc, tuple<_Args1...> __first_args, tuple<_Args2...> __second_args)
: _Base1(__pc, std::move(__first_args), typename __make_tuple_indices<sizeof...(_Args1)>::type()),
_Base2(__pc, std::move(__second_args), typename __make_tuple_indices<sizeof...(_Args2)>::type()) {}
#endif

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename _Base1::reference first() _NOEXCEPT {
return static_cast<_Base1&>(*this).__get();
}

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename _Base1::const_reference first() const _NOEXCEPT {
return static_cast<_Base1 const&>(*this).__get();
}

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 typename _Base2::reference second() _NOEXCEPT {
return static_cast<_Base2&>(*this).__get();
}
#ifndef _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR typename _Base2::const_reference second() const _NOEXCEPT {
return static_cast<_Base2 const&>(*this).__get();
}

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR static _Base1* __get_first_base(__compressed_pair* __pair) _NOEXCEPT {
return static_cast<_Base1*>(__pair);
}
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR static _Base2* __get_second_base(__compressed_pair* __pair) _NOEXCEPT {
return static_cast<_Base2*>(__pair);
}

_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void swap(__compressed_pair& __x)
_NOEXCEPT_(__is_nothrow_swappable<_T1>::value&& __is_nothrow_swappable<_T2>::value) {
using std::swap;
swap(first(), __x.first());
swap(second(), __x.second());
}
template <class _ToPad>
class __compressed_pair_padding {
char __padding_[(!is_object<_ToPad>::value || (is_empty<_ToPad>::value && !__libcpp_is_final<_ToPad>::value))
? 0
: sizeof(_ToPad) - __libcpp_datasizeof<_ToPad>::value];
};

template <class _T1, class _T2>
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void
swap(__compressed_pair<_T1, _T2>& __x, __compressed_pair<_T1, _T2>& __y)
_NOEXCEPT_(__is_nothrow_swappable<_T1>::value&& __is_nothrow_swappable<_T2>::value) {
__x.swap(__y);
}
# define _LIBCPP_COMPRESSED_PAIR_PADDING(T, Name) _LIBCPP_NO_UNIQUE_ADDRESS __compressed_pair_padding<T> Name
# define _LIBCPP_COMPRESSED_PAIR(T1, Name1, T2, Name2) \
[[__gnu__::__aligned__(_LIBCPP_ALIGNOF(T2))]] _LIBCPP_NO_UNIQUE_ADDRESS T1 Name1; \
_LIBCPP_COMPRESSED_PAIR_PADDING(T1, Name1##_padding_); \
_LIBCPP_NO_UNIQUE_ADDRESS T2 Name2; \
_LIBCPP_COMPRESSED_PAIR_PADDING(T2, Name2##_padding_)
#else
# define _LIBCPP_COMPRESSED_PAIR_PADDING(T, Name)
# define _LIBCPP_COMPRESSED_PAIR(T1, Name1, T2, Name2) \
_LIBCPP_NO_UNIQUE_ADDRESS T1 Name1; \
_LIBCPP_NO_UNIQUE_ADDRESS T2 Name2;
#endif // _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING

_LIBCPP_END_NAMESPACE_STD

_LIBCPP_POP_MACROS

#endif // _LIBCPP___MEMORY_COMPRESSED_PAIR_H
45 changes: 21 additions & 24 deletions libcxx/include/__memory/shared_ptr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,11 +196,17 @@ class _LIBCPP_EXPORTED_FROM_ABI __shared_weak_count : private __shared_count {

template <class _Tp, class _Dp, class _Alloc>
class __shared_ptr_pointer : public __shared_weak_count {
__compressed_pair<__compressed_pair<_Tp, _Dp>, _Alloc> __data_;
[[using __gnu__: __aligned__(_LIBCPP_ALIGNOF(_Dp)),
__aligned__(_LIBCPP_ALIGNOF(_Alloc))]] _LIBCPP_NO_UNIQUE_ADDRESS _Tp __ptr_;
_LIBCPP_COMPRESSED_PAIR_PADDING(_Tp, __ptr_padding_);
_LIBCPP_NO_UNIQUE_ADDRESS _Dp __deleter_;
_LIBCPP_COMPRESSED_PAIR_PADDING(_Dp, __deleter_padding_);
_LIBCPP_NO_UNIQUE_ADDRESS _Alloc __alloc_;
_LIBCPP_COMPRESSED_PAIR_PADDING(_Alloc, __alloc_padding_);

public:
_LIBCPP_HIDE_FROM_ABI __shared_ptr_pointer(_Tp __p, _Dp __d, _Alloc __a)
: __data_(__compressed_pair<_Tp, _Dp>(__p, std::move(__d)), std::move(__a)) {}
: __ptr_(__p), __deleter_(std::move(__d)), __alloc_(std::move(__a)) {}

#ifndef _LIBCPP_HAS_NO_RTTI
_LIBCPP_HIDE_FROM_ABI_VIRTUAL const void* __get_deleter(const type_info&) const _NOEXCEPT override;
Expand All @@ -215,15 +221,15 @@ class __shared_ptr_pointer : public __shared_weak_count {

template <class _Tp, class _Dp, class _Alloc>
const void* __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__get_deleter(const type_info& __t) const _NOEXCEPT {
return __t == typeid(_Dp) ? std::addressof(__data_.first().second()) : nullptr;
return __t == typeid(_Dp) ? std::addressof(__deleter_) : nullptr;
}

#endif // _LIBCPP_HAS_NO_RTTI

template <class _Tp, class _Dp, class _Alloc>
void __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__on_zero_shared() _NOEXCEPT {
__data_.first().second()(__data_.first().first());
__data_.first().second().~_Dp();
__deleter_(__ptr_);
__deleter_.~_Dp();
}

template <class _Tp, class _Dp, class _Alloc>
Expand All @@ -232,8 +238,8 @@ void __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__on_zero_shared_weak() _NOEXCEPT {
typedef allocator_traits<_Al> _ATraits;
typedef pointer_traits<typename _ATraits::pointer> _PTraits;

_Al __a(__data_.second());
__data_.second().~_Alloc();
_Al __a(__alloc_);
__alloc_.~_Alloc();
__a.deallocate(_PTraits::pointer_to(*this), 1);
}

Expand Down Expand Up @@ -299,28 +305,19 @@ struct __shared_ptr_emplace : __shared_weak_count {
// with Allocator construction for _Tp. To allow implementing P0674 in C++20,
// we now use a properly aligned char buffer while making sure that we maintain
// the same layout that we had when we used a compressed pair.
using _CompressedPair = __compressed_pair<_Alloc, _Tp>;
struct _ALIGNAS_TYPE(_CompressedPair) _Storage {
char __blob_[sizeof(_CompressedPair)];
struct [[using __gnu__: __aligned__(_LIBCPP_ALIGNOF(_Alloc)), __aligned__(_LIBCPP_ALIGNOF(_Tp))]] _Storage {
union {
struct {
_LIBCPP_COMPRESSED_PAIR(_Alloc, __alloc_, _Tp, __elem_);
};
};

_LIBCPP_HIDE_FROM_ABI explicit _Storage(_Alloc&& __a) { ::new ((void*)__get_alloc()) _Alloc(std::move(__a)); }
_LIBCPP_HIDE_FROM_ABI ~_Storage() { __get_alloc()->~_Alloc(); }
_LIBCPP_HIDE_FROM_ABI _Alloc* __get_alloc() _NOEXCEPT {
_CompressedPair* __as_pair = reinterpret_cast<_CompressedPair*>(__blob_);
typename _CompressedPair::_Base1* __first = _CompressedPair::__get_first_base(__as_pair);
_Alloc* __alloc = reinterpret_cast<_Alloc*>(__first);
return __alloc;
}
_LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI _Tp* __get_elem() _NOEXCEPT {
_CompressedPair* __as_pair = reinterpret_cast<_CompressedPair*>(__blob_);
typename _CompressedPair::_Base2* __second = _CompressedPair::__get_second_base(__as_pair);
_Tp* __elem = reinterpret_cast<_Tp*>(__second);
return __elem;
}
_LIBCPP_HIDE_FROM_ABI _Alloc* __get_alloc() _NOEXCEPT { return std::addressof(__alloc_); }
_LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_CFI _Tp* __get_elem() _NOEXCEPT { return std::addressof(__elem_); }
};

static_assert(_LIBCPP_ALIGNOF(_Storage) == _LIBCPP_ALIGNOF(_CompressedPair), "");
static_assert(sizeof(_Storage) == sizeof(_CompressedPair), "");
_Storage __storage_;
};

Expand Down
Loading

0 comments on commit 8d3053c

Please sign in to comment.