Deep copy (#303)

* Add deep copy option to avoid the need for xnack+

---------

Co-authored-by: Alan Dayton <6393677+adayton1@users.noreply.github.com>

Author: liu15

RELEASE_NOTES.md

@@ -14,6 +14,9 @@ The format of this file is based on [Keep a Changelog](http://keepachangelog.com
 
 ## [Unreleased] - Release date YYYY-MM-DD
 
+### Added
+- Added CARE\_DEEP\_COPY\_RAW\_PTR configuration option.
+
 ### Added
 - Added ATOMIC\_SUB, ATOMIC\_LOAD, ATOMIC\_STORE, ATOMIC\_EXCHANGE, and ATOMIC\_CAS macros.
 - Added TSAN\_ONLY\_ATOMIC\_\* macros to suppress tsan data race reports. Controlled by CARE\_ENABLE\_TSAN\_ONLY\_ATOMICS configuration option.

cmake/SetupOptions.cmake

@@ -19,6 +19,7 @@ option(CARE_ENABLE_IMPLICIT_CONVERSIONS "Enable implicit conversions to-from raw
 # CHAI must also be configured with the same settings for implicit conversions.
 set(CHAI_ENABLE_IMPLICIT_CONVERSIONS ${CARE_ENABLE_IMPLICIT_CONVERSIONS} CACHE BOOL "Enable implicit conversions to-from raw pointers")
 option(CARE_LEGACY_COMPATIBILITY_MODE "Enable legacy compatibility mode" OFF)
+option(CARE_DEEP_COPY_RAW_PTR "Use deep copy for managed array initialization from raw pointer" OFF)
 option(CARE_ENABLE_MANAGED_PTR "Enable managed_ptr aliases, tests, and reproducer" ON)
 option(CARE_DISABLE_RAJAPLUGIN "Disable use of the RAJA plugin. WILL ALSO DISABLE MEMORY MOTION." OFF)
 option(CARE_ENABLE_EXTERN_INSTANTIATE "Enable extern instantiation of template functions" OFF)

src/care/config.h.in

@@ -19,6 +19,7 @@
 #ifndef CARE_LEGACY_COMPATIBILITY_MODE
 #cmakedefine01 CARE_LEGACY_COMPATIBILITY_MODE
 #endif
+#cmakedefine CARE_DEEP_COPY_RAW_PTR
 #cmakedefine CARE_ENABLE_MANAGED_PTR
 #cmakedefine CARE_DISABLE_RAJAPLUGIN
 #cmakedefine CARE_ENABLE_EXTERN_INSTANTIATE

src/care/host_device_ptr.h

@@ -23,6 +23,7 @@
 // Std library headers
 #include <cstddef>
 
+
 namespace care {
    ///////////////////////////////////////////////////////////////////////////
    /// @struct _kv
@@ -138,13 +139,20 @@ namespace care {
       {
       }
 
-#if defined (CHAI_DISABLE_RM) || defined(CHAI_THIN_GPU_ALLOCATE)
       ///
       /// @author Peter Robinson
       ///
       /// Construct from a raw pointer, size, and name
       /// This is defined when the CHAI resource manager is disabled
       ///
+#if defined(CARE_DEEP_COPY_RAW_PTR)
+      host_device_ptr<T>(T* from, size_t size, const char * name)
+         : MA(size)
+      {
+         std::copy_n(from, size, (T_non_const*)MA::data());
+      }
+#else /* defined(CARE_DEEP_COPY_RAW_PTR) */
+#if defined (CHAI_DISABLE_RM) || defined(CHAI_THIN_GPU_ALLOCATE)
       host_device_ptr<T>(T* from, size_t size, const char * name)
          : MA(from, nullptr, size, nullptr)
       {
@@ -173,6 +181,7 @@ namespace care {
          }
       }
 #endif
+#endif /* defined(CARE_DEEP_COPY_RAW_PTR) */
 
       ///
       /// @author Peter Robinson
@@ -412,6 +421,16 @@ namespace care {
       void freeDeviceMemory(T_non_const ** CPU_destination,
                             size_t elems,
                             bool deregisterPointer=true) {
+#if defined(CARE_DEEP_COPY_RAW_PTR)
+         // if there is a pointer to update ...
+         if (CPU_destination != nullptr) {
+            if (*CPU_destination == nullptr) {
+               *CPU_destination = (T_non_const *) std::malloc(elems*sizeof(T));
+            }
+            std::copy_n(MA::cdata(), elems, *CPU_destination);
+         }
+         MA::free();
+#else /* defined(CARE_DEEP_COPY_RAW_PTR) */
 #if !defined(CHAI_DISABLE_RM) 
 #if defined(CHAI_GPUCC) || CARE_ENABLE_GPU_SIMULATION_MODE
          if (CPU_destination != nullptr) {
@@ -441,6 +460,7 @@ namespace care {
             arrayManager->deregisterPointer(MA::m_pointer_record,true);
             CHAICallback::deregisterRecord(MA::m_pointer_record);
          }
+         
 #else // no resource manager active
 #if defined(CHAI_THIN_GPU_ALLOCATE) // GPU allocated thin wrapped
          // ... then sync to ensure data is up to date
@@ -464,6 +484,7 @@ namespace care {
             }
          }
 #endif
+#endif /* defined(CARE_DEEP_COPY_RAW_PTR) */
       }
 
       CARE_HOST_DEVICE void pick(int idx, T_non_const& val) const  {