First heterogeneous CLUE workflow.

Configuration/ProcessModifiers/python/heterogeneousCLUE_cff.py

@@ -0,0 +1,5 @@
+import FWCore.ParameterSet.Config as cms
+
+# This modifier runs the heterogeneous version of CLUE on the EE part of HGCAL
+# in  the HLT Phase2 Simplified menu. All seeded HLT Paths will keep on using the regular CPU version.
+heterogeneousCLUE = cms.Modifier()

DataFormats/HGCalReco/BuildFile.xml

@@ -5,6 +5,10 @@
 <use name="DataFormats/TrackReco"/>
 <use name="DataFormats/GeometryVector"/>
 <use name="eigen"/>
+<use name="DataFormats/Portable"/>
+<use name="DataFormats/SoATemplate"/>
+<use name="HeterogeneousCore/AlpakaInterface"/>
+<flags ALPAKA_BACKENDS="cuda rocm"/>
 <export>
   <lib name="1"/>
 </export>

DataFormats/HGCalReco/interface/HGCalSoACells.h

@@ -0,0 +1,27 @@
+#ifndef DataFormats_HGCalReco_interface_HGCalSoACells_h
+#define DataFormats_HGCalReco_interface_HGCalSoACells_h
+
+#include <Eigen/Core>
+#include <Eigen/Dense>
+
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+#include "DataFormats/SoATemplate/interface/SoAView.h"
+
+// SoA layout with dim1, dim2, weight, sigmaNoise, recHitsIndex layer and cellsCount fields
+GENERATE_SOA_LAYOUT(HGCalSoACellsLayout,
+                    // columns: one value per element
+                    SOA_COLUMN(float, dim1),
+                    SOA_COLUMN(float, dim2),
+                    SOA_COLUMN(float, dim3),
+                    SOA_COLUMN(int, layer),
+                    SOA_COLUMN(float, weight),
+                    SOA_COLUMN(float, sigmaNoise),
+                    SOA_COLUMN(unsigned int, recHitIndex),
+                    SOA_COLUMN(uint32_t, detid),
+                    SOA_COLUMN(float, time),
+                    SOA_COLUMN(float, timeError))
+
+using HGCalSoACells = HGCalSoACellsLayout<>;
+
+#endif  // DataFormats_PortableTestObjects_interface_TestSoA_h

DataFormats/HGCalReco/interface/HGCalSoACellsHostCollection.h

@@ -0,0 +1,10 @@
+#ifndef DataFormats_HGCalReco_interface_HGCalSoACellsHostCollection_h
+#define DataFormats_HGCalReco_interface_HGCalSoACellsHostCollection_h
+
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACells.h"
+
+// SoA with x, y, z, id fields in host memory
+using HGCalSoACellsHostCollection = PortableHostCollection<HGCalSoACells>;
+
+#endif  // DataFormats_HGCalReco_interface_HGCalSoACellsHostCollection_h

DataFormats/HGCalReco/interface/HGCalSoACellsOut.h

@@ -0,0 +1,20 @@
+#ifndef DataFormats_HGCalReco_interface_HGCalSoACellsOut_h
+#define DataFormats_HGCalReco_interface_HGCalSoACellsOut_h
+
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+#include "DataFormats/SoATemplate/interface/SoAView.h"
+
+// SoA layout with delta, rho, weight, nearestHigher, clusterIndex, layer, isSeed, and cellsCount fields
+GENERATE_SOA_LAYOUT(HGCalSoACellsOutLayout,
+                    // columns: one value per element
+                    SOA_COLUMN(float, delta),
+                    SOA_COLUMN(float, rho),
+                    SOA_COLUMN(unsigned int, nearestHigher),
+                    SOA_COLUMN(int, clusterIndex),
+                    SOA_COLUMN(uint8_t, isSeed),
+                    SOA_SCALAR(unsigned int, numberOfClustersScalar))
+
+using HGCalSoACellsOut = HGCalSoACellsOutLayout<>;
+
+#endif

DataFormats/HGCalReco/interface/HGCalSoACellsOutHostCollection.h

@@ -0,0 +1,10 @@
+#ifndef DataFormats_HGCalReco_interface_HGCalSoACellsOutHostCollection_h
+#define DataFormats_HGCalReco_interface_HGCalSoACellsOutHostCollection_h
+
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACellsOut.h"
+
+// SoA with delta, rho, weight, nearestHigher, clusterIndex, layer, isSeed, and cellsCount fields in host memory
+using HGCalSoACellsOutHostCollection = PortableHostCollection<HGCalSoACellsOut>;
+
+#endif  // DataFormats_HGCalReco_interface_HGCalSoACellsOutHostCollection_h

DataFormats/HGCalReco/interface/HGCalSoAClusters.h

@@ -0,0 +1,20 @@
+#ifndef DataFormats_HGCalReco_interface_HGCalSoAClusters_h
+#define DataFormats_HGCalReco_interface_HGCalSoAClusters_h
+
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+#include "DataFormats/SoATemplate/interface/SoAView.h"
+
+GENERATE_SOA_LAYOUT(HGCalSoAClustersLayout,
+                    // columns: one value per element
+                    SOA_COLUMN(float, x),
+                    SOA_COLUMN(float, y),
+                    SOA_COLUMN(float, z),
+                    SOA_COLUMN(float, energy),
+                    SOA_COLUMN(int, cells),  // number of hits in the cluster
+                    SOA_COLUMN(int, seed)    // This is the index of the seed of each cluster inside the RecHit SoA
+)
+
+using HGCalSoAClusters = HGCalSoAClustersLayout<>;
+
+#endif

DataFormats/HGCalReco/interface/HGCalSoAClustersHostCollection.h

@@ -0,0 +1,9 @@
+#ifndef DataFormats_HGCalReco_interface_HGCalSoAClustersHostCollection_h
+#define DataFormats_HGCalReco_interface_HGCalSoAClustersHostCollection_h
+
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoAClusters.h"
+
+using HGCalSoAClustersHostCollection = PortableHostCollection<HGCalSoAClusters>;
+
+#endif  // DataFormats_HGCalReco_interface_HGCalSoAClustersHostCollection_h

DataFormats/HGCalReco/interface/HGCalSoAClustersService.h

@@ -0,0 +1,19 @@
+#ifndef DataFormats_HGCalReco_interface_HGCalSoAClustersService_h
+#define DataFormats_HGCalReco_interface_HGCalSoAClustersService_h
+
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+#include "DataFormats/SoATemplate/interface/SoAView.h"
+
+GENERATE_SOA_LAYOUT(
+    HGCalSoAClustersServiceLayout,
+    // columns: one value per element
+    SOA_COLUMN(float, total_weight),
+    SOA_COLUMN(float, total_weight_log),
+    SOA_COLUMN(float, maxEnergyValue),
+    SOA_COLUMN(int, maxEnergyIndex)  // Index in the RecHitSoA of the rechit with highest energy in each cluster
+)
+
+using HGCalSoAClustersService = HGCalSoAClustersServiceLayout<>;
+
+#endif

DataFormats/HGCalReco/interface/HGCalSoAClustersServiceHostCollection.h

@@ -0,0 +1,9 @@
+#ifndef DataFormats_HGCalReco_interface_HGCalSoAClustersServiceHostCollection_h
+#define DataFormats_HGCalReco_interface_HGCalSoAClustersServiceHostCollection_h
+
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoAClustersService.h"
+
+using HGCalSoAClustersServiceHostCollection = PortableHostCollection<HGCalSoAClustersService>;
+
+#endif  // DataFormats_HGCalReco_interface_HGCalSoAClustersServiceHostCollection_h

DataFormats/HGCalReco/interface/alpaka/HGCalSoACellsDeviceCollection.h

@@ -0,0 +1,15 @@
+#ifndef DataFormats_PortableTestObjects_interface_alpaka_HGCalSoACellsDeviceCollection_h
+#define DataFormats_PortableTestObjects_interface_alpaka_HGCalSoACellsDeviceCollection_h
+
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACells.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  // SoA with x, y, z, id fields in device global memory
+  using HGCalSoACellsDeviceCollection = PortableCollection<HGCalSoACells>;
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#endif  // DataFormats_PortableTestObjects_interface_alpaka_HGCalSoACellsDeviceCollection_h

DataFormats/HGCalReco/interface/alpaka/HGCalSoACellsOutDeviceCollection.h

@@ -0,0 +1,15 @@
+#ifndef DataFormats_PortableTestObjects_interface_alpaka_HGCalSoACellsOutDeviceCollection_h
+#define DataFormats_PortableTestObjects_interface_alpaka_HGCalSoACellsOutDeviceCollection_h
+
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACellsOut.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  // SoA with delta, rho, weight, nearestHigher, clusterIndex, layer, isSeed, and cellsCount fields in device global memory
+  using HGCalSoACellsOutDeviceCollection = PortableCollection<HGCalSoACellsOut>;
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#endif  // DataFormats_PortableTestObjects_interface_alpaka_HGCalSoACellsOutDeviceCollection_h

DataFormats/HGCalReco/interface/alpaka/HGCalSoAClustersDeviceCollection.h

@@ -0,0 +1,14 @@
+#ifndef DataFormats_PortableTestObjects_interface_alpaka_HGCalSoAClustersDeviceCollection_h
+#define DataFormats_PortableTestObjects_interface_alpaka_HGCalSoAClustersDeviceCollection_h
+
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoAClusters.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  using HGCalSoAClustersDeviceCollection = PortableCollection<HGCalSoAClusters>;
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#endif  // DataFormats_PortableTestObjects_interface_alpaka_HGCalSoAClustersDeviceCollection_h

DataFormats/HGCalReco/interface/alpaka/HGCalSoAClustersServiceDeviceCollection.h

@@ -0,0 +1,14 @@
+#ifndef DataFormats_PortableTestObjects_interface_alpaka_HGCalSoAClustersServiceDeviceCollection_h
+#define DataFormats_PortableTestObjects_interface_alpaka_HGCalSoAClustersServiceDeviceCollection_h
+
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoAClustersService.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  using HGCalSoAClustersServiceDeviceCollection = PortableCollection<HGCalSoAClustersService>;
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#endif  // DataFormats_PortableTestObjects_interface_alpaka_HGCalSoAClustersServiceDeviceCollection_h

DataFormats/HGCalReco/src/alpaka/classes_cuda.h

@@ -0,0 +1,7 @@
+#include "DataFormats/Common/interface/DeviceProduct.h"
+#include "DataFormats/Common/interface/Wrapper.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACells.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACellsOut.h"
+#include "DataFormats/HGCalReco/interface/alpaka/HGCalSoACellsDeviceCollection.h"
+#include "DataFormats/HGCalReco/interface/alpaka/HGCalSoACellsOutDeviceCollection.h"
+#include "DataFormats/HGCalReco/interface/alpaka/HGCalSoAClustersDeviceCollection.h"

DataFormats/HGCalReco/src/alpaka/classes_cuda_def.xml

@@ -0,0 +1,13 @@
+<lcgdict>
+  <class name="alpaka_cuda_async::HGCalSoACellsDeviceCollection" persistent="false"/>
+  <class name="edm::DeviceProduct<alpaka_cuda_async::HGCalSoACellsDeviceCollection>" persistent="false"/>
+  <class name="edm::Wrapper<edm::DeviceProduct<alpaka_cuda_async::HGCalSoACellsDeviceCollection>>" persistent="false"/>
+
+  <class name="alpaka_cuda_async::HGCalSoACellsOutDeviceCollection" persistent="false"/>
+  <class name="edm::DeviceProduct<alpaka_cuda_async::HGCalSoACellsOutDeviceCollection>" persistent="false"/>
+  <class name="edm::Wrapper<edm::DeviceProduct<alpaka_cuda_async::HGCalSoACellsOutDeviceCollection>>" persistent="false"/>
+
+  <class name="alpaka_cuda_async::HGCalSoAClustersDeviceCollection" persistent="false"/>
+  <class name="edm::DeviceProduct<alpaka_cuda_async::HGCalSoAClustersDeviceCollection>" persistent="false"/>
+  <class name="edm::Wrapper<edm::DeviceProduct<alpaka_cuda_async::HGCalSoAClustersDeviceCollection>>" persistent="false"/>
+</lcgdict>

DataFormats/HGCalReco/src/alpaka/classes_rocm.h

@@ -0,0 +1,7 @@
+#include "DataFormats/Common/interface/DeviceProduct.h"
+#include "DataFormats/Common/interface/Wrapper.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACells.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACellsOut.h"
+#include "DataFormats/HGCalReco/interface/alpaka/HGCalSoACellsDeviceCollection.h"
+#include "DataFormats/HGCalReco/interface/alpaka/HGCalSoACellsOutDeviceCollection.h"
+#include "DataFormats/HGCalReco/interface/alpaka/HGCalSoAClustersDeviceCollection.h"

DataFormats/HGCalReco/src/alpaka/classes_rocm_def.xml

@@ -0,0 +1,13 @@
+<lcgdict>
+  <class name="alpaka_rocm_async::HGCalSoACellsDeviceCollection" persistent="false"/>
+  <class name="edm::DeviceProduct<alpaka_rocm_async::HGCalSoACellsDeviceCollection>" persistent="false"/>
+  <class name="edm::Wrapper<edm::DeviceProduct<alpaka_rocm_async::HGCalSoACellsDeviceCollection>>" persistent="false"/>
+
+  <class name="alpaka_rocm_async::HGCalSoACellsOutDeviceCollection" persistent="false"/>
+  <class name="edm::DeviceProduct<alpaka_rocm_async::HGCalSoACellsOutDeviceCollection>" persistent="false"/>
+  <class name="edm::Wrapper<edm::DeviceProduct<alpaka_rocm_async::HGCalSoACellsOutDeviceCollection>>" persistent="false"/>
+
+  <class name="alpaka_rocm_async::HGCalSoAClustersDeviceCollection" persistent="false"/>
+  <class name="edm::DeviceProduct<alpaka_rocm_async::HGCalSoAClustersDeviceCollection>" persistent="false"/>
+  <class name="edm::Wrapper<edm::DeviceProduct<alpaka_rocm_async::HGCalSoAClustersDeviceCollection>>" persistent="false"/>
+</lcgdict>

DataFormats/HGCalReco/src/classes.h

@@ -5,3 +5,9 @@
 #include "DataFormats/HGCalReco/interface/TICLSeedingRegion.h"
 #include "DataFormats/HGCalReco/interface/TICLCandidate.h"
 #include "DataFormats/Common/interface/Wrapper.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACellsHostCollection.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACells.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACellsOut.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoACellsOutHostCollection.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoAClusters.h"
+#include "DataFormats/HGCalReco/interface/HGCalSoAClustersHostCollection.h"

DataFormats/HGCalReco/src/classes_def.xml

@@ -55,4 +55,58 @@
   <class name="std::vector<TICLCandidate>" />
   <class name="edm::Wrapper<TICLCandidate>" />
   <class name="edm::Wrapper<std::vector<TICLCandidate> >" />
+  <class  name="HGCalSoACells"/>
+
+  <class name="HGCalSoACellsHostCollection"/>
+  <!-- Read streamer declaration for dictionary. It ensures allocation of buffer and data copying to it -->
+  <read
+    sourceClass="HGCalSoACellsHostCollection"
+    targetClass="HGCalSoACellsHostCollection"
+    version="[1-]"
+    source="HGCalSoACells layout_;"
+    target="buffer_,layout_,view_"
+    embed="false">
+  <![CDATA[
+    HGCalSoACellsHostCollection::ROOTReadStreamer(newObj, onfile.layout_);
+  ]]>
+  </read>
+  <class name="edm::Wrapper<HGCalSoACellsHostCollection>" splitLevel="0"/>
+
+  <class name="HGCalSoACellsOut"/>
+
+  <!-- Collection declaration for dictionary -->
+  <class name="HGCalSoACellsOutHostCollection"/>
+
+  <!-- Read streamer declaration for dictionary. It ensures allocation of buffer and data copying to it -->
+  <read
+    sourceClass="HGCalSoACellsOutHostCollection"
+    targetClass="HGCalSoACellsOutHostCollection"
+    version="[1-]"
+    source="HGCalSoACellsOut layout_;"
+    target="buffer_,layout_,view_"
+    embed="false">
+  <![CDATA[
+    HGCalSoACellsOutHostCollection::ROOTReadStreamer(newObj, onfile.layout_);
+  ]]>
+  </read>
+
+  <class name="edm::Wrapper<HGCalSoACellsOutHostCollection>" splitLevel="0"/>
+
+  <class  name="HGCalSoAClusters"/>
+
+  <class name="HGCalSoAClustersHostCollection"/>
+  <!-- Read streamer declaration for dictionary. It ensures allocation of buffer and data copying to it -->
+  <read
+    sourceClass="HGCalSoAClustersHostCollection"
+    targetClass="HGCalSoAClustersHostCollection"
+    version="[1-]"
+    source="HGCalSoAClusters layout_;"
+    target="buffer_,layout_,view_"
+    embed="false">
+  <![CDATA[
+    HGCalSoAClustersHostCollection::ROOTReadStreamer(newObj, onfile.layout_);
+  ]]>
+  </read>
+  <class name="edm::Wrapper<HGCalSoAClustersHostCollection>" splitLevel="0"/>
+
 </lcgdict>

DataFormats/Portable/scripts/portableHostCollectionHints

@@ -29,5 +29,25 @@ if len(layouts) > 1:
   print("")
 print("  <!-- Collection declaration for dictionary -->")
 print("  <class name=\"%s\"/>"% collectionName)
+print()
+print("  <!-- Read streamer declaration for dictionary. It ensures allocation of buffer and data copying to it -->")
+print("  <read")
+print("    sourceClass=\"%s\""% collectionName)
+print("    targetClass=\"%s\""% collectionName)
+print("    version=\"[1-]\"")
+if len(layouts) < 2:
+  print("    source=\"%s layout_;\""% layouts[0])
+  print("    target=\"buffer_,layout_,view_\"")
+else:
+  print("    source=\"%s::Implementation impl_;\""% collectionName)
+  print("    target=\"buffer_,impl_\"")
+print("    embed=\"false\">")
+print("  <![CDATA[")
+if len(layouts) < 2:
+  print("    %s::ROOTReadStreamer(newObj, onfile.layout_);"% collectionName)
+else:
+  print("    %s::ROOTReadStreamer(newObj, onfile.impl_);"% collectionName)
+print("  ]]>")
+print("  </read>\n")
 print("  <class name=\"edm::Wrapper<%s>\" splitLevel=\"0\"/>"% collectionName)
-print("</lcgdict>")
+print("</lcgdict>")

HLTrigger/Configuration/python/HLT_75e33/modules/hgcalLayerClustersFromSoAProducer_cfi.py

@@ -0,0 +1,14 @@
+import FWCore.ParameterSet.Config as cms
+from ..psets.hgcal_reco_constants_cfi import HGCAL_reco_constants as HGCAL_reco_constants
+
+hgCalLayerClustersFromSoAProducer = cms.EDProducer("HGCalLayerClustersFromSoAProducer",
+    detector = cms.string('EE'),
+    hgcalRecHitsLayerClustersSoA = cms.InputTag("hgCalSoARecHitsLayerClustersProducer"),
+    hgcalRecHitsSoA = cms.InputTag("hgCalSoARecHitsProducer"),
+    mightGet = cms.optional.untracked.vstring,
+    nHitsTime = cms.uint32(3),
+    srcDevice = cms.InputTag("hgCalSoALayerClustersProducer"),
+    timeClname = cms.string('timeLayerCluster')
+)
+
+

HLTrigger/Configuration/python/HLT_75e33/modules/hgcalSoALayerClustersProducer_cfi.py

@@ -0,0 +1,13 @@
+import FWCore.ParameterSet.Config as cms
+from ..psets.hgcal_reco_constants_cfi import HGCAL_reco_constants as HGCAL_reco_constants
+
+hgCalSoALayerClustersProducer = cms.EDProducer("HGCalSoALayerClustersProducer@alpaka",
+    alpaka = cms.untracked.PSet(
+        backend = cms.untracked.string('cuda_async')
+    ),
+    hgcalRecHitsLayerClustersSoA = cms.InputTag("hgCalSoARecHitsLayerClustersProducer"),
+    hgcalRecHitsSoA = cms.InputTag("hgCalSoARecHitsProducer"),
+    mightGet = cms.optional.untracked.vstring,
+    positionDeltaRho2 = cms.double(1.69),
+    thresholdW0 = cms.double(2.9)
+)

HLTrigger/Configuration/python/HLT_75e33/modules/hgcalSoARecHitsLayerClustersProducer_cfi.py

@@ -0,0 +1,10 @@
+import FWCore.ParameterSet.Config as cms
+from ..psets.hgcal_reco_constants_cfi import HGCAL_reco_constants as HGCAL_reco_constants
+
+hgCalSoARecHitsLayerClustersProducer = cms.EDProducer("HGCalSoARecHitsLayerClustersProducer@alpaka",
+    alpaka = cms.untracked.PSet(
+        backend = cms.untracked.string('cuda_async')
+    ),
+    hgcalRecHitsSoA = cms.InputTag("hgCalSoARecHitsProducer"),
+    mightGet = cms.optional.untracked.vstring
+)