Merge CMSSW_10_6_X into CMSSW_10_6_DEVEL_X.

Configuration/PyReleaseValidation/python/MatrixReader.py

@@ -234,6 +234,8 @@ def readMatrix(self, fileNameIn, useInput=None, refRel=None, fromScratch=None):
 
             for (stepI,step) in enumerate(stepList):
                 stepName=step
+                if self.relvalModule.steps[stepName] is None:
+                    continue
                 if self.wm:
                     #cannot put a certain number of things in wm
                     if stepName in [

Configuration/PyReleaseValidation/python/relval_2023.py

@@ -22,7 +22,7 @@
 numWFIB.extend([22034.0]) #2023D23
 numWFIB.extend([22434.0]) #2023D24
 numWFIB.extend([24034.0]) #2023D28
-numWFIB.extend([27434.0]) #2023D35
+numWFIB.extend([27434.0,27434.21,27634.21]) #2023D35, prodlike, prodlike PU
 numWFIB.extend([23234.0]) #2023D36
 numWFIB.extend([23634.0]) #2023D37
 numWFIB.extend([27834.0]) #2023D38

Configuration/PyReleaseValidation/python/relval_steps.py

@@ -2945,11 +2945,12 @@ def gen2018HiMix(fragment,howMuch):
 for ds in defaultDataSets:
     key='MinBias_14TeV_pythia8_TuneCUETP8M1'+'_'+ds
     name=baseDataSetReleaseBetter[key]
-#    if '2017' in name or '2018' in name:
     if '2017' in name:
     	PUDataSets[ds]={'-n':10,'--pileup':'AVE_35_BX_25ns','--pileup_input':'das:/RelValMinBias_13/%s/GEN-SIM'%(name,)}
     elif '2018' in name:
     	PUDataSets[ds]={'-n':10,'--pileup':'AVE_50_BX_25ns','--pileup_input':'das:/RelValMinBias_13/%s/GEN-SIM'%(name,)}
+    elif '2023' in name:
+    	PUDataSets[ds]={'-n':10,'--pileup':'AVE_200_BX_25ns','--pileup_input':'das:/RelValMinBias_14TeV/%s/GEN-SIM'%(name,)}
     else:
         PUDataSets[ds]={'-n':10,'--pileup':'AVE_35_BX_25ns','--pileup_input':'das:/RelValMinBias_14TeV/%s/GEN-SIM'%(name,)}
 
@@ -3054,6 +3055,15 @@ def gen2018HiMix(fragment,howMuch):
                                       '--geometry' : geom
                                       }
 
+    upgradeStepDict['MiniAODFullGlobal'][k] = {'-s':'PAT',
+                                      '--conditions':gt,
+                                      '--datatier':'MINIAODSIM',
+                                      '-n':'10',
+                                      '--runUnscheduled':'',
+                                      '--eventcontent':'MINIAODSIM',
+                                      '--geometry' : geom
+                                      }
+
     upgradeStepDict['HARVESTFull'][k]={'-s':'HARVESTING:@standardValidation+@standardDQM+@ExtraHLT+@miniAODValidation+@miniAODDQM',
                                     '--conditions':gt,
                                     '--mc':'',
@@ -3158,6 +3168,17 @@ def gen2018HiMix(fragment,howMuch):
         stepName = step + upgradeSteps['killStuckTBM']['suffix']
         upgradeStepDict[stepName][k] = merge([{'--customise': 'SimTracker/SiPixelDigitizer/customiseStuckTBMSimulation.activateStuckTBMSimulation2018NoPU'}, upgradeStepDict[step][k]])
 
+    for step in upgradeSteps['ProdLike']['steps']:
+        stepName = step + upgradeSteps['ProdLike']['suffix']
+        if 'Reco' in step:
+            upgradeStepDict[stepName][k] = merge([{'-s': 'RAW2DIGI,L1Reco,RECO,RECOSIM', '--datatier':'GEN-SIM-RECO', '--eventcontent':'FEVTDEBUGHLT'}, upgradeStepDict[step][k]])
+        elif 'MiniAOD' in step:
+            # the separate miniAOD step is used here
+            upgradeStepDict[stepName][k] = deepcopy(upgradeStepDict[step][k])
+        if 'HARVEST' in step:
+            # remove step
+            upgradeStepDict[stepName][k] = None
+
     # setup PU
     if k2 in PUDataSets:
         # Setup premixing stage1
@@ -3186,7 +3207,10 @@ def gen2018HiMix(fragment,howMuch):
             for step in upgradeSteps[stepType]['PU']:
                 stepName = step + upgradeSteps[stepType]['suffix']
                 stepNamePU = step + 'PU' + upgradeSteps[stepType]['suffix']
-                upgradeStepDict[stepNamePU][k]=merge([PUDataSets[k2],upgradeStepDict[stepName][k]])
+                if upgradeStepDict[stepName][k] is None:
+                    upgradeStepDict[stepNamePU][k] = None
+                else:
+                    upgradeStepDict[stepNamePU][k]=merge([PUDataSets[k2],upgradeStepDict[stepName][k]])
 
                 # Setup premixing stage2
                 if "Digi" in step or "Reco" in step:
@@ -3232,5 +3256,8 @@ def gen2018HiMix(fragment,howMuch):
         for key in [key for year in upgradeKeys for key in upgradeKeys[year]]:
             k=step+'_'+key
             if step in upgradeStepDict and key in upgradeStepDict[step]:
-                steps[k]=merge([upgradeStepDict[step][key]])
+                if upgradeStepDict[step][key] is None:
+                    steps[k]=None
+                else:
+                    steps[k]=merge([upgradeStepDict[step][key]])
 

Configuration/PyReleaseValidation/python/relval_upgrade.py

@@ -60,6 +60,9 @@ def makeStepName(key,frag,step,suffix):
                         stepList[stepType].append(stepMaker(key,"SingleNuE10_cf",s,upgradeSteps[stepType]['suffix']))
                     elif (stepType is not 'baseline') and ( ('PU' in step and step.replace('PU','') in upgradeSteps[stepType]['PU']) or (step in upgradeSteps[stepType]['steps']) ):
                         stepList[stepType].append(stepMaker(key,frag[:-4],step,upgradeSteps[stepType]['suffix']))
+                        # hack to add an extra step
+                        if stepType == 'ProdLike' and 'RecoFullGlobal' in step:
+                            stepList[stepType].append(stepMaker(key,frag[:-4],step.replace('RecoFullGlobal','MiniAODFullGlobal'),upgradeSteps[stepType]['suffix']))
                     else:
                         stepList[stepType].append(stepMaker(key,frag[:-4],step,upgradeSteps['baseline']['suffix']))
 
@@ -94,6 +97,10 @@ def makeStepName(key,frag,step,suffix):
             if any(upgradeDatasetFromFragment[frag]==nfrag for nfrag in tbmFrags) and '2018' in key:
                 workflows[numWF+upgradeSteps['killStuckTBM']['offset']] = [ upgradeDatasetFromFragment[frag], stepList['killStuckTBM']]
 
+            # workflow for profiling
+            if upgradeDatasetFromFragment[frag]=="TTbar_14TeV" and '2023' in key:
+                workflows[numWF+upgradeSteps['ProdLike']['offset']] = [ upgradeDatasetFromFragment[frag]+"_ProdLike", stepList['ProdLike']]
+
             # premixing stage1, only for NuGun
             if upgradeDatasetFromFragment[frag]=="NuGun" and 'PU' in key and '2023' in key:
                 workflows[numWF+upgradeSteps['Premix']['offset']] = [upgradeDatasetFromFragment[frag], stepList['Premix']]

Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py

@@ -98,6 +98,7 @@
         'HARVESTFullGlobal',
         'ALCAFull',
         'NanoFull',
+        'MiniAODFullGlobal',
     ],
     'PU' : [
         'DigiFullTrigger',
@@ -107,6 +108,7 @@
         'RecoFull',
         'HARVESTFull',
         'HARVESTFullGlobal',
+        'MiniAODFullGlobal',
     ],
     'suffix' : '',
     'offset' : 0.0,
@@ -164,6 +166,20 @@
     'suffix' : '_Timing',
     'offset' : 0.11,
 }
+upgradeSteps['ProdLike'] = {
+    'steps' : [
+        'RecoFullGlobal',
+        'HARVESTFullGlobal',
+        'MiniAODFullGlobal',
+    ],
+    'PU' : [
+        'RecoFullGlobal',
+        'HARVESTFullGlobal',
+        'MiniAODFullGlobal',
+    ],
+    'suffix' : '_ProdLike',
+    'offset' : 0.21,
+}
 upgradeSteps['Neutron'] = {
     'steps' : [
         'GenSimFull',