rucio aso testsuite tasks (#8091)

test/statusTrackingTasks/psetManyEDM.py

@@ -0,0 +1,27 @@
+from __future__ import division
+import FWCore.ParameterSet.Config as cms
+
+process = cms.Process('NoSplit')
+
+process.source = cms.Source("PoolSource", fileNames = cms.untracked.vstring('root://cms-xrd-global.cern.ch///store/mc/HC/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/00000/8ADD04E5-1776-E711-A1BA-FA163E6741E0.root'))
+process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(10))
+process.options = cms.untracked.PSet(wantSummary = cms.untracked.bool(True))
+
+process.output = cms.OutputModule("PoolOutputModule",
+    outputCommands = cms.untracked.vstring("drop *", "keep recoTracks_globalMuons_*_*"),
+    fileName = cms.untracked.string('output.root'),
+    dataset = cms.untracked.PSet(
+        filterName = cms.untracked.string('OutOne'),
+    )
+)
+process.out1 = cms.EndPath(process.output)
+
+process.secondoutput = cms.OutputModule("PoolOutputModule",
+    outputCommands = cms.untracked.vstring("drop *", "keep recoTracks_globalMuons_*_*"),
+    fileName = cms.untracked.string('secondoutput.root'),
+    dataset = cms.untracked.PSet(
+        filterName = cms.untracked.string('OutTwo'),
+    )
+
+)
+process.out2 = cms.EndPath(process.secondoutput)

test/statusTrackingTasks/rucio_transfers.py

@@ -0,0 +1,48 @@
+from __future__ import division
+import time
+from WMCore.Configuration import Configuration
+import os
+import datetime
+from CRABClient.UserUtilities import getUsernameFromCRIC
+
+now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+filename_nopy =  __file__.split('/')[-1][:-3]
+
+config = Configuration()
+
+config.section_("General")
+config.General.instance = os.getenv('REST_Instance','test12')
+#config.General.restHost = ''
+#config.General.dbInstance = ''
+config.General.requestName = filename_nopy + '_' + now_str
+config.General.workArea = '/tmp/crabStatusTracking'
+config.General.transferLogs=False
+
+config.section_("JobType")
+config.JobType.pluginName = 'Analysis'
+config.JobType.psetName = 'pset.py'
+
+config.section_("Data")
+config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM')
+
+config.Data.splitting = 'LumiBased'
+config.Data.unitsPerJob = 1
+config.JobType.maxJobRuntimeMin = 60
+config.Data.totalUnits = 40
+
+config.Data.publication = True
+testName = "ruciotransfers-%d" % int(time.time())
+#testName = 'ruciotransfers-1'
+config.Data.outputDatasetTag = testName
+# rucio
+config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('user', getUsernameFromCRIC(), testName,)
+
+config.section_("User")
+
+config.section_("Site")
+config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*']
+#config.Site.blacklist = ['T2_ES_IFCA']
+config.Site.storageSite = 'T2_CH_CERN'
+
+config.section_("Debug")
+config.Debug.scheddName = 'crab3@vocms059.cern.ch'

test/statusTrackingTasks/rucio_transfers_group.py

@@ -0,0 +1,48 @@
+from __future__ import division
+import time
+from WMCore.Configuration import Configuration
+import os
+import datetime
+from CRABClient.UserUtilities import getUsernameFromCRIC
+
+now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+filename_nopy =  __file__.split('/')[-1][:-3]
+
+config = Configuration()
+
+config.section_("General")
+config.General.instance = os.getenv('REST_Instance','test12')
+#config.General.restHost = ''
+#config.General.dbInstance = ''
+config.General.requestName = filename_nopy + '_' + now_str
+config.General.workArea = '/tmp/crabStatusTracking'
+config.General.transferLogs=False
+
+config.section_("JobType")
+config.JobType.pluginName = 'Analysis'
+config.JobType.psetName = 'pset.py'
+
+config.section_("Data")
+config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM')
+
+config.Data.splitting = 'LumiBased'
+config.Data.unitsPerJob = 1
+config.JobType.maxJobRuntimeMin = 60
+config.Data.totalUnits = 40
+
+config.Data.publication = True
+testName = "ruciotransfers-%d" % int(time.time())
+#testName = 'ruciotransfers-1'
+config.Data.outputDatasetTag = testName
+# rucio
+config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('group', 'crab_test', testName,)
+
+config.section_("User")
+
+config.section_("Site")
+config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*']
+#config.Site.blacklist = ['T2_ES_IFCA']
+config.Site.storageSite = 'T2_IT_Legnaro'
+
+config.section_("Debug")
+config.Debug.scheddName = 'crab3@vocms059.cern.ch'

test/statusTrackingTasks/rucio_transfers_manyedm_nopublication.py

@@ -0,0 +1,48 @@
+from __future__ import division
+import time
+from WMCore.Configuration import Configuration
+import os
+import datetime
+from CRABClient.UserUtilities import getUsernameFromCRIC
+
+now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+filename_nopy =  __file__.split('/')[-1][:-3]
+
+config = Configuration()
+
+config.section_("General")
+config.General.instance = os.getenv('REST_Instance','test12')
+#config.General.restHost = ''
+#config.General.dbInstance = ''
+config.General.requestName = filename_nopy + '_' + now_str
+config.General.workArea = '/tmp/crabStatusTracking'
+config.General.transferLogs=False
+
+config.section_("JobType")
+config.JobType.pluginName = 'Analysis'
+config.JobType.psetName = 'psetManyEDM.py'
+
+config.section_("Data")
+config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM')
+
+config.Data.splitting = 'LumiBased'
+config.Data.unitsPerJob = 1
+config.JobType.maxJobRuntimeMin = 60
+config.Data.totalUnits = 40
+
+config.Data.publication = False
+testName = "ruciotransfers-%d" % int(time.time())
+#testName = 'ruciotransfers-1'
+config.Data.outputDatasetTag = testName
+# rucio
+config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('user', getUsernameFromCRIC(), testName,)
+
+config.section_("User")
+
+config.section_("Site")
+config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*']
+#config.Site.blacklist = ['T2_ES_IFCA']
+config.Site.storageSite = 'T2_CH_CERN'
+
+config.section_("Debug")
+config.Debug.scheddName = 'crab3@vocms059.cern.ch'

test/statusTrackingTasks/rucio_transfers_nopublication.py

@@ -0,0 +1,48 @@
+from __future__ import division
+import time
+from WMCore.Configuration import Configuration
+import os
+import datetime
+from CRABClient.UserUtilities import getUsernameFromCRIC
+
+now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+filename_nopy =  __file__.split('/')[-1][:-3]
+
+config = Configuration()
+
+config.section_("General")
+config.General.instance = os.getenv('REST_Instance','test12')
+#config.General.restHost = ''
+#config.General.dbInstance = ''
+config.General.requestName = filename_nopy + '_' + now_str
+config.General.workArea = '/tmp/crabStatusTracking'
+config.General.transferLogs=False
+
+config.section_("JobType")
+config.JobType.pluginName = 'Analysis'
+config.JobType.psetName = 'pset.py'
+
+config.section_("Data")
+config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM')
+
+config.Data.splitting = 'LumiBased'
+config.Data.unitsPerJob = 1
+config.JobType.maxJobRuntimeMin = 60
+config.Data.totalUnits = 40
+
+config.Data.publication = False
+testName = "ruciotransfers-%d" % int(time.time())
+#testName = 'ruciotransfers-1'
+config.Data.outputDatasetTag = testName
+# rucio
+config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('user', getUsernameFromCRIC(), testName,)
+
+config.section_("User")
+
+config.section_("Site")
+config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*']
+#config.Site.blacklist = ['T2_ES_IFCA']
+config.Site.storageSite = 'T2_CH_CERN'
+
+config.section_("Debug")
+config.Debug.scheddName = 'crab3@vocms059.cern.ch'

test/statusTrackingTasks/rucio_transfers_withnoneedm.py

@@ -0,0 +1,50 @@
+from __future__ import division
+import time
+from WMCore.Configuration import Configuration
+import os
+import datetime
+from CRABClient.UserUtilities import getUsernameFromCRIC
+
+now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+filename_nopy =  __file__.split('/')[-1][:-3]
+
+config = Configuration()
+
+config.section_("General")
+config.General.instance = os.getenv('REST_Instance','test12')
+#config.General.restHost = ''
+#config.General.dbInstance = ''
+config.General.requestName = filename_nopy + '_' + now_str
+config.General.workArea = '/tmp/crabStatusTracking'
+config.General.transferLogs=False
+
+config.section_("JobType")
+config.JobType.pluginName = 'Analysis'
+config.JobType.psetName = 'pset.py'
+config.JobType.scriptExe = 'withNoneEDM.sh'
+config.JobType.outputFiles = ['miniaodfake.root']
+
+config.section_("Data")
+config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM')
+
+config.Data.splitting = 'LumiBased'
+config.Data.unitsPerJob = 1
+config.JobType.maxJobRuntimeMin = 60
+config.Data.totalUnits = 40
+
+config.Data.publication = True
+testName = "ruciotransfers-%d" % int(time.time())
+#testName = 'ruciotransfers-1'
+config.Data.outputDatasetTag = testName
+# rucio
+config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('user', getUsernameFromCRIC(), testName,)
+
+config.section_("User")
+
+config.section_("Site")
+config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*']
+#config.Site.blacklist = ['T2_ES_IFCA']
+config.Site.storageSite = 'T2_CH_CERN'
+
+config.section_("Debug")
+config.Debug.scheddName = 'crab3@vocms059.cern.ch'

test/statusTrackingTasks/rucio_transfers_withnoneedm_nopublication.py

@@ -0,0 +1,50 @@
+from __future__ import division
+import time
+from WMCore.Configuration import Configuration
+import os
+import datetime
+from CRABClient.UserUtilities import getUsernameFromCRIC
+
+now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+filename_nopy =  __file__.split('/')[-1][:-3]
+
+config = Configuration()
+
+config.section_("General")
+config.General.instance = os.getenv('REST_Instance','test12')
+#config.General.restHost = ''
+#config.General.dbInstance = ''
+config.General.requestName = filename_nopy + '_' + now_str
+config.General.workArea = '/tmp/crabStatusTracking'
+config.General.transferLogs=False
+
+config.section_("JobType")
+config.JobType.pluginName = 'Analysis'
+config.JobType.psetName = 'pset.py'
+config.JobType.scriptExe = 'withNoneEDM.sh'
+config.JobType.outputFiles = ['miniaodfake.root']
+
+config.section_("Data")
+config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM')
+
+config.Data.splitting = 'LumiBased'
+config.Data.unitsPerJob = 1
+config.JobType.maxJobRuntimeMin = 60
+config.Data.totalUnits = 40
+
+config.Data.publication = False
+testName = "ruciotransfers-%d" % int(time.time())
+#testName = 'ruciotransfers-1'
+config.Data.outputDatasetTag = testName
+# rucio
+config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('user', getUsernameFromCRIC(), testName,)
+
+config.section_("User")
+
+config.section_("Site")
+config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*']
+#config.Site.blacklist = ['T2_ES_IFCA']
+config.Site.storageSite = 'T2_CH_CERN'
+
+config.section_("Debug")
+config.Debug.scheddName = 'crab3@vocms059.cern.ch'

test/statusTrackingTasks/withNoneEDM.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -euo pipefail
+set -x
+
+dd if=/dev/urandom of=miniaodfake.root bs=1M count=10
+
+cmsRun -j FrameworkJobReport.xml -p PSet.py