From 39aef1a1eccf893b1a2d9aea9e85956565c1f81c Mon Sep 17 00:00:00 2001 From: Thanayut Seethongchuen Date: Fri, 8 Dec 2023 20:27:38 +0100 Subject: [PATCH] rucio aso testsuite tasks (#8091) --- test/statusTrackingTasks/psetManyEDM.py | 27 ++++++++++ test/statusTrackingTasks/rucio_transfers.py | 48 ++++++++++++++++++ .../rucio_transfers_group.py | 48 ++++++++++++++++++ .../rucio_transfers_manyedm_nopublication.py | 48 ++++++++++++++++++ .../rucio_transfers_nopublication.py | 48 ++++++++++++++++++ .../rucio_transfers_withnoneedm.py | 50 +++++++++++++++++++ ...cio_transfers_withnoneedm_nopublication.py | 50 +++++++++++++++++++ test/statusTrackingTasks/withNoneEDM.sh | 8 +++ 8 files changed, 327 insertions(+) create mode 100644 test/statusTrackingTasks/psetManyEDM.py create mode 100644 test/statusTrackingTasks/rucio_transfers.py create mode 100644 test/statusTrackingTasks/rucio_transfers_group.py create mode 100644 test/statusTrackingTasks/rucio_transfers_manyedm_nopublication.py create mode 100644 test/statusTrackingTasks/rucio_transfers_nopublication.py create mode 100644 test/statusTrackingTasks/rucio_transfers_withnoneedm.py create mode 100644 test/statusTrackingTasks/rucio_transfers_withnoneedm_nopublication.py create mode 100644 test/statusTrackingTasks/withNoneEDM.sh diff --git a/test/statusTrackingTasks/psetManyEDM.py b/test/statusTrackingTasks/psetManyEDM.py new file mode 100644 index 0000000000..b0c91228bf --- /dev/null +++ b/test/statusTrackingTasks/psetManyEDM.py @@ -0,0 +1,27 @@ +from __future__ import division +import FWCore.ParameterSet.Config as cms + +process = cms.Process('NoSplit') + +process.source = cms.Source("PoolSource", fileNames = cms.untracked.vstring('root://cms-xrd-global.cern.ch///store/mc/HC/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/00000/8ADD04E5-1776-E711-A1BA-FA163E6741E0.root')) +process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(10)) +process.options = cms.untracked.PSet(wantSummary = cms.untracked.bool(True)) + +process.output = cms.OutputModule("PoolOutputModule", + outputCommands = cms.untracked.vstring("drop *", "keep recoTracks_globalMuons_*_*"), + fileName = cms.untracked.string('output.root'), + dataset = cms.untracked.PSet( + filterName = cms.untracked.string('OutOne'), + ) +) +process.out1 = cms.EndPath(process.output) + +process.secondoutput = cms.OutputModule("PoolOutputModule", + outputCommands = cms.untracked.vstring("drop *", "keep recoTracks_globalMuons_*_*"), + fileName = cms.untracked.string('secondoutput.root'), + dataset = cms.untracked.PSet( + filterName = cms.untracked.string('OutTwo'), + ) + +) +process.out2 = cms.EndPath(process.secondoutput) diff --git a/test/statusTrackingTasks/rucio_transfers.py b/test/statusTrackingTasks/rucio_transfers.py new file mode 100644 index 0000000000..8d21320cdd --- /dev/null +++ b/test/statusTrackingTasks/rucio_transfers.py @@ -0,0 +1,48 @@ +from __future__ import division +import time +from WMCore.Configuration import Configuration +import os +import datetime +from CRABClient.UserUtilities import getUsernameFromCRIC + +now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") +filename_nopy = __file__.split('/')[-1][:-3] + +config = Configuration() + +config.section_("General") +config.General.instance = os.getenv('REST_Instance','test12') +#config.General.restHost = '' +#config.General.dbInstance = '' +config.General.requestName = filename_nopy + '_' + now_str +config.General.workArea = '/tmp/crabStatusTracking' +config.General.transferLogs=False + +config.section_("JobType") +config.JobType.pluginName = 'Analysis' +config.JobType.psetName = 'pset.py' + +config.section_("Data") +config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM') + +config.Data.splitting = 'LumiBased' +config.Data.unitsPerJob = 1 +config.JobType.maxJobRuntimeMin = 60 +config.Data.totalUnits = 40 + +config.Data.publication = True +testName = "ruciotransfers-%d" % int(time.time()) +#testName = 'ruciotransfers-1' +config.Data.outputDatasetTag = testName +# rucio +config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('user', getUsernameFromCRIC(), testName,) + +config.section_("User") + +config.section_("Site") +config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*'] +#config.Site.blacklist = ['T2_ES_IFCA'] +config.Site.storageSite = 'T2_CH_CERN' + +config.section_("Debug") +config.Debug.scheddName = 'crab3@vocms059.cern.ch' diff --git a/test/statusTrackingTasks/rucio_transfers_group.py b/test/statusTrackingTasks/rucio_transfers_group.py new file mode 100644 index 0000000000..201bf0281f --- /dev/null +++ b/test/statusTrackingTasks/rucio_transfers_group.py @@ -0,0 +1,48 @@ +from __future__ import division +import time +from WMCore.Configuration import Configuration +import os +import datetime +from CRABClient.UserUtilities import getUsernameFromCRIC + +now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") +filename_nopy = __file__.split('/')[-1][:-3] + +config = Configuration() + +config.section_("General") +config.General.instance = os.getenv('REST_Instance','test12') +#config.General.restHost = '' +#config.General.dbInstance = '' +config.General.requestName = filename_nopy + '_' + now_str +config.General.workArea = '/tmp/crabStatusTracking' +config.General.transferLogs=False + +config.section_("JobType") +config.JobType.pluginName = 'Analysis' +config.JobType.psetName = 'pset.py' + +config.section_("Data") +config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM') + +config.Data.splitting = 'LumiBased' +config.Data.unitsPerJob = 1 +config.JobType.maxJobRuntimeMin = 60 +config.Data.totalUnits = 40 + +config.Data.publication = True +testName = "ruciotransfers-%d" % int(time.time()) +#testName = 'ruciotransfers-1' +config.Data.outputDatasetTag = testName +# rucio +config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('group', 'crab_test', testName,) + +config.section_("User") + +config.section_("Site") +config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*'] +#config.Site.blacklist = ['T2_ES_IFCA'] +config.Site.storageSite = 'T2_IT_Legnaro' + +config.section_("Debug") +config.Debug.scheddName = 'crab3@vocms059.cern.ch' diff --git a/test/statusTrackingTasks/rucio_transfers_manyedm_nopublication.py b/test/statusTrackingTasks/rucio_transfers_manyedm_nopublication.py new file mode 100644 index 0000000000..5ddd1cb696 --- /dev/null +++ b/test/statusTrackingTasks/rucio_transfers_manyedm_nopublication.py @@ -0,0 +1,48 @@ +from __future__ import division +import time +from WMCore.Configuration import Configuration +import os +import datetime +from CRABClient.UserUtilities import getUsernameFromCRIC + +now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") +filename_nopy = __file__.split('/')[-1][:-3] + +config = Configuration() + +config.section_("General") +config.General.instance = os.getenv('REST_Instance','test12') +#config.General.restHost = '' +#config.General.dbInstance = '' +config.General.requestName = filename_nopy + '_' + now_str +config.General.workArea = '/tmp/crabStatusTracking' +config.General.transferLogs=False + +config.section_("JobType") +config.JobType.pluginName = 'Analysis' +config.JobType.psetName = 'psetManyEDM.py' + +config.section_("Data") +config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM') + +config.Data.splitting = 'LumiBased' +config.Data.unitsPerJob = 1 +config.JobType.maxJobRuntimeMin = 60 +config.Data.totalUnits = 40 + +config.Data.publication = False +testName = "ruciotransfers-%d" % int(time.time()) +#testName = 'ruciotransfers-1' +config.Data.outputDatasetTag = testName +# rucio +config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('user', getUsernameFromCRIC(), testName,) + +config.section_("User") + +config.section_("Site") +config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*'] +#config.Site.blacklist = ['T2_ES_IFCA'] +config.Site.storageSite = 'T2_CH_CERN' + +config.section_("Debug") +config.Debug.scheddName = 'crab3@vocms059.cern.ch' diff --git a/test/statusTrackingTasks/rucio_transfers_nopublication.py b/test/statusTrackingTasks/rucio_transfers_nopublication.py new file mode 100644 index 0000000000..e6089733bf --- /dev/null +++ b/test/statusTrackingTasks/rucio_transfers_nopublication.py @@ -0,0 +1,48 @@ +from __future__ import division +import time +from WMCore.Configuration import Configuration +import os +import datetime +from CRABClient.UserUtilities import getUsernameFromCRIC + +now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") +filename_nopy = __file__.split('/')[-1][:-3] + +config = Configuration() + +config.section_("General") +config.General.instance = os.getenv('REST_Instance','test12') +#config.General.restHost = '' +#config.General.dbInstance = '' +config.General.requestName = filename_nopy + '_' + now_str +config.General.workArea = '/tmp/crabStatusTracking' +config.General.transferLogs=False + +config.section_("JobType") +config.JobType.pluginName = 'Analysis' +config.JobType.psetName = 'pset.py' + +config.section_("Data") +config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM') + +config.Data.splitting = 'LumiBased' +config.Data.unitsPerJob = 1 +config.JobType.maxJobRuntimeMin = 60 +config.Data.totalUnits = 40 + +config.Data.publication = False +testName = "ruciotransfers-%d" % int(time.time()) +#testName = 'ruciotransfers-1' +config.Data.outputDatasetTag = testName +# rucio +config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('user', getUsernameFromCRIC(), testName,) + +config.section_("User") + +config.section_("Site") +config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*'] +#config.Site.blacklist = ['T2_ES_IFCA'] +config.Site.storageSite = 'T2_CH_CERN' + +config.section_("Debug") +config.Debug.scheddName = 'crab3@vocms059.cern.ch' diff --git a/test/statusTrackingTasks/rucio_transfers_withnoneedm.py b/test/statusTrackingTasks/rucio_transfers_withnoneedm.py new file mode 100644 index 0000000000..acf8a8e749 --- /dev/null +++ b/test/statusTrackingTasks/rucio_transfers_withnoneedm.py @@ -0,0 +1,50 @@ +from __future__ import division +import time +from WMCore.Configuration import Configuration +import os +import datetime +from CRABClient.UserUtilities import getUsernameFromCRIC + +now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") +filename_nopy = __file__.split('/')[-1][:-3] + +config = Configuration() + +config.section_("General") +config.General.instance = os.getenv('REST_Instance','test12') +#config.General.restHost = '' +#config.General.dbInstance = '' +config.General.requestName = filename_nopy + '_' + now_str +config.General.workArea = '/tmp/crabStatusTracking' +config.General.transferLogs=False + +config.section_("JobType") +config.JobType.pluginName = 'Analysis' +config.JobType.psetName = 'pset.py' +config.JobType.scriptExe = 'withNoneEDM.sh' +config.JobType.outputFiles = ['miniaodfake.root'] + +config.section_("Data") +config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM') + +config.Data.splitting = 'LumiBased' +config.Data.unitsPerJob = 1 +config.JobType.maxJobRuntimeMin = 60 +config.Data.totalUnits = 40 + +config.Data.publication = True +testName = "ruciotransfers-%d" % int(time.time()) +#testName = 'ruciotransfers-1' +config.Data.outputDatasetTag = testName +# rucio +config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('user', getUsernameFromCRIC(), testName,) + +config.section_("User") + +config.section_("Site") +config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*'] +#config.Site.blacklist = ['T2_ES_IFCA'] +config.Site.storageSite = 'T2_CH_CERN' + +config.section_("Debug") +config.Debug.scheddName = 'crab3@vocms059.cern.ch' diff --git a/test/statusTrackingTasks/rucio_transfers_withnoneedm_nopublication.py b/test/statusTrackingTasks/rucio_transfers_withnoneedm_nopublication.py new file mode 100644 index 0000000000..de1f055afa --- /dev/null +++ b/test/statusTrackingTasks/rucio_transfers_withnoneedm_nopublication.py @@ -0,0 +1,50 @@ +from __future__ import division +import time +from WMCore.Configuration import Configuration +import os +import datetime +from CRABClient.UserUtilities import getUsernameFromCRIC + +now_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") +filename_nopy = __file__.split('/')[-1][:-3] + +config = Configuration() + +config.section_("General") +config.General.instance = os.getenv('REST_Instance','test12') +#config.General.restHost = '' +#config.General.dbInstance = '' +config.General.requestName = filename_nopy + '_' + now_str +config.General.workArea = '/tmp/crabStatusTracking' +config.General.transferLogs=False + +config.section_("JobType") +config.JobType.pluginName = 'Analysis' +config.JobType.psetName = 'pset.py' +config.JobType.scriptExe = 'withNoneEDM.sh' +config.JobType.outputFiles = ['miniaodfake.root'] + +config.section_("Data") +config.Data.inputDataset = os.getenv('inputDataset','/GenericTTbar/HC-CMSSW_9_2_6_91X_mcRun1_realistic_v2-v2/AODSIM') + +config.Data.splitting = 'LumiBased' +config.Data.unitsPerJob = 1 +config.JobType.maxJobRuntimeMin = 60 +config.Data.totalUnits = 40 + +config.Data.publication = False +testName = "ruciotransfers-%d" % int(time.time()) +#testName = 'ruciotransfers-1' +config.Data.outputDatasetTag = testName +# rucio +config.Data.outLFNDirBase = '/store/%s/rucio/%s/%s' % ('user', getUsernameFromCRIC(), testName,) + +config.section_("User") + +config.section_("Site") +config.Site.whitelist = ['T1_*','T2_US_*','T2_IT_*','T2_DE_*','T2_ES_*','T2_FR_*','T2_UK_*'] +#config.Site.blacklist = ['T2_ES_IFCA'] +config.Site.storageSite = 'T2_CH_CERN' + +config.section_("Debug") +config.Debug.scheddName = 'crab3@vocms059.cern.ch' diff --git a/test/statusTrackingTasks/withNoneEDM.sh b/test/statusTrackingTasks/withNoneEDM.sh new file mode 100644 index 0000000000..0b3c245178 --- /dev/null +++ b/test/statusTrackingTasks/withNoneEDM.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail +set -x + +dd if=/dev/urandom of=miniaodfake.root bs=1M count=10 + +cmsRun -j FrameworkJobReport.xml -p PSet.py