From 17673c741a3bc752773902459bbec2afda59bb8c Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Wed, 8 Jan 2025 12:09:39 -0600 Subject: [PATCH] Updated test of edm::Ref use from a merged ROOT file - consolidated two cfg files into one with command line options - modernized module configuration syntax - extended test to include case where different branches are in the different files --- FWCore/Integration/test/ref_merge_cfg.py | 26 +++++--- .../Integration/test/ref_merge_prod1_cfg.py | 39 ------------ .../Integration/test/ref_merge_prod2_cfg.py | 34 ---------- FWCore/Integration/test/ref_merge_prod_cfg.py | 63 +++++++++++++++++++ FWCore/Integration/test/ref_merge_test_cfg.py | 18 ++++-- FWCore/Integration/test/run_RefMerge.sh | 20 ++++-- 6 files changed, 110 insertions(+), 90 deletions(-) delete mode 100644 FWCore/Integration/test/ref_merge_prod1_cfg.py delete mode 100644 FWCore/Integration/test/ref_merge_prod2_cfg.py create mode 100644 FWCore/Integration/test/ref_merge_prod_cfg.py diff --git a/FWCore/Integration/test/ref_merge_cfg.py b/FWCore/Integration/test/ref_merge_cfg.py index 0a1be4dd71ef6..237dc54eea621 100644 --- a/FWCore/Integration/test/ref_merge_cfg.py +++ b/FWCore/Integration/test/ref_merge_cfg.py @@ -1,17 +1,27 @@ import FWCore.ParameterSet.Config as cms +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test ConditionalTasks.') + +parser.add_argument("--inFile1", help="first file to read") +parser.add_argument("--inFile2", help="second file to read") +parser.add_argument("--outFile", help="name of output file", default="ref_merge.root") + +args = parser.parse_args() + process = cms.Process("MERGE") -process.source = cms.Source("PoolSource", - fileNames = cms.untracked.vstring("file:ref_merge_prod1.root", - "file:ref_merge_prod2.root") +from IOPool.Input.modules import PoolSource +process.source = PoolSource(fileNames = [f"file:{args.inFile1}", + f"file:{args.inFile2}"] ) -process.out = cms.OutputModule("PoolOutputModule", - fileName = cms.untracked.string("ref_merge.root") - ) +from IOPool.Output.modules import PoolOutputModule +process.out = PoolOutputModule(fileName = args.outFile) -process.tester = cms.EDAnalyzer("OtherThingAnalyzer", - other = cms.untracked.InputTag("d","testUserTag")) +from FWCore.Integration.modules import OtherThingAnalyzer +process.tester = OtherThingAnalyzer(other = ("d","testUserTag")) process.o = cms.EndPath(process.out+process.tester) diff --git a/FWCore/Integration/test/ref_merge_prod1_cfg.py b/FWCore/Integration/test/ref_merge_prod1_cfg.py deleted file mode 100644 index 68da8ecec0abf..0000000000000 --- a/FWCore/Integration/test/ref_merge_prod1_cfg.py +++ /dev/null @@ -1,39 +0,0 @@ -import FWCore.ParameterSet.Config as cms -process = cms.Process("PROD") - -process.source = cms.Source("EmptySource", - firstLuminosityBlock = cms.untracked.uint32(1), - firstEvent = cms.untracked.uint32(1) -) - -process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(10)) - -process.a = cms.EDProducer("IntProducer", - ivalue = cms.int32(1)) - -process.b = cms.EDProducer("IntProducer", - ivalue = cms.int32(2)) - -process.c = cms.EDProducer("ThingProducer") - -process.d = cms.EDProducer("OtherThingProducer", - thingTag=cms.InputTag("c")) - -process.o = cms.OutputModule("PoolOutputModule", - outputCommands = cms.untracked.vstring("drop *", - "keep edmtestThings_*_*_*", - "keep edmtestOtherThings_*_*_*"), - fileName = cms.untracked.string("ref_merge_prod1.root") - ) - -process.p = cms.Path(process.a+process.b+process.c*process.d) - -process.tester = cms.EDAnalyzer("OtherThingAnalyzer", - other = cms.untracked.InputTag("d","testUserTag")) - -process.out = cms.EndPath(process.o+process.tester) - - - - - diff --git a/FWCore/Integration/test/ref_merge_prod2_cfg.py b/FWCore/Integration/test/ref_merge_prod2_cfg.py deleted file mode 100644 index 7f630d613249d..0000000000000 --- a/FWCore/Integration/test/ref_merge_prod2_cfg.py +++ /dev/null @@ -1,34 +0,0 @@ -#No edmtest::IntProduct's added to this output file -import FWCore.ParameterSet.Config as cms -process = cms.Process("PROD") - -process.source = cms.Source("EmptySource", - firstLuminosityBlock = cms.untracked.uint32(2), - firstEvent = cms.untracked.uint32(20) -) - -process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(10)) - -process.c = cms.EDProducer("ThingProducer") - -process.d = cms.EDProducer("OtherThingProducer", - thingTag=cms.InputTag("c")) - -process.o = cms.OutputModule("PoolOutputModule", - outputCommands = cms.untracked.vstring("drop *", - "keep edmtestThings_*_*_*", - "keep edmtestOtherThings_*_*_*"), - fileName = cms.untracked.string("ref_merge_prod2.root") - ) - -process.p = cms.Path(process.c*process.d) - -process.tester = cms.EDAnalyzer("OtherThingAnalyzer", - other = cms.untracked.InputTag("d","testUserTag")) - -process.out = cms.EndPath(process.o+process.tester) - - - - - diff --git a/FWCore/Integration/test/ref_merge_prod_cfg.py b/FWCore/Integration/test/ref_merge_prod_cfg.py new file mode 100644 index 0000000000000..016a016316214 --- /dev/null +++ b/FWCore/Integration/test/ref_merge_prod_cfg.py @@ -0,0 +1,63 @@ +import FWCore.ParameterSet.Config as cms + +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test ConditionalTasks.') + +parser.add_argument("--extraProducers", help="Add extra producers to configuration", action="store_true") +parser.add_argument("--fileName", help="name of output file") +parser.add_argument("--firstLumi", help="LuminosityBlock number for first lumi", type = int, default=1) +parser.add_argument("--keepAllProducts", help="Keep all products made in the job", action="store_true") +parser.add_argument("--dropThings", help="drop the Things collections so the refs will not function", action="store_true") + +args = parser.parse_args() + + +process = cms.Process("PROD") + +nEvents = 10 +from FWCore.Modules.modules import EmptySource +process.source = EmptySource( + firstLuminosityBlock = args.firstLumi, + firstEvent = nEvents*(args.firstLumi-1)+1 +) + +process.maxEvents.input = nEvents + +if args.extraProducers: + from FWCore.Framework.modules import IntProducer + process.a = IntProducer(ivalue = 1) + + process.b = IntProducer(ivalue = 2) + +from FWCore.Integration.modules import ThingProducer, OtherThingProducer, OtherThingAnalyzer +process.c = ThingProducer() + +process.d = OtherThingProducer(thingTag="c") + +outputs = [] +if not args.keepAllProducts: + outputs = ["drop *", + "keep edmtestOtherThings_*_*_*"] + if not args.dropThings: + outputs.append("keep edmtestThings_*_*_*") + + +from IOPool.Output.modules import PoolOutputModule +process.o = PoolOutputModule(outputCommands = outputs, + fileName = args.fileName + ) +if args.extraProducers: + process.p = cms.Path(process.a+process.b+process.c*process.d) +else: + process.p = cms.Path(process.c*process.d) + +process.tester = OtherThingAnalyzer(other = ("d","testUserTag")) + +process.out = cms.EndPath(process.o+process.tester) + + + + + diff --git a/FWCore/Integration/test/ref_merge_test_cfg.py b/FWCore/Integration/test/ref_merge_test_cfg.py index 4e5c5283bdcd8..bfd521d553f08 100644 --- a/FWCore/Integration/test/ref_merge_test_cfg.py +++ b/FWCore/Integration/test/ref_merge_test_cfg.py @@ -1,12 +1,20 @@ import FWCore.ParameterSet.Config as cms +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test Refs after merge.') + +parser.add_argument("--fileName", help="file to read") + +args = parser.parse_args() + process = cms.Process("TEST") -process.source = cms.Source("PoolSource", - fileNames = cms.untracked.vstring("file:ref_merge.root") - ) +from IOPool.Input.modules import PoolSource +process.source = PoolSource(fileNames = [f"file:{args.fileName}"]) -process.tester = cms.EDAnalyzer("OtherThingAnalyzer", - other = cms.untracked.InputTag("d","testUserTag")) +from FWCore.Integration.modules import OtherThingAnalyzer +process.tester = OtherThingAnalyzer(other = ("d","testUserTag")) process.e = cms.EndPath(process.tester) diff --git a/FWCore/Integration/test/run_RefMerge.sh b/FWCore/Integration/test/run_RefMerge.sh index d1643036544a2..95f75d76cf3e7 100755 --- a/FWCore/Integration/test/run_RefMerge.sh +++ b/FWCore/Integration/test/run_RefMerge.sh @@ -6,16 +6,28 @@ function die { echo Failure $1: status $2 ; exit $2 ; } LOCAL_TEST_DIR=${SCRAM_TEST_PATH} echo ${test}prod1 ------------------------------------------------------------ - cmsRun ${LOCAL_TEST_DIR}/${test}prod1_cfg.py || die "cmsRun ${test}prod1_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --extraProducers --fileName 'ref_merge_prod1.root' || die "cmsRun ${test}prod_cfg.py --extraProducers" $? echo ${test}prod2 ------------------------------------------------------------ - cmsRun ${LOCAL_TEST_DIR}/${test}prod2_cfg.py || die "cmsRun ${test}prod2_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --firstLumi 10 --fileName 'ref_merge_prod2.root'|| die "cmsRun ${test}prod_cfg.py" $? echo ${test}MERGE------------------------------------------------------------ - cmsRun ${LOCAL_TEST_DIR}/${test}cfg.py || die "cmsRun ${test}cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/${test}cfg.py --inFile1 'ref_merge_prod1.root' --inFile2 'ref_merge_prod2.root' --outFile 'ref_merge.root' || die "cmsRun ${test}cfg.py" $? echo ${test}test------------------------------------------------------------ - cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py || die "cmsRun ${test}test_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --fileName 'ref_merge.root' || die "cmsRun ${test}test_cfg.py" $? + + echo ${test}keepAllProd ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --extraProducers --keepAllProducts --fileName 'ref_merge_prod_all.root' || die "cmsRun ${test}prod_cfg.py --keepAllProducts" $? + + echo ${test}MERGE_keepAll1st ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}cfg.py --inFile1 'ref_merge_prod_all.root' --inFile2 'ref_merge_prod2.root' --outFile 'ref_merge_all1st.root' || die "cmsRun ${test}cfg.py" $? + + echo ${test}test_all1st------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --fileName 'ref_merge_all1st.root' || die "cmsRun ${test}test_cfg.py all1st" $? + + #note having all be the second file does not work as PoolSource enforces that subsequent files must have a strict subset + # of the branches in the first file read echo ${test}test------------------------------------------------------------ cmsRun ${LOCAL_TEST_DIR}/${test}subprocess_cfg.py || die "cmsRun ${test}subprocess_cfg.py" $?