Run PR gpu utests/relvals on both CUDA and ROCm GPUs

cleanup-cmssdt

@@ -51,6 +51,7 @@ DIRS="lizard flawfinder invalid-includes cmssw-afs-eos-comparison ubsan_logs ib-
 DIRS="${DIRS} check_headers valgrind HLT-Validation ib-static-analysis ib-baseline-tests ib-dqm-tests profiling igprof"
 DIRS="${DIRS} iwyu material-budget das_query build-any-ib check-unused-cmsdist-packages class_versions"
 DIRS="${DIRS} test-os-alma8 test-os-cs8 test-os-ubi8 test-os-lxplus8 test-os-rhel8 test-os-rocky8 test-os-el8 cms-containers-run-cmssw-test"
+DIRS="${DIRS} baseLineComparisonsCUDA baseLineComparisonsROCM"
 for dir in ${DIRS}; do
   [ -d ${JENKINS_ARTIFACTS}/$dir ] || continue
   DIRS_PROCESSED="${DIRS_PROCESSED} ${dir}"

cmssw-pr-test-config

@@ -20,6 +20,12 @@ elif [ "$CMSSW_VER" -ge 1300 ] ; then
 else
   PR_TEST_MATRIX_EXTRAS_GPU=11634.586,11634.587
 fi
+if [ X"$PR_TEST_MATRIX_EXTRAS_GPU" != X"" ]; then
+  for GPU_T in ${ALL_GPU_TYPES[@]} ; do
+    GPU_T_UC = $(echo ${GPU_T} | tr '[a-z]' '[A-Z]')
+    eval "PR_TEST_MATRIX_EXTRAS_${GPU_T_UC}=$PR_TEST_MATRIX_EXTRAS_GPU"
+  done
+fi
 PR_TEST_MATRIX_EXTRAS_PROFILING=29834.21,13034.21
 PR_TEST_MATRIX_EXTRAS_HIGH_STATS=35034.0
 PR_TEST_MATRIX_EXTRAS_NANO=all

gpu_flavors.txt

@@ -0,0 +1,2 @@
+cuda
+rocm

pr_testing/run-pr-relvals.sh

@@ -1,5 +1,20 @@
 #!/bin/bash -ex
+function is_in_array() {
+    local value="$1"
+    shift
+    local array=("$@")
+
+    for item in "${array[@]}"; do
+        if [[ "$item" == "$value" ]]; then
+            return 0  # Found match
+        fi
+    done
+    return 1  # No match
+}
+
 source $(dirname $0)/setup-pr-test-env.sh
+readarray -t ALL_GPU_TYPES < ${CMS_BOT_DIR}/gpu_flavors.txt
+
 GH_CONTEXT="relvals"
 GH_COMP_CONTEXT="comparison"
 UC_TEST_FLAVOR=$(echo ${TEST_FLAVOR} | tr '[a-z]' '[A-Z]')
@@ -17,10 +32,12 @@ echo "${MATRIX_ARGS}"  | tr ';' '\n' | while IFS= read -r args; do
   if [ $(echo "${args}" | sed 's|.*-l ||;s| .*||' | tr ',' '\n' | grep '^all$' | wc -l) -gt 0 ] ; then
     OPTS=""
     case "${TEST_FLAVOR}" in
-      gpu ) OPTS="-w gpu" ;;
       high_stats ) ;;
       nano ) OPTS="-w nano" ;;
-      * ) ;;
+      * ) if is_in_array "${TEST_FLAVOR}" "${ALL_GPU_TYPES[@]}" ; then
+            OPTS="-w gpu"
+          fi
+          ;;
     esac
     ALL_WFS=$(runTheMatrix.py -n ${OPTS} ${args} | grep -v ' workflows ' | grep '^[1-9][0-9]*\(.[0-9][0-9]*\|\)\s' | sed 's| .*||' | tr '\n' ',' | sed 's|,$||')
     args=$(echo "${args}" | sed "s|all|${ALL_WFS}|")

pr_testing/run-pr-unittests.sh

@@ -10,57 +10,58 @@ cd $WORKSPACE/${CMSSW_VERSION}
 CMSSW_PKG_COUNT=$(ls -d $LOCALRT/src/*/* | wc -l)
 REPORT_OPTS="--report-url ${PR_RESULT_URL} $NO_POST"
 
-rm -f ${RESULTS_DIR}/unittestGPU.txt
-mark_commit_status_all_prs 'unittests/gpu' 'pending' -u "${BUILD_URL}" -d "Running tests" || true
+rm -f ${RESULTS_DIR}/unittest${GPU_FLAVOR}.txt
+mark_commit_status_all_prs "unittests/${GPU_FLAVOR}" 'pending' -u "${BUILD_URL}" -d "Running tests" || true
 echo '--------------------------------------'
-mkdir -p $WORKSPACE/gpuUnitTests
+mkdir -p $WORKSPACE/${GPU_FLAVOR}UnitTests
 let UT_TIMEOUT=7200+${CMSSW_PKG_COUNT}*20
-UTESTS_CMD="USER_UNIT_TESTS=cuda timeout ${UT_TIMEOUT} scram b -v -k -j ${NCPU}  unittests "
+gpu_t_lc=$(echo ${GPU_T} | tr '[A-Z]' '[a-z]')
+UTESTS_CMD="USER_UNIT_TESTS=${gpu_t_lc} timeout ${UT_TIMEOUT} scram b -v -k -j ${NCPU}  unittests "
 echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
 scram build echo_LD_LIBRARY_PATH || true
 scram build -r echo_CXX || true
 cms_major=$(echo ${CMSSW_IB} | cut -d_ -f2)
 cms_minor=$(echo ${CMSSW_IB} | cut -d_ -f3)
 cms_ver="$(echo 00${cms_major} | sed -E 's|^.*(..)$|\1|')$(echo 00${cms_minor} | sed -E 's|^.*(..)$|\1|')"
-echo $UTESTS_CMD > $WORKSPACE/gpuUnitTests/log.txt
-(eval $UTESTS_CMD && echo 'ALL_OK') > $WORKSPACE/gpuUnitTests/log.txt 2>&1 || true
+echo $UTESTS_CMD > $WORKSPACE/${GPU_FLAVOR}UnitTests/log.txt
+(eval $UTESTS_CMD && echo 'ALL_OK') > $WORKSPACE/${GPU_FLAVOR}UnitTests/log.txt 2>&1 || true
 echo 'END OF UNIT TESTS'
 echo '--------------------------------------'
 
-TEST_ERRORS=$(grep -ai 'had errors\|recipe for target' $WORKSPACE/gpuUnitTests/log.txt | sed "s|'||g;s|.*recipe for target *||;s|.*unittests_|---> test |;s| failed$| timeout|" || true)
-TEST_ERRORS=`grep -ai "had errors" $WORKSPACE/gpuUnitTests/log.txt` || true
-GENERAL_ERRORS=`grep -a "ALL_OK" $WORKSPACE/gpuUnitTests/log.txt` || true
+TEST_ERRORS=$(grep -ai 'had errors\|recipe for target' $WORKSPACE/${GPU_FLAVOR}UnitTests/log.txt | sed "s|'||g;s|.*recipe for target *||;s|.*unittests_|---> test |;s| failed$| timeout|" || true)
+TEST_ERRORS=`grep -ai "had errors" $WORKSPACE/${GPU_FLAVOR}UnitTests/log.txt` || true
+GENERAL_ERRORS=`grep -a "ALL_OK" $WORKSPACE/${GPU_FLAVOR}UnitTests/log.txt` || true
 
 if [ "X$TEST_ERRORS" != "X" -o "X$GENERAL_ERRORS" = "X" ]; then
-  echo "Errors in the gpu unit tests"
-  echo 'GPU_UNIT_TEST_RESULTS;ERROR,GPU Unit Tests,See Log,gpuUnitTests' >> ${RESULTS_DIR}/unittestGPU.txt
+  echo "Errors in the ${GPU_FLAVOR} unit tests"
+  echo "${GPU_FLAVOR}_UNIT_TEST_RESULTS;ERROR,GPU Unit Tests,See Log,${GPU_FLAVOR}UnitTests" >> ${RESULTS_DIR}/unittest${GPU_FLAVOR}.txt
   ALL_OK=false
   UNIT_TESTS_OK=false
-  $CMS_BOT_DIR/report-pull-request-results PARSE_GPU_UNIT_TESTS_FAIL -f $WORKSPACE/gpuUnitTests/log.txt --report-file ${RESULTS_DIR}/14-unittestGPU-report.res ${REPORT_OPTS}
-  echo "GpuUnitTests" > ${RESULTS_DIR}/14-failed.res
+  $CMS_BOT_DIR/report-pull-request-results PARSE_${GPU_FLAVOR}_UNIT_TESTS_FAIL -f $WORKSPACE/${GPU_FLAVOR}UnitTests/log.txt --report-file ${RESULTS_DIR}/14-unittest${GPU_FLAVOR}-report.res ${REPORT_OPTS}
+  echo "${GPU_FLAVOR}UnitTests" > ${RESULTS_DIR}/14-failed.res
 else
-  echo 'GPU_UNIT_TEST_RESULTS;OK,GPU Unit Tests,See Log,gpuUnitTests' >> ${RESULTS_DIR}/unittestGPU.txt
+  echo "${GPU_FLAVOR}_UNIT_TEST_RESULTS;OK,GPU Unit Tests,See Log,${GPU_FLAVOR}UnitTests" >> ${RESULTS_DIR}/unittest${GPU_FLAVOR}.txt
 fi
-echo "<html><head></head><body>" > $WORKSPACE/gpuUnitTests/success.html
-cp $WORKSPACE/gpuUnitTests/success.html $WORKSPACE/gpuUnitTests/failed.html
+echo "<html><head></head><body>" > $WORKSPACE/${GPU_FLAVOR}UnitTests/success.html
+cp $WORKSPACE/${GPU_FLAVOR}UnitTests/success.html $WORKSPACE/${GPU_FLAVOR}UnitTests/failed.html
 UT_ERR=false
 utlog="testing.log"
 for t in $(find $WORKSPACE/$CMSSW_IB/tmp/${SCRAM_ARCH}/src -name ${utlog} -type f | sed "s|$WORKSPACE/$CMSSW_IB/tmp/${SCRAM_ARCH}/||;s|/${utlog}$||") ; do
-  mkdir -p $WORKSPACE/gpuUnitTests/${t}
-  mv $WORKSPACE/$CMSSW_IB/tmp/${SCRAM_ARCH}/${t}/${utlog} $WORKSPACE/gpuUnitTests/${t}/
-  if [ $(grep -a '^\-\-\-> test  *[^ ]*  *succeeded$' $WORKSPACE/gpuUnitTests/${t}/${utlog} | wc -l) -gt 0 ] ; then
-    echo "<a href='${t}/${utlog}'>${t}</a><br/>" >> $WORKSPACE/gpuUnitTests/success.html
+  mkdir -p $WORKSPACE/${GPU_FLAVOR}UnitTests/${t}
+  mv $WORKSPACE/$CMSSW_IB/tmp/${SCRAM_ARCH}/${t}/${utlog} $WORKSPACE/${GPU_FLAVOR}UnitTests/${t}/
+  if [ $(grep -a '^\-\-\-> test  *[^ ]*  *succeeded$' $WORKSPACE/${GPU_FLAVOR}UnitTests/${t}/${utlog} | wc -l) -gt 0 ] ; then
+    echo "<a href='${t}/${utlog}'>${t}</a><br/>" >> $WORKSPACE/${GPU_FLAVOR}UnitTests/success.html
   else
-    echo "<a href='${t}/${utlog}'>${t}</a><br/>" >> $WORKSPACE/gpuUnitTests/failed.html
+    echo "<a href='${t}/${utlog}'>${t}</a><br/>" >> $WORKSPACE/${GPU_FLAVOR}UnitTests/failed.html
     UT_ERR=true
   fi
 done
-if ! $UT_ERR ; then echo "No unit test failed" >> $WORKSPACE/gpuUnitTests/failed.html ; fi
-echo "</body></html>" >> $WORKSPACE/gpuUnitTests/success.html
-echo "</body></html>" >> $WORKSPACE/gpuUnitTests/failed.html
+if ! $UT_ERR ; then echo "No unit test failed" >> $WORKSPACE/${GPU_FLAVOR}UnitTests/failed.html ; fi
+echo "</body></html>" >> $WORKSPACE/${GPU_FLAVOR}UnitTests/success.html
+echo "</body></html>" >> $WORKSPACE/${GPU_FLAVOR}UnitTests/failed.html
 prepare_upload_results
 if $UNIT_TESTS_OK ; then
-  mark_commit_status_all_prs 'unittests/gpu' 'success' -u "${BUILD_URL}" -d "Passed"
+  mark_commit_status_all_prs "unittests/${GPU_FLAVOR}" 'success' -u "${BUILD_URL}" -d "Passed"
 else
-  mark_commit_status_all_prs 'unittests/gpu' 'error' -u "${BUILD_URL}" -d "Some unit tests were failed."
+  mark_commit_status_all_prs "unittests/${GPU_FLAVOR}" 'error' -u "${BUILD_URL}" -d "Some unit tests were failed."
 fi

pr_testing/test_multiple_prs.sh

@@ -79,6 +79,22 @@ function process_changed_files() {
   sort -u "$directlyChangedFiles" $WORKSPACE/indirectly-changed-files.txt > "$allChangedFiles"
 }
 
+function is_in_array() {
+    local value="$1"
+    shift
+    local array=("$@")
+
+    for item in "${array[@]}"; do
+        if [[ "$item" == "$value" ]]; then
+            return 0  # Found match
+        fi
+    done
+    return 1  # No match
+}
+
+readarray -t ALL_GPU_TYPES < ${CMS_BOT_DIR}/gpu_flavors.txt
+export ALL_GPU_TYPES
+ALL_GPU_TYPES_UC=( $(echo ${ALL_GPU_TYPES[@]} | tr '[a-z]' '[A-Z]') )
 # Constants
 echo LD_LIBRARY_PATH=${LD_LIBRARY_PATH} || true
 ls ${LD_LIBRARY_PATH} || true
@@ -91,7 +107,7 @@ PR_TESTING_DIR=${CMS_BOT_DIR}/pr_testing
 COMMON=${CMS_BOT_DIR}/common
 CONFIG_MAP=$CMS_BOT_DIR/config.map
 [ "${USE_IB_TAG}" != "true" ] && export USE_IB_TAG=false
-[ "${EXTRA_RELVALS_TESTS}" = "" ] && EXTRA_RELVALS_TESTS="GPU THREADING HIGH_STATS NANO"
+[ "${EXTRA_RELVALS_TESTS}" = "" ] && EXTRA_RELVALS_TESTS="THREADING HIGH_STATS NANO ${ALL_GPU_TYPES_UC[@]}"
 EXTRA_RELVALS_TESTS=$(echo ${EXTRA_RELVALS_TESTS} | tr ' ' '\n' | grep -v THREADING | tr '\n' ' ')
 # ---
 # doc: Input variable
@@ -1326,7 +1342,9 @@ if [ "X$BUILD_OK" = Xtrue -a "$RUN_TESTS" = "true" ]; then
   fi
   if [ $(echo ${ENABLE_BOT_TESTS} | tr ',' ' ' | tr ' ' '\n' | grep '^GPU$' | wc -l) -gt 0 -a X"${DISABLE_GPU_TESTS}" != X"true" ] ; then
     DO_GPU_TESTS=true
-    mark_commit_status_all_prs 'unittests/gpu' 'pending' -u "${BUILD_URL}" -d "Waiting for tests to start"
+    for GPU_T in ${ENABLE_GPU_FLAVORS[@]} ; do
+      mark_commit_status_all_prs "unittests/${GPU_T}" 'pending' -u "${BUILD_URL}" -d "Waiting for tests to start"
+    done
   fi
   if [ $(echo ${ENABLE_BOT_TESTS} | tr ',' ' ' | tr ' ' '\n' | grep '^HLT_P2_TIMING$' | wc -l) -gt 0 ] ; then
     if [ $(echo ${ARCHITECTURE}   | grep "_amd64_" | wc -l) -gt 0 ] ; then
@@ -1499,7 +1517,10 @@ if [ "X$DO_ADDON_TESTS" = Xtrue ]; then
 fi
 
 if [ "X$DO_GPU_TESTS" = Xtrue ]; then
-  cp $WORKSPACE/test-env.txt $WORKSPACE/run-unittests.prop
+  for GPU_T in ${ENABLE_GPU_FLAVORS[@]}; do
+    cp $WORKSPACE/test-env.txt $WORKSPACE/run-unittests-${GPU_T}.prop
+    echo "GPU_FLAVOR=${GPU_T}" >> $WORKSPACE/run-unittests-${GPU_T}.prop
+  done
 fi
 
 if ${BUILD_EXTERNAL} ; then
@@ -1510,7 +1531,7 @@ fi
 
 if [ "${DO_PROFILING}" = "true" ]  ; then
   PROFILING_WORKFLOWS=$($CMS_BOT_DIR/cmssw-pr-test-config _PROFILING | tr ',' ' ')
-  for wf in ${PROFILING_WORKFLOWS};do
+  for wf in ${PROFILING_WORKFLOWS}; do
     cp $WORKSPACE/test-env.txt $WORKSPACE/run-profiling-$wf.prop
     echo "PROFILING_WORKFLOWS=${wf}" >> $WORKSPACE/run-profiling-$wf.prop
   done
@@ -1525,3 +1546,4 @@ if [ "${DO_HLT_P2_INTEGRATION}" = "true" ] ;  then
 fi
 
 rm -f $WORKSPACE/test-env.txt
+

process_pr.py

@@ -143,7 +143,8 @@ def format(s, **kwds):
 REGEX_IGNORE_FILE_COUNT = r"\+file-count"
 TEST_WAIT_GAP = 720
 ALL_CHECK_FUNCTIONS = None
-EXTRA_RELVALS_TESTS = ["threading", "gpu", "high-stats", "nano"]
+ALL_GPU_FLAVORS = open(join(dirname(__file__), "gpu_flavors.txt"), "r").read().splitlines()
+EXTRA_RELVALS_TESTS = ["threading", "gpu", "high-stats", "nano"] + ALL_GPU_FLAVORS
 EXTRA_RELVALS_TESTS_OPTS = "_" + "|_".join(EXTRA_RELVALS_TESTS)
 EXTRA_TESTS = (
     "|".join(EXTRA_RELVALS_TESTS)
@@ -168,7 +169,7 @@ def format(s, **kwds):
     "disable_poison": ["true|false", "DISABLE_POISON"],
     "use_ib_tag": ["true|false", "USE_IB_TAG"],
     "baseline": ["self|default", "USE_BASELINE"],
-    "set_env": ["[A-Z][A-Z0-9_]+(\s*,\s*[A-Z][A-Z0-9_]+|)*", "CMSBOT_SET_ENV"],
+    "set_env": [r"[A-Z][A-Z0-9_]+(\s*,\s*[A-Z][A-Z0-9_]+|)*", "CMSBOT_SET_ENV"],
     "skip_test(s|)": [format(r"(%(tests)s)(\s*,\s*(%(tests)s))*", tests=SKIP_TESTS), "SKIP_TESTS"],
     "dry_run": ["true|false", "DRY_RUN"],
     "jenkins_(slave|node)": [JENKINS_NODES, "RUN_ON_SLAVE"],
@@ -1315,7 +1316,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
         elif re.match(REGEX_EX_ENABLE_TESTS, first_line, re.I):
             comment_emoji = "-1"
             if valid_commenter:
-                enable_tests, ignore = check_enable_bot_tests(first_line.split(" ", 1)[-1])
+                enable_tests, _ = check_enable_bot_tests(first_line.split(" ", 1)[-1])
                 comment_emoji = "+1"
         elif re.match(r"^allow\s+@([^ ]+)\s+test\s+rights$", first_line, re.I):
             comment_emoji = "-1"
@@ -1375,6 +1376,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
                 first_line, comment_lines, repository
             )
             if test_params_m:
+                # Error in parameters
                 test_params_msg = str(comment.id) + ":" + test_params_m
                 test_params_comment = comment
                 continue
@@ -1589,6 +1591,21 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
                         set_comment_emoji_cache(dryRun, bot_cache, comment, repository)
 
     # end of parsing comments section
+
+    # Extract enabled GPU flavors and remove them from enable_tests
+    new_enable_tests = []
+    enabled_gpu_flavors = set()
+    for test in enable_tests.split():
+        if test == "GPU":
+            enabled_gpu_flavors.update([x.upper() for x in ALL_GPU_FLAVORS])
+        elif test.lower() in ALL_GPU_FLAVORS:
+            enabled_gpu_flavors.add(test)
+        else:
+            new_enable_tests.append(test)
+
+    new_enable_tests.extend(list(enabled_gpu_flavors))
+    enable_tests = new_enable_tests
+
     # Check if it needs to be automatically closed.
     if mustClose:
         if issue.state == "open":

report-pull-request-results.py

@@ -411,7 +411,7 @@ def read_unit_tests_file(unit_tests_file):
     send_message_pr(message)
 
 
-def read_gpu_tests_file(unit_tests_file):
+def read_gpu_tests_file(unit_tests_file, gpu_flavor="GPU"):
     errors_found = ""
     err_cnt = 0
     for line in openlog(unit_tests_file):
@@ -423,8 +423,8 @@ def read_gpu_tests_file(unit_tests_file):
                 continue
             errors_found += line
     message = (
-        "\n## GPU Unit Tests\n\nI found %s errors in the following unit tests:\n\n<pre>%s</pre>"
-        % (err_cnt, errors_found)
+        "\n## %s Unit Tests\n\nI found %s errors in the following unit tests:\n\n<pre>%s</pre>"
+        % (gpu_flavor, err_cnt, errors_found)
     )
     send_message_pr(message)
 
@@ -606,6 +606,8 @@ def complain_missing_param(param_name):
 GITLOG_FILE_BASE_URL = "%s/git-recent-commits.json" % options.report_url
 GIT_CMS_MERGE_TOPIC_BASE_URL = "%s/git-merge-result" % options.report_url
 
+ACTION = ACTION.upper()
+
 if ACTION == "GET_BASE_MESSAGE":
     get_base_message()
 elif ACTION == "PARSE_UNIT_TESTS_FAIL":
@@ -630,6 +632,10 @@ def complain_missing_param(param_name):
     read_material_budget_log_file(options.unit_tests_file)
 elif ACTION == "MERGE_COMMITS":
     add_to_report(get_recent_merges_message())
+elif ACTION == "PARSE_CUDA_UNIT_TESTS_FAIL":
+    read_gpu_tests_file(options.unit_tests_file, "CUDA")
+elif ACTION == "PARSE_ROCM_UNIT_TESTS_FAIL":
+    read_gpu_tests_file(options.unit_tests_file, "ROCm")
 elif ACTION == "PARSE_GPU_UNIT_TESTS_FAIL":
     read_gpu_tests_file(options.unit_tests_file)
 else:

run-ib-pr-matrix.sh

@@ -1,6 +1,20 @@
 #!/bin/sh -ex
+function is_in_array() {
+    local value="$1"
+    shift
+    local array=("$@")
+
+    for item in "${array[@]}"; do
+        if [[ "$item" == "$value" ]]; then
+            return 0  # Found match
+        fi
+    done
+    return 1  # No match
+}
+
 TEST_FLAVOR=$1
 CMS_BOT_DIR=$(cd $(dirname $0) >/dev/null 2>&1; pwd -P)
+readarray -t ALL_GPU_TYPES < ${CMS_BOT_DIR}/gpu_flavors.txt
 ARTIFACT_DIR="ib-baseline-tests/${RELEASE_FORMAT}/${ARCHITECTURE}/${REAL_ARCH}/matrix${TEST_FLAVOR}-results"
 source $CMS_BOT_DIR/jenkins-artifacts
 #Run on any machine to see which workflows should be run
@@ -9,10 +23,12 @@ if [ "${CHECK_WORKFLOWS}" = "true" ] ; then
   send_jenkins_artifacts ${WORKSPACE}/workflows-${BUILD_ID}.log ${ARTIFACT_DIR}/workflows-${BUILD_ID}.log
   OPTS=""
   case "${TEST_FLAVOR}" in
-    gpu ) OPTS="-w gpu" ;;
     high_stats ) ;;
     nano ) OPTS="-w nano" ;;
-    * ) ;;
+    * ) if is_in_array "${TEST_FLAVOR}" "${ALL_GPU_TYPES[@]}" ; then
+          OPTS="-w gpu"
+        fi
+        ;;
   esac
   REL_WFS=""
   if has_jenkins_artifacts ${ARTIFACT_DIR} -d ; then
@@ -47,14 +63,20 @@ UC_TEST_FLAVOR=$(echo ${TEST_FLAVOR} | tr '[a-z]' '[A-Z]')
 pushd "$WORKSPACE/matrix-results"
   NJOBS=$(nproc)
   CMD_OPTS=""
-  if ${PRODUCTION_RELEASE} && cmsDriver.py --help | grep -q '\-\-maxmem_profile'  ; then CMD_OPTS="--maxmem_profile" ; fi
+  if ${PRODUCTION_RELEASE} && cmsDriver.py --help | grep -q '\-\-maxmem_profile'  ; then
+    if [ "TEST_FLAVOR" != "rocm" ]; then
+      CMD_OPTS="--maxmem_profile"
+    fi
+  fi
   case "${TEST_FLAVOR}" in
-    gpu )        MATRIX_ARGS="-w gpu ${MATRIX_ARGS}" ;;
     high_stats ) CMD_OPTS="-n 500" ; MATRIX_ARGS="-i all ${MATRIX_ARGS}" ;;
     threading )  MATRIX_ARGS="-i all -t 4 ${MATRIX_ARGS}" ; let NJOBS=(${NJOBS}/4)+1 ;;
     nano )       MATRIX_ARGS="-w nano -i all ${MATRIX_ARGS}" ;;
     input )      MATRIX_ARGS="-i all --maxSteps=2 ${MATRIX_ARGS}" ; CMD_OPTS="-n 1 --prefix ${CMS_BOT_DIR}/pr_testing/retry-command.sh" ; export CMS_BOT_RETRY_COUNT=3 ;;
-    * ) ;;
+    * ) if is_in_array "${TEST_FLAVOR}" "${ALL_GPU_TYPES[@]}" ; then
+          MATRIX_ARGS="-w gpu ${MATRIX_ARGS}"
+        fi
+        ;;
   esac
   [ $(runTheMatrix.py --help | grep 'job-reports' | wc -l) -gt 0 ] && MATRIX_ARGS="--job-reports $MATRIX_ARGS"
   [ -f ${CMSSW_RELEASE_BASE}/src/Validation/Performance/python/TimeMemoryJobReport.py ] && CMD_OPTS="${CMD_OPTS} --customise Validation/Performance/TimeMemoryJobReport.customiseWithTimeMemoryJobReport"

tests/PRActionData/TestProcessPr.test_abort.json

@@ -110,7 +110,7 @@
                 "SKIP_TESTS": "header,static",
                 "PULL_REQUESTS": "iarspider-cmssw/cmssw#17 cms-sw/cms-bot#2134",
                 "RELEASE_FORMAT": "CMSSW_14_1_CPP20_X",
-                "EXTRA_RELVALS_TESTS": "THREADING GPU HIGH_STATS NANO",
+                "EXTRA_RELVALS_TESTS": "THREADING GPU HIGH_STATS NANO CUDA ROCM",
                 "CONTEXT_PREFIX": "cms/17"
             }
         }

tests/PRActionData/TestProcessPr.test_draft_pr_ready.json

@@ -70,7 +70,7 @@
             "filename": "trigger-tests-iarspider-cmssw-cmssw-21.properties",
             "data": {
                 "PULL_REQUESTS": "iarspider-cmssw/cmssw#21",
-                "EXTRA_RELVALS_TESTS": "THREADING GPU HIGH_STATS NANO",
+                "EXTRA_RELVALS_TESTS": "THREADING GPU HIGH_STATS NANO CUDA ROCM",
                 "CONTEXT_PREFIX": "cms/21"
             }
         }

tests/PRActionData/TestProcessPr.test_draft_pr_start_test.json

@@ -70,7 +70,7 @@
             "filename": "trigger-tests-iarspider-cmssw-cmssw-21.properties",
             "data": {
                 "PULL_REQUESTS": "iarspider-cmssw/cmssw#21",
-                "EXTRA_RELVALS_TESTS": "THREADING GPU HIGH_STATS NANO",
+                "EXTRA_RELVALS_TESTS": "THREADING GPU HIGH_STATS NANO CUDA ROCM",
                 "CONTEXT_PREFIX": "cms/21"
             }
         }