Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jenkinsfile rework #1096

Draft
wants to merge 25 commits into
base: develop
Choose a base branch
from
Draft

Jenkinsfile rework #1096

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
94bffef
Update Jenkinsfile
FernandoAndrade-NOAA Nov 22, 2024
85100d7
Merge branch 'NOAA-EMC:develop' into ci-testing
FernandoAndrade-NOAA Nov 22, 2024
a350aa9
Update Jenkinsfile rm baseline logic
FernandoAndrade-NOAA Nov 22, 2024
4e8e665
Merge branch 'NOAA-EMC:develop' into ci-testing
FernandoAndrade-NOAA Dec 3, 2024
b754b12
Update Jenkinsfile
FernandoAndrade-NOAA Dec 11, 2024
2538d8f
Update Jenkinsfile
FernandoAndrade-NOAA Dec 11, 2024
5d38cf2
[AutoRT] Hera Job Completed.
epic-cicd-jenkins Dec 11, 2024
0cc61b9
[AutoRT] Hera Job Completed.
epic-cicd-jenkins Dec 11, 2024
a970bf5
Update Jenkinsfile
FernandoAndrade-NOAA Dec 11, 2024
a4fcf9b
Update Jenkinsfile
FernandoAndrade-NOAA Dec 11, 2024
b65ecf4
Merge branch 'develop' into ci-testing
FernandoAndrade-NOAA Jan 21, 2025
577f339
[AutoRT] Hera Job Completed.
epic-cicd-jenkins Jan 21, 2025
ea2a3e3
Update Jenkinsfile remove BL label ref
FernandoAndrade-NOAA Jan 21, 2025
7c2c94e
Update Jenkinsfile
FernandoAndrade-NOAA Jan 21, 2025
96a6791
Update Jenkinsfile
FernandoAndrade-NOAA Jan 21, 2025
4c0dd29
fix typo
FernandoAndrade-NOAA Jan 22, 2025
8696745
[AutoRT] Orion Job Completed.
epic-cicd-jenkins Jan 22, 2025
da50e3a
Merge branch 'develop' into ci-testing
FernandoAndrade-NOAA Jan 22, 2025
f67a5a7
[AutoRT] Hera Job Completed.
epic-cicd-jenkins Jan 22, 2025
a097288
[AutoRT] Hercules Job Completed.
epic-cicd-jenkins Jan 22, 2025
5c01e23
[AutoRT] Hercules Job Completed.
epic-cicd-jenkins Jan 22, 2025
dce1f68
Merge branch 'develop' into ci-testing
FernandoAndrade-NOAA Jan 24, 2025
212095c
[AutoRT] Hera Job Completed.
epic-cicd-jenkins Jan 24, 2025
ad9ce47
[AutoRT] Orion Job Completed.
epic-cicd-jenkins Jan 24, 2025
8e176f5
[AutoRT] Orion Job Completed.
epic-cicd-jenkins Jan 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
372 changes: 143 additions & 229 deletions ci/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,238 +1,152 @@
def formatComment(machine) {
script {
def buildResult = currentBuild.resultIsBetterOrEqualTo("SUCCESS") ? "PASSED" : "FAILED"
def commentMessage = "${machine} UPP Regression Tests ${buildResult}"
if (fileExists('ci/changed_results.txt')) {
echo 'changed_results.txt file found'
sh "cat ci/changed_results.txt"
def changedResults = readFile(file: 'ci/changed_results.txt')
def failMessage = commentMessage + "\n\n" + changedResults
def formattedMessage = failMessage.replaceAll("\n", "\\\\n")
postGitHubComment(formattedMessage)
sh "rm -rf ci/changed_results.txt"
} else {
echo 'changed_results.txt file not found'
postGitHubComment(commentMessage)
}
}
matchedNode = []
for (label in pullRequest.labels) {
listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect {
node -> node.getLabelString().contains(label) ? node.name : null

if ((label.matches(node.getLabelString()+"-(.*)"))) {
matchedNode += node.getLabelString()
}
}
}

modifiedLabels = matchedNode.collect{"'" + it + "'"}
def generateStage(nodeLabel) {
return {
stage("Running on ${nodeLabel}") {
node(nodeLabel) {
cleanWs()
checkout scm
script {
try {
echo "Running on ${nodeLabel}"
sh '''
pwd
cd tests
export machine=${NODE_NAME}
export PATH=$PATH:~/bin
echo $CHANGE_ID
export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/NOAA-EMC/UPP/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url')
export FORK_BRANCH=$(curl --silent https://api.github.com/repos/NOAA-EMC/UPP/pulls/$CHANGE_ID | jq -r '.head.ref')

if [[ $machine =~ "Hercules" ]]
then
echo "Running regression tests on $machine"
export dprefix=/work2/noaa/$ACCNR/$USER
cd $WORKSPACE/ci
sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh
export ACCNR=epic
./rt.sh -a ${ACCNR} | tee $WORKSPACE/tests/logs/RT-run-$machine.log
cd $WORKSPACE/tests/logs/
cp rt.log.HERCULES /work/noaa/epic/role-epic/jenkins/workspace
git remote -v
git fetch --no-recurse-submodules origin
git reset FETCH_HEAD --hard
cd .. && cd .. && cd ..
cp rt.log.HERCULES $WORKSPACE/tests/logs/
cd $WORKSPACE/tests/
elif [[ $machine =~ "Orion" ]]
then
echo "Running regression tests on $machine"
export dprefix=/work2/noaa/$ACCNR/$USER
cd $WORKSPACE/ci
sed "s|/noaa/stmp/|/noaa/$ACCNR/stmp/|g" -i rt.sh
./rt.sh -a ${ACCNR} | tee $WORKSPACE/tests/logs/RT-run-$machine.log
cd $WORKSPACE/tests/logs/
cp rt.log.ORION /work/noaa/epic/role-epic/jenkins/workspace
git remote -v
git fetch --no-recurse-submodules origin
git reset FETCH_HEAD --hard
cd .. && cd .. && cd ..
cp rt.log.ORION $WORKSPACE/tests/logs/
cd $WORKSPACE/tests/
elif [[ $machine =~ "Hera" ]]
then
echo "Running regression tests on $machine"
export ACCNR=epic
cd $WORKSPACE/ci
./rt.sh -a ${ACCNR} | tee $WORKSPACE/tests/logs/RT-run-$machine.log
cd $WORKSPACE/tests/logs
cp rt.log.HERA /scratch2/NAGAPE/epic/role.epic/jenkins/workspace
git remote -v
git fetch --no-recurse-submodules origin
git reset FETCH_HEAD --hard
cd .. && cd .. && cd ..
cp rt.log.HERA $WORKSPACE/tests/logs/
cd $WORKSPACE/tests/
else
echo "Running regression tests on $machine"
./rt.sh -a ${ACCNR} | tee $WORKSPACE/tests/logs/RT-run-$machine.log
fi

git config user.email "ecc.platform@noaa.gov"
git config user.name "epic-cicd-jenkins"
export machine_name_logs=$(echo $machine | awk '{ print toupper($1) }')
echo "Testing concluded...removing labels for $machine from $GIT_URL"
git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1
git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1
git add logs/rt.log.$machine_name_logs
git commit -m "[AutoRT] $machine Job Completed.\n\n\n on-behalf-of @ufs-community <ecc.platform@noaa.gov>"
git pull sshorigin $FORK_BRANCH
git push sshorigin HEAD:$FORK_BRANCH

tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log

GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4)
GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1)

def postGitHubComment(commentMessage) {
script {
withCredentials([string(credentialsId: 'GithubJenkinsNew', variable: 'ACCESS_TOKEN')]) {
def apiUrl = "https://api.github.com/repos/NOAA-EMC/UPP/issues/${env.CHANGE_ID}/comments"
def curlCommand = "curl -s -H \"Authorization: token " + ACCESS_TOKEN + "\" \\\n" +
"-X POST -d '{\"body\": \"" + commentMessage + "\"}' \\\n" +
"\"" + apiUrl + "\""
curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/$machine-RT

'''
}
catch(err) {
sh '''
export machine=${NODE_NAME}
export CHANGE_ID=${CHANGE_ID}
export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/NOAA-EMC/UPP/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url')
export FORK_BRANCH=$(curl --silent https://api.github.com/repos/NOAA-EMC/UPP/pulls/$CHANGE_ID | jq -r '.head.ref')
cd $WORKSPACE/tests
git config user.email "ecc.platform@noaa.gov"
git config user.name "epic-cicd-jenkins"
export machine_name_logs=$(echo $machine | awk '{ print toupper($1) }')
echo "Testing concluded...removing labels for $machine from $GIT_URL"
git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1
git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1

def response = sh(script: curlCommand, returnStatus: true)
if (response == 0) {
echo "Comment added successfully to PR #${env.CHANGE_ID}"
} else {
error "Failed to add comment to PR #${env.CHANGE_ID}"
}
}
}
tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log

GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4)
GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1)

curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/$machine-RT
'''
currentBuild.result = 'FAILURE'
}
}
}
}
}
}

def parallelStagesMap = modifiedLabels.collectEntries {
["${it}" : generateStage(it)]
}

pipeline {
agent none
stages {
stage('UPP Regression Tests') {
agent {
label 'built-in'
}
steps {
script {
for (label in pullRequest.labels) {
if ((label.matches("orion"))) {
env.CHOICE_NODE='orion'
}
else if ((label.matches("hera"))) {
env.CHOICE_NODE='hera'
}
else if ((label.matches("hercules"))) {
env.CHOICE_NODE='hercules'
}
else if ((label.matches("jet"))) {
env.CHOICE_NODE='jet'
}
else {
env.CHOICE_NODE='none'
}
}
// Why do I need another if..block, because it just works this way.
if (CHOICE_NODE == 'orion') {
echo "Starting up orion ${CHOICE_NODE}...this might take 5-10 minutes...please be patient."
}
else if (CHOICE_NODE == 'jet') {
echo "Starting up jet ${CHOICE_NODE}...this might take 5-10 minutes...please be patient."
}
else if (CHOICE_NODE == 'hercules') {
echo "Starting up hera ${CHOICE_NODE}...this might take 5-10 minutes...please be patient."
}
else if (CHOICE_NODE == 'hera') {
echo "Starting up hera ${CHOICE_NODE}...this might take 5-10 minutes...please be patient."
}
else {
echo "${CHOICE_NODE} is NOT a platform, moving on..."
}
}
}
}
stage('Run UPP RTs on Orion') {
agent {
label "orion"
}
environment {
ACCNR = 'epic'
NODE_PATH = '/work/noaa/epic/UPP/jenkins-ci/orion'
}
steps {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
cleanWs()
checkout scm
sh '''
echo $(pwd)
cd ci/
echo $CHANGE_ID
export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/NOAA-EMC/UPP/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url')
export FORK_BRANCH=$(curl --silent https://api.github.com/repos/NOAA-EMC/UPP/pulls/$CHANGE_ID | jq -r '.head.ref')
./rt.sh -a ${ACCNR} -r `pwd`/rundir
cd ../tests/logs
git remote -v
git fetch --no-recurse-submodules origin
git config user.email "ecc.platform@noaa.gov"
git config user.name "epic-cicd-jenkins"
echo "Orion testing concluded..."
git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1
git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1
git pull sshorigin $FORK_BRANCH
git add rt.log.ORION
git commit -m "Orion Jobs Completed."
git push sshorigin HEAD:$FORK_BRANCH
'''
}
}
post {
always {
script {
formatComment("Orion")
}
}
}
}
stage('Run UPP RTs on Hera') {
agent {
label "hera"
}
environment {
ACCNR = 'epic'
NODE_PATH = '/scratch2/NAGAPE/epic/UPP/jenkins-ci'
}
steps {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
cleanWs()
checkout scm
sh '''
echo $(pwd)
cd ci/
echo $CHANGE_ID
export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/NOAA-EMC/UPP/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url')
export FORK_BRANCH=$(curl --silent https://api.github.com/repos/NOAA-EMC/UPP/pulls/$CHANGE_ID | jq -r '.head.ref')
./rt.sh -a ${ACCNR} -r `pwd`/rundir
cd ../tests/logs
git remote -v
git fetch --no-recurse-submodules origin
git config user.email "ecc.platform@noaa.gov"
git config user.name "epic-cicd-jenkins"
echo "Hera testing concluded..."
git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1
git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1
git pull sshorigin $FORK_BRANCH
git add rt.log.HERA
git commit -m "Hera Jobs Completed."
git push sshorigin HEAD:$FORK_BRANCH
'''
}
}
post {
always {
script {
formatComment("Hera")
}
}
}
}
stage('Run UPP RTs on Hercules') {
agent {
label "hercules"
}
environment {
ACCNR = 'epic'
NODE_PATH = '/work/noaa/epic/UPP/jenkins-ci/hercules'
}
steps {
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
cleanWs()
checkout scm
sh '''
echo $(pwd)
cd ci/
echo $CHANGE_ID
export SSH_ORIGIN=$(curl --silent https://api.github.com/repos/NOAA-EMC/UPP/pulls/$CHANGE_ID | jq -r '.head.repo.ssh_url')
export FORK_BRANCH=$(curl --silent https://api.github.com/repos/NOAA-EMC/UPP/pulls/$CHANGE_ID | jq -r '.head.ref')
./rt.sh -a ${ACCNR} -r `pwd`/rundir
cd ../tests/logs
git remote -v
git fetch --no-recurse-submodules origin
git config user.email "ecc.platform@noaa.gov"
git config user.name "epic-cicd-jenkins"
echo "Hercules testing concluded..."
git remote -v | grep -w sshorigin > /dev/null 2>&1 && git remote remove sshorigin > /dev/null 2>&1
git remote add sshorigin $SSH_ORIGIN > /dev/null 2>&1
git pull sshorigin $FORK_BRANCH
git add rt.log.HERCULES
git commit -m "Hercules Jobs Completed."
git push sshorigin HEAD:$FORK_BRANCH
'''
}
}
post {
always {
script {
formatComment("Hercules")
}
}
}
}
// TO BE ADDED...
// stage('Run UPP RTs on Jet') {
// agent {
// label "jet"
// }
// environment {
// ACCNR = 'hfv3gfs'
// NODE_PATH = '/mnt/lfs4/HFIP/hfv3gfs/role.epic/'
// }
// steps {
// catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
// cleanWs()
// checkout scm
// sh '''

// '''
// }
// }
// post {
// always {
// script {
// formatComment("Jet")
// }
// }
// }
// }
}
agent none
environment {
ACCNR = 'epic'
AWS_PROD_ACCOUNT_ID = credentials('AWS_PROD_ACCOUNT_ID')
AWS_PROD_SNS_TOPIC = credentials('AWS_PROD_SNS_TOPIC')
GITHUB_TOKEN = credentials('GithubJenkinsNew')
GIT_URL = 'https://github.com/NOAA-EMC/UPP.git'
}
stages {
stage('Run Regression Tests in Parallel') {
steps {
script {
parallel parallelStagesMap
}
}
}
}
}
Loading