Skip to content

Commit 4d1b12a

Browse files
zhangzujianzhangzujian
committed
ci: fix valgrind result analysis (#2853)
1 parent e1b7919 commit 4d1b12a

File tree

1 file changed

+179
-13
lines changed

1 file changed

+179
-13
lines changed

‎.github/workflows/build-x86-image.yaml

+179-13
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ jobs:
2828
build-kube-ovn-base:
2929
name: Build kube-ovn-base
3030
runs-on: ubuntu-22.04
31+
outputs:
32+
build-base: ${{ steps.build.outputs.build-base }}
3133
steps:
3234
- uses: actions/checkout@v3
3335
with:
@@ -116,7 +118,7 @@ jobs:
116118
go mod tidy
117119
git diff --exit-code
118120
make lint
119-
if [ "x${{ env.BUILD_BASE }}" = "x1" ]; then
121+
if [ ${{ needs.build-kube-ovn-base.outputs.build-base || 0 }} = 1 ]; then
120122
TAG=$(cat VERSION)
121123
docker tag kubeovn/kube-ovn-base:$TAG-amd64 kubeovn/kube-ovn-base:$TAG
122124
docker tag kubeovn/kube-ovn-base:$TAG-amd64-no-avx512 kubeovn/kube-ovn-base:$TAG-no-avx512
@@ -369,19 +371,29 @@ jobs:
369371
kubectl -n kube-system rollout status deploy ovn-central
370372
kubectl -n kube-system delete po -l app=ovs
371373
kubectl -n kube-system wait pod -l app=ovs --for condition=Ready --timeout=90s
374+
while true; do
375+
if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then
376+
break
377+
fi
378+
sleep 1
379+
done
372380
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovn
373381
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovs
374382
375-
exit_code=0
376-
find kubectl-ko-log -type f -name '*.valgrind.*' | while read f; do
383+
for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
384+
echo "Checking if valgrind log file for $daemon exists..."
385+
find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
386+
done
387+
388+
find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
377389
if grep -qw 'definitely lost' "$f"; then
378-
exit_code=1
379-
echo $f; cat "$f";
390+
echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
391+
echo $f
392+
cat "$f"
393+
exit 1
380394
fi;
381395
done
382396
383-
exit $exit_code
384-
385397
k8s-netpol-e2e:
386398
name: Kubernetes Network Policy E2E
387399
if: |
@@ -462,6 +474,9 @@ jobs:
462474
- name: Load image
463475
run: docker load --input kube-ovn.tar
464476

477+
- name: Export debug image tag
478+
run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
479+
465480
- name: Create kind cluster
466481
run: |
467482
sudo pip3 install j2cli
@@ -471,12 +486,57 @@ jobs:
471486
sudo chown -R $(id -un). ~/.kube/
472487
473488
- name: Install Kube-OVN
489+
env:
490+
VERSION: ${{ env.DEBUG_TAG }}
491+
DEBUG_WRAPPER: valgrind
474492
run: make kind-install-${{ matrix.ip-family }}
475493

476494
- name: Run E2E
477495
working-directory: ${{ env.E2E_DIR }}
478496
run: make k8s-netpol-e2e
479497

498+
- name: kubectl ko log
499+
if: failure()
500+
run: |
501+
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log all
502+
mv kubectl-ko-log.tar.gz k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
503+
504+
- name: upload kubectl ko log
505+
uses: actions/upload-artifact@v3
506+
if: failure()
507+
with:
508+
name: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log
509+
path: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
510+
511+
- name: Check valgrind result
512+
run: |
513+
kubectl -n kube-system rollout restart deploy ovn-central
514+
kubectl -n kube-system rollout status deploy ovn-central
515+
kubectl -n kube-system delete po -l app=ovs
516+
kubectl -n kube-system wait pod -l app=ovs --for condition=Ready --timeout=90s
517+
while true; do
518+
if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then
519+
break
520+
fi
521+
sleep 1
522+
done
523+
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovn
524+
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovs
525+
526+
for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
527+
echo "Checking if valgrind log file for $daemon exists..."
528+
find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
529+
done
530+
531+
find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
532+
if grep -qw 'definitely lost' "$f"; then
533+
echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
534+
echo $f
535+
cat "$f"
536+
exit 1
537+
fi;
538+
done
539+
480540
k8s-netpol-legacy-e2e:
481541
name: Kubernetes Network Policy Legacy E2E
482542
if: |
@@ -557,6 +617,9 @@ jobs:
557617
- name: Load image
558618
run: docker load --input kube-ovn.tar
559619

620+
- name: Export debug image tag
621+
run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
622+
560623
- name: Create kind cluster
561624
run: |
562625
sudo pip3 install j2cli
@@ -566,12 +629,57 @@ jobs:
566629
sudo chown -R $(id -un). ~/.kube/
567630
568631
- name: Install Kube-OVN
632+
env:
633+
VERSION: ${{ env.DEBUG_TAG }}
634+
DEBUG_WRAPPER: valgrind
569635
run: make kind-install-${{ matrix.ip-family }}
570636

571637
- name: Run E2E
572638
working-directory: ${{ env.E2E_DIR }}
573639
run: make k8s-netpol-legacy-e2e
574640

641+
- name: kubectl ko log
642+
if: failure()
643+
run: |
644+
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log all
645+
mv kubectl-ko-log.tar.gz k8s-netpol-legacy-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
646+
647+
- name: upload kubectl ko log
648+
uses: actions/upload-artifact@v3
649+
if: failure()
650+
with:
651+
name: k8s-netpol-legacy-e2e-${{ matrix.ip-family }}-ko-log
652+
path: k8s-netpol-legacy-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
653+
654+
- name: Check valgrind result
655+
run: |
656+
kubectl -n kube-system rollout restart deploy ovn-central
657+
kubectl -n kube-system rollout status deploy ovn-central
658+
kubectl -n kube-system delete po -l app=ovs
659+
kubectl -n kube-system wait pod -l app=ovs --for condition=Ready --timeout=90s
660+
while true; do
661+
if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then
662+
break
663+
fi
664+
sleep 1
665+
done
666+
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovn
667+
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovs
668+
669+
for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
670+
echo "Checking if valgrind log file for $daemon exists..."
671+
find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
672+
done
673+
674+
find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
675+
if grep -qw 'definitely lost' "$f"; then
676+
echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
677+
echo $f
678+
cat "$f"
679+
exit 1
680+
fi;
681+
done
682+
575683
cyclonus-netpol-e2e:
576684
name: Cyclonus Network Policy E2E
577685
if: |
@@ -629,6 +737,9 @@ jobs:
629737
- name: Load image
630738
run: docker load --input kube-ovn.tar
631739

740+
- name: Export debug image tag
741+
run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"
742+
632743
- name: Create kind cluster
633744
run: |
634745
sudo pip3 install j2cli
@@ -638,12 +749,57 @@ jobs:
638749
sudo chown -R $(id -un). ~/.kube/
639750
640751
- name: Install Kube-OVN
752+
env:
753+
VERSION: ${{ env.DEBUG_TAG }}
754+
DEBUG_WRAPPER: valgrind
641755
run: make kind-install-${{ matrix.ip-family }}
642756

643757
- name: Run E2E
644758
working-directory: ${{ env.E2E_DIR }}
645759
run: make cyclonus-netpol-e2e
646760

761+
- name: kubectl ko log
762+
if: failure()
763+
run: |
764+
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log all
765+
mv kubectl-ko-log.tar.gz cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
766+
767+
- name: upload kubectl ko log
768+
uses: actions/upload-artifact@v3
769+
if: failure()
770+
with:
771+
name: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log
772+
path: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
773+
774+
- name: Check valgrind result
775+
run: |
776+
kubectl -n kube-system rollout restart deploy ovn-central
777+
kubectl -n kube-system rollout status deploy ovn-central
778+
kubectl -n kube-system delete po -l app=ovs
779+
kubectl -n kube-system wait pod -l app=ovs --for condition=Ready --timeout=90s
780+
while true; do
781+
if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then
782+
break
783+
fi
784+
sleep 1
785+
done
786+
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovn
787+
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovs
788+
789+
for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
790+
echo "Checking if valgrind log file for $daemon exists..."
791+
find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
792+
done
793+
794+
find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
795+
if grep -qw 'definitely lost' "$f"; then
796+
echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
797+
echo $f
798+
cat "$f"
799+
exit 1
800+
fi;
801+
done
802+
647803
kube-ovn-conformance-e2e:
648804
name: Kube-OVN Conformance E2E
649805
needs:
@@ -749,19 +905,29 @@ jobs:
749905
kubectl -n kube-system rollout status deploy ovn-central
750906
kubectl -n kube-system delete po -l app=ovs
751907
kubectl -n kube-system wait pod -l app=ovs --for condition=Ready --timeout=90s
908+
while true; do
909+
if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then
910+
break
911+
fi
912+
sleep 1
913+
done
752914
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovn
753915
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovs
754916
755-
exit_code=0
756-
find kubectl-ko-log -type f -name '*.valgrind.*' | while read f; do
917+
for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
918+
echo "Checking if valgrind log file for $daemon exists..."
919+
find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
920+
done
921+
922+
find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
757923
if grep -qw 'definitely lost' "$f"; then
758-
exit_code=1
759-
echo $f; cat "$f";
924+
echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
925+
echo $f
926+
cat "$f"
927+
exit 1
760928
fi;
761929
done
762930
763-
exit $exit_code
764-
765931
- name: Cleanup
766932
run: sh dist/images/cleanup.sh
767933

0 commit comments

Comments
 (0)