Merge branch 'bugfix' into fix_aws_endpoints

quirinziessler · Mar 4, 2025 · 131994e · 131994e
2 parents 75fb024 + fa9bd73
commit 131994e
Show file tree

Hide file tree

Showing 295 changed files with 4,637 additions and 2,722 deletions.
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -60,3 +60,8 @@ localization:
     - any-glob-to-any-file:
       - dojo/locale/*
       - dojo/locale/**/*
+
+lint:
+  - changed-files:
+    - any-glob-to-any-file:
+      - ruff.toml
diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml
@@ -41,6 +41,8 @@ categories:
     label: 'ui'
   - title: '🗣 Updates in localization'
     label: 'localization'
+  - title: '🔧 Improved code quality with linters'
+    label: 'lint'
   - title: '🧰 Maintenance'
     collapse-after: 3
     labels:

diff --git a/.github/workflows/build-docker-images-for-testing.yml b/.github/workflows/build-docker-images-for-testing.yml
@@ -1,23 +1,38 @@
 name: "Build Docker Images For Testing"
 
 on:
+  # inspired by https://github.com/orgs/community/discussions/26801 we take the runs-on as parameter so we can run arm64 on native hardwarde
   workflow_dispatch:
+    inputs:
+        runner:
+          type: string
+          default: "ubuntu-latest"
   workflow_call:
+    inputs:
+        runner:
+          type: string
+          default: "ubuntu-latest"
 
 jobs:
   build:
-    # build with docker so we can use layer caching
     name: Build Docker Images
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.runner }}
     strategy:
       matrix:
+        # integration tests are only build (and run) on debian x64
         docker-image: [django, nginx, integration-tests]
         os: [alpine, debian]
+        runner: ["${{ inputs.runner }}"]
         exclude:
-          - docker-image: integration-tests
-            os: alpine
+            - docker-image: integration-tests
+              os: alpine
+            - docker-image: integration-tests
+              runner: ubuntu-24.04-arm
 
     steps:
+      - name: Show Platform
+        run: echo Runner value for this run = ${{ inputs.runner }}
+
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -28,29 +43,26 @@ jobs:
         run: echo "IMAGE_REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
-        with:
-          buildkitd-flags: --debug
-          driver-opts: image=moby/buildkit:master # needed to get the fix for https://github.com/moby/buildkit/issues/2426
-
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+
       - name: Build
         id: docker_build
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
-        timeout-minutes: 10
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
+        timeout-minutes: 15
         env:
           DOCKER_BUILD_CHECKS_ANNOTATIONS: false
         with:
           context: .
           push: false
-          tags: defectdojo/defectdojo-${{ matrix.docker-image }}:${{ matrix.os }}
+          tags: defectdojo/defectdojo-${{ matrix.docker-image }}:${{ matrix.os }},${{ env.IMAGE_REPOSITORY }}/defectdojo-${{ matrix.docker-image }}:${{ matrix.os }}
           file: Dockerfile.${{ matrix.docker-image }}-${{ matrix.os }}
-          outputs: type=docker,dest=${{ matrix.docker-image }}-${{ matrix.os }}_img
+          outputs: type=docker,dest=${{ matrix.docker-image }}-${{ matrix.os }}-${{ inputs.runner }}_img
 
       # export docker images to be used in next jobs below
       - name: Upload image ${{ matrix.docker-image }} as artifact
-        timeout-minutes: 10
-        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        timeout-minutes: 15
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
-          name: built-docker-image-${{ matrix.docker-image }}-${{ matrix.os }}
-          path: ${{ matrix.docker-image }}-${{ matrix.os }}_img
+          name: built-docker-image-${{ matrix.docker-image }}-${{ matrix.os }}-${{ inputs.runner }}
+          path: ${{ matrix.docker-image }}-${{ matrix.os }}-${{ inputs.runner }}_img
           retention-days: 1
diff --git a/.github/workflows/fetch-oas.yml b/.github/workflows/fetch-oas.yml
@@ -51,7 +51,7 @@ jobs:
         run: docker compose down
 
       - name: Upload oas.${{ matrix.file-type }} as artifact
-        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: oas-${{ matrix.file-type }}
           path: oas.${{ matrix.file-type }}

diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
@@ -24,7 +24,7 @@ jobs:
           node-version: '22.5.1'
 
       - name: Cache dependencies
-        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -45,7 +45,7 @@ jobs:
 
       # load docker images from build jobs
       - name: Load images from artifacts
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
         with:
           path: built-docker-image
           pattern: built-docker-image-*
@@ -54,15 +54,15 @@ jobs:
       - name: Load docker images
         timeout-minutes: 10
         run: |-
-             docker load -i built-docker-image/nginx-${{ matrix.os }}_img
-             docker load -i built-docker-image/django-${{ matrix.os }}_img
-             docker load -i built-docker-image/integration-tests-debian_img
+             docker load -i built-docker-image/nginx-${{ matrix.os }}-ubuntu-latest_img
+             docker load -i built-docker-image/django-${{ matrix.os }}-ubuntu-latest_img
+             docker load -i built-docker-image/integration-tests-debian-ubuntu-latest_img
              docker images
 
       - name: Set integration-test mode
         run: ln -s docker-compose.override.integration_tests.yml docker-compose.override.yml
 
-      - name: Start Dojo 
+      - name: Start Dojo
         run: docker compose up --no-deps -d postgres nginx celerybeat celeryworker mailhog uwsgi redis
         env:
           DJANGO_VERSION: ${{ matrix.os }}

diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
@@ -48,18 +48,18 @@ jobs:
           minikube status
 
       - name: Load images from artifacts
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
         with:
           path: built-docker-image
           pattern: built-docker-image-*
           merge-multiple: true
 
       - name: Load docker images
-        timeout-minutes: 10
+        timeout-minutes: 15
         run: |-
              eval $(minikube docker-env)
-             docker load -i built-docker-image/nginx-${{ matrix.os }}_img
-             docker load -i built-docker-image/django-${{ matrix.os }}_img
+             docker load -i built-docker-image/nginx-${{ matrix.os }}-ubuntu-latest_img
+             docker load -i built-docker-image/django-${{ matrix.os }}-ubuntu-latest_img
              docker images
 
       - name: Configure HELM repos
@@ -85,6 +85,7 @@ jobs:
                ./helm/defectdojo \
                --set django.ingress.enabled=true \
                --set imagePullPolicy=Never \
+               --set initializer.keepSeconds="-1" \
                ${{ env[matrix.databases] }} \
                ${{ env[matrix.brokers] }} \
                --set createSecret=true \
@@ -124,10 +125,15 @@ jobs:
              RETRY=0
              while :
                do
+               DJANGO_IP=$(kubectl get svc defectdojo-django -o jsonpath='{.spec.clusterIP}')
                OUT=$(kubectl run  curl --quiet=true --image=curlimages/curl:7.73.0 \
                   --overrides='{ "apiVersion": "v1" }' \
-                  --restart=Never -i --rm -- -s -m 20 -I --header "Host: $DD_HOSTNAME" http://`kubectl get service defectdojo-django -o json \
-                 |  jq -r '.spec.clusterIP'`/login?next=/)
+                  --restart=Never -i --rm -- \
+                    --silent \
+                    --max-time 20 \
+                    --head \
+                    --header "Host: $DD_HOSTNAME" \
+                    http://$DJANGO_IP/login?next=/)
                echo $OUT
                CR=`echo $OUT | egrep "^HTTP" | cut -d' ' -f2`
                echo $CR
@@ -148,6 +154,26 @@ jobs:
                 break
                fi
              done
+             ADMIN_PASS=$(kubectl get secret/defectdojo -o jsonpath='{.data.DD_ADMIN_PASSWORD}' | base64 -d)
+             echo "Simple API check"
+             DJANGO_IP=$(kubectl get svc defectdojo-django -o jsonpath='{.spec.clusterIP}')
+             CR=$(kubectl run  curl --quiet=true --image=curlimages/curl:7.73.0 \
+               --overrides='{ "apiVersion": "v1" }' \
+               --restart=Never -i --rm -- \
+                 --silent \
+                 --max-time 20 \
+                 --header "Host: $DD_HOSTNAME" \
+                 --data-raw "username=admin&password=$ADMIN_PASS" \
+                 --output /dev/null \
+                 --write-out "%{http_code}\n" \
+                 http://$DJANGO_IP/api/v2/api-token-auth/)
+             echo $CR
+             if [[ $CR -ne 200 ]]; then
+              echo "ERROR: login is not possible; got HTTP code $CR"
+              exit 1
+             else
+              echo "Result received"
+             fi
              echo "Final Check of components"
              errors=`kubectl get pods | grep Error | awk '{print  $1}'`
              if [[ ! -z  $errors ]]; then

diff --git a/.github/workflows/release-2-tag-docker-push.yml b/.github/workflows/release-2-tag-docker-push.yml
@@ -3,7 +3,7 @@ name: "Release-2: Tag, Release, Push"
 env:
   GIT_USERNAME: "DefectDojo release bot"
   GIT_EMAIL: "dojo-release-bot@users.noreply.github.com"
-  workflow_name: 'release 2 tag release docker push' # needed in cache key, which doesn't support comma's
+
 on:
   workflow_dispatch:
     inputs:
@@ -32,7 +32,7 @@ jobs:
         run: |
           git tag -a ${{ github.event.inputs.release_number }} -m "[bot] release ${{ github.event.inputs.release_number }}"
           git push origin ${{ github.event.inputs.release_number }}
-  
+
   release-helm-chart:
     needs: tag
     uses: ./.github/workflows/release-x-manual-helm-chart.yml

diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml
@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Load OAS files from artifacts
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
         with:
           pattern: oas-*
 

diff --git a/.github/workflows/release-x-manual-docker-containers.yml b/.github/workflows/release-x-manual-docker-containers.yml
@@ -3,7 +3,7 @@ name: "release-X: Release docker containers"
 env:
   GIT_USERNAME: "DefectDojo release bot"
   GIT_EMAIL: "dojo-release-bot@users.noreply.github.com"
-  workflow_name: 'release docker containers' # needed in cache key, which doesn't support comma's
+
 on:
   workflow_dispatch:
     inputs:
@@ -29,7 +29,6 @@ jobs:
       matrix:
           docker-image: [django, nginx]
           os: [alpine, debian]
-          platform: [amd64]
     steps:
       - name: Login to DockerHub
         uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
@@ -47,11 +46,11 @@ jobs:
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
 
       - name: Build and push images with debian
         if: ${{ matrix.os  == 'debian' }}
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         env:
           DOCKER_BUILD_CHECKS_ANNOTATIONS: false
           REPO_ORG: ${{ env.repoorg }}
@@ -64,7 +63,7 @@ jobs:
 
       - name: Build and push images with alpine
         if: ${{ matrix.os  == 'alpine' }}
-        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         env:
           DOCKER_BUILD_CHECKS_ANNOTATIONS: false
           REPO_ORG: ${{ env.repoorg }}

diff --git a/.github/workflows/release-x-manual-helm-chart.yml b/.github/workflows/release-x-manual-helm-chart.yml
@@ -47,7 +47,7 @@ jobs:
           git config --global user.email "${{ env.GIT_EMAIL }}"
       
       - name: Set up Helm
-        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
+        uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
 
       - name: Configure HELM repos
         run: |-

diff --git a/.github/workflows/rest-framework-tests.yml b/.github/workflows/rest-framework-tests.yml
@@ -2,11 +2,15 @@ name: Rest Framework Unit Tests
 
 on:
   workflow_call:
+    inputs:
+        runner:
+          type: string
+          default: "ubuntu-latest"
 
 jobs:
   unit_tests:
     name: Rest Framework Unit Tests
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.runner }}
 
     strategy:
       matrix:
@@ -20,7 +24,7 @@ jobs:
 
       # load docker images from build jobs
       - name: Load images from artifacts
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
         with:
           path: built-docker-image
           pattern: built-docker-image-*
@@ -29,8 +33,8 @@ jobs:
       - name: Load docker images
         timeout-minutes: 10
         run: |-
-             docker load -i built-docker-image/nginx-${{ matrix.os }}_img
-             docker load -i built-docker-image/django-${{ matrix.os }}_img
+             docker load -i built-docker-image/nginx-${{ matrix.os }}-${{ inputs.runner }}_img
+             docker load -i built-docker-image/django-${{ matrix.os }}-${{ inputs.runner }}_img
              docker images
 
       # run tests with docker compose
@@ -43,7 +47,7 @@ jobs:
 
       # no celery or initializer needed for unit tests
       - name: Unit tests
-        timeout-minutes: 10
+        timeout-minutes: 15
         run: docker compose up --no-deps --exit-code-from uwsgi uwsgi
         env:
           DJANGO_VERSION: ${{ matrix.os }}

diff --git a/.github/workflows/test-helm-chart.yml b/.github/workflows/test-helm-chart.yml
@@ -20,7 +20,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0
+        uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
 
       - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -12,19 +12,33 @@ on:
 
 jobs:
   build-docker-containers:
+    strategy:
+        matrix:
+          runner: ["ubuntu-latest", "ubuntu-24.04-arm"] # we need to pass runner names instead of amd64/amd64
+        fail-fast: false
     uses: ./.github/workflows/build-docker-images-for-testing.yml
     secrets: inherit
-
+    with:
+      runner: ${{ matrix.runner }}
+
   test-rest-framework:
+    strategy:
+        matrix:
+          runner: ["ubuntu-latest", "ubuntu-24.04-arm"]
+        fail-fast: false
     needs: build-docker-containers
     uses: ./.github/workflows/rest-framework-tests.yml
     secrets: inherit
+    with:
+      runner: ${{ matrix.runner}}
 
+  # only run integration tests on ubuntu-latest (x64)
   test-user-interface:
     needs: build-docker-containers
     uses: ./.github/workflows/integration-tests.yml
     secrets: inherit
-
+
+  # only run k8s tests on ubuntu-latest (x64)
   test-k8s:
     needs: build-docker-containers
     uses: ./.github/workflows/k8s-tests.yml