test-target.yml

# "target" refers to the integration (a.k.a. check) that we want to test.
# This action is a template that we reuse in other actions such as test-all or pr-test.
name: Test target

on:
  workflow_call:
    inputs:
      job-name:
        required: true
        type: string
      target:
        required: true
        type: string
      platform:
        required: true
        type: string
      runner:
        required: true
        type: string
      repo:
        required: true
        type: string

      python-version:
        required: false
        default: ""
        type: string
      standard:
        required: false
        default: false
        type: boolean
      benchmark:
        required: false
        default: false
        type: boolean
      latest:
        required: false
        default: false
        type: boolean
      minimum-base-package:
        required: false
        default: false
        type: boolean
      # integrations-core repo no longer tests Python 2, only Python 3.
      # We keep the options in the workflow for community and marketplace partners.
      # They may still continue to test Python 2 and Agent 6.
      test-py2:
        required: false
        default: false
        type: boolean
      test-py3:
        required: false
        default: true
        type: boolean
      agent-image:
        required: false
        default: ""
        type: string
      agent-image-py2:
        required: false
        default: ""
        type: string
      agent-image-windows:
        required: false
        default: ""
        type: string
      agent-image-windows-py2:
        required: false
        default: ""
        type: string
      traces-artifact-name:
        required: false
        default: "traces"
        type: string
      trace-agent-port:
        required: false
        default: "8126"
        type: string
      setup-env-vars:
        required: false
        default: ""
        type: string
      pytest-args:
        description: "Arguments to pass to pytest"
        required: false
        type: string
        default: ""

defaults:
  run:
    shell: bash

jobs:
  run:
    name: "${{ inputs.job-name }}"
    runs-on: ${{ fromJson(inputs.runner) }}

    env:
      FORCE_COLOR: "1"
      PYTHON_VERSION: "${{ inputs.python-version || '3.12' }}"
      PYTHON_FILTER: "${{ (inputs.test-py2 && !inputs.test-py3) && '2.7' || (!inputs.test-py2 && inputs.test-py3) && (inputs.python-version || '3.12') || '' }}"
      SKIP_ENV_NAME: "${{ (inputs.test-py2 && !inputs.test-py3) && 'py3.*' || (!inputs.test-py2 && inputs.test-py3) && 'py2.*' || '' }}"
      # Windows E2E requires Windows containers
      DDEV_E2E_AGENT: "${{ inputs.platform == 'windows' && (inputs.agent-image-windows || 'datadog/agent-dev:master-py3-win-servercore') || inputs.agent-image }}"
      DDEV_E2E_AGENT_PY2: "${{ inputs.platform == 'windows' && (inputs.agent-image-windows-py2 || 'datadog/agent-dev:master-py2-win-servercore') || inputs.agent-image-py2 }}"
      # Test results for later processing
      TEST_RESULTS_BASE_DIR: "test-results"
      # Tracing to monitor our test suite
      DD_ENV: "ci"
      DD_SERVICE: "ddev-integrations-${{ inputs.repo }}"
      DD_TAGS: "team:agent-integrations,platform:${{ inputs.platform }},integration:${{ inputs.target }}"
      DD_TRACE_ANALYTICS_ENABLED: "true"
      # Capture traces for a separate job to do the submission
      TRACE_CAPTURE_BASE_DIR: "trace-captures"
      TRACE_CAPTURE_LOG: "trace-captures/output.log"

    steps:

    - name: Set environment variables with sanitized paths
      run: |
        # We want to replace leading dots as they will make directories hidden, which will cause them to be ignored by upload-artifact and EnricoMi/publish-unit-test-result-action
        JOB_NAME=$(echo "${{ inputs.job-name }}" | sed 's/^\./Dot/')

        echo "TEST_RESULTS_DIR=$TEST_RESULTS_BASE_DIR/$JOB_NAME" >> $GITHUB_ENV
        echo "TRACE_CAPTURE_FILE=$TRACE_CAPTURE_BASE_DIR/$JOB_NAME" >> $GITHUB_ENV

    - name: Set up Windows
      if: runner.os == 'Windows'
      run: |-
        # Enable disk performance counters
        diskperf -y

    - uses: actions/checkout@v4

    - name: Set up Python 2.7
      if: inputs.test-py2
      run: |
        if [ "$RUNNER_OS" == "Linux" ]; then
          sudo apt update
          sudo apt install python2 python2-dev
          sudo update-alternatives --install /usr/bin/python python /usr/bin/python2 1
        elif [ "$RUNNER_OS" == "Windows" ]; then
          choco install python2
        else
          echo "$RUNNER_OS not supported"
          exit 1
        fi
      shell: bash

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@v5
      with:
        python-version: "${{ env.PYTHON_VERSION }}"
        cache: 'pip'

    - name: Restore cache
      if: inputs.repo == 'core'
      uses: actions/cache/restore@v4
      with:
        path: ${{ runner.os == 'Windows' && '~\AppData\Local\pip\Cache' || runner.os == 'macOS' && '~/Library/Caches/pip' || '~/.cache/pip' }}
        key: >-
          ${{ format(
            'v01-python-{0}-{1}-{2}-{3}',
            env.pythonLocation,
            hashFiles('datadog_checks_base/pyproject.toml'),
            hashFiles('datadog_checks_dev/pyproject.toml'),
            hashFiles('ddev/pyproject.toml')
          )}}
        restore-keys: |-
          v01-python-${{ env.pythonLocation }}

    - name: Install ddev from local folder
      if: inputs.repo == 'core'
      run: |-
        pip install -e ./datadog_checks_dev[cli]
        pip install -e ./ddev

    - name: Install ddev from PyPI
      if: inputs.repo != 'core'
      run: pip install ddev

    - name: Configure ddev
      run: |-
        ddev config set repos.${{ inputs.repo }} .
        ddev config set repo ${{ inputs.repo }}

    - name: Lint
      run: ddev test --lint ${{ inputs.target }}

    - name: Prepare for testing
      env: >-
        ${{ fromJson(inputs.setup-env-vars || format(
          '{{
            "PYTHONUNBUFFERED": "1",
            "DOCKER_USERNAME": "{0}",
            "DOCKER_ACCESS_TOKEN": "{1}",
            "ORACLE_DOCKER_USERNAME": "{2}",
            "ORACLE_DOCKER_PASSWORD": "{3}",
            "SINGLESTORE_LICENSE": "{4}",
            "DD_GITHUB_USER": "{5}",
            "DD_GITHUB_TOKEN": "{6}"
          }}',
          secrets.DOCKER_USERNAME,
          secrets.DOCKER_ACCESS_TOKEN,
          secrets.ORACLE_DOCKER_USERNAME,
          secrets.ORACLE_DOCKER_PASSWORD,
          secrets.SINGLESTORE_LICENSE,
          github.actor,
          secrets.GITHUB_TOKEN
        ))}}
      run: ddev ci setup ${{ inputs.target }}

    - name: Set up trace capturing
      if: inputs.repo == 'core'
      env:
        PYTHONUNBUFFERED: "1"
      run: |-
        mkdir "${{ env.TRACE_CAPTURE_BASE_DIR }}"
        python .ddev/ci/scripts/traces.py capture --port "${{ inputs.trace-agent-port }}" --record-file "${{ env.TRACE_CAPTURE_FILE }}" > "${{ env.TRACE_CAPTURE_LOG }}" 2>&1 &

    - name: Run Unit & Integration tests
      if: inputs.standard && !inputs.minimum-base-package
      env:
        DDEV_TEST_ENABLE_TRACING: "${{ inputs.repo == 'core' && '1' || '0' }}"
      run: |
        if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then
          set +e # Disable immediate exit
          ddev test --cov --junit ${{ inputs.target }} -- ${{ inputs.pytest-args }} -k "not fips"
          exit_code=$?
          if [ $exit_code -eq 5 ]; then
            # Flaky test count can be zero, this is done to avoid pipeline failure
            echo "No tests were collected."
            exit 0
          else
            exit $exit_code
          fi
        else
          ddev test --cov --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- {0} -k "not fips"', inputs.pytest-args) || '-- -k "not fips"' }}
        fi

    - name: Run Unit & Integration tests with minimum version of base package
      if: inputs.standard && inputs.minimum-base-package
      run: |
        if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then
          set +e # Disable immediate exit
          ddev test --compat --recreate --junit ${{ inputs.target }} -- ${{ inputs.pytest-args }} -k "not fips"
          exit_code=$?
          if [ $exit_code -eq 5 ]; then
            # Flaky test count can be zero, this is done to avoid pipeline failure
            echo "No tests were collected."
            exit 0
          else
            exit $exit_code
          fi
        else
          ddev test --compat --recreate --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- {0} -k "not fips"', inputs.pytest-args) || '-- -k "not fips"' }}
        fi

    - name: Run E2E tests with latest base package
      if: inputs.standard && inputs.repo == 'core' && !inputs.minimum-base-package
      env:
        DD_API_KEY: "${{ secrets.DD_API_KEY }}"
      run: |
        # '-- all' is passed for e2e tests if pytest args are provided
        # This is done to avoid ddev from interpreting the arguments as environments
        #   instead of pytest-args, because by default if no environment is provided
        #   after -- it will run all environments. So when pytests args are provided
        #   ddev will interpret '-m' as an environment to run the e2e test on and fails
        # This is not required when no pytest args are provided and it will run all environments
        #   by default
        if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then
          set +e # Disable immediate exit
          ddev env test --base --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} -k "not fips"
          exit_code=$?
          if [ $exit_code -eq 5 ]; then
            # Flaky test count can be zero, this is done to avoid pipeline failure
            echo "No tests were collected."
            exit 0
          else
            exit $exit_code
          fi
        elif [ '${{ inputs.pytest-args }}' = '-m "not flaky"' ]; then
          set +e # Disable immediate exit
          ddev env test --base --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} -k "not fips"
          exit_code=$?
          if [ $exit_code -eq 5 ]; then
            # Flaky test count can be zero, this is done to avoid pipeline failure
            echo "No tests were collected."
            exit 0
          else
            exit $exit_code
          fi
        else
          ddev env test --base --new-env --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- all {0} -k "not fips"', inputs.pytest-args) || '-- all -k "not fips"' }}
        fi

    - name: Run E2E tests
      if: inputs.standard && inputs.repo != 'core'
      env:
        DD_API_KEY: "${{ secrets.DD_API_KEY }}"
      run: |
        # '-- all' is passed for e2e tests if pytest args are provided
        # This is done to avoid ddev from interpreting the arguments as environments
        #   instead of pytest-args, because by default if no environment is provided
        #   after -- it will run all environments. So when pytests args are provided
        #   ddev will interpret '-m' as an environment to run the e2e test on and fails
        # This is not required when no pytest args are provided and it will run all environments
        #   by default
        if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then
          set +e # Disable immediate exit
          ddev env test --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} -k "not fips"
          exit_code=$?
          if [ $exit_code -eq 5 ]; then
            # Flaky test count can be zero, this is done to avoid pipeline failure
            echo "No tests were collected."
            exit 0
          else
            exit $exit_code
          fi
        elif [ '${{ inputs.pytest-args }}' = '-m "not flaky"' ]; then
          set +e # Disable immediate exit
          ddev env test --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} -k "not fips"
          exit_code=$?
          if [ $exit_code -eq 5 ]; then
            # Flaky test count can be zero, this is done to avoid pipeline failure
            echo "No tests were collected."
            exit 0
          else
            exit $exit_code
          fi
        else
          ddev env test --new-env --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- all {0} -k "not fips"', inputs.pytest-args) || '-- all -k "not fips"' }}
        fi

    - name: Run benchmarks
      if: inputs.benchmark
      run: ddev test --bench --junit ${{ inputs.target }}

    - name: Run tests and verify support for the latest version
      if: inputs.latest
      run: ddev test --latest --junit ${{ inputs.target }}

    - name: Run E2E tests for the latest version
      if: inputs.latest
      env:
        DD_API_KEY: "${{ secrets.DD_API_KEY }}"
        DDEV_TEST_ENABLE_TRACING: "${{ inputs.repo == 'core' && '1' || '0' }}"
      run: |
        # '-- all' is passed for e2e tests if pytest args are provided
        # This is done to avoid ddev from interpreting the arguments as environments
        #   instead of pytest-args, because by default if no environment is provided
        #   after -- it will run all environments. So when pytests args are provided
        #   ddev will interpret '-m' as an environment to run the e2e test on and fails
        # This is not required when no pytest args are provided and it will run all environments
        #   by default
        if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then
          set +e # Disable immediate exit
          ddev env test --base --new-env --junit ${{ inputs.target }}:latest -- all ${{ inputs.pytest-args }} -k "not fips"
          exit_code=$?
          if [ $exit_code -eq 5 ]; then
            # Flaky test count can be zero, this is done to avoid pipeline failure
            echo "No tests were collected."
            exit 0
          else
            exit $exit_code
          fi
        elif [ '${{ inputs.pytest-args }}' = '-m "not flaky"' ]; then
          set +e # Disable immediate exit
          ddev env test --base --new-env --junit ${{ inputs.target }}:latest -- all ${{ inputs.pytest-args }}
          exit_code=$?
          if [ $exit_code -eq 5 ]; then
            # Flaky test count can be zero, this is done to avoid pipeline failure
            echo "No tests were collected."
            exit 0
          else
            exit $exit_code
          fi
        else
          ddev env test --base --new-env --junit ${{ inputs.target }}:latest ${{ inputs.pytest-args != '' && format('-- all {0} -k "not fips"', inputs.pytest-args) || '-- all -k "not fips"' }}
        fi

    - name: View trace log
      if: inputs.repo == 'core' && always()
      run: cat "${{ env.TRACE_CAPTURE_LOG }}"

    - name: Upload captured traces
      if: inputs.repo == 'core' && always()
      uses: actions/upload-artifact@v4
      with:
        name: "${{ inputs.traces-artifact-name }}-${{ inputs.target }}-${{ inputs.platform }}"
        path: "${{ env.TRACE_CAPTURE_FILE }}"

    - name: Finalize test results
      if: always()
      run: |-
        mkdir -p "${{ env.TEST_RESULTS_DIR }}"
        if [[ -d ${{ inputs.target }}/.junit ]]; then
          mv ${{ inputs.target }}/.junit/*.xml "${{ env.TEST_RESULTS_DIR }}"
        fi

    - name: Upload test results
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: "test-results-${{ inputs.target }}-${{ inputs.platform }}"
        path: "${{ env.TEST_RESULTS_BASE_DIR }}"

    - name: Upload coverage data
      if: >
        inputs.standard &&
        !github.event.repository.private &&
        always() &&
        inputs.pytest-args != '-m flaky'
      # Flaky tests will have low coverage, don't upload it to avoid pipeline failure
      uses: codecov/codecov-action@v4
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        files: "${{ inputs.target }}/coverage.xml"
        flags: "${{ inputs.target }}"