From 1832c69c269e1c9f15c3eb36f7a1bd59a861ab73 Mon Sep 17 00:00:00 2001 From: h-chlor <43164320+h-chlor@users.noreply.github.com> Date: Wed, 19 Jan 2022 18:25:43 +0200 Subject: [PATCH] Add SonarQube to CI (#8362) --- .github/actions/ci-java-tests/action.yml | 28 ++ .github/actions/ci-py-tests/action.yml | 81 +++++ .github/actions/ci-tests-runner/action.yml | 187 ++++++++++ ...tect-changed-modules-and-build-reports.yml | 42 --- .github/workflows/sonar-scan.yml | 73 ++++ docs/contributing-to-airbyte/README.md | 2 + .../sonar-qube-workflow.md | 34 ++ pyproject.toml | 20 +- .../__init__.py | 0 .../ci_changes_detection/__init__.py | 0 .../ci_changes_detection/main.py | 89 +++++ .../ci_sonar_qube/__init__.py | 12 + .../ci_sonar_qube/log_parsers.py | 312 ++++++++++++++++ tools/ci_code_validator/ci_sonar_qube/main.py | 60 +++ .../ci_sonar_qube/sonar_qube_api.py | 342 ++++++++++++++++++ tools/ci_code_validator/setup.py | 43 +++ tools/ci_code_validator/tests/__init__.py | 0 .../black_smell_package_report.json | 14 + .../isort_smell_package_report.json | 14 + .../mypy_smell_package_report.json | 52 +++ .../simple_files/without_issues_report.json | 1 + .../tests/simple_package/__init__.py | 0 .../tests/simple_package/valid_file.py | 13 + .../tests/simple_smell_package/__init__.py | 0 .../simple_smell_package/invalid_file.py | 15 + .../tests/test_detect_changed_modules.py | 74 ++++ .../tests/test_sq_project.py | 20 + tools/ci_code_validator/tests/test_tools.py | 102 ++++++ tools/ci_credentials/setup.py | 2 +- tools/ci_static_check_reports/__init__.py | 3 - .../__init__.py | 3 - .../main.py | 100 ----- .../ci_detect_changed_modules/main.py | 52 --- tools/ci_static_check_reports/setup.py | 29 -- .../unit_tests/__init__.py | 3 - .../test_build_static_checkers_reports.py | 42 --- .../unit_tests/test_detect_changed_modules.py | 58 --- 37 files changed, 1584 insertions(+), 338 deletions(-) create mode 100644 .github/actions/ci-java-tests/action.yml create mode 100644 .github/actions/ci-py-tests/action.yml create mode 100644 .github/actions/ci-tests-runner/action.yml delete mode 100644 .github/workflows/detect-changed-modules-and-build-reports.yml create mode 100644 .github/workflows/sonar-scan.yml create mode 100644 docs/contributing-to-airbyte/sonar-qube-workflow.md rename tools/{ci_static_check_reports/ci_detect_changed_modules => ci_code_validator}/__init__.py (100%) create mode 100644 tools/ci_code_validator/ci_changes_detection/__init__.py create mode 100644 tools/ci_code_validator/ci_changes_detection/main.py create mode 100644 tools/ci_code_validator/ci_sonar_qube/__init__.py create mode 100644 tools/ci_code_validator/ci_sonar_qube/log_parsers.py create mode 100644 tools/ci_code_validator/ci_sonar_qube/main.py create mode 100644 tools/ci_code_validator/ci_sonar_qube/sonar_qube_api.py create mode 100644 tools/ci_code_validator/setup.py create mode 100644 tools/ci_code_validator/tests/__init__.py create mode 100644 tools/ci_code_validator/tests/simple_files/black_smell_package_report.json create mode 100644 tools/ci_code_validator/tests/simple_files/isort_smell_package_report.json create mode 100644 tools/ci_code_validator/tests/simple_files/mypy_smell_package_report.json create mode 100644 tools/ci_code_validator/tests/simple_files/without_issues_report.json create mode 100644 tools/ci_code_validator/tests/simple_package/__init__.py create mode 100644 tools/ci_code_validator/tests/simple_package/valid_file.py create mode 100644 tools/ci_code_validator/tests/simple_smell_package/__init__.py create mode 100644 tools/ci_code_validator/tests/simple_smell_package/invalid_file.py create mode 100644 tools/ci_code_validator/tests/test_detect_changed_modules.py create mode 100644 tools/ci_code_validator/tests/test_sq_project.py create mode 100644 tools/ci_code_validator/tests/test_tools.py delete mode 100644 tools/ci_static_check_reports/__init__.py delete mode 100644 tools/ci_static_check_reports/ci_build_python_static_checkers_reports/__init__.py delete mode 100644 tools/ci_static_check_reports/ci_build_python_static_checkers_reports/main.py delete mode 100644 tools/ci_static_check_reports/ci_detect_changed_modules/main.py delete mode 100644 tools/ci_static_check_reports/setup.py delete mode 100644 tools/ci_static_check_reports/unit_tests/__init__.py delete mode 100644 tools/ci_static_check_reports/unit_tests/test_build_static_checkers_reports.py delete mode 100644 tools/ci_static_check_reports/unit_tests/test_detect_changed_modules.py diff --git a/.github/actions/ci-java-tests/action.yml b/.github/actions/ci-java-tests/action.yml new file mode 100644 index 0000000000000..b98fa86dcae48 --- /dev/null +++ b/.github/actions/ci-java-tests/action.yml @@ -0,0 +1,28 @@ +name: "Runner CI Java Tests" +description: "Runner CI Java Tests" +inputs: + module-name: + required: true + module-folder: + required: true + +runs: + using: "composite" + steps: + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: '17' + + - name: "Build" + shell: bash + run: | + rm -rf ${{ inputs.module-folder }}/.venv ${{ inputs.module-folder }}/build + ROOT_DIR=$(git rev-parse --show-toplevel) + ARG=:$(python -c "import os; print(os.path.relpath('${{ inputs.module-folder }}', start='${ROOT_DIR}').replace('/', ':') )") + echo "./gradlew --no-daemon $ARG:build" + ./gradlew --no-daemon "$ARG:clean" + ./gradlew --no-daemon "$ARG:build" + + + diff --git a/.github/actions/ci-py-tests/action.yml b/.github/actions/ci-py-tests/action.yml new file mode 100644 index 0000000000000..c1f50251d5579 --- /dev/null +++ b/.github/actions/ci-py-tests/action.yml @@ -0,0 +1,81 @@ +name: "Runner CI Python Tests" +description: "Runner CI Python Tests" +inputs: + module-name: + required: true + module-folder: + required: true +outputs: + coverage-paths: + description: "Coverage Paths" + value: ${{ steps.build-coverage-reports.outputs.coverage-paths }} + flake8-logs: + description: "Flake8 Logs" + value: ${{ steps.build-linter-reports.outputs.flake8-logs }} + mypy-logs: + description: "MyPy Logs" + value: ${{ steps.build-linter-reports.outputs.mypy-logs }} + black-diff: + description: "Black Diff" + value: ${{ steps.build-linter-reports.outputs.black-diff }} + isort-diff: + description: "Isort Diff" + value: ${{ steps.build-linter-reports.outputs.isort-diff }} +runs: + using: "composite" + steps: + - name: Build Coverage Reports + id: build-coverage-reports + shell: bash + working-directory: ${{ inputs.module-folder }} + run: | + virtualenv .venv + source .venv/bin/activate + JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}' + pip install coverage[toml]~=6.2 + mkdir -p .venv/source-acceptance-test + mkdir -p reports + SAT_DIR=$(git rev-parse --show-toplevel)/airbyte-integrations/bases/source-acceptance-test + PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml + git ls-tree -r HEAD --name-only $SAT_DIR | while read src; do cp -f $src .venv/source-acceptance-test; done + pip install build + python -m build .venv/source-acceptance-test + pip install .venv/source-acceptance-test/dist/source_acceptance_test-*.whl + [ -f requirements.txt ] && pip install --quiet -r requirements.txt + pip install .[tests] + coverage run --rcfile=${PYPROJECT_CONFIG} -m pytest ./unit_tests || true + coverage xml --rcfile=${PYPROJECT_CONFIG} -o reports/coverage.xml || true + + rm -rf .venv + echo "::set-output name=coverage-paths::reports/coverage.xml" + + - name: Build Linter Reports + id: build-linter-reports + shell: bash + working-directory: ${{ inputs.module-folder }} + run: | + JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}' + REPORT_FOLDER=reports + PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml + + # run mypy + pip install lxml~=4.7 mypy~=0.910 . + mypy . --config-file=${PYPROJECT_CONFIG} | tee reports/mypy.log || true + + # run black + pip install black~=21.12b0 + XDG_CACHE_HOME=/dev/null black --config ${PYPROJECT_CONFIG} --diff . | tee reports/black.diff + + # run isort + pip install isort~=5.10.1 + cp ${PYPROJECT_CONFIG} ./pyproject.toml + isort --diff . | tee reports/isort.diff + + # run flake8 + pip install mccabe~=0.6.1 pyproject-flake8~=0.0.1a2 + pflake8 --exit-zero . | grep ^. | tee reports/flake.txt + + echo "::set-output name=mypy-logs::reports/mypy.log" + echo "::set-output name=black-diff::reports/black.diff" + echo "::set-output name=isort-diff::reports/isort.diff" + echo "::set-output name=flake8-logs::reports/flake.txt" diff --git a/.github/actions/ci-tests-runner/action.yml b/.github/actions/ci-tests-runner/action.yml new file mode 100644 index 0000000000000..3fa01cccb84ac --- /dev/null +++ b/.github/actions/ci-tests-runner/action.yml @@ -0,0 +1,187 @@ +name: "Setup CI Tests Env" +description: "Setup CI Tests Env for all module types" +inputs: + module-name: + description: "Unique module name. e.g.: connectors/source-s3, connectors/destination-s3" + required: true + module-folder: + description: "Path to module folder" + required: true + module-lang: + description: "Detected module language. Available values: py, java" + required: true + sonar-gcp-access-key: + required: true + sonar-token: + description: "Access token for using SonarQube API" + required: true + pull-request-id: + description: "Unique PR ID. For example: airbyte/1234" + default: "0" + token: + required: true + remove-sonar-project: + description: "This flag should be used if needed to remove sonar project after using" + default: false +runs: + using: "composite" + steps: + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Tests of CI + shell: bash + run: | + # all CI python packages have the prefix "ci_" + pip install --quiet tox==3.24.4 + tox -r -c ./tools/tox_ci.ini + pip install --quiet -e ./tools/ci_* + echo "::echo::off" + + - name: Auth with gcloud CLI + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ inputs.sonar-gcp-access-key }} + project_id: dataline-integration-testing + export_default_credentials: true + + - name: Create IAP tunnel + id: gcloud-tunnel + shell: bash + run: | + while true; do + PORT=$(( ((RANDOM<<15)|RANDOM) % 49152 + 10000 )) + status="$(nc -z 127.0.0.1 $PORT < /dev/null &>/dev/null; echo $?)" + if [ "${status}" != "0" ]; then + echo "$PORT is free to use"; + break; + fi + done + IPS=($(hostname -I)) + LOCAL_IP_PORT="${IPS[0]}:${PORT}" + gcloud compute start-iap-tunnel sonarqube-1-vm 80 --local-host-port=${LOCAL_IP_PORT} --zone=europe-central2-a --project dataline-integration-testing & + echo ::set-output name=pid::$! + echo "::set-output name=sonar-host::http://${LOCAL_IP_PORT}/" + echo "::echo::on" + + - name: Python Tests + id: ci-py-tests + if: ${{ inputs.module-lang == 'py' }} + uses: ./.github/actions/ci-py-tests + with: + module-name: ${{ inputs.module-name }} + module-folder: ${{ inputs.module-folder }} + + - name: Java Tests + id: ci-java-tests + if: ${{ inputs.module-lang == 'java' }} + uses: ./.github/actions/ci-java-tests + with: + module-name: ${{ inputs.module-name }} + module-folder: ${{ inputs.module-folder }} + + + + + + - name: Prepare SQ Options + shell: bash + id: sq-options + working-directory: ${{ inputs.module-folder }} + run: | + REPORT_FOLDER=reports + mkdir -p ${REPORT_FOLDER} + declare -a REPORT_FILES + declare -a OPTIONS + if [ ${{ inputs.module-lang }} == 'py' ]; then + [ -f ${{ steps.ci-py-tests.outputs.mypy-logs }} ] && ci_sonar_qube --mypy_log ${{ steps.ci-py-tests.outputs.mypy-logs }} --output_file ${REPORT_FOLDER}/issues_mypy.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} + [ -f ${{ steps.ci-py-tests.outputs.mypy-logs }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_mypy.json) + + [ -f ${{ steps.ci-py-tests.outputs.black-diff }} ] && ci_sonar_qube --black_diff ${{ steps.ci-py-tests.outputs.black-diff }} --output_file ${REPORT_FOLDER}/issues_black.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} + [ -f ${{ steps.ci-py-tests.outputs.black-diff }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_black.json) + + [ -f ${{ steps.ci-py-tests.outputs.isort-diff }} ] && ci_sonar_qube --isort_diff ${{ steps.ci-py-tests.outputs.isort-diff }} --output_file ${REPORT_FOLDER}/issues_isort.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} + [ -f ${{ steps.ci-py-tests.outputs.isort-diff }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_isort.json) + + [ -f ${{ steps.ci-py-tests.outputs.coverage-paths }} ] && OPTIONS+=("-Dsonar.python.coverage.reportPaths=${{ steps.ci-py-tests.outputs.coverage-paths }}") + [ -f ${{ steps.ci-py-tests.outputs.flake8-logs }} ] && OPTIONS+=("-Dsonar.python.flake8.reportPaths=${{ steps.ci-py-tests.outputs.flake8-logs }}") + fi + if [ ${{ inputs.module-lang }} == 'java' ]; then + [ -d "./src/main/java" ] && OPTIONS+=("-Dsonar.sources=./src/main/java") + [ -d "./src/test/java" ] && OPTIONS+=("-Dsonar.tests=./src/test/java") + [ -d "./build/test-results" ] && OPTIONS+=("-Dsonar.junit.reportsPath=./build/test-results") + [ -f "./build/jacoco/test.exec" ] && OPTIONS+=("-Dsonar.jacoco.reportPaths=./build/jacoco/test.exec") + [ -d "./build/classes/java/main" ] && OPTIONS+=("-Dsonar.java.binaries=./build/classes/java/main") + [ -d "./build/classes/java/test" ] && OPTIONS+=("-Dsonar.test.binaries=./build/classes/java/test") + + fi + + # join the array to string format + echo ::set-output name=external_reports::$(IFS=, ; echo "${REPORT_FILES[*]}") + echo ::set-output name=options::$(IFS=' ' ; echo "${OPTIONS[*]}") + + - name: Create SonarQube Project + shell: bash + id: create-sq-project + run: | + ci_sonar_qube --pr ${{ inputs.pull-request-id }} --create --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} + echo "::set-output name=sq_project_name::$(ci_sonar_qube --pr ${{ inputs.pull-request-id }} --print_key --module ${{ inputs.module-name }})" + ROOT_DIR=$(git rev-parse --show-toplevel) + MODULE_DIR=$(python -c "print('${{ inputs.module-folder }}'.replace('${ROOT_DIR}', '.'))") + echo "::set-output name=module_dir::${MODULE_DIR}" + + - name: SonarQube Scan + + uses: sonarsource/sonarqube-scan-action@master + env: + SONAR_TOKEN: ${{ inputs.sonar-token }} + SONAR_HOST_URL: ${{ steps.gcloud-tunnel.outputs.sonar-host }} + with: + projectBaseDir: ${{ steps.create-sq-project.outputs.module_dir }} + args: > + -Dsonar.projectKey=${{ steps.create-sq-project.outputs.sq_project_name }} + -Dsonar.verbose=true + -Dsonar.working.directory=/tmp/scannerwork + -Dsonar.language=${{ inputs.module-lang }} + -Dsonar.sourceEncoding=UTF-8 + -Dsonar.projectBaseDir=${{ steps.create-sq-project.outputs.module_dir }} + -Dsonar.exclusions=reports/**,*.toml + -Dsonar.externalIssuesReportPaths=${{ steps.sq-options.outputs.external_reports }} + ${{ steps.sq-options.outputs.options }} + + - name: Generate SonarQube Report + shell: bash + id: generate-sq-report + run: | + # delay because SQ needs time for processing of all input data + sleep 10 + REPORT_FILE=/tmp/sq_report_$RANDOM.md + ci_sonar_qube --pr ${{ inputs.pull-request-id }} --report ${REPORT_FILE} --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} + body="$(cat ${REPORT_FILE})" + body="${body//'%'/'%25'}" + body="${body//$'\n'/'%0A'}" + body="${body//$'\r'/'%0D'}" + echo "::set-output name=sq-report::$body" + + - name: Add Comment + if: ${{ github.event_name == 'pull_request' }} + uses: peter-evans/commit-comment@v1 + with: + body: ${{ steps.generate-sq-report.outputs.sq-report }} + token: ${{ inputs.token }} + + - name: Remove SonarQube Project + if: ${{ inputs.remove-sonar-project == true }} + shell: bash + id: remove-sq-project + run: | + ci_sonar_qube --pr ${{ inputs.pull-request-id }} --remove --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} + + - name: Remove IAP tunnel + if: always() + shell: bash + run: | + kill ${{ steps.gcloud-tunnel.outputs.pid }} diff --git a/.github/workflows/detect-changed-modules-and-build-reports.yml b/.github/workflows/detect-changed-modules-and-build-reports.yml deleted file mode 100644 index ca5d9f9e730d2..0000000000000 --- a/.github/workflows/detect-changed-modules-and-build-reports.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Detect Changed Modules and Build Reports -on: - push: -jobs: - detect-changed-modules: - name: Detect Changed Modules - timeout-minutes: 5 - runs-on: ubuntu-latest - outputs: - changed-modules: ${{ steps.detect-changed-modules.outputs.changed-modules }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - fetch-depth: 1000 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Intall Requirements - run: pip install ./tools/ci_static_check_reports/. - - name: Detect Changed Modules - id: detect-changed-modules - run: | - git fetch - echo "::set-output name=changed-modules::'$(ci_detect_changed_modules $(git diff --name-only $(git merge-base HEAD origin/master)))'" - build-reports: - name: Build Python Static Checkers Reports - needs: - - detect-changed-modules - runs-on: ubuntu-latest - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Intall Requirements - run: pip install ./tools/ci_static_check_reports/. - - name: Build Reports - run: ci_build_python_checkers_reports ${{needs.detect-changed-modules.outputs.changed-modules}} diff --git a/.github/workflows/sonar-scan.yml b/.github/workflows/sonar-scan.yml new file mode 100644 index 0000000000000..db57aef0b7f69 --- /dev/null +++ b/.github/workflows/sonar-scan.yml @@ -0,0 +1,73 @@ +name: Sonar Scan +on: + pull_request: + types: [opened, synchronize, reopened, closed, ready_for_review] + +jobs: + + detect-changes: + name: Detect Changed Modules + timeout-minutes: 5 + runs-on: ubuntu-latest + outputs: + changed-modules: ${{ steps.detect-changed-modules.outputs.changed-modules }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + fetch-depth: 1000 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Intall Requirements + run: | + pip install ./tools/ci_common_utils ./tools/ci_code_validator[tests] + pytest ./tools/ci_code_validator + - name: Detect Changed Modules + id: detect-changed-modules + run: | + git fetch + CHANGES=$(ci_changes_detection $(git diff --name-only $(git merge-base HEAD origin/master)) | jq -c .) + echo "::set-output name=changed-modules::{ \"include\": $CHANGES }" + + run-ci-tests: + if: github.event.pull_request.draft == false + needs: detect-changes + name: Tests for ${{ matrix.module }} + runs-on: ubuntu-latest + + strategy: + matrix: ${{fromJson(needs.detect-changes.outputs.changed-modules)}} + env: + MODULE_NAME: ${{ matrix.module }} + MODULE_LANG: ${{ matrix.lang }} + MODULE_FOLDER: ${{ matrix.folder }} + ENV_NAME: "github" + + + steps: + - name: Print Settings + run: | + echo "Module: ${{ env.MODULE_NAME }}, Lang: ${{ env.MODULE_LANG }}, Folder: ${{ env.MODULE_FOLDER }}" + - name: Checkout Airbyte + if: ${{ env.ENV_NAME == 'github' }} + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Run Tests Runner + id: run-python-tests + uses: ./.github/actions/ci-tests-runner + with: + token: ${{ secrets.GITHUB_TOKEN }} + module-name: ${{ env.MODULE_NAME }} + module-folder: ${{ env.MODULE_FOLDER }} + module-lang: ${{ env.MODULE_LANG }} + sonar-token: ${{ secrets.SONAR_TOKEN }} + sonar-gcp-access-key: ${{ secrets.GCP_SONAR_SA_KEY }} + pull-request-id: "${{ github.repository }}/${{ github.event.pull_request.number }}" + remove-sonar-project: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }} + + + diff --git a/docs/contributing-to-airbyte/README.md b/docs/contributing-to-airbyte/README.md index 650bb3073341f..b38bd1138681a 100644 --- a/docs/contributing-to-airbyte/README.md +++ b/docs/contributing-to-airbyte/README.md @@ -120,3 +120,5 @@ As soon as you are done with your development, just put up a PR. You're also alw ‌Once your PR passes, we will merge it. +## **Airbyte CI workflows** +* [Testing by SonarQube](sonar-qube-workflow.md) diff --git a/docs/contributing-to-airbyte/sonar-qube-workflow.md b/docs/contributing-to-airbyte/sonar-qube-workflow.md new file mode 100644 index 0000000000000..5314655118ba8 --- /dev/null +++ b/docs/contributing-to-airbyte/sonar-qube-workflow.md @@ -0,0 +1,34 @@ +# SonarQube workflow + +## Goals + The Airbyte monorepo receives contributions from a lot of developers, and there is no way around human errors while merging PRs. +Likely every language has different tools for testing and validation of source files. And while it's best practice to lint and validate code before pushing to git branches, it doesn't always happen. +But it is optional, and as rule as we detect possible problems after launch test/publish commands only. Therefore, using of automated CI code validation can provided the following benefits: +* Problem/vulnerability reports available when the PR was created. And developers would fix bugs and remove smells before code reviews. +* Reviewers would be sure all standard checks were made and code changes satisfy the requirements. +* Set of tools and their options can be changed anytime globally. +* Progress of code changes are saved in SonarQube and this information helps to analyse quality of the product integrally and also its separate parts. + + +## UML diagram +![image](https://user-images.githubusercontent.com/11213273/149561440-0aceaa30-8f82-4e5b-9ee5-77bdcfd87695.png) + + +## Used tools +### Python +* [flake8](https://flake8.pycqa.org/en/stable/) +* [mypy](https://mypy.readthedocs.io/en/stable/) +* [isort](https://pycqa.github.io/isort/) +* [black](https://black.readthedocs.io/en/stable/) +* [coverage](https://coverage.readthedocs.io/en/6.2/) + +All Python tools use the common [pyproject.toml](https://github.com/airbytehq/airbyte/blob/master/pyproject.toml) file. + +### Common tools +* [SonarQube Scanner](https://docs.sonarqube.org/latest/analysis/scan/sonarscanner/) + +## Access to SonarQube +The Airbyte project uses a custom SonarQube instance. Access to it is explained [here](https://github.com/airbytehq/airbyte-cloud/wiki/IAP-tunnel-to-the-SonarQube-instance). + +## SonarQube settings +The SonarQube server uses default settings. All customisations are implemented into the Github WorkFlows. More details are [here](https://github.com/airbytehq/airbyte/tree/master/.github/actions/ci-tests-runner/action.yml) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 374c87023d494..773d47806fe7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,34 +6,44 @@ target-version = ["py37"] fail_under = 100 skip_empty = true sort = "-cover" - +omit = [ + "*_tests/**", + "setup.py" +] [tool.flake8] -extend-exclude = ".venv" +extend-exclude = [ + ".venv", + "build", + "models", + ".eggs", + ".tox" +] + max-complexity = 10 max-line-length = 140 [tool.isort] profile = "black" -color_output = true +color_output = false skip_gitignore = true [tool.mypy] platform = "linux" +exclude = "build" # Strictness +ignore_missing_imports = true allow_redefinition = true disallow_incomplete_defs = true disallow_untyped_defs = true no_implicit_reexport = true no_strict_optional = true strict_equality = true - # Output pretty = true show_column_numbers = true show_error_codes = true show_error_context = true - # Warnings warn_redundant_casts = true warn_return_any = true diff --git a/tools/ci_static_check_reports/ci_detect_changed_modules/__init__.py b/tools/ci_code_validator/__init__.py similarity index 100% rename from tools/ci_static_check_reports/ci_detect_changed_modules/__init__.py rename to tools/ci_code_validator/__init__.py diff --git a/tools/ci_code_validator/ci_changes_detection/__init__.py b/tools/ci_code_validator/ci_changes_detection/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tools/ci_code_validator/ci_changes_detection/main.py b/tools/ci_code_validator/ci_changes_detection/main.py new file mode 100644 index 0000000000000..9c945a12d3291 --- /dev/null +++ b/tools/ci_code_validator/ci_changes_detection/main.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# +import json +import sys +from pathlib import Path +from typing import Dict, List, Optional + +from ci_sonar_qube import ROOT_DIR + +from ci_common_utils import Logger + +# Filenames used to detect whether the dir is a module +LANGUAGE_MODULE_ID_FILE = { + ".py": "setup.py", + # TODO: Add ID files for other languages +} + +LOGGER = Logger() + + +def folder_generator(dir_path: Path) -> Path: + while dir_path and str(dir_path) != dir_path.root and dir_path != dir_path.parent: + if dir_path.is_dir(): + yield dir_path + dir_path = dir_path.parent + + +def find_py_module(changed_path: Path) -> Optional[Path]: + """All Python connectors have setup.py file into own sortware folders""" + for dir_path in folder_generator(changed_path): + setup_py_file = dir_path / "setup.py" + if setup_py_file.is_file(): + return dir_path + return None + + +def find_java_module(changed_path: Path) -> Optional[Path]: + """All Java connectors have a folder src/main/java into own folders""" + for dir_path in folder_generator(changed_path): + required_java_dir = dir_path / "src/main/java" + if required_java_dir.is_dir(): + return dir_path + return None + + +def list_changed_modules(changed_files: List[str]) -> List[Dict[str, str]]: + """ + changed_filed are the list of files which were modified in current branch. + E.g. changed_files = ["tools/ci_static_check_reports/__init__.py", "tools/ci_static_check_reports/setup.py", ...] + """ + module_folders = {} + for file_path in changed_files: + if not file_path.startswith("/"): + file_path = ROOT_DIR / file_path + else: + file_path = Path(file_path) + module_folder = find_py_module(file_path) + if module_folder: + module_folders[module_folder] = "py" + continue + module_folder = find_java_module(file_path) + if module_folder: + module_folders[module_folder] = "java" + + modules = [] + for module_folder, lang in module_folders.items(): + module_folder = str(module_folder) + if "airbyte-integrations/connectors" not in module_folder: + # now we need to detect connectors only + LOGGER.info(f"skip the folder {module_folder}...") + continue + parts = module_folder.split("/") + module_name = "/".join(parts[-2:]) + modules.append({"folder": module_folder, "lang": lang, "module": module_name}) + LOGGER.info(f"Detected the module: {module_name}({lang}) in the folder: {module_folder}") + # _, file_extension = os.path.splitext(file_path) + # find_base_path(file_path, modules, file_ext=file_extension, unique_modules=unique_modules) + return modules + + +def main() -> int: + changed_modules = list_changed_modules(sys.argv[1:]) + print(json.dumps(changed_modules)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_code_validator/ci_sonar_qube/__init__.py b/tools/ci_code_validator/ci_sonar_qube/__init__.py new file mode 100644 index 0000000000000..0be1d14aa82d5 --- /dev/null +++ b/tools/ci_code_validator/ci_sonar_qube/__init__.py @@ -0,0 +1,12 @@ +import os +from pathlib import Path + +from ci_common_utils import Logger + +LOGGER = Logger() + +ROOT_DIR = Path(os.getcwd()) +while str(ROOT_DIR) != "/" and not (ROOT_DIR / "gradlew").is_file(): + ROOT_DIR = ROOT_DIR.parent +if str(ROOT_DIR) == "/": + LOGGER.critical("this script must be executed into the Airbite repo only") diff --git a/tools/ci_code_validator/ci_sonar_qube/log_parsers.py b/tools/ci_code_validator/ci_sonar_qube/log_parsers.py new file mode 100644 index 0000000000000..8389db9e03125 --- /dev/null +++ b/tools/ci_code_validator/ci_sonar_qube/log_parsers.py @@ -0,0 +1,312 @@ +import json +import os +import re +from collections import defaultdict +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Callable, TextIO, List, Optional, Mapping, Any + +try: + # these packages are not always needed + from mypy.errorcodes import error_codes as mypy_error_codes, ErrorCode + from unidiff import PatchSet +except ModuleNotFoundError: + pass + +from .sonar_qube_api import SonarQubeApi + +HERE = Path(os.getcwd()) +RE_MYPY_LINE = re.compile(r"^(.+):(\d+):(\d+):") +RE_MYPY_LINE_WO_COORDINATES = re.compile(r"^(.+): error: (.+)") + + +class IssueSeverity(Enum): + blocker = "BLOCKER" + critical = "CRITICAL" + major = "MAJOR" + minor = "MINOR" + info = "INFO" + + +@dataclass +class Rule: + class Type(Enum): + code_smell = "CODE_SMELL" + bug = "BUG" + vulnerability = "VULNERABILITY" + security_hotspot = "SECURITY_HOTSPOT" + + rule_type: Type + key: str + name: str + description: str + tool_name: str + template: str + severity: IssueSeverity + + @property + def unique_key(self): + return f"{self.tool_name}_{self.key}".replace("-", "_") + + @property + def sq_key(self): + lang_part = self.template.split(":")[0] + return f"{lang_part}:{self.tool_name}_{self.key}".replace("-", "_") + + +def generate_mypy_rules() -> Mapping[str, Rule]: + try: + addl_code = ErrorCode( + code="unknown", + description="Unknown error", + category="General", + ) + except NameError: + return [] + return {f"[{err.code}]": Rule( + rule_type=Rule.Type.code_smell, + key=err.code, + name=err.code.replace("-", " ").capitalize() + " (mypy)", + description=err.description, + tool_name="mypy", + severity=IssueSeverity.minor, + template="python:CommentRegularExpression" + ) for err in list(mypy_error_codes.values()) + [addl_code]} + + +class LogParser(SonarQubeApi): + _mypy_rules: Mapping[str, Rule] = generate_mypy_rules() + _black_rule = Rule( + rule_type=Rule.Type.code_smell, + key="need_format", + name="Should be formatted (black)", + description='Please run one of the commands: "black --config ./pyproject.toml " or "./gradlew format"', + tool_name="black", + severity=IssueSeverity.minor, + template="python:CommentRegularExpression" + ) + + _isort_rule = Rule( + rule_type=Rule.Type.code_smell, + key="need_format", + name="Should be formatted (isort)", + description='Please run one of the commands: "isort " or "./gradlew format"', + tool_name="isort", + severity=IssueSeverity.minor, + template="python:CommentRegularExpression" + ) + + @dataclass + class Issue: + path: str + + rule: Rule + description: str + + line_number: int = None # 1-indexed + column_number: int = None # 1-indexed + + def to_json(self): + data = { + "engineId": self.rule.tool_name, + "ruleId": self.rule.sq_key, + "severity": self.rule.severity.value, + "type": self.rule.rule_type.value, + "primaryLocation": { + "message": self.description, + "filePath": self.checked_path, + } + } + if self.line_number is not None: + data["primaryLocation"]["textRange"] = { + "startLine": self.line_number, + "endLine": self.line_number, + "startColumn": self.column_number - 1, # 0-indexed + "endColumn": self.column_number, # 0-indexed + } + return data + + @property + def checked_path(self): + if self.path.startswith(str(HERE) + "/"): + # remove a parent part of path + return self.path[len(str(HERE) + "/"):] + return self.path + + def __init__(self, output_file: str, host: str, token: str): + super().__init__(host=host, token=token, pr_name="0") + self.output_file = output_file + + def prepare_file(func: Callable) -> Callable: + def intra(self, input_file: str) -> int: + if not os.path.exists(input_file): + self.logger.critical(f"not found input file: {input_file}") + with open(input_file, "r") as file: + issues = func(self, file) + self._save_all_rules(issues) + data = self._issues2dict(issues) + with open(self.output_file, "w") as output_file: + output_file.write(json.dumps(data)) + self.logger.info(f"the file {self.output_file} was updated") + return 0 + return 1 + + return intra + + def _save_all_rules(self, issues: List[Issue]) -> bool: + """Checks and create SQ rules if needed""" + if not issues: + return False + rules = defaultdict(list) + for issue in issues: + rules[issue.rule.tool_name].append(issue.rule) + for tool_name, tool_rules in rules.items(): + exist_rules = [rule["key"] for rule in self._get_list(f"rules/search?include_external=true&q={tool_name}", "rules")] + grouped_rules = {rule.sq_key: rule for rule in tool_rules} + for sq_key, rule in grouped_rules.items(): + if sq_key in exist_rules: + # was created before + continue + self.logger.info(f"try to create the rule: {sq_key}") + body = { + "custom_key": rule.unique_key, + "markdown_description": rule.description, + "name": rule.name, + "severity": rule.severity.value, + "type": rule.rule_type.value, + "template_key": rule.template + } + self._post("rules/create", body) + self.logger.info(f"the rule {sq_key} was created") + return True + + def _issues2dict(self, issues: List[Issue]) -> Mapping[str, Any]: + """ + { + "issues": [ + { + "engineId": "test", + "ruleId": "rule1", + "severity":"BLOCKER", + "type":"CODE_SMELL", + "primaryLocation": { + "message": "fully-fleshed issue", + "filePath": "sources/A.java", + "textRange": { + "startLine": 30, + "endLine": 30, + "startColumn": 9, + "endColumn": 14 + } + } + }, + ... + ]}""" + return { + "issues": [issue.to_json() for issue in issues] + } + + @prepare_file + def from_mypy(self, file: TextIO) -> List[Issue]: + buff = None + items = [] + + for line in file: + line = line.strip() + if RE_MYPY_LINE.match(line): + if buff: + items.append(self.__parse_mypy_issue(buff)) + buff = [] + if buff is not None: + buff.append(line) + if buff is None: + # mypy can return an error without line/column values + file.seek(0) + for line in file: + m = RE_MYPY_LINE_WO_COORDINATES.match(line.strip()) + if not m: + continue + items.append(self.Issue( + path=m.group(1).strip(), + description=m.group(2).strip(), + rule=self._mypy_rules["[unknown]"], + )) + self.logger.info(f"detected an error without coordinates: {line}") + + items.append(self.__parse_mypy_issue(buff)) + return [i for i in items if i] + + @classmethod + def __parse_mypy_issue(cls, lines: List[str]) -> Optional[Issue]: + """" + An example of log response: + source_airtable/helpers.py:8:1: error: Library stubs not installed for + "requests" (or incompatible with Python 3.7) [import] + import requests + ^ + source_airtable/helpers.py:8:1: note: Hint: "python3 -m pip install types-requests" + """ + if not lines: + return None + path, line_number, column_number, error_or_note, *others = " ".join(lines).split(":") + if "test" in Path(path).name: + cls.logger.info(f"skip the test file: {path}") + return None + if error_or_note.strip() == "note": + return None + others = ":".join(others) + rule = None + for code in cls._mypy_rules: + if code in others: + rule = cls._mypy_rules[code] + others = re.sub(r"\s+", " ", others.replace(code, ". Code line: ")) + break + if not rule: + cls.logger.warning(f"couldn't parse the lines: {lines}") + return None + + description = others.split("^")[0] + + return cls.Issue( + path=path.strip(), + line_number=int(line_number.strip()), + column_number=int(column_number.strip()), + description=description.strip(), + rule=rule, + ) + + @staticmethod + def __parse_diff(lines: List[str]) -> Mapping[str, int]: + """Converts diff lines to mapping: + {file1: , file2: } + """ + patch = PatchSet(lines, metadata_only=True) + return {updated_file.path: len(updated_file) for updated_file in patch} + + @prepare_file + def from_black(self, file: TextIO) -> List[Issue]: + return [self.Issue( + path=path, + description=f"{count} code part(s) should be updated.", + rule=self._black_rule, + ) for path, count in self.__parse_diff(file.readlines()).items()] + + @prepare_file + def from_isort(self, file: TextIO) -> List[Issue]: + changes = defaultdict(lambda: 0) + for path, count in self.__parse_diff(file.readlines()).items(): + # check path value + # path in isort diff file has the following format + # :before|after + if path.endswith(":before"): + path = path[:-len(":before")] + elif path.endswith(":after"): + path = path[:-len(":after")] + changes[path] += count + + return [self.Issue( + path=path, + description=f"{count} code part(s) should be updated.", + rule=self._isort_rule, + ) for path, count in changes.items()] diff --git a/tools/ci_code_validator/ci_sonar_qube/main.py b/tools/ci_code_validator/ci_sonar_qube/main.py new file mode 100644 index 0000000000000..17c8370f1afd4 --- /dev/null +++ b/tools/ci_code_validator/ci_sonar_qube/main.py @@ -0,0 +1,60 @@ +import argparse +import sys + +from .log_parsers import LogParser +from .sonar_qube_api import SonarQubeApi + + +def main() -> int: + convert_key = len(set(["--mypy_log", "--black_diff", "--isort_diff"]) & set(sys.argv)) > 0 + need_print_key = "--print_key" in sys.argv + + parser = argparse.ArgumentParser(description='Working with SonarQube instance.') + parser.add_argument('--host', help='SonarQube host', required=not need_print_key, type=str) + parser.add_argument('--token', help='SonarQube token', required=not need_print_key, type=str) + parser.add_argument('--pr', help='PR unique name. Example: airbyte/1231', type=str, default=None) + + name_value = parser.add_mutually_exclusive_group(required=not convert_key) + name_value.add_argument('--project', help='Name of future project', type=str) + name_value.add_argument('--module', help='Name of future module project', type=str) + + command = parser.add_mutually_exclusive_group(required=not convert_key) + command.add_argument('--print_key', help='Return a generate SonarQube key', action="store_true") + command.add_argument('--report', help='generate .md file with current issues of a project') + command.add_argument('--create', help='create a project', action="store_true") + command.add_argument('--remove', help='remove project', action="store_true") + + parser.add_argument('--mypy_log', help='Path to MyPy Logs', required=False, type=str) + parser.add_argument('--black_diff', help='Path to Black Diff', required=False, type=str) + parser.add_argument('--isort_diff', help='Path to iSort Diff', required=False, type=str) + parser.add_argument('--output_file', help='Path of output file', required=convert_key, type=str) + + args = parser.parse_args() + if convert_key: + parser = LogParser(output_file=args.output_file, host=args.host, token=args.token) + if args.mypy_log: + return parser.from_mypy(args.mypy_log) + if args.black_diff: + return parser.from_black(args.black_diff) + if args.isort_diff: + return parser.from_isort(args.isort_diff) + api = SonarQubeApi(host=args.host, token=args.token, pr_name=args.pr) + + project_name = api.module2project(args.module) if args.module else args.project + + if args.create: + return 0 if api.create_project(project_name=project_name) else 1 + elif args.remove: + return 0 if api.remove_project(project_name=project_name) else 1 + elif args.print_key: + data = api.prepare_project_settings(project_name) + print(data["project"], file=sys.stdout) + return 0 + elif args.report: + return 0 if api.generate_report(project_name=project_name, report_file=args.report) else 1 + api.logger.critical("not set any action...") + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/ci_code_validator/ci_sonar_qube/sonar_qube_api.py b/tools/ci_code_validator/ci_sonar_qube/sonar_qube_api.py new file mode 100644 index 0000000000000..1b96f5ec6e785 --- /dev/null +++ b/tools/ci_code_validator/ci_sonar_qube/sonar_qube_api.py @@ -0,0 +1,342 @@ +import itertools +import re +from functools import reduce +from typing import Mapping, Any, Optional, List +from urllib.parse import urljoin + +import requests +from mdutils.mdutils import MdUtils +from requests.auth import HTTPBasicAuth + +from ci_common_utils import Logger + +AIRBYTE_PROJECT_PREFIX = "airbyte" +RE_RULE_NAME = re.compile(r"(.+):[A-Za-z]+(\d+)") + +REPORT_METRICS = ( + "alert_status", + # "quality_gate_details", + "bugs", "new_bugs", + "reliability_rating", "new_reliability_rating", + "vulnerabilities", "new_vulnerabilities", + "security_rating", "new_security_rating", + # "security_hotspots", "new_security_hotspots", + # "security_hotspots_reviewed", "new_security_hotspots_reviewed", + # "security_review_rating", "new_security_review_rating", + "code_smells", "new_code_smells", + # "sqale_rating", "new_maintainability_rating", + # "sqale_index", "new_technical_debt", + "coverage", "new_coverage", + "lines_to_cover", "new_lines_to_cover", + "tests", + "duplicated_lines_density", "new_duplicated_lines_density", + "duplicated_blocks", + "ncloc", + # "ncloc_language_distribution", + # "projects", + # "lines", "new_lines" +) + +RATINGS = { + 1.0: "A", + 2.0: "B", + 3.0: "C", + 4.0: "D", + 5.0: "F", +} + + +class SonarQubeApi: + """https://sonarcloud.io/web_api""" + logger = Logger() + + def __init__(self, host: str, token: str, pr_name: str): + + self._host = host + self._token = token + + # split the latest name part + self._pr_id = (pr_name or '').split("/")[-1] + if not self._pr_id.isdigit(): + self.logger.critical(f"PR id should be integer. Current value: {pr_name}") + + self._pr_id = int(self._pr_id) + # check token + # https://sonarcloud.io/web_api/api/authentication/validate + if not self._host: + return + resp = self._get("authentication/validate") + if not resp["valid"]: + self.logger.critical("provided token is not valid") + + @property + def __auth(self): + return HTTPBasicAuth(self._token, '') + + def __parse_response(self, url: str, response: requests.Response) -> Mapping[str, Any]: + if response.status_code == 204: + # empty response + return {} + elif response.status_code != 200: + self.logger.critical(f"API error for {url}: [{response.status_code}] {response.json()['errors']}") + return response.json() + + def generate_url(self, endpoint: str) -> str: + return reduce(urljoin, [self._host, "/api/", endpoint]) + + def _post(self, endpoint: str, json: Mapping[str, Any]) -> Mapping[str, Any]: + url = self.generate_url(endpoint) + return self.__parse_response(url, requests.post(url, auth=self.__auth, params=json, json=json)) + + def _get(self, endpoint: str) -> Mapping[str, Any]: + url = self.generate_url(endpoint) + return self.__parse_response(url, requests.get(url, auth=self.__auth)) + + def _get_list(self, endpoint: str, list_name: str) -> List[Mapping[str, Any]]: + + page = 0 + items = [] + while True: + page += 1 + url = endpoint + "&" if "?" in endpoint else "?" + f"p={page}" + data = self._get(url) + items += data[list_name] + total = data.get("total") or data.get("paging", {}).get("total", 0) + if len(items) >= total: + break + return items + + @classmethod + def module2project(cls, module_name: str) -> str: + """""" + parts = module_name.split("/") + if len(parts) != 2: + cls.logger.critical("module name must have the format: component/module") + return f"{AIRBYTE_PROJECT_PREFIX}:{parts[0].lower()}:{parts[1].lower().replace('_', '-')}" + + def __correct_project_name(self, project_name: str) -> str: + return f"pr:{self._pr_id}:{project_name}" if self._pr_id else f"master:{project_name}" + + def __search_project(self, project_name: str) -> Optional[Mapping[str, Any]]: + """https://sonarcloud.io/web_api/api/projects/search""" + data = self._get(f"projects/search?q={project_name}") + exists_projects = data["components"] + if len(exists_projects) > 1: + self.logger.critical(f"there are several projects with the name '{project_name}'") + elif len(exists_projects) == 0: + return None + return exists_projects[0] + + def prepare_project_settings(self, project_name: str) -> Mapping[str, str]: + title = re.sub('[:_-]', ' ', project_name).replace("connectors_", "").title() + if self._pr_id: + title += f"(#{self._pr_id})" + + project_name = self.__correct_project_name(project_name) + return { + "name": title, + "project": project_name, + "visibility": "private", + } + + def create_project(self, project_name: str) -> bool: + """https://sonarcloud.io/web_api/api/projects/create""" + data = self.prepare_project_settings(project_name) + project_name = data["project"] + exists_project = self.__search_project(project_name) + if exists_project: + self.logger.info(f"The project '{project_name}' was created before") + return True + + self._post("projects/create", data) + self.logger.info(f"The project '{project_name}' was created") + return True + + def remove_project(self, project_name: str) -> bool: + """https://sonarcloud.io/web_api/api/projects/delete""" + project_name = self.prepare_project_settings(project_name)["project"] + + exists_project = self.__search_project(project_name) + if exists_project is None: + self.logger.info(f"not found the project '{project_name}'") + return True + body = { + "project": project_name + } + self._post("projects/delete", body) + self.logger.info(f"The project '{project_name}' was removed") + return True + + def generate_report(self, project_name: str, report_file: str) -> bool: + project_data = self.prepare_project_settings(project_name) + + md_file = MdUtils(file_name=report_file) + md_file.new_line(f'### SonarQube report for {project_data["name"]}') + + project_name = project_data["project"] + + issues = self._get_list(f"issues/search?componentKeys={project_name}&additionalFields=_all", "issues") + rules = {} + for rule_key in set(issue["rule"] for issue in issues): + key_parts = rule_key.split(":") + while len(key_parts) > 2: + key_parts.pop(0) + key = ":".join(key_parts) + + data = self._get(f"rules/search?rule_key={key}")["rules"] + if not data: + data = self._get(f"rules/show?key={rule_key}")["rule"] + else: + data = data[0] + + description = data["name"] + public_name = key + link = None + if rule_key.startswith("external_"): + public_name = key.replace("external_", "") + if not data["isExternal"]: + # this is custom rule + description = data["htmlDesc"] + if public_name.startswith("flake"): + # single link for all descriptions + link = "https://flake8.pycqa.org/en/latest/user/error-codes.html" + elif "isort_" in public_name: + link = "https://pycqa.github.io/isort/index.html" + elif "black_" in public_name: + link = "https://black.readthedocs.io/en/stable/the_black_code_style/index.html" + else: + # link's example + # https://rules.sonarsource.com/python/RSPEC-6287 + m = RE_RULE_NAME.match(public_name) + if not m: + # for local server + link = f"{self._host}coding_rules?open={key}&rule_key={key}" + else: + # to public SQ docs + link = f"https://rules.sonarsource.com/{m.group(1)}/RSPEC-{m.group(2)}" + if link: + public_name = md_file.new_inline_link( + link=link, + text=public_name + ) + + rules[rule_key] = (public_name, description) + + data = self._get(f"measures/component?component={project_name}&additionalFields=metrics&metricKeys={','.join(REPORT_METRICS)}") + measures = {} + total_coverage = None + for measure in data["component"]["measures"]: + metric = measure["metric"] + if measure["metric"].startswith("new_") and measure.get("periods"): + # we need to show values for last sync period only + last_period = max(measure["periods"], key=lambda period: period["index"]) + value = last_period["value"] + else: + value = measure.get("value") + measures[metric] = value + # group overall and latest values + measures = {metric: (value, measures.get(f"new_{metric}")) for metric, value in measures.items() if + not metric.startswith("new_")} + metrics = {} + for metric in data["metrics"]: + # if metric["key"] not in measures: + # continue + metrics[metric["key"]] = (metric["name"], metric["type"]) + + md_file.new_line('#### Measures') + + values = [] + for metric, (overall_value, latest_value) in measures.items(): + if metric not in metrics: + continue + name, metric_type = metrics[metric] + value = overall_value if (latest_value is None or latest_value == "0") else latest_value + + if metric_type == "PERCENT": + value = str(round(float(value), 1)) + elif metric_type == "INT": + value = int(float(value)) + elif metric_type == "LEVEL": + pass + elif metric_type == "RATING": + value = int(float(value)) + for k, v in RATINGS.items(): + if value <= k: + value = v + break + if metric == "coverage": + total_coverage = value + values.append([name, value]) + + values += [ + ("Blocker Issues", sum(map(lambda i: i["severity"] == "BLOCKER", issues))), + ("Critical Issues", sum(map(lambda i: i["severity"] == "CRITICAL", issues))), + ("Major Issues", sum(map(lambda i: i["severity"] == "MAJOR", issues))), + ("Minor Issues", sum(map(lambda i: i["severity"] == "MINOR", issues))), + ] + + while len(values) % 3: + values.append(("", "")) + table_items = ["Name", "Value"] * 3 + list(itertools.chain.from_iterable(values)) + md_file.new_table(columns=6, rows=int(len(values) / 3 + 1), text=table_items, text_align='left') + md_file.new_line() + if issues: + md_file.new_line('#### Detected Issues') + table_items = [ + "Rule", "File", "Description", "Message" + ] + for issue in issues: + rule_name, description = rules[issue["rule"]] + path = issue["component"].split(":")[-1].split("/") + # need to show only 2 last path parts + while len(path) > 2: + path.pop(0) + path = "/".join(path) + + # add line number in the end + if issue.get("line"): + path += f':{issue["line"]}' + table_items += [ + f'{rule_name} ({issue["severity"]})', + path, + description, + issue["message"], + ] + + md_file.new_table(columns=4, rows=len(issues) + 1, text=table_items, text_align='left') + coverage_files = [(k, v) for k, v in self.load_coverage_component(project_name).items()] + if total_coverage is not None: + md_file.new_line(f'#### Coverage ({total_coverage}%)') + while len(coverage_files) % 2: + coverage_files.append(("", "")) + table_items = ["File", "Coverage"] * 2 + list(itertools.chain.from_iterable(coverage_files)) + md_file.new_table(columns=4, rows=int(len(coverage_files) / 2 + 1), text=table_items, text_align='left') + md_file.create_md_file() + self.logger.info(f"The {report_file} was generated") + return True + + def load_coverage_component(self, base_component: str, dir_path: str = None) -> Mapping[str, Any]: + + page = 0 + coverage_files = {} + read_count = 0 + while True: + page += 1 + component = base_component + if dir_path: + component += f":{dir_path}" + url = f"measures/component_tree?p={page}&component={component}&additionalFields=metrics&metricKeys=coverage,uncovered_lines,uncovered_conditions&strategy=children" + data = self._get(url) + read_count += len(data["components"]) + for component in data["components"]: + if component["qualifier"] == "DIR": + coverage_files.update(self.load_coverage_component(base_component, component["path"])) + continue + elif not component["measures"]: + continue + elif component["qualifier"] == "FIL": + coverage_files[component["path"]] = [m["value"] for m in component["measures"] if m["metric"] == "coverage"][0] + if data["paging"]["total"] <= read_count: + break + + return coverage_files diff --git a/tools/ci_code_validator/setup.py b/tools/ci_code_validator/setup.py new file mode 100644 index 0000000000000..87d4c2ad3b3fb --- /dev/null +++ b/tools/ci_code_validator/setup.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "requests", + "ci_common_utils", + "unidiff", + "mdutils~=1.3.1" +] + +TEST_REQUIREMENTS = [ + "requests-mock", + "pytest", + "black", + "mypy", + "lxml", + "isort" +] + +setup( + version="0.0.0", + name="ci_code_validator", + description="Load and extract CI secrets for test suites", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + python_requires='>=3.7', + extras_require={ + "tests": TEST_REQUIREMENTS, + + }, + entry_points={ + 'console_scripts': [ + 'ci_sonar_qube = ci_sonar_qube.main:main', + 'ci_changes_detection = ci_changes_detection.main:main', + ], + }, +) diff --git a/tools/ci_code_validator/tests/__init__.py b/tools/ci_code_validator/tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tools/ci_code_validator/tests/simple_files/black_smell_package_report.json b/tools/ci_code_validator/tests/simple_files/black_smell_package_report.json new file mode 100644 index 0000000000000..f5ba0fa01f794 --- /dev/null +++ b/tools/ci_code_validator/tests/simple_files/black_smell_package_report.json @@ -0,0 +1,14 @@ +{ + "issues": [ + { + "engineId": "black", + "ruleId": "python:black_need_format", + "severity": "MINOR", + "type": "CODE_SMELL", + "primaryLocation": { + "message": "1 code part(s) should be updated.", + "filePath": "simple_smell_package/invalid_file.py" + } + } + ] +} diff --git a/tools/ci_code_validator/tests/simple_files/isort_smell_package_report.json b/tools/ci_code_validator/tests/simple_files/isort_smell_package_report.json new file mode 100644 index 0000000000000..34c4c7c63f61d --- /dev/null +++ b/tools/ci_code_validator/tests/simple_files/isort_smell_package_report.json @@ -0,0 +1,14 @@ +{ + "issues": [ + { + "engineId": "isort", + "ruleId": "python:isort_need_format", + "severity": "MINOR", + "type": "CODE_SMELL", + "primaryLocation": { + "message": "1 code part(s) should be updated.", + "filePath": "simple_smell_package/invalid_file.py" + } + } + ] +} diff --git a/tools/ci_code_validator/tests/simple_files/mypy_smell_package_report.json b/tools/ci_code_validator/tests/simple_files/mypy_smell_package_report.json new file mode 100644 index 0000000000000..968becdca1dca --- /dev/null +++ b/tools/ci_code_validator/tests/simple_files/mypy_smell_package_report.json @@ -0,0 +1,52 @@ +{ + "issues": [ + { + "engineId": "mypy", + "ruleId": "python:mypy_return_value", + "severity": "MINOR", + "type": "CODE_SMELL", + "primaryLocation": { + "message": "Incompatible return value type (got \"int\", expected \"str\") . Code line: return 1000", + "filePath": "simple_smell_package/invalid_file.py", + "textRange": { + "startLine": 11, + "endLine": 11, + "startColumn": 11, + "endColumn": 12 + } + } + }, + { + "engineId": "mypy", + "ruleId": "python:mypy_no_redef", + "severity": "MINOR", + "type": "CODE_SMELL", + "primaryLocation": { + "message": "Name \"fake_func\" already defined on line 10 . Code line: def fake_func(i):", + "filePath": "simple_smell_package/invalid_file.py", + "textRange": { + "startLine": 14, + "endLine": 14, + "startColumn": 0, + "endColumn": 1 + } + } + }, + { + "engineId": "mypy", + "ruleId": "python:mypy_no_untyped_def", + "severity": "MINOR", + "type": "CODE_SMELL", + "primaryLocation": { + "message": "Function is missing a type annotation . Code line: def fake_func(i):", + "filePath": "simple_smell_package/invalid_file.py", + "textRange": { + "startLine": 14, + "endLine": 14, + "startColumn": 0, + "endColumn": 1 + } + } + } + ] +} diff --git a/tools/ci_code_validator/tests/simple_files/without_issues_report.json b/tools/ci_code_validator/tests/simple_files/without_issues_report.json new file mode 100644 index 0000000000000..2eb23288a67d0 --- /dev/null +++ b/tools/ci_code_validator/tests/simple_files/without_issues_report.json @@ -0,0 +1 @@ +{"issues": []} \ No newline at end of file diff --git a/tools/ci_code_validator/tests/simple_package/__init__.py b/tools/ci_code_validator/tests/simple_package/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tools/ci_code_validator/tests/simple_package/valid_file.py b/tools/ci_code_validator/tests/simple_package/valid_file.py new file mode 100644 index 0000000000000..5ad71b1c936f3 --- /dev/null +++ b/tools/ci_code_validator/tests/simple_package/valid_file.py @@ -0,0 +1,13 @@ +# don't valid it by auto-linters becaise this file is used for testing +import os +from pathlib import Path + +LONG_STRING = """aaaaaaaaaaaaaaaa""" + + +def func() -> bool: + return Path(os.getcwd()).is_dir() is True + + +def func2(i: int) -> int: + return i * 10 diff --git a/tools/ci_code_validator/tests/simple_smell_package/__init__.py b/tools/ci_code_validator/tests/simple_smell_package/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tools/ci_code_validator/tests/simple_smell_package/invalid_file.py b/tools/ci_code_validator/tests/simple_smell_package/invalid_file.py new file mode 100644 index 0000000000000..b99a26f148566 --- /dev/null +++ b/tools/ci_code_validator/tests/simple_smell_package/invalid_file.py @@ -0,0 +1,15 @@ +# don't valid it by auto-linters because this file is used for testing +import pathlib +import os + + + +LONG_STRING = """aaaaaaaaaaaaaaaaaaaaaaaaaawwwwwwwwwwwwwwwwwwwwwwwww mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm mmmmmmmmmmmmmm wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww""" + + +def fake_func() -> str: + return 1000 + + +def fake_func(i): + return i * 10 diff --git a/tools/ci_code_validator/tests/test_detect_changed_modules.py b/tools/ci_code_validator/tests/test_detect_changed_modules.py new file mode 100644 index 0000000000000..cd1956fc17fd6 --- /dev/null +++ b/tools/ci_code_validator/tests/test_detect_changed_modules.py @@ -0,0 +1,74 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# +from typing import List, Set + +import pytest +from ci_changes_detection.main import list_changed_modules +from ci_sonar_qube import ROOT_DIR + + +@pytest.mark.parametrize( + "changed_files,changed_modules", + [ + (["path/to/file1", "file2.txt", "path/to/file3.txt"], []), + ( + [ + "airbyte-integrations/connectors/source-asana/source_asana/streams.py", + "airbyte-integrations/connectors/source-asana/source_asana/source.py", + "airbyte-integrations/connectors/source-braintree/integration_tests/abnormal_state.json", + ], + [ + {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/source-asana"), "lang": "py", + "module": "connectors/source-asana"}, + {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/source-braintree"), "lang": "py", + "module": "connectors/source-braintree"}, + ], + ), + ( + [ + "airbyte-integrations/connectors/destination-mongodb/build.gradle", + "airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java", + "airbyte-integrations/connectors/destination-s3/Dockerfile", + ], + [ + {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/destination-mongodb"), "lang": "java", + "module": "connectors/destination-mongodb"}, + {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/destination-s3"), "lang": "java", + "module": "connectors/destination-s3"}, + ], + ), + ( + [ + "airbyte-integrations/connectors/source-s3/Dockerfile", + "airbyte-integrations/connectors/destination-s3/Dockerfile", + "tools/ci_code_validator" + ], + [ + {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/source-s3"), "lang": "py", + "module": "connectors/source-s3"}, + {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/destination-s3"), "lang": "java", + "module": "connectors/destination-s3"}, + ], + ), + ( + [ + "airbyte-integrations/connectors/source-s3/Dockerfile", + "airbyte-integrations/connectors/destination-s3/Dockerfile", + "tools/ci_code_validator" + ], + [ + {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/source-s3"), "lang": "py", + "module": "connectors/source-s3"}, + {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/destination-s3"), "lang": "java", + "module": "connectors/destination-s3"}, + ], + ), + + ], + ids=["incorrect_files", "py_modules_only", "java_modules_only", "mix_modules", "absolute_paths"], +) +def test_list_changed_modules(changed_files: List[str], changed_modules: Set[str]) -> None: + calculated_changed_modules = list_changed_modules(changed_files) + + assert calculated_changed_modules == changed_modules diff --git a/tools/ci_code_validator/tests/test_sq_project.py b/tools/ci_code_validator/tests/test_sq_project.py new file mode 100644 index 0000000000000..4fb0ea1b38149 --- /dev/null +++ b/tools/ci_code_validator/tests/test_sq_project.py @@ -0,0 +1,20 @@ +import pytest +import requests_mock +from ci_sonar_qube.sonar_qube_api import SonarQubeApi + + +@pytest.mark.parametrize( + "module_name,pr, expected_title, expected_key", + [ + ("connectors/source-s3", "airbyte/1234", "Airbyte Connectors Source S3(#1234)", "pr:1234:airbyte:connectors:source-s3"), + ("tools/ci_code_validator", "airbyte/1111", "Airbyte Tools Ci Code Validator(#1111)", "pr:1111:airbyte:tools:ci-code-validator"), + ("airbyte-cdk/python", "0", "Airbyte Airbyte Cdk Python", "master:airbyte:airbyte-cdk:python"), + ] +) +def test_module2project(module_name, pr, expected_title, expected_key): + with requests_mock.Mocker() as m: + m.get('/api/authentication/validate', json={"valid": True}) + api = SonarQubeApi(host="http://fake.com/", token="", pr_name=pr) + project_settings = api.prepare_project_settings(api.module2project(module_name)) + assert project_settings["name"] == expected_title + assert project_settings["project"] == expected_key diff --git a/tools/ci_code_validator/tests/test_tools.py b/tools/ci_code_validator/tests/test_tools.py new file mode 100644 index 0000000000000..3a9b7593062ad --- /dev/null +++ b/tools/ci_code_validator/tests/test_tools.py @@ -0,0 +1,102 @@ +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest +import requests_mock + +from ci_code_validator.ci_sonar_qube.log_parsers import LogParser + +HERE = Path(__file__).parent +PACKAGE_DIR = HERE / "simple_package" +SMELL_PACKAGE_DIR = HERE / "simple_smell_package" +SIMPLE_FILES = HERE / "simple_files" +WITHOUT_ISSUE_REPORT = SIMPLE_FILES / "without_issues_report.json" + +ISORT_CMD = """isort --diff {package_dir}""" # config file should be in a started folder +BLACK_CMD = r"""black --config {toml_config_file} --diff {package_dir}""" +MYPY_CMD = r"""mypy {package_dir} --config-file={toml_config_file}""" + + +@pytest.fixture(scope="session") +def toml_config_file() -> Path: + root_dir = HERE + while str(root_dir) != root_dir.root: + config_file = root_dir / "pyproject.toml" + if config_file.is_file(): + return config_file + root_dir = root_dir.parent + raise Exception("can't found pyproject.toml") + + +@pytest.fixture(autouse=True) +def prepare_toml_file(toml_config_file): + pyproject_toml = Path(os.getcwd()) / "pyproject.toml" + if toml_config_file != pyproject_toml and not pyproject_toml.is_file(): + shutil.copy(toml_config_file, pyproject_toml) + yield + if toml_config_file != pyproject_toml and pyproject_toml.is_file(): + os.remove(str(pyproject_toml)) + + +@pytest.mark.parametrize( + "cmd,package_dir,expected_file", + [ + ( + "mypy {package_dir} --config-file={toml_config_file}", + SMELL_PACKAGE_DIR, + SIMPLE_FILES / "mypy_smell_package_report.json" + ), + ( + "mypy {package_dir} --config-file={toml_config_file}", + PACKAGE_DIR, + WITHOUT_ISSUE_REPORT + ), + ( + "black --config {toml_config_file} --diff {package_dir}", + SMELL_PACKAGE_DIR, + HERE / "simple_files/black_smell_package_report.json" + ), + ( + "black --config {toml_config_file} --diff {package_dir}", + PACKAGE_DIR, + WITHOUT_ISSUE_REPORT + ), + ( + ISORT_CMD, + SMELL_PACKAGE_DIR, + HERE / "simple_files/isort_smell_package_report.json" + ), + ( + ISORT_CMD, + PACKAGE_DIR, + WITHOUT_ISSUE_REPORT, + ), + ], + ids=["mypy_failed", "mypy_pass", "black_failed", "black_pass", "isort_failed", "isort_pass"] +) +def test_tool(tmp_path, toml_config_file, cmd, package_dir, expected_file): + cmd = cmd.format(package_dir=package_dir, toml_config_file=toml_config_file) + + proc = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, _ = proc.communicate() + file_log = tmp_path / "temp.log" + file_log.write_bytes(out) + assert file_log.is_file() is True + issues_file = tmp_path / "issues.json" + with requests_mock.Mocker() as m: + m.get('/api/authentication/validate', json={"valid": True}) + m.get("/api/rules/search", json={"rules": []}) + m.post("/api/rules/create", json={}) + parser = LogParser(issues_file, host="http://fake.com/", token="fake_token") + assert getattr(parser, f'from_{cmd.split(" ")[0]}')(file_log) == 0 + + assert issues_file.is_file() is True + data = json.loads(issues_file.read_text()) + for issue in data["issues"]: + issue["primaryLocation"]["filePath"] = "/".join(issue["primaryLocation"]["filePath"].split("/")[-2:]) + + expected_data = json.loads(Path(expected_file).read_text()) + assert json.dumps(data, sort_keys=True) == json.dumps(expected_data, sort_keys=True) diff --git a/tools/ci_credentials/setup.py b/tools/ci_credentials/setup.py index 6c072e755223d..f4c4136d534ff 100644 --- a/tools/ci_credentials/setup.py +++ b/tools/ci_credentials/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["requests", "ci_common_utils", "pytest"] +MAIN_REQUIREMENTS = ["requests", "ci_common_utils"] TEST_REQUIREMENTS = ["requests-mock"] diff --git a/tools/ci_static_check_reports/__init__.py b/tools/ci_static_check_reports/__init__.py deleted file mode 100644 index 46b7376756ec6..0000000000000 --- a/tools/ci_static_check_reports/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/tools/ci_static_check_reports/ci_build_python_static_checkers_reports/__init__.py b/tools/ci_static_check_reports/ci_build_python_static_checkers_reports/__init__.py deleted file mode 100644 index 46b7376756ec6..0000000000000 --- a/tools/ci_static_check_reports/ci_build_python_static_checkers_reports/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/tools/ci_static_check_reports/ci_build_python_static_checkers_reports/main.py b/tools/ci_static_check_reports/ci_build_python_static_checkers_reports/main.py deleted file mode 100644 index 6e779b3e4e1f2..0000000000000 --- a/tools/ci_static_check_reports/ci_build_python_static_checkers_reports/main.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# -import argparse -import json -import os -import sys -from typing import Dict, List - -from invoke import Context - -sys.path.insert(0, "airbyte-integrations/connectors") -from tasks import CONFIG_FILE, TOOLS_VERSIONS, _run_task # noqa - -TASK_COMMANDS: Dict[str, List[str]] = { - "black": [ - f"pip install black~={TOOLS_VERSIONS['black']}", - f"XDG_CACHE_HOME={os.devnull} black -v {{check_option}} --diff {{source_path}}/. > {{reports_path}}/black.txt", - ], - "coverage": [ - "pip install .", - f"pip install coverage[toml]~={TOOLS_VERSIONS['coverage']}", - "coverage xml --rcfile={toml_config_file} -o {reports_path}/coverage.xml", - ], - "flake": [ - f"pip install mccabe~={TOOLS_VERSIONS['mccabe']}", - f"pip install pyproject-flake8~={TOOLS_VERSIONS['flake']}", - f"pip install flake8-junit-report~={TOOLS_VERSIONS['flake_junit']}", - "pflake8 -v {source_path} --output-file={reports_path}/flake.txt --bug-report", - "flake8_junit {reports_path}/flake.txt {reports_path}/flake.xml", - "rm -f {reports_path}/flake.txt", - ], - "isort": [ - f"pip install colorama~={TOOLS_VERSIONS['colorama']}", - f"pip install isort~={TOOLS_VERSIONS['isort']}", - "isort -v {check_option} {source_path}/. > {reports_path}/isort.txt", - ], - "mypy": [ - "pip install .", - f"pip install lxml~={TOOLS_VERSIONS['lxml']}", - f"pip install mypy~={TOOLS_VERSIONS['mypy']}", - "mypy {source_path} --config-file={toml_config_file} --cobertura-xml-report={reports_path}", - ], - "test": [ - "mkdir {venv}/source-acceptance-test", - "cp -f $(git ls-tree -r HEAD --name-only {source_acceptance_test_path} | tr '\n' ' ') {venv}/source-acceptance-test", - "pip install build", - f"python -m build {os.path.join('{venv}', 'source-acceptance-test')}", - f"pip install {os.path.join('{venv}', 'source-acceptance-test', 'dist', 'source_acceptance_test-*.whl')}", - "[ -f requirements.txt ] && pip install -r requirements.txt 2> /dev/null", - "pip install .", - "pip install .[tests]", - "pip install pytest-cov", - "pytest -v --cov={source_path} --cov-report xml:{reports_path}/pytest.xml {source_path}/unit_tests", - ], -} - - -def build_static_checkers_reports(modules: list, static_checker_reports_path: str) -> int: - ctx = Context() - toml_config_file = os.path.join(os.getcwd(), "pyproject.toml") - - for module_path in modules: - reports_path = f"{os.getcwd()}/{static_checker_reports_path}/{module_path}" - if not os.path.exists(reports_path): - os.makedirs(reports_path) - - for checker in TASK_COMMANDS: - _run_task( - ctx, - f"{os.getcwd()}/{module_path}", - checker, - module_path=module_path, - multi_envs=True, - check_option="", - task_commands=TASK_COMMANDS, - toml_config_file=toml_config_file, - reports_path=reports_path, - source_acceptance_test_path=os.path.join(os.getcwd(), "airbyte-integrations/bases/source-acceptance-test"), - ) - return 0 - - -def main() -> int: - parser = argparse.ArgumentParser(description="Working with Python Static Report Builder.") - parser.add_argument("changed_modules", nargs="*") - parser.add_argument("--static-checker-reports-path", help="SonarQube host", required=False, type=str, default="static_checker_reports") - - args = parser.parse_args() - changed_python_module_paths = [ - module["dir"] - for module in json.loads(args.changed_modules[0]) - if module["lang"] == "py" and os.path.exists(module["dir"]) and "setup.py" in os.listdir(module["dir"]) - ] - print("Changed python modules: ", changed_python_module_paths) - return build_static_checkers_reports(changed_python_module_paths, static_checker_reports_path=args.static_checker_reports_path) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/ci_static_check_reports/ci_detect_changed_modules/main.py b/tools/ci_static_check_reports/ci_detect_changed_modules/main.py deleted file mode 100644 index a2a68c3be06f1..0000000000000 --- a/tools/ci_static_check_reports/ci_detect_changed_modules/main.py +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# -import json -import os -import sys -from typing import Dict, List, Set - -# Filenames used to detect whether the dir is a module -LANGUAGE_MODULE_ID_FILE = { - ".py": "setup.py", - # TODO: Add ID files for other languages -} - - -def find_base_path(path: str, modules: List[Dict[str, str]], unique_modules: Set[str], file_ext: str = "", lookup_file: str = None) -> None: - filename, file_extension = os.path.splitext(path) - lookup_file = lookup_file or LANGUAGE_MODULE_ID_FILE.get(file_extension) - - dir_path = os.path.dirname(filename) - if dir_path and os.path.exists(dir_path): - is_module_root = lookup_file in os.listdir(dir_path) - if is_module_root: - if dir_path not in unique_modules: - modules.append({"dir": dir_path, "lang": file_ext[1:]}) - unique_modules.add(dir_path) - else: - find_base_path(dir_path, modules, unique_modules, file_ext=file_extension, lookup_file=lookup_file) - - -def list_changed_modules(changed_files: List[str]) -> List[Dict[str, str]]: - """ - changed_filed are the list of files which were modified in current branch. - E.g. changed_files = ["tools/ci_static_check_reports/__init__.py", "tools/ci_static_check_reports/setup.py", ...] - """ - - modules: List[Dict[str, str]] = [] - unique_modules: set = set() - for file_path in changed_files: - _, file_extension = os.path.splitext(file_path) - find_base_path(file_path, modules, file_ext=file_extension, unique_modules=unique_modules) - return modules - - -def main() -> int: - changed_modules = list_changed_modules(sys.argv[1:]) - print(json.dumps(changed_modules)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/ci_static_check_reports/setup.py b/tools/ci_static_check_reports/setup.py deleted file mode 100644 index 4241142328843..0000000000000 --- a/tools/ci_static_check_reports/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -TEST_REQUIREMENTS = [ - "pytest~=6.1", -] - -setup( - name="ci_static_check_reports", - description="CI tool to detect changed modules and then generate static check reports.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=["invoke~=1.6.0", "virtualenv~=20.10.0"], - package_data={"": ["*.json", "schemas/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, - entry_points={ - "console_scripts": [ - "ci_detect_changed_modules = ci_detect_changed_modules.main:main", - "ci_build_python_checkers_reports = ci_build_python_static_checkers_reports.main:main", - ], - }, -) diff --git a/tools/ci_static_check_reports/unit_tests/__init__.py b/tools/ci_static_check_reports/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec6..0000000000000 --- a/tools/ci_static_check_reports/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/tools/ci_static_check_reports/unit_tests/test_build_static_checkers_reports.py b/tools/ci_static_check_reports/unit_tests/test_build_static_checkers_reports.py deleted file mode 100644 index 77b9437d4a727..0000000000000 --- a/tools/ci_static_check_reports/unit_tests/test_build_static_checkers_reports.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# -import os -import subprocess - -import pytest - - -@pytest.mark.parametrize( - "changed_module,should_build_reports", - [ - ('[{"dir": "tools/ci_static_check_reports", "lang": "py"}]', True), - ('[{"dir": "airbyte-integrations/connectors/destination-bigquery", "lang": "java"}]', False), - ('[{"dir": "airbyte-integrations/connectors/not-existing-module", "lang": "other"}]', False), - ], -) -def test_build_static_checkers_reports(changed_module: str, should_build_reports: bool) -> None: - subprocess.call(["ci_build_python_checkers_reports", changed_module], shell=False) - static_checker_reports_path = f"static_checker_reports/{changed_module}" - - static_checker_reports_path_exists = os.path.exists(static_checker_reports_path) - black_exists = os.path.exists(os.path.join(static_checker_reports_path, "black.txt")) - coverage_exists = os.path.exists(os.path.join(static_checker_reports_path, "coverage.xml")) - flake_exists = os.path.exists(os.path.join(static_checker_reports_path, "flake.xml")) - isort_exists = os.path.exists(os.path.join(static_checker_reports_path, "isort.txt")) - cobertura_exists = os.path.exists(os.path.join(static_checker_reports_path, "cobertura.xml")) - pytest_exists = os.path.exists(os.path.join(static_checker_reports_path, "pytest.xml")) - report_paths_exist = [ - static_checker_reports_path_exists, - black_exists, - coverage_exists, - flake_exists, - isort_exists, - cobertura_exists, - pytest_exists, - ] - - if should_build_reports: - assert all(report_paths_exist) - else: - assert not all(report_paths_exist) diff --git a/tools/ci_static_check_reports/unit_tests/test_detect_changed_modules.py b/tools/ci_static_check_reports/unit_tests/test_detect_changed_modules.py deleted file mode 100644 index 468e7dc21ac09..0000000000000 --- a/tools/ci_static_check_reports/unit_tests/test_detect_changed_modules.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# -from typing import List, Set - -import pytest -from ci_detect_changed_modules.main import list_changed_modules - - -@pytest.mark.parametrize( - "changed_files,changed_modules", - [ - (["path/to/file1", "file2.txt", "path/to/file3.txt"], []), - ( - [ - "airbyte-cdk/python/airbyte_cdk/entrypoint.py", - "airbyte-cdk/python/airbyte_cdk/file1", - "airbyte-cdk/python/airbyte_cdk/file2.py", - ], - [{"dir": "airbyte-cdk/python", "lang": "py"}], - ), - ( - [ - "airbyte-cdk/python/airbyte_cdk/entrypoint.py", - "airbyte-integrations/connectors/source-asana/source_asana/streams.py", - "airbyte-integrations/connectors/source-asana/source_asana/source.py", - "airbyte-integrations/connectors/source-braintree/integration_tests/abnormal_state.json", - ], - [{"dir": "airbyte-cdk/python", "lang": "py"}, {"dir": "airbyte-integrations/connectors/source-asana", "lang": "py"}], - ), - ( - [], - [], - ), - # TODO: update test after non-python modules are supported - ( - [ - "airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/main/" - "java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java" - ], - [], - ), - ( - ["airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json"], - [], - ), - ( - ["airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/destination.py"], - [ - {"dir": "airbyte-integrations/connectors/destination-amazon-sqs", "lang": "py"}, - ], - ), - ], -) -def test_list_changed_modules(changed_files: List[str], changed_modules: Set[str]) -> None: - calculated_changed_modules = list_changed_modules(changed_files) - - assert calculated_changed_modules == changed_modules