🔧 Pinterest Schema updated

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.35.5-alpha
+current_version = 0.35.14-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.35.5-alpha
+VERSION=0.35.14-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data
@@ -50,7 +50,6 @@ CONFIG_DATABASE_PASSWORD=
 CONFIG_DATABASE_URL=
 CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.1.001
 
-
 ### AIRBYTE SERVICES ###
 TEMPORAL_HOST=airbyte-temporal:7233
 INTERNAL_API_HOST=airbyte-server:8001
@@ -88,3 +87,7 @@ MAX_SYNC_WORKERS=5
 MAX_SPEC_WORKERS=5
 MAX_CHECK_WORKERS=5
 MAX_DISCOVER_WORKERS=5
+
+
+### FEATURE FLAGS ###
+NEW_SCHEDULER=false

.env.dev

@@ -27,4 +27,3 @@ SYNC_JOB_MAX_TIMEOUT_DAYS=3
 
 # Sentry
 SENTRY_DSN=""
-

.github/actions/ci-java-tests/action.yml

@@ -0,0 +1,28 @@
+name: "Runner CI Java Tests"
+description: "Runner CI Java Tests"
+inputs:
+  module-name:
+    required: true
+  module-folder:
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Java
+      uses: actions/setup-java@v1
+      with:
+        java-version: '17'
+
+    - name: "Build"
+      shell: bash
+      run: |
+        rm -rf ${{ inputs.module-folder }}/.venv ${{ inputs.module-folder }}/build
+        ROOT_DIR=$(git rev-parse --show-toplevel)
+        ARG=:$(python -c "import os; print(os.path.relpath('${{ inputs.module-folder }}', start='${ROOT_DIR}').replace('/', ':') )")
+        echo "./gradlew --no-daemon $ARG:build"
+        ./gradlew --no-daemon "$ARG:clean"
+        ./gradlew --no-daemon "$ARG:build"
+
+
+

.github/actions/ci-py-tests/action.yml

@@ -0,0 +1,85 @@
+name: "Runner CI Python Tests"
+description: "Runner CI Python Tests"
+inputs:
+  module-name:
+    required: true
+  module-folder:
+    required: true
+outputs:
+  coverage-paths:
+    description: "Coverage Paths"
+    value: ${{ steps.build-coverage-reports.outputs.coverage-paths }}
+  flake8-logs:
+    description: "Flake8 Logs"
+    value: ${{ steps.build-linter-reports.outputs.flake8-logs }}
+  mypy-logs:
+    description: "MyPy Logs"
+    value: ${{ steps.build-linter-reports.outputs.mypy-logs }}
+  black-diff:
+    description: "Black Diff"
+    value: ${{ steps.build-linter-reports.outputs.black-diff }}
+  isort-diff:
+    description: "Isort Diff"
+    value: ${{ steps.build-linter-reports.outputs.isort-diff }}
+runs:
+  using: "composite"
+  steps:
+    - name: Build Coverage Reports
+      id: build-coverage-reports
+      shell: bash
+      working-directory: ${{ inputs.module-folder }}
+      run: |
+        virtualenv .venv
+        source .venv/bin/activate
+        JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
+        pip install coverage[toml]~=6.2
+        mkdir -p .venv/source-acceptance-test
+        mkdir -p reports
+        SAT_DIR=$(git rev-parse --show-toplevel)/airbyte-integrations/bases/source-acceptance-test
+        PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
+        git ls-tree -r HEAD --name-only $SAT_DIR | while read src; do cp -f $src .venv/source-acceptance-test; done
+        pip install build
+        python -m build .venv/source-acceptance-test
+        pip install .venv/source-acceptance-test/dist/source_acceptance_test-*.whl
+        [ -f requirements.txt ] && pip install --quiet -r requirements.txt
+        pip install .[tests]
+        coverage run --rcfile=${PYPROJECT_CONFIG} -m pytest ./unit_tests || true
+        coverage xml --rcfile=${PYPROJECT_CONFIG} -o reports/coverage.xml || true
+
+        rm -rf .venv
+        echo "::set-output name=coverage-paths::reports/coverage.xml"
+
+    - name: Upload coverage to Codecov
+      if: ${{ always() }}
+      uses: codecov/codecov-action@v2
+
+    - name: Build Linter Reports
+      id: build-linter-reports
+      shell: bash
+      working-directory: ${{ inputs.module-folder }}
+      run: |
+        JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
+        REPORT_FOLDER=reports
+        PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
+
+        # run mypy
+        pip install lxml~=4.7 mypy~=0.910 .
+        mypy . --config-file=${PYPROJECT_CONFIG} | tee reports/mypy.log || true
+
+        # run black
+        pip install black~=21.12b0
+        XDG_CACHE_HOME=/dev/null black --config ${PYPROJECT_CONFIG} --diff . | tee reports/black.diff
+
+        # run isort
+        pip install isort~=5.10.1
+        cp ${PYPROJECT_CONFIG} ./pyproject.toml
+        isort --diff . | tee reports/isort.diff
+
+        # run flake8
+        pip install mccabe~=0.6.1 pyproject-flake8~=0.0.1a2
+        pflake8 --exit-zero . | grep ^. | tee reports/flake.txt
+
+        echo "::set-output name=mypy-logs::reports/mypy.log"
+        echo "::set-output name=black-diff::reports/black.diff"
+        echo "::set-output name=isort-diff::reports/isort.diff"
+        echo "::set-output name=flake8-logs::reports/flake.txt"

.github/actions/ci-tests-runner/action.yml

@@ -0,0 +1,187 @@
+name: "Setup CI Tests Env"
+description: "Setup CI Tests Env for all module types"
+inputs:
+  module-name:
+    description: "Unique module name. e.g.: connectors/source-s3, connectors/destination-s3"
+    required: true
+  module-folder:
+    description: "Path to module folder"
+    required: true
+  module-lang:
+    description: "Detected module language. Available values: py, java"
+    required: true
+  sonar-gcp-access-key:
+    required: true
+  sonar-token:
+    description: "Access token for using SonarQube API"
+    required: true
+  pull-request-id:
+    description: "Unique PR ID. For example: airbyte/1234"
+    default: "0"
+  token:
+    required: true
+  remove-sonar-project:
+    description: "This flag should be used if needed to remove sonar project after using"
+    default: false
+runs:
+  using: "composite"
+  steps:
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+
+    - name: Tests of CI
+      shell: bash
+      run: |
+        # all CI python packages have the prefix "ci_"
+        pip install --quiet tox==3.24.4
+        tox -r -c ./tools/tox_ci.ini
+        pip install --quiet -e ./tools/ci_*
+        echo "::echo::off"
+
+    - name: Auth with gcloud CLI
+      uses: google-github-actions/setup-gcloud@v0
+      with:
+        service_account_key: ${{ inputs.sonar-gcp-access-key }}
+        project_id: dataline-integration-testing
+        export_default_credentials: true
+
+    - name: Create IAP tunnel
+      id: gcloud-tunnel
+      shell: bash
+      run: |
+          while true; do
+            PORT=$(( ((RANDOM<<15)|RANDOM) % 49152 + 10000 ))
+            status="$(nc -z 127.0.0.1 $PORT < /dev/null &>/dev/null; echo $?)"
+            if [ "${status}" != "0" ]; then
+                echo "$PORT is free to use";
+                break;
+            fi
+          done
+          IPS=($(hostname -I))
+          LOCAL_IP_PORT="${IPS[0]}:${PORT}"
+          gcloud compute start-iap-tunnel sonarqube-1-vm 80 --local-host-port=${LOCAL_IP_PORT} --zone=europe-central2-a --project dataline-integration-testing &
+          echo ::set-output name=pid::$!
+          echo "::set-output name=sonar-host::http://${LOCAL_IP_PORT}/"
+          echo "::echo::on"
+
+    - name: Python Tests
+      id: ci-py-tests
+      if: ${{ inputs.module-lang == 'py' }}
+      uses: ./.github/actions/ci-py-tests
+      with:
+        module-name: ${{ inputs.module-name }}
+        module-folder: ${{ inputs.module-folder }}
+
+    - name: Java Tests
+      id: ci-java-tests
+      if: ${{ inputs.module-lang == 'java' }}
+      uses: ./.github/actions/ci-java-tests
+      with:
+        module-name: ${{ inputs.module-name }}
+        module-folder: ${{ inputs.module-folder }}
+
+
+
+
+
+    - name: Prepare SQ Options
+      shell: bash
+      id: sq-options
+      working-directory: ${{ inputs.module-folder }}
+      run: |
+        REPORT_FOLDER=reports
+        mkdir -p ${REPORT_FOLDER}
+        declare -a REPORT_FILES
+        declare -a OPTIONS
+        if [ ${{ inputs.module-lang }} == 'py' ]; then
+          [ -f ${{ steps.ci-py-tests.outputs.mypy-logs }} ] && ci_sonar_qube --mypy_log ${{ steps.ci-py-tests.outputs.mypy-logs }} --output_file ${REPORT_FOLDER}/issues_mypy.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
+          [ -f ${{ steps.ci-py-tests.outputs.mypy-logs }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_mypy.json)
+
+          [ -f ${{ steps.ci-py-tests.outputs.black-diff }} ] && ci_sonar_qube --black_diff ${{ steps.ci-py-tests.outputs.black-diff }} --output_file ${REPORT_FOLDER}/issues_black.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
+          [ -f ${{ steps.ci-py-tests.outputs.black-diff }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_black.json)
+
+          [ -f ${{ steps.ci-py-tests.outputs.isort-diff }} ] && ci_sonar_qube --isort_diff ${{ steps.ci-py-tests.outputs.isort-diff }} --output_file ${REPORT_FOLDER}/issues_isort.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
+          [ -f ${{ steps.ci-py-tests.outputs.isort-diff }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_isort.json)
+
+          [ -f ${{ steps.ci-py-tests.outputs.coverage-paths }} ] && OPTIONS+=("-Dsonar.python.coverage.reportPaths=${{ steps.ci-py-tests.outputs.coverage-paths }}")
+          [ -f ${{ steps.ci-py-tests.outputs.flake8-logs }} ] && OPTIONS+=("-Dsonar.python.flake8.reportPaths=${{ steps.ci-py-tests.outputs.flake8-logs }}")
+        fi
+        if [ ${{ inputs.module-lang }} == 'java' ]; then
+             [ -d "./src/main/java" ] && OPTIONS+=("-Dsonar.sources=./src/main/java")
+             [ -d "./src/test/java" ] && OPTIONS+=("-Dsonar.tests=./src/test/java")
+             [ -d "./build/test-results" ] && OPTIONS+=("-Dsonar.junit.reportsPath=./build/test-results")
+             [ -f "./build/jacoco/test.exec" ] && OPTIONS+=("-Dsonar.jacoco.reportPaths=./build/jacoco/test.exec")
+             [ -d "./build/classes/java/main" ] && OPTIONS+=("-Dsonar.java.binaries=./build/classes/java/main")
+             [ -d "./build/classes/java/test" ] && OPTIONS+=("-Dsonar.test.binaries=./build/classes/java/test")
+
+        fi
+
+        # join the array to string format
+        echo ::set-output name=external_reports::$(IFS=, ; echo "${REPORT_FILES[*]}")
+        echo ::set-output name=options::$(IFS=' ' ; echo "${OPTIONS[*]}")
+
+    - name: Create SonarQube Project
+      shell: bash
+      id: create-sq-project
+      run: |
+        ci_sonar_qube --pr ${{ inputs.pull-request-id }} --create --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
+        echo "::set-output name=sq_project_name::$(ci_sonar_qube --pr ${{ inputs.pull-request-id }} --print_key --module ${{ inputs.module-name }})"
+        ROOT_DIR=$(git rev-parse --show-toplevel)
+        MODULE_DIR=$(python -c "print('${{ inputs.module-folder }}'.replace('${ROOT_DIR}', '.'))")
+        echo "::set-output name=module_dir::${MODULE_DIR}"
+
+    - name: SonarQube Scan
+
+      uses: sonarsource/sonarqube-scan-action@master
+      env:
+        SONAR_TOKEN: ${{ inputs.sonar-token }}
+        SONAR_HOST_URL: ${{ steps.gcloud-tunnel.outputs.sonar-host }}
+      with:
+          projectBaseDir: ${{ steps.create-sq-project.outputs.module_dir }}
+          args: >
+            -Dsonar.projectKey=${{ steps.create-sq-project.outputs.sq_project_name }}
+            -Dsonar.verbose=true
+            -Dsonar.working.directory=/tmp/scannerwork
+            -Dsonar.language=${{ inputs.module-lang }}
+            -Dsonar.sourceEncoding=UTF-8
+            -Dsonar.projectBaseDir=${{ steps.create-sq-project.outputs.module_dir }}
+            -Dsonar.exclusions=reports/**,*.toml
+            -Dsonar.externalIssuesReportPaths=${{ steps.sq-options.outputs.external_reports }}
+            ${{ steps.sq-options.outputs.options }}
+
+    - name: Generate SonarQube Report
+      shell: bash
+      id: generate-sq-report
+      run: |
+        # delay because SQ needs time for processing of all input data
+        sleep 10
+        REPORT_FILE=/tmp/sq_report_$RANDOM.md
+        ci_sonar_qube --pr ${{ inputs.pull-request-id }} --report ${REPORT_FILE} --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
+        body="$(cat ${REPORT_FILE})"
+        body="${body//'%'/'%25'}"
+        body="${body//$'\n'/'%0A'}"
+        body="${body//$'\r'/'%0D'}"
+        echo "::set-output name=sq-report::$body"
+
+    - name: Add Comment
+      if: ${{ github.event_name == 'pull_request' }}
+      uses: peter-evans/commit-comment@v1
+      with:
+        body: ${{ steps.generate-sq-report.outputs.sq-report }}
+        token: ${{ inputs.token }}
+
+    - name: Remove SonarQube Project
+      if: ${{ inputs.remove-sonar-project == true }}
+      shell: bash
+      id: remove-sq-project
+      run: |
+        ci_sonar_qube --pr ${{ inputs.pull-request-id }} --remove --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
+
+    - name: Remove IAP tunnel
+      if: always()
+      shell: bash
+      run: |
+        kill ${{ steps.gcloud-tunnel.outputs.pid }}