Skip to content

Commit 1223795

Browse files
annalvova05annalvova05
committed
Merge remote-tracking branch 'origin/master' into alvova/7098-tiktok-oauth
# Conflicts: # airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java
2 parents b150d04 + b8cd772 commit 1223795

File tree

175 files changed

+3559
-1053
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

175 files changed

+3559
-1053
lines changed

‎.bumpversion.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[bumpversion]
2-
current_version = 0.35.5-alpha
2+
current_version = 0.35.6-alpha
33
commit = False
44
tag = False
55
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

‎.env

+5-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111

1212
### SHARED ###
13-
VERSION=0.35.5-alpha
13+
VERSION=0.35.6-alpha
1414

1515
# When using the airbyte-db via default docker image
1616
CONFIG_ROOT=/data
@@ -88,3 +88,7 @@ MAX_SYNC_WORKERS=5
8888
MAX_SPEC_WORKERS=5
8989
MAX_CHECK_WORKERS=5
9090
MAX_DISCOVER_WORKERS=5
91+
92+
93+
### FEATURE FLAGS ###
94+
NEW_SCHEDULER=false

‎.github/actions/ci-java-tests/action.yml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: "Runner CI Java Tests"
2+
description: "Runner CI Java Tests"
3+
inputs:
4+
module-name:
5+
required: true
6+
module-folder:
7+
required: true
8+
9+
runs:
10+
using: "composite"
11+
steps:
12+
- name: Install Java
13+
uses: actions/setup-java@v1
14+
with:
15+
java-version: '17'
16+
17+
- name: "Build"
18+
shell: bash
19+
run: |
20+
rm -rf ${{ inputs.module-folder }}/.venv ${{ inputs.module-folder }}/build
21+
ROOT_DIR=$(git rev-parse --show-toplevel)
22+
ARG=:$(python -c "import os; print(os.path.relpath('${{ inputs.module-folder }}', start='${ROOT_DIR}').replace('/', ':') )")
23+
echo "./gradlew --no-daemon $ARG:build"
24+
./gradlew --no-daemon "$ARG:clean"
25+
./gradlew --no-daemon "$ARG:build"
26+
27+
28+

‎.github/actions/ci-py-tests/action.yml

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: "Runner CI Python Tests"
2+
description: "Runner CI Python Tests"
3+
inputs:
4+
module-name:
5+
required: true
6+
module-folder:
7+
required: true
8+
outputs:
9+
coverage-paths:
10+
description: "Coverage Paths"
11+
value: ${{ steps.build-coverage-reports.outputs.coverage-paths }}
12+
flake8-logs:
13+
description: "Flake8 Logs"
14+
value: ${{ steps.build-linter-reports.outputs.flake8-logs }}
15+
mypy-logs:
16+
description: "MyPy Logs"
17+
value: ${{ steps.build-linter-reports.outputs.mypy-logs }}
18+
black-diff:
19+
description: "Black Diff"
20+
value: ${{ steps.build-linter-reports.outputs.black-diff }}
21+
isort-diff:
22+
description: "Isort Diff"
23+
value: ${{ steps.build-linter-reports.outputs.isort-diff }}
24+
runs:
25+
using: "composite"
26+
steps:
27+
- name: Build Coverage Reports
28+
id: build-coverage-reports
29+
shell: bash
30+
working-directory: ${{ inputs.module-folder }}
31+
run: |
32+
virtualenv .venv
33+
source .venv/bin/activate
34+
JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
35+
pip install coverage[toml]~=6.2
36+
mkdir -p .venv/source-acceptance-test
37+
mkdir -p reports
38+
SAT_DIR=$(git rev-parse --show-toplevel)/airbyte-integrations/bases/source-acceptance-test
39+
PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
40+
git ls-tree -r HEAD --name-only $SAT_DIR | while read src; do cp -f $src .venv/source-acceptance-test; done
41+
pip install build
42+
python -m build .venv/source-acceptance-test
43+
pip install .venv/source-acceptance-test/dist/source_acceptance_test-*.whl
44+
[ -f requirements.txt ] && pip install --quiet -r requirements.txt
45+
pip install .[tests]
46+
coverage run --rcfile=${PYPROJECT_CONFIG} -m pytest ./unit_tests || true
47+
coverage xml --rcfile=${PYPROJECT_CONFIG} -o reports/coverage.xml || true
48+
49+
rm -rf .venv
50+
echo "::set-output name=coverage-paths::reports/coverage.xml"
51+
52+
- name: Build Linter Reports
53+
id: build-linter-reports
54+
shell: bash
55+
working-directory: ${{ inputs.module-folder }}
56+
run: |
57+
JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
58+
REPORT_FOLDER=reports
59+
PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
60+
61+
# run mypy
62+
pip install lxml~=4.7 mypy~=0.910 .
63+
mypy . --config-file=${PYPROJECT_CONFIG} | tee reports/mypy.log || true
64+
65+
# run black
66+
pip install black~=21.12b0
67+
XDG_CACHE_HOME=/dev/null black --config ${PYPROJECT_CONFIG} --diff . | tee reports/black.diff
68+
69+
# run isort
70+
pip install isort~=5.10.1
71+
cp ${PYPROJECT_CONFIG} ./pyproject.toml
72+
isort --diff . | tee reports/isort.diff
73+
74+
# run flake8
75+
pip install mccabe~=0.6.1 pyproject-flake8~=0.0.1a2
76+
pflake8 --exit-zero . | grep ^. | tee reports/flake.txt
77+
78+
echo "::set-output name=mypy-logs::reports/mypy.log"
79+
echo "::set-output name=black-diff::reports/black.diff"
80+
echo "::set-output name=isort-diff::reports/isort.diff"
81+
echo "::set-output name=flake8-logs::reports/flake.txt"

‎.github/actions/ci-tests-runner/action.yml

+187
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
name: "Setup CI Tests Env"
2+
description: "Setup CI Tests Env for all module types"
3+
inputs:
4+
module-name:
5+
description: "Unique module name. e.g.: connectors/source-s3, connectors/destination-s3"
6+
required: true
7+
module-folder:
8+
description: "Path to module folder"
9+
required: true
10+
module-lang:
11+
description: "Detected module language. Available values: py, java"
12+
required: true
13+
sonar-gcp-access-key:
14+
required: true
15+
sonar-token:
16+
description: "Access token for using SonarQube API"
17+
required: true
18+
pull-request-id:
19+
description: "Unique PR ID. For example: airbyte/1234"
20+
default: "0"
21+
token:
22+
required: true
23+
remove-sonar-project:
24+
description: "This flag should be used if needed to remove sonar project after using"
25+
default: false
26+
runs:
27+
using: "composite"
28+
steps:
29+
30+
- name: Set up Python
31+
uses: actions/setup-python@v2
32+
with:
33+
python-version: 3.7
34+
35+
- name: Tests of CI
36+
shell: bash
37+
run: |
38+
# all CI python packages have the prefix "ci_"
39+
pip install --quiet tox==3.24.4
40+
tox -r -c ./tools/tox_ci.ini
41+
pip install --quiet -e ./tools/ci_*
42+
echo "::echo::off"
43+
44+
- name: Auth with gcloud CLI
45+
uses: google-github-actions/setup-gcloud@v0
46+
with:
47+
service_account_key: ${{ inputs.sonar-gcp-access-key }}
48+
project_id: dataline-integration-testing
49+
export_default_credentials: true
50+
51+
- name: Create IAP tunnel
52+
id: gcloud-tunnel
53+
shell: bash
54+
run: |
55+
while true; do
56+
PORT=$(( ((RANDOM<<15)|RANDOM) % 49152 + 10000 ))
57+
status="$(nc -z 127.0.0.1 $PORT < /dev/null &>/dev/null; echo $?)"
58+
if [ "${status}" != "0" ]; then
59+
echo "$PORT is free to use";
60+
break;
61+
fi
62+
done
63+
IPS=($(hostname -I))
64+
LOCAL_IP_PORT="${IPS[0]}:${PORT}"
65+
gcloud compute start-iap-tunnel sonarqube-1-vm 80 --local-host-port=${LOCAL_IP_PORT} --zone=europe-central2-a --project dataline-integration-testing &
66+
echo ::set-output name=pid::$!
67+
echo "::set-output name=sonar-host::http://${LOCAL_IP_PORT}/"
68+
echo "::echo::on"
69+
70+
- name: Python Tests
71+
id: ci-py-tests
72+
if: ${{ inputs.module-lang == 'py' }}
73+
uses: ./.github/actions/ci-py-tests
74+
with:
75+
module-name: ${{ inputs.module-name }}
76+
module-folder: ${{ inputs.module-folder }}
77+
78+
- name: Java Tests
79+
id: ci-java-tests
80+
if: ${{ inputs.module-lang == 'java' }}
81+
uses: ./.github/actions/ci-java-tests
82+
with:
83+
module-name: ${{ inputs.module-name }}
84+
module-folder: ${{ inputs.module-folder }}
85+
86+
87+
88+
89+
90+
- name: Prepare SQ Options
91+
shell: bash
92+
id: sq-options
93+
working-directory: ${{ inputs.module-folder }}
94+
run: |
95+
REPORT_FOLDER=reports
96+
mkdir -p ${REPORT_FOLDER}
97+
declare -a REPORT_FILES
98+
declare -a OPTIONS
99+
if [ ${{ inputs.module-lang }} == 'py' ]; then
100+
[ -f ${{ steps.ci-py-tests.outputs.mypy-logs }} ] && ci_sonar_qube --mypy_log ${{ steps.ci-py-tests.outputs.mypy-logs }} --output_file ${REPORT_FOLDER}/issues_mypy.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
101+
[ -f ${{ steps.ci-py-tests.outputs.mypy-logs }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_mypy.json)
102+
103+
[ -f ${{ steps.ci-py-tests.outputs.black-diff }} ] && ci_sonar_qube --black_diff ${{ steps.ci-py-tests.outputs.black-diff }} --output_file ${REPORT_FOLDER}/issues_black.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
104+
[ -f ${{ steps.ci-py-tests.outputs.black-diff }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_black.json)
105+
106+
[ -f ${{ steps.ci-py-tests.outputs.isort-diff }} ] && ci_sonar_qube --isort_diff ${{ steps.ci-py-tests.outputs.isort-diff }} --output_file ${REPORT_FOLDER}/issues_isort.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
107+
[ -f ${{ steps.ci-py-tests.outputs.isort-diff }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_isort.json)
108+
109+
[ -f ${{ steps.ci-py-tests.outputs.coverage-paths }} ] && OPTIONS+=("-Dsonar.python.coverage.reportPaths=${{ steps.ci-py-tests.outputs.coverage-paths }}")
110+
[ -f ${{ steps.ci-py-tests.outputs.flake8-logs }} ] && OPTIONS+=("-Dsonar.python.flake8.reportPaths=${{ steps.ci-py-tests.outputs.flake8-logs }}")
111+
fi
112+
if [ ${{ inputs.module-lang }} == 'java' ]; then
113+
[ -d "./src/main/java" ] && OPTIONS+=("-Dsonar.sources=./src/main/java")
114+
[ -d "./src/test/java" ] && OPTIONS+=("-Dsonar.tests=./src/test/java")
115+
[ -d "./build/test-results" ] && OPTIONS+=("-Dsonar.junit.reportsPath=./build/test-results")
116+
[ -f "./build/jacoco/test.exec" ] && OPTIONS+=("-Dsonar.jacoco.reportPaths=./build/jacoco/test.exec")
117+
[ -d "./build/classes/java/main" ] && OPTIONS+=("-Dsonar.java.binaries=./build/classes/java/main")
118+
[ -d "./build/classes/java/test" ] && OPTIONS+=("-Dsonar.test.binaries=./build/classes/java/test")
119+
120+
fi
121+
122+
# join the array to string format
123+
echo ::set-output name=external_reports::$(IFS=, ; echo "${REPORT_FILES[*]}")
124+
echo ::set-output name=options::$(IFS=' ' ; echo "${OPTIONS[*]}")
125+
126+
- name: Create SonarQube Project
127+
shell: bash
128+
id: create-sq-project
129+
run: |
130+
ci_sonar_qube --pr ${{ inputs.pull-request-id }} --create --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
131+
echo "::set-output name=sq_project_name::$(ci_sonar_qube --pr ${{ inputs.pull-request-id }} --print_key --module ${{ inputs.module-name }})"
132+
ROOT_DIR=$(git rev-parse --show-toplevel)
133+
MODULE_DIR=$(python -c "print('${{ inputs.module-folder }}'.replace('${ROOT_DIR}', '.'))")
134+
echo "::set-output name=module_dir::${MODULE_DIR}"
135+
136+
- name: SonarQube Scan
137+
138+
uses: sonarsource/sonarqube-scan-action@master
139+
env:
140+
SONAR_TOKEN: ${{ inputs.sonar-token }}
141+
SONAR_HOST_URL: ${{ steps.gcloud-tunnel.outputs.sonar-host }}
142+
with:
143+
projectBaseDir: ${{ steps.create-sq-project.outputs.module_dir }}
144+
args: >
145+
-Dsonar.projectKey=${{ steps.create-sq-project.outputs.sq_project_name }}
146+
-Dsonar.verbose=true
147+
-Dsonar.working.directory=/tmp/scannerwork
148+
-Dsonar.language=${{ inputs.module-lang }}
149+
-Dsonar.sourceEncoding=UTF-8
150+
-Dsonar.projectBaseDir=${{ steps.create-sq-project.outputs.module_dir }}
151+
-Dsonar.exclusions=reports/**,*.toml
152+
-Dsonar.externalIssuesReportPaths=${{ steps.sq-options.outputs.external_reports }}
153+
${{ steps.sq-options.outputs.options }}
154+
155+
- name: Generate SonarQube Report
156+
shell: bash
157+
id: generate-sq-report
158+
run: |
159+
# delay because SQ needs time for processing of all input data
160+
sleep 10
161+
REPORT_FILE=/tmp/sq_report_$RANDOM.md
162+
ci_sonar_qube --pr ${{ inputs.pull-request-id }} --report ${REPORT_FILE} --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
163+
body="$(cat ${REPORT_FILE})"
164+
body="${body//'%'/'%25'}"
165+
body="${body//$'\n'/'%0A'}"
166+
body="${body//$'\r'/'%0D'}"
167+
echo "::set-output name=sq-report::$body"
168+
169+
- name: Add Comment
170+
if: ${{ github.event_name == 'pull_request' }}
171+
uses: peter-evans/commit-comment@v1
172+
with:
173+
body: ${{ steps.generate-sq-report.outputs.sq-report }}
174+
token: ${{ inputs.token }}
175+
176+
- name: Remove SonarQube Project
177+
if: ${{ inputs.remove-sonar-project == true }}
178+
shell: bash
179+
id: remove-sq-project
180+
run: |
181+
ci_sonar_qube --pr ${{ inputs.pull-request-id }} --remove --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }}
182+
183+
- name: Remove IAP tunnel
184+
if: always()
185+
shell: bash
186+
run: |
187+
kill ${{ steps.gcloud-tunnel.outputs.pid }}

‎.github/workflows/detect-changed-modules-and-build-reports.yml

-42
This file was deleted.

0 commit comments

Comments
 (0)