Skip to content

Commit 2e416f2

Browse files
rpopovrpopov
committed
Merge branch 'main' into 50395-2-2
2 parents 588ca53 + ca68c5c commit 2e416f2

File tree

75 files changed

+3334
-711
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+3334
-711
lines changed

‎.github/workflows/autofix-command.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ jobs:
7575
- name: Set up Poetry
7676
uses: Gr1N/setup-poetry@v9
7777
with:
78-
poetry-version: "1.8.4"
78+
poetry-version: "2.0.1"
7979
- name: Set up Python
8080
uses: actions/setup-python@v5
8181
with:

‎.github/workflows/connector-tests.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ jobs:
128128
- name: Set up Python
129129
uses: actions/setup-python@v5
130130
with:
131-
python-version: "3.10"
131+
python-version: "3.11"
132132
# Create initial pending status for test report
133133
- name: Create Pending Test Report Status
134134
if: steps.no_changes.outputs.status != 'cancelled'

‎.github/workflows/pdoc_preview.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ jobs:
1616
- name: Set up Poetry
1717
uses: Gr1N/setup-poetry@v9
1818
with:
19-
poetry-version: "1.8.4"
19+
poetry-version: "2.0.1"
2020
- name: Set up Python
2121
uses: actions/setup-python@v5
2222
with:

‎.github/workflows/pdoc_publish.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ jobs:
3333
- name: Set up Poetry
3434
uses: Gr1N/setup-poetry@v9
3535
with:
36-
poetry-version: "1.8.4"
36+
poetry-version: "2.0.1"
3737
- name: Set up Python
3838
uses: actions/setup-python@v5
3939
with:

‎.github/workflows/poetry-lock-command.yml

+12-1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,17 @@ jobs:
3636
with:
3737
pr: ${{ github.event.inputs.pr }}
3838

39+
- name: Check for blank or missing poetry.lock
40+
run: |
41+
if [ ! -s "poetry.lock" ]; then
42+
echo "poetry.lock missing or blank. Fetching from main branch..."
43+
git fetch origin main
44+
git checkout origin/main -- poetry.lock
45+
echo "Lock file restored from main."
46+
else
47+
echo "poetry.lock found. Proceeding."
48+
fi
49+
3950
- name: Get PR info
4051
id: pr-info
4152
run: |
@@ -72,7 +83,7 @@ jobs:
7283
- name: Set up Poetry
7384
uses: Gr1N/setup-poetry@v9
7485
with:
75-
poetry-version: "1.8.4"
86+
poetry-version: "2.0.1"
7687
- name: Set up Python
7788
uses: actions/setup-python@v5
7889
with:

‎.github/workflows/pytest_fast.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
- name: Set up Poetry
1818
uses: Gr1N/setup-poetry@v9
1919
with:
20-
poetry-version: "1.8.4"
20+
poetry-version: "2.0.1"
2121

2222
- name: Check Poetry lock file is current
2323
run: poetry check
@@ -44,7 +44,7 @@ jobs:
4444
- name: Set up Poetry
4545
uses: Gr1N/setup-poetry@v9
4646
with:
47-
poetry-version: "1.8.4"
47+
poetry-version: "2.0.1"
4848
- name: Set up Python
4949
uses: actions/setup-python@v5
5050
with:

‎.github/workflows/pytest_matrix.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030
python-version: [
3131
"3.10",
3232
"3.11",
33-
#'3.12', # Currently blocked by Pendulum
33+
# "3.12", # `requests-cache` blocker: https://github.com/airbytehq/airbyte-python-cdk/issues/299
3434
]
3535
os: [
3636
Ubuntu,
@@ -61,7 +61,7 @@ jobs:
6161
uses: Gr1N/setup-poetry@v9
6262
if: steps.changes.outputs.src == 'true'
6363
with:
64-
poetry-version: "1.8.4"
64+
poetry-version: "2.0.1"
6565
- name: Set up Python
6666
uses: actions/setup-python@v5
6767
if: steps.changes.outputs.src == 'true'

‎.github/workflows/python_lint.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
- name: Set up Poetry
1818
uses: Gr1N/setup-poetry@v9
1919
with:
20-
poetry-version: "1.8.4"
20+
poetry-version: "2.0.1"
2121
- name: Set up Python
2222
uses: actions/setup-python@v5
2323
with:

‎.github/workflows/test-command.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ jobs:
8585
- name: Set up Poetry
8686
uses: Gr1N/setup-poetry@v9
8787
with:
88-
poetry-version: "1.8.4"
88+
poetry-version: "2.0.1"
8989
- name: Set up Python
9090
uses: actions/setup-python@v5
9191
with:

‎.pre-commit-config.yaml

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# See https://pre-commit.com for more information
2+
# See https://pre-commit.com/hooks.html for more hooks
3+
exclude: |
4+
(?x)(
5+
# Python/system files
6+
^.*/__init__\.py$|
7+
^.*?/\.venv/.*$|
8+
^.*?/node_modules/.*$|
9+
10+
# Generated/test files
11+
^.*?/\.pytest_cache/.*$|
12+
^.*?/__pycache__/.*$|
13+
^.*?/\.mypy_cache/.*$|
14+
^.*?/\.ruff_cache/.*$
15+
16+
# Package management
17+
^.*?/poetry\.lock$|
18+
^.*?/package-lock\.json$|
19+
^.*?/pnpm-lock\.yaml$|
20+
21+
# Build and test artifacts
22+
^.*?/build/.*$|
23+
^.*?/dist/.*$|
24+
^.*?/\.coverage$|
25+
^.*?/coverage\.xml$|
26+
)
27+
28+
repos:
29+
- repo: https://github.com/pre-commit/pre-commit-hooks
30+
rev: v4.5.0
31+
hooks:
32+
- id: trailing-whitespace
33+
- id: end-of-file-fixer
34+
- id: check-yaml
35+
- id: check-added-large-files
36+
- id: check-toml
37+
38+
- repo: https://github.com/astral-sh/ruff-pre-commit
39+
rev: v0.8.3
40+
hooks:
41+
# Run the linter with repo-defined settings
42+
- id: ruff
43+
args: [--fix]
44+
45+
# Run the formatter with repo-defined settings
46+
- id: ruff-format
47+
48+
- repo: https://github.com/pre-commit/mirrors-prettier
49+
rev: v3.0.3
50+
hooks:
51+
- id: prettier
52+
types_or: [json, yaml]
53+
additional_dependencies:
54+
- prettier@3.0.3
55+
56+
- repo: local
57+
hooks:
58+
- id: addlicense
59+
name: Add license headers
60+
entry: addlicense -c "Airbyte, Inc." -l apache -v -f LICENSE_SHORT
61+
language: golang
62+
additional_dependencies: [github.com/google/addlicense@v1.1.1]
63+
files: \.py$

‎LICENSE_SHORT

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Copyright (c) 2025 Airbyte, Inc., all rights reserved.

‎airbyte_cdk/cli/source_declarative_manifest/_run.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import sys
2222
import traceback
2323
from collections.abc import Mapping
24-
from datetime import datetime
2524
from pathlib import Path
2625
from typing import Any, cast
2726

@@ -44,6 +43,7 @@
4443
)
4544
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
4645
from airbyte_cdk.sources.source import TState
46+
from airbyte_cdk.utils.datetime_helpers import ab_datetime_now
4747

4848

4949
class SourceLocalYaml(YamlDeclarativeSource):
@@ -101,7 +101,7 @@ def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
101101
type=Type.TRACE,
102102
trace=AirbyteTraceMessage(
103103
type=TraceType.ERROR,
104-
emitted_at=int(datetime.now().timestamp() * 1000),
104+
emitted_at=ab_datetime_now().to_epoch_millis(),
105105
error=AirbyteErrorTraceMessage(
106106
message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
107107
stack_trace=traceback.format_exc(),
@@ -191,7 +191,7 @@ def create_declarative_source(
191191
type=Type.TRACE,
192192
trace=AirbyteTraceMessage(
193193
type=TraceType.ERROR,
194-
emitted_at=int(datetime.now().timestamp() * 1000),
194+
emitted_at=ab_datetime_now().to_epoch_millis(),
195195
error=AirbyteErrorTraceMessage(
196196
message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
197197
stack_trace=traceback.format_exc(),

‎airbyte_cdk/connector_builder/connector_builder_handler.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
#
44

55
import dataclasses
6-
from datetime import datetime
76
from typing import Any, List, Mapping
87

98
from airbyte_cdk.connector_builder.message_grouper import MessageGrouper
@@ -21,6 +20,7 @@
2120
ModelToComponentFactory,
2221
)
2322
from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
23+
from airbyte_cdk.utils.datetime_helpers import ab_datetime_now
2424
from airbyte_cdk.utils.traced_exception import AirbyteTracedException
2525

2626
DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5
@@ -114,4 +114,4 @@ def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
114114

115115

116116
def _emitted_at() -> int:
117-
return int(datetime.now().timestamp()) * 1000
117+
return ab_datetime_now().to_epoch_millis()

‎airbyte_cdk/sources/declarative/async_job/job_orchestrator.py

+7-7
Original file line numberDiff line numberDiff line change
@@ -437,10 +437,10 @@ def create_and_get_completed_partitions(self) -> Iterable[AsyncPartition]:
437437
yield from self._process_running_partitions_and_yield_completed_ones()
438438
self._wait_on_status_update()
439439
except Exception as exception:
440+
LOGGER.warning(
441+
f"Caught exception that stops the processing of the jobs: {exception}. Traceback: {traceback.format_exc()}"
442+
)
440443
if self._is_breaking_exception(exception):
441-
LOGGER.warning(
442-
f"Caught exception that stops the processing of the jobs: {exception}"
443-
)
444444
self._abort_all_running_jobs()
445445
raise exception
446446

@@ -482,16 +482,16 @@ def _is_breaking_exception(self, exception: Exception) -> bool:
482482
and exception.failure_type == FailureType.config_error
483483
)
484484

485-
def fetch_records(self, partition: AsyncPartition) -> Iterable[Mapping[str, Any]]:
485+
def fetch_records(self, async_jobs: Iterable[AsyncJob]) -> Iterable[Mapping[str, Any]]:
486486
"""
487-
Fetches records from the given partition's jobs.
487+
Fetches records from the given jobs.
488488
489489
Args:
490-
partition (AsyncPartition): The partition containing the jobs.
490+
async_jobs Iterable[AsyncJob]: The list of AsyncJobs.
491491
492492
Yields:
493493
Iterable[Mapping[str, Any]]: The fetched records from the jobs.
494494
"""
495-
for job in partition.jobs:
495+
for job in async_jobs:
496496
yield from self._job_repository.fetch_records(job)
497497
self._job_repository.delete(job)

‎airbyte_cdk/sources/declarative/auth/jwt.py

+17-11
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#
44

55
import base64
6+
import json
67
from dataclasses import InitVar, dataclass
78
from datetime import datetime
89
from typing import Any, Mapping, Optional, Union
@@ -104,21 +105,21 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
104105
)
105106

106107
def _get_jwt_headers(self) -> dict[str, Any]:
107-
""" "
108+
"""
108109
Builds and returns the headers used when signing the JWT.
109110
"""
110-
headers = self._additional_jwt_headers.eval(self.config)
111+
headers = self._additional_jwt_headers.eval(self.config, json_loads=json.loads)
111112
if any(prop in headers for prop in ["kid", "alg", "typ", "cty"]):
112113
raise ValueError(
113114
"'kid', 'alg', 'typ', 'cty' are reserved headers and should not be set as part of 'additional_jwt_headers'"
114115
)
115116

116117
if self._kid:
117-
headers["kid"] = self._kid.eval(self.config)
118+
headers["kid"] = self._kid.eval(self.config, json_loads=json.loads)
118119
if self._typ:
119-
headers["typ"] = self._typ.eval(self.config)
120+
headers["typ"] = self._typ.eval(self.config, json_loads=json.loads)
120121
if self._cty:
121-
headers["cty"] = self._cty.eval(self.config)
122+
headers["cty"] = self._cty.eval(self.config, json_loads=json.loads)
122123
headers["alg"] = self._algorithm
123124
return headers
124125

@@ -130,18 +131,19 @@ def _get_jwt_payload(self) -> dict[str, Any]:
130131
exp = now + self._token_duration if isinstance(self._token_duration, int) else now
131132
nbf = now
132133

133-
payload = self._additional_jwt_payload.eval(self.config)
134+
payload = self._additional_jwt_payload.eval(self.config, json_loads=json.loads)
134135
if any(prop in payload for prop in ["iss", "sub", "aud", "iat", "exp", "nbf"]):
135136
raise ValueError(
136137
"'iss', 'sub', 'aud', 'iat', 'exp', 'nbf' are reserved properties and should not be set as part of 'additional_jwt_payload'"
137138
)
138139

139140
if self._iss:
140-
payload["iss"] = self._iss.eval(self.config)
141+
payload["iss"] = self._iss.eval(self.config, json_loads=json.loads)
141142
if self._sub:
142-
payload["sub"] = self._sub.eval(self.config)
143+
payload["sub"] = self._sub.eval(self.config, json_loads=json.loads)
143144
if self._aud:
144-
payload["aud"] = self._aud.eval(self.config)
145+
payload["aud"] = self._aud.eval(self.config, json_loads=json.loads)
146+
145147
payload["iat"] = now
146148
payload["exp"] = exp
147149
payload["nbf"] = nbf
@@ -151,7 +153,7 @@ def _get_secret_key(self) -> str:
151153
"""
152154
Returns the secret key used to sign the JWT.
153155
"""
154-
secret_key: str = self._secret_key.eval(self.config)
156+
secret_key: str = self._secret_key.eval(self.config, json_loads=json.loads)
155157
return (
156158
base64.b64encode(secret_key.encode()).decode()
157159
if self._base64_encode_secret_key
@@ -176,7 +178,11 @@ def _get_header_prefix(self) -> Union[str, None]:
176178
"""
177179
Returns the header prefix to be used when attaching the token to the request.
178180
"""
179-
return self._header_prefix.eval(self.config) if self._header_prefix else None
181+
return (
182+
self._header_prefix.eval(self.config, json_loads=json.loads)
183+
if self._header_prefix
184+
else None
185+
)
180186

181187
@property
182188
def auth_header(self) -> str:

0 commit comments

Comments
 (0)