Skip to content

Commit 99875c4

Browse files
andriikorotkovandriikorotkov
authored
GCS Destination: MLP documentation (Beta) (#11499)
* updated gcs destination spec end documentation * fixed remarks * updated version * updated destination_specs yaml file
1 parent 2f850b9 commit 99875c4

File tree

5 files changed

+110
-91
lines changed

5 files changed

+110
-91
lines changed

‎airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@
8383
- name: Google Cloud Storage (GCS)
8484
destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a
8585
dockerRepository: airbyte/destination-gcs
86-
dockerImageTag: 0.2.0
86+
dockerImageTag: 0.2.1
8787
documentationUrl: https://docs.airbyte.io/integrations/destinations/gcs
8888
icon: googlecloudstorage.svg
8989
resourceRequirements:

‎airbyte-config/init/src/main/resources/seed/destination_specs.yaml

+60-44
Original file line numberDiff line numberDiff line change
@@ -1274,7 +1274,7 @@
12741274
- "overwrite"
12751275
- "append"
12761276
supportsNamespaces: true
1277-
- dockerImage: "airbyte/destination-gcs:0.2.0"
1277+
- dockerImage: "airbyte/destination-gcs:0.2.1"
12781278
spec:
12791279
documentationUrl: "https://docs.airbyte.io/integrations/destinations/gcs"
12801280
connectionSpecification:
@@ -1284,32 +1284,36 @@
12841284
required:
12851285
- "gcs_bucket_name"
12861286
- "gcs_bucket_path"
1287-
- "gcs_bucket_region"
12881287
- "credential"
12891288
- "format"
12901289
additionalProperties: false
12911290
properties:
12921291
gcs_bucket_name:
12931292
title: "GCS Bucket Name"
1293+
order: 1
12941294
type: "string"
1295-
description: "You can find the bucket name in the App Engine Admin console\
1296-
\ Application Settings page, under the label Google Cloud Storage Bucket."
1295+
description: "You can find the bucket name in the App Engine Admin console\
1296+
\ Application Settings page, under the label Google Cloud Storage Bucket.\
1297+
\ Read more <a href=\"https://cloud.google.com/storage/docs/naming-buckets\"\
1298+
>here</a>."
12971299
examples:
12981300
- "airbyte_sync"
12991301
gcs_bucket_path:
1302+
title: "GCS Bucket Path"
13001303
description: "GCS Bucket Path string Subdirectory under the above bucket\
13011304
\ to sync the data into."
1305+
order: 2
13021306
type: "string"
13031307
examples:
13041308
- "data_sync/test"
13051309
gcs_bucket_region:
1306-
title: "GCS Bucket Region"
1310+
title: "GCS Bucket Region (Optional)"
13071311
type: "string"
1308-
default: ""
1309-
description: "Select a Region of the GCS Bucket."
1312+
order: 3
1313+
default: "us"
1314+
description: "Select a Region of the GCS Bucket. Read more <a href=\"https://cloud.google.com/storage/docs/locations\"\
1315+
>here</a>."
13101316
enum:
1311-
- ""
1312-
- "-- North America --"
13131317
- "northamerica-northeast1"
13141318
- "northamerica-northeast2"
13151319
- "us-central1"
@@ -1319,18 +1323,15 @@
13191323
- "us-west2"
13201324
- "us-west3"
13211325
- "us-west4"
1322-
- "-- South America --"
13231326
- "southamerica-east1"
13241327
- "southamerica-west1"
1325-
- "-- Europe --"
13261328
- "europe-central2"
13271329
- "europe-north1"
13281330
- "europe-west1"
13291331
- "europe-west2"
13301332
- "europe-west3"
13311333
- "europe-west4"
13321334
- "europe-west6"
1333-
- "-- Asia --"
13341335
- "asia-east1"
13351336
- "asia-east2"
13361337
- "asia-northeast1"
@@ -1340,20 +1341,22 @@
13401341
- "asia-south2"
13411342
- "asia-southeast1"
13421343
- "asia-southeast2"
1343-
- "-- Australia --"
13441344
- "australia-southeast1"
13451345
- "australia-southeast2"
1346-
- "-- Multi-regions --"
13471346
- "asia"
13481347
- "eu"
13491348
- "us"
1350-
- "-- Dual-regions --"
13511349
- "asia1"
13521350
- "eur4"
13531351
- "nam4"
13541352
credential:
1355-
title: "Credential"
1353+
title: "Authentication"
1354+
description: "An HMAC key is a type of credential and can be associated\
1355+
\ with a service account or a user account in Cloud Storage. Read more\
1356+
\ <a href=\"https://cloud.google.com/storage/docs/authentication/hmackeys\"\
1357+
>here</a>."
13561358
type: "object"
1359+
order: 0
13571360
oneOf:
13581361
- title: "HMAC Key"
13591362
required:
@@ -1368,25 +1371,35 @@
13681371
default: "HMAC_KEY"
13691372
hmac_key_access_id:
13701373
type: "string"
1371-
description: "HMAC key access ID. When linked to a service account,\
1372-
\ this ID is 61 characters long; when linked to a user account,\
1373-
\ it is 24 characters long."
1374-
title: "HMAC Key Access ID"
1374+
description: "When linked to a service account, this ID is 61 characters\
1375+
\ long; when linked to a user account, it is 24 characters long.\
1376+
\ Read more <a href=\"https://cloud.google.com/storage/docs/authentication/hmackeys#overview\"\
1377+
>here</a>."
1378+
title: "Access ID"
13751379
airbyte_secret: true
1380+
order: 0
13761381
examples:
13771382
- "1234567890abcdefghij1234"
13781383
hmac_key_secret:
13791384
type: "string"
13801385
description: "The corresponding secret for the access ID. It is a\
1381-
\ 40-character base-64 encoded string."
1382-
title: "HMAC Key Secret"
1386+
\ 40-character base-64 encoded string. Read more <a href=\"https://cloud.google.com/storage/docs/authentication/hmackeys#secrets\"\
1387+
>here</a>."
1388+
title: "Secret"
13831389
airbyte_secret: true
1390+
order: 1
13841391
examples:
13851392
- "1234567890abcdefghij1234567890ABCDEFGHIJ"
13861393
format:
13871394
title: "Output Format"
13881395
type: "object"
1389-
description: "Output data format"
1396+
description: "Output data format. One of the following formats must be selected\
1397+
\ - <a href=\"https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-avro#advantages_of_avro\"\
1398+
>AVRO</a> format, <a href=\"https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet#parquet_schemas\"\
1399+
>PARQUET</a> format, <a href=\"https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-csv#loading_csv_data_into_a_table\"\
1400+
>CSV</a> format, or <a href=\"https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json#loading_json_data_into_a_new_table\"\
1401+
>JSONL</a> format."
1402+
order: 4
13901403
oneOf:
13911404
- title: "Avro: Apache Avro"
13921405
required:
@@ -1416,15 +1429,14 @@
14161429
- title: "Deflate"
14171430
required:
14181431
- "codec"
1419-
- "compression_level"
14201432
properties:
14211433
codec:
14221434
type: "string"
14231435
enum:
14241436
- "Deflate"
14251437
default: "Deflate"
14261438
compression_level:
1427-
title: "Deflate level"
1439+
title: "Deflate level (Optional)"
14281440
description: "0: no compression & fastest, 9: best compression\
14291441
\ & slowest."
14301442
type: "integer"
@@ -1443,16 +1455,22 @@
14431455
- title: "xz"
14441456
required:
14451457
- "codec"
1446-
- "compression_level"
14471458
properties:
14481459
codec:
14491460
type: "string"
14501461
enum:
14511462
- "xz"
14521463
default: "xz"
14531464
compression_level:
1454-
title: "Compression Level"
1455-
description: "See <a href=\"https://commons.apache.org/proper/commons-compress/apidocs/org/apache/commons/compress/compressors/xz/XZCompressorOutputStream.html#XZCompressorOutputStream-java.io.OutputStream-int-\"\
1465+
title: "Compression Level (Optional)"
1466+
description: "The presets 0-3 are fast presets with medium compression.\
1467+
\ The presets 4-6 are fairly slow presets with high compression.\
1468+
\ The default preset is 6. The presets 7-9 are like the preset\
1469+
\ 6 but use bigger dictionaries and have higher compressor\
1470+
\ and decompressor memory requirements. Unless the uncompressed\
1471+
\ size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is\
1472+
\ waste of memory to use the presets 7, 8, or 9, respectively.\
1473+
\ Read more <a href=\"https://commons.apache.org/proper/commons-compress/apidocs/org/apache/commons/compress/compressors/xz/XZCompressorOutputStream.html#XZCompressorOutputStream-java.io.OutputStream-int-\"\
14561474
>here</a> for details."
14571475
type: "integer"
14581476
default: 6
@@ -1461,15 +1479,14 @@
14611479
- title: "zstandard"
14621480
required:
14631481
- "codec"
1464-
- "compression_level"
14651482
properties:
14661483
codec:
14671484
type: "string"
14681485
enum:
14691486
- "zstandard"
14701487
default: "zstandard"
14711488
compression_level:
1472-
title: "Compression Level"
1489+
title: "Compression Level (Optional)"
14731490
description: "Negative levels are 'fast' modes akin to lz4 or\
14741491
\ snappy, levels above 9 are generally for archival purposes,\
14751492
\ and levels above 18 use a lot of memory."
@@ -1478,7 +1495,7 @@
14781495
minimum: -5
14791496
maximum: 22
14801497
include_checksum:
1481-
title: "Include Checksum"
1498+
title: "Include Checksum (Optional)"
14821499
description: "If true, include a checksum with each data block."
14831500
type: "boolean"
14841501
default: false
@@ -1492,7 +1509,7 @@
14921509
- "snappy"
14931510
default: "snappy"
14941511
part_size_mb:
1495-
title: "Block Size (MB) for GCS multipart upload"
1512+
title: "Block Size (MB) for GCS multipart upload (Optional)"
14961513
description: "This is the size of a \"Part\" being buffered in memory.\
14971514
\ It limits the memory usage when writing. Larger values will allow\
14981515
\ to upload a bigger files and improve the speed, but consumes9\
@@ -1504,7 +1521,6 @@
15041521
- title: "CSV: Comma-Separated Values"
15051522
required:
15061523
- "format_type"
1507-
- "flattening"
15081524
properties:
15091525
format_type:
15101526
type: "string"
@@ -1513,15 +1529,15 @@
15131529
default: "CSV"
15141530
flattening:
15151531
type: "string"
1516-
title: "Normalization (Flattening)"
1517-
description: "Whether the input json data should be normalized (flattened)\
1532+
title: "Normalization (Optional)"
1533+
description: "Whether the input JSON data should be normalized (flattened)\
15181534
\ in the output CSV. Please refer to docs for details."
15191535
default: "No flattening"
15201536
enum:
15211537
- "No flattening"
15221538
- "Root level flattening"
15231539
part_size_mb:
1524-
title: "Block Size (MB) for GCS multipart upload"
1540+
title: "Block Size (MB) for GCS multipart upload (Optional)"
15251541
description: "This is the size of a \"Part\" being buffered in memory.\
15261542
\ It limits the memory usage when writing. Larger values will allow\
15271543
\ to upload a bigger files and improve the speed, but consumes9\
@@ -1540,7 +1556,7 @@
15401556
- "JSONL"
15411557
default: "JSONL"
15421558
part_size_mb:
1543-
title: "Block Size (MB) for GCS multipart upload"
1559+
title: "Block Size (MB) for GCS multipart upload (Optional)"
15441560
description: "This is the size of a \"Part\" being buffered in memory.\
15451561
\ It limits the memory usage when writing. Larger values will allow\
15461562
\ to upload a bigger files and improve the speed, but consumes9\
@@ -1559,9 +1575,10 @@
15591575
- "Parquet"
15601576
default: "Parquet"
15611577
compression_codec:
1562-
title: "Compression Codec"
1578+
title: "Compression Codec (Optional)"
15631579
description: "The compression algorithm used to compress data pages."
15641580
type: "string"
1581+
default: "UNCOMPRESSED"
15651582
enum:
15661583
- "UNCOMPRESSED"
15671584
- "SNAPPY"
@@ -1570,9 +1587,8 @@
15701587
- "BROTLI"
15711588
- "LZ4"
15721589
- "ZSTD"
1573-
default: "UNCOMPRESSED"
15741590
block_size_mb:
1575-
title: "Block Size (Row Group Size) (MB)"
1591+
title: "Block Size (Row Group Size) (MB) (Optional)"
15761592
description: "This is the size of a row group being buffered in memory.\
15771593
\ It limits the memory usage when writing. Larger values will improve\
15781594
\ the IO when reading, but consume more memory when writing. Default:\
@@ -1582,15 +1598,15 @@
15821598
examples:
15831599
- 128
15841600
max_padding_size_mb:
1585-
title: "Max Padding Size (MB)"
1601+
title: "Max Padding Size (MB) (Optional)"
15861602
description: "Maximum size allowed as padding to align row groups.\
15871603
\ This is also the minimum size of a row group. Default: 8 MB."
15881604
type: "integer"
15891605
default: 8
15901606
examples:
15911607
- 8
15921608
page_size_kb:
1593-
title: "Page Size (KB)"
1609+
title: "Page Size (KB) (Optional)"
15941610
description: "The page size is for compression. A block is composed\
15951611
\ of pages. A page is the smallest unit that must be read fully\
15961612
\ to access a single record. If this value is too small, the compression\
@@ -1600,7 +1616,7 @@
16001616
examples:
16011617
- 1024
16021618
dictionary_page_size_kb:
1603-
title: "Dictionary Page Size (KB)"
1619+
title: "Dictionary Page Size (KB) (Optional)"
16041620
description: "There is one dictionary page per column per row group\
16051621
\ when dictionary encoding is used. The dictionary page size works\
16061622
\ like the page size but for dictionary. Default: 1024 KB."
@@ -1609,7 +1625,7 @@
16091625
examples:
16101626
- 1024
16111627
dictionary_encoding:
1612-
title: "Dictionary Encoding"
1628+
title: "Dictionary Encoding (Optional)"
16131629
description: "Default: true."
16141630
type: "boolean"
16151631
default: true

‎airbyte-integrations/connectors/destination-gcs/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION destination-gcs
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=0.2.0
19+
LABEL io.airbyte.version=0.2.1
2020
LABEL io.airbyte.name=airbyte/destination-gcs

0 commit comments

Comments
 (0)