Skip to content

Commit

Permalink
update desrcription for bq destination setup
Browse files Browse the repository at this point in the history
  • Loading branch information
yurii-bidiuk committed Jan 18, 2022
1 parent c4b365c commit ce12128
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 10 deletions.
20 changes: 13 additions & 7 deletions airbyte-config/init/src/main/resources/seed/destination_specs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,9 @@
big_query_client_buffer_size_mb:
title: "Google BigQuery client chunk size"
description: "Google BigQuery client's chunk(buffer) size (MIN=1, MAX =\
\ 15) for each table. The default 15MiB value is used if not set explicitly.\
\ 15) for each table. The size that will be written by a single RPC. Written data will be \
\ buffered and only flushed upon reaching this size or closing the channel. \
\ The default 15MiB value is used if not set explicitly. \
\ It's recommended to decrease value for big data sets migration for less\
\ HEAP memory consumption and avoiding crashes. For more details refer\
\ to https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html"
Expand Down Expand Up @@ -271,9 +273,11 @@
airbyte_secret: true
transformation_priority:
type: "string"
description: "When running custom transformations or Basic normalization,\
\ running queries on interactive mode can hit BQ limits, choosing batch\
\ will solve those limits."
description: "Interactive run type means that the query is executed as soon as possible, \
\ and these queries count towards concurrent rate limit and daily limit. \
\ Batch queries are queued and started as soon as idle resources are available \
\ in the BigQuery shared resource pool, which usually occurs within a few minutes. \
\ Batch queries don’t count towards your concurrent rate limit."
title: "Transformation Query Run Type"
default: "interactive"
enum:
Expand Down Expand Up @@ -393,9 +397,11 @@
big_query_client_buffer_size_mb:
title: "Google BigQuery client chunk size"
description: "Google BigQuery client's chunk (buffer) size (MIN = 1, MAX\
\ = 15) for each table. It defaults to 15MiB. Smaller chunk size means\
\ less memory consumption, and is recommended for big data sets. For more\
\ details refer to the documentation <a href=\"https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html\"\
\ = 15) for each table. The size that will be written by a single RPC. Written data will be \
\ buffered and only flushed upon reaching this size or closing the channel. \
\ It defaults to 15MiB. Smaller chunk size means less memory consumption, \
\ and is recommended for big data sets. For more details refer \
\ to the documentation <a href=\"https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html\"\
>here</a>"
type: "integer"
minimum: 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"properties": {
"big_query_client_buffer_size_mb": {
"title": "Google BigQuery client chunk size",
"description": "Google BigQuery client's chunk (buffer) size (MIN = 1, MAX = 15) for each table. It defaults to 15MiB. Smaller chunk size means less memory consumption, and is recommended for big data sets. For more details refer to the documentation <a href=\"https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html\">here</a>",
"description": "Google BigQuery client's chunk (buffer) size (MIN = 1, MAX = 15) for each table. The size that will be written by a single RPC. Written data will be buffered and only flushed upon reaching this size or closing the channel. It defaults to 15MiB. Smaller chunk size means less memory consumption, and is recommended for big data sets. For more details refer to the documentation <a href=\"https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html\">here</a>",
"type": "integer",
"minimum": 1,
"maximum": 15,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"properties": {
"big_query_client_buffer_size_mb": {
"title": "Google BigQuery client chunk size",
"description": "Google BigQuery client's chunk(buffer) size (MIN=1, MAX = 15) for each table. The default 15MiB value is used if not set explicitly. It's recommended to decrease value for big data sets migration for less HEAP memory consumption and avoiding crashes. For more details refer to https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html",
"description": "Google BigQuery client's chunk(buffer) size (MIN=1, MAX = 15) for each table. The size that will be written by a single RPC. Written data will be buffered and only flushed upon reaching this size or closing the channel. The default 15MiB value is used if not set explicitly. It's recommended to decrease value for big data sets migration for less HEAP memory consumption and avoiding crashes. For more details refer to https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html",
"type": "integer",
"minimum": 1,
"maximum": 15,
Expand Down Expand Up @@ -77,7 +77,7 @@
},
"transformation_priority": {
"type": "string",
"description": "When running custom transformations or Basic normalization, running queries on interactive mode can hit BQ limits, choosing batch will solve those limits.",
"description": "Interactive run type means that the query is executed as soon as possible, and these queries count towards concurrent rate limit and daily limit. Batch queries are queued and started as soon as idle resources are available in the BigQuery shared resource pool, which usually occurs within a few minutes. Batch queries don’t count towards your concurrent rate limit.",
"title": "Transformation Query Run Type",
"default": "interactive",
"enum": ["interactive", "batch"]
Expand Down

0 comments on commit ce12128

Please sign in to comment.