diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json index 07e795e2e3889..e4137456ad996 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json @@ -2,7 +2,7 @@ "destinationDefinitionId": "4816b78f-1489-44c1-9060-4b19d5fa9362", "name": "S3", "dockerRepository": "airbyte/destination-s3", - "dockerImageTag": "0.2.4", + "dockerImageTag": "0.2.5", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/s3", "icon": "s3.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 2b5cad4c78691..1e561ba8289d8 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -167,7 +167,7 @@ - name: S3 destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 dockerRepository: airbyte/destination-s3 - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.5 documentationUrl: https://docs.airbyte.io/integrations/destinations/s3 icon: s3.svg - name: SFTP-JSON diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 06520bfa6bcd0..9276dac6c07b6 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3408,7 +3408,7 @@ supported_destination_sync_modes: - "append" - "overwrite" -- dockerImage: "airbyte/destination-s3:0.2.4" +- dockerImage: "airbyte/destination-s3:0.2.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/s3" connectionSpecification: @@ -3419,8 +3419,6 @@ - "s3_bucket_name" - "s3_bucket_path" - "s3_bucket_region" - - "access_key_id" - - "secret_access_key" - "format" additionalProperties: false properties: @@ -3478,14 +3476,16 @@ access_key_id: type: "string" description: "The access key id to access the S3 bucket. Airbyte requires\ - \ Read and Write permissions to the given bucket." + \ Read and Write permissions to the given bucket, if not set, Airbyte\ + \ will rely on Instance Profile." title: "S3 Key Id" airbyte_secret: true examples: - "A012345678910EXAMPLE" secret_access_key: type: "string" - description: "The corresponding secret to the access key id." + description: "The corresponding secret to the access key id, if S3 Key Id\ + \ is set, then S3 Access Key must also be provided" title: "S3 Access Key" airbyte_secret: true examples: diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index 7334a212258e6..7850e4f00e2a7 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-s3 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-s3/README.md b/airbyte-integrations/connectors/destination-s3/README.md index e163606e68a71..a5b7f9bb7c4cc 100644 --- a/airbyte-integrations/connectors/destination-s3/README.md +++ b/airbyte-integrations/connectors/destination-s3/README.md @@ -8,6 +8,7 @@ As a community contributor, you will need access to AWS to run the integration t - Create an S3 bucket for testing. - Get your `access_key_id` and `secret_access_key` that can read and write to the above bucket. +- if you leave `access_key_id` and `secret_access_key` in blank, the authentication will rely on the instance profile authentication - Paste the bucket and key information into the config files under [`./sample_secrets`](./sample_secrets). - Rename the directory from `sample_secrets` to `secrets`. - Feel free to modify the config files with different settings in the acceptance test file (e.g. `S3CsvDestinationAcceptanceTest.java`, method `getFormatConfig`), as long as they follow the schema defined in [spec.json](src/main/resources/spec.json). diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java index 3aea3ceceed3a..79ea47bee1853 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.s3; +import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; @@ -87,8 +88,8 @@ public static S3DestinationConfig getS3DestinationConfig(final JsonNode config) config.get("s3_bucket_name").asText(), bucketPath, config.get("s3_bucket_region").asText(), - config.get("access_key_id").asText(), - config.get("secret_access_key").asText(), + config.get("access_key_id") == null ? "" : config.get("access_key_id").asText(), + config.get("secret_access_key") == null ? "" : config.get("secret_access_key").asText(), partSize, format); } @@ -128,7 +129,18 @@ public S3FormatConfig getFormatConfig() { public AmazonS3 getS3Client() { final AWSCredentials awsCreds = new BasicAWSCredentials(accessKeyId, secretAccessKey); - if (endpoint == null || endpoint.isEmpty()) { + if (accessKeyId.isEmpty() && !secretAccessKey.isEmpty() + || !accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { + throw new RuntimeException("Either both accessKeyId and secretAccessKey should be provided, or neither"); + } + + if (accessKeyId.isEmpty() && secretAccessKey.isEmpty()) { + return AmazonS3ClientBuilder.standard() + .withCredentials(new InstanceProfileCredentialsProvider(false)) + .build(); + } + + else if (endpoint == null || endpoint.isEmpty()) { return AmazonS3ClientBuilder.standard() .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .withRegion(bucketRegion) diff --git a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json index ceb0d8998cf2e..1251f6a400d99 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json @@ -12,8 +12,6 @@ "s3_bucket_name", "s3_bucket_path", "s3_bucket_region", - "access_key_id", - "secret_access_key", "format" ], "additionalProperties": false, @@ -72,14 +70,14 @@ }, "access_key_id": { "type": "string", - "description": "The access key id to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket.", + "description": "The access key id to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket, if not set, Airbyte will rely on Instance Profile.", "title": "S3 Key Id", "airbyte_secret": true, "examples": ["A012345678910EXAMPLE"] }, "secret_access_key": { "type": "string", - "description": "The corresponding secret to the access key id.", + "description": "The corresponding secret to the access key id, if S3 Key Id is set, then S3 Access Key must also be provided", "title": "S3 Access Key", "airbyte_secret": true, "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"] diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index d9ac12b8c1fcb..8ef5ee823d3ad 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -199,7 +199,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A #### Requirements 1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in separate VPCs\). -2. An S3 bucket with credentials. +2. An S3 bucket with credentials or an instanceprofile with read/write permissions configured for the host (ec2, eks). #### Setup Guide @@ -211,18 +211,22 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A * **S3 Bucket Region** * **Access Key Id** * See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. + * See [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) on how to create a instanceprofile. * We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. + * If the Access Key and Secret Access Key are not provided, the authentication will rely on the instanceprofile. * **Secret Access Key** * Corresponding key to the above key id. * Make sure your S3 bucket is accessible from the machine running Airbyte. * This depends on your networking setup. * You can check AWS S3 documentation with a tutorial on how to properly configure your S3's access [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). + * If you will use instance profile authentication, make sure the role has permission to read/write on the bucket. * The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check connection tool in the UI. ## CHANGELOG | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.2.5 | 2022-01-13 | [\#9399](https://github.com/airbytehq/airbyte/pull/9399) | Use instance profile authentication if credentials are not provided | | 0.2.4 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | | 0.2.3 | 2022-01-11 | [\#9367](https://github.com/airbytehq/airbyte/pull/9367) | Avro & Parquet: support array field with unknown item type; default any improperly typed field to string. | | 0.2.2 | 2021-12-21 | [\#8574](https://github.com/airbytehq/airbyte/pull/8574) | Added namespace to Avro and Parquet record types |