From f048db832f14ecbdaf5862f4d94c200cfac55955 Mon Sep 17 00:00:00 2001 From: Kees Duvekot Date: Mon, 12 Apr 2021 10:55:38 +0200 Subject: [PATCH 1/8] include additional kinesis-settings added the following settings: IncludeTransactionDetails IncludePartitionValue PartitionIncludeSchemaTable IncludeTableAlterOperations IncludeControlDetails IncludeNullAndEmpty --- aws/resource_aws_dms_endpoint.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/aws/resource_aws_dms_endpoint.go b/aws/resource_aws_dms_endpoint.go index 9aad32ba2e45..94de642d2c36 100644 --- a/aws/resource_aws_dms_endpoint.go +++ b/aws/resource_aws_dms_endpoint.go @@ -367,9 +367,15 @@ func resourceAwsDmsEndpointCreate(d *schema.ResourceData, meta interface{}) erro } case "kinesis": request.KinesisSettings = &dms.KinesisSettings{ - MessageFormat: aws.String(d.Get("kinesis_settings.0.message_format").(string)), - ServiceAccessRoleArn: aws.String(d.Get("kinesis_settings.0.service_access_role_arn").(string)), - StreamArn: aws.String(d.Get("kinesis_settings.0.stream_arn").(string)), + MessageFormat: aws.String(d.Get("kinesis_settings.0.message_format").(string)), + ServiceAccessRoleArn: aws.String(d.Get("kinesis_settings.0.service_access_role_arn").(string)), + StreamArn: aws.String(d.Get("kinesis_settings.0.stream_arn").(string)), + IncludeTransactionDetails: aws.Bool(d.Get("kinesis_settings.0.include_transaction_details").(bool)), + IncludePartitionValue: aws.Bool(d.Get("kinesis_settings.0.include_partition_value").(bool)), + PartitionIncludeSchemaTable: aws.Bool(d.Get("kinesis_settings.0.partition_include_schema_table").(bool)), + IncludeTableAlterOperations: aws.Bool(d.Get("kinesis_settings.0.include_table_alter_operations").(bool)), + IncludeControlDetails: aws.Bool(d.Get("kinesis_settings.0.include_control_details").(bool)), + IncludeNullAndEmpty: aws.Bool(d.Get("kinesis_settings.0.include_null_and_empty").(bool)), } case "mongodb": request.MongoDbSettings = &dms.MongoDbSettings{ @@ -596,6 +602,12 @@ func resourceAwsDmsEndpointUpdate(d *schema.ResourceData, meta interface{}) erro request.KinesisSettings = &dms.KinesisSettings{ ServiceAccessRoleArn: aws.String(d.Get("kinesis_settings.0.service_access_role_arn").(string)), StreamArn: aws.String(d.Get("kinesis_settings.0.stream_arn").(string)), + IncludeTransactionDetails: aws.Bool(d.Get("kinesis_settings.0.include_transaction_details").(bool)), + IncludePartitionValue: aws.Bool(d.Get("kinesis_settings.0.include_partition_value").(bool)), + PartitionIncludeSchemaTable: aws.Bool(d.Get("kinesis_settings.0.partition_include_schema_table").(bool)), + IncludeTableAlterOperations: aws.Bool(d.Get("kinesis_settings.0.include_table_alter_operations").(bool)), + IncludeControlDetails: aws.Bool(d.Get("kinesis_settings.0.include_control_details").(bool)), + IncludeNullAndEmpty: aws.Bool(d.Get("kinesis_settings.0.include_null_and_empty").(bool)), } request.EngineName = aws.String(d.Get("engine_name").(string)) // Must be included (should be 'kinesis') hasChanges = true @@ -802,6 +814,11 @@ func flattenDmsKinesisSettings(settings *dms.KinesisSettings) []map[string]inter "message_format": aws.StringValue(settings.MessageFormat), "service_access_role_arn": aws.StringValue(settings.ServiceAccessRoleArn), "stream_arn": aws.StringValue(settings.StreamArn), + "include_transaction_details": aws.BoolValue(settings.IncludeTransactionDetails), + "include_partition_value": aws.BoolValue(settings.IncludePartitionValue), + "partition_include_schema_table": aws.BoolValue(settings.PartitionIncludeSchemaTable), + "include_control_details": aws.BoolValue(settings.IncludeControlDetails), + "include_null_and_empty": aws.BoolValue(settings.IncludeNullAndEmpty), } return []map[string]interface{}{m} From 529da55ee542842b78a1a3938047adcdbfe0c61c Mon Sep 17 00:00:00 2001 From: Kees Duvekot Date: Mon, 12 Apr 2021 11:19:51 +0200 Subject: [PATCH 2/8] formatting --- aws/resource_aws_dms_endpoint.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aws/resource_aws_dms_endpoint.go b/aws/resource_aws_dms_endpoint.go index 94de642d2c36..97ef8bb908b0 100644 --- a/aws/resource_aws_dms_endpoint.go +++ b/aws/resource_aws_dms_endpoint.go @@ -600,8 +600,8 @@ func resourceAwsDmsEndpointUpdate(d *schema.ResourceData, meta interface{}) erro // "An error occurred (InvalidParameterValueException) when calling the ModifyEndpoint // operation: Message format cannot be modified for kinesis endpoints." request.KinesisSettings = &dms.KinesisSettings{ - ServiceAccessRoleArn: aws.String(d.Get("kinesis_settings.0.service_access_role_arn").(string)), - StreamArn: aws.String(d.Get("kinesis_settings.0.stream_arn").(string)), + ServiceAccessRoleArn: aws.String(d.Get("kinesis_settings.0.service_access_role_arn").(string)), + StreamArn: aws.String(d.Get("kinesis_settings.0.stream_arn").(string)), IncludeTransactionDetails: aws.Bool(d.Get("kinesis_settings.0.include_transaction_details").(bool)), IncludePartitionValue: aws.Bool(d.Get("kinesis_settings.0.include_partition_value").(bool)), PartitionIncludeSchemaTable: aws.Bool(d.Get("kinesis_settings.0.partition_include_schema_table").(bool)), @@ -811,9 +811,9 @@ func flattenDmsKinesisSettings(settings *dms.KinesisSettings) []map[string]inter } m := map[string]interface{}{ - "message_format": aws.StringValue(settings.MessageFormat), - "service_access_role_arn": aws.StringValue(settings.ServiceAccessRoleArn), - "stream_arn": aws.StringValue(settings.StreamArn), + "message_format": aws.StringValue(settings.MessageFormat), + "service_access_role_arn": aws.StringValue(settings.ServiceAccessRoleArn), + "stream_arn": aws.StringValue(settings.StreamArn), "include_transaction_details": aws.BoolValue(settings.IncludeTransactionDetails), "include_partition_value": aws.BoolValue(settings.IncludePartitionValue), "partition_include_schema_table": aws.BoolValue(settings.PartitionIncludeSchemaTable), From fd534102280cb7f9182fbd0adcfa85447bb48875 Mon Sep 17 00:00:00 2001 From: Kees Duvekot Date: Mon, 12 Apr 2021 12:49:20 +0200 Subject: [PATCH 3/8] added schema.Resource added the new elements to the schema of the Kinesis settings --- aws/resource_aws_dms_endpoint.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/aws/resource_aws_dms_endpoint.go b/aws/resource_aws_dms_endpoint.go index 97ef8bb908b0..9ad60d38c050 100644 --- a/aws/resource_aws_dms_endpoint.go +++ b/aws/resource_aws_dms_endpoint.go @@ -189,6 +189,35 @@ func resourceAwsDmsEndpoint() *schema.Resource { Type: schema.TypeString, Optional: true, ValidateFunc: validateArn, + "include_transaction_details": { + Type: schema.TypeBool, + Optional: true, + Default: false, + }, + "include_partition_value": { + Type: schema.TypeBool, + Optional: true, + Default: false, + }, + "partition_include_schema_table": { + Type: schema.TypeBool, + Optional: true, + Default: false, + }, + "include_table_alter_operations": { + Type: schema.TypeBool, + Optional: true, + Default: false, + }, + "include_control_details": { + Type: schema.TypeBool, + Optional: true, + Default: false, + }, + "include_null_and_empty": { + Type: schema.TypeBool, + Optional: true, + Default: false, }, }, }, From c84de3e325889e9a9d60d629d132f6ff67299297 Mon Sep 17 00:00:00 2001 From: Kees Duvekot Date: Mon, 12 Apr 2021 12:58:19 +0200 Subject: [PATCH 4/8] fix typo removed a "}," that should not have been removed :-) --- aws/resource_aws_dms_endpoint.go | 1 + 1 file changed, 1 insertion(+) diff --git a/aws/resource_aws_dms_endpoint.go b/aws/resource_aws_dms_endpoint.go index 9ad60d38c050..c6e0040d6fda 100644 --- a/aws/resource_aws_dms_endpoint.go +++ b/aws/resource_aws_dms_endpoint.go @@ -189,6 +189,7 @@ func resourceAwsDmsEndpoint() *schema.Resource { Type: schema.TypeString, Optional: true, ValidateFunc: validateArn, + }, "include_transaction_details": { Type: schema.TypeBool, Optional: true, From a58f8cbf004a2b48fbba27dfc8b3baf573549ce3 Mon Sep 17 00:00:00 2001 From: Kees Duvekot Date: Mon, 12 Apr 2021 13:18:22 +0200 Subject: [PATCH 5/8] added TestCheckResourceAttr for Kinesis endpoints --- aws/resource_aws_dms_endpoint_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/aws/resource_aws_dms_endpoint_test.go b/aws/resource_aws_dms_endpoint_test.go index 79723c311876..620f727c6208 100644 --- a/aws/resource_aws_dms_endpoint_test.go +++ b/aws/resource_aws_dms_endpoint_test.go @@ -392,6 +392,12 @@ func TestAccAwsDmsEndpoint_Kinesis(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "kinesis_settings.#", "1"), resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.message_format", "json"), resource.TestCheckResourceAttrPair(resourceName, "kinesis_settings.0.stream_arn", "aws_kinesis_stream.stream1", "arn"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_transaction_details", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_partition_value", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.partition_include_schema_table", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_table_alter_operations", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_control_details", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_null_and_empty", false), ), }, { @@ -407,6 +413,12 @@ func TestAccAwsDmsEndpoint_Kinesis(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "kinesis_settings.#", "1"), resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.message_format", "json"), resource.TestCheckResourceAttrPair(resourceName, "kinesis_settings.0.stream_arn", "aws_kinesis_stream.stream2", "arn"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_transaction_details", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_partition_value", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.partition_include_schema_table", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_table_alter_operations", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_control_details", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_null_and_empty", false), ), }, }, From 17642c182a9c738f493b6bcd80e28baaff6706b6 Mon Sep 17 00:00:00 2001 From: Kees Duvekot Date: Mon, 12 Apr 2021 14:15:05 +0200 Subject: [PATCH 6/8] update fales to "false" --- aws/resource_aws_dms_endpoint_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/aws/resource_aws_dms_endpoint_test.go b/aws/resource_aws_dms_endpoint_test.go index 620f727c6208..3a3e5c37c472 100644 --- a/aws/resource_aws_dms_endpoint_test.go +++ b/aws/resource_aws_dms_endpoint_test.go @@ -392,12 +392,12 @@ func TestAccAwsDmsEndpoint_Kinesis(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "kinesis_settings.#", "1"), resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.message_format", "json"), resource.TestCheckResourceAttrPair(resourceName, "kinesis_settings.0.stream_arn", "aws_kinesis_stream.stream1", "arn"), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_transaction_details", false), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_partition_value", false), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.partition_include_schema_table", false), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_table_alter_operations", false), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_control_details", false), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_null_and_empty", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_transaction_details", "false"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_partition_value", "false"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.partition_include_schema_table", "false"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_table_alter_operations", "false"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_control_details", "false"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_null_and_empty", "false"), ), }, { @@ -413,12 +413,12 @@ func TestAccAwsDmsEndpoint_Kinesis(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "kinesis_settings.#", "1"), resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.message_format", "json"), resource.TestCheckResourceAttrPair(resourceName, "kinesis_settings.0.stream_arn", "aws_kinesis_stream.stream2", "arn"), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_transaction_details", false), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_partition_value", false), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.partition_include_schema_table", false), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_table_alter_operations", false), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_control_details", false), - resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_null_and_empty", false), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_transaction_details", "false"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_partition_value", "false"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.partition_include_schema_table", "false"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_table_alter_operations", "false"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_control_details", "false"), + resource.TestCheckResourceAttr(resourceName, "kinesis_settings.0.include_null_and_empty", "false"), ), }, }, From ed1d5414b300db8fa2776e3b47e51522f26437de Mon Sep 17 00:00:00 2001 From: Kees Duvekot Date: Mon, 12 Apr 2021 14:28:24 +0200 Subject: [PATCH 7/8] update documentation --- website/docs/r/dms_endpoint.html.markdown | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/website/docs/r/dms_endpoint.html.markdown b/website/docs/r/dms_endpoint.html.markdown index f9d964913f34..3fbfabaafb6e 100644 --- a/website/docs/r/dms_endpoint.html.markdown +++ b/website/docs/r/dms_endpoint.html.markdown @@ -98,6 +98,12 @@ The `kinesis_settings` configuration block supports the following arguments: * `message_format` - (Optional) Output format for the records created. Defaults to `json`. Valid values are `json` and `json_unformatted` (a single line with no tab). * `service_access_role_arn` - (Optional) Amazon Resource Name (ARN) of the IAM Role with permissions to write to the Kinesis data stream. * `stream_arn` - (Optional) Amazon Resource Name (ARN) of the Kinesis data stream. +* `include_transaction_details` - (Optional) Provides detailed transaction information from the source database. The default is `false`. +* `include_partition_value` - (Optional) Shows the partition value within the Kinesis message output, unless the partition type is schema-table-type. The default is `false`. +* `partition_include_schema_table` - (Optional) Prefixes schema and table names to partition values, when the partition type is primary-key-type. The default is `false`. +* `include_table_alter_operations` - (Optional) Includes any data definition language (DDL) operations that change the table in the control data. The default is `false`. +* `include_control_details` - (Optional) Shows detailed control information for table definition, column definition, and table and column changes in the Kinesis message output. The default is `false`. +* `include_null_and_empty` - (Optional) Include NULL and empty columns in the target. The default is `false`. ### mongodb_settings Arguments From a2edd9d0840aab22ee1ebbc613b9e80c677ce4b4 Mon Sep 17 00:00:00 2001 From: Kees Duvekot Date: Mon, 12 Apr 2021 14:35:54 +0200 Subject: [PATCH 8/8] format changes --- aws/resource_aws_dms_endpoint.go | 36 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/aws/resource_aws_dms_endpoint.go b/aws/resource_aws_dms_endpoint.go index c6e0040d6fda..513c31f5ec55 100644 --- a/aws/resource_aws_dms_endpoint.go +++ b/aws/resource_aws_dms_endpoint.go @@ -191,34 +191,34 @@ func resourceAwsDmsEndpoint() *schema.Resource { ValidateFunc: validateArn, }, "include_transaction_details": { - Type: schema.TypeBool, - Optional: true, - Default: false, + Type: schema.TypeBool, + Optional: true, + Default: false, }, "include_partition_value": { - Type: schema.TypeBool, - Optional: true, - Default: false, + Type: schema.TypeBool, + Optional: true, + Default: false, }, "partition_include_schema_table": { - Type: schema.TypeBool, - Optional: true, - Default: false, + Type: schema.TypeBool, + Optional: true, + Default: false, }, "include_table_alter_operations": { - Type: schema.TypeBool, - Optional: true, - Default: false, + Type: schema.TypeBool, + Optional: true, + Default: false, }, "include_control_details": { - Type: schema.TypeBool, - Optional: true, - Default: false, + Type: schema.TypeBool, + Optional: true, + Default: false, }, "include_null_and_empty": { - Type: schema.TypeBool, - Optional: true, - Default: false, + Type: schema.TypeBool, + Optional: true, + Default: false, }, }, },