From a3dfceb964d38861211b3920a0ed8f610df73168 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 7 Mar 2023 13:45:53 +0000 Subject: [PATCH 01/32] removal: duplicate sentence --- packages/@aws-cdk/aws-glue/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index 83cb4a1c82434..5c4d369dd4b3a 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -225,8 +225,6 @@ new glue.Table(this, 'MyTable', { }); ``` -By default, an S3 bucket will be created to store the table's data and stored in the bucket root. You can also manually pass the `bucket` and `s3Prefix`: - ### Partition Keys To improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window: From 3daf56a17f431c5728173abf2cc3df6768ab8188 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 7 Mar 2023 14:49:57 +0000 Subject: [PATCH 02/32] addition: removal policies, dont want them in account after test --- packages/@aws-cdk/aws-glue/test/integ.table.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts index d9d543a124d16..a291ac9987f6d 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.table.ts @@ -9,7 +9,9 @@ const app = new cdk.App(); const stack = new cdk.Stack(app, 'aws-cdk-glue'); -const bucket = new s3.Bucket(stack, 'DataBucket'); +const bucket = new s3.Bucket(stack, 'DataBucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, +}); const database = new glue.Database(stack, 'MyDatabase', { databaseName: 'my_database', @@ -84,11 +86,14 @@ const encryptedTable = new glue.Table(stack, 'MyEncryptedTable', { partitionKeys, dataFormat: glue.DataFormat.JSON, encryption: glue.TableEncryption.KMS, - encryptionKey: new kms.Key(stack, 'MyKey'), + encryptionKey: new kms.Key(stack, 'MyKey', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + }), }); new glue.Table(stack, 'MyPartitionFilteredTable', { database, + bucket, tableName: 'partition_filtered_table', columns, dataFormat: glue.DataFormat.JSON, From 37bcd790a1742e2acdacb27c6ee52e38d352d0c9 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 7 Mar 2023 14:50:18 +0000 Subject: [PATCH 03/32] addition: integ test construct --- packages/@aws-cdk/aws-glue/test/integ.table.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts index a291ac9987f6d..28ab83fa465c7 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.table.ts @@ -1,8 +1,10 @@ #!/usr/bin/env node +/* eslint-disable import/no-extraneous-dependencies */ import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; import * as s3 from '@aws-cdk/aws-s3'; import * as cdk from '@aws-cdk/core'; +import * as integ from '@aws-cdk/integ-tests'; import * as glue from '../lib'; const app = new cdk.App(); @@ -109,4 +111,8 @@ avroTable.grantReadWrite(anotherUser); jsonTable.grantReadWrite(anotherUser); parquetTable.grantReadWrite(anotherUser); +new integ.IntegTest(app, 'aws-cdk-glue-table-integ', { + testCases: [stack], +}); + app.synth(); From 1b5494b0873a75c489f48407bd4fa51799eda3e1 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 7 Mar 2023 14:51:41 +0000 Subject: [PATCH 04/32] adding storageParameters attribute to construct --- packages/@aws-cdk/aws-glue/lib/table.ts | 8 ++ .../aws-cdk-glue.assets.json | 6 +- .../aws-cdk-glue.template.json | 19 ++- ...efaultTestDeployAssert8BFB5B70.assets.json | 19 +++ ...aultTestDeployAssert8BFB5B70.template.json | 36 ++++++ .../test/integ.table.js.snapshot/cdk.out | 2 +- .../test/integ.table.js.snapshot/integ.json | 12 +- .../integ.table.js.snapshot/manifest.json | 78 ++++++++++-- .../test/integ.table.js.snapshot/tree.json | 120 +++++++++++++----- .../@aws-cdk/aws-glue/test/integ.table.ts | 4 + packages/@aws-cdk/aws-glue/test/table.test.ts | 31 ++++- 11 files changed, 265 insertions(+), 70 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json create mode 100644 packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index 83efe3f86f772..1d60fb338a4b4 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -183,6 +183,13 @@ export interface TableProps { * @default - The parameter is not defined */ readonly enablePartitionFiltering?: boolean; + + /** + * Description of the physical storage of this table. + * + * @default - The parameter is not defined + */ + readonly storageParameters?: { [key: string]: any }; } /** @@ -330,6 +337,7 @@ export class Table extends Resource implements ITable { serdeInfo: { serializationLibrary: props.dataFormat.serializationLibrary.className, }, + parameters: props.storageParameters, }, tableType: 'EXTERNAL_TABLE', diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index 5381bea5e40ca..801bb4a355413 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { - "version": "20.0.0", + "version": "30.1.0", "files": { - "eef5abdc0f1ee16e5be447f60688757df6726f3c2d1d06c136e9bbdb99d96e1f": { + "6da233c1065398a34bfbdbcde82f5fc76cbb7ce6db0316b2093f4e2ed653d68b": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "eef5abdc0f1ee16e5be447f60688757df6726f3c2d1d06c136e9bbdb99d96e1f.json", + "objectKey": "6da233c1065398a34bfbdbcde82f5fc76cbb7ce6db0316b2093f4e2ed653d68b.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json index 92dfc8e76e2d5..037757eb5a250 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -2,8 +2,8 @@ "Resources": { "DataBucketE3889A50": { "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" }, "MyDatabase1E2517DB": { "Type": "AWS::Glue::Database", @@ -328,8 +328,8 @@ "Version": "2012-10-17" } }, - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" }, "MyEncryptedTableBucket7B28486D": { "Type": "AWS::S3::Bucket", @@ -423,11 +423,6 @@ } } }, - "MyPartitionFilteredTableBucket6ACAA137": { - "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" - }, "MyPartitionFilteredTable324BA27A": { "Type": "AWS::Glue::Table", "Properties": { @@ -477,13 +472,17 @@ [ "s3://", { - "Ref": "MyPartitionFilteredTableBucket6ACAA137" + "Ref": "DataBucketE3889A50" }, "/" ] ] }, "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "Parameters": { + "separatorChar": ",", + "skip.header.line.count": 2 + }, "SerdeInfo": { "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" }, diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json new file mode 100644 index 0000000000000..7af33a9b07de8 --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json @@ -0,0 +1,19 @@ +{ + "version": "30.1.0", + "files": { + "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { + "source": { + "path": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json new file mode 100644 index 0000000000000..ad9d0fb73d1dd --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json @@ -0,0 +1,36 @@ +{ + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/cdk.out index 588d7b269d34f..b72fef144f05c 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"20.0.0"} \ No newline at end of file +{"version":"30.1.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/integ.json index 1f604630bc610..b6071e0578e5c 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/integ.json @@ -1,14 +1,12 @@ { - "version": "20.0.0", + "version": "30.1.0", "testCases": { - "integ.table": { + "aws-cdk-glue-table-integ/DefaultTest": { "stacks": [ "aws-cdk-glue" ], - "diffAssets": false, - "stackUpdateWorkflow": true + "assertionStack": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert", + "assertionStackName": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70" } - }, - "synthContext": {}, - "enableLookups": false + } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json index 931c2276f10a3..c5d0230990320 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json @@ -1,12 +1,6 @@ { - "version": "20.0.0", + "version": "30.1.0", "artifacts": { - "Tree": { - "type": "cdk:tree", - "properties": { - "file": "tree.json" - } - }, "aws-cdk-glue.assets": { "type": "cdk:asset-manifest", "properties": { @@ -23,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/eef5abdc0f1ee16e5be447f60688757df6726f3c2d1d06c136e9bbdb99d96e1f.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/6da233c1065398a34bfbdbcde82f5fc76cbb7ce6db0316b2093f4e2ed653d68b.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -93,12 +87,6 @@ "data": "MyEncryptedTable981A88C6" } ], - "/aws-cdk-glue/MyPartitionFilteredTable/Bucket/Resource": [ - { - "type": "aws:cdk:logicalId", - "data": "MyPartitionFilteredTableBucket6ACAA137" - } - ], "/aws-cdk-glue/MyPartitionFilteredTable/Table": [ { "type": "aws:cdk:logicalId", @@ -140,9 +128,71 @@ "type": "aws:cdk:logicalId", "data": "CheckBootstrapVersion" } + ], + "MyPartitionFilteredTableBucket6ACAA137": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPartitionFilteredTableBucket6ACAA137", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } ] }, "displayName": "aws-cdk-glue" + }, + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json", + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets" + ], + "metadata": { + "/aws-cdk-glue-table-integ/DefaultTest/DeployAssert/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/aws-cdk-glue-table-integ/DefaultTest/DeployAssert/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert" + }, + "Tree": { + "type": "cdk:tree", + "properties": { + "file": "tree.json" + } } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json index 441cf0ac8901f..f4719d1d5a380 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json @@ -4,14 +4,6 @@ "id": "App", "path": "", "children": { - "Tree": { - "id": "Tree", - "path": "Tree", - "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.1.85" - } - }, "aws-cdk-glue": { "id": "aws-cdk-glue", "path": "aws-cdk-glue", @@ -600,28 +592,6 @@ "id": "MyPartitionFilteredTable", "path": "aws-cdk-glue/MyPartitionFilteredTable", "children": { - "Bucket": { - "id": "Bucket", - "path": "aws-cdk-glue/MyPartitionFilteredTable/Bucket", - "children": { - "Resource": { - "id": "Resource", - "path": "aws-cdk-glue/MyPartitionFilteredTable/Bucket/Resource", - "attributes": { - "aws:cdk:cloudformation:type": "AWS::S3::Bucket", - "aws:cdk:cloudformation:props": {} - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-s3.CfnBucket", - "version": "0.0.0" - } - } - }, - "constructInfo": { - "fqn": "@aws-cdk/aws-s3.Bucket", - "version": "0.0.0" - } - }, "Table": { "id": "Table", "path": "aws-cdk-glue/MyPartitionFilteredTable/Table", @@ -649,7 +619,7 @@ [ "s3://", { - "Ref": "MyPartitionFilteredTableBucket6ACAA137" + "Ref": "DataBucketE3889A50" }, "/" ] @@ -684,6 +654,10 @@ "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", "serdeInfo": { "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "parameters": { + "separatorChar": ",", + "skip.header.line.count": 2 } }, "tableType": "EXTERNAL_TABLE" @@ -1095,17 +1069,95 @@ "fqn": "@aws-cdk/aws-iam.User", "version": "0.0.0" } + }, + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-cdk-glue/BootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-cdk-glue/CheckBootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/core.Stack", + "version": "0.0.0" + } + }, + "aws-cdk-glue-table-integ": { + "id": "aws-cdk-glue-table-integ", + "path": "aws-cdk-glue-table-integ", + "children": { + "DefaultTest": { + "id": "DefaultTest", + "path": "aws-cdk-glue-table-integ/DefaultTest", + "children": { + "Default": { + "id": "Default", + "path": "aws-cdk-glue-table-integ/DefaultTest/Default", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.1.264" + } + }, + "DeployAssert": { + "id": "DeployAssert", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert", + "children": { + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert/BootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert/CheckBootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/core.Stack", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests.IntegTestCase", + "version": "0.0.0" + } } }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests.IntegTest", + "version": "0.0.0" + } + }, + "Tree": { + "id": "Tree", + "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.1.85" + "version": "10.1.264" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.1.85" + "fqn": "@aws-cdk/core.App", + "version": "0.0.0" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts index 28ab83fa465c7..2a24cafade46c 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.table.ts @@ -100,6 +100,10 @@ new glue.Table(stack, 'MyPartitionFilteredTable', { columns, dataFormat: glue.DataFormat.JSON, enablePartitionFiltering: true, + storageParameters: { + 'separatorChar': ',', + 'skip.header.line.count': 2, + }, }); const user = new iam.User(stack, 'MyUser'); diff --git a/packages/@aws-cdk/aws-glue/test/table.test.ts b/packages/@aws-cdk/aws-glue/test/table.test.ts index 520c6a8528f49..7c35195e1a946 100644 --- a/packages/@aws-cdk/aws-glue/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue/test/table.test.ts @@ -1,4 +1,4 @@ -import { Template, Match } from '@aws-cdk/assertions'; +import { Match, Template } from '@aws-cdk/assertions'; import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; import * as s3 from '@aws-cdk/aws-s3'; @@ -1600,6 +1600,35 @@ test('can specify a physical name', () => { }); }); +test('storage descriptor parameters', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + new glue.Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + storageParameters: { + 'skip.header.line.count': 2, + 'separatorChar': ',', + }, + }); + + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { + TableInput: { + StorageDescriptor: { + Parameters: { + 'skip.header.line.count': 2, + 'separatorChar': ',', + }, + }, + }, + }); +}); + function createTable(props: Pick>): void { const stack = new cdk.Stack(); new glue.Table(stack, 'table', { From 19d5a54c2aa67fcb9f489d8dff0ab7a184c111d3 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 7 Mar 2023 14:57:56 +0000 Subject: [PATCH 05/32] specific comment --- packages/@aws-cdk/aws-glue/lib/table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index 1d60fb338a4b4..f9850b90dec52 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -185,7 +185,7 @@ export interface TableProps { readonly enablePartitionFiltering?: boolean; /** - * Description of the physical storage of this table. + * The user-supplied properties for the description of the physical storage of this table. * * @default - The parameter is not defined */ From ced55e50c2031f7458aa80e652496f3aa702f883 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 7 Mar 2023 15:01:54 +0000 Subject: [PATCH 06/32] addition: README for the storageParameters --- packages/@aws-cdk/aws-glue/README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index 5c4d369dd4b3a..874bcb5c120f9 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -225,6 +225,25 @@ new glue.Table(this, 'MyTable', { }); ``` +Glue tables can be configured to contain user-defined properties, to describe the physical storage of table data, through the `storageParameters` property: + +```ts +declare const myBucket: s3.Bucket; +declare const myDatabase: glue.Database; +new glue.Table(this, 'MyTable', { + database: myDatabase, + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + storageParameters: { + 'skip.header.line.count': 1, + 'separatorChar': ',', + } +}); +``` + ### Partition Keys To improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window: From a615848862c6d6b2bbb69894f4be05752d69a547 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 8 Mar 2023 11:57:42 +0000 Subject: [PATCH 07/32] removing quote from README --- packages/@aws-cdk/aws-glue/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index 874bcb5c120f9..fd358ac880697 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -239,7 +239,7 @@ new glue.Table(this, 'MyTable', { dataFormat: glue.DataFormat.JSON, storageParameters: { 'skip.header.line.count': 1, - 'separatorChar': ',', + separatorChar: ',', } }); ``` From 7a90bc01e9d287606eb751a780f3c96283f8aee7 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 8 Mar 2023 12:15:51 +0000 Subject: [PATCH 08/32] updating paramter comment --- packages/@aws-cdk/aws-glue/lib/table.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index f9850b90dec52..a900f71a41ddb 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -187,6 +187,16 @@ export interface TableProps { /** * The user-supplied properties for the description of the physical storage of this table. * + * There are reserved keys that are used by AWS Glue. They CAN be mutated, but they are best left alone. + * + * The key/value pairs that are allowed to be submitted are not limited, however their functionality is not guaranteed. + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under "TABLE PROPERTIES" contains a non-exhaustive list of the keys that have functionality. + * + * @example + * { foo: 'bar' } // will have no effect + * { 'skip.header.line.count': 1 } // will set the number of header lines to skip to 1. + * * @default - The parameter is not defined */ readonly storageParameters?: { [key: string]: any }; From c729a3ff36d670490fc492e06d6ff79e657436ad Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 8 Mar 2023 12:24:43 +0000 Subject: [PATCH 09/32] integ test for non-functioning key --- .../integ.table.js.snapshot/aws-cdk-glue.assets.json | 4 ++-- .../aws-cdk-glue.template.json | 3 ++- .../test/integ.table.js.snapshot/manifest.json | 11 +---------- .../aws-glue/test/integ.table.js.snapshot/tree.json | 3 ++- packages/@aws-cdk/aws-glue/test/integ.table.ts | 1 + 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index 801bb4a355413..503a49d6b195a 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { "version": "30.1.0", "files": { - "6da233c1065398a34bfbdbcde82f5fc76cbb7ce6db0316b2093f4e2ed653d68b": { + "553f4bc301289a3e09b1cd03c07892e4c62da6b35d057a57fe1613c230c27ef6": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "6da233c1065398a34bfbdbcde82f5fc76cbb7ce6db0316b2093f4e2ed653d68b.json", + "objectKey": "553f4bc301289a3e09b1cd03c07892e4c62da6b35d057a57fe1613c230c27ef6.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json index 037757eb5a250..cdbd13c00f014 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -481,7 +481,8 @@ "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", "Parameters": { "separatorChar": ",", - "skip.header.line.count": 2 + "skip.header.line.count": 2, + "foo": "bar" }, "SerdeInfo": { "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json index c5d0230990320..e4b022f44d479 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/6da233c1065398a34bfbdbcde82f5fc76cbb7ce6db0316b2093f4e2ed653d68b.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/553f4bc301289a3e09b1cd03c07892e4c62da6b35d057a57fe1613c230c27ef6.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -128,15 +128,6 @@ "type": "aws:cdk:logicalId", "data": "CheckBootstrapVersion" } - ], - "MyPartitionFilteredTableBucket6ACAA137": [ - { - "type": "aws:cdk:logicalId", - "data": "MyPartitionFilteredTableBucket6ACAA137", - "trace": [ - "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" - ] - } ] }, "displayName": "aws-cdk-glue" diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json index f4719d1d5a380..409cc8e9d0ebf 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json @@ -657,7 +657,8 @@ }, "parameters": { "separatorChar": ",", - "skip.header.line.count": 2 + "skip.header.line.count": 2, + "foo": "bar" } }, "tableType": "EXTERNAL_TABLE" diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts index 2a24cafade46c..880da80f3fde0 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.table.ts @@ -103,6 +103,7 @@ new glue.Table(stack, 'MyPartitionFilteredTable', { storageParameters: { 'separatorChar': ',', 'skip.header.line.count': 2, + 'foo': 'bar', }, }); From 0c5fa9a79c25fbc38c0127272a6559e8f35138d0 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Wed, 8 Mar 2023 14:22:47 +0000 Subject: [PATCH 10/32] updating README --- packages/@aws-cdk/aws-glue/lib/table.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index a900f71a41ddb..ed43be6058fb6 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -185,7 +185,7 @@ export interface TableProps { readonly enablePartitionFiltering?: boolean; /** - * The user-supplied properties for the description of the physical storage of this table. + * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources. * * There are reserved keys that are used by AWS Glue. They CAN be mutated, but they are best left alone. * @@ -194,8 +194,20 @@ export interface TableProps { * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under "TABLE PROPERTIES" contains a non-exhaustive list of the keys that have functionality. * * @example - * { foo: 'bar' } // will have no effect - * { 'skip.header.line.count': 1 } // will set the number of header lines to skip to 1. + * + * declare const glueDatabase: glue.IDatabase; + * const table = new glue.Table(this, 'Table', { + * database: glueDatabase, + * columns: [{ + * name: 'col1', + * type: glue.Schema.STRING, + * }], + * dataFormat: glue.DataFormat.CSV, + * storageParameters: { + * foo: 'bar', // Will have no effect + * 'skip.header.line.count': 1, // Will be used to skip the first line of the file + * }, + * }); * * @default - The parameter is not defined */ From a71a336372b1aa14d72e3968eeed95583a4f11f9 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 10 Mar 2023 16:20:13 +0000 Subject: [PATCH 11/32] addition: adding enums --- packages/@aws-cdk/aws-glue/lib/table.ts | 286 +++++++++++++++++- .../@aws-cdk/aws-glue/test/integ.table.ts | 8 +- packages/@aws-cdk/aws-glue/test/table.test.ts | 6 +- 3 files changed, 289 insertions(+), 11 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index ed43be6058fb6..3cccfd1f001c2 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -75,6 +75,282 @@ export enum TableEncryption { CLIENT_SIDE_KMS = 'CSE-KMS' } +/** + * Properties for creating a new Table. + */ +export enum StorageParameters { + /** + * The number of rows to skip at the top of a CSV file when the table is being created. + */ + SKIP_HEADER_LINE_COUNT = 'skip.header.line.count', + + /** + * Determines whether data handling is on for the table. + */ + DATA_CLEANSING_ENABLED = 'data_cleansing_enabled', + + /** + * The type of compression used on the table, when the file name does not contain an extension. This value overrides the compression type specified through the extension. + * + * Assign this parameter to a value of `StorageParameters.CompressionType` + */ + COMPRESSION_TYPE = 'compression_type', + + /** + * Specifies the action to perform when query results contain invalid UTF-8 character values. + * + * Assign this parameter to a value of `StorageParameters.InvalidCharHandlingAction` + */ + INVALID_CHAR_HANDLING = 'invalid_char_handling', + + /** + * Specifies the replacement character to use when you set `INVALID_CHAR_HANDLING` to `REPLACE`. + */ + REPLACEMENT_CHAR = 'replacement_char', + + /** + * Specifies the action to perform when ORC data contains an integer (for example, BIGINT or int64) that is larger than the column definition (for example, SMALLINT or int16). + * + * Assign this parameter to a value of `StorageParameters.NumericOverflowHandlingAction` + */ + NUMERIC_OVERFLOW_HANDLING = 'numeric_overflow_handling', + + /** + * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARBYTE data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. + * + * Assign this parameter to a value of `StorageParameters.SurplusBytesHandlingAction` + */ + SURPLUS_BYTES_HANDLING = 'surplus_bytes_handling', + + /** + * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARCHAR, CHAR, or string data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. + * + * Assign this parameter to a value of `StorageParameters.SurplusCharHandlingAction` + */ + SURPLUS_CHAR_HANDLING = 'surplus_char_handling', + + /** + * Identifies if the file contains less or more values for a row than the number of columns specified in the external table definition. This property is only available for an uncompressed text file format. + * + * Assign this parameter to a value of `StorageParameters.ColumnCountMismatchHandling` + */ + COLUMN_COUNT_MISMATCH_HANDLING = 'column_count_mismatch_handling', + + /** + * A property that sets the numRows value for the table definition. To explicitly update an external table's statistics, set the numRows property to indicate the size of the table. Amazon Redshift doesn't analyze external tables to generate the table statistics that the query optimizer uses to generate a query plan. If table statistics aren't set for an external table, Amazon Redshift generates a query execution plan based on an assumption that external tables are the larger tables and local tables are the smaller tables. + */ + NUM_ROWS = 'num_rows', + + /** + * A property that sets number of rows to skip at the beginning of each source file. + */ + SERIALIZATION_NULL_FORMAT = 'serialization.null.format', + + /** + * A property that sets the column mapping type for tables that use ORC data format. This property is ignored for other data formats. If this property is omitted, columns are mapped by `OrcColumnMappingType.NAME` by default. + * + * Assign this parameter to a value of `StorageParameters.OrcColumnMappingType` + */ + ORC_SCHEMA_RESOLUTION = 'orc.schema.resolution', + + /** + * A property that sets whether CREATE EXTERNAL TABLE AS should write data in parallel. By default, CREATE EXTERNAL TABLE AS writes data in parallel to multiple files, according to the number of slices in the cluster. The default option is on. When 'write.parallel' is set to off, CREATE EXTERNAL TABLE AS writes to one or more data files serially onto Amazon S3. This table property also applies to any subsequent INSERT statement into the same external table. + * + * Assign this parameter to a value of `StorageParameters.WriteParallel` + */ + WRITE_PARALLEL = 'write.parallel', + + /** + * A property that sets the maximum size (in MB) of each file written to Amazon S3 by CREATE EXTERNAL TABLE AS. The size must be a valid integer between 5 and 6200. The default maximum file size is 6,200 MB. This table property also applies to any subsequent INSERT statement into the same external table. + */ + WRITE_MAX_FILESIZE_MB = 'write.maxfilesize.mb', + + /** + * You can specify an AWS Key Management Service key to enable Server–Side Encryption (SSE) for Amazon S3 objects. + */ + WRITE_KMS_KEY_ID = 'write.kms.key.id', +} + +/** + * This list is not exhaustive, use the `custom` method to add custom parameters. + */ +export namespace StorageParameters { + + /** + * The compression type used by `StorageParameters.COMPRESSION_TYPE`. + */ + export enum CompressionType { + /** + * No compression. + */ + NONE = 'none', + + /** + * Burrows-Wheeler compression. + */ + BZIP2 = 'bzip2', + + /** + * Deflate compression. + */ + GZIP = 'gzip', + + /** + * Compression algorithm focused on high compression and decompression speeds, rather than the maximum possible compression. + */ + SNAPPY = 'snappy', + } + + /** + * The action to assign to `StorageParameters.INVALID_CHAR_HANDLING`. + */ + export enum InvalidCharHandlingAction { + /** + * Doesn't perform invalid character handling. + */ + DISABLED = 'DISABLED', + + /** + * Cancels queries that return data containing invalid UTF-8 values. + */ + FAIL = 'FAIL', + + /** + * Replaces invalid UTF-8 values with null. + */ + SET_TO_NULL = 'SET_TO_NULL', + + /** + * Replaces each value in the row with null. + */ + DROP_ROW = 'DROP_ROW', + + /** + * Replaces the invalid character with the replacement character you specify using `StorageParameters.REPLACEMENT_CHAR`. + */ + REPLACE = 'REPLACE', + } + + /** + * The action to assign to `StorageParameters.COLUMN_COUNT_MISMATCH_HANDLING`. + */ + export enum NumericOverflowHandlingAction { + /** + * Invalid character handling is turned off. + */ + DISABLED = 'DISABLED', + + /** + * Cancel the query when the data includes invalid characters. + */ + FAIL = 'FAIL', + + /** + * Set invalid characters to null. + */ + SET_TO_NULL = 'SET_TO_NULL', + + /** + * Set each value in the row to null. + */ + DROP_ROW = 'DROP_ROW', + } + + /** + * The action to assign to `StorageParameters.COLUMN_COUNT_MISMATCH_HANDLING`. + */ + export enum SurplusBytesHandlingAction { + /** + * Replaces data that exceeds the column width with null. + */ + SET_TO_NULL = 'SET_TO_NULL', + + /** + * Doesn't perform surplus byte handling. + */ + DISABLED = 'DISABLED', + + /** + * Cancels queries that return data exceeding the column width. + */ + FAIL = 'FAIL', + + /** + * Drop all rows that contain data exceeding column width. + */ + DROP_ROW = 'DROP_ROW', + + /** + * Removes the characters that exceed the maximum number of characters defined for the column. + */ + TRUNCATE = 'TRUNCATE', + } + + /** + * The action to assign to `StorageParameters.COLUMN_COUNT_MISMATCH_HANDLING`. + */ + export enum SurplusCharHandlingAction { + /** + * Replaces data that exceeds the column width with null. + */ + SET_TO_NULL = 'SET_TO_NULL', + + /** + * Doesn't perform surplus character handling. + */ + DISABLED = 'DISABLED', + + /** + * Cancels queries that return data exceeding the column width. + */ + FAIL = 'FAIL', + + /** + * Replaces each value in the row with null. + */ + DROP_ROW = 'DROP_ROW', + + /** + * Removes the characters that exceed the maximum number of characters defined for the column. + */ + TRUNCATE = 'TRUNCATE', + } + + /** + * The action to assign to `StorageParameters.COLUMN_COUNT_MISMATCH_HANDLING`. + */ + export enum ColumnCountMismatchHandlingAction { + /** + * Column count mismatch handling is turned off. + */ + DISABLED = 'DISABLED', + + /** + * Fail the query if the column count mismatch is detected. + */ + FAIL = 'FAIL', + + /** + * Fill missing values with NULL and ignore the additional values in each row. + */ + SET_TO_NULL = 'SET_TO_NULL', + + /** + * Drop all rows that contain column count mismatch error from the scan. + */ + DROP_ROW = 'DROP_ROW', + } + + /** + * A custom storage parameter. + * + * @param key the key of the parameter + */ + export function custom(key: string): string { + return key; + } +} + export interface TableAttributes { readonly tableArn: string; readonly tableName: string; @@ -187,12 +463,10 @@ export interface TableProps { /** * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources. * - * There are reserved keys that are used by AWS Glue. They CAN be mutated, but they are best left alone. + * Some keys will be auto-populated by glue crawlers, however, you can override them by specifying the key and value in this property. * * The key/value pairs that are allowed to be submitted are not limited, however their functionality is not guaranteed. * - * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under "TABLE PROPERTIES" contains a non-exhaustive list of the keys that have functionality. - * * @example * * declare const glueDatabase: glue.IDatabase; @@ -204,8 +478,10 @@ export interface TableProps { * }], * dataFormat: glue.DataFormat.CSV, * storageParameters: { - * foo: 'bar', // Will have no effect - * 'skip.header.line.count': 1, // Will be used to skip the first line of the file + * glue.StorageParameters.SKIP_HEADER_LINE_COUNT: '1', + * glue.StorageParameters.COMPRESSION_TYPE: glue.StorageParameters.CompressionType.GZIP, + * glue.StorageParameters.custom('foo'): 'bar', // Will have no effect + * glue.StorageParameters.custom('separatorChar'): ',', // Will describe the separator char used in the data * }, * }); * diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts index 880da80f3fde0..5c0b74c89ee9d 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.table.ts @@ -101,9 +101,11 @@ new glue.Table(stack, 'MyPartitionFilteredTable', { dataFormat: glue.DataFormat.JSON, enablePartitionFiltering: true, storageParameters: { - 'separatorChar': ',', - 'skip.header.line.count': 2, - 'foo': 'bar', + [glue.StorageParameters.custom('separatorChar')]: ',', + [glue.StorageParameters.SKIP_HEADER_LINE_COUNT]: '1', + [glue.StorageParameters.COMPRESSION_TYPE]: glue.StorageParameters.CompressionType.GZIP, + [glue.StorageParameters.INVALID_CHAR_HANDLING]: glue.StorageParameters.InvalidCharHandlingAction.DISABLED, + [glue.StorageParameters.custom('foo')]: 'bar', }, }); diff --git a/packages/@aws-cdk/aws-glue/test/table.test.ts b/packages/@aws-cdk/aws-glue/test/table.test.ts index 7c35195e1a946..f4e2d3efdd681 100644 --- a/packages/@aws-cdk/aws-glue/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue/test/table.test.ts @@ -4,7 +4,7 @@ import * as kms from '@aws-cdk/aws-kms'; import * as s3 from '@aws-cdk/aws-s3'; import * as cdk from '@aws-cdk/core'; import * as glue from '../lib'; -import { PartitionIndex } from '../lib'; +import { PartitionIndex, StorageParameters } from '../lib'; import { CfnTable } from '../lib/glue.generated'; test('unpartitioned JSON table', () => { @@ -1612,8 +1612,8 @@ test('storage descriptor parameters', () => { }], dataFormat: glue.DataFormat.JSON, storageParameters: { - 'skip.header.line.count': 2, - 'separatorChar': ',', + [StorageParameters.SKIP_HEADER_LINE_COUNT]: '2', + // [StorageParameters.custom('separatorChar')]: ',', }, }); From fba89cc6572acb9db5094e52d931ab941ff2114a Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 23 Mar 2023 15:12:36 +0000 Subject: [PATCH 12/32] modification: StorageParameter as namespace for Keys, not values --- packages/@aws-cdk/aws-glue/lib/table.ts | 305 ++++++++++-------- packages/@aws-cdk/aws-glue/package.json | 8 +- .../@aws-cdk/aws-glue/test/integ.table.ts | 21 +- packages/@aws-cdk/aws-glue/test/table.test.ts | 14 +- 4 files changed, 194 insertions(+), 154 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index 3cccfd1f001c2..926cf7fa3cd4f 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -175,180 +175,195 @@ export enum StorageParameters { * This list is not exhaustive, use the `custom` method to add custom parameters. */ export namespace StorageParameters { + export function custom(key: string): StorageParameters { + return key as StorageParameters; + } +} +/** + * The compression type used by `StorageParameters.COMPRESSION_TYPE`. + */ +export enum CompressionType { /** - * The compression type used by `StorageParameters.COMPRESSION_TYPE`. + * No compression. */ - export enum CompressionType { - /** - * No compression. - */ - NONE = 'none', + NONE = 'none', - /** - * Burrows-Wheeler compression. - */ - BZIP2 = 'bzip2', + /** + * Burrows-Wheeler compression. + */ + BZIP2 = 'bzip2', - /** - * Deflate compression. - */ - GZIP = 'gzip', + /** + * Deflate compression. + */ + GZIP = 'gzip', - /** - * Compression algorithm focused on high compression and decompression speeds, rather than the maximum possible compression. - */ - SNAPPY = 'snappy', - } + /** + * Compression algorithm focused on high compression and decompression speeds, rather than the maximum possible compression. + */ + SNAPPY = 'snappy', +} +/** + * The action to assign to `INVALID_CHAR_HANDLING`. + */ +export enum InvalidCharHandlingAction { /** - * The action to assign to `StorageParameters.INVALID_CHAR_HANDLING`. + * Doesn't perform invalid character handling. */ - export enum InvalidCharHandlingAction { - /** - * Doesn't perform invalid character handling. - */ - DISABLED = 'DISABLED', + DISABLED = 'DISABLED', - /** - * Cancels queries that return data containing invalid UTF-8 values. - */ - FAIL = 'FAIL', + /** + * Cancels queries that return data containing invalid UTF-8 values. + */ + FAIL = 'FAIL', - /** - * Replaces invalid UTF-8 values with null. - */ - SET_TO_NULL = 'SET_TO_NULL', + /** + * Replaces invalid UTF-8 values with null. + */ + SET_TO_NULL = 'SET_TO_NULL', - /** - * Replaces each value in the row with null. - */ - DROP_ROW = 'DROP_ROW', + /** + * Replaces each value in the row with null. + */ + DROP_ROW = 'DROP_ROW', - /** - * Replaces the invalid character with the replacement character you specify using `StorageParameters.REPLACEMENT_CHAR`. - */ - REPLACE = 'REPLACE', - } + /** + * Replaces the invalid character with the replacement character you specify using `REPLACEMENT_CHAR`. + */ + REPLACE = 'REPLACE', +} +/** + * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + */ +export enum NumericOverflowHandlingAction { /** - * The action to assign to `StorageParameters.COLUMN_COUNT_MISMATCH_HANDLING`. + * Invalid character handling is turned off. */ - export enum NumericOverflowHandlingAction { - /** - * Invalid character handling is turned off. - */ - DISABLED = 'DISABLED', + DISABLED = 'DISABLED', - /** - * Cancel the query when the data includes invalid characters. - */ - FAIL = 'FAIL', + /** + * Cancel the query when the data includes invalid characters. + */ + FAIL = 'FAIL', - /** - * Set invalid characters to null. - */ - SET_TO_NULL = 'SET_TO_NULL', + /** + * Set invalid characters to null. + */ + SET_TO_NULL = 'SET_TO_NULL', - /** - * Set each value in the row to null. - */ - DROP_ROW = 'DROP_ROW', - } + /** + * Set each value in the row to null. + */ + DROP_ROW = 'DROP_ROW', +} + +/** + * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + */ +export enum SurplusBytesHandlingAction { + /** + * Replaces data that exceeds the column width with null. + */ + SET_TO_NULL = 'SET_TO_NULL', /** - * The action to assign to `StorageParameters.COLUMN_COUNT_MISMATCH_HANDLING`. + * Doesn't perform surplus byte handling. */ - export enum SurplusBytesHandlingAction { - /** - * Replaces data that exceeds the column width with null. - */ - SET_TO_NULL = 'SET_TO_NULL', + DISABLED = 'DISABLED', - /** - * Doesn't perform surplus byte handling. - */ - DISABLED = 'DISABLED', + /** + * Cancels queries that return data exceeding the column width. + */ + FAIL = 'FAIL', - /** - * Cancels queries that return data exceeding the column width. - */ - FAIL = 'FAIL', + /** + * Drop all rows that contain data exceeding column width. + */ + DROP_ROW = 'DROP_ROW', - /** - * Drop all rows that contain data exceeding column width. - */ - DROP_ROW = 'DROP_ROW', + /** + * Removes the characters that exceed the maximum number of characters defined for the column. + */ + TRUNCATE = 'TRUNCATE', +} - /** - * Removes the characters that exceed the maximum number of characters defined for the column. - */ - TRUNCATE = 'TRUNCATE', - } +/** + * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + */ +export enum SurplusCharHandlingAction { + /** + * Replaces data that exceeds the column width with null. + */ + SET_TO_NULL = 'SET_TO_NULL', /** - * The action to assign to `StorageParameters.COLUMN_COUNT_MISMATCH_HANDLING`. + * Doesn't perform surplus character handling. */ - export enum SurplusCharHandlingAction { - /** - * Replaces data that exceeds the column width with null. - */ - SET_TO_NULL = 'SET_TO_NULL', + DISABLED = 'DISABLED', - /** - * Doesn't perform surplus character handling. - */ - DISABLED = 'DISABLED', + /** + * Cancels queries that return data exceeding the column width. + */ + FAIL = 'FAIL', - /** - * Cancels queries that return data exceeding the column width. - */ - FAIL = 'FAIL', + /** + * Replaces each value in the row with null. + */ + DROP_ROW = 'DROP_ROW', - /** - * Replaces each value in the row with null. - */ - DROP_ROW = 'DROP_ROW', + /** + * Removes the characters that exceed the maximum number of characters defined for the column. + */ + TRUNCATE = 'TRUNCATE', +} - /** - * Removes the characters that exceed the maximum number of characters defined for the column. - */ - TRUNCATE = 'TRUNCATE', - } +/** + * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + */ +export enum ColumnCountMismatchHandlingAction { + /** + * Column count mismatch handling is turned off. + */ + DISABLED = 'DISABLED', /** - * The action to assign to `StorageParameters.COLUMN_COUNT_MISMATCH_HANDLING`. + * Fail the query if the column count mismatch is detected. */ - export enum ColumnCountMismatchHandlingAction { - /** - * Column count mismatch handling is turned off. - */ - DISABLED = 'DISABLED', + FAIL = 'FAIL', - /** - * Fail the query if the column count mismatch is detected. - */ - FAIL = 'FAIL', + /** + * Fill missing values with NULL and ignore the additional values in each row. + */ + SET_TO_NULL = 'SET_TO_NULL', - /** - * Fill missing values with NULL and ignore the additional values in each row. - */ - SET_TO_NULL = 'SET_TO_NULL', + /** + * Drop all rows that contain column count mismatch error from the scan. + */ + DROP_ROW = 'DROP_ROW', +} - /** - * Drop all rows that contain column count mismatch error from the scan. - */ - DROP_ROW = 'DROP_ROW', - } +/** + * A property of the Storage Descriptor. + */ +export interface StorageParameter { + /** + * The key of the property. If you want to use a custom key, use `StorageParameters.custom()`. + */ + readonly key: StorageParameters; /** - * A custom storage parameter. - * - * @param key the key of the parameter + * The value of the property. There are enums that are available for some of the values, or you can assign your own. */ - export function custom(key: string): string { - return key; - } + readonly value: ColumnCountMismatchHandlingAction + | CompressionType + | InvalidCharHandlingAction + | NumericOverflowHandlingAction + | SurplusBytesHandlingAction + | SurplusCharHandlingAction + | string; } export interface TableAttributes { @@ -471,23 +486,24 @@ export interface TableProps { * * declare const glueDatabase: glue.IDatabase; * const table = new glue.Table(this, 'Table', { + * storageParameters: [ + * { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: '1' }, + * { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.CompressionType.GZIP }, + * { key: glue.StorageParameters.custom('foo'), value: 'bar' }, // Will have no effect + * { key: glue.StorageParameters.custom('separatorChar'), value: ',' }, // Will describe the separator char used in the data + * ], + * // ... * database: glueDatabase, * columns: [{ * name: 'col1', * type: glue.Schema.STRING, * }], * dataFormat: glue.DataFormat.CSV, - * storageParameters: { - * glue.StorageParameters.SKIP_HEADER_LINE_COUNT: '1', - * glue.StorageParameters.COMPRESSION_TYPE: glue.StorageParameters.CompressionType.GZIP, - * glue.StorageParameters.custom('foo'): 'bar', // Will have no effect - * glue.StorageParameters.custom('separatorChar'): ',', // Will describe the separator char used in the data - * }, * }); * * @default - The parameter is not defined */ - readonly storageParameters?: { [key: string]: any }; + readonly storageParameters?: StorageParameter[]; } /** @@ -580,6 +596,11 @@ export class Table extends Resource implements ITable { */ public readonly partitionIndexes?: PartitionIndex[]; + /** + * The tables' storage descriptor properties. + */ + public readonly storageDescriptor?: StorageParameter[]; + /** * Partition indexes must be created one at a time. To avoid * race conditions, we store the resource and add dependencies @@ -602,6 +623,7 @@ export class Table extends Resource implements ITable { validateSchema(props.columns, props.partitionKeys); this.columns = props.columns; this.partitionKeys = props.partitionKeys; + this.storageDescriptor = props.storageParameters; this.compressed = props.compressed ?? false; const { bucket, encryption, encryptionKey } = createBucket(this, props); @@ -635,7 +657,10 @@ export class Table extends Resource implements ITable { serdeInfo: { serializationLibrary: props.dataFormat.serializationLibrary.className, }, - parameters: props.storageParameters, + parameters: props.storageParameters ? props.storageParameters.reduce((acc, param) => { + acc[param.key] = param.value; + return acc; + }, {} as { [key: string]: string }) : undefined, }, tableType: 'EXTERNAL_TABLE', diff --git a/packages/@aws-cdk/aws-glue/package.json b/packages/@aws-cdk/aws-glue/package.json index 1f48ff3f0d2fa..1431eaa397ec6 100644 --- a/packages/@aws-cdk/aws-glue/package.json +++ b/packages/@aws-cdk/aws-glue/package.json @@ -163,7 +163,13 @@ "docs-public-apis:@aws-cdk/aws-glue.ClassificationString.PARQUET", "docs-public-apis:@aws-cdk/aws-glue.ClassificationString.ORC", "docs-public-apis:@aws-cdk/aws-glue.ClassificationString.value", - "events-method-signature:@aws-cdk/aws-glue.Job.onStateChange" + "events-method-signature:@aws-cdk/aws-glue.Job.onStateChange", + "no-unused-type:@aws-cdk/aws-glue.ColumnCountMismatchHandlingAction", + "no-unused-type:@aws-cdk/aws-glue.CompressionType", + "no-unused-type:@aws-cdk/aws-glue.InvalidCharHandlingAction", + "no-unused-type:@aws-cdk/aws-glue.NumericOverflowHandlingAction", + "no-unused-type:@aws-cdk/aws-glue.SurplusBytesHandlingAction", + "no-unused-type:@aws-cdk/aws-glue.SurplusCharHandlingAction" ] }, "awscdkio": { diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts index 5c0b74c89ee9d..540c91b641ac4 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.table.ts @@ -100,13 +100,20 @@ new glue.Table(stack, 'MyPartitionFilteredTable', { columns, dataFormat: glue.DataFormat.JSON, enablePartitionFiltering: true, - storageParameters: { - [glue.StorageParameters.custom('separatorChar')]: ',', - [glue.StorageParameters.SKIP_HEADER_LINE_COUNT]: '1', - [glue.StorageParameters.COMPRESSION_TYPE]: glue.StorageParameters.CompressionType.GZIP, - [glue.StorageParameters.INVALID_CHAR_HANDLING]: glue.StorageParameters.InvalidCharHandlingAction.DISABLED, - [glue.StorageParameters.custom('foo')]: 'bar', - }, +}); + +new glue.Table(stack, 'MyTableWithStorageDescriptorParameters', { + database, + bucket, + tableName: 'table_with_storage_descriptor_parameters', + columns, + dataFormat: glue.DataFormat.JSON, + storageParameters: [ + { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: '1' }, + { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.CompressionType.GZIP }, + { key: glue.StorageParameters.custom('foo'), value: 'bar' }, // Will have no effect + { key: glue.StorageParameters.custom('separatorChar'), value: ',' }, // Will describe the separator char used in the data + ], }); const user = new iam.User(stack, 'MyUser'); diff --git a/packages/@aws-cdk/aws-glue/test/table.test.ts b/packages/@aws-cdk/aws-glue/test/table.test.ts index f4e2d3efdd681..cf26c41846cb5 100644 --- a/packages/@aws-cdk/aws-glue/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue/test/table.test.ts @@ -4,7 +4,7 @@ import * as kms from '@aws-cdk/aws-kms'; import * as s3 from '@aws-cdk/aws-s3'; import * as cdk from '@aws-cdk/core'; import * as glue from '../lib'; -import { PartitionIndex, StorageParameters } from '../lib'; +import { PartitionIndex } from '../lib'; import { CfnTable } from '../lib/glue.generated'; test('unpartitioned JSON table', () => { @@ -1611,18 +1611,20 @@ test('storage descriptor parameters', () => { type: glue.Schema.STRING, }], dataFormat: glue.DataFormat.JSON, - storageParameters: { - [StorageParameters.SKIP_HEADER_LINE_COUNT]: '2', - // [StorageParameters.custom('separatorChar')]: ',', - }, + storageParameters: [ + { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: '2' }, + { key: glue.StorageParameters.custom('separatorChar'), value: ',' }, + { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.CompressionType.GZIP }, + ], }); Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', { TableInput: { StorageDescriptor: { Parameters: { - 'skip.header.line.count': 2, + 'skip.header.line.count': '2', 'separatorChar': ',', + 'compression_type': 'gzip', }, }, }, From 1c83552c5a4d1d4c91d03a4ea879b046e977ce03 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 23 Mar 2023 15:33:16 +0000 Subject: [PATCH 13/32] modification: creating base parameter --- packages/@aws-cdk/aws-glue/lib/table.ts | 27 ++++-- packages/@aws-cdk/aws-glue/package.json | 8 +- .../aws-cdk-glue.assets.json | 4 +- .../aws-cdk-glue.template.json | 71 ++++++++++++++- .../integ.table.js.snapshot/manifest.json | 8 +- .../test/integ.table.js.snapshot/tree.json | 89 ++++++++++++++++++- 6 files changed, 183 insertions(+), 24 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index 926cf7fa3cd4f..ab8397ee1ab95 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -345,25 +345,36 @@ export enum ColumnCountMismatchHandlingAction { DROP_ROW = 'DROP_ROW', } +interface BaseStorageParameter { + /** + * The key of the property. If you want to use a custom key, use `StorageParameters.custom()`. + */ + readonly key: StorageParameters; +} + /** * A property of the Storage Descriptor. */ -export interface StorageParameter { +export interface CustomStorageParameter extends BaseStorageParameter { /** - * The key of the property. If you want to use a custom key, use `StorageParameters.custom()`. + * The value of the property. */ - readonly key: StorageParameters; + readonly value: string; +} +/** + * A property of the Storage Descriptor, but with a value provided by the CDK. + */ +export interface StorageParameter extends BaseStorageParameter { /** - * The value of the property. There are enums that are available for some of the values, or you can assign your own. + * The value of the property. This is a pre-defined value. */ readonly value: ColumnCountMismatchHandlingAction | CompressionType | InvalidCharHandlingAction | NumericOverflowHandlingAction | SurplusBytesHandlingAction - | SurplusCharHandlingAction - | string; + | SurplusCharHandlingAction; } export interface TableAttributes { @@ -503,7 +514,7 @@ export interface TableProps { * * @default - The parameter is not defined */ - readonly storageParameters?: StorageParameter[]; + readonly storageParameters?: (StorageParameter | CustomStorageParameter)[]; } /** @@ -599,7 +610,7 @@ export class Table extends Resource implements ITable { /** * The tables' storage descriptor properties. */ - public readonly storageDescriptor?: StorageParameter[]; + public readonly storageDescriptor?: (StorageParameter | CustomStorageParameter)[]; /** * Partition indexes must be created one at a time. To avoid diff --git a/packages/@aws-cdk/aws-glue/package.json b/packages/@aws-cdk/aws-glue/package.json index 1431eaa397ec6..1f48ff3f0d2fa 100644 --- a/packages/@aws-cdk/aws-glue/package.json +++ b/packages/@aws-cdk/aws-glue/package.json @@ -163,13 +163,7 @@ "docs-public-apis:@aws-cdk/aws-glue.ClassificationString.PARQUET", "docs-public-apis:@aws-cdk/aws-glue.ClassificationString.ORC", "docs-public-apis:@aws-cdk/aws-glue.ClassificationString.value", - "events-method-signature:@aws-cdk/aws-glue.Job.onStateChange", - "no-unused-type:@aws-cdk/aws-glue.ColumnCountMismatchHandlingAction", - "no-unused-type:@aws-cdk/aws-glue.CompressionType", - "no-unused-type:@aws-cdk/aws-glue.InvalidCharHandlingAction", - "no-unused-type:@aws-cdk/aws-glue.NumericOverflowHandlingAction", - "no-unused-type:@aws-cdk/aws-glue.SurplusBytesHandlingAction", - "no-unused-type:@aws-cdk/aws-glue.SurplusCharHandlingAction" + "events-method-signature:@aws-cdk/aws-glue.Job.onStateChange" ] }, "awscdkio": { diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index 503a49d6b195a..8931164150d66 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { "version": "30.1.0", "files": { - "553f4bc301289a3e09b1cd03c07892e4c62da6b35d057a57fe1613c230c27ef6": { + "eddae78388dd938d03998219503a0c3e6662678efa97b8099af7c02290533da6": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "553f4bc301289a3e09b1cd03c07892e4c62da6b35d057a57fe1613c230c27ef6.json", + "objectKey": "eddae78388dd938d03998219503a0c3e6662678efa97b8099af7c02290533da6.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json index cdbd13c00f014..e06c2701dd5dc 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -440,6 +440,70 @@ "has_encrypted_data": false, "partition_filtering.enabled": true }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyTableWithStorageDescriptorParametersTable1A347345": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "table_with_storage_descriptor_parameters generated by CDK", + "Name": "table_with_storage_descriptor_parameters", + "Parameters": { + "classification": "json", + "has_encrypted_data": false + }, "StorageDescriptor": { "Columns": [ { @@ -480,9 +544,10 @@ }, "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", "Parameters": { - "separatorChar": ",", - "skip.header.line.count": 2, - "foo": "bar" + "skip.header.line.count": "1", + "compression_type": "gzip", + "foo": "bar", + "separatorChar": "," }, "SerdeInfo": { "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json index e4b022f44d479..39f1708bbe923 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/553f4bc301289a3e09b1cd03c07892e4c62da6b35d057a57fe1613c230c27ef6.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/eddae78388dd938d03998219503a0c3e6662678efa97b8099af7c02290533da6.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -93,6 +93,12 @@ "data": "MyPartitionFilteredTable324BA27A" } ], + "/aws-cdk-glue/MyTableWithStorageDescriptorParameters/Table": [ + { + "type": "aws:cdk:logicalId", + "data": "MyTableWithStorageDescriptorParametersTable1A347345" + } + ], "/aws-cdk-glue/MyUser/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json index 409cc8e9d0ebf..1d183f87c0c56 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/tree.json @@ -612,6 +612,88 @@ "has_encrypted_data": false, "partition_filtering.enabled": true }, + "storageDescriptor": { + "location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "compressed": false, + "storedAsSubDirectories": false, + "columns": [ + { + "name": "col1", + "type": "string" + }, + { + "name": "col2", + "type": "string", + "comment": "col2 comment" + }, + { + "name": "col3", + "type": "array" + }, + { + "name": "col4", + "type": "map" + }, + { + "name": "col5", + "type": "struct" + } + ], + "inputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "serdeInfo": { + "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + } + }, + "tableType": "EXTERNAL_TABLE" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.CfnTable", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue.Table", + "version": "0.0.0" + } + }, + "MyTableWithStorageDescriptorParameters": { + "id": "MyTableWithStorageDescriptorParameters", + "path": "aws-cdk-glue/MyTableWithStorageDescriptorParameters", + "children": { + "Table": { + "id": "Table", + "path": "aws-cdk-glue/MyTableWithStorageDescriptorParameters/Table", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Table", + "aws:cdk:cloudformation:props": { + "catalogId": { + "Ref": "AWS::AccountId" + }, + "databaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "tableInput": { + "name": "table_with_storage_descriptor_parameters", + "description": "table_with_storage_descriptor_parameters generated by CDK", + "parameters": { + "classification": "json", + "has_encrypted_data": false + }, "storageDescriptor": { "location": { "Fn::Join": [ @@ -656,9 +738,10 @@ "serializationLibrary": "org.openx.data.jsonserde.JsonSerDe" }, "parameters": { - "separatorChar": ",", - "skip.header.line.count": 2, - "foo": "bar" + "skip.header.line.count": "1", + "compression_type": "gzip", + "foo": "bar", + "separatorChar": "," } }, "tableType": "EXTERNAL_TABLE" From 64242f32827b0440f037c22d3bdebfea592f666f Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 23 Mar 2023 15:56:58 +0000 Subject: [PATCH 14/32] modification: updating README --- packages/@aws-cdk/aws-glue/README.md | 11 +++++++---- packages/@aws-cdk/aws-glue/lib/table.ts | 4 +++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index fd358ac880697..d32a6383fc006 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -231,16 +231,19 @@ Glue tables can be configured to contain user-defined properties, to describe th declare const myBucket: s3.Bucket; declare const myDatabase: glue.Database; new glue.Table(this, 'MyTable', { + storageParameters: [ + { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: '1' }, + { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.CompressionType.GZIP }, + { key: glue.StorageParameters.custom('foo'), value: 'bar' }, // Will have no effect + { key: glue.StorageParameters.custom('separatorChar'), value: ',' }, // Will describe the separator char used in the data + ], + // ... database: myDatabase, columns: [{ name: 'col1', type: glue.Schema.STRING, }], dataFormat: glue.DataFormat.JSON, - storageParameters: { - 'skip.header.line.count': 1, - separatorChar: ',', - } }); ``` diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index ab8397ee1ab95..bd09af6ee2048 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -489,9 +489,11 @@ export interface TableProps { /** * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources. * + * The key/value pairs that are allowed to be submitted are not limited, however their functionality is not guaranteed. + * * Some keys will be auto-populated by glue crawlers, however, you can override them by specifying the key and value in this property. * - * The key/value pairs that are allowed to be submitted are not limited, however their functionality is not guaranteed. + * @see https://docs.aws.amazon.com/glue/latest/dg/table-properties-crawler.html * * @example * From d4da5b571cff41791bdeff5e13b06cca04550b00 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 23 Mar 2023 15:57:12 +0000 Subject: [PATCH 15/32] modification: updating README --- packages/@aws-cdk/aws-glue/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index d32a6383fc006..498d90d08d73b 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -228,7 +228,6 @@ new glue.Table(this, 'MyTable', { Glue tables can be configured to contain user-defined properties, to describe the physical storage of table data, through the `storageParameters` property: ```ts -declare const myBucket: s3.Bucket; declare const myDatabase: glue.Database; new glue.Table(this, 'MyTable', { storageParameters: [ From db3cc725dcd900fb9dbeb948d1c4b5dfd3e020e1 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 3 Apr 2023 14:24:20 +0000 Subject: [PATCH 16/32] modification: using alpha integ module --- packages/@aws-cdk/aws-glue/package.json | 8 ++++---- packages/@aws-cdk/aws-glue/test/integ.table.ts | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/package.json b/packages/@aws-cdk/aws-glue/package.json index b0ca1fbd4d29c..0a97367f14bed 100644 --- a/packages/@aws-cdk/aws-glue/package.json +++ b/packages/@aws-cdk/aws-glue/package.json @@ -82,15 +82,15 @@ "license": "Apache-2.0", "devDependencies": { "@aws-cdk/cdk-build-tools": "0.0.0", - "@aws-cdk/integ-runner": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", + "@aws-cdk/integ-runner": "0.0.0", + "@aws-cdk/integ-tests-alpha": "^0.0.0", "@aws-cdk/pkglint": "0.0.0", "@types/jest": "^27.5.2", - "jest": "^27.5.1", "aws-cdk-lib": "0.0.0", - "constructs": "^10.0.0" + "constructs": "^10.0.0", + "jest": "^27.5.1" }, - "dependencies": {}, "homepage": "https://github.com/aws/aws-cdk", "peerDependencies": { "aws-cdk-lib": "^0.0.0", diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts index d44846ae8350d..4376ccf53701d 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.table.ts @@ -1,6 +1,5 @@ #!/usr/bin/env node -/* eslint-disable import/no-extraneous-dependencies */ -import * as integ from '@aws-cdk/integ-tests'; +import * as integ from '@aws-cdk/integ-tests-alpha'; import * as cdk from 'aws-cdk-lib'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as kms from 'aws-cdk-lib/aws-kms'; From d14beff86c2fb8837bb8d04337bc948e8fd42439 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 4 Apr 2023 14:59:57 +0000 Subject: [PATCH 17/32] creating class like enum --- packages/@aws-cdk/aws-glue/lib/index.ts | 5 +- packages/@aws-cdk/aws-glue/lib/table.ts | 278 +----------------------- 2 files changed, 8 insertions(+), 275 deletions(-) diff --git a/packages/@aws-cdk/aws-glue/lib/index.ts b/packages/@aws-cdk/aws-glue/lib/index.ts index f4895390b3081..b6cce2dda25b4 100644 --- a/packages/@aws-cdk/aws-glue/lib/index.ts +++ b/packages/@aws-cdk/aws-glue/lib/index.ts @@ -1,11 +1,12 @@ // AWS::Glue CloudFormation Resources: +export * from './code'; export * from './connection'; export * from './data-format'; export * from './database'; export * from './job'; export * from './job-executable'; -export * from './code'; export * from './schema'; export * from './security-configuration'; -export * from './table'; \ No newline at end of file +export * from './storage-parameter'; +export * from './table'; diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index 403983923e236..d28349561b7b9 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -1,14 +1,15 @@ +import { ArnFormat, Fn, IResource, Lazy, Names, Resource, Stack } from 'aws-cdk-lib'; +import { CfnTable } from 'aws-cdk-lib/aws-glue'; import * as iam from 'aws-cdk-lib/aws-iam'; import * as kms from 'aws-cdk-lib/aws-kms'; import * as s3 from 'aws-cdk-lib/aws-s3'; -import { ArnFormat, Fn, IResource, Lazy, Names, Resource, Stack } from 'aws-cdk-lib'; import * as cr from 'aws-cdk-lib/custom-resources'; import { AwsCustomResource } from 'aws-cdk-lib/custom-resources'; import { Construct } from 'constructs'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; -import { CfnTable } from 'aws-cdk-lib/aws-glue'; import { Column } from './schema'; +import { ColumnCountMismatchHandlingAction, CompressionType, InvalidCharHandlingAction, NumericOverflowHandlingAction, StorageParameters, SurplusBytesHandlingAction, SurplusCharHandlingAction } from './storage-parameter'; /** * Properties of a Partition Index. @@ -75,276 +76,6 @@ export enum TableEncryption { CLIENT_SIDE_KMS = 'CSE-KMS' } -/** - * Properties for creating a new Table. - */ -export enum StorageParameters { - /** - * The number of rows to skip at the top of a CSV file when the table is being created. - */ - SKIP_HEADER_LINE_COUNT = 'skip.header.line.count', - - /** - * Determines whether data handling is on for the table. - */ - DATA_CLEANSING_ENABLED = 'data_cleansing_enabled', - - /** - * The type of compression used on the table, when the file name does not contain an extension. This value overrides the compression type specified through the extension. - * - * Assign this parameter to a value of `StorageParameters.CompressionType` - */ - COMPRESSION_TYPE = 'compression_type', - - /** - * Specifies the action to perform when query results contain invalid UTF-8 character values. - * - * Assign this parameter to a value of `StorageParameters.InvalidCharHandlingAction` - */ - INVALID_CHAR_HANDLING = 'invalid_char_handling', - - /** - * Specifies the replacement character to use when you set `INVALID_CHAR_HANDLING` to `REPLACE`. - */ - REPLACEMENT_CHAR = 'replacement_char', - - /** - * Specifies the action to perform when ORC data contains an integer (for example, BIGINT or int64) that is larger than the column definition (for example, SMALLINT or int16). - * - * Assign this parameter to a value of `StorageParameters.NumericOverflowHandlingAction` - */ - NUMERIC_OVERFLOW_HANDLING = 'numeric_overflow_handling', - - /** - * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARBYTE data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. - * - * Assign this parameter to a value of `StorageParameters.SurplusBytesHandlingAction` - */ - SURPLUS_BYTES_HANDLING = 'surplus_bytes_handling', - - /** - * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARCHAR, CHAR, or string data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. - * - * Assign this parameter to a value of `StorageParameters.SurplusCharHandlingAction` - */ - SURPLUS_CHAR_HANDLING = 'surplus_char_handling', - - /** - * Identifies if the file contains less or more values for a row than the number of columns specified in the external table definition. This property is only available for an uncompressed text file format. - * - * Assign this parameter to a value of `StorageParameters.ColumnCountMismatchHandling` - */ - COLUMN_COUNT_MISMATCH_HANDLING = 'column_count_mismatch_handling', - - /** - * A property that sets the numRows value for the table definition. To explicitly update an external table's statistics, set the numRows property to indicate the size of the table. Amazon Redshift doesn't analyze external tables to generate the table statistics that the query optimizer uses to generate a query plan. If table statistics aren't set for an external table, Amazon Redshift generates a query execution plan based on an assumption that external tables are the larger tables and local tables are the smaller tables. - */ - NUM_ROWS = 'num_rows', - - /** - * A property that sets number of rows to skip at the beginning of each source file. - */ - SERIALIZATION_NULL_FORMAT = 'serialization.null.format', - - /** - * A property that sets the column mapping type for tables that use ORC data format. This property is ignored for other data formats. If this property is omitted, columns are mapped by `OrcColumnMappingType.NAME` by default. - * - * Assign this parameter to a value of `StorageParameters.OrcColumnMappingType` - */ - ORC_SCHEMA_RESOLUTION = 'orc.schema.resolution', - - /** - * A property that sets whether CREATE EXTERNAL TABLE AS should write data in parallel. By default, CREATE EXTERNAL TABLE AS writes data in parallel to multiple files, according to the number of slices in the cluster. The default option is on. When 'write.parallel' is set to off, CREATE EXTERNAL TABLE AS writes to one or more data files serially onto Amazon S3. This table property also applies to any subsequent INSERT statement into the same external table. - * - * Assign this parameter to a value of `StorageParameters.WriteParallel` - */ - WRITE_PARALLEL = 'write.parallel', - - /** - * A property that sets the maximum size (in MB) of each file written to Amazon S3 by CREATE EXTERNAL TABLE AS. The size must be a valid integer between 5 and 6200. The default maximum file size is 6,200 MB. This table property also applies to any subsequent INSERT statement into the same external table. - */ - WRITE_MAX_FILESIZE_MB = 'write.maxfilesize.mb', - - /** - * You can specify an AWS Key Management Service key to enable Server–Side Encryption (SSE) for Amazon S3 objects. - */ - WRITE_KMS_KEY_ID = 'write.kms.key.id', -} - -/** - * This list is not exhaustive, use the `custom` method to add custom parameters. - */ -export namespace StorageParameters { - export function custom(key: string): StorageParameters { - return key as StorageParameters; - } -} - -/** - * The compression type used by `StorageParameters.COMPRESSION_TYPE`. - */ -export enum CompressionType { - /** - * No compression. - */ - NONE = 'none', - - /** - * Burrows-Wheeler compression. - */ - BZIP2 = 'bzip2', - - /** - * Deflate compression. - */ - GZIP = 'gzip', - - /** - * Compression algorithm focused on high compression and decompression speeds, rather than the maximum possible compression. - */ - SNAPPY = 'snappy', -} - -/** - * The action to assign to `INVALID_CHAR_HANDLING`. - */ -export enum InvalidCharHandlingAction { - /** - * Doesn't perform invalid character handling. - */ - DISABLED = 'DISABLED', - - /** - * Cancels queries that return data containing invalid UTF-8 values. - */ - FAIL = 'FAIL', - - /** - * Replaces invalid UTF-8 values with null. - */ - SET_TO_NULL = 'SET_TO_NULL', - - /** - * Replaces each value in the row with null. - */ - DROP_ROW = 'DROP_ROW', - - /** - * Replaces the invalid character with the replacement character you specify using `REPLACEMENT_CHAR`. - */ - REPLACE = 'REPLACE', -} - -/** - * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. - */ -export enum NumericOverflowHandlingAction { - /** - * Invalid character handling is turned off. - */ - DISABLED = 'DISABLED', - - /** - * Cancel the query when the data includes invalid characters. - */ - FAIL = 'FAIL', - - /** - * Set invalid characters to null. - */ - SET_TO_NULL = 'SET_TO_NULL', - - /** - * Set each value in the row to null. - */ - DROP_ROW = 'DROP_ROW', -} - -/** - * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. - */ -export enum SurplusBytesHandlingAction { - /** - * Replaces data that exceeds the column width with null. - */ - SET_TO_NULL = 'SET_TO_NULL', - - /** - * Doesn't perform surplus byte handling. - */ - DISABLED = 'DISABLED', - - /** - * Cancels queries that return data exceeding the column width. - */ - FAIL = 'FAIL', - - /** - * Drop all rows that contain data exceeding column width. - */ - DROP_ROW = 'DROP_ROW', - - /** - * Removes the characters that exceed the maximum number of characters defined for the column. - */ - TRUNCATE = 'TRUNCATE', -} - -/** - * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. - */ -export enum SurplusCharHandlingAction { - /** - * Replaces data that exceeds the column width with null. - */ - SET_TO_NULL = 'SET_TO_NULL', - - /** - * Doesn't perform surplus character handling. - */ - DISABLED = 'DISABLED', - - /** - * Cancels queries that return data exceeding the column width. - */ - FAIL = 'FAIL', - - /** - * Replaces each value in the row with null. - */ - DROP_ROW = 'DROP_ROW', - - /** - * Removes the characters that exceed the maximum number of characters defined for the column. - */ - TRUNCATE = 'TRUNCATE', -} - -/** - * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. - */ -export enum ColumnCountMismatchHandlingAction { - /** - * Column count mismatch handling is turned off. - */ - DISABLED = 'DISABLED', - - /** - * Fail the query if the column count mismatch is detected. - */ - FAIL = 'FAIL', - - /** - * Fill missing values with NULL and ignore the additional values in each row. - */ - SET_TO_NULL = 'SET_TO_NULL', - - /** - * Drop all rows that contain column count mismatch error from the scan. - */ - DROP_ROW = 'DROP_ROW', -} - interface BaseStorageParameter { /** * The key of the property. If you want to use a custom key, use `StorageParameters.custom()`. @@ -671,7 +402,8 @@ export class Table extends Resource implements ITable { serializationLibrary: props.dataFormat.serializationLibrary.className, }, parameters: props.storageParameters ? props.storageParameters.reduce((acc, param) => { - acc[param.key] = param.value; + const key = param.key.key; + acc[key] = param.value; return acc; }, {} as { [key: string]: string }) : undefined, }, From cc812d30d6cb1949fe75d0d86d76e501bda0c76b Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 4 Apr 2023 15:00:16 +0000 Subject: [PATCH 18/32] class like enum --- .../aws-glue/lib/storage-parameter.ts | 269 ++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 packages/@aws-cdk/aws-glue/lib/storage-parameter.ts diff --git a/packages/@aws-cdk/aws-glue/lib/storage-parameter.ts b/packages/@aws-cdk/aws-glue/lib/storage-parameter.ts new file mode 100644 index 0000000000000..43d718698bd58 --- /dev/null +++ b/packages/@aws-cdk/aws-glue/lib/storage-parameter.ts @@ -0,0 +1,269 @@ +/** + * The compression type used by `StorageParameters.COMPRESSION_TYPE`. + */ +export enum CompressionType { + /** + * No compression. + */ + NONE = 'none', + + /** + * Burrows-Wheeler compression. + */ + BZIP2 = 'bzip2', + + /** + * Deflate compression. + */ + GZIP = 'gzip', + + /** + * Compression algorithm focused on high compression and decompression speeds, rather than the maximum possible compression. + */ + SNAPPY = 'snappy', +} + +/** + * The action to assign to `INVALID_CHAR_HANDLING`. + */ +export enum InvalidCharHandlingAction { + /** + * Doesn't perform invalid character handling. + */ + DISABLED = 'DISABLED', + + /** + * Cancels queries that return data containing invalid UTF-8 values. + */ + FAIL = 'FAIL', + + /** + * Replaces invalid UTF-8 values with null. + */ + SET_TO_NULL = 'SET_TO_NULL', + + /** + * Replaces each value in the row with null. + */ + DROP_ROW = 'DROP_ROW', + + /** + * Replaces the invalid character with the replacement character you specify using `REPLACEMENT_CHAR`. + */ + REPLACE = 'REPLACE', +} + +/** + * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + */ +export enum NumericOverflowHandlingAction { + /** + * Invalid character handling is turned off. + */ + DISABLED = 'DISABLED', + + /** + * Cancel the query when the data includes invalid characters. + */ + FAIL = 'FAIL', + + /** + * Set invalid characters to null. + */ + SET_TO_NULL = 'SET_TO_NULL', + + /** + * Set each value in the row to null. + */ + DROP_ROW = 'DROP_ROW', +} + +/** + * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + */ +export enum SurplusBytesHandlingAction { + /** + * Replaces data that exceeds the column width with null. + */ + SET_TO_NULL = 'SET_TO_NULL', + + /** + * Doesn't perform surplus byte handling. + */ + DISABLED = 'DISABLED', + + /** + * Cancels queries that return data exceeding the column width. + */ + FAIL = 'FAIL', + + /** + * Drop all rows that contain data exceeding column width. + */ + DROP_ROW = 'DROP_ROW', + + /** + * Removes the characters that exceed the maximum number of characters defined for the column. + */ + TRUNCATE = 'TRUNCATE', +} + +/** + * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + */ +export enum SurplusCharHandlingAction { + /** + * Replaces data that exceeds the column width with null. + */ + SET_TO_NULL = 'SET_TO_NULL', + + /** + * Doesn't perform surplus character handling. + */ + DISABLED = 'DISABLED', + + /** + * Cancels queries that return data exceeding the column width. + */ + FAIL = 'FAIL', + + /** + * Replaces each value in the row with null. + */ + DROP_ROW = 'DROP_ROW', + + /** + * Removes the characters that exceed the maximum number of characters defined for the column. + */ + TRUNCATE = 'TRUNCATE', +} + +/** + * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + */ +export enum ColumnCountMismatchHandlingAction { + /** + * Column count mismatch handling is turned off. + */ + DISABLED = 'DISABLED', + + /** + * Fail the query if the column count mismatch is detected. + */ + FAIL = 'FAIL', + + /** + * Fill missing values with NULL and ignore the additional values in each row. + */ + SET_TO_NULL = 'SET_TO_NULL', + + /** + * Drop all rows that contain column count mismatch error from the scan. + */ + DROP_ROW = 'DROP_ROW', +} + +export class StorageParameters { + /** + * The number of rows to skip at the top of a CSV file when the table is being created. + */ + public static readonly SKIP_HEADER_LINE_COUNT = new StorageParameters('skip.header.line.count'); + + /** + * Determines whether data handling is on for the table. + */ + public static readonly DATA_CLEANSING_ENABLED = new StorageParameters('data_cleansing_enabled'); + + /** + * The type of compression used on the table, when the file name does not contain an extension. This value overrides the compression type specified through the extension. + * + * Assign this parameter to a value of `StorageParameters.CompressionType` + */ + public static readonly COMPRESSION_TYPE = new StorageParameters('compression_type'); + + /** + * Specifies the action to perform when query results contain invalid UTF-8 character values. + * + * Assign this parameter to a value of `StorageParameters.InvalidCharHandlingAction` + */ + public static readonly INVALID_CHAR_HANDLING = new StorageParameters('invalid_char_handling'); + + /** + * Specifies the replacement character to use when you set `INVALID_CHAR_HANDLING` to `REPLACE`. + */ + public static readonly REPLACEMENT_CHAR = new StorageParameters('replacement_char'); + + /** + * Specifies the action to perform when ORC data contains an integer (for example, BIGINT or int64) that is larger than the column definition (for example, SMALLINT or int16). + * + * Assign this parameter to a value of `StorageParameters.NumericOverflowHandlingAction` + */ + public static readonly NUMERIC_OVERFLOW_HANDLING = new StorageParameters('numeric_overflow_handling'); + + /** + * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARBYTE data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. + * + * Assign this parameter to a value of `StorageParameters.SurplusBytesHandlingAction` + */ + public static readonly SURPLUS_BYTES_HANDLING = new StorageParameters('surplus_bytes_handling'); + + /** + * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARCHAR, CHAR, or string data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. + * + * Assign this parameter to a value of `StorageParameters.SurplusCharHandlingAction` + */ + public static readonly SURPLUS_CHAR_HANDLING = new StorageParameters('surplus_char_handling'); + + /** + * Identifies if the file contains less or more values for a row than the number of columns specified in the external table definition. This property is only available for an uncompressed text file format. + * + * Assign this parameter to a value of `StorageParameters.ColumnCountMismatchHandling` + */ + public static readonly COLUMN_COUNT_MISMATCH_HANDLING = new StorageParameters('column_count_mismatch_handling'); + + /** + * A property that sets the numRows value for the table definition. To explicitly update an external table's statistics, set the numRows property to indicate the size of the table. Amazon Redshift doesn't analyze external tables to generate the table statistics that the query optimizer uses to generate a query plan. If table statistics aren't set for an external table, Amazon Redshift generates a query execution plan based on an assumption that external tables are the larger tables and local tables are the smaller tables. + */ + public static readonly NUM_ROWS = new StorageParameters('num_rows'); + + /** + * A property that sets number of rows to skip at the beginning of each source file. + * + * Assign this parameter to a value of `StorageParameters.SerializationNullFormat` + */ + public static readonly SERIALIZATION_NULL_FORMAT = new StorageParameters('serialization.null.format'); + + /** + * A property that sets the column mapping type for tables that use ORC data format. This property is ignored for other data formats. If this property is omitted, columns are mapped by `OrcColumnMappingType.NAME` by default. + * + * Assign this parameter to a value of `StorageParameters.OrcColumnMappingType` + */ + public static readonly ORC_SCHEMA_RESOLUTION = new StorageParameters('orc.schema.resolution'); + + /** + * A property that sets whether CREATE EXTERNAL TABLE AS should write data in parallel. By default, CREATE EXTERNAL TABLE AS writes data in parallel to multiple files, according to the number of slices in the cluster. The default option is on. When 'write.parallel' is set to off, CREATE EXTERNAL TABLE AS writes to one or more data files serially onto Amazon S3. This table property also applies to any subsequent INSERT statement into the same external table. + * + * Assign this parameter to a value of `StorageParameters.WriteParallel` + */ + public static readonly WRITE_PARALLEL = new StorageParameters('write.parallel'); + + /** + * A property that sets the maximum size (in MB) of each file written to Amazon S3 by CREATE EXTERNAL TABLE AS. The size must be a valid integer between 5 and 6200. The default maximum file size is 6,200 MB. This table property also applies to any subsequent INSERT statement into the same external table. + */ + public static readonly WRITE_MAX_FILESIZE_MB = new StorageParameters('write.maxfilesize.mb'); + + /** + * You can specify an AWS Key Management Service key to enable Server–Side Encryption (SSE) for Amazon S3 objects. + */ + public static readonly WRITE_KMS_KEY_ID = new StorageParameters('write.kms.key.id'); + + /** + * A custom storage parameter. + * @param key The key of the parameter. + */ + public static custom(key: string) { + return new StorageParameters(key); + } + + protected constructor(public readonly key: string) {} +} \ No newline at end of file From bc6db051b2b08b2567dc30dfa5aad2c643dd74e6 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 13 Apr 2023 10:32:03 +0000 Subject: [PATCH 19/32] removing extra value --- packages/@aws-cdk/aws-glue/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index 9c3f697b498f5..6936262258517 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -227,7 +227,6 @@ new glue.Table(this, 'MyTable', { storageParameters: [ { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: '1' }, { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.CompressionType.GZIP }, - { key: glue.StorageParameters.custom('foo'), value: 'bar' }, // Will have no effect { key: glue.StorageParameters.custom('separatorChar'), value: ',' }, // Will describe the separator char used in the data ], // ... From bd7a5d6df2bd98fe120e7d3a2eb5bc8ad479e1f4 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 13 Apr 2023 11:12:16 +0000 Subject: [PATCH 20/32] comment update --- packages/@aws-cdk/aws-glue/lib/storage-parameter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue/lib/storage-parameter.ts b/packages/@aws-cdk/aws-glue/lib/storage-parameter.ts index 43d718698bd58..c506328a2d9a2 100644 --- a/packages/@aws-cdk/aws-glue/lib/storage-parameter.ts +++ b/packages/@aws-cdk/aws-glue/lib/storage-parameter.ts @@ -24,7 +24,7 @@ export enum CompressionType { } /** - * The action to assign to `INVALID_CHAR_HANDLING`. + * Specifies the action to perform when query results contain invalid UTF-8 character values. */ export enum InvalidCharHandlingAction { /** From 715dc36e51eec7d99eb9954f17f78f800d231fe6 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Thu, 13 Apr 2023 15:38:44 +0000 Subject: [PATCH 21/32] changing to use enum like class --- packages/@aws-cdk/aws-glue-alpha/README.md | 6 +-- .../aws-glue-alpha/lib/storage-parameter.ts | 54 +++++++++++++++++++ packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 42 ++++----------- .../aws-glue-alpha/test/integ.table.ts | 8 +-- .../aws-glue-alpha/test/table.test.ts | 6 +-- 5 files changed, 75 insertions(+), 41 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index 6936262258517..5895f3e59a464 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -225,9 +225,9 @@ Glue tables can be configured to contain user-defined properties, to describe th declare const myDatabase: glue.Database; new glue.Table(this, 'MyTable', { storageParameters: [ - { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: '1' }, - { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.CompressionType.GZIP }, - { key: glue.StorageParameters.custom('separatorChar'), value: ',' }, // Will describe the separator char used in the data + { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: glue.StorageParameterValue.custom('1') }, + { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.StorageParameterValue.compressionType(glue.CompressionType.GZIP) }, + { key: glue.StorageParameters.custom('separatorChar'), value: glue.StorageParameterValue.custom(',') }, // Will describe the separator char used in the data ], // ... database: myDatabase, diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts b/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts index c506328a2d9a2..c83e15dd765a0 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts @@ -163,6 +163,60 @@ export enum ColumnCountMismatchHandlingAction { DROP_ROW = 'DROP_ROW', } +export class StorageParameterValue { + /** + * Assigns a value of `CompressionType` + */ + public static compressionType(value: CompressionType): StorageParameterValue { + return new StorageParameterValue(value); + } + + /** + * Assigns a value of `InvalidCharHandlingAction` + */ + public static invalidCharHandling(value: InvalidCharHandlingAction): StorageParameterValue { + return new StorageParameterValue(value); + } + + /** + * Assigns a value of `NumericOverflowHandlingAction` + */ + public static numericOverflowHandling(value: NumericOverflowHandlingAction): StorageParameterValue { + return new StorageParameterValue(value); + } + + /** + * Assigns a value of `SurplusBytesHandlingAction` + */ + public static surplusBytesHandling(value: SurplusBytesHandlingAction): StorageParameterValue { + return new StorageParameterValue(value); + } + + /** + * Assigns a value of `SurplusCharHandlingAction` + */ + public static surplusCharHandling(value: SurplusCharHandlingAction): StorageParameterValue { + return new StorageParameterValue(value); + } + + /** + * Assigns a value of `ColumnCountMismatchHandlingAction` + */ + public static columnCountMismatchHandling(value: ColumnCountMismatchHandlingAction): StorageParameterValue { + return new StorageParameterValue(value); + } + + /** + * A custom storage parameter value that can be used to assign to any storage parameter. + * @param value - The value of the storage parameter. + */ + public static custom(value: string): StorageParameterValue { + return new StorageParameterValue(value); + } + + protected constructor(public readonly value: string) {} +} + export class StorageParameters { /** * The number of rows to skip at the top of a CSV file when the table is being created. diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index 7cebdb04e9045..d7be9f01d44a6 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -9,7 +9,7 @@ import { Construct } from 'constructs'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; import { Column } from './schema'; -import { ColumnCountMismatchHandlingAction, CompressionType, InvalidCharHandlingAction, NumericOverflowHandlingAction, StorageParameters, SurplusBytesHandlingAction, SurplusCharHandlingAction } from './storage-parameter'; +import { StorageParameterValue, StorageParameters } from './storage-parameter'; /** * Properties of a Partition Index. @@ -74,36 +74,16 @@ export enum TableEncryption { CLIENT_SIDE_KMS = 'CSE-KMS' } -interface BaseStorageParameter { +export interface StorageParameter { /** * The key of the property. If you want to use a custom key, use `StorageParameters.custom()`. */ readonly key: StorageParameters; -} -/** - * A property of the Storage Descriptor. - */ -export interface CustomStorageParameter extends BaseStorageParameter { - /** - * The value of the property. - */ - readonly value: string; -} - -/** - * A property of the Storage Descriptor, but with a value provided by the CDK. - */ -export interface StorageParameter extends BaseStorageParameter { /** - * The value of the property. This is a pre-defined value. + * The value of the property. If you want to use a custom value, use `StorageParameter.custom()`. */ - readonly value: ColumnCountMismatchHandlingAction - | CompressionType - | InvalidCharHandlingAction - | NumericOverflowHandlingAction - | SurplusBytesHandlingAction - | SurplusCharHandlingAction; + readonly value: StorageParameterValue; } export interface TableAttributes { @@ -229,10 +209,10 @@ export interface TableProps { * declare const glueDatabase: glue.IDatabase; * const table = new glue.Table(this, 'Table', { * storageParameters: [ - * { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: '1' }, - * { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.CompressionType.GZIP }, - * { key: glue.StorageParameters.custom('foo'), value: 'bar' }, // Will have no effect - * { key: glue.StorageParameters.custom('separatorChar'), value: ',' }, // Will describe the separator char used in the data + * { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: glue.StorageParameterValue.custom('1') }, + * { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.StorageParameterValue.compressionType(glue.CompressionType.GZIP) }, + * { key: glue.StorageParameters.custom('foo'), value: glue.StorageParameterValue.custom('bar') }, // Will have no effect + * { key: glue.StorageParameters.custom('separatorChar'), value: glue.StorageParameterValue.custom(',') }, // Will describe the separator char used in the data * ], * // ... * database: glueDatabase, @@ -245,7 +225,7 @@ export interface TableProps { * * @default - The parameter is not defined */ - readonly storageParameters?: (StorageParameter | CustomStorageParameter)[]; + readonly storageParameters?: StorageParameter[]; } /** @@ -341,7 +321,7 @@ export class Table extends Resource implements ITable { /** * The tables' storage descriptor properties. */ - public readonly storageDescriptor?: (StorageParameter | CustomStorageParameter)[]; + public readonly storageDescriptor?: StorageParameter[]; /** * Partition indexes must be created one at a time. To avoid @@ -401,7 +381,7 @@ export class Table extends Resource implements ITable { }, parameters: props.storageParameters ? props.storageParameters.reduce((acc, param) => { const key = param.key.key; - acc[key] = param.value; + acc[key] = param.value.value; return acc; }, {} as { [key: string]: string }) : undefined, }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts index 4376ccf53701d..2e4add123f953 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts @@ -109,10 +109,10 @@ new glue.Table(stack, 'MyTableWithStorageDescriptorParameters', { columns, dataFormat: glue.DataFormat.JSON, storageParameters: [ - { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: '1' }, - { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.CompressionType.GZIP }, - { key: glue.StorageParameters.custom('foo'), value: 'bar' }, // Will have no effect - { key: glue.StorageParameters.custom('separatorChar'), value: ',' }, // Will describe the separator char used in the data + { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: glue.StorageParameterValue.custom('1') }, + { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.StorageParameterValue.compressionType(glue.CompressionType.GZIP) }, + { key: glue.StorageParameters.custom('foo'), value: glue.StorageParameterValue.custom('bar') }, // Will have no effect + { key: glue.StorageParameters.custom('separatorChar'), value: glue.StorageParameterValue.custom(',') }, // Will describe the separator char used in the data ], }); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts index 29356c1b410b6..131190fc0f811 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts @@ -1612,9 +1612,9 @@ test('storage descriptor parameters', () => { }], dataFormat: glue.DataFormat.JSON, storageParameters: [ - { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: '2' }, - { key: glue.StorageParameters.custom('separatorChar'), value: ',' }, - { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.CompressionType.GZIP }, + { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: glue.StorageParameterValue.custom('2') }, + { key: glue.StorageParameters.custom('separatorChar'), value: glue.StorageParameterValue.custom(',') }, + { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.StorageParameterValue.compressionType(glue.CompressionType.GZIP) }, ], }); From c3c3b78a0fc3c1f5d4de6f3169b3aa810152cf5c Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 14 Apr 2023 14:30:10 +0000 Subject: [PATCH 22/32] integ --- .../aws-cdk-glue.assets.json | 8 +- .../aws-cdk-glue.template.json | 1736 +++++++++-------- ...efaultTestDeployAssert8BFB5B70.assets.json | 2 +- .../test/integ.table.js.snapshot/integ.json | 8 +- .../integ.table.js.snapshot/manifest.json | 14 +- .../test/integ.table.js.snapshot/tree.json | 106 +- 6 files changed, 961 insertions(+), 913 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index 9a13d80727722..4b14d29b52f16 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { - "version": "30.1.0", + "version": "31.0.0", "files": { - "eddae78388dd938d03998219503a0c3e6662678efa97b8099af7c02290533da6": { + "575ff5e7aa88089b2f961dec32c432bf462cbc875db69d584d780e9404ae8176": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,11 +9,11 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "eddae78388dd938d03998219503a0c3e6662678efa97b8099af7c02290533da6.json", + "objectKey": "575ff5e7aa88089b2f961dec32c432bf462cbc875db69d584d780e9404ae8176.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } } }, "dockerImages": {} -} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json index fd18cae8f1d9a..55e8f24d3b187 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -1,887 +1,917 @@ { - "Resources": { - "DataBucketE3889A50": { - "Type": "AWS::S3::Bucket", - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" + "Resources": { + "DataBucketE3889A50": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "MyDatabase1E2517DB": { + "Type": "AWS::Glue::Database", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyDatabase1E2517DB": { - "Type": "AWS::Glue::Database", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseInput": { - "Name": "my_database" - } - } + "DatabaseInput": { + "Name": "my_database" + } + } + }, + "AVROTable58646ABF": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "AVROTable58646ABF": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "avro_table generated by CDK", - "Name": "avro_table", - "Parameters": { - "classification": "avro", - "has_encrypted_data": true - }, - "PartitionKeys": [ - { - "Name": "year", - "Type": "smallint" - } - ], - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.apache.hadoop.hive.serde2.avro.AvroSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } - } + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" }, - "CSVTableE499CABA": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "csv_table generated by CDK", - "Name": "csv_table", - "Parameters": { - "classification": "csv", - "has_encrypted_data": true - }, - "PartitionKeys": [ - { - "Name": "year", - "Type": "smallint" - } - ], - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.apache.hadoop.hive.serde2.OpenCSVSerde" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } + "TableInput": { + "Description": "avro_table generated by CDK", + "Name": "avro_table", + "Parameters": { + "classification": "avro", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.apache.hadoop.hive.serde2.avro.AvroSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "CSVTableE499CABA": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "JSONTable00348F1D": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "json_table generated by CDK", - "Name": "json_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": true - }, - "PartitionKeys": [ - { - "Name": "year", - "Type": "smallint" - } - ], - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } - } + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" }, - "ParquetTableE84E985F": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "parquet_table generated by CDK", - "Name": "parquet_table", - "Parameters": { - "classification": "parquet", - "has_encrypted_data": true - }, - "PartitionKeys": [ - { - "Name": "year", - "Type": "smallint" - } - ], - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } + "TableInput": { + "Description": "csv_table generated by CDK", + "Name": "csv_table", + "Parameters": { + "classification": "csv", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" } - }, - "MyKey6AB29FA6": { - "Type": "AWS::KMS::Key", - "Properties": { - "KeyPolicy": { - "Statement": [ - { - "Action": "kms:*", - "Effect": "Allow", - "Principal": { - "AWS": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::", - { - "Ref": "AWS::AccountId" - }, - ":root" - ] - ] - } - }, - "Resource": "*" - } - ], - "Version": "2012-10-17" - } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.apache.hadoop.hive.serde2.OpenCSVSerde" }, - "UpdateReplacePolicy": "Delete", - "DeletionPolicy": "Delete" + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "JSONTable00348F1D": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyEncryptedTableBucket7B28486D": { - "Type": "AWS::S3::Bucket", - "Properties": { - "BucketEncryption": { - "ServerSideEncryptionConfiguration": [ - { - "ServerSideEncryptionByDefault": { - "KMSMasterKeyID": { - "Fn::GetAtt": ["MyKey6AB29FA6", "Arn"] - }, - "SSEAlgorithm": "aws:kms" - } - } - ] - } + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "json_table generated by CDK", + "Name": "json_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" + } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] }, - "UpdateReplacePolicy": "Retain", - "DeletionPolicy": "Retain" + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "ParquetTableE84E985F": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyEncryptedTable981A88C6": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "my_encrypted_table generated by CDK", - "Name": "my_encrypted_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": true - }, - "PartitionKeys": [ - { - "Name": "year", - "Type": "smallint" - } - ], - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "MyEncryptedTableBucket7B28486D" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "parquet_table generated by CDK", + "Name": "parquet_table", + "Parameters": { + "classification": "parquet", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" + } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyKey6AB29FA6": { + "Type": "AWS::KMS::Key", + "Properties": { + "KeyPolicy": { + "Statement": [ + { + "Action": "kms:*", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root" + ] + ] } + }, + "Resource": "*" } - }, - "MyPartitionFilteredTable324BA27A": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" + ], + "Version": "2012-10-17" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "MyEncryptedTableBucket7B28486D": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + { + "ServerSideEncryptionByDefault": { + "KMSMasterKeyID": { + "Fn::GetAtt": [ + "MyKey6AB29FA6", + "Arn" + ] }, - "TableInput": { - "Description": "partition_filtered_table generated by CDK", - "Name": "partition_filtered_table", - "Parameters": { - "classification": "json", - "has_encrypted_data": true, - "partition_filtering.enabled": true - }, - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } + "SSEAlgorithm": "aws:kms" + } } + ] + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "MyEncryptedTable981A88C6": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyTableWithStorageDescriptorParametersTable1A347345": { - "Type": "AWS::Glue::Table", - "Properties": { - "CatalogId": { - "Ref": "AWS::AccountId" - }, - "DatabaseName": { - "Ref": "MyDatabase1E2517DB" - }, - "TableInput": { - "Description": "table_with_storage_descriptor_parameters generated by CDK", - "Name": "table_with_storage_descriptor_parameters", - "Parameters": { - "classification": "json", - "has_encrypted_data": false - }, - "StorageDescriptor": { - "Columns": [ - { - "Name": "col1", - "Type": "string" - }, - { - "Comment": "col2 comment", - "Name": "col2", - "Type": "string" - }, - { - "Name": "col3", - "Type": "array" - }, - { - "Name": "col4", - "Type": "map" - }, - { - "Name": "col5", - "Type": "struct" - } - ], - "Compressed": false, - "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", - "Location": { - "Fn::Join": [ - "", - [ - "s3://", - { - "Ref": "DataBucketE3889A50" - }, - "/" - ] - ] - }, - "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", - "Parameters": { - "skip.header.line.count": "1", - "compression_type": "gzip", - "foo": "bar", - "separatorChar": "," - }, - "SerdeInfo": { - "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" - }, - "StoredAsSubDirectories": false - }, - "TableType": "EXTERNAL_TABLE" - } + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "my_encrypted_table generated by CDK", + "Name": "my_encrypted_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "MyEncryptedTableBucket7B28486D" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyPartitionFilteredTable324BA27A": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "MyUserDC45028B": { - "Type": "AWS::IAM::User" + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" }, - "MyUserDefaultPolicy7B897426": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "glue:BatchCreatePartition", - "glue:BatchDeletePartition", - "glue:BatchGetPartition", - "glue:CreatePartition", - "glue:DeletePartition", - "glue:GetPartition", - "glue:GetPartitions", - "glue:GetTable", - "glue:GetTableVersion", - "glue:GetTableVersions", - "glue:GetTables", - "glue:UpdatePartition" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":glue:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":table/", - { - "Ref": "MyDatabase1E2517DB" - }, - "/", - { - "Ref": "CSVTableE499CABA" - } - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":glue:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":table/", - { - "Ref": "MyDatabase1E2517DB" - }, - "/", - { - "Ref": "MyEncryptedTable981A88C6" - } - ] - ] - } - ] - }, - { - "Action": [ - "s3:Abort*", - "s3:DeleteObject*", - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] - }, - { - "Fn::GetAtt": ["MyEncryptedTableBucket7B28486D", "Arn"] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] - }, - "/*" - ] - ] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": ["MyEncryptedTableBucket7B28486D", "Arn"] - }, - "/*" - ] - ] - } - ] - }, - { - "Action": [ - "kms:Decrypt", - "kms:DescribeKey", - "kms:Encrypt", - "kms:GenerateDataKey*", - "kms:ReEncrypt*" - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": ["MyKey6AB29FA6", "Arn"] - } - } - ], - "Version": "2012-10-17" - }, - "PolicyName": "MyUserDefaultPolicy7B897426", - "Users": [ - { - "Ref": "MyUserDC45028B" - } + "TableInput": { + "Description": "partition_filtered_table generated by CDK", + "Name": "partition_filtered_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": true, + "partition_filtering.enabled": true + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" ] - } + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyTableWithStorageDescriptorParametersTable1A347345": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" }, - "AnotherUser254B09E3": { - "Type": "AWS::IAM::User" + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" }, - "AnotherUserDefaultPolicyDBDB9923": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "glue:BatchCreatePartition", - "glue:BatchDeletePartition", - "glue:BatchGetPartition", - "glue:CreatePartition", - "glue:DeletePartition", - "glue:GetPartition", - "glue:GetPartitions", - "glue:GetTable", - "glue:GetTableVersion", - "glue:GetTableVersions", - "glue:GetTables", - "glue:UpdatePartition" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":glue:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":table/", - { - "Ref": "MyDatabase1E2517DB" - }, - "/", - { - "Ref": "AVROTable58646ABF" - } - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":glue:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":table/", - { - "Ref": "MyDatabase1E2517DB" - }, - "/", - { - "Ref": "JSONTable00348F1D" - } - ] - ] - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":glue:", - { - "Ref": "AWS::Region" - }, - ":", - { - "Ref": "AWS::AccountId" - }, - ":table/", - { - "Ref": "MyDatabase1E2517DB" - }, - "/", - { - "Ref": "ParquetTableE84E985F" - } - ] - ] - } - ] - }, - { - "Action": [ - "s3:Abort*", - "s3:DeleteObject*", - "s3:GetBucket*", - "s3:GetObject*", - "s3:List*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging" - ], - "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] - }, - "/*" - ] - ] - } - ] - } - ], - "Version": "2012-10-17" + "TableInput": { + "Description": "table_with_storage_descriptor_parameters generated by CDK", + "Name": "table_with_storage_descriptor_parameters", + "Parameters": { + "classification": "json", + "has_encrypted_data": true + }, + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Comment": "col2 comment", + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "array" + }, + { + "Name": "col4", + "Type": "map" + }, + { + "Name": "col5", + "Type": "struct" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "Parameters": { + "skip.header.line.count": "1", + "compression_type": "gzip", + "foo": "bar", + "separatorChar": "," + }, + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "MyUserDC45028B": { + "Type": "AWS::IAM::User" + }, + "MyUserDefaultPolicy7B897426": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "glue:BatchCreatePartition", + "glue:BatchDeletePartition", + "glue:BatchGetPartition", + "glue:CreatePartition", + "glue:DeletePartition", + "glue:GetPartition", + "glue:GetPartitions", + "glue:GetTable", + "glue:GetTableVersion", + "glue:GetTableVersions", + "glue:GetTables", + "glue:UpdatePartition" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "CSVTableE499CABA" + } + ] + ] }, - "PolicyName": "AnotherUserDefaultPolicyDBDB9923", - "Users": [ - { - "Ref": "AnotherUser254B09E3" - } + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "MyEncryptedTable981A88C6" + } + ] + ] + } + ] + }, + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" + ] + }, + { + "Fn::GetAtt": [ + "MyEncryptedTableBucket7B28486D", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" + ] + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "MyEncryptedTableBucket7B28486D", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey*", + "kms:ReEncrypt*" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "MyKey6AB29FA6", + "Arn" ] + } } - } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyUserDefaultPolicy7B897426", + "Users": [ + { + "Ref": "MyUserDC45028B" + } + ] + } }, - "Parameters": { - "BootstrapVersion": { - "Type": "AWS::SSM::Parameter::Value", - "Default": "/cdk-bootstrap/hnb659fds/version", - "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" - } + "AnotherUser254B09E3": { + "Type": "AWS::IAM::User" }, - "Rules": { - "CheckBootstrapVersion": { - "Assertions": [ + "AnotherUserDefaultPolicyDBDB9923": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "glue:BatchCreatePartition", + "glue:BatchDeletePartition", + "glue:BatchGetPartition", + "glue:CreatePartition", + "glue:DeletePartition", + "glue:GetPartition", + "glue:GetPartitions", + "glue:GetTable", + "glue:GetTableVersion", + "glue:GetTableVersions", + "glue:GetTables", + "glue:UpdatePartition" + ], + "Effect": "Allow", + "Resource": [ { - "Assert": { - "Fn::Not": [ - { - "Fn::Contains": [ - ["1", "2", "3", "4", "5"], - { - "Ref": "BootstrapVersion" - } - ] - } + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "AVROTable58646ABF" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "JSONTable00348F1D" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "ParquetTableE84E985F" + } + ] + ] + } + ] + }, + { + "Action": [ + "s3:Abort*", + "s3:DeleteObject*", + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*", + "s3:PutObject", + "s3:PutObjectLegalHold", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectVersionTagging" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" ] - }, - "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + }, + "/*" + ] + ] } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "AnotherUserDefaultPolicyDBDB9923", + "Users": [ + { + "Ref": "AnotherUser254B09E3" + } + ] + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." } + ] } -} + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json index 7af33a9b07de8..a69e06b12518c 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json @@ -1,5 +1,5 @@ { - "version": "30.1.0", + "version": "31.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json index 81cea509f1f56..276316af49ccd 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json @@ -1,10 +1,12 @@ { - "version": "30.1.0", + "version": "31.0.0", "testCases": { "aws-cdk-glue-table-integ/DefaultTest": { - "stacks": ["aws-cdk-glue"], + "stacks": [ + "aws-cdk-glue" + ], "assertionStack": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert", "assertionStackName": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70" } } -} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json index fbd55f00e24ce..6497dc2a31b7a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "30.1.0", + "version": "31.0.0", "artifacts": { "aws-cdk-glue.assets": { "type": "cdk:asset-manifest", @@ -17,17 +17,21 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/eddae78388dd938d03998219503a0c3e6662678efa97b8099af7c02290533da6.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/575ff5e7aa88089b2f961dec32c432bf462cbc875db69d584d780e9404ae8176.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", - "additionalDependencies": ["aws-cdk-glue.assets"], + "additionalDependencies": [ + "aws-cdk-glue.assets" + ], "lookupRole": { "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", "requiresBootstrapStackVersion": 8, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" } }, - "dependencies": ["aws-cdk-glue.assets"], + "dependencies": [ + "aws-cdk-glue.assets" + ], "metadata": { "/aws-cdk-glue/DataBucket/Resource": [ { @@ -188,4 +192,4 @@ } } } -} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json index 2b068387f67bd..f8edfa17f3f90 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json @@ -20,13 +20,13 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "@aws-cdk/aws-s3.CfnBucket", + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-s3.Bucket", + "fqn": "aws-cdk-lib.aws_s3.Bucket", "version": "0.0.0" } }, @@ -49,13 +49,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnDatabase", + "fqn": "aws-cdk-lib.aws_glue.CfnDatabase", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Database", + "fqn": "@aws-cdk/aws-glue-alpha.Database", "version": "0.0.0" } }, @@ -137,13 +137,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -225,13 +225,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -313,13 +313,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -401,13 +401,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -452,13 +452,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-kms.CfnKey", + "fqn": "aws-cdk-lib.aws_kms.CfnKey", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-kms.Key", + "fqn": "aws-cdk-lib.aws_kms.Key", "version": "0.0.0" } }, @@ -482,7 +482,10 @@ "serverSideEncryptionByDefault": { "sseAlgorithm": "aws:kms", "kmsMasterKeyId": { - "Fn::GetAtt": ["MyKey6AB29FA6", "Arn"] + "Fn::GetAtt": [ + "MyKey6AB29FA6", + "Arn" + ] } } } @@ -491,13 +494,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-s3.CfnBucket", + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-s3.Bucket", + "fqn": "aws-cdk-lib.aws_s3.Bucket", "version": "0.0.0" } }, @@ -575,13 +578,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -658,13 +661,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -689,7 +692,7 @@ "description": "table_with_storage_descriptor_parameters generated by CDK", "parameters": { "classification": "json", - "has_encrypted_data": false + "has_encrypted_data": true }, "storageDescriptor": { "location": { @@ -746,13 +749,13 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.CfnTable", + "fqn": "aws-cdk-lib.aws_glue.CfnTable", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-glue.Table", + "fqn": "@aws-cdk/aws-glue-alpha.Table", "version": "0.0.0" } }, @@ -768,7 +771,7 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.CfnUser", + "fqn": "aws-cdk-lib.aws_iam.CfnUser", "version": "0.0.0" } }, @@ -873,7 +876,10 @@ "Effect": "Allow", "Resource": [ { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" + ] }, { "Fn::GetAtt": [ @@ -921,7 +927,10 @@ ], "Effect": "Allow", "Resource": { - "Fn::GetAtt": ["MyKey6AB29FA6", "Arn"] + "Fn::GetAtt": [ + "MyKey6AB29FA6", + "Arn" + ] } } ], @@ -936,19 +945,19 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.CfnPolicy", + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.Policy", + "fqn": "aws-cdk-lib.aws_iam.Policy", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.User", + "fqn": "aws-cdk-lib.aws_iam.User", "version": "0.0.0" } }, @@ -964,7 +973,7 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.CfnUser", + "fqn": "aws-cdk-lib.aws_iam.CfnUser", "version": "0.0.0" } }, @@ -1096,7 +1105,10 @@ "Effect": "Allow", "Resource": [ { - "Fn::GetAtt": ["DataBucketE3889A50", "Arn"] + "Fn::GetAtt": [ + "DataBucketE3889A50", + "Arn" + ] }, { "Fn::Join": [ @@ -1126,19 +1138,19 @@ } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.CfnPolicy", + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.Policy", + "fqn": "aws-cdk-lib.aws_iam.Policy", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/aws-iam.User", + "fqn": "aws-cdk-lib.aws_iam.User", "version": "0.0.0" } }, @@ -1146,7 +1158,7 @@ "id": "BootstrapVersion", "path": "aws-cdk-glue/BootstrapVersion", "constructInfo": { - "fqn": "@aws-cdk/core.CfnParameter", + "fqn": "aws-cdk-lib.CfnParameter", "version": "0.0.0" } }, @@ -1154,13 +1166,13 @@ "id": "CheckBootstrapVersion", "path": "aws-cdk-glue/CheckBootstrapVersion", "constructInfo": { - "fqn": "@aws-cdk/core.CfnRule", + "fqn": "aws-cdk-lib.CfnRule", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/core.Stack", + "fqn": "aws-cdk-lib.Stack", "version": "0.0.0" } }, @@ -1177,7 +1189,7 @@ "path": "aws-cdk-glue-table-integ/DefaultTest/Default", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.1.264" + "version": "10.1.270" } }, "DeployAssert": { @@ -1188,7 +1200,7 @@ "id": "BootstrapVersion", "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert/BootstrapVersion", "constructInfo": { - "fqn": "@aws-cdk/core.CfnParameter", + "fqn": "aws-cdk-lib.CfnParameter", "version": "0.0.0" } }, @@ -1196,25 +1208,25 @@ "id": "CheckBootstrapVersion", "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert/CheckBootstrapVersion", "constructInfo": { - "fqn": "@aws-cdk/core.CfnRule", + "fqn": "aws-cdk-lib.CfnRule", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/core.Stack", + "fqn": "aws-cdk-lib.Stack", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/integ-tests.IntegTestCase", + "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", "version": "0.0.0" } } }, "constructInfo": { - "fqn": "@aws-cdk/integ-tests.IntegTest", + "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", "version": "0.0.0" } }, @@ -1223,13 +1235,13 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.1.264" + "version": "10.1.270" } } }, "constructInfo": { - "fqn": "@aws-cdk/core.App", + "fqn": "aws-cdk-lib.App", "version": "0.0.0" } } -} +} \ No newline at end of file From a0081f76774a24f19475da2d5e151ee3779ea0c0 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Tue, 2 May 2023 09:31:04 +0000 Subject: [PATCH 23/32] updating comment --- packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index d7be9f01d44a6..7ecd411ee77cb 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -81,7 +81,7 @@ export interface StorageParameter { readonly key: StorageParameters; /** - * The value of the property. If you want to use a custom value, use `StorageParameter.custom()`. + * The value of the property. If you want to use a custom value, use `StorageParameterValue.custom()`. */ readonly value: StorageParameterValue; } From 8e7b80224e7dcbaa0fe2019202f8c37d314728d8 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 26 Jun 2023 11:58:12 +0000 Subject: [PATCH 24/32] fixing pkglint --- packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts index 2e4add123f953..ab18b958c9c6b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts @@ -6,7 +6,6 @@ import * as kms from 'aws-cdk-lib/aws-kms'; import * as s3 from 'aws-cdk-lib/aws-s3'; import * as glue from '../lib'; - const app = new cdk.App(); const stack = new cdk.Stack(app, 'aws-cdk-glue'); From 6fe3da282939fd9ee70b6fd4c7253b6aa10e740b Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 17 Jul 2023 10:48:12 +0000 Subject: [PATCH 25/32] adding @see for external tables to redshift documentation --- packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index dc1471375d288..1dfae0b0ce57b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -204,6 +204,8 @@ export interface TableProps { * * @see https://docs.aws.amazon.com/glue/latest/dg/table-properties-crawler.html * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ + * * @example * * declare const glueDatabase: glue.IDatabase; From 993fc4bcb86d2fbb9dc220134e472b5dd7b97241 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 28 Jul 2023 14:13:33 +0000 Subject: [PATCH 26/32] renaming storage descriptor attribute --- packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index 1dfae0b0ce57b..cc160d0473b95 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -323,7 +323,7 @@ export class Table extends Resource implements ITable { /** * The tables' storage descriptor properties. */ - public readonly storageDescriptor?: StorageParameter[]; + public readonly storageParameters?: StorageParameter[]; /** * Partition indexes must be created one at a time. To avoid @@ -347,7 +347,7 @@ export class Table extends Resource implements ITable { validateSchema(props.columns, props.partitionKeys); this.columns = props.columns; this.partitionKeys = props.partitionKeys; - this.storageDescriptor = props.storageParameters; + this.storageParameters = props.storageParameters; this.compressed = props.compressed ?? false; const { bucket, encryption, encryptionKey } = createBucket(this, props); From b6ce8655877e2f3baf5d7d1b8f5e7cc8c9bd3845 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 28 Jul 2023 14:31:34 +0000 Subject: [PATCH 27/32] creating test case for duplicate keys --- .../aws-glue-alpha/test/table.test.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts index 131190fc0f811..73c9cefa54e4d 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts @@ -1478,6 +1478,27 @@ describe('validate', () => { encryption: glue.TableEncryption.CLIENT_SIDE_KMS, })).not.toThrow(); }); + + test('unique storage descriptor parameters', () => { + const app = new cdk.App(); + const stack = new cdk.Stack(app, 'Stack'); + const database = new glue.Database(stack, 'Database'); + + expect(() => new glue.Table(stack, 'Table', { + database, + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + storageParameters: [ + glue.StorageParameter.skipHeaderLineCount(2), + glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + glue.StorageParameter.custom('foo', 'bar'), + glue.StorageParameter.custom(glue.StorageParameters.COMPRESSION_TYPE, 'true'), + ], + })).toThrowError('Duplicate storage parameter key: compression_type'); + }); }); test('Table.fromTableArn', () => { From fd3c961803baaa1a80b54c62a9c005cec84f12cd Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 28 Jul 2023 14:34:35 +0000 Subject: [PATCH 28/32] changing API to be more simple --- packages/@aws-cdk/aws-glue-alpha/README.md | 6 +- .../aws-glue-alpha/lib/storage-parameter.ts | 246 ++++++++++++------ packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 30 +-- packages/@aws-cdk/aws-glue-alpha/package.json | 3 +- .../aws-glue-alpha/test/integ.table.ts | 8 +- .../aws-glue-alpha/test/table.test.ts | 8 +- packages/aws-cdk/vendor/noctilucent | 1 + 7 files changed, 198 insertions(+), 104 deletions(-) create mode 160000 packages/aws-cdk/vendor/noctilucent diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index c97ae8c2c7a59..533b728282211 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -229,9 +229,9 @@ Glue tables can be configured to contain user-defined properties, to describe th declare const myDatabase: glue.Database; new glue.Table(this, 'MyTable', { storageParameters: [ - { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: glue.StorageParameterValue.custom('1') }, - { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.StorageParameterValue.compressionType(glue.CompressionType.GZIP) }, - { key: glue.StorageParameters.custom('separatorChar'), value: glue.StorageParameterValue.custom(',') }, // Will describe the separator char used in the data + glue.StorageParameter.skipHeaderLineCount(1), // note this is a number. The static method should convert it to a string. + glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + glue.StorageParameter.custom('separatorChar', ',') ], // ... database: myDatabase, diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts b/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts index c83e15dd765a0..ec71d25727954 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts @@ -1,5 +1,7 @@ /** - * The compression type used by `StorageParameters.COMPRESSION_TYPE`. + * The compression type. + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ > _"compression_type"_ */ export enum CompressionType { /** @@ -25,6 +27,8 @@ export enum CompressionType { /** * Specifies the action to perform when query results contain invalid UTF-8 character values. + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ > _"invalid_char_handling"_ */ export enum InvalidCharHandlingAction { /** @@ -54,7 +58,9 @@ export enum InvalidCharHandlingAction { } /** - * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + * Specifies the action to perform when ORC data contains an integer (for example, BIGINT or int64) that is larger than the column definition (for example, SMALLINT or int16). + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ > _"numeric_overflow_handling"_ */ export enum NumericOverflowHandlingAction { /** @@ -79,7 +85,9 @@ export enum NumericOverflowHandlingAction { } /** - * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARBYTE data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ > _"surplus_bytes_handling"_ */ export enum SurplusBytesHandlingAction { /** @@ -109,7 +117,9 @@ export enum SurplusBytesHandlingAction { } /** - * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARCHAR, CHAR, or string data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ > _"surplus_char_handling"_ */ export enum SurplusCharHandlingAction { /** @@ -139,7 +149,9 @@ export enum SurplusCharHandlingAction { } /** - * The action to assign to `COLUMN_COUNT_MISMATCH_HANDLING`. + * Identifies if the file contains less or more values for a row than the number of columns specified in the external table definition. This property is only available for an uncompressed text file format. + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ > _"column_count_mismatch_handling"_ */ export enum ColumnCountMismatchHandlingAction { /** @@ -163,161 +175,249 @@ export enum ColumnCountMismatchHandlingAction { DROP_ROW = 'DROP_ROW', } -export class StorageParameterValue { +/** + * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARCHAR, CHAR, or string data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ > _"surplus_char_handling"_ + */ +export enum WriteParallel { /** - * Assigns a value of `CompressionType` + * Write data in parallel. */ - public static compressionType(value: CompressionType): StorageParameterValue { - return new StorageParameterValue(value); - } + ON = 'on', /** - * Assigns a value of `InvalidCharHandlingAction` + * Write data serially. */ - public static invalidCharHandling(value: InvalidCharHandlingAction): StorageParameterValue { - return new StorageParameterValue(value); - } + OFF = 'off', +} +/** + * Specifies how to map columns when the table uses ORC data format. + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ > _"orc.schema.resolution"_ + */ +export enum OrcColumnMappingType { /** - * Assigns a value of `NumericOverflowHandlingAction` + * Map columns by name. */ - public static numericOverflowHandling(value: NumericOverflowHandlingAction): StorageParameterValue { - return new StorageParameterValue(value); - } + NAME = 'name', /** - * Assigns a value of `SurplusBytesHandlingAction` + * Map columns by position. */ - public static surplusBytesHandling(value: SurplusBytesHandlingAction): StorageParameterValue { - return new StorageParameterValue(value); - } + POSITION = 'position', +} +/** + * The storage parameter keys that are currently known, this list is not exhaustive and other keys may be used. + */ +export enum StorageParameters { /** - * Assigns a value of `SurplusCharHandlingAction` + * The number of rows to skip at the top of a CSV file when the table is being created. */ - public static surplusCharHandling(value: SurplusCharHandlingAction): StorageParameterValue { - return new StorageParameterValue(value); - } + SKIP_HEADER_LINE_COUNT = 'skip.header.line.count', /** - * Assigns a value of `ColumnCountMismatchHandlingAction` + * Determines whether data handling is on for the table. */ - public static columnCountMismatchHandling(value: ColumnCountMismatchHandlingAction): StorageParameterValue { - return new StorageParameterValue(value); - } + DATA_CLEANSING_ENABLED = 'data_cleansing_enabled', /** - * A custom storage parameter value that can be used to assign to any storage parameter. - * @param value - The value of the storage parameter. + * The type of compression used on the table, when the file name does not contain an extension. This value overrides the compression type specified through the extension. */ - public static custom(value: string): StorageParameterValue { - return new StorageParameterValue(value); - } + COMPRESSION_TYPE = 'compression_type', + + /** + * Specifies the action to perform when query results contain invalid UTF-8 character values. + */ + INVALID_CHAR_HANDLING = 'invalid_char_handling', - protected constructor(public readonly value: string) {} + /** + * Specifies the replacement character to use when you set `INVALID_CHAR_HANDLING` to `REPLACE`. + */ + REPLACEMENT_CHAR = 'replacement_char', + + /** + * Specifies the action to perform when ORC data contains an integer (for example, BIGINT or int64) that is larger than the column definition (for example, SMALLINT or int16). + */ + NUMERIC_OVERFLOW_HANDLING = 'numeric_overflow_handling', + + /** + * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARBYTE data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. + */ + SURPLUS_BYTES_HANDLING = 'surplus_bytes_handling', + + /** + * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARCHAR, CHAR, or string data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. + */ + SURPLUS_CHAR_HANDLING = 'surplus_char_handling', + + /** + * Identifies if the file contains less or more values for a row than the number of columns specified in the external table definition. This property is only available for an uncompressed text file format. + */ + COLUMN_COUNT_MISMATCH_HANDLING = 'column_count_mismatch_handling', + + /** + * A property that sets the numRows value for the table definition. To explicitly update an external table's statistics, set the numRows property to indicate the size of the table. Amazon Redshift doesn't analyze external tables to generate the table statistics that the query optimizer uses to generate a query plan. If table statistics aren't set for an external table, Amazon Redshift generates a query execution plan based on an assumption that external tables are the larger tables and local tables are the smaller tables. + */ + NUM_ROWS = 'num_rows', + + /** + * A property that sets number of rows to skip at the beginning of each source file. + */ + SERIALIZATION_NULL_FORMAT = 'serialization.null.format', + + /** + * A property that sets the column mapping type for tables that use ORC data format. This property is ignored for other data formats. + */ + ORC_SCHEMA_RESOLUTION = 'orc.schema.resolution', + + /** + * A property that sets whether CREATE EXTERNAL TABLE AS should write data in parallel. When 'write.parallel' is set to off, CREATE EXTERNAL TABLE AS writes to one or more data files serially onto Amazon S3. This table property also applies to any subsequent INSERT statement into the same external table. + */ + WRITE_PARALLEL = 'write.parallel', + + /** + * A property that sets the maximum size (in MB) of each file written to Amazon S3 by CREATE EXTERNAL TABLE AS. The size must be a valid integer between 5 and 6200. The default maximum file size is 6,200 MB. This table property also applies to any subsequent INSERT statement into the same external table. + */ + WRITE_MAX_FILESIZE_MB = 'write.maxfilesize.mb', + + /** + * You can specify an AWS Key Management Service key to enable Server–Side Encryption (SSE) for Amazon S3 objects. + */ + WRITE_KMS_KEY_ID = 'write.kms.key.id', } -export class StorageParameters { +/** + * A storage parameter. The list of storage parameters available is not exhaustive and other keys may be used. + * + * If you would like to specify a storage parameter that is not available as a static member of this class, use the `StorageParameter.custom` method. + * + * The list of storage parameters currently known within the CDK is listed. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/table-properties-crawler.html + * + * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under _"TABLE PROPERTIES"_ + */ +export class StorageParameter { /** * The number of rows to skip at the top of a CSV file when the table is being created. */ - public static readonly SKIP_HEADER_LINE_COUNT = new StorageParameters('skip.header.line.count'); + public static skipHeaderLineCount(value: number): StorageParameter { + return new StorageParameter('skip.header.line.count', value.toString()); + } /** * Determines whether data handling is on for the table. */ - public static readonly DATA_CLEANSING_ENABLED = new StorageParameters('data_cleansing_enabled'); + public static dataCleansingEnabled(value: boolean): StorageParameter { + return new StorageParameter('data_cleansing_enabled', value.toString()); + } /** * The type of compression used on the table, when the file name does not contain an extension. This value overrides the compression type specified through the extension. - * - * Assign this parameter to a value of `StorageParameters.CompressionType` */ - public static readonly COMPRESSION_TYPE = new StorageParameters('compression_type'); + public static compressionType(value: CompressionType): StorageParameter { + return new StorageParameter('compression_type', value); + } /** * Specifies the action to perform when query results contain invalid UTF-8 character values. - * - * Assign this parameter to a value of `StorageParameters.InvalidCharHandlingAction` */ - public static readonly INVALID_CHAR_HANDLING = new StorageParameters('invalid_char_handling'); + public static invalidCharHandling(value: InvalidCharHandlingAction): StorageParameter { + return new StorageParameter('invalid_char_handling', value); + } /** * Specifies the replacement character to use when you set `INVALID_CHAR_HANDLING` to `REPLACE`. */ - public static readonly REPLACEMENT_CHAR = new StorageParameters('replacement_char'); + public static replacementChar(value: string): StorageParameter { + return new StorageParameter('replacement_char', value); + } /** * Specifies the action to perform when ORC data contains an integer (for example, BIGINT or int64) that is larger than the column definition (for example, SMALLINT or int16). - * - * Assign this parameter to a value of `StorageParameters.NumericOverflowHandlingAction` */ - public static readonly NUMERIC_OVERFLOW_HANDLING = new StorageParameters('numeric_overflow_handling'); + public static numericOverflowHandling(value: NumericOverflowHandlingAction): StorageParameter { + return new StorageParameter('numeric_overflow_handling', value); + } /** * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARBYTE data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. - * - * Assign this parameter to a value of `StorageParameters.SurplusBytesHandlingAction` */ - public static readonly SURPLUS_BYTES_HANDLING = new StorageParameters('surplus_bytes_handling'); + public static surplusBytesHandling(value: SurplusBytesHandlingAction): StorageParameter { + return new StorageParameter('surplus_bytes_handling', value); + } /** * Specifies how to handle data being loaded that exceeds the length of the data type defined for columns containing VARCHAR, CHAR, or string data. By default, Redshift Spectrum sets the value to null for data that exceeds the width of the column. - * - * Assign this parameter to a value of `StorageParameters.SurplusCharHandlingAction` */ - public static readonly SURPLUS_CHAR_HANDLING = new StorageParameters('surplus_char_handling'); + public static surplusCharHandling(value: SurplusCharHandlingAction): StorageParameter { + return new StorageParameter('surplus_char_handling', value); + } /** * Identifies if the file contains less or more values for a row than the number of columns specified in the external table definition. This property is only available for an uncompressed text file format. - * - * Assign this parameter to a value of `StorageParameters.ColumnCountMismatchHandling` */ - public static readonly COLUMN_COUNT_MISMATCH_HANDLING = new StorageParameters('column_count_mismatch_handling'); + public static columnCountMismatchHandling(value: ColumnCountMismatchHandlingAction): StorageParameter { + return new StorageParameter('column_count_mismatch_handling', value); + } /** * A property that sets the numRows value for the table definition. To explicitly update an external table's statistics, set the numRows property to indicate the size of the table. Amazon Redshift doesn't analyze external tables to generate the table statistics that the query optimizer uses to generate a query plan. If table statistics aren't set for an external table, Amazon Redshift generates a query execution plan based on an assumption that external tables are the larger tables and local tables are the smaller tables. */ - public static readonly NUM_ROWS = new StorageParameters('num_rows'); + public static numRows(value: number): StorageParameter { + return new StorageParameter('num_rows', value.toString()); + } /** * A property that sets number of rows to skip at the beginning of each source file. - * - * Assign this parameter to a value of `StorageParameters.SerializationNullFormat` */ - public static readonly SERIALIZATION_NULL_FORMAT = new StorageParameters('serialization.null.format'); + public static serializationNullFormat(value: string): StorageParameter { + return new StorageParameter('serialization.null.format', value); + } /** * A property that sets the column mapping type for tables that use ORC data format. This property is ignored for other data formats. If this property is omitted, columns are mapped by `OrcColumnMappingType.NAME` by default. * - * Assign this parameter to a value of `StorageParameters.OrcColumnMappingType` + * @default OrcColumnMappingType.NAME */ - public static readonly ORC_SCHEMA_RESOLUTION = new StorageParameters('orc.schema.resolution'); + public static orcSchemaResolution(value: OrcColumnMappingType): StorageParameter { + return new StorageParameter('orc.schema.resolution', value); + } /** - * A property that sets whether CREATE EXTERNAL TABLE AS should write data in parallel. By default, CREATE EXTERNAL TABLE AS writes data in parallel to multiple files, according to the number of slices in the cluster. The default option is on. When 'write.parallel' is set to off, CREATE EXTERNAL TABLE AS writes to one or more data files serially onto Amazon S3. This table property also applies to any subsequent INSERT statement into the same external table. + * A property that sets whether CREATE EXTERNAL TABLE AS should write data in parallel. When 'write.parallel' is set to off, CREATE EXTERNAL TABLE AS writes to one or more data files serially onto Amazon S3. This table property also applies to any subsequent INSERT statement into the same external table. * - * Assign this parameter to a value of `StorageParameters.WriteParallel` + * @default WriteParallel.ON */ - public static readonly WRITE_PARALLEL = new StorageParameters('write.parallel'); + public static writeParallel(value: WriteParallel): StorageParameter { + return new StorageParameter('write.parallel', value); + } /** * A property that sets the maximum size (in MB) of each file written to Amazon S3 by CREATE EXTERNAL TABLE AS. The size must be a valid integer between 5 and 6200. The default maximum file size is 6,200 MB. This table property also applies to any subsequent INSERT statement into the same external table. */ - public static readonly WRITE_MAX_FILESIZE_MB = new StorageParameters('write.maxfilesize.mb'); + public static writeMaxFileSizeMb(value: number): StorageParameter { + return new StorageParameter('write.maxfilesize.mb', value.toString()); + } /** * You can specify an AWS Key Management Service key to enable Server–Side Encryption (SSE) for Amazon S3 objects. */ - public static readonly WRITE_KMS_KEY_ID = new StorageParameters('write.kms.key.id'); + public static writeKmsKeyId(value: string): StorageParameter { + return new StorageParameter('write.kms.key.id', value); + } /** * A custom storage parameter. - * @param key The key of the parameter. + * @param key - The key of the storage parameter. + * @param value - The value of the storage parameter. */ - public static custom(key: string) { - return new StorageParameters(key); + public static custom(key: string, value: any): StorageParameter { + return new StorageParameter(key, value.toString()); } - protected constructor(public readonly key: string) {} + protected constructor(public readonly key: string, public readonly value: string) {} } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index cc160d0473b95..164fb2acd730a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -9,7 +9,7 @@ import { Construct } from 'constructs'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; import { Column } from './schema'; -import { StorageParameterValue, StorageParameters } from './storage-parameter'; +import { StorageParameter } from './storage-parameter'; /** * Properties of a Partition Index. @@ -74,18 +74,6 @@ export enum TableEncryption { CLIENT_SIDE_KMS = 'CSE-KMS' } -export interface StorageParameter { - /** - * The key of the property. If you want to use a custom key, use `StorageParameters.custom()`. - */ - readonly key: StorageParameters; - - /** - * The value of the property. If you want to use a custom value, use `StorageParameterValue.custom()`. - */ - readonly value: StorageParameterValue; -} - export interface TableAttributes { readonly tableArn: string; readonly tableName: string; @@ -211,10 +199,11 @@ export interface TableProps { * declare const glueDatabase: glue.IDatabase; * const table = new glue.Table(this, 'Table', { * storageParameters: [ - * { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: glue.StorageParameterValue.custom('1') }, - * { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.StorageParameterValue.compressionType(glue.CompressionType.GZIP) }, - * { key: glue.StorageParameters.custom('foo'), value: glue.StorageParameterValue.custom('bar') }, // Will have no effect - * { key: glue.StorageParameters.custom('separatorChar'), value: glue.StorageParameterValue.custom(',') }, // Will describe the separator char used in the data + * glue.StorageParameter.skipHeaderLineCount(1), + * glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + * glue.StorageParameter.custom('foo', 'bar'), // Will have no effect + * glue.StorageParameter.custom('separatorChar', ','), // Will describe the separator char used in the data + * glue.StorageParameter.custom(glue.StorageParameters.COMPRESSION_TYPE, 'true'), // Will have no effect * ], * // ... * database: glueDatabase, @@ -382,8 +371,11 @@ export class Table extends Resource implements ITable { serializationLibrary: props.dataFormat.serializationLibrary.className, }, parameters: props.storageParameters ? props.storageParameters.reduce((acc, param) => { - const key = param.key.key; - acc[key] = param.value.value; + if (param.key in acc) { + throw new Error(`Duplicate storage parameter key: ${param.key}`); + } + const key = param.key; + acc[key] = param.value; return acc; }, {} as { [key: string]: string }) : undefined, }, diff --git a/packages/@aws-cdk/aws-glue-alpha/package.json b/packages/@aws-cdk/aws-glue-alpha/package.json index 762d3ac51d8bf..8aea2b8ca6983 100644 --- a/packages/@aws-cdk/aws-glue-alpha/package.json +++ b/packages/@aws-cdk/aws-glue-alpha/package.json @@ -84,9 +84,8 @@ "@aws-cdk/cdk-build-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/integ-runner": "0.0.0", - "@aws-cdk/integ-tests-alpha": "^0.0.0", - "@aws-cdk/pkglint": "0.0.0", "@aws-cdk/integ-tests-alpha": "0.0.0", + "@aws-cdk/pkglint": "0.0.0", "@types/jest": "^29.5.1", "aws-cdk-lib": "0.0.0", "constructs": "^10.0.0", diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts index ab18b958c9c6b..7c8fc3ff117de 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts @@ -108,10 +108,10 @@ new glue.Table(stack, 'MyTableWithStorageDescriptorParameters', { columns, dataFormat: glue.DataFormat.JSON, storageParameters: [ - { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: glue.StorageParameterValue.custom('1') }, - { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.StorageParameterValue.compressionType(glue.CompressionType.GZIP) }, - { key: glue.StorageParameters.custom('foo'), value: glue.StorageParameterValue.custom('bar') }, // Will have no effect - { key: glue.StorageParameters.custom('separatorChar'), value: glue.StorageParameterValue.custom(',') }, // Will describe the separator char used in the data + glue.StorageParameter.skipHeaderLineCount(1), + glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + glue.StorageParameter.custom('foo', 'bar'), // Will have no effect + glue.StorageParameter.custom('separatorChar', ','), // Will describe the separator char used in the data ], }); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts index 73c9cefa54e4d..487a5c5343e14 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/table.test.ts @@ -1633,9 +1633,10 @@ test('storage descriptor parameters', () => { }], dataFormat: glue.DataFormat.JSON, storageParameters: [ - { key: glue.StorageParameters.SKIP_HEADER_LINE_COUNT, value: glue.StorageParameterValue.custom('2') }, - { key: glue.StorageParameters.custom('separatorChar'), value: glue.StorageParameterValue.custom(',') }, - { key: glue.StorageParameters.COMPRESSION_TYPE, value: glue.StorageParameterValue.compressionType(glue.CompressionType.GZIP) }, + glue.StorageParameter.skipHeaderLineCount(2), + glue.StorageParameter.compressionType(glue.CompressionType.GZIP), + glue.StorageParameter.custom('foo', 'bar'), + glue.StorageParameter.custom('separatorChar', ','), ], }); @@ -1645,6 +1646,7 @@ test('storage descriptor parameters', () => { Parameters: { 'skip.header.line.count': '2', 'separatorChar': ',', + 'foo': 'bar', 'compression_type': 'gzip', }, }, diff --git a/packages/aws-cdk/vendor/noctilucent b/packages/aws-cdk/vendor/noctilucent new file mode 160000 index 0000000000000..6da7c9fade55f --- /dev/null +++ b/packages/aws-cdk/vendor/noctilucent @@ -0,0 +1 @@ +Subproject commit 6da7c9fade55f8443bba7b8fdfcd4ebfe5208fb1 From 3abcbf1b1613b7420c37833fb7a3df1b9445891c Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Fri, 28 Jul 2023 14:43:06 +0000 Subject: [PATCH 29/32] removing vendor --- packages/aws-cdk/vendor/noctilucent | 1 - 1 file changed, 1 deletion(-) delete mode 160000 packages/aws-cdk/vendor/noctilucent diff --git a/packages/aws-cdk/vendor/noctilucent b/packages/aws-cdk/vendor/noctilucent deleted file mode 160000 index 6da7c9fade55f..0000000000000 --- a/packages/aws-cdk/vendor/noctilucent +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6da7c9fade55f8443bba7b8fdfcd4ebfe5208fb1 From 13346da1546d587d9ca52e276e2a5adcc68086ac Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 31 Jul 2023 15:36:30 +0000 Subject: [PATCH 30/32] misc --- packages/@aws-cdk/aws-glue-alpha/README.md | 2 +- packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index 533b728282211..64ac86f8c6b9b 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -229,7 +229,7 @@ Glue tables can be configured to contain user-defined properties, to describe th declare const myDatabase: glue.Database; new glue.Table(this, 'MyTable', { storageParameters: [ - glue.StorageParameter.skipHeaderLineCount(1), // note this is a number. The static method should convert it to a string. + glue.StorageParameter.skipHeaderLineCount(1), glue.StorageParameter.compressionType(glue.CompressionType.GZIP), glue.StorageParameter.custom('separatorChar', ',') ], diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts b/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts index ec71d25727954..50b1ad934d712 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/storage-parameter.ts @@ -420,4 +420,4 @@ export class StorageParameter { } protected constructor(public readonly key: string, public readonly value: string) {} -} \ No newline at end of file +} From 5e0759b821e8d8f2d2317427f8020b680b26f8b1 Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 31 Jul 2023 15:37:04 +0000 Subject: [PATCH 31/32] broken example --- packages/@aws-cdk/aws-glue-alpha/lib/table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts index 164fb2acd730a..f62e6af2010a4 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/table.ts @@ -203,7 +203,7 @@ export interface TableProps { * glue.StorageParameter.compressionType(glue.CompressionType.GZIP), * glue.StorageParameter.custom('foo', 'bar'), // Will have no effect * glue.StorageParameter.custom('separatorChar', ','), // Will describe the separator char used in the data - * glue.StorageParameter.custom(glue.StorageParameters.COMPRESSION_TYPE, 'true'), // Will have no effect + * glue.StorageParameter.custom(glue.StorageParameters.WRITE_PARALLEL, 'off'), * ], * // ... * database: glueDatabase, From 098a80cb3163074decc15b0cf66dcc47a287ba3e Mon Sep 17 00:00:00 2001 From: Rizbir Khan Date: Mon, 31 Jul 2023 16:15:23 +0000 Subject: [PATCH 32/32] integ test with additional param --- .../aws-cdk-glue.assets.json | 6 +++--- .../aws-cdk-glue.template.json | 3 ++- ...integDefaultTestDeployAssert8BFB5B70.assets.json | 2 +- .../test/integ.table.js.snapshot/cdk.out | 2 +- .../test/integ.table.js.snapshot/integ.json | 2 +- .../test/integ.table.js.snapshot/manifest.json | 13 +++++++++++-- .../test/integ.table.js.snapshot/tree.json | 7 ++++--- .../@aws-cdk/aws-glue-alpha/test/integ.table.ts | 1 + 8 files changed, 24 insertions(+), 12 deletions(-) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json index 4b14d29b52f16..d714c2d54aaba 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.assets.json @@ -1,7 +1,7 @@ { - "version": "31.0.0", + "version": "33.0.0", "files": { - "575ff5e7aa88089b2f961dec32c432bf462cbc875db69d584d780e9404ae8176": { + "dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754": { "source": { "path": "aws-cdk-glue.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "575ff5e7aa88089b2f961dec32c432bf462cbc875db69d584d780e9404ae8176.json", + "objectKey": "dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json index 55e8f24d3b187..f4415b64a1333 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/aws-cdk-glue.template.json @@ -547,7 +547,8 @@ "skip.header.line.count": "1", "compression_type": "gzip", "foo": "bar", - "separatorChar": "," + "separatorChar": ",", + "write.parallel": "off" }, "SerdeInfo": { "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json index a69e06b12518c..670371bb340e7 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json @@ -1,5 +1,5 @@ { - "version": "31.0.0", + "version": "33.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/cdk.out index 7925065efbcc4..560dae10d018f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"31.0.0"} \ No newline at end of file +{"version":"33.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json index 276316af49ccd..ab7e38c81b5c6 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "31.0.0", + "version": "33.0.0", "testCases": { "aws-cdk-glue-table-integ/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json index 6497dc2a31b7a..02bab428fcf54 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "31.0.0", + "version": "33.0.0", "artifacts": { "aws-cdk-glue.assets": { "type": "cdk:asset-manifest", @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/575ff5e7aa88089b2f961dec32c432bf462cbc875db69d584d780e9404ae8176.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/dc6c1c5f05a8e365822e6d61c41b6fc6afd58d20a2784614b906ae1587c68754.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -134,6 +134,15 @@ "type": "aws:cdk:logicalId", "data": "CheckBootstrapVersion" } + ], + "MyPartitionFilteredTableBucket6ACAA137": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPartitionFilteredTableBucket6ACAA137", + "trace": [ + "!!DESTRUCTIVE_CHANGES: WILL_DESTROY" + ] + } ] }, "displayName": "aws-cdk-glue" diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json index f8edfa17f3f90..71c545fff8226 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.js.snapshot/tree.json @@ -741,7 +741,8 @@ "skip.header.line.count": "1", "compression_type": "gzip", "foo": "bar", - "separatorChar": "," + "separatorChar": ",", + "write.parallel": "off" } }, "tableType": "EXTERNAL_TABLE" @@ -1189,7 +1190,7 @@ "path": "aws-cdk-glue-table-integ/DefaultTest/Default", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.1.270" + "version": "10.2.69" } }, "DeployAssert": { @@ -1235,7 +1236,7 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.1.270" + "version": "10.2.69" } } }, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts index 7c8fc3ff117de..5633ccac75c1f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts @@ -112,6 +112,7 @@ new glue.Table(stack, 'MyTableWithStorageDescriptorParameters', { glue.StorageParameter.compressionType(glue.CompressionType.GZIP), glue.StorageParameter.custom('foo', 'bar'), // Will have no effect glue.StorageParameter.custom('separatorChar', ','), // Will describe the separator char used in the data + glue.StorageParameter.custom(glue.StorageParameters.WRITE_PARALLEL, 'off'), ], });