Add: GET_SECRETS_CRITICAL and GET_SECRETS_CRITICAL envvars to fine-grained disabling of cloud secrets manager for critical secrets and plain envvars [GS-41].

Add: Secret and KMS access policies to the "template-sam.yml" file [GS-41].
Add: generate_cf_dynamodb and deploy_dynamodb to Makefile [GS-84].
Add: "scripts/aws_dynamodb/generate_dynamodb_cf/generate_dynamodb_cf.py" and its ".sh" to generate the "cf-template-dynamodb.yml" file in the project's scripts directory [GS-84].
Add: "scripts/aws_dynamodb/run-dynamodb-deploy.sh" to deploy generated "cf-template-dynamodb.yml" [GS-84].
Add: "scripts/aws_cf_processor/run-cf-deployment.sh" enhanced to simulate the EC2 + ALB in AWS LocalStack [GS-97].
Add: "scripts/aws_cf_processor/test_localstack.sh" to test localstack EC2 functionality with the LOCALSTACK_AUTH_TOKEN envvar [GS-97].
Change: set APP_DB_URI when GET_SECRETS_ENABLED=0 or GET_SECRETS_CRITICAL=0 in "run_aws.sh" [GS-41].
Change: "aws_secrets_manager.sh" enhanced to use "run-cf-deployment.sh" [GS-41].
Change: "cf_template_kms_key.yml" and "cf_template_secrets.yml" renamed to "cf-template-kms-key.yml" and "cf-template-secrets.yml" [GS-41].
Change: "scripts/aws_ec2_elb/ec2_elb_manager.sh" removed [GS-96].
Fix: issue reporting the "_placeholder" missing parameter in the SAM template in verify_base_names() of big_lambdas_manager.sh.
Fix: "scripts/aws_ec2_elb/run-ec2-cloud-deploy.sh" and "cf-template-ec2-elb.yml" issues to finish the EC2 + ELB deployments (EBS volume encryption postponed) [GS-96].

Author: tomkat-cr

.gitignore

@@ -72,4 +72,7 @@ scripts/dns/Dockerfile
 
 response.json
 
-tmp
+tmp
+
+localstack_requirements.txt
+venv

CHANGELOG.md

@@ -22,15 +22,18 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch
 ---
 
 ### New
-Add password and API Keys to AWS Secrets [GS-41].
-Add: AWS Secrets manager to Makefile [GS-41].
-Add: EC2+ALB App deployment [GS-96].
-Add: delpoy_ec2 and deploy_ecr_creation to Makefile [GS-96].
+Add password and API Keys to AWS Secrets using AWS CloudFormation [GS-41].
+Add AWS Secrets manager to Makefile [GS-41].
+Add EC2+ALB App deployment using AWS CloudFormation [GS-96].
+Add depLoy_ec2 and deploy_ecr_creation to Makefile [GS-96].
+Add DynamoDB tables creation from the JSON configs using AWS CloudFormation [GS-84].
+Add "run-cf-deployment.sh" to standarize all Cloudformation calls [GS-96].
 
 ### Changes
 Change APP_STAGE dynamic assignment in run_aws.sh, set_chalice_cnf.sh, and big_lambdas_manager.sh, and secure_local_server/docker_entrypoint.sh [GS-41].
-Change __pycache__ removal simplified in big_lambdas_manager.sh [GS-96].
-Change APP_DB_URI and ther secrets assignment removed in big_lambdas_manager.sh, docker-compose-big-lambda-AL2.yml, docker-compose-big-lambda-Alpine.yml [GS-41].
+__pycache__ removal simplified in big_lambdas_manager.sh [GS-96].
+APP_DB_URI and the secrets assignment removed in big_lambdas_manager.sh, docker-compose-big-lambda-AL2.yml, docker-compose-big-lambda-Alpine.yml [GS-41].
+Remove all envvars from "template-sam.yml" [GS-96].
 
 ### Fixes
 Fix 'USER_AGENT environment variable not set...' LangSmith warning message removed in run_aws.sh, big_lambdas_manager.sh, aws_big_lambda/template-sam.yml, and secure_local_server/docker_entrypoint.sh.

Makefile

@@ -160,6 +160,9 @@ create_aws_config:
 generate_sam_dynamodb:
 	sh node_modules/genericsuite-be-scripts/scripts/aws_big_lambda/generate_sam_dynamodb/run_generate_sam_dynamodb.sh
 
+generate_cf_dynamodb:
+	sh node_modules/genericsuite-be-scripts/scripts/aws_dynamodb/generate_dynamodb_cf/generate_dynamodb_cf.sh
+
 ## Deployment
 
 deploy_qa: create_s3_bucket_qa
@@ -187,8 +190,16 @@ deploy_ecr_creation:
 	sh node_modules/genericsuite-be-scripts/scripts/aws_ec2_elb/run-fastapi-ecr-creation.sh
 
 deploy_ec2:
+	# E.g.
+	# CICD_MODE=0 ACTION=run STAGE=qa TARGET=ec2 ECR_DOCKER_IMAGE_TAG=0.0.16 make deploy_ec2
+	# CICD_MODE=0 ACTION=destroy STAGE=qa TARGET=ec2 ECR_DOCKER_IMAGE_TAG=0.0.16 make deploy_ec2
 	sh node_modules/genericsuite-be-scripts/scripts/aws_ec2_elb/run-ec2-cloud-deploy.sh
 
+deploy_dynamodb:
+	# CICD_MODE=0 ACTION=run STAGE=qa TARGET=dynamodb ENGINE=localstack make deploy_dynamodb
+	# CICD_MODE=0 ACTION=run STAGE=qa TARGET=dynamodb make deploy_dynamodb
+	sh node_modules/genericsuite-be-scripts/scripts/aws_dynamodb/run-dynamodb-deploy.sh
+
 deploy: deploy_qa
 
 ## Secrets
@@ -198,6 +209,10 @@ generate_seed:
 	sh node_modules/genericsuite-be-scripts/scripts/cryptography/run_generate_seed.sh
 
 aws_secrets:
+	# E.g.
+	# CICD_MODE=0 ACTION=run STAGE=qa TARGET=kms make aws_secrets
+	# CICD_MODE=0 ACTION=run STAGE=qa TARGET=kms ENGINE=localstack make aws_secrets
+	# CICD_MODE=0 ACTION=run STAGE=qa TARGET=secrets make aws_secrets
 	sh node_modules/genericsuite-be-scripts/scripts/aws_secrets/aws_secrets_manager.sh
 
 # aws_secrets_create:

scripts/aws/run_aws.sh

@@ -159,7 +159,9 @@ if [[ "$1" = "run_local" || "$1" = "" ]]; then
 
     export APP_DB_ENGINE=$(eval echo \$APP_DB_ENGINE_${STAGE_UPPERCASE})
     export APP_DB_NAME=$(eval echo \$APP_DB_NAME_${STAGE_UPPERCASE})
-    # export APP_DB_URI=$(eval echo \$APP_DB_URI_${STAGE_UPPERCASE})
+    if [[ "${GET_SECRETS_ENABLED}" = 0 || "${GET_SECRETS_CRITICAL}" = "0" ]]; then
+        export APP_DB_URI=$(eval echo \$APP_DB_URI_${STAGE_UPPERCASE})
+    fi
     export APP_CORS_ORIGIN="$(eval echo \"\$APP_CORS_ORIGIN_${STAGE_UPPERCASE}\")"
     export AWS_S3_CHATBOT_ATTACHMENTS_BUCKET=$(eval echo \$AWS_S3_CHATBOT_ATTACHMENTS_BUCKET_${STAGE_UPPERCASE})
 

scripts/aws_big_lambda/big_lambdas_manager.sh

@@ -555,7 +555,8 @@ get_ssl_cert_arn() {
 }
 
 verify_base_names() {
-    base_names=("CLOUD_PROVIDER AWS_REGION APP_DB_URI APP_DB_NAME APP_DB_ENGINE APP_NAME APP_SECRET_KEY APP_SUPERADMIN_EMAIL APP_HOST_NAME STORAGE_URL_SEED GIT_SUBMODULE_LOCAL_PATH")
+    local names="CLOUD_PROVIDER AWS_REGION APP_DB_URI APP_DB_NAME APP_DB_ENGINE APP_NAME APP_SECRET_KEY APP_SUPERADMIN_EMAIL APP_HOST_NAME STORAGE_URL_SEED GIT_SUBMODULE_LOCAL_PATH"
+    local base_names=(${names})
     ERROR_FLAG=0
 
     for base_name in "${base_names[@]}"; do

scripts/aws_big_lambda/generate_sam_dynamodb/generate_sam_dynamodb.py

@@ -108,7 +108,8 @@ def get_dynamodb_definition(config: dict, table_prefix: str) -> dict:
 
 def generate_dynamodb_definitions(basedir: str, table_prefix: str):
     """
-    Generates DynamoDB table definitions from frontend and backend config files.
+    Generates DynamoDB table definitions from frontend and backend
+    config files.
     """
     dynamodb_definitions = {}
 
@@ -120,16 +121,19 @@ def generate_dynamodb_definitions(basedir: str, table_prefix: str):
         for file in files:
             _ = DEBUG and print(f'File: {file}')
             if file.endswith('.json'):
-                with open(os.path.join(root, file), 'r', encoding="utf-8") as f:
+                with open(os.path.join(root, file), 'r',
+                          encoding="utf-8") as f:
                     config = json.load(f)
                     # if file exists in backend config, merge them
                     file_path = os.path.join(basedir, 'backend', file)
                     if os.path.exists(file_path):
                         with open(file_path, 'r', encoding="utf-8") as f:
                             config.update(json.load(f))
-                    table_definition = get_dynamodb_definition(config, table_prefix)
+                    table_definition = get_dynamodb_definition(config,
+                                                               table_prefix)
                     if table_definition:
-                        # table_name = table_definition['Properties']['TableName']
+                        # table_name = \
+                        #   table_definition['Properties']['TableName']
                         # dynamodb_definitions[table_name] = table_definition
                         dynamodb_definitions.update(table_definition)
 
@@ -145,7 +149,8 @@ def generate_sam_dynamodb():
     print('')
 
     if len(sys.argv) < 4:
-        print('Usage: python generate_sam_dynamodb.py <base_config_path> <target_template_path> <table_prefix>')
+        print('Usage: python generate_sam_dynamodb.py <base_config_path>' +
+              '<target_template_path> <table_prefix>')
         sys.exit(1)
 
     base_config_path = sys.argv[1]
@@ -156,7 +161,8 @@ def generate_sam_dynamodb():
     print(f'target_template_path: {target_template_path}')
     print('')
 
-    dynamodb_def = generate_dynamodb_definitions(basedir=base_config_path,
+    dynamodb_def = generate_dynamodb_definitions(
+        basedir=base_config_path,
         table_prefix=table_prefix)
     # Open the target template file to write the DynamoDB table definition
     with open(target_template_path, 'w', encoding="utf-8") as f:

scripts/aws_big_lambda/template-sam.yml

@@ -54,6 +54,34 @@ Resources:
               - logs:CreateLogStream
               - logs:PutLogEvents
               Resource: arn:*:logs:*:*:*
+        - PolicyName: Ec2SecretsAccessPolicy
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - secretsmanager:GetSecretValue
+                Resource: '*'
+                # Resource: 
+                # #   - arn:aws:secretsmanager:*:*:secret:AWS_SECRETS_MANAGER_SECRETS_NAME_placeholder
+                #   - !Sub arn:aws:secretsmanager:*:*:secret:${AsmSecretsName}
+                # #   - arn:aws:secretsmanager:*:*:secret:AWS_SECRETS_MANAGER_ENVS_NAME_placeholder
+                #   - !Sub arn:aws:secretsmanager:*:*:secret:${AsmEnvsName}
+        - PolicyName: Ec2KmsAccessPolicy
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - kms:Decrypt
+                  - kms:GenerateDataKey*
+                  - kms:CreateGrant
+                Resource: !Sub 'arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/*'
+                # Resource: !Sub 'arn:aws:kms:${AWS::Region}:${AWS::AccountId}:alias/${KmsKeyAlias}'
+                # Resource: '*'
+                # Resource: 
+                # # - arn:aws:kms:AWS_REGION_placeholder:AWS_ACCOUNT_ID_placeholder:alias/AWS_KMS_KEY_ALIAS_placeholder
+                #   - !Sub arn:aws:kms:${AwsRegion}:${AwsAccountId}:alias/${KmsKeyAlias}
 
   APIHandler:
     Type: AWS::Serverless::Function
@@ -72,7 +100,12 @@ Resources:
           APP_NAME: APP_NAME_placeholder
           APP_STAGE: APP_STAGE_placeholder
           CLOUD_PROVIDER: CLOUD_PROVIDER_placeholder
-          AWS_REGION: AWS_REGION_placeholder
+          # AWS_REGION: AWS_REGION_placeholder
+          # Lambda was unable to configure your environment variables because the environment variables you have provided contains reserved keys that are currently not supported for modification. Reserved keys used in this request: AWS_REGION
+
+          # GET_SECRETS_ENABLED: 0
+          # GET_SECRETS_ENVVARS: 0
+          # GET_SECRETS_CRITICAL: 0
 
           # AI_ASSISTANT_NAME: AI_ASSISTANT_NAME_placeholder
           # APP_VERSION: APP_VERSION_placeholder