Refactor to allow individual tf modules

address #25 (comment)

Author: rivernews

cloud_environments/dev/module.tf

@@ -1,7 +1,8 @@
 module main {
-  source = "../../cloud_module"
+  source = "../../cloud_module/pipeline"
   environment_name = var.environment_name
   project_alias = var.project_alias
   slack_signing_secret = var.slack_signing_secret
   slack_post_webhook_url = var.slack_post_webhook_url
+  repo_dir = var.repo_dir
 }

cloud_environments/dev/module_variables.tf

@@ -18,3 +18,8 @@ variable environment_name {
   default = ""
   description = "Empty string for Production, otherwise the environment name e.g. dev, stage, etc, make sure to use lowercase (s3 bucket only allows lower)"
 }
+
+variable repo_dir {
+  type = string
+  description = "The absolute path of git repository path"
+}

cloud_environments/dev_table/.terraform.lock.hcl

@@ -0,0 +1,6 @@
+terraform {
+  backend "s3" {
+    bucket = "iriversland-cloud"
+    key    = "terraform/media-literacy-dev--table.remote-terraform.tfstate"
+  }
+}

cloud_environments/dev_table/backend.tf

@@ -0,0 +1 @@
+terraform init -backend-config=local.backend.credentials.tfvars

cloud_environments/dev_table/local.backend.init.sh

@@ -0,0 +1,5 @@
+module main_table {
+  source = "${var.repo_dir}/cloud_module/dynamodb"
+  environment_name = var.environment_name
+  project_alias = var.project_alias
+}

cloud_environments/dev_table/module.tf

@@ -0,0 +1,15 @@
+variable "project_alias" {
+  type        = string
+  description = "Name prefix used for step function and related resources, including the domain name, so please only use [0-9a-z_-]"
+}
+
+variable environment_name {
+  type = string
+  default = ""
+  description = "Empty string for Production, otherwise the environment name e.g. dev, stage, etc, make sure to use lowercase (s3 bucket only allows lower)"
+}
+
+variable repo_dir {
+  type = string
+  description = "The absolute path of git repository path"
+}

cloud_environments/dev_table/module_variables.tf

@@ -0,0 +1,4 @@
+set -e
+
+REPO_DIR=$(git rev-parse --show-toplevel)
+ENV=dev_table sh "${REPO_DIR}/cloud_environments/terraform.sh" "$@"

cloud_environments/dev_table/terraform.sh

@@ -1,7 +1,8 @@
 module main {
-  source = "../../cloud_module"
+  source = "../../cloud_module/pipeline"
   environment_name = var.environment_name
   project_alias = var.project_alias
   slack_signing_secret = var.slack_signing_secret
   slack_post_webhook_url = var.slack_post_webhook_url
+  repo_dir = var.repo_dir
 }

cloud_environments/production/module.tf

@@ -18,3 +18,8 @@ variable environment_name {
   default = ""
   description = "Empty string for Production, otherwise the environment name e.g. dev, stage, etc, make sure to use lowercase (s3 bucket only allows lower)"
 }
+
+variable repo_dir {
+  type = string
+  description = "The absolute path of git repository path"
+}

cloud_environments/production/module_variables.tf

@@ -1,8 +1,9 @@
 set -e
 
-DEPLOY_DIR=$(git rev-parse --show-toplevel)/cloud_environments/${ENV:-production}
-GOLANG_SRC_DIR=$(git rev-parse --show-toplevel)/lambda_golang
-PYTHON_SRC_DIR=$(git rev-parse --show-toplevel)/lambda
+REPO_DIR=$(git rev-parse --show-toplevel)
+DEPLOY_DIR=${REPO_DIR}/cloud_environments/${ENV:-production}
+GOLANG_SRC_DIR=${REPO_DIR}/lambda_golang
+PYTHON_SRC_DIR=${REPO_DIR}/lambda
 
 set -o allexport
 . ${DEPLOY_DIR}/local.backend.credentials.tfvars
@@ -14,6 +15,7 @@ TF_VAR_project_alias=media-literacy
 TF_VAR_environment_name=${ENV:-}
 TF_VAR_slack_signing_secret=${slack_signing_secret}
 TF_VAR_slack_post_webhook_url=${slack_post_webhook_url}
+TF_VAR_repo_dir=${REPO_DIR}
 set +o allexport
 
 # below is just for testing golang build! The actual build command is in terraform lambda module `command` property

cloud_environments/terraform.sh

@@ -0,0 +1,43 @@
+resource "aws_dynamodb_table" "basic-dynamodb-table" {
+  name           = "GameScores"
+  billing_mode   = "PROVISIONED"
+  read_capacity  = 20
+  write_capacity = 20
+  hash_key       = "UserId"
+  range_key      = "GameTitle"
+
+  attribute {
+    name = "UserId"
+    type = "S"
+  }
+
+  attribute {
+    name = "GameTitle"
+    type = "S"
+  }
+
+  attribute {
+    name = "TopScore"
+    type = "N"
+  }
+
+  ttl {
+    attribute_name = "TimeToExist"
+    enabled        = false
+  }
+
+  global_secondary_index {
+    name               = "GameTitleIndex"
+    hash_key           = "GameTitle"
+    range_key          = "TopScore"
+    write_capacity     = 10
+    read_capacity      = 10
+    projection_type    = "INCLUDE"
+    non_key_attributes = ["UserId"]
+  }
+
+  tags = {
+    Name        = "dynamodb-table-1"
+    Environment = "production"
+  }
+}

cloud_module/dynamodb/table.tf

@@ -0,0 +1,15 @@
+variable "project_alias" {
+  type        = string
+  description = "Name prefix used for step function and related resources, including the domain name, so please only use [0-9a-z_-]"
+}
+
+variable environment_name {
+  type = string
+  default = ""
+  description = "Empty string for Production, otherwise the environment name e.g. dev, stage, etc, make sure to use lowercase (s3 bucket only allows lower)"
+}
+
+locals {
+  project_name = var.environment_name != "" ? "${var.project_alias}-${var.environment_name}" : "${var.project_alias}"
+  environment = var.environment_name != "" ? var.environment_name : "prod_table"
+}

cloud_module/dynamodb/variables.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">= 1.0.2" # minimum version that supports M1
+
+  required_providers {
+    # please use env var to pass over credentials
+    # https://registry.terraform.io/providers/hashicorp/aws/latest/docs
+    aws = ">= 4.9.0" # minimum version based on `terraform init` error message
+  }
+}

cloud_module/dynamodb/version.tf

@@ -56,7 +56,7 @@ module "slack_command_lambda" {
   description   = "Lambda function for slack command for environment ${local.project_name}"
   handler       = "slack_command_controller.lambda_handler"
   runtime     = "python3.8"
-  source_path = "${path.module}/../lambda/src/slack_command_controller.py"
+  source_path = "${var.repo_dir}/lambda/src/slack_command_controller.py"
 
   layers = [
     module.lambda_layer.lambda_layer_arn

cloud_module/README.md cloud_module/pipeline/README.md

@@ -10,7 +10,7 @@ module "lambda_layer" {
   runtime     = "python3.8"
   compatible_runtimes = ["python3.8"]
   source_path = [{
-    path = "${path.module}/../lambda/layer"
+    path = "${var.repo_dir}/lambda/layer"
     pip_requirements = true
     # Make sure the follow the Layer Structure
     # https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path
@@ -67,7 +67,7 @@ module "scraper_lambda" {
   # Based on tf https://github.com/terraform-aws-modules/terraform-aws-lambda/blob/master/examples/build-package/main.tf#L111
   # Based on golang https://github.com/snsinfu/terraform-lambda-example/blob/master/Makefile#L23
   source_path = [{
-    path = "${path.module}/../lambda_golang/"
+    path = "${var.repo_dir}/lambda_golang/"
     commands = ["${local.go_build_flags} go build ./cmd/landing", ":zip"]
     patterns = ["landing"]
   }]

cloud_module/api.tf cloud_module/pipeline/api.tf

@@ -29,7 +29,7 @@ module "pipeline_queue_consumer_lambda" {
   description   = "Consumer lambda function for ${local.project_name} pipeline queue"
   handler       = "pipeline_queue_consumer.lambda_handler"
   runtime     = "python3.8"
-  source_path = "${path.module}/../lambda/src/pipeline_queue_consumer.py"
+  source_path = "${var.repo_dir}/lambda/src/pipeline_queue_consumer.py"
   publish = true
 
   layers = [

cloud_module/archive.tf cloud_module/pipeline/archive.tf

@@ -36,7 +36,7 @@ module landing_parse_metadata_lambda {
   runtime       = "go1.x"
 
   source_path = [{
-    path = "${path.module}/../lambda_golang/"
+    path = "${var.repo_dir}/lambda_golang/"
     commands = ["${local.go_build_flags} go build ./cmd/landing_metadata", ":zip"]
     patterns = ["landing_metadata"]
   }]
@@ -101,7 +101,7 @@ module fetch_story_lambda {
   runtime       = "go1.x"
 
   source_path = [{
-    path = "${path.module}/../lambda_golang/"
+    path = "${var.repo_dir}/lambda_golang/"
     commands = ["${local.go_build_flags} go build ./cmd/story", ":zip"]
     patterns = ["story"]
   }]

cloud_module/domain.tf cloud_module/pipeline/domain.tf

@@ -34,7 +34,7 @@ module "stories_queue_consumer_lambda" {
   handler       = "stories"
   runtime     = "go1.x"
   source_path = [{
-    path = "${path.module}/../lambda_golang/"
+    path = "${var.repo_dir}/lambda_golang/"
     commands = ["${local.go_build_flags} go build ./cmd/stories", ":zip"]
     patterns = ["stories"]
   }]

cloud_module/global_ssm.tf cloud_module/pipeline/global_ssm.tf

@@ -19,6 +19,11 @@ variable environment_name {
   description = "Empty string for Production, otherwise the environment name e.g. dev, stage, etc, make sure to use lowercase (s3 bucket only allows lower)"
 }
 
+variable repo_dir {
+  type = string
+  description = "The absolute path of git repository path"
+}
+
 locals {
   project_name = var.environment_name != "" ? "${var.project_alias}-${var.environment_name}" : "${var.project_alias}"
   environment = var.environment_name != "" ? var.environment_name : "prod"