Skip to content

Commit d816fe6

Browse files
rivernewsrivernews
committed
Complete tf surgery; Identify all TODOs in golang
For #25
1 parent ec37c70 commit d816fe6

File tree

19 files changed

+348
-216
lines changed

19 files changed

+348
-216
lines changed

‎.gitignore

+3-1
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22
**/builds/**
33

44
lambda_golang/landing
5+
lambda_golang/landing_s3_trigger
6+
lambda_golang/landing_metadata_cronjob
57
lambda_golang/stories
6-
lambda_golang/landing_metadata
78
lambda_golang/story
9+
lambda_golang/stories_finalizer
810
venv
911

1012
# Binaries for programs and plugins

‎cloud_environments/terraform.sh

+4-2
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,11 @@ set +o allexport
2222
if (
2323
cd $GOLANG_SRC_DIR && \
2424
go build ./cmd/landing && \
25-
go build ./cmd/landing_metadata && \
25+
go build ./cmd/landing_s3_trigger && \
26+
go build ./cmd/landing_metadata_cronjob && \
2627
go build ./cmd/stories && \
2728
go build ./cmd/story && \
29+
go build ./cmd/stories_finalizer && \
2830
cd $PYTHON_SRC_DIR && python -m compileall layer src
2931
); then
3032
cd $DEPLOY_DIR
@@ -37,7 +39,7 @@ if (
3739
# https://github.com/terraform-aws-modules/terraform-aws-step-functions/issues/20
3840
# terraform "$@" \
3941
# -target=module.main.module.scraper_lambda \
40-
# -target=module.main.module.landing_parse_metadata_lambda
42+
# -target=module.main.module.landing_metadata_cronjob_lambda
4143

4244
terraform "$@"
4345
else

‎cloud_module/dynamodb/table.tf

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
resource "aws_ssm_parameter" "media_table" {
22
name = "/app/media-literacy/table"
33
type = "String"
4-
value = aws_dynamodb_table.media_table.arn
4+
value = "${aws_dynamodb_table.media_table.arn},${aws_dynamodb_table.media_table.id}"
55
}
66

77
// https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table#attributes-reference

‎cloud_module/pipeline/global_ssm.tf

+4
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,8 @@ data aws_ssm_parameter media_table {
99
locals {
1010
newssite_economy_tokens = split(",", data.aws_ssm_parameter.newssite_economy.value)
1111
newssite_economy_alias = local.newssite_economy_tokens[2]
12+
13+
_media_table_tokens = split(",", data.aws_ssm_parameter.media_table)
14+
media_table_arn = local._media_table_tokens[0]
15+
media_table_id = local._media_table_tokens[1]
1216
}

‎cloud_module/pipeline/landing_s3_trigger.tf

-38
This file was deleted.

‎cloud_module/pipeline/s3_triggers.tf

+152
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
resource "aws_s3_bucket_notification" "bucket_notification" {
2+
bucket = data.aws_s3_bucket.archive.id
3+
4+
lambda_function {
5+
lambda_function_arn = module.landing_s3_trigger_lambda.lambda_function_arn
6+
events = ["s3:ObjectCreated:*"]
7+
filter_prefix = "${local.newssite_economy_alias}/"
8+
filter_suffix = "landing.html"
9+
}
10+
11+
lambda_function {
12+
lambda_function_arn = module.landing_metadata_s3_trigger_lambda.lambda_function_arn
13+
events = ["s3:ObjectCreated:*"]
14+
filter_prefix = "${local.newssite_economy_alias}/"
15+
filter_suffix = "/metadata.json"
16+
}
17+
18+
depends_on = [
19+
aws_lambda_permission.allow_bucket_trigger_by_landing,
20+
aws_lambda_permission.allow_bucket_trigger_by_landing_metadata
21+
]
22+
}
23+
24+
resource "aws_lambda_permission" "allow_bucket_trigger_by_landing" {
25+
statement_id = "AllowExecutionFromS3Bucket"
26+
action = "lambda:InvokeFunction"
27+
function_name = module.landing_s3_trigger_lambda.lambda_function_arn
28+
principal = "s3.amazonaws.com"
29+
source_arn = data.aws_s3_bucket.archive.arn
30+
}
31+
32+
resource "aws_lambda_permission" "allow_bucket_trigger_by_landing_metadata" {
33+
statement_id = "AllowExecutionFromS3Bucket"
34+
action = "lambda:InvokeFunction"
35+
function_name = module.landing_metadata_s3_trigger_lambda.lambda_function_arn
36+
principal = "s3.amazonaws.com"
37+
source_arn = data.aws_s3_bucket.archive.arn
38+
}
39+
40+
module "landing_s3_trigger_lambda" {
41+
source = "terraform-aws-modules/lambda/aws"
42+
create_function = true
43+
function_name = "${local.project_name}-landing-s3-trigger-lambda"
44+
description = "Put a landing page in db"
45+
handler = "landing_s3_trigger"
46+
runtime = "go1.x"
47+
48+
source_path = [{
49+
path = "${var.repo_dir}/lambda_golang/"
50+
commands = ["${local.go_build_flags} go build ./cmd/landing_s3_trigger", ":zip"]
51+
patterns = ["landing_s3_trigger"]
52+
}]
53+
54+
timeout = 900
55+
cloudwatch_logs_retention_in_days = 7
56+
publish = true
57+
58+
attach_policy_statements = true
59+
policy_statements = {
60+
allow_db_put = {
61+
effect = "Allow",
62+
actions = [
63+
"dynamodb:PutItem",
64+
],
65+
resources = [media_table_arn]
66+
}
67+
}
68+
69+
environment_variables = {
70+
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
71+
LOG_LEVEL = "DEBUG"
72+
DEBUG = "true"
73+
DYNAMODB_TABLE_ID = media_table_id
74+
}
75+
76+
tags = {
77+
Project = local.project_name
78+
}
79+
}
80+
81+
module "landing_metadata_s3_trigger_lambda" {
82+
source = "terraform-aws-modules/lambda/aws"
83+
84+
create_function = true
85+
function_name = "${local.project_name}-fetch-stories"
86+
description = "Fetch ${local.project_name} stories; triggered by metadata.json creation"
87+
handler = "stories"
88+
runtime = "go1.x"
89+
source_path = [{
90+
path = "${var.repo_dir}/lambda_golang/"
91+
commands = ["${local.go_build_flags} go build ./cmd/stories", ":zip"]
92+
patterns = ["stories"]
93+
}]
94+
publish = true
95+
96+
timeout = 900
97+
cloudwatch_logs_retention_in_days = 7
98+
99+
reserved_concurrent_executions = -1
100+
101+
# allow lambda to invoke step function
102+
attach_policy_json = true
103+
policy_json = <<EOF
104+
{
105+
"Version": "2012-10-17",
106+
"Statement": [
107+
{
108+
"Effect": "Allow",
109+
"Action": [
110+
"states:StartExecution"
111+
],
112+
"Resource": ["${module.batch_stories_sfn.state_machine_arn}"]
113+
}
114+
]
115+
}
116+
EOF
117+
118+
attach_policy_statements = true
119+
policy_statements = {
120+
s3_archive_bucket = {
121+
effect = "Allow",
122+
actions = [
123+
"s3:GetObject"
124+
],
125+
resources = [
126+
"${data.aws_s3_bucket.archive.arn}/*",
127+
]
128+
}
129+
s3_archive_bucket_check_404 = {
130+
effect = "Allow",
131+
actions = [
132+
"s3:ListBucket",
133+
],
134+
resources = [
135+
"${data.aws_s3_bucket.archive.arn}",
136+
]
137+
}
138+
}
139+
140+
environment_variables = {
141+
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
142+
LOGLEVEL = "DEBUG"
143+
ENV = local.environment
144+
145+
S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
146+
SFN_ARN = module.batch_stories_sfn.state_machine_arn
147+
}
148+
149+
tags = {
150+
Project = local.project_name
151+
}
152+
}

‎cloud_module/pipeline/scheduler.tf

+83
Original file line numberDiff line numberDiff line change
@@ -60,3 +60,86 @@ data "aws_iam_policy_document" "scheduler" {
6060
}
6161
}
6262
}
63+
64+
65+
resource "aws_cloudwatch_event_rule" "landing_metadata_scheduler" {
66+
count = var.environment_name == "" ? 1 : 0
67+
68+
name = "${local.project_name}-schedule-start-metadata-for-landing"
69+
# schedule experssion
70+
# https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html
71+
schedule_expression = "rate(1 hours)"
72+
description = "Every hour to give courtesy to the website"
73+
}
74+
75+
resource "aws_cloudwatch_event_target" "landing_metadata_scheduler_event_target" {
76+
count = var.environment_name == "" ? 1 : 0
77+
78+
target_id = "${local.project_name}-schedule-start-metadata-for-landing-event-target"
79+
rule = aws_cloudwatch_event_rule.landing_metadata_scheduler.0.name
80+
arn = module.landing_metadata_cronjob_lambda.lambda_function_arn
81+
}
82+
83+
module landing_metadata_cronjob_lambda {
84+
source = "terraform-aws-modules/lambda/aws"
85+
create_function = true
86+
function_name = "${local.project_name}-batch-stories-fetch-parse"
87+
description = "Query landing pages in db; compute & archive their metadata"
88+
handler = "landing_metadata_cronjob"
89+
runtime = "go1.x"
90+
91+
source_path = [{
92+
path = "${var.repo_dir}/lambda_golang/"
93+
commands = ["${local.go_build_flags} go build ./cmd/landing_metadata_cronjob", ":zip"]
94+
patterns = ["landing_metadata_cronjob"]
95+
}]
96+
97+
timeout = 900
98+
cloudwatch_logs_retention_in_days = 7
99+
100+
publish = true
101+
102+
attach_policy_statements = true
103+
policy_statements = {
104+
allow_db_query = {
105+
effect = "Allow",
106+
actions = [
107+
"dynamodb:Query",
108+
"dynamodb:UpdateItem",
109+
],
110+
resources = [media_table_arn]
111+
}
112+
s3_archive_bucket = {
113+
effect = "Allow",
114+
actions = [
115+
"s3:PutObject",
116+
],
117+
resources = [
118+
"${data.aws_s3_bucket.archive.arn}/*",
119+
]
120+
}
121+
# enable getting 404 instead of 403 in case of not found
122+
# https://stackoverflow.com/a/19808954/9814131
123+
s3_archive_bucket_check_404 = {
124+
effect = "Allow",
125+
actions = [
126+
"s3:ListBucket",
127+
],
128+
resources = [
129+
"${data.aws_s3_bucket.archive.arn}",
130+
]
131+
}
132+
}
133+
134+
environment_variables = {
135+
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
136+
LOG_LEVEL = "DEBUG"
137+
DEBUG = "true"
138+
S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
139+
DYNAMODB_TABLE_ID = media_table_id
140+
}
141+
142+
tags = {
143+
Project = local.project_name
144+
}
145+
}

‎cloud_module/pipeline/sfn_def/batch_stories_def.json

+6
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@
3232
}
3333
}
3434
},
35+
"Next": "Stories-Finalizer",
36+
"End": false
37+
},
38+
"Stories-Finalizer": {
39+
"Type":"Task",
40+
"Resource": "${STORIES_FINALIZER_LAMBDA_ARN}",
3541
"End": true
3642
}
3743
}

0 commit comments

Comments
 (0)