Skip to content

Commit 4ec0693

Browse files
rivernewsrivernews
authored
Ready to test (#36)
* Ready to test * Fix db field first char not lowercase Tracked by #25 (comment) * Fix permission of db index, S3 pull Tracked by #25 (comment) * All tests complete Tracked by #25 (comment)
1 parent 2c8f841 commit 4ec0693

File tree

24 files changed

+444
-369
lines changed

24 files changed

+444
-369
lines changed

‎.gitignore

+5-6
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
**credential**
22
**/builds/**
33

4-
lambda_golang/landing
5-
lambda_golang/landing_s3_trigger
6-
lambda_golang/landing_metadata_cronjob
7-
lambda_golang/stories
8-
lambda_golang/story
9-
lambda_golang/stories_finalizer
4+
lambda_golang/*
5+
!lambda_golang/go.mod
6+
!lambda_golang/go.sum
7+
!lambda_golang/*/
8+
!lambda_golang/*/**
109
venv
1110

1211
# Binaries for programs and plugins

‎cloud_environments/terraform.sh

-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ set +o allexport
2222
if (
2323
cd $GOLANG_SRC_DIR && \
2424
go build ./cmd/landing && \
25-
go build ./cmd/landing_s3_trigger && \
2625
go build ./cmd/landing_metadata_cronjob && \
2726
go build ./cmd/stories && \
2827
go build ./cmd/story && \

‎cloud_module/dynamodb/table.tf

+15-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ resource "aws_ssm_parameter" "media_table" {
66

77
// https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table#attributes-reference
88
resource "aws_dynamodb_table" "media_table" {
9-
name = "Mediatable"
9+
name = "${title(replace("${var.project_alias}_${var.environment_name}", "-", "_"))}"
1010
billing_mode = "PROVISIONED"
1111
read_capacity = 20
1212
write_capacity = 20
@@ -23,8 +23,12 @@ resource "aws_dynamodb_table" "media_table" {
2323
type = "S"
2424
}
2525

26+
attribute {
27+
name = "s3Key"
28+
type = "S"
29+
}
30+
2631
// other fields
27-
// S3 key
2832
// docType = {landing | story | landingMetadata | ...}
2933
// events
3034

@@ -58,6 +62,15 @@ resource "aws_dynamodb_table" "media_table" {
5862
non_key_attributes = ["s3Key"]
5963
}
6064

65+
global_secondary_index {
66+
name = "s3KeyIndex"
67+
hash_key = "s3Key"
68+
range_key = "createdAt"
69+
write_capacity = 10
70+
read_capacity = 10
71+
projection_type = "KEYS_ONLY"
72+
}
73+
6174
tags = {
6275
Project = local.project_name
6376
Environment = var.environment_name

‎cloud_module/pipeline/global_ssm.tf

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ locals {
1010
newssite_economy_tokens = split(",", data.aws_ssm_parameter.newssite_economy.value)
1111
newssite_economy_alias = local.newssite_economy_tokens[2]
1212

13-
_media_table_tokens = split(",", data.aws_ssm_parameter.media_table)
13+
_media_table_tokens = split(",", data.aws_ssm_parameter.media_table.value)
1414
media_table_arn = local._media_table_tokens[0]
1515
media_table_id = local._media_table_tokens[1]
1616
}

‎cloud_module/pipeline/lambda.tf

+9-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ module "step_function" {
5959
module "scraper_lambda" {
6060
source = "terraform-aws-modules/lambda/aws"
6161
create_function = true
62-
function_name = "${local.project_name}-scraper-lambda"
62+
function_name = "${local.project_name}-landing-lambda"
6363
description = "Lambda function for scraping"
6464
handler = "landing"
6565
runtime = "go1.x"
@@ -82,6 +82,13 @@ module "scraper_lambda" {
8282

8383
attach_policy_statements = true
8484
policy_statements = {
85+
allow_db_query = {
86+
effect = "Allow",
87+
actions = [
88+
"dynamodb:PutItem"
89+
],
90+
resources = [local.media_table_arn]
91+
}
8592
s3_archive_bucket = {
8693
effect = "Allow",
8794
actions = [
@@ -98,6 +105,7 @@ module "scraper_lambda" {
98105
S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
99106

100107
NEWSSITE_ECONOMY = data.aws_ssm_parameter.newssite_economy.value
108+
DYNAMODB_TABLE_ID = local.media_table_id
101109
}
102110

103111
tags = {

‎cloud_module/pipeline/s3_triggers.tf

+12-58
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,6 @@
11
resource "aws_s3_bucket_notification" "bucket_notification" {
22
bucket = data.aws_s3_bucket.archive.id
33

4-
lambda_function {
5-
lambda_function_arn = module.landing_s3_trigger_lambda.lambda_function_arn
6-
events = ["s3:ObjectCreated:*"]
7-
filter_prefix = "${local.newssite_economy_alias}/"
8-
filter_suffix = "landing.html"
9-
}
10-
114
lambda_function {
125
lambda_function_arn = module.landing_metadata_s3_trigger_lambda.lambda_function_arn
136
events = ["s3:ObjectCreated:*"]
@@ -16,19 +9,10 @@ resource "aws_s3_bucket_notification" "bucket_notification" {
169
}
1710

1811
depends_on = [
19-
aws_lambda_permission.allow_bucket_trigger_by_landing,
2012
aws_lambda_permission.allow_bucket_trigger_by_landing_metadata
2113
]
2214
}
2315

24-
resource "aws_lambda_permission" "allow_bucket_trigger_by_landing" {
25-
statement_id = "AllowExecutionFromS3Bucket"
26-
action = "lambda:InvokeFunction"
27-
function_name = module.landing_s3_trigger_lambda.lambda_function_arn
28-
principal = "s3.amazonaws.com"
29-
source_arn = data.aws_s3_bucket.archive.arn
30-
}
31-
3216
resource "aws_lambda_permission" "allow_bucket_trigger_by_landing_metadata" {
3317
statement_id = "AllowExecutionFromS3Bucket"
3418
action = "lambda:InvokeFunction"
@@ -37,52 +21,11 @@ resource "aws_lambda_permission" "allow_bucket_trigger_by_landing_metadata" {
3721
source_arn = data.aws_s3_bucket.archive.arn
3822
}
3923

40-
module "landing_s3_trigger_lambda" {
41-
source = "terraform-aws-modules/lambda/aws"
42-
create_function = true
43-
function_name = "${local.project_name}-landing-s3-trigger-lambda"
44-
description = "Put a landing page in db"
45-
handler = "landing_s3_trigger"
46-
runtime = "go1.x"
47-
48-
source_path = [{
49-
path = "${var.repo_dir}/lambda_golang/"
50-
commands = ["${local.go_build_flags} go build ./cmd/landing_s3_trigger", ":zip"]
51-
patterns = ["landing_s3_trigger"]
52-
}]
53-
54-
timeout = 900
55-
cloudwatch_logs_retention_in_days = 7
56-
publish = true
57-
58-
attach_policy_statements = true
59-
policy_statements = {
60-
allow_db_put = {
61-
effect = "Allow",
62-
actions = [
63-
"dynamodb:PutItem",
64-
],
65-
resources = [media_table_arn]
66-
}
67-
}
68-
69-
environment_variables = {
70-
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
71-
LOG_LEVEL = "DEBUG"
72-
DEBUG = "true"
73-
DYNAMODB_TABLE_ID = media_table_id
74-
}
75-
76-
tags = {
77-
Project = local.project_name
78-
}
79-
}
80-
8124
module "landing_metadata_s3_trigger_lambda" {
8225
source = "terraform-aws-modules/lambda/aws"
8326

8427
create_function = true
85-
function_name = "${local.project_name}-fetch-stories"
28+
function_name = "${local.project_name}-stories-lambda"
8629
description = "Fetch ${local.project_name} stories; triggered by metadata.json creation"
8730
handler = "stories"
8831
runtime = "go1.x"
@@ -117,6 +60,15 @@ EOF
11760

11861
attach_policy_statements = true
11962
policy_statements = {
63+
allow_db_put = {
64+
effect = "Allow",
65+
actions = [
66+
"dynamodb:UpdateItem",
67+
],
68+
resources = [
69+
local.media_table_arn,
70+
]
71+
}
12072
s3_archive_bucket = {
12173
effect = "Allow",
12274
actions = [
@@ -141,8 +93,10 @@ EOF
14193
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
14294
LOGLEVEL = "DEBUG"
14395
ENV = local.environment
96+
DEBUG = "true"
14497

14598
S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
99+
DYNAMODB_TABLE_ID = local.media_table_id
146100
SFN_ARN = module.batch_stories_sfn.state_machine_arn
147101
}
148102

‎cloud_module/pipeline/scheduler.tf

+7-3
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ resource "aws_cloudwatch_event_target" "landing_metadata_scheduler_event_target"
8383
module landing_metadata_cronjob_lambda {
8484
source = "terraform-aws-modules/lambda/aws"
8585
create_function = true
86-
function_name = "${local.project_name}-batch-stories-fetch-parse"
86+
function_name = "${local.project_name}-landing-metadata-cronjob-lambda"
8787
description = "Query landing pages in db; compute & archive their metadata"
8888
handler = "landing_metadata_cronjob"
8989
runtime = "go1.x"
@@ -107,11 +107,15 @@ module landing_metadata_cronjob_lambda {
107107
"dynamodb:Query",
108108
"dynamodb:UpdateItem",
109109
],
110-
resources = [media_table_arn]
110+
resources = [
111+
local.media_table_arn,
112+
"${local.media_table_arn}/index/metadataIndex"
113+
]
111114
}
112115
s3_archive_bucket = {
113116
effect = "Allow",
114117
actions = [
118+
"s3:GetObject",
115119
"s3:PutObject",
116120
],
117121
resources = [
@@ -136,7 +140,7 @@ module landing_metadata_cronjob_lambda {
136140
LOG_LEVEL = "DEBUG"
137141
DEBUG = "true"
138142
S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
139-
DYNAMODB_TABLE_ID = media_table_id
143+
DYNAMODB_TABLE_ID = local.media_table_id
140144
}
141145

142146
tags = {

‎cloud_module/pipeline/sfn_def/batch_stories_def.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"Parameters": {
1313
"story.$": "$$.Map.Item.Value",
1414
"newsSiteAlias.$": "$.newsSiteAlias",
15+
"landingPageUuid.$": "$.landingPageUuid",
1516
"landingPageTimeStamp.$": "$.landingPageTimeStamp"
1617
},
1718
"Iterator": {
@@ -32,8 +33,7 @@
3233
}
3334
}
3435
},
35-
"Next": "Stories-Finalizer",
36-
"End": false
36+
"Next": "Stories-Finalizer"
3737
},
3838
"Stories-Finalizer": {
3939
"Type":"Task",

‎cloud_module/pipeline/stories_sfn.tf

+19-4
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ module batch_stories_sfn {
1515
service_integrations = {
1616
lambda = {
1717
lambda = [
18-
module.fetch_story_lambda.lambda_function_arn
18+
module.fetch_story_lambda.lambda_function_arn,
1919
module.stories_finalizer_lambda.lambda_function_arn
2020
]
2121
}
@@ -31,7 +31,7 @@ module batch_stories_sfn {
3131
module fetch_story_lambda {
3232
source = "terraform-aws-modules/lambda/aws"
3333
create_function = true
34-
function_name = "${local.project_name}-fetch-story"
34+
function_name = "${local.project_name}-story-lambda"
3535
description = "Fetch and archive a story page"
3636
handler = "story"
3737
runtime = "go1.x"
@@ -49,6 +49,15 @@ module fetch_story_lambda {
4949

5050
attach_policy_statements = true
5151
policy_statements = {
52+
allow_db_put = {
53+
effect = "Allow",
54+
actions = [
55+
"dynamodb:PutItem",
56+
],
57+
resources = [
58+
local.media_table_arn,
59+
]
60+
}
5261
s3_archive_bucket = {
5362
effect = "Allow",
5463
actions = [
@@ -77,6 +86,8 @@ module fetch_story_lambda {
7786
LOG_LEVEL = "DEBUG"
7887
DEBUG = "true"
7988
S3_ARCHIVE_BUCKET = data.aws_s3_bucket.archive.id
89+
90+
DYNAMODB_TABLE_ID = local.media_table_id
8091
}
8192

8293
tags = {
@@ -107,17 +118,21 @@ module "stories_finalizer_lambda" {
107118
allow_db_put = {
108119
effect = "Allow",
109120
actions = [
121+
"dynamodb:Query",
110122
"dynamodb:UpdateItem",
111123
],
112-
resources = [media_table_arn]
124+
resources = [
125+
local.media_table_arn,
126+
"${local.media_table_arn}/index/s3KeyIndex"
127+
]
113128
}
114129
}
115130

116131
environment_variables = {
117132
SLACK_WEBHOOK_URL = var.slack_post_webhook_url
118133
LOG_LEVEL = "DEBUG"
119134
DEBUG = "true"
120-
DYNAMODB_TABLE_ID = media_table_id
135+
DYNAMODB_TABLE_ID = local.media_table_id
121136
}
122137

123138
tags = {

‎lambda_golang/cmd/landing/main.go

+14-2
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/rivernews/GoTools"
1111

1212
"github.com/rivernews/media-literacy/pkg/cloud"
13+
"github.com/rivernews/media-literacy/pkg/common"
1314
"github.com/rivernews/media-literacy/pkg/newssite"
1415
)
1516

@@ -29,7 +30,7 @@ type LambdaResponse struct {
2930
func HandleRequest(ctx context.Context, name LambdaEvent) (LambdaResponse, error) {
3031
newsSite := newssite.GetNewsSite("NEWSSITE_ECONOMY")
3132

32-
bodyText := newssite.Fetch(newsSite.LandingURL)
33+
bodyText := common.Fetch(newsSite.LandingURL)
3334
GoTools.Logger("INFO", "In golang runtime now!\n\n```\n "+bodyText[:500]+"\n ...```\n End of message")
3435

3536
// scraper
@@ -56,10 +57,21 @@ func HandleRequest(ctx context.Context, name LambdaEvent) (LambdaResponse, error
5657
GoTools.Logger("INFO", successMessage)
5758

5859
// S3 archive
60+
landingPageS3Key := fmt.Sprintf("%s/daily-headlines/%s/landing.html", newsSite.Alias, common.Now())
5961
cloud.Archive(cloud.ArchiveArgs{
6062
BodyText: bodyText,
61-
Key: fmt.Sprintf("%s/daily-headlines/%s/landing.html", newsSite.Alias, newssite.Now()),
63+
Key: landingPageS3Key,
6264
})
65+
out := cloud.DynamoDBPutItem(ctx, newssite.MediaTableItem{
66+
S3Key: landingPageS3Key,
67+
DocType: newssite.DOCTYPE_LANDING,
68+
Events: []newssite.MediaTableItemEvent{
69+
newssite.GetEventLandingPageFetched(newsSite.Alias, landingPageS3Key),
70+
newssite.GetEventLandingMetadataRequested(landingPageS3Key),
71+
},
72+
IsDocTypeWaitingForMetadata: newssite.DOCTYPE_LANDING,
73+
})
74+
GoTools.Logger("DEBUG", fmt.Sprintf("```%s```\n", GoTools.AsJson(out)))
6375

6476
return LambdaResponse{
6577
OK: true,

0 commit comments

Comments
 (0)