fix compile error; progress in metadata cronjob add query

Author: rivernews

lambda_golang/cmd/landing_metadata_cronjob/main.go

@@ -36,16 +36,12 @@ type LambdaResponse struct {
 }
 
 func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse, error) {
-
-	// TODO: create a cronjob rule in tf
-	// TODO: point cronjob to this func
-
-	// TODO: change from s3Event to fetch from dynamoDB (query)
+	items := cloud.DynamoDBQueryWaitingMetadata(ctx, newssite.DOCTYPE_LANDING)
 
 	GoTools.Logger("INFO", "Landing page metadata.json generator launched")
 
-	for _, record := range s3Event.Records {
-		landingPageS3Key := record.S3.Object.URLDecodedKey
+	for _, landingItem := range *items {
+		landingPageS3Key := landingItem.S3Key
 		GoTools.Logger("INFO", fmt.Sprintf("Captured landing page at %s", landingPageS3Key))
 
 		landingPageHtmlText := cloud.Pull(landingPageS3Key)

lambda_golang/cmd/landing_s3_trigger/main.go

@@ -16,8 +16,6 @@ import (
 	//"github.com/aws/aws-sdk-go-v2/config"
 	//"github.com/aws/aws-sdk-go-v2/service/sqs"
 
-	"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
-
 	// local packages
 	"github.com/rivernews/media-literacy/pkg/cloud"
 	"github.com/rivernews/media-literacy/pkg/newssite"
@@ -43,21 +41,17 @@ func HandleRequest(ctx context.Context, s3Event events.S3Event) (LambdaResponse,
 		landingPageS3Key := record.S3.Object.URLDecodedKey
 		GoTools.Logger("INFO", fmt.Sprintf("Captured landing page at %s", landingPageS3Key))
 
-		// remember to configure permissions in tf
-
-		// TODO: let landing s3 trigger switch to point to this func
-
-		out := cloud.DynamoDBPutItem(ctx, map[string]any{
-			"s3Key":   landingPageS3Key,
-			"docType": string(newssite.DOCTYPE_LANDING),
-			"events": []newssite.MediaTableItemEvent{
+		out := cloud.DynamoDBPutItem(ctx, newssite.MediaTableItem{
+			S3Key:   landingPageS3Key,
+			DocType: newssite.DOCTYPE_LANDING,
+			Events: []newssite.MediaTableItemEvent{
 				{
 					EventName: newssite.EVENT_LANDING_PAGE_FETCHED,
 					Detail:    fmt.Sprintf("Landing page html stored in S3 and triggers event to store in media table"),
 					EventTime: newssite.Now(),
 				},
 			},
-			"isDocTypeWaitingForMetadata": &types.AttributeValueMemberS{Value: string(newssite.DOCTYPE_LANDING)},
+			IsDocTypeWaitingForMetadata: newssite.DOCTYPE_LANDING,
 		})
 
 		GoTools.Logger("DEBUG", fmt.Sprintf("```%s```\n", GoTools.AsJson(out)))

lambda_golang/pkg/cloud/dynamodb_common.go

@@ -7,7 +7,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue"
 	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
 	"github.com/rivernews/GoTools"
+	"github.com/rivernews/media-literacy/pkg/newssite"
 
 	"github.com/google/uuid"
 )
@@ -24,27 +26,30 @@ func SharedDynamoDBClient() *dynamodb.Client {
 	return dynamoDBClient
 }
 
-func DynamoDBPutItem(ctx context.Context, item map[string]any) *dynamodb.PutItemOutput {
+func getTableName() string {
 	tableID := GoTools.GetEnvVarHelper("DYNAMODB_TABLE_ID")
 	if tableID == "" {
 		GoTools.Logger("ERROR", "DYNAMODB_TABLE_ID is required please set this env var")
 	}
+	return tableID
+}
 
-	if _, exist := item["uuid"]; !exist {
-		item["uuid"] = uuid.New().String()
+func DynamoDBPutItem(ctx context.Context, item any) *dynamodb.PutItemOutput {
+	dynamoDBItem, err := attributevalue.MarshalMap(item)
+	if err != nil {
+		GoTools.Logger("ERROR", err.Error())
 	}
 
-	if _, exist := item["createdAt"]; !exist {
-		// TODO
+	if _, exist := dynamoDBItem["uuid"]; !exist {
+		dynamoDBItem["uuid"] = uuid.New().String()
 	}
 
-	dynamoDBItem, err := attributevalue.MarshalMap(item)
-	if err != nil {
-		GoTools.Logger("ERROR", err.Error())
+	if _, exist := dynamoDBItem["createdAt"]; !exist {
+		dynamoDBItem["createdAt"] = newssite.Now()
 	}
 
 	out, err := dynamoDBClient.PutItem(ctx, &dynamodb.PutItemInput{
-		TableName: aws.String(tableID),
+		TableName: aws.String(getTableName()),
 		Item:      dynamoDBItem,
 	})
 
@@ -54,3 +59,29 @@ func DynamoDBPutItem(ctx context.Context, item map[string]any) *dynamodb.PutItem
 
 	return out
 }
+
+func DynamoDBQueryWaitingMetadata(ctx context.Context, docType newssite.DocType) *[]newssite.MediaTableItem {
+	out, err := dynamoDBClient.Query(ctx, &dynamodb.QueryInput{
+		TableName:              aws.String(getTableName()),
+		IndexName:              aws.String(string(newssite.METADATA_INDEX)),
+		KeyConditionExpression: aws.String("isDocTypeWaitingForMetadata = :isDocTypeWaitingForMetadata and createdAt < :createdAt"),
+		ExpressionAttributeValues: map[string]types.AttributeValue{
+			":isDocTypeWaitingForMetadata": &types.AttributeValueMemberS{Value: string(docType)},
+			":createdAt":                   &types.AttributeValueMemberS{Value: newssite.Now()},
+		},
+	})
+	if err != nil {
+		GoTools.Logger("ERROR", err.Error())
+	}
+
+	var results []newssite.MediaTableItem
+	attributevalue.UnmarshalListOfMaps(out.Items, &results)
+
+	return &results
+}
+
+func DynamoDBUpdateItem() {
+	// TODO: remove attribute
+
+	// TODO: add event to `events`
+}

lambda_golang/pkg/newssite/table.go

@@ -3,8 +3,8 @@ package newssite
 type EventName string
 
 const (
-	// `landing`
-	EVENT_LANDING_PAGE_REQUESTED EventName = "LANDING_PAGE_REQUESTED" ✅ 📩
+	// `landing`  ✅ 📩
+	EVENT_LANDING_PAGE_REQUESTED EventName = "LANDING_PAGE_REQUESTED"
 	// +`landing_s3_trigger` (put in db) ✅
 	EVENT_LANDING_PAGE_FETCHED EventName = "LANDING_PAGE_FETCHED"
 	// @`landing_metadata` -> `landing_metadata_cronjob` (query db; store metadata) ✅ (cronjob trigger) ✅
@@ -29,4 +29,18 @@ type DocType string
 
 const (
 	DOCTYPE_LANDING DocType = "LANDING"
+	DOCTYPE_STORY   DocType = "STORY"
 )
+
+type TableIndex string
+
+const (
+	METADATA_INDEX TableIndex = "metadataIndex"
+)
+
+type MediaTableItem struct {
+	S3Key                       string                `json:"s3Key"`
+	DocType                     DocType               `json:"docType"`
+	Events                      []MediaTableItemEvent `json:"events"`
+	IsDocTypeWaitingForMetadata DocType               `json:"isDocTypeWaitingForMetadata,omitempty"`
+}