Merge branch 'revert'

Author: Kai Hendry

.gitignore

@@ -0,0 +1 @@
+lambda2sns

.travis.yml

@@ -1,30 +1,31 @@
 sudo: required
 language: go
 go:
-- tip
+  - tip
 before_install:
-- curl https://raw.githubusercontent.com/apex/apex/master/install.sh | sudo sh
+  - pip install --user awscli
+  - pip install --user aws-sam-cli
 install:
-- go get -t ./...
+  - go get -t ./...
 deploy:
- # dev
- - provider: script
-   script: make dev
-   skip_cleanup: true
-   on:
-      branch: master
- # demo
- - provider: script
-   script: AWS_ACCESS_KEY_ID=$AWS_915001051872_ID AWS_SECRET_ACCESS_KEY=$AWS_915001051872_SECRET make demo
-   skip_cleanup: true
-   on:
-      branch: master
- # production
- - provider: script
-   script: AWS_ACCESS_KEY_ID=$AWS_192458993663_ID AWS_SECRET_ACCESS_KEY=$AWS_192458993663_SECRET make prod
-   skip_cleanup: true
-   on:
+  # dev
+  - provider: script
+    script: make deploy
+    skip_cleanup: true
+    on:
       branch: master
+  # demo
+  - provider: script
+    script: AWS_ACCESS_KEY_ID=$AWS_915001051872_ID AWS_SECRET_ACCESS_KEY=$AWS_915001051872_SECRET make demo
+    skip_cleanup: true
+    on:
+       branch: master
+  # production
+  - provider: script
+    script: AWS_ACCESS_KEY_ID=$AWS_192458993663_ID AWS_SECRET_ACCESS_KEY=$AWS_192458993663_SECRET make prod
+    skip_cleanup: true
+    on:
+       branch: master
 
 env:
   - GO111MODULE=on

Makefile

@@ -1,29 +1,33 @@
-all:
-	@echo make {dev,demo,prod} to deploy
+PROJECT = $(shell basename $(CURDIR))
+STACK_NAME ?= $(PROJECT)
+AWS_REGION = ap-southeast-1
+DEPLOY_S3_BUCKET = dev-media-unee-t
+DEPLOY_S3_PREFIX = lambda2sns
 
-logs:
-	apex -r ap-southeast-1 --env dev logs -f
+.PHONY: deps clean build test deploy
 
-dev:
-	@echo $$AWS_ACCESS_KEY_ID
-	apex -r ap-southeast-1 --env dev deploy
+# https://blog.deleu.dev/leveraging-aws-sqs-retry-mechanism-lambda/
 
-demo:
-	@echo $$AWS_ACCESS_KEY_ID
-	apex -r ap-southeast-1 --env demo deploy
+deps:
+	go mod tidy
 
-prod:
-	@echo $$AWS_ACCESS_KEY_ID
-	apex -r ap-southeast-1 --env prod deploy
+build: deps
+	GOOS=linux GOARCH=amd64 go build -o lambda2sns .
 
-testdev:
-	apex -r ap-southeast-1 --env dev invoke simple < event.json
+test:
+	go test ./...
 
-testdemo:
-	apex -r ap-southeast-1 --env demo invoke simple < event.json
+logs:
+	sam logs -n alambda_simple -t
 
-testprod:
-	apex -r ap-southeast-1 --env prod invoke simple < event.json
+destroy:
+	aws cloudformation delete-stack \
+		--stack-name $(STACK_NAME)
 
+deploy: build
+	sam validate --template template.yaml
+	sam package --template-file template.yaml --s3-bucket $(DEPLOY_S3_BUCKET) --s3-prefix $(DEPLOY_S3_PREFIX) --output-template-file packaged.yaml
+	sam deploy --template-file ./packaged.yaml --stack-name $(STACK_NAME) --capabilities CAPABILITY_IAM
 
-.PHONY: dev demo prod testdev testdemo testprod
+lint:
+	cfn-lint template.yaml

README.md

@@ -1,5 +1,13 @@
 [![Build Status](https://travis-ci.org/unee-t/lambda2sns.svg?branch=master)](https://travis-ci.org/unee-t/lambda2sns)
 
+<img src="https://media.dev.unee-t.com/2019-08-02/lambda2sns.png" alt="Lambda2sns">
+
+lambda2sns originally started life as a bridge for [Aurora CALL
+mysql.lambda_async](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Integrating.Lambda.html)
+payloads to SNS to be subscribed to. It has evolved to do a lot more, with most
+of the complexity coming from the requirement to write results back to the
+originating database.
+
 	lambda: arn:aws:lambda:ap-southeast-1:812644853088:function:alambda_simple
 	sns: arn:aws:sns:ap-southeast-1:812644853088:atest
 
@@ -25,12 +33,6 @@ Then you should get an email of the JSON payload.
 
 # Setup
 
-## lambda role
-
-Assuming project.demo.json is created with correct profile.
-
-	apex -r ap-southeast-1 --env demo init
-
 ## sns topic
 
 	[hendry@t480s alambda]$ aws --profile uneet-demo sns create-topic --name atest

event.json

@@ -72,6 +72,26 @@ func main() {
 	lambda.Start(handler)
 }
 
+type SQSevent struct {
+	Records []struct {
+		MessageID     string `json:"messageId"`
+		ReceiptHandle string `json:"receiptHandle"`
+		Body          string `json:"body"`
+		Attributes    struct {
+			ApproximateReceiveCount          string `json:"ApproximateReceiveCount"`
+			SentTimestamp                    string `json:"SentTimestamp"`
+			SenderID                         string `json:"SenderId"`
+			ApproximateFirstReceiveTimestamp string `json:"ApproximateFirstReceiveTimestamp"`
+		} `json:"attributes"`
+		MessageAttributes struct {
+		} `json:"messageAttributes"`
+		Md5OfBody      string `json:"md5OfBody"`
+		EventSource    string `json:"eventSource"`
+		EventSourceARN string `json:"eventSourceARN"`
+		AwsRegion      string `json:"awsRegion"`
+	} `json:"Records"`
+}
+
 func handler(ctx context.Context, evt json.RawMessage) error {
 
 	c := withRequestID{}
@@ -84,27 +104,44 @@ func handler(ctx context.Context, evt json.RawMessage) error {
 		log.Warn("no requestID context")
 	}
 
-	// if err := DB.Ping(); err != nil {
-	// 	c.log.WithError(err).Fatal("failed to ping DB")
-	// }
-	// c.log.WithField("evt", evt).Info("ping")
-
+	var sqsMessage SQSevent
 	var dat map[string]interface{}
-	if err := json.Unmarshal(evt, &dat); err != nil {
-		return err
+
+	// Check if SQS event https://github.com/unee-t/lambda2sns/issues/21
+	err := json.Unmarshal(evt, &sqsMessage)
+	if err == nil && len(sqsMessage.Records) > 0 {
+		log.WithField("body", sqsMessage.Records[0].Body).Info("SQS interface")
+		err = json.Unmarshal([]byte(sqsMessage.Records[0].Body), &dat)
+		if err != nil {
+			return err
+		}
+	} else {
+		log.Info("Lambda interface")
+		err = json.Unmarshal(evt, &dat)
+		if err != nil {
+			return err
+		}
 	}
 
+	log.WithField("dat", dat).Info("Payload")
+
+	// What type of payload is this?
 	_, actionType := dat["actionType"].(string)
+	// Use dat to replace evt, since it might be parsed out of SQS
+	evt, err = json.Marshal(dat)
+	if err != nil {
+		return err
+	}
 
 	if actionType {
-		c.log.WithField("payload", evt).Info("actionType")
+		c.log.WithField("evt", evt).Info("actionType")
 		err := c.actionTypeDB(evt)
 		if err != nil {
-			// c.log.WithError(err).Error("actionTypeDB")
+			c.log.WithError(err).Error("actionTypeDB")
 			return err
 		}
 	} else {
-		c.log.WithField("payload", evt).Info("postChangeMessage")
+		c.log.WithField("evt", evt).Info("postChangeMessage")
 		err := c.postChangeMessage(evt)
 		if err != nil {
 			c.log.WithError(err).Error("postChangeMessage")
@@ -130,6 +167,8 @@ func (c withRequestID) actionTypeDB(evt json.RawMessage) (err error) {
 
 	var act actionType
 
+	log.WithField("evt", evt).Info("in actiontypeDB")
+
 	if err := json.Unmarshal(evt, &act); err != nil {
 		c.log.WithError(err).Fatal("unable to unmarshall payload")
 		return err

findError.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+source ./.env
+test -f "$1" || exit
+if jq -e . >/dev/null 2>&1 <"$1"; then
+aws --profile uneet-dev sqs send-message --queue-url $QUEUE --message-body file://$1
+else
+	echo $1 is not JSON
+fi

functions/simple/main.go main.gofunctions/simple/main.go renamed to main.go

@@ -0,0 +1,86 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: SAM Lambda2SNS
+
+Globals:
+  Function:
+    Runtime: go1.x
+    Timeout: 60
+
+Resources:
+  SAMLambda:
+    Type: AWS::Serverless::Function
+    Properties:
+      VpcConfig:
+        SecurityGroupIds:
+          - sg-66390301
+        SubnetIds:
+          - subnet-0ff046ccc4e3b6281
+          - subnet-0938728dfb344b970
+          - subnet-0e123bd457c082cff
+      Role: !GetAtt LambdaRole.Arn
+      CodeUri: .
+      FunctionName: alambda_simple
+      Tracing: Active
+      Handler: lambda2sns
+      Runtime: go1.x
+      DeadLetterQueue:
+        Type: SQS
+        TargetArn: !GetAtt [RetryDeleteQueue, Arn]
+
+  RetrySAMLambda:
+    Type: AWS::Serverless::Function
+    Properties:
+      Role: !GetAtt LambdaRole.Arn
+      CodeUri: .
+      FunctionName: alambda_simple_retry
+      VpcConfig:
+        SecurityGroupIds:
+          - sg-66390301
+        SubnetIds:
+          - subnet-0ff046ccc4e3b6281
+          - subnet-0938728dfb344b970
+          - subnet-0e123bd457c082cff
+      Tracing: Active
+      Handler: lambda2sns
+      Runtime: go1.x
+      Events:
+        SQSEvent:
+          Type: SQS
+          Properties:
+            Queue: !GetAtt [RetryDeleteQueue, Arn]
+            BatchSize: 1
+
+  RetryDeleteDeadLetterQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      MessageRetentionPeriod: 1209600
+
+  RetryDeleteQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      VisibilityTimeout: 90
+      MessageRetentionPeriod: 604800
+      RedrivePolicy:
+        deadLetterTargetArn: !GetAtt [RetryDeleteDeadLetterQueue, Arn]
+        maxReceiveCount: 3
+
+  LambdaRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: [lambda.amazonaws.com]
+            Action: ['sts:AssumeRole']
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/AWSLambdaExecute
+        - arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess
+        - arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaRole
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole
+        - arn:aws:iam::aws:policy/AmazonSNSFullAccess
+        - arn:aws:iam::aws:policy/AmazonSQSFullAccess
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaSQSQueueExecutionRole