Merge pull request #1 from awslabs/master

Merge from Master

Author: chrisjameswilliams

Networking.md

@@ -4,15 +4,15 @@
 
 In this model, your Redshift cluster is in a VPC Subnet, and we recommend using [AWS Lambda VPC Endpoints](http://docs.aws.amazon.com/lambda/latest/dg/vpc.html) to manage access to your cluster. The following diagram represents the configuration required to successfully run this solution in a VPC:
 
-![VPC Connectivity](VPCConnectivity.png)
+![VPC Connectivity](img/VPCConnectivity.png)
 
 In this architecture, you expose your AWS Lambda function into a set of private VPC subnets, and then select a security group for your Lambda function. Ideally this would be the same security group as the Redshift cluster was using, but you can also use security group grants to enable access between two SG's. Your Lambda function must also have internet egress enabled so it can read its configuration from DynamoDB, and the easiest way to do this is to use [VPC NAT Gateway](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/vpc-nat-gateway.html). The following steps should be undertaken:
 
 _VPC_
 
 To use Lambda in a VPC, we recommend having both a private and public subnet per AZ. Your Lambda function is enabled to run within the Private AZ, and a NAT Gateway is created in the Public AZ. Your Private Subnets will each have their own Route Table, with a route for 0.0.0.0/0 (public internet) routed to the NAT Gateway in the same AZ. Your Public Subnets will all share a Route Table which routes 0.0.0.0/0 to the Internet Gateway.
 
-![VPCPerAZ](VPCPerAZ.png)
+![VPCPerAZ](img/VPCPerAZ.png)
 
 _AWS Lambda_
 
@@ -47,5 +47,5 @@ INSERT using GRANT (http://docs.aws.amazon.com/redshift/latest/dg/r_GRANT.html).
 
 To configure environment variable of your Lambda function:
 
-1. Add a variable. Key is "https_proxy".
-2. Fill a variable. For example, http://proxy.example.org:3128
+1. Add a variable. Key is `https_proxy`.
+2. Fill a variable. For example `http://proxy.example.org:3128`

README.md

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# publish deploy.yaml to regional buckets
+for r in `aws ec2 describe-regions --query Regions[*].RegionName --output text`; do aws s3 cp deploy.yaml s3://awslabs-code-$r/LambdaRedshiftLoader/deploy.yaml --acl public-read --region $r; done

build.sh renamed to bin/build.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+#set -x
+
+ver=`cat package.json | grep "version" | cut -d: -f2 | sed -e "s/[\"\,]//g" | tr -d '[:space:]'`
+token=AWSLambdaRedshiftLoader
+
+for r in `aws ec2 describe-regions --query Regions[*].RegionName --output text`; do
+    aws s3 cp dist/$token-$ver.zip s3://awslabs-code-$r/LambdaRedshiftLoader/$token-$ver.zip --acl public-read --region $r;
+done

bin/deploy-yaml.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+for r in `aws ec2 describe-regions --query Regions[*].RegionName --output text`; do echo "|$r |  [<img src=\"https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png\" target=\”_blank\”>](https://console.aws.amazon.com/cloudformation/home?region=$r#/stacks/new?stackName=LambdaRedshiftLoader&templateURL=https://s3-$r.amazonaws.com/awslabs-code-$r/LambdaRedshiftLoader/deploy.yaml) |"; done
+

bin/deploy.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#set -x
+
+ver=`cat package.json | grep "version" | cut -d: -f2 | sed -e "s/[\"\,]//g" | tr -d '[:space:]'`
+token=AWSLambdaRedshiftLoader
+project_prefix=LambdaRedshiftLoader
+
+for r in `aws ec2 describe-regions --query Regions[*].RegionName --output text`; do
+    if [ "$r" == "us-east-1" ]; then
+        region="."
+    else
+        region=".$r."
+    fi
+    prefix="amazonaws.com/awslabs-code-$r/$project_prefix/$token-$ver.zip"
+    link="https://s3$region$prefix"
+    echo "| $r | [s3://awslabs-code-$r/$project_prefix/$token-$ver.zip]($link)"
+done

bin/generate-readme-table.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+#set -x
+
+ver=$1
+token=AWSLambdaRedshiftLoader
+
+for r in `aws ec2 describe-regions --query Regions[*].RegionName --output text`; do
+    aws s3 rm s3://awslabs-code-$r/LambdaRedshiftLoader/$token-$ver.zip --region $r;
+done

bin/generate-s3table.sh

@@ -81,10 +81,7 @@ exports.createTables = function (dynamoDB, callback) {
             KeyType: 'HASH'
         }],
         TableName: filesTable,
-        ProvisionedThroughput: {
-            ReadCapacityUnits: 5,
-            WriteCapacityUnits: 5
-        }
+        BillingMode: "PAY_PER_REQUEST"
     };
     var configKey = s3prefix;
     var configSpec = {
@@ -97,10 +94,7 @@ exports.createTables = function (dynamoDB, callback) {
             KeyType: 'HASH'
         }],
         TableName: configTable,
-        ProvisionedThroughput: {
-            ReadCapacityUnits: 5,
-            WriteCapacityUnits: 5
-        }
+        BillingMode: "PAY_PER_REQUEST"
     };
 
     var batchKey = batchId;
@@ -127,10 +121,7 @@ exports.createTables = function (dynamoDB, callback) {
             KeyType: 'RANGE'
         }],
         TableName: batchTable,
-        ProvisionedThroughput: {
-            ReadCapacityUnits: 1,
-            WriteCapacityUnits: 5
-        },
+        BillingMode: "PAY_PER_REQUEST",
         GlobalSecondaryIndexes: [{
             IndexName: batchStatusGSI,
             KeySchema: [{
@@ -389,7 +380,7 @@ exports.getS3NotificationConfiguration = function (s3, bucket, prefix, functionA
             if (data.LambdaFunctionConfigurations && data.LambdaFunctionConfigurations.length > 0) {
                 var matchConfigId;
                 data.LambdaFunctionConfigurations.map(function (item) {
-                    if (item.Filter.Key.FilterRules) {
+                    if (item && item.Filter && item.Filter.Key && item.Filter.Key.FilterRules) {
                         item.Filter.Key.FilterRules.map(function (filter) {
                             if (filter.Name === 'Prefix' && filter.Value === prefix) {
                                 if (item.LambdaFunctionArn === functionArn) {
@@ -704,6 +695,21 @@ exports.deleteFile = function (dynamoDB, region, file, callback) {
     });
 }
 
+exports.queryFile = function (dynamoDB, region, file, callback) {
+    var fileItem = {
+        Key: {
+            loadFile: {
+                S: file
+            }
+        },
+        TableName: filesTable
+    };
+
+    dynamoDB.getItem(fileItem, function (err, data) {
+        callback(err, data);
+    });
+}
+
 exports.reprocessFile = function (dynamoDB, s3, region, file, callback) {
     // get the file so we know what the current batch ID is
     var fileItem = {

bin/remove-version.sh

@@ -9,6 +9,8 @@
 	"table": "sensor_data_raw",
 	"columnList": "",
 	"truncateTable": "N",
+    "presql": "",
+    "postsql": "",
 	"userName": "master",
 	"userPwd": "            ",
 	"df": "CSV",
@@ -27,4 +29,4 @@
 	"batchTimeoutSecs": "120",
 	"copyOptions": "",
 	"symmetricKey": ""
-}
+}