Merge pull request #7 from Frameio/ks/DEVREL-589

Adds Backup to S3. Setup repo for many examples.

Author: kylenstone

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Frame.io
+Copyright (c) 2020 Frame.io
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

examples/custom-action-offload-to-s3/README.md

@@ -0,0 +1,40 @@
+# frameio-custom-action-offload-to-s3
+
+A Custom Actions workflow for backing up Frame.io assets to your own S3 bucket.
+
+## Instructions for use
+
+This repo contains sample code demonstrating a deployment of Custom Actions in the context of a serverless workflow.
+
+The code is pre-formatted for insertion into two Lambda functions.  An HTTP endpoint should be created in AWS API Gateway and configured as the Trigger for the initial Lambda.  
+
+Once the initial Lambda runs, it will gather data from the Frame.io API and invoke a second Lambda function which completes the upload to S3 using a buffered file write.
+
+The benefit of this workflow is that a Frame.io customer does not have to run a persistent server to download assets and re-upload them to a backup system.  This is accomplished automatically and cheaply with serverless technology.
+
+## Requirements
+- Frame.io developer token
+- Node JS 12.x runtime
+- AWS account with the ability to set IAM permissions
+
+This function requires two environment variables:
+- `FRAMEIO_TOKEN`: an API token with appropriate permissions to get assets from Frame.io
+- `BUCKET_NAME`: an AWS S3 bucket where files will be uploaded.
+
+## Components
+- Frame.io Custom Action
+- AWS API Gateway
+- Two AWS Lambda functions
+- S3 (data is streamed to S3 using the [aws-sdk](https://aws.amazon.com/sdk-for-node-js/) [S3.putObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) method)
+- First Lambda function (`lambda.js`) receive the Custom Actions event and parses the Frame.io asset.
+
+## Optional enhancements
+
+As written, this lambda function simply triggers an upload of a single asset.  There are many ways it could be enhanced or optimized:
+
+- Lambda functions are restricted to a maximum runtime of 15 minutes which limits the size of a file you can reliably upload.  To upload files larger than 2-3 GBs,  additional Lambdas could be spawned and tasked with performing one part of a distributed multi-part file upload.
+- AWS Step Functions provide an alternative and arguably better soluting to fan-out Lambdas for a large upload job.
+
+The integration to Frame.io could be expanded to cover many other workflows:
+- The `fetchDataFromAPI()` function can be modified to target [proxy media files](https://support.frame.io/en/articles/13321-what-is-my-video-converted-to-when-it-s-uploaded), allowing you to incorporate Frame.io's media conversion pipeline into your backup workflow.
+- Instead of using Custom Actions, you could write your own integration to our API to back up many Projects or Folders worth of files.  See the [Frame.io API Guides]](https://docs.frame.io/docs) for more details.

examples/custom-action-offload-to-s3/firstLambda.js

@@ -0,0 +1,26 @@
+const { fetchAsset, invokeLambda } = require('./modules/api');
+
+exports.handler = async function (event, context) {
+    // Save the X-ray Trace ID and and this Lambda's function name.
+    // We'll pass them to our second 'file handler' Lambda.
+    const firstLambdaTraceID = process.env._X_AMZN_TRACE_ID;
+    const caller = context.functionName;
+
+    let id = JSON.parse(event.body).resource.id;
+    let { url, name} = await fetchAsset(id);
+
+    try {
+        await invokeLambda(caller, firstLambdaTraceID, url, name);
+        let returnPayload = {
+            statusCode: 202, 
+            body: JSON.stringify({
+                'title': 'Job received',
+                'description': `Your backup job for '${name}' has been triggered.`,
+                'traceID': `${firstLambdaTraceID}`
+            })
+        };
+        return returnPayload;
+    } catch(err) {
+        return (`Hit a problem: ${err.message}`);
+    }
+};

examples/custom-action-offload-to-s3/modules/api.js

@@ -0,0 +1,56 @@
+const AWS = require('aws-sdk');
+const fetch = require('node-fetch');
+
+async function fetchAsset (id) {
+    const token = process.env.FRAMEIO_TOKEN;
+    let url = `http://api.frame.io/v2/assets/${id}`;
+    let requestOptions = {
+      method: 'GET',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${token}`
+      }
+    };
+    try {
+        let response = await fetch(url, requestOptions);
+        let result = await response.json();
+        return { url: result.original, name: result.name };
+    } catch(err) {
+        return (`error: ${err}`);
+    }
+}
+
+function invokeLambda (caller, firstLambdaTraceID, url, name) { 
+
+    const lambda = new AWS.Lambda();
+
+    let req = {
+        FunctionName: 'kstone-custom-action-offload-to-s3-second-lambda',
+        InvocationType: 'Event', // returns statusCode 202 on success. See invoke() SDK for info
+        Payload: JSON.stringify({
+            caller: caller,
+            firstLambdaTraceID: firstLambdaTraceID,
+            url: url, 
+            name: name })
+    };
+
+    return lambda.invoke(req).promise();
+}
+
+async function s3Uploader(url, name) {
+
+    const s3 = new AWS.S3();
+    
+    let response = await fetch(url);
+    let stream = response.body;
+
+    const params = {
+        Bucket: process.env.BUCKET_NAME,
+        Key: name,
+        Body: stream
+    };
+    
+    return (s3.upload(params).promise());
+}
+
+module.exports = { fetchAsset, invokeLambda, s3Uploader };

examples/custom-action-offload-to-s3/node-fetch-layer/nodeFetchLayer.zip

@@ -0,0 +1,15 @@
+{
+  "name": "node-fetch-layer",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "build": "npm install && mkdir -p nodejs && cp -r node_modules nodejs/ && zip -r node-fetch-layer.zip nodejs"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "node-fetch": "^2.6.0"
+  }
+}

examples/custom-action-offload-to-s3/node-fetch-layer/package-lock.json

@@ -0,0 +1,17 @@
+const { s3Uploader } = require('./modules/api');
+
+exports.handler = async (event) => {
+    let { caller, firstLambdaTraceID, url, name } = event;
+    
+    // Logs for convenient searching in X-Ray and CloudWatch
+    console.log(`Second Lambda trace ID: ${process.env._X_AMZN_TRACE_ID}`);
+    console.log(`Called by ${caller} with trace ID: ${firstLambdaTraceID}. Begin uploading ${name}...`);
+
+    try {
+        await s3Uploader(url, name);
+    } catch(err) {
+        return (`error: ${err}`);
+    }
+    
+    return (console.log(`Done uploading ${name}!`));
+  };

examples/custom-action-offload-to-s3/node-fetch-layer/package.json

@@ -0,0 +1,98 @@
+{
+    "resource": "/kstone-custom-action-offload-to-s3",
+    "path": "/kstone-custom-action-offload-to-s3",
+    "httpMethod": "POST",
+    "headers": {
+      "content-type": "application/json",
+      "Host": "188hrg8gxb.execute-api.us-east-2.amazonaws.com",
+      "User-Agent": "Frame.io API",
+      "X-Amzn-Trace-Id": "Root=1-5f4ad89b-b52895fe5e608f091fb6b7d8",
+      "X-Forwarded-For": "52.44.243.35",
+      "X-Forwarded-Port": "443",
+      "X-Forwarded-Proto": "https",
+      "x-frameio-request-timestamp": "1598740635",
+      "x-frameio-signature": "v0=fdd82f426536c5a927f29eeab7e01e177c4c0ca0a33f2827817c32e530fddaa2"
+    },
+    "multiValueHeaders": {
+      "content-type": [
+        "application/json"
+      ],
+      "Host": [
+        "188hrg8gxb.execute-api.us-east-2.amazonaws.com"
+      ],
+      "User-Agent": [
+        "Frame.io API"
+      ],
+      "X-Amzn-Trace-Id": [
+        "Root=1-5f4ad89b-b52895fe5e608f091fb6b7d8"
+      ],
+      "X-Forwarded-For": [
+        "52.44.243.35"
+      ],
+      "X-Forwarded-Port": [
+        "443"
+      ],
+      "X-Forwarded-Proto": [
+        "https"
+      ],
+      "x-frameio-request-timestamp": [
+        "1598740635"
+      ],
+      "x-frameio-signature": [
+        "v0=fdd82f426536c5a927f29eeab7e01e177c4c0ca0a33f2827817c32e530fddaa2"
+      ]
+    },
+    "queryStringParameters": null,
+    "multiValueQueryStringParameters": null,
+    "pathParameters": null,
+    "stageVariables": null,
+    "requestContext": {
+      "resourceId": "51f4b5",
+      "resourcePath": "/kstone-custom-action-offload-to-s3",
+      "httpMethod": "POST",
+      "extendedRequestId": "SDbIUGgdCYcF33A=",
+      "requestTime": "29/Aug/2020:22:37:15 +0000",
+      "path": "/default/kstone-custom-action-offload-to-s3",
+      "accountId": "720872555379",
+      "protocol": "HTTP/1.1",
+      "stage": "default",
+      "domainPrefix": "188hrg8gxb",
+      "requestTimeEpoch": 1598740635596,
+      "requestId": "eeb69611-0501-4d2f-89fd-be9b8fce3e64",
+      "identity": {
+        "cognitoIdentityPoolId": null,
+        "accountId": null,
+        "cognitoIdentityId": null,
+        "caller": null,
+        "sourceIp": "52.44.243.35",
+        "principalOrgId": null,
+        "accessKey": null,
+        "cognitoAuthenticationType": null,
+        "cognitoAuthenticationProvider": null,
+        "userArn": null,
+        "userAgent": "Frame.io API",
+        "user": null
+      },
+      "domainName": "188hrg8gxb.execute-api.us-east-2.amazonaws.com",
+      "apiId": "188hrg8gxb"
+    },
+    "body": {
+      "action_id": "44f40b4a-65e9-479a-b2bf-40ec57111c88",
+      "interaction_id": "706e2345-14d7-4784-9e76-81a42e206915",
+      "project": {
+        "id": "5065bd0f-7d61-48f2-beec-7bc4313d0b23"
+      },
+      "resource": {
+        "id": "ce611de8-79b8-4bf5-bdf3-0b5a72f7dbb6",
+        "type": "asset"
+      },
+      "team": {
+        "id": "aa891687-4b1e-4150-9b6d-9e4911c5b436"
+      },
+      "type": "custom.action.demo",
+      "user": {
+        "id": "59c9ade1-311b-4c3b-8231-b9d88e9a1a85"
+      }
+    },
+    "isBase64Encoded": false
+  }