From 9490ef5bbd374b49c49e57f0aabfb825bd4ce602 Mon Sep 17 00:00:00 2001 From: Sean Stavropoulos Date: Fri, 19 Aug 2016 11:19:42 -0700 Subject: [PATCH] Add support for file streaming to S3 via ExAws 1.0.0-rc1 --- CHANGELOG.md | 4 +++- README.md | 21 ++++++++++++------ lib/arc/storage/s3.ex | 48 +++++++++++++++++++++++++++++++--------- mix.exs | 22 +++++++++++++----- mix.lock | 18 +++++++++------ test/storage/s3_test.exs | 2 ++ 6 files changed, 84 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6a8285..202553f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog -## Pre-released (Will be v0.6.0) +## v0.6.0-rc1 (2016-10-04) + * (Dependencies) - Removed `httpoison` as an optional dependency, added `sweet_xml` and `hackney` as optional dependencies (required if using S3). + * (Enhancement) File streaming to S3 - Allows the uploading of large files to S3 without reading to memory first. * (Enhancement) Allow Arc to transform and store directly on binary input. * (Bugfix - backwards incompatible) Return error tuple rather than raising `Arc.ConvertError` if the transformation fails. * (Bugfix) Update `:crypto` usage to `:crypto.strong_rand_bytes` diff --git a/README.md b/README.md index 90af8ca..a4447ce 100644 --- a/README.md +++ b/README.md @@ -9,20 +9,26 @@ Browse the readme below, or jump to [a full example](#full-example). ## Installation -Add the latest stable release to your `mix.exs` file: +Add the latest stable release to your `mix.exs` file, along with the required dependencies for `ExAws` if appropriate: ```elixir defp deps do [ - arc: "~> 0.5.2", - ex_aws: "~> 0.4.10", # Required if using Amazon S3 - httpoison: "~> 0.7", # Required if using Amazon S3 - poison: "~> 1.2" # Required if using Amazon S3 + arc: "~> 0.6.0-rc1", + + # If using Amazon S3: + ex_aws: "~> 1.0.0-rc1", + + # Per the ExAws requirements: + httpoison: "~> 0.7", + hackney: "~> 1.5", # Use Hackeny OR Poison - whichever you prefer + poison: "~> 2.0", # Use Hackeny OR Poison - whichever you prefer + sweet_xml: "~> 0.5" ] end ``` -If you plan on using Amazon's S3 Storage, you must also add `ex_aws` and `httpoison` the following applications as startup dependencies your application's `mix.exs` file: +If you plan on using Amazon's S3 Storage, you must also add `ex_aws`, `hackney`, and `poison` as startup dependencies your application's `mix.exs` file: ```elixir def application do @@ -30,7 +36,8 @@ def application do mod: { MyApp, [] }, applications: [ :ex_aws, - :httpoison + :hackney, + :poison ] ] end diff --git a/lib/arc/storage/s3.ex b/lib/arc/storage/s3.ex index 683e076..a50cf76 100644 --- a/lib/arc/storage/s3.ex +++ b/lib/arc/storage/s3.ex @@ -1,4 +1,5 @@ defmodule Arc.Storage.S3 do + require Logger @default_expiry_time 60*5 def put(definition, version, {file, scope}) do @@ -10,10 +11,7 @@ defmodule Arc.Storage.S3 do definition.s3_object_headers(version, {file, scope}) |> Dict.put(:acl, acl) - case ExAws.S3.put_object(bucket, s3_key, extract_binary(file), s3_options) do - {:ok, _res} -> {:ok, file.file_name} - {:error, error} -> {:error, error} - end + do_put(file, s3_key, s3_options) end def url(definition, version, file_and_scope, options \\ []) do @@ -24,7 +22,9 @@ defmodule Arc.Storage.S3 do end def delete(definition, version, {file, scope}) do - ExAws.S3.delete_object bucket, s3_key(definition, version, {file, scope}) + bucket + |> ExAws.S3.delete_object(s3_key(definition, version, {file, scope})) + |> ExAws.request() :ok end @@ -33,13 +33,45 @@ defmodule Arc.Storage.S3 do # Private # + # If the file is stored as a binary in-memory, send to AWS in a single request + defp do_put(file=%Arc.File{binary: file_binary}, s3_key, s3_options) when is_binary(file_binary) do + ExAws.S3.put_object(bucket(), s3_key, file_binary, s3_options) + |> ExAws.request() + |> case do + {:ok, _res} -> {:ok, file.file_name} + {:error, error} -> {:error, error} + end + end + + # Stream the file and upload to AWS as a multi-part upload + defp do_put(file, s3_key, s3_options) do + + try do + file.path + |> ExAws.S3.Upload.stream_file() + |> ExAws.S3.upload(bucket(), s3_key, s3_options) + |> ExAws.request() + |> case do + # :done -> {:ok, file.file_name} + {:ok, :done} -> {:ok, file.file_name} + {:error, error} -> {:error, error} + end + rescue + e in ExAws.Error -> + Logger.error(inspect e) + Logger.error(e.message) + {:error, :invalid_bucket} + end + end + defp build_url(definition, version, file_and_scope, _options) do Path.join host, s3_key(definition, version, file_and_scope) end defp build_signed_url(definition, version, file_and_scope, options) do expires_in = Keyword.get(options, :expire_in, @default_expiry_time) - {:ok, url} = ExAws.S3.presigned_url(:get, bucket, s3_key(definition, version, file_and_scope), [expires_in: expires_in, virtual_host: virtual_host]) + config = ExAws.Config.new(:s3, Application.get_all_env(:ex_aws)) + {:ok, url} = ExAws.S3.presigned_url(config, :get, bucket, s3_key(definition, version, file_and_scope), [expires_in: expires_in, virtual_host: virtual_host]) url end @@ -73,8 +105,4 @@ defmodule Arc.Storage.S3 do name -> name end end - - defp extract_binary(file) do - file.binary || File.read!(file.path) - end end diff --git a/mix.exs b/mix.exs index 9c01145..cd2dc56 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Arc.Mixfile do use Mix.Project - @version "0.5.3" + @version "0.6.0-rc1" def project do [app: :arc, @@ -28,15 +28,25 @@ defmodule Arc.Mixfile do end def application do - [applications: [:logger]] + [ + applications: [ + :logger + ] ++ applications(Mix.env) + ] end + def applications(:test), do: [:ex_aws, :poison, :hackney] + def applications(_), do: [] + defp deps do [ - {:ex_aws, "~> 0.4.10 or ~> 0.5.0", optional: true}, - {:poison, "~> 1.2 or ~> 2.0", optional: true}, - {:httpoison, "~> 0.7", optional: true}, - {:mock, "~> 0.1.1", only: :test} + {:ex_aws, "~> 1.0.0-rc.1", optional: true}, + {:mock, "~> 0.1.1", only: :test}, + + # If using Amazon S3: + {:hackney, "~> 1.5", optional: true}, + {:poison, "~> 2.0", optional: true}, + {:sweet_xml, "~> 0.5", optional: true} ] end end diff --git a/mix.lock b/mix.lock index b427b8c..2d4ae73 100644 --- a/mix.lock +++ b/mix.lock @@ -1,10 +1,14 @@ -%{"certifi": {:hex, :certifi, "0.3.0", "389d4b126a47895fe96d65fcf8681f4d09eca1153dc2243ed6babad0aac1e763", [:rebar3], []}, - "ex_aws": {:hex, :ex_aws, "0.4.13", "80578054ccd8fda9e7ebb8c84d94075e15db4e29665c7c435244d3d065dc3667", [:mix], [{:httpoison, "~> 0.7", [hex: :httpoison, optional: true]}, {:httpotion, "~> 2.0", [hex: :httpotion, optional: true]}, {:jsx, "~> 2.5", [hex: :jsx, optional: true]}, {:poison, "~> 1.2", [hex: :poison, optional: true]}, {:sweet_xml, "~> 0.5", [hex: :sweet_xml, optional: true]}]}, - "hackney": {:hex, :hackney, "1.4.7", "fcca8e6ba215de6225cc9b56230a3ef441ddaee05d963a39b56ca7b064026342", [:rebar3], [{:certifi, "0.3.0", [hex: :certifi, optional: false]}, {:idna, "1.0.2", [hex: :idna, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:ssl_verify_hostname, "1.0.5", [hex: :ssl_verify_hostname, optional: false]}]}, - "httpoison": {:hex, :httpoison, "0.8.0", "52a958d40b2aa46da418cdf6d8dfd82ba83e94d5e60920dfa5f40c05b34fe073", [:mix], [{:hackney, "~> 1.4.4", [hex: :hackney, optional: false]}]}, - "idna": {:hex, :idna, "1.0.2", "397e3d001c002319da75759b0a81156bf11849c71d565162436d50020cb7265e", [:make], []}, +%{"certifi": {:hex, :certifi, "0.4.0", "a7966efb868b179023618d29a407548f70c52466bf1849b9e8ebd0e34b7ea11f", [:rebar3], []}, + "ex_aws": {:hex, :ex_aws, "1.0.0-rc.1", "1fc21b65686892fe24ea0ff699e347c30ab538f69f9ed15b6cb488cc6e1900d9", [:mix], [{:gen_stage, "~> 0.5.0", [hex: :gen_stage, optional: false]}, {:hackney, "~> 1.5", [hex: :hackney, optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, optional: true]}, {:poison, "~> 1.2 or ~> 2.0", [hex: :poison, optional: true]}, {:sweet_xml, "~> 0.5", [hex: :sweet_xml, optional: true]}]}, + "gen_stage": {:hex, :gen_stage, "0.5.0", "758068f3a81286e342a609d668e9624714c2e4d43cc26699ead04d1680fde6c6", [:mix], []}, + "hackney": {:hex, :hackney, "1.6.1", "ddd22d42db2b50e6a155439c8811b8f6df61a4395de10509714ad2751c6da817", [:rebar3], [{:certifi, "0.4.0", [hex: :certifi, optional: false]}, {:idna, "1.2.0", [hex: :idna, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:ssl_verify_fun, "1.1.0", [hex: :ssl_verify_fun, optional: false]}]}, + "httpoison": {:hex, :httpoison, "0.9.1", "6c2b4eaf2588a6f3ef29663d28c992531ca3f0bc832a97e0359bc822978e1c5d", [:mix], [{:hackney, "~> 1.6.0", [hex: :hackney, optional: false]}]}, + "idna": {:hex, :idna, "1.2.0", "ac62ee99da068f43c50dc69acf700e03a62a348360126260e87f2b54eced86b2", [:rebar3], []}, "meck": {:hex, :meck, "0.8.2", "f15f7d513e14ec8c8dee9a95d4ae585b3e5a88bf0fa6a7573240d6ddb58a7236", [:make, :rebar], []}, + "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []}, "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []}, "mock": {:hex, :mock, "0.1.1", "e21469ca27ba32aa7b18b61699db26f7a778171b21c0e5deb6f1218a53278574", [:mix], [{:meck, "~> 0.8.2", [hex: :meck, optional: false]}]}, - "poison": {:hex, :poison, "1.5.0", "f2f4f460623a6f154683abae34352525e1d918380267cdbd949a07ba57503248", [:mix], []}, - "ssl_verify_hostname": {:hex, :ssl_verify_hostname, "1.0.5", "2e73e068cd6393526f9fa6d399353d7c9477d6886ba005f323b592d389fb47be", [:make], []}} + "poison": {:hex, :poison, "2.2.0", "4763b69a8a77bd77d26f477d196428b741261a761257ff1cf92753a0d4d24a63", [:mix], []}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.0", "edee20847c42e379bf91261db474ffbe373f8acb56e9079acb6038d4e0bf414f", [:rebar, :make], []}, + "ssl_verify_hostname": {:hex, :ssl_verify_hostname, "1.0.5"}, + "sweet_xml": {:hex, :sweet_xml, "0.6.1", "a56f235171f35a32807ce44798dedc748ce249fca574674fecd29c1321cab0de", [:mix], []}} diff --git a/test/storage/s3_test.exs b/test/storage/s3_test.exs index ec27fa3..ddc0dc9 100644 --- a/test/storage/s3_test.exs +++ b/test/storage/s3_test.exs @@ -98,6 +98,8 @@ defmodule ArcTest.Storage.S3 do # Application.put_env :ex_aws, :s3, [scheme: "https://", host: "s3.amazonaws.com", region: "us-west-2"] Application.put_env :ex_aws, :access_key_id, System.get_env("ARC_TEST_S3_KEY") Application.put_env :ex_aws, :secret_access_key, System.get_env("ARC_TEST_S3_SECRET") + # Application.put_env :ex_aws, :region, "us-east-1" + # Application.put_env :ex_aws, :scheme, "https://" end @tag :s3