From 0a359541740e6646e94c46e871c9bfe1b8b7a19f Mon Sep 17 00:00:00 2001
From: Devon Estes <devon.c.estes@gmail.com>
Date: Thu, 22 Nov 2018 11:19:39 +0100
Subject: [PATCH 1/2] Update to new formatter behaviour

This updates this formatter to the new Benchee.Formatter behaviour and
removes supoprt for the legacy stuff that we're dropping in 1.0.
---
 .travis.yml                                   |   8 -
 lib/benchee/formatters/html.ex                | 294 ++++++++--------
 mix.exs                                       |   8 +-
 mix.lock                                      |  42 ++-
 .../formatters/html_integration_test.exs      |  42 +--
 test/benchee/formatters/html_test.exs         | 329 ++++++++----------
 6 files changed, 315 insertions(+), 408 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7d1ac9a..60caedc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,18 +1,10 @@
 language: elixir
 elixir:
-  - 1.5.3
   - 1.6.6
   - 1.7.4
 otp_release:
-  - 19.3
   - 20.3
   - 21.1
-matrix:
-  exclude:
-    - elixir: 1.5.3
-      otp_release: 21.1
-    - elixir: 1.7.4
-      otp_release: 19.3
 
 before_script:
   - MIX_ENV=test mix compile --warnings-as-errors
diff --git a/lib/benchee/formatters/html.ex b/lib/benchee/formatters/html.ex
index 9e53ad0..acf2d9f 100644
--- a/lib/benchee/formatters/html.ex
+++ b/lib/benchee/formatters/html.ex
@@ -1,170 +1,70 @@
 defmodule Benchee.Formatters.HTML do
-  use Benchee.Formatter
-
-  require EEx
-  alias Benchee.{Suite, Statistics, Configuration}
-  alias Benchee.Conversion
-  alias Benchee.Conversion.{Duration, Count, DeviationPercent}
-  alias Benchee.Utility.FileCreation
-  alias Benchee.Formatters.JSON
-
-  # Major pages
-  EEx.function_from_file(:defp, :comparison, "priv/templates/comparison.html.eex", [
-    :input_name,
-    :suite,
-    :units,
-    :suite_json,
-    :inline_assets
-  ])
-
-  EEx.function_from_file(:defp, :job_detail, "priv/templates/job_detail.html.eex", [
-    :input_name,
-    :job_name,
-    :job_statistics,
-    :system,
-    :units,
-    :job_json,
-    :inline_assets
-  ])
-
-  EEx.function_from_file(:defp, :index, "priv/templates/index.html.eex", [
-    :names_to_paths,
-    :system,
-    :inline_assets
-  ])
-
-  # Partials
-  EEx.function_from_file(:defp, :head, "priv/templates/partials/head.html.eex", [:inline_assets])
-  EEx.function_from_file(:defp, :header, "priv/templates/partials/header.html.eex", [:input_name])
-
-  EEx.function_from_file(:defp, :js_includes, "priv/templates/partials/js_includes.html.eex", [
-    :inline_assets
-  ])
-
-  EEx.function_from_file(
-    :defp,
-    :version_note,
-    "priv/templates/partials/version_note.html.eex",
-    []
-  )
-
-  EEx.function_from_file(:defp, :input_label, "priv/templates/partials/input_label.html.eex", [
-    :input_name
-  ])
-
-  EEx.function_from_file(:defp, :data_table, "priv/templates/partials/data_table.html.eex", [
-    :statistics,
-    :units,
-    :options
-  ])
-
-  EEx.function_from_file(:defp, :system_info, "priv/templates/partials/system_info.html.eex", [
-    :system,
-    :options
-  ])
-
-  EEx.function_from_file(:defp, :footer, "priv/templates/partials/footer.html.eex", [
-    :dependencies
-  ])
-
-  # Small wrappers to have default arguments
-  defp render_data_table(statistics, units, options \\ []) do
-    data_table(statistics, units, options)
-  end
-
-  defp render_system_info(system, options \\ [visible: false]) do
-    system_info(system, options)
-  end
-
-  defp render_footer do
-    footer(%{
-      benchee: Application.spec(:benchee, :vsn),
-      benchee_html: Application.spec(:benchee_html, :vsn)
-    })
-  end
-
   @moduledoc """
   Functionality for converting Benchee benchmarking results to an HTML page
   with plotly.js generated graphs and friends.
 
   ## Examples
 
-      list = Enum.to_list(1..10_000)
-      map_fun = fn(i) -> [i, i * i] end
+    list = Enum.to_list(1..10_000)
+    map_fun = fn(i) -> [i, i * i] end
 
-      Benchee.run(%{
-        "flat_map"    => fn -> Enum.flat_map(list, map_fun) end,
-        "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten end
+    Benchee.run(
+      %{
+        "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
+        "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten() end
       },
-        formatters: [
-          &Benchee.Formatters.HTML.output/1,
-          &Benchee.Formatters.Console.output/1
-        ],
-        formatter_options: [html: [file: "samples_output/flat_map.html"]],
-      )
-
+      formatters: [
+        Benchee.Formatters.Console,
+        {Benchee.Formatters.HTML, file: "samples_output/flat_map.html"}
+      ]
+    )
   """
 
-  @doc """
-  Transforms the statistical results from benchmarking to html to be written
-  somewhere, such as a file through `IO.write/2`.
+  @behaviour Benchee.Formatter
 
-  Returns a map from file name/path to file content along with formatter options.
-  """
-  @spec format(Suite.t()) :: {%{Suite.key() => String.t()}, map}
-  def format(suite) do
-    suite
-    |> default_configuration
-    |> do_format
-  end
+  require EEx
 
-  @default_filename "benchmarks/output/results.html"
-  @default_auto_open true
-  @default_inline_assets false
-  defp default_configuration(suite) do
-    opts =
-      suite.configuration.formatter_options
-      |> Map.get(:html, %{})
-      |> Map.put_new(:file, @default_filename)
-      |> Map.put_new(:auto_open, @default_auto_open)
-      |> Map.put_new(:inline_assets, @default_inline_assets)
-
-    updated_configuration = %Configuration{suite.configuration | formatter_options: %{html: opts}}
-    load_specs_for_versions()
-    %Suite{suite | configuration: updated_configuration}
-  end
+  alias Benchee.{
+    Configuration,
+    Conversion,
+    Formatters.JSON,
+    Suite,
+    Statistics,
+    Utility.FileCreation
+  }
 
-  defp load_specs_for_versions do
+  alias Conversion.{Duration, Count, DeviationPercent}
+
+  @doc """
+  Transforms the statistical results from benchmarking to html reports.
+
+  Returns a map from file name/path to file content.
+  """
+  @spec format(Suite.t(), map) :: %{Suite.key() => String.t()}
+  def format(
+        %Suite{
+          scenarios: scenarios,
+          system: system,
+          configuration: %Configuration{unit_scaling: unit_scaling}
+        },
+        opts
+      ) do
     _ = Application.load(:benchee)
     _ = Application.load(:benchee_html)
-  end
+    %{file: filename, inline_assets: inline_assets} = default_configuration(opts)
 
-  defp do_format(%Suite{
-         scenarios: scenarios,
-         system: system,
-         configuration: %{
-           formatter_options: %{html: options = %{file: filename, inline_assets: inline_assets}},
-           unit_scaling: unit_scaling
-         }
-       }) do
-    data =
-      scenarios
-      |> Enum.group_by(fn scenario -> scenario.input_name end)
-      |> Enum.map(fn tagged_scenarios ->
-        reports_for_input(tagged_scenarios, system, filename, unit_scaling, inline_assets)
-      end)
-      |> add_index(filename, system, inline_assets)
-      |> List.flatten()
-      |> Map.new()
-
-    {data, options}
+    scenarios
+    |> Enum.group_by(fn scenario -> scenario.input_name end)
+    |> Enum.map(fn tagged_scenarios ->
+      reports_for_input(tagged_scenarios, system, filename, unit_scaling, inline_assets)
+    end)
+    |> add_index(filename, system, inline_assets)
+    |> List.flatten()
+    |> Map.new()
   end
 
   @doc """
-  Uses output of `Benchee.Formatters.HTML.format/1` to transform the statistics
-  output to HTML with JS, but also already writes it to files defined in the
-  initial configuration under `formatter_options: [html: [file:
-  "benchmark_out/my.html"]]`.
+  Writes the output of `Benchee.Formatters.HTML.format/2` to disk.
 
   Generates the following files:
 
@@ -173,22 +73,33 @@ defmodule Benchee.Formatters.HTML do
   * for each job a detail page with more detailed run time graphs for that
     particular job (one per benchmark input)
   """
-  @spec write({%{Suite.key() => String.t()}, map}) :: :ok
-  def write({data, %{file: filename, auto_open: auto_open?, inline_assets: inline_assets?}}) do
-    prepare_folder_structure(filename, inline_assets?)
+  @spec write(%{Suite.key() => String.t()}, map) :: :ok
+  def write(data, opts) do
+    %{
+      file: filename,
+      auto_open: auto_open?,
+      inline_assets: inline_assets?
+    } = default_configuration(opts)
 
+    prepare_folder_structure(filename, inline_assets?)
     FileCreation.each(data, filename)
-
     if auto_open?, do: open_report(filename)
-
     :ok
   end
 
+  @default_filename "benchmarks/output/results.html"
+  @default_auto_open true
+  @default_inline_assets false
+  defp default_configuration(opts) do
+    opts
+    |> Map.put_new(:file, @default_filename)
+    |> Map.put_new(:auto_open, @default_auto_open)
+    |> Map.put_new(:inline_assets, @default_inline_assets)
+  end
+
   defp prepare_folder_structure(filename, inline_assets?) do
     base_directory = create_base_directory(filename)
-
     unless inline_assets?, do: copy_asset_files(base_directory)
-
     base_directory
   end
 
@@ -339,4 +250,79 @@ defmodule Benchee.Formatters.HTML do
       {:win32, _} -> "explorer"
     end
   end
+
+  # Major pages
+  EEx.function_from_file(:defp, :comparison, "priv/templates/comparison.html.eex", [
+    :input_name,
+    :suite,
+    :units,
+    :suite_json,
+    :inline_assets
+  ])
+
+  EEx.function_from_file(:defp, :job_detail, "priv/templates/job_detail.html.eex", [
+    :input_name,
+    :job_name,
+    :job_statistics,
+    :system,
+    :units,
+    :job_json,
+    :inline_assets
+  ])
+
+  EEx.function_from_file(:defp, :index, "priv/templates/index.html.eex", [
+    :names_to_paths,
+    :system,
+    :inline_assets
+  ])
+
+  # Partials
+  EEx.function_from_file(:defp, :head, "priv/templates/partials/head.html.eex", [:inline_assets])
+  EEx.function_from_file(:defp, :header, "priv/templates/partials/header.html.eex", [:input_name])
+
+  EEx.function_from_file(:defp, :js_includes, "priv/templates/partials/js_includes.html.eex", [
+    :inline_assets
+  ])
+
+  EEx.function_from_file(
+    :defp,
+    :version_note,
+    "priv/templates/partials/version_note.html.eex",
+    []
+  )
+
+  EEx.function_from_file(:defp, :input_label, "priv/templates/partials/input_label.html.eex", [
+    :input_name
+  ])
+
+  EEx.function_from_file(:defp, :data_table, "priv/templates/partials/data_table.html.eex", [
+    :statistics,
+    :units,
+    :options
+  ])
+
+  EEx.function_from_file(:defp, :system_info, "priv/templates/partials/system_info.html.eex", [
+    :system,
+    :options
+  ])
+
+  EEx.function_from_file(:defp, :footer, "priv/templates/partials/footer.html.eex", [
+    :dependencies
+  ])
+
+  # Small wrappers to have default arguments
+  defp render_data_table(statistics, units, options \\ []) do
+    data_table(statistics, units, options)
+  end
+
+  defp render_system_info(system, options \\ [visible: false]) do
+    system_info(system, options)
+  end
+
+  defp render_footer do
+    footer(%{
+      benchee: Application.spec(:benchee, :vsn),
+      benchee_html: Application.spec(:benchee_html, :vsn)
+    })
+  end
 end
diff --git a/mix.exs b/mix.exs
index 153b7d0..46b8938 100644
--- a/mix.exs
+++ b/mix.exs
@@ -47,14 +47,14 @@ defmodule BencheeHTML.Mixfile do
   # Type "mix help deps" for more examples and options
   defp deps do
     [
-      {:benchee, "~> 0.12"},
-      {:benchee_json, "~> 0.5"},
+      {:benchee, github: "PragTob/benchee"},
+      {:benchee_json, github: "PragTob/benchee_json"},
       {:excoveralls, "~> 0.8.1", only: :test},
       {:mix_test_watch, "~> 0.2", only: :dev},
-      {:credo, "~> 0.4", only: :dev},
+      {:credo, "~> 1.0", only: :dev},
       {:ex_doc, "~> 0.11", only: :dev},
       {:earmark, "~> 1.2", only: :dev},
-      {:dialyxir, "~> 0.5", only: :dev, runtime: false}
+      {:dialyxir, "~> 1.0.0-rc.4", only: :dev, runtime: false}
     ]
   end
 
diff --git a/mix.lock b/mix.lock
index 5e77bd5..36fce95 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,24 +1,30 @@
 %{
-  "benchee": {:hex, :benchee, "0.12.0", "771e919505cbe399df5f49b8a21c56af5225542ca5c877e0baa0c503efb4e626", [:mix], [{:deep_merge, "~> 0.1", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"},
-  "benchee_json": {:hex, :benchee_json, "0.5.0", "75878fd9944093ced03e9c214e6e1429da147de43e2f4a4cbc4377294ed5d029", [:mix], [{:benchee, "~> 0.12", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
-  "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], []},
-  "certifi": {:hex, :certifi, "2.0.0", "a0c0e475107135f76b8c1d5bc7efb33cd3815cb3cf3dea7aefdd174dabead064", [:rebar3], [], "hexpm"},
-  "credo": {:hex, :credo, "0.8.6", "335f723772d35da499b5ebfdaf6b426bfb73590b6fcbc8908d476b75f8cbca3f", [:mix], [{:bunt, "~> 0.2.0", [repo: "hexpm", hex: :bunt, optional: false]}], "hexpm"},
-  "deep_merge": {:hex, :deep_merge, "0.1.1", "c27866a7524a337b6a039eeb8dd4f17d458fd40fbbcb8c54661b71a22fffe846", [:mix], []},
-  "dialyxir": {:hex, :dialyxir, "0.5.0", "5bc543f9c28ecd51b99cc1a685a3c2a1a93216990347f259406a910cf048d1d7", [:mix], []},
-  "earmark": {:hex, :earmark, "1.2.1", "7ad3f203ab84d31832814483c834e006cf88949f061a4b50d7e783147572280f", [:mix], []},
-  "ex_doc": {:hex, :ex_doc, "0.15.1", "d5f9d588fd802152516fccfdb96d6073753f77314fcfee892b15b6724ca0d596", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, optional: false]}]},
-  "excoveralls": {:hex, :excoveralls, "0.8.1", "0bbf67f22c7dbf7503981d21a5eef5db8bbc3cb86e70d3798e8c802c74fa5e27", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:hackney, ">= 0.12.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
+  "benchee": {:git, "https://github.com/PragTob/benchee.git", "ee9f7233f94ea8b38795be24c7e5714761d45add", []},
+  "benchee_json": {:git, "https://github.com/PragTob/benchee_json.git", "7c719bb18de353888de031a4c3f6d7e32e186015", []},
+  "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
+  "certifi": {:hex, :certifi, "2.4.2", "75424ff0f3baaccfd34b1214184b6ef616d89e420b258bb0a5ea7d7bc628f7f0", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"},
+  "credo": {:hex, :credo, "1.0.0", "aaa40fdd0543a0cf8080e8c5949d8c25f0a24e4fc8c1d83d06c388f5e5e0ea42", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
+  "deep_merge": {:hex, :deep_merge, "0.2.0", "c1050fa2edf4848b9f556fba1b75afc66608a4219659e3311d9c9427b5b680b3", [:mix], [], "hexpm"},
+  "dialyxir": {:hex, :dialyxir, "1.0.0-rc.4", "71b42f5ee1b7628f3e3a6565f4617dfb02d127a0499ab3e72750455e986df001", [:mix], [{:erlex, "~> 0.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm"},
+  "earmark": {:hex, :earmark, "1.3.0", "17f0c38eaafb4800f746b457313af4b2442a8c2405b49c645768680f900be603", [:mix], [], "hexpm"},
+  "erlex": {:hex, :erlex, "0.1.6", "c01c889363168d3fdd23f4211647d8a34c0f9a21ec726762312e08e083f3d47e", [:mix], [], "hexpm"},
+  "ex_doc": {:hex, :ex_doc, "0.19.1", "519bb9c19526ca51d326c060cb1778d4a9056b190086a8c6c115828eaccea6cf", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.7", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
+  "excoveralls": {:hex, :excoveralls, "0.8.2", "b941a08a1842d7aa629e0bbc969186a4cefdd035bad9fe15d43aaaaaeb8fae36", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:hackney, ">= 0.12.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
   "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm"},
+  "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"},
   "fs": {:hex, :fs, "2.12.0", "ad631efacc9a5683c8eaa1b274e24fa64a1b8eb30747e9595b93bec7e492e25e", [:rebar3], []},
-  "hackney": {:hex, :hackney, "1.11.0", "4951ee019df102492dabba66a09e305f61919a8a183a7860236c0fde586134b6", [:rebar3], [{:certifi, "2.0.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
-  "idna": {:hex, :idna, "5.1.0", "d72b4effeb324ad5da3cab1767cb16b17939004e789d8c0ad5b70f3cea20c89a", [:rebar3], [{:unicode_util_compat, "0.3.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
-  "jason": {:hex, :jason, "1.0.0", "0f7cfa9bdb23fed721ec05419bcee2b2c21a77e926bce0deda029b5adc716fe2", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},
+  "hackney": {:hex, :hackney, "1.14.3", "b5f6f5dcc4f1fba340762738759209e21914516df6be440d85772542d4a5e412", [:rebar3], [{:certifi, "2.4.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
+  "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"},
+  "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"},
   "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm"},
-  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []},
-  "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []},
-  "mix_test_watch": {:hex, :mix_test_watch, "0.3.3", "70859889a8d1d43d1b75d69d87258a301f43209a17787cdb2bd9cab42adf271d", [:mix], [{:fs, "~> 2.12", [hex: :fs, optional: false]}]},
+  "makeup": {:hex, :makeup, "0.5.5", "9e08dfc45280c5684d771ad58159f718a7b5788596099bdfb0284597d368a882", [:mix], [{:nimble_parsec, "~> 0.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.10.0", "0f09c2ddf352887a956d84f8f7e702111122ca32fbbc84c2f0569b8b65cbf7fa", [:mix], [{:makeup, "~> 0.5.5", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"},
+  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
+  "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"},
+  "mix_test_watch": {:hex, :mix_test_watch, "0.9.0", "c72132a6071261893518fa08e121e911c9358713f62794a90c95db59042af375", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm"},
+  "nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"},
+  "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], []},
-  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], []},
-  "unicode_util_compat": {:hex, :unicode_util_compat, "0.3.1", "a1f612a7b512638634a603c8f401892afbf99b8ce93a45041f8aaca99cadb85e", [:rebar3], [], "hexpm"},
+  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
+  "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"},
 }
diff --git a/test/benchee/formatters/html_integration_test.exs b/test/benchee/formatters/html_integration_test.exs
index 6f0df2b..495d214 100644
--- a/test/benchee/formatters/html_integration_test.exs
+++ b/test/benchee/formatters/html_integration_test.exs
@@ -16,44 +16,7 @@ defmodule Benchee.Formatters.HTMLIntegrationTest do
     benchee_options = [
       time: 0.01,
       warmup: 0.02,
-      formatters: [Benchee.Formatters.HTML],
-      formatter_options: [html: [file: @file_path, auto_open: false]]
-    ]
-
-    assertion_data = %{
-      comparison_path: @comparison_path,
-      test_directory: @test_directory,
-      file_path: @file_path,
-      base_name: @base_name
-    }
-
-    basic_test(benchee_options, assertion_data)
-  end
-
-  test "works with the old school function as formatter" do
-    benchee_options = [
-      time: 0.01,
-      warmup: 0.02,
-      formatters: [&Benchee.Formatters.HTML.output/1],
-      formatter_options: [html: [file: @file_path, auto_open: false]]
-    ]
-
-    assertion_data = %{
-      comparison_path: @comparison_path,
-      test_directory: @test_directory,
-      file_path: @file_path,
-      base_name: @base_name
-    }
-
-    basic_test(benchee_options, assertion_data)
-  end
-
-  test "works fine with the legacy configuration format" do
-    benchee_options = [
-      time: 0.01,
-      warmup: 0.02,
-      formatters: [Benchee.Formatters.HTML],
-      html: [file: @file_path, auto_open: false]
+      formatters: [{Benchee.Formatters.HTML, file: @file_path, auto_open: false}]
     ]
 
     assertion_data = %{
@@ -70,8 +33,7 @@ defmodule Benchee.Formatters.HTMLIntegrationTest do
     benchee_options = [
       time: 0.01,
       warmup: 0.02,
-      formatters: [Benchee.Formatters.HTML],
-      formatter_options: [html: [auto_open: false]]
+      formatters: [{Benchee.Formatters.HTML, auto_open: false}]
     ]
 
     assertion_data = %{
diff --git a/test/benchee/formatters/html_test.exs b/test/benchee/formatters/html_test.exs
index c96a0d4..1048548 100644
--- a/test/benchee/formatters/html_test.exs
+++ b/test/benchee/formatters/html_test.exs
@@ -7,6 +7,7 @@ defmodule Benchee.Formatters.HTMLTest do
   @test_directory "test_output"
   @filename "#{@test_directory}/my.html"
   @expected_filename "#{@test_directory}/my_some_input_comparison.html"
+  @default_options %{file: @filename, auto_open: false}
   @system_info %{
     elixir: "1.4.0",
     erlang: "19.1",
@@ -35,147 +36,167 @@ defmodule Benchee.Formatters.HTMLTest do
   }
   @sample_suite %Benchee.Suite{
     scenarios: [@scenario],
-    system: @system_info,
-    configuration: %Benchee.Configuration{
-      formatter_options: %{html: %{file: @filename, auto_open: false}}
-    }
+    system: @system_info
   }
-  test ".format returns an HTML-ish string for every input" do
-    {format, _} = HTML.format(@sample_suite)
 
-    Enum.each(format, fn {_, html} ->
-      assert html =~ ~r/<html>.+<article>.+<\/html>/si
-    end)
-  end
+  describe "format/2" do
+    test "returns an HTML-ish string for every input" do
+      @sample_suite
+      |> HTML.format(@default_options)
+      |> Enum.each(fn {_, html} ->
+        assert html =~ ~r/<html>.+<article>.+<\/html>/si
+      end)
+    end
+
+    test "has the important suite data in the html result" do
+      Enum.each(comparison_and_job_htmls(), fn html ->
+        assert_includes(
+          html,
+          [
+            "[190,200,210]",
+            "\"average\":200.0",
+            "\"median\":190.0",
+            "\"ips\":5.0e3",
+            "My Job",
+            ">3<",
+            ">190 ns<",
+            ">210 ns<",
+            ">200 ns<",
+            "5 K"
+          ]
+        )
+      end)
+    end
+
+    test "has system info in the html result" do
+      Enum.each(comparison_and_job_htmls(), fn html ->
+        assert_includes(
+          html,
+          [
+            "Elixir: #{@system_info[:elixir]}",
+            "Erlang: #{@system_info[:erlang]}",
+            "Operating system: #{@system_info[:os]}",
+            "Available memory: #{@system_info[:available_memory]}",
+            "CPU Information: #{@system_info[:cpu_speed]}",
+            "Number of Available Cores: #{@system_info[:num_cores]}"
+          ]
+        )
+      end)
+    end
+
+    test "scales the run times to μs" do
+      statistics = %Benchee.Statistics{
+        average: 1500.0,
+        ips: 666.66,
+        std_dev: 150,
+        std_dev_ratio: 0.1,
+        std_dev_ips: 66.66,
+        median: 1400.0,
+        sample_size: 3,
+        minimum: 1300,
+        maximum: 1700
+      }
 
-  test ".format has the important suite data in the html result" do
-    Enum.each(comparison_and_job_htmls(), fn html ->
-      assert_includes(
-        html,
-        [
-          "[190,200,210]",
-          "\"average\":200.0",
-          "\"median\":190.0",
-          "\"ips\":5.0e3",
-          "My Job",
-          ">3<",
-          ">190 μs<",
-          ">210 μs<",
-          ">200 μs<",
-          "5 K"
-        ]
-      )
-    end)
-  end
+      scenario = %Benchee.Benchmark.Scenario{
+        @scenario
+        | run_time_statistics: statistics
+      }
 
-  test ".format has system info in the html result" do
-    Enum.each(comparison_and_job_htmls(), fn html ->
-      assert_includes(
-        html,
-        [
-          "Elixir: #{@system_info[:elixir]}",
-          "Erlang: #{@system_info[:erlang]}",
-          "Operating system: #{@system_info[:os]}",
-          "Available memory: #{@system_info[:available_memory]}",
-          "CPU Information: #{@system_info[:cpu_speed]}",
-          "Number of Available Cores: #{@system_info[:num_cores]}"
-        ]
-      )
-    end)
-  end
+      suite = %Benchee.Suite{@sample_suite | scenarios: [scenario]}
 
-  test ".format also scales the run times to ms" do
-    statistics = %Benchee.Statistics{
-      average: 1500.0,
-      ips: 666.66,
-      std_dev: 150,
-      std_dev_ratio: 0.1,
-      std_dev_ips: 66.66,
-      median: 1400.0,
-      sample_size: 3,
-      minimum: 1300,
-      maximum: 1700
-    }
+      Enum.each(comparison_and_job_htmls(suite), fn html ->
+        assert_includes(
+          html,
+          [">1.50 μs<", ">666.66<", ">1.40 μs<", ">1.30 μs<", ">1.70 μs<"]
+        )
+      end)
+    end
 
-    scenario = %Benchee.Benchmark.Scenario{
-      @scenario
-      | run_time_statistics: statistics
-    }
+    test "produces the right JSON data without the input level" do
+      %{["Some Input", "comparison"] => html} = HTML.format(@sample_suite, @default_options)
+      assert html =~ "\"statistics\":{\"My Job\""
+    end
 
-    suite = %Benchee.Suite{@sample_suite | scenarios: [scenario]}
+    test "shows the units alright" do
+      Enum.each(comparison_and_job_htmls(), fn html ->
+        assert html =~ "±"
+        assert html =~ "ns"
+      end)
+    end
 
-    Enum.each(comparison_and_job_htmls(suite), fn html ->
-      assert_includes(
-        html,
-        [">1.50 ms<", ">666.66<", ">1.40 ms<", ">1.30 ms<", ">1.70 ms<"]
-      )
-    end)
-  end
+    test "mentions the input" do
+      @sample_suite
+      |> HTML.format(@default_options)
+      |> Enum.each(fn {_, html} ->
+        assert html =~ "Some Input"
+      end)
+    end
+
+    test "does not render the label if no input was given" do
+      marker = Benchee.Benchmark.no_input()
+
+      %Benchee.Suite{
+        scenarios: [
+          %Benchee.Benchmark.Scenario{
+            job_name: "My Job",
+            name: "My Job",
+            run_times: [190, 200, 210],
+            input_name: marker,
+            input: marker,
+            run_time_statistics: %Benchee.Statistics{
+              average: 200.0,
+              ips: 5000.0,
+              std_dev: 20,
+              std_dev_ratio: 0.1,
+              std_dev_ips: 500,
+              median: 190.0,
+              sample_size: 3,
+              minimum: 190,
+              maximum: 210
+            }
+          }
+        ],
+        system: @system_info
+      }
+      |> HTML.format(@default_options)
+      |> Enum.each(fn {_, html} ->
+        refute html =~ "#{marker}"
+        refute html =~ "input-label"
+      end)
+    end
 
-  test ".format produces the right JSON data without the input level" do
-    {%{["Some Input", "comparison"] => html}, _} = HTML.format(@sample_suite)
+    test "does not copy assets when inlining is on" do
+      options = %{file: @filename, auto_open: false, inline_assets: true}
 
-    assert html =~ "\"statistics\":{\"My Job\""
-  end
+      capture_io(fn ->
+        @sample_suite
+        |> HTML.format(options)
+        |> HTML.write(options)
+      end)
 
-  test ".format shows the units alright" do
-    Enum.each(comparison_and_job_htmls(), fn html ->
-      assert html =~ "±"
-      assert html =~ "μs"
-    end)
-  end
+      assert File.exists?(@expected_filename)
 
-  defp comparison_and_job_htmls(suite \\ @sample_suite) do
-    {%{["Some Input", "comparison"] => comparison_html, ["Some Input", "My Job"] => job_html}, _} =
-      HTML.format(suite)
+      content = File.read!(@expected_filename)
+      assets_inlined(content, ["<style>", "<script>"])
 
-    [comparison_html, job_html]
-  end
+      refute File.exists?("#{@test_directory}/assets/stylesheets/benchee.css")
+      refute File.exists?("#{@test_directory}/assets/fontello/css/fontello.css")
 
-  test ".format mentions the input" do
-    {format, _} = HTML.format(@sample_suite)
-
-    Enum.each(format, fn {_, html} ->
-      assert html =~ "Some Input"
-    end)
+      refute File.exists?("#{@test_directory}/assets/javascripts/benchee.js")
+      refute File.exists?("#{@test_directory}/assets/javascripts/plotly-1.30.1.min.js")
+    after
+      if File.exists?(@test_directory), do: File.rm_rf!(@test_directory)
+    end
   end
 
-  test ".format does not render the label if no input was given" do
-    marker = Benchee.Benchmark.no_input()
-
-    suite = %Benchee.Suite{
-      scenarios: [
-        %Benchee.Benchmark.Scenario{
-          job_name: "My Job",
-          name: "My Job",
-          run_times: [190, 200, 210],
-          input_name: marker,
-          input: marker,
-          run_time_statistics: %Benchee.Statistics{
-            average: 200.0,
-            ips: 5000.0,
-            std_dev: 20,
-            std_dev_ratio: 0.1,
-            std_dev_ips: 500,
-            median: 190.0,
-            sample_size: 3,
-            minimum: 190,
-            maximum: 210
-          }
-        }
-      ],
-      system: @system_info,
-      configuration: %Benchee.Configuration{
-        formatter_options: %{html: %{file: @filename, auto_open: false}}
-      }
-    }
-
-    {format, _} = HTML.format(suite)
+  defp comparison_and_job_htmls(
+         suite \\ @sample_suite,
+         options \\ %{html: %{file: @filename, auto_open: false}}
+       ) do
+    %{["Some Input", "comparison"] => comparison_html, ["Some Input", "My Job"] => job_html} =
+      HTML.format(suite, options)
 
-    Enum.each(format, fn {_, html} ->
-      refute html =~ "#{marker}"
-      refute html =~ "input-label"
-    end)
+    [comparison_html, job_html]
   end
 
   defp assert_includes(html, expected_contents) do
@@ -184,69 +205,9 @@ defmodule Benchee.Formatters.HTMLTest do
     end)
   end
 
-  test ".output returns the suite again unchanged, produces files" do
-    capture_io(fn ->
-      return = Benchee.Formatters.HTML.output(@sample_suite)
-      assert return == @sample_suite
-    end)
-
-    assert File.exists?(@expected_filename)
-    assert_assets_copied()
-
-    content = File.read!(@expected_filename)
-    assert_includes(content, ["My Job", "average"])
-  after
-    if File.exists?(@test_directory), do: File.rm_rf!(@test_directory)
-  end
-
-  test "assets should not be copied when assets inlining is on" do
-    suite = %Benchee.Suite{
-      scenarios: [@scenario],
-      system: @system_info,
-      configuration: %Benchee.Configuration{
-        formatter_options: %{html: %{file: @filename, auto_open: false, inline_assets: true}}
-      }
-    }
-
-    capture_io(fn ->
-      return = Benchee.Formatters.HTML.output(suite)
-      assert return == suite
-    end)
-
-    assert File.exists?(@expected_filename)
-
-    content = File.read!(@expected_filename)
-    assets_inlined(content, ["<style>", "<script>"])
-
-    refute File.exists?("#{@test_directory}/assets/stylesheets/benchee.css")
-    refute File.exists?("#{@test_directory}/assets/fontello/css/fontello.css")
-
-    refute File.exists?("#{@test_directory}/assets/javascripts/benchee.js")
-    refute File.exists?("#{@test_directory}/assets/javascripts/plotly-1.30.1.min.js")
-  after
-    if File.exists?(@test_directory), do: File.rm_rf!(@test_directory)
-  end
-
   defp assets_inlined(html, assets) do
     Enum.each(assets, fn asset ->
       assert html =~ asset
     end)
   end
-
-  test ".output lets you know where it put the html" do
-    output =
-      capture_io(fn ->
-        Benchee.Formatters.HTML.output(@sample_suite)
-      end)
-
-    assert output =~ @expected_filename
-    assert File.exists?(@expected_filename)
-  after
-    if File.exists?(@test_directory), do: File.rm_rf!(@test_directory)
-  end
-
-  defp assert_assets_copied do
-    assert File.exists?("#{@test_directory}/assets/javascripts/benchee.js")
-    assert File.exists?("#{@test_directory}/assets/javascripts/plotly-1.30.1.min.js")
-  end
 end

From 79614dce262baaf02ed5db619bf6401f98bfb824 Mon Sep 17 00:00:00 2001
From: Devon Estes <devon.c.estes@gmail.com>
Date: Thu, 22 Nov 2018 18:22:19 +0100
Subject: [PATCH 2/2] Add memory measurements

Now we can see some memory stuff in HTML form!
---
 lib/benchee/formatters/html.ex                | 128 +++++++++++++-----
 mix.exs                                       |   2 +-
 mix.lock                                      |   2 +-
 priv/templates/comparison.html.eex            |  27 +++-
 .../templates/memory_scenario_detail.html.eex |  55 ++++++++
 priv/templates/partials/data_table.html.eex   |  26 ++--
 ....eex => run_time_scenario_detail.html.eex} |  22 +--
 samples/run.exs                               |  19 +--
 .../formatters/html_integration_test.exs      |  16 ++-
 test/benchee/formatters/html_test.exs         |  45 +++++-
 10 files changed, 260 insertions(+), 82 deletions(-)
 create mode 100644 priv/templates/memory_scenario_detail.html.eex
 rename priv/templates/{job_detail.html.eex => run_time_scenario_detail.html.eex} (66%)

diff --git a/lib/benchee/formatters/html.ex b/lib/benchee/formatters/html.ex
index acf2d9f..8915a61 100644
--- a/lib/benchee/formatters/html.ex
+++ b/lib/benchee/formatters/html.ex
@@ -118,53 +118,89 @@ defmodule Benchee.Formatters.HTML do
 
   defp reports_for_input({input_name, scenarios}, system, filename, unit_scaling, inline_assets) do
     units = Conversion.units(scenarios, unit_scaling)
-    job_reports = job_reports(input_name, scenarios, system, units, inline_assets)
+    scenario_reports = scenario_reports(input_name, scenarios, system, units, inline_assets)
     comparison = comparison_report(input_name, scenarios, system, filename, units, inline_assets)
-    [comparison | job_reports]
+    [comparison | scenario_reports]
   end
 
-  defp job_reports(input_name, scenarios, system, units, inline_assets) do
+  defp scenario_reports(input_name, scenarios, system, units, inline_assets) do
     # extract some of me to benchee_json pretty please?
-    Enum.map(scenarios, fn scenario ->
-      job_json =
-        JSON.encode!(%{
-          statistics: scenario.run_time_statistics,
-          run_times: scenario.run_times
-        })
-
-      {
-        [input_name, scenario.name],
-        job_detail(
-          input_name,
-          scenario.name,
-          scenario.run_time_statistics,
-          system,
-          units,
-          job_json,
-          inline_assets
-        )
-      }
+    Enum.flat_map(scenarios, fn scenario ->
+      inputs =
+        if scenario.memory_usage_statistics do
+          [
+            {"run_time", scenario.run_time_statistics, scenario.run_times,
+             :run_time_scenario_detail},
+            {"memory", scenario.memory_usage_statistics, scenario.memory_usages,
+             :memory_scenario_detail}
+          ]
+        else
+          [
+            {"run_time", scenario.run_time_statistics, scenario.run_times,
+             :run_time_scenario_detail}
+          ]
+        end
+
+      Enum.map(inputs, fn {label, stats, raw_measurements, template_fun} ->
+        scenario_json =
+          JSON.encode!(%{
+            statistics: stats,
+            raw_measurements: raw_measurements
+          })
+
+        {
+          [label, input_name, scenario.name],
+          apply(__MODULE__, template_fun, [
+            input_name,
+            scenario.name,
+            stats,
+            system,
+            units,
+            scenario_json,
+            inline_assets
+          ])
+        }
+      end)
     end)
   end
 
   defp comparison_report(input_name, scenarios, system, filename, units, inline_assets) do
     input_json = JSON.format_scenarios_for_input(scenarios)
 
-    sorted_statistics =
+    sorted_run_time_statistics =
+      scenarios
+      |> Statistics.sort()
+      |> Enum.map(fn scenario ->
+        {scenario.name, %{statistics: scenario.run_time_statistics}}
+      end)
+      |> Map.new()
+
+    sorted_memory_statistics =
       scenarios
       |> Statistics.sort()
       |> Enum.map(fn scenario ->
-        {scenario.name, %{run_time_statistics: scenario.run_time_statistics}}
+        {scenario.name, %{statistics: scenario.memory_usage_statistics}}
       end)
       |> Map.new()
 
+    sorted_memory_statistics =
+      if Enum.all?(sorted_memory_statistics, fn
+        {_, %{statistics: nil}} -> true
+        _ -> false
+      end) do
+        nil
+      else
+        sorted_memory_statistics
+      end
+
     input_run_times =
       scenarios
       |> Enum.map(fn scenario -> {scenario.name, scenario.run_times} end)
       |> Map.new()
 
     input_suite = %{
-      statistics: sorted_statistics,
+      run_time_statistics: sorted_run_time_statistics,
+      memory_usage_statistics: sorted_memory_statistics,
       run_times: input_run_times,
       system: system,
       job_count: length(scenarios),
@@ -208,6 +244,8 @@ defmodule Benchee.Formatters.HTML do
     Duration.format({Duration.scale(duration, unit), unit})
   end
 
+  defp format_count(nil, _), do: nil
+
   defp format_count(count, unit) do
     Count.format({Count.scale(count, unit), unit})
   end
@@ -260,15 +298,35 @@ defmodule Benchee.Formatters.HTML do
     :inline_assets
   ])
 
-  EEx.function_from_file(:defp, :job_detail, "priv/templates/job_detail.html.eex", [
-    :input_name,
-    :job_name,
-    :job_statistics,
-    :system,
-    :units,
-    :job_json,
-    :inline_assets
-  ])
+  EEx.function_from_file(
+    :def,
+    :memory_scenario_detail,
+    "priv/templates/memory_scenario_detail.html.eex",
+    [
+      :input_name,
+      :scenario_name,
+      :scenario_statistics,
+      :system,
+      :units,
+      :scenario_json,
+      :inline_assets
+    ]
+  )
+
+  EEx.function_from_file(
+    :def,
+    :run_time_scenario_detail,
+    "priv/templates/run_time_scenario_detail.html.eex",
+    [
+      :input_name,
+      :scenario_name,
+      :scenario_statistics,
+      :system,
+      :units,
+      :scenario_json,
+      :inline_assets
+    ]
+  )
 
   EEx.function_from_file(:defp, :index, "priv/templates/index.html.eex", [
     :names_to_paths,
@@ -311,7 +369,7 @@ defmodule Benchee.Formatters.HTML do
   ])
 
   # Small wrappers to have default arguments
-  defp render_data_table(statistics, units, options \\ []) do
+  defp render_data_table(statistics, units, options) do
     data_table(statistics, units, options)
   end
 
diff --git a/mix.exs b/mix.exs
index 46b8938..97330c2 100644
--- a/mix.exs
+++ b/mix.exs
@@ -48,7 +48,7 @@ defmodule BencheeHTML.Mixfile do
   defp deps do
     [
       {:benchee, github: "PragTob/benchee"},
-      {:benchee_json, github: "PragTob/benchee_json"},
+      {:benchee_json, github: "PragTob/benchee_json", branch: "add-memory-measurements"},
       {:excoveralls, "~> 0.8.1", only: :test},
       {:mix_test_watch, "~> 0.2", only: :dev},
       {:credo, "~> 1.0", only: :dev},
diff --git a/mix.lock b/mix.lock
index 36fce95..cf4dd73 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,6 +1,6 @@
 %{
   "benchee": {:git, "https://github.com/PragTob/benchee.git", "ee9f7233f94ea8b38795be24c7e5714761d45add", []},
-  "benchee_json": {:git, "https://github.com/PragTob/benchee_json.git", "7c719bb18de353888de031a4c3f6d7e32e186015", []},
+  "benchee_json": {:git, "https://github.com/PragTob/benchee_json.git", "bede232dee5363c74e14e84b6da16e3a9da41610", [branch: "add-memory-measurements"]},
   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
   "certifi": {:hex, :certifi, "2.4.2", "75424ff0f3baaccfd34b1214184b6ef616d89e420b258bb0a5ea7d7bc628f7f0", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"},
   "credo": {:hex, :credo, "1.0.0", "aaa40fdd0543a0cf8080e8c5949d8c25f0a24e4fc8c1d83d06c388f5e5e0ea42", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
diff --git a/priv/templates/comparison.html.eex b/priv/templates/comparison.html.eex
index ecd7817..4122b37 100644
--- a/priv/templates/comparison.html.eex
+++ b/priv/templates/comparison.html.eex
@@ -9,18 +9,16 @@
 
     <article>
       <h2>
-        Comparison
+        Run Time Comparison
         <a href="/PragTob/benchee_html/wiki/Chart-Types#comparison">
           <i
           class="benchee-help-icon icon-question-circle-o"
-          title="Comparisons of the different benchmarking jobs.
-For Iterations per Second higher is better, the black whiskers denote standard deviation. otherwise it is runtime so lower is better.
-The Box Plot shows a distribution of run times with the median and outliers."
+          title="Comparisons of the different benchmarking jobs. For Iterations per Second higher is better, the black whiskers denote standard deviation. otherwise it is runtime so lower is better. The Box Plot shows a distribution of run times with the median and outliers."
           ></i>
         </a>
       </h2>
       <section>
-        <%= render_data_table(suite.statistics, units, path_information: {suite.filename, input_name}) %>
+        <%= render_data_table(suite.run_time_statistics, units, path_information: {suite.filename, input_name}, run_time: true) %>
       </section>
 
       <section class="benchee-plot-container">
@@ -32,6 +30,23 @@ The Box Plot shows a distribution of run times with the median and outliers."
       </section>
     </article>
 
+    <%= if suite.memory_usage_statistics do %>
+      <article>
+        <h2>
+          Memory Usage Comparison
+          <a href="/PragTob/benchee_html/wiki/Chart-Types#comparison">
+            <i
+            class="benchee-help-icon icon-question-circle-o"
+            title="This is a memory usage comparison"
+            ></i>
+          </a>
+        </h2>
+        <section>
+          <%= render_data_table(suite.memory_usage_statistics, units, path_information: {suite.filename, input_name}, run_time: false) %>
+        </section>
+      </article>
+    <% end %>
+
     <footer>
       <%= render_footer() %>
     </footer>
@@ -41,7 +56,7 @@ The Box Plot shows a distribution of run times with the median and outliers."
     <script>
       var suite      = <%= suite_json %>;
       var runTimes   = suite.run_times;
-      var statistics = suite.statistics;
+      var statistics = suite.run_time_statistics;
       var sortOrder  = suite.sort_order;
 
       var inputHeadline = "<%= input_headline(input_name) %>"
diff --git a/priv/templates/memory_scenario_detail.html.eex b/priv/templates/memory_scenario_detail.html.eex
new file mode 100644
index 0000000..7239a20
--- /dev/null
+++ b/priv/templates/memory_scenario_detail.html.eex
@@ -0,0 +1,55 @@
+<!DOCTYPE html>
+<html>
+  <%= head(inline_assets) %>
+  <body>
+
+    <%= header(input_name) %>
+
+    <%= render_system_info(system) %>
+
+    <article>
+      <h2>
+        <a name="<%= scenario_name %>"></a>
+        <%= scenario_name %>
+        <a href="/PragTob/benchee_html/wiki/Chart-Types#individual">
+          <i
+            class="benchee-help-icon icon-question-circle-o"
+            title="Graphs showing the individual memory usage distribution for <%= scenario_name %>.
+Run Times Histogram shows the distribution of occurrences of specific run times in buckets.
+Raw Run Times shows all individual recorded run times in the order they were recorded in."
+          ></i>
+        </a>
+      </h2>
+
+      <section>
+        <%= render_data_table(%{scenario_name => %{statistics: scenario_statistics}}, units, run_time: false) %>
+      </section>
+
+      <%= if scenario_statistics.minimum != scenario_statistics.maximum do %>
+        <section class="benchee-plot-container">
+          <div id="sorted-run-times" class="plot" data-job-name="<%= scenario_name %>"></div>
+        </section>
+        <section class="benchee-plot-container">
+          <div id="raw-run-times" class="plot" data-job-name="<%= scenario_name %>"></div>
+        </section>
+      <% end %>
+    </article>
+
+    <footer>
+      <%= render_footer() %>
+    </footer>
+
+    <%= js_includes(inline_assets) %>
+
+    <script>
+      var measurements = <%= scenario_json %>;
+      var rawMeasurements = measurements.raw_measurements;
+      var statistics = measurements.statistics;
+
+      var inputHeadline = "<%= input_headline(input_name) %>"
+
+      drawRawRunTimeCharts(rawMeasurements, inputHeadline, statistics);
+      drawRunTimeHistograms(rawMeasurements, inputHeadline);
+    </script>
+  </body>
+</html>
diff --git a/priv/templates/partials/data_table.html.eex b/priv/templates/partials/data_table.html.eex
index 9d68648..52d89aa 100644
--- a/priv/templates/partials/data_table.html.eex
+++ b/priv/templates/partials/data_table.html.eex
@@ -3,7 +3,9 @@
   <thead>
     <tr>
       <th class="text-left">Name</th>
-      <th>Iterations per Second</th>
+      <%= if options[:run_time] do %>
+        <th>Iterations per Second</th>
+      <% end %>
       <th>Average</th>
       <th>Deviation</th>
       <th>median</th>
@@ -13,7 +15,7 @@
     </tr>
   </thead>
   <tbody>
-    <%= for {name, %{run_time_statistics: statistics}} <- statistics do %>
+    <%= for {name, %{statistics: statistics}} <- statistics do %>
       <tr>
         <%= case Keyword.get(options, :path_information) do %>
         <% {base_name, input_name} -> %>
@@ -21,12 +23,20 @@
         <% nil -> %>
           <td class="text-left"><%= name %></td>
         <% end %>
-        <td><%= format_count statistics.ips, units.ips %></td>
-        <td><%= format_duration statistics.average, units.run_time %></td>
-        <td><%= format_percent statistics.std_dev_ratio %></td>
-        <td><%= format_duration statistics.median, units.run_time %></td>
-        <td><%= format_duration statistics.minimum, units.run_time %></td>
-        <td><%= format_duration statistics.maximum, units.run_time %></td>
+        <%= if options[:run_time] do %>
+          <td><%= format_count statistics.ips, units.ips %></td>
+          <td><%= format_duration statistics.average, units.run_time %></td>
+          <td><%= format_percent statistics.std_dev_ratio %></td>
+          <td><%= format_duration statistics.median, units.run_time %></td>
+          <td><%= format_duration statistics.minimum, units.run_time %></td>
+          <td><%= format_duration statistics.maximum, units.run_time %></td>
+        <% else %>
+          <td><%= format_duration statistics.average, units.memory %></td>
+          <td><%= format_percent statistics.std_dev_ratio %></td>
+          <td><%= format_duration statistics.median, units.memory %></td>
+          <td><%= format_duration statistics.minimum, units.memory %></td>
+          <td><%= format_duration statistics.maximum, units.memory %></td>
+        <% end %>
         <td><%= statistics.sample_size %></td>
       </tr>
     <% end %>
diff --git a/priv/templates/job_detail.html.eex b/priv/templates/run_time_scenario_detail.html.eex
similarity index 66%
rename from priv/templates/job_detail.html.eex
rename to priv/templates/run_time_scenario_detail.html.eex
index db4e91d..47cb942 100644
--- a/priv/templates/job_detail.html.eex
+++ b/priv/templates/run_time_scenario_detail.html.eex
@@ -9,12 +9,12 @@
 
     <article>
       <h2>
-        <a name="<%= job_name %>"></a>
-        <%= job_name %>
+        <a name="<%= scenario_name %>"></a>
+        <%= scenario_name %>
         <a href="/PragTob/benchee_html/wiki/Chart-Types#individual">
           <i
             class="benchee-help-icon icon-question-circle-o"
-            title="Graphs showing the individual run time distribution for <%= job_name %>.
+            title="Graphs showing the individual run time distribution for <%= scenario_name %>.
 Run Times Histogram shows the distribution of occurrences of specific run times in buckets.
 Raw Run Times shows all individual recorded run times in the order they were recorded in."
           ></i>
@@ -22,14 +22,14 @@ Raw Run Times shows all individual recorded run times in the order they were rec
       </h2>
 
       <section>
-        <%= render_data_table(%{job_name => %{run_time_statistics: job_statistics}}, units) %>
+        <%= render_data_table(%{scenario_name => %{statistics: scenario_statistics}}, units, run_time: true) %>
       </section>
 
       <section class="benchee-plot-container">
-        <div id="sorted-run-times" class="plot" data-job-name="<%= job_name %>"></div>
+        <div id="sorted-run-times" class="plot" data-job-name="<%= scenario_name %>"></div>
       </section>
       <section class="benchee-plot-container">
-        <div id="raw-run-times" class="plot" data-job-name="<%= job_name %>"></div>
+        <div id="raw-run-times" class="plot" data-job-name="<%= scenario_name %>"></div>
       </section>
     </article>
 
@@ -40,14 +40,14 @@ Raw Run Times shows all individual recorded run times in the order they were rec
     <%= js_includes(inline_assets) %>
 
     <script>
-      var measurements = <%= job_json %>;
-      var runTimes     = measurements.run_times;
-      var statistics   = measurements.statistics;
+      var measurements = <%= scenario_json %>;
+      var rawMeasurements = measurements.raw_measurements;
+      var statistics = measurements.statistics;
 
       var inputHeadline = "<%= input_headline(input_name) %>"
 
-      drawRawRunTimeCharts(runTimes, inputHeadline, statistics);
-      drawRunTimeHistograms(runTimes, inputHeadline);
+      drawRawRunTimeCharts(rawMeasurements, inputHeadline, statistics);
+      drawRunTimeHistograms(rawMeasurements, inputHeadline);
     </script>
   </body>
 </html>
diff --git a/samples/run.exs b/samples/run.exs
index 83378bc..1dbda7f 100644
--- a/samples/run.exs
+++ b/samples/run.exs
@@ -1,14 +1,15 @@
 # It is possible to use multiple formatters so that you have both the Console
 # output and an HTML file.
 list = Enum.to_list(1..10_000)
-map_fun = fn(i) -> [i, i * i] end
+map_fun = fn i -> [i, i * i] end
 
-Benchee.run(%{
-  "flat_map"    => fn -> Enum.flat_map(list, map_fun) end,
-  "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten end
-},
-  formatters: [
-    Benchee.Formatters.HTML,
-    Benchee.Formatters.Console
-  ]
+Benchee.run(
+  %{
+    "flat_map" => fn -> Enum.flat_map(list, map_fun) end,
+    "map.flatten" => fn -> list |> Enum.map(map_fun) |> List.flatten() end
+  },
+  formatters: [{Benchee.Formatters.HTML, auto_open: false}],
+  time: 0.05,
+  memory_time: 0.05,
+  warmup: 0.01
 )
diff --git a/test/benchee/formatters/html_integration_test.exs b/test/benchee/formatters/html_integration_test.exs
index 495d214..e7e2464 100644
--- a/test/benchee/formatters/html_integration_test.exs
+++ b/test/benchee/formatters/html_integration_test.exs
@@ -15,6 +15,7 @@ defmodule Benchee.Formatters.HTMLIntegrationTest do
   test "works just fine" do
     benchee_options = [
       time: 0.01,
+      memory_time: 0.01,
       warmup: 0.02,
       formatters: [{Benchee.Formatters.HTML, file: @file_path, auto_open: false}]
     ]
@@ -32,6 +33,7 @@ defmodule Benchee.Formatters.HTMLIntegrationTest do
   test "works fine with filename not provided" do
     benchee_options = [
       time: 0.01,
+      memory_time: 0.01,
       warmup: 0.02,
       formatters: [{Benchee.Formatters.HTML, auto_open: false}]
     ]
@@ -51,7 +53,7 @@ defmodule Benchee.Formatters.HTMLIntegrationTest do
       Benchee.run(
         %{
           "Sleep" => fn -> :timer.sleep(10) end,
-          "Sleep longer" => fn -> :timer.sleep(20) end
+          "List" => fn -> [:rand.uniform()] end
         },
         benchee_options
       )
@@ -59,19 +61,25 @@ defmodule Benchee.Formatters.HTMLIntegrationTest do
       assert File.exists?(assertion_data.comparison_path)
 
       assert File.exists?(
-               "#{assertion_data.test_directory}/#{assertion_data.base_name}_sleep.html"
+               "#{assertion_data.test_directory}/#{assertion_data.base_name}_memory_sleep.html"
+             )
+      assert File.exists?(
+               "#{assertion_data.test_directory}/#{assertion_data.base_name}_run_time_sleep.html"
              )
 
       assert File.exists?(
-               "#{assertion_data.test_directory}/#{assertion_data.base_name}_sleep_longer.html"
+               "#{assertion_data.test_directory}/#{assertion_data.base_name}_memory_list.html"
              )
 
+      assert File.exists?(
+               "#{assertion_data.test_directory}/#{assertion_data.base_name}_run_time_list.html"
+             )
       assert File.exists?(assertion_data.file_path)
       html = File.read!(assertion_data.comparison_path)
 
       assert html =~ "<body>"
       assert html =~ "Sleep"
-      assert html =~ "Sleep longer"
+      assert html =~ "List"
       assert html =~ "ips-comparison"
       assert html =~ "System info</a>"
       assert html =~ "benchee version"
diff --git a/test/benchee/formatters/html_test.exs b/test/benchee/formatters/html_test.exs
index 1048548..72c7800 100644
--- a/test/benchee/formatters/html_test.exs
+++ b/test/benchee/formatters/html_test.exs
@@ -20,6 +20,7 @@ defmodule Benchee.Formatters.HTMLTest do
     job_name: "My Job",
     name: "My Job",
     run_times: [190, 200, 210],
+    memory_usages: [190, 200, 210],
     input_name: "Some Input",
     input: "Some Input",
     run_time_statistics: %Benchee.Statistics{
@@ -32,6 +33,17 @@ defmodule Benchee.Formatters.HTMLTest do
       sample_size: 3,
       minimum: 190,
       maximum: 210
+    },
+    memory_usage_statistics: %Benchee.Statistics{
+      average: 200.0,
+      ips: nil,
+      std_dev: 20,
+      std_dev_ratio: 0.1,
+      std_dev_ips: nil,
+      median: 190.0,
+      sample_size: 3,
+      minimum: 190,
+      maximum: 210
     }
   }
   @sample_suite %Benchee.Suite{
@@ -49,7 +61,9 @@ defmodule Benchee.Formatters.HTMLTest do
     end
 
     test "has the important suite data in the html result" do
-      Enum.each(comparison_and_job_htmls(), fn html ->
+      [comparison_html, run_time_html, memory_html] = comparison_and_job_htmls()
+
+      Enum.each([comparison_html, run_time_html], fn html ->
         assert_includes(
           html,
           [
@@ -66,6 +80,17 @@ defmodule Benchee.Formatters.HTMLTest do
           ]
         )
       end)
+
+      assert_includes(memory_html, [
+        "[190,200,210]",
+        "\"average\":200.0",
+        "\"median\":190.0",
+        "My Job",
+        ">3<",
+        ">190 B<",
+        ">210 B<",
+        ">200 B<"
+      ])
     end
 
     test "has system info in the html result" do
@@ -104,7 +129,9 @@ defmodule Benchee.Formatters.HTMLTest do
 
       suite = %Benchee.Suite{@sample_suite | scenarios: [scenario]}
 
-      Enum.each(comparison_and_job_htmls(suite), fn html ->
+      [comparison_html, run_time_html, _memory_html] = comparison_and_job_htmls(suite)
+
+      Enum.each([comparison_html, run_time_html], fn html ->
         assert_includes(
           html,
           [">1.50 μs<", ">666.66<", ">1.40 μs<", ">1.30 μs<", ">1.70 μs<"]
@@ -114,11 +141,12 @@ defmodule Benchee.Formatters.HTMLTest do
 
     test "produces the right JSON data without the input level" do
       %{["Some Input", "comparison"] => html} = HTML.format(@sample_suite, @default_options)
-      assert html =~ "\"statistics\":{\"My Job\""
+      assert html =~ "\"run_time_statistics\":{\"My Job\""
     end
 
     test "shows the units alright" do
-      Enum.each(comparison_and_job_htmls(), fn html ->
+      [comparison_html, run_time_html, _memory_html] = comparison_and_job_htmls()
+      Enum.each([comparison_html, run_time_html], fn html ->
         assert html =~ "±"
         assert html =~ "ns"
       end)
@@ -193,10 +221,13 @@ defmodule Benchee.Formatters.HTMLTest do
          suite \\ @sample_suite,
          options \\ %{html: %{file: @filename, auto_open: false}}
        ) do
-    %{["Some Input", "comparison"] => comparison_html, ["Some Input", "My Job"] => job_html} =
-      HTML.format(suite, options)
+    assert %{
+             ["Some Input", "comparison"] => comparison_html,
+             ["run_time", "Some Input", "My Job"] => run_time_html,
+             ["memory", "Some Input", "My Job"] => memory_html
+           } = HTML.format(suite, options)
 
-    [comparison_html, job_html]
+    [comparison_html, run_time_html, memory_html]
   end
 
   defp assert_includes(html, expected_contents) do