Merge pull request #282 from JuliaIO/kms/update-benchmarks

Update benchmarks to Julia 1.x

bench/bench.jl

@@ -4,22 +4,22 @@ using ArgParse
 using JSON
 
 
-function bench(f, simulate=false)
+function bench(f, flags, parsefile, simulate=false)
     fp = joinpath(JSON_DATA_DIR, string(f, ".json"))
     if !isfile(fp)
         println("Downloading benchmark file...")
         download(DATA_SOURCES[f], fp)
     end
     GC.gc()  # run gc so it doesn't affect benchmarks
-    t = if args["parse"]["parse-file"]
+    t = if parsefile
         @elapsed JSON.parsefile(fp)
     else
         data = read(fp, String)
         @elapsed JSON.Parser.parse(data)
     end
 
     if !simulate
-        printstyled(" [Bench$FLAGS] "; color=:yellow)
+        printstyled(" [Bench$flags] "; color=:yellow)
         println(f, " ", t, " seconds")
     end
     t
@@ -35,58 +35,64 @@ const DATA_SOURCES = Dict(
     "citylots" => "https://raw.githubusercontent.com/zemirco/sf-city-lots-json/master/citylots.json",
     "twitter" => "https://raw.githubusercontent.com/miloyip/nativejson-benchmark/v1.0.0/data/twitter.json")
 
-@add_arg_table s begin
-    "parse"
-        action = :command
-        help = "Run a JSON parser benchmark"
-    "list"
-        action = :command
-        help = "List available JSON files for use"
-end
+function main()
+    @add_arg_table s begin
+        "parse"
+            action = :command
+            help = "Run a JSON parser benchmark"
+        "list"
+            action = :command
+            help = "List available JSON files for use"
+    end
 
-@add_arg_table s["parse"] begin
-    "--include-compile", "-c"
-        help = "If set, include the compile time in measurements"
-        action = :store_true
-    "--parse-file", "-f"
-        help = "If set, measure JSON.parsefile, hence including IO time"
-        action = :store_true
-    "file"
-        help = "The JSON file to benchmark (leave out to benchmark all)"
-        required = false
-end
+    @add_arg_table s["parse"] begin
+        "--include-compile", "-c"
+            help = "If set, include the compile time in measurements"
+            action = :store_true
+        "--parse-file", "-f"
+            help = "If set, measure JSON.parsefile, hence including IO time"
+            action = :store_true
+        "file"
+            help = "The JSON file to benchmark (leave out to benchmark all)"
+            required = false
+    end
 
-const args = parse_args(ARGS, s)
+    args = parse_args(ARGS, s)
 
-if args["%COMMAND%"] == "parse"
-    const FLAGS = string(
-            args["parse"]["include-compile"] ? "C" : "",
-            args["parse"]["parse-file"] ? "F" : "")
+    if args["%COMMAND%"] == "parse"
+        include_compile = args["parse"]["include-compile"]
+        parsefile = args["parse"]["parse-file"]
 
-    if args["parse"]["file"] ≠ nothing
-        const file = args["parse"]["file"]
+        flags = string(include_compile ? "C" : "",
+                       parsefile ? "F" : "")
 
-        if !args["parse"]["include-compile"]
-            bench(file, true)
-        end
-        bench(file)
-    else
-        times = 1.0
-        if args["parse"]["include-compile"]
-            error("Option --include-compile can only be used for single file.")
-        end
-        for k in sort(collect(keys(DATA_SOURCES)))
-            bench(k, true)  # warm up compiler
+        if args["parse"]["file"] ≠ nothing
+            file = args["parse"]["file"]
+
+            if !include_compile
+                bench(file, flags, parsefile, true)
+            end
+            bench(file, flags, parsefile)
+        else
+            times = 1.0
+            if include_compile
+                error("Option --include-compile can only be used for single file.")
+            end
+            for k in sort(collect(keys(DATA_SOURCES)))
+                bench(k, flags, parsefile, true)  # warm up compiler
+            end
+            for k in sort(collect(keys(DATA_SOURCES)))
+                times *= bench(k, flags, parsefile)  # do benchmark
+            end
+            printstyled(" [Bench$flags] ", color=:yellow)
+            println("Total (G.M.) ", times^(1/length(DATA_SOURCES)), " seconds")
         end
+    elseif args["%COMMAND%"] == "list"
+        println("Available benchmarks are:")
         for k in sort(collect(keys(DATA_SOURCES)))
-            times *= bench(k)  # do benchmark
+            println(" • $k")
         end
-        print_with_color(:yellow, " [Bench$FLAGS] ")
-        println("Total (G.M.) ", times^(1/length(DATA_SOURCES)), " seconds")
-    end
-elseif args["%COMMAND%"] == "list"
-    println("Available benchmarks are:")
-    for k in sort(collect(keys(DATA_SOURCES)))
-        println(" • $k")
     end
 end
+
+main()

bench/bench.py

@@ -0,0 +1,19 @@
+from functools import reduce
+from textwrap import dedent as dd
+from timeit import repeat
+
+
+sources = ["canada", "citm_catalog", "citylots", "twitter"]
+
+min_times = []
+for source in sources:
+    s = dd(f"""\
+    with open("../data/{source}.json") as f:
+        json.load(f)""")
+    times = repeat(stmt=s, setup="import json", repeat=3, number=1)
+    t = reduce(min, times)
+    print(f"{source} {t:0.06f} seconds")
+    min_times.append(t)
+
+geo_mean = reduce(lambda a, b: a*b, min_times)**(1/len(min_times))
+print(f"Total (G.M): {geo_mean:0.06f}")