From 92e187e07ac722422211da5cb7b6a867f72999c0 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 13 Jul 2018 16:41:45 +0200 Subject: [PATCH] wip: generate precompile as part of build process --- Make.inc | 3 ++ Makefile | 10 ++++- base/.gitignore | 1 + base/Makefile | 1 + base/options.jl | 1 + base/precompile.jl | 2 + contrib/fixup_precompile.jl | 76 +++++++++++++++++++++++----------- contrib/generate_precompile.jl | 40 ++++++++++++++++++ contrib/precompile_replay.jl | 53 ++++++++++++++++++++++++ src/gf.c | 12 +++--- src/jloptions.c | 16 +++++++ src/julia.h | 1 + stdlib/REPL/test/fake_repl.jl | 49 ++++++++++++++++++++++ stdlib/REPL/test/repl.jl | 48 +-------------------- 14 files changed, 234 insertions(+), 79 deletions(-) create mode 100644 contrib/generate_precompile.jl create mode 100644 contrib/precompile_replay.jl create mode 100644 stdlib/REPL/test/fake_repl.jl diff --git a/Make.inc b/Make.inc index 8cb2c1014fa5a4..60538570a68eb3 100644 --- a/Make.inc +++ b/Make.inc @@ -134,6 +134,9 @@ else NO_GIT := 1 endif +# Setting for local precompile +JULIA_LOCAL_PRECOMPILE ?= 1 + # Julia's Semantic Versioning system labels the three decimal places in a version number as # the major, minor and patch versions. Typically the major version would be incremented # whenever a backwards-incompatible change is made, the minor version would be incremented diff --git a/Makefile b/Makefile index c7ac7258e61209..4e74540060c9c5 100644 --- a/Makefile +++ b/Makefile @@ -74,13 +74,20 @@ julia-ui-release julia-ui-debug : julia-ui-% : julia-src-% julia-sysimg : julia-base julia-ui-$(JULIA_BUILD_MODE) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys.ji JULIA_EXECUTABLE='$(JULIA_EXECUTABLE)' +julia-sysimg-precompile-release julia-sysimg-precompile-debug : julia-sysimg-precompile-% : julia-sysimg-% +ifeq ($(JULIA_LOCAL_PRECOMPILE), 1) + @echo Generating precompile statements for the local system, disable with JULIA_LOCAL_PRECOMPILE=0 in Make.user + @$(JULIA_EXECUTABLE) $(JULIAHOME)/contrib/generate_precompile.jl + @$(MAKE) julia-sysimg-$* +endif + julia-sysimg-release : julia-sysimg julia-ui-release @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys.$(SHLIB_EXT) julia-sysimg-debug : julia-sysimg julia-ui-debug @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) -julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-% julia-symlink julia-libccalltest julia-base-cache +julia-debug julia-release : julia-% : julia-ui-% julia-sysimg-precompile-% julia-symlink julia-libccalltest julia-base-cache debug release : % : julia-% @@ -567,6 +574,7 @@ distcleanall: cleanall julia-debug julia-release julia-deps \ julia-ui-release julia-ui-debug julia-src-release julia-src-debug \ julia-symlink julia-base julia-sysimg julia-sysimg-release julia-sysimg-debug \ + julia-sysimg-precompile-release julia-sysimg-precompile-debug \ test testall testall1 test clean distcleanall cleanall clean-* \ run-julia run-julia-debug run-julia-release run \ install binary-dist light-source-dist.tmp light-source-dist \ diff --git a/base/.gitignore b/base/.gitignore index 49b6f3f2d7808b..610d88371ad824 100644 --- a/base/.gitignore +++ b/base/.gitignore @@ -7,3 +7,4 @@ /version_git.jl /version_git.jl.phony /userimg.jl +/precompile_local.jl \ No newline at end of file diff --git a/base/Makefile b/base/Makefile index 21c7ad457a1404..d509a80a1fa668 100644 --- a/base/Makefile +++ b/base/Makefile @@ -224,4 +224,5 @@ clean: -rm -f $(BUILDDIR)/file_constants.jl -rm -f $(BUILDDIR)/version_git.jl -rm -f $(BUILDDIR)/version_git.jl.phony + -rm -f $(BUILDDIR)/precompile_local.jl -rm -f $(build_private_libdir)/lib*.$(SHLIB_EXT)* diff --git a/base/options.jl b/base/options.jl index 4561d299651763..0abf2d55fd866b 100644 --- a/base/options.jl +++ b/base/options.jl @@ -26,6 +26,7 @@ struct JLOptions warn_overwrite::Int8 can_inline::Int8 polly::Int8 + trace_compile::Int8 fast_math::Int8 worker::Int8 cookie::Ptr{UInt8} diff --git a/base/precompile.jl b/base/precompile.jl index 55c5af8487a94a..a8eb96f54c8d75 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -16,6 +16,8 @@ for (_pkgid, _mod) in Base.loaded_modules @eval PrecompileStagingArea $(Symbol(_mod)) = $_mod end end +f = joinpath(@__DIR__, "precompile_local.jl") +isfile(f) && Base.include(PrecompileStagingArea, f) @eval PrecompileStagingArea begin precompile(Tuple{Type{Array{Base.StackTraces.StackFrame, 1}}, UndefInitializer, Int64}) precompile(Tuple{Type{Array{Union{Nothing, String}, 1}}, UndefInitializer, Int64}) diff --git a/contrib/fixup_precompile.jl b/contrib/fixup_precompile.jl index f1139a68e9bbec..da9c4c2ca1fc53 100644 --- a/contrib/fixup_precompile.jl +++ b/contrib/fixup_precompile.jl @@ -11,22 +11,24 @@ const HEADER = """ # Steps to regenerate this file: # 1. Remove all `precompile` calls # 2. Rebuild system image -# 3. Enable TRACE_COMPILE in options.h and rebuild -# 4. Run `./julia 2> precompiles.txt` and do various things. +# 4. Run `./julia --trace-compile 2> precompiles.txt` and do various things. # 5. Run `./julia contrib/fixup_precompile.jl precompiles.txt to overwrite `precompile.jl` # or ./julia contrib/fixup_precompile.jl --merge precompiles.txt to merge into existing # `precompile.jl` """ -function fixup_precompile(new_precompile_file; merge=false) - old_precompile_file = joinpath(Sys.BINDIR, "..", "..", "base", "precompile.jl") +function fixup_precompile(new_precompile_file; merge, keep_anonymous, header, output) precompile_statements = Set{String}() - for file in [new_precompile_file; merge ? old_precompile_file : []] + isfile(output) || touch(output) + for file in [new_precompile_file; merge ? output : []] for line in eachline(file) line = strip(line) # filter out closures, which might have different generated names in different environments - occursin(r"#[0-9]", line) && continue + if !keep_anonymous && occursin(r"#[0-9]", line) + continue + end + # Other stuff than precompile statements might have been written to STDERR startswith(line, "precompile(Tuple{") || continue # Ok, add the line @@ -34,37 +36,61 @@ function fixup_precompile(new_precompile_file; merge=false) end end - open(old_precompile_file, "w") do f - println(f, HEADER) - println(f, """ - let - PrecompileStagingArea = Module() - for (_pkgid, _mod) in Base.loaded_modules - if !(_pkgid.name in ("Main", "Core", "Base")) - @eval PrecompileStagingArea \$(Symbol(_mod)) = \$_mod + open(output, "w") do f + if header + println(f, HEADER) + println(f, """ + let + PrecompileStagingArea = Module() + for (_pkgid, _mod) in Base.loaded_modules + if !(_pkgid.name in ("Main", "Core", "Base")) + @eval PrecompileStagingArea \$(Symbol(_mod)) = \$_mod + end end + f = joinpath(@__DIR__, "precompile_local.jl") + isfile(f) && include(PrecompileStagingArea, f) + @eval PrecompileStagingArea begin + """) end - @eval PrecompileStagingArea begin""") for statement in sort(collect(precompile_statements)) isgpl = needs_USE_GPL_LIBS(statement) isgpl && print(f, "if Base.USE_GPL_LIBS\n ") println(f, statement) isgpl && println(f, "end") end - println(f, "end\nend") + if header + println(f, "end\nend") + end end if merge - "Merged $new_precompile_file into $old_precompile_file" + "Merged $new_precompile_file into $output" else - "Overwrite $old_precompile_file with $new_precompile_file" + "Overwrote $output with $new_precompile_file" end end -if length(ARGS) == 1 - fixup_precompile(joinpath(pwd(), ARGS[1])) -elseif length(ARGS) == 2 - @assert ARGS[1] == "--merge" - fixup_precompile(joinpath(pwd(), ARGS[2]); merge = true) -else - error("invalid arguments") +function runit() + output = joinpath(Sys.BINDIR, "..", "..", "base", "precompile.jl") + merge = false + keep_anonymous = false + header=true + for arg in ARGS[1:end-1] + if arg == "--merge" + merge = true + elseif arg == "--keep-anonymous" + keep_anonymous = true + elseif arg == "--no-header" + header = false + elseif startswith(arg, "--output") + output = split(arg, "=")[2] + else + error("unknown argument $arg") + end + end + fixup_precompile(joinpath(pwd(), ARGS[end]); merge=merge, keep_anonymous=keep_anonymous, header=header, output=output) end + +running_as_script = abspath(PROGRAM_FILE) == @__FILE__ +if running_as_script + runit() +end \ No newline at end of file diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl new file mode 100644 index 00000000000000..e94271b0b42ec1 --- /dev/null +++ b/contrib/generate_precompile.jl @@ -0,0 +1,40 @@ + +tmp = tempname() +if haskey(Base.loaded_modules, Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL")) + # Record precompile statements when starting a julia session with a repl + run(pipeline(`$(Base.julia_cmd()) --trace-compile=yes -e ' + @async while true + sleep(0.01) + isdefined(Base, :active_repl) && exit(0) + end' -i`; stderr = tmp)) + # Replay a REPL script + repl_replay = joinpath(@__DIR__, "precompile_replay.jl") + run(pipeline(`$(Base.julia_cmd()) --trace-compile=yes $repl_replay`; stderr=tmp, append=true)) +else + # No REPL, just record the startup + run(pipeline(`$(Base.julia_cmd()) --trace-compile=yes -e 'exit(0)'`; stderr=tmp)) +end + +# Replace the fake terminal with the real terminal and filter out everything we compiled in Main +precompiles = readlines(tmp) +new_precompiles = String[] +for line in precompiles + line = replace(line, "FakeTerminals.FakeTerminal" => "REPL.Terminals.TTYTerminal") + (occursin(r"Main.", line) || occursin(r"FakeTerminals.", line)) && continue + push!(new_precompiles, line) +end +write(tmp, join(new_precompiles, '\n')) + +# Only write the precompile in case it is different + +include("fixup_precompile.jl") +precompile_local = joinpath(@__DIR__, "..", "base/precompile_local.jl") +tmp2 = tempname() +isfile(precompile_local) && cp(precompile_local, tmp2) +fixup_precompile(tmp; merge=true, keep_anonymous=true, header=false, output=tmp2) +# Only update timestamp if different +if !isfile(precompile_local) || (isfile(precompile_local) && (read(tmp2, String) != read(precompile_local, String))) + println("Updatingf...") + mv(tmp2, precompile_local; force=true) +end +rm(tmp) \ No newline at end of file diff --git a/contrib/precompile_replay.jl b/contrib/precompile_replay.jl new file mode 100644 index 00000000000000..0b947384b27f13 --- /dev/null +++ b/contrib/precompile_replay.jl @@ -0,0 +1,53 @@ +import REPL +include(joinpath(Sys.STDLIB, "REPL", "test", "FakeTerminals.jl")) +import .FakeTerminals.FakeTerminal +include(joinpath(Sys.STDLIB, "REPL", "test", "fake_repl.jl")) + +const CTRL_C = '\x03' +const UP_ARROW = "\e[A" +const DOWN_ARROW = "\e[B" + +# TODO: Have a utility to generate this from a real REPL session? +precompile_script = """ +2+2 +println("Hello") +@time 1+1 +?reinterpret +;ls +using Ra\t$CTRL_C +\\alpha\t$CTRL_C +\e[200~paste here ;)\e[201~"$CTRL_C +$UP_ARROW$DOWN_ARROW +""" + +# Writing ^C to the repl will cause sigint, so let's not die on that +ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 0) + +fake_repl() do stdin_write, stdout_read, repl + repl.specialdisplay = REPL.REPLDisplay(repl) + repl.history_file = false + + repltask = @async begin + REPL.run_repl(repl) + end + + global inc = false + global b = Condition() + global c = Condition() + let cmd = "\"Hello REPL\"" + write(stdin_write, "Main.inc || wait(Main.b); r = $cmd; notify(Main.c); r\r") + end + inc = true + notify(b) + wait(c) + + write(stdin_write, precompile_script) + + s = readavailable(stdout_read) + + # Close REPL ^D + write(stdin_write, '\x04') + Base._wait(repltask) + + nothing +end \ No newline at end of file diff --git a/src/gf.c b/src/gf.c index 8a108087bee43a..3851807632dd6a 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1097,13 +1097,13 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype if (entry != NULL) { jl_method_t *m = entry->func.method; if (!jl_has_call_ambiguities((jl_value_t*)tt, m)) { -#ifdef TRACE_COMPILE - if (!jl_has_free_typevars((jl_value_t*)tt)) { - jl_printf(JL_STDERR, "precompile("); - jl_static_show(JL_STDERR, (jl_value_t*)tt); - jl_printf(JL_STDERR, ")\n"); + if (jl_options.trace_compile) { + if (!jl_has_free_typevars((jl_value_t*)tt)) { + jl_printf(JL_STDERR, "precompile("); + jl_static_show(JL_STDERR, (jl_value_t*)tt); + jl_printf(JL_STDERR, ")\n"); + } } -#endif if (!mt_cache) { intptr_t nspec = (mt == jl_type_type_mt ? m->nargs + 1 : mt->max_args + 2); jl_compilation_sig(tt, env, m, nspec, &newparams); diff --git a/src/jloptions.c b/src/jloptions.c index 147e0b8a28c7f2..9d2a30e2d0fa76 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -55,6 +55,11 @@ jl_options_t jl_options = { 0, // quiet 0, // method overwrite warning 1, // can_inline JL_OPTIONS_POLLY_ON, // polly +#ifdef TRACE_COMPILE + 1, // trace_compile +#else + 0, // trace_compile +#endif JL_OPTIONS_FAST_MATH_DEFAULT, 0, // worker NULL, // cookie @@ -159,6 +164,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_warn_overwrite, opt_inline, opt_polly, + opt_trace_compile, opt_math_mode, opt_worker, opt_bind_to, @@ -215,6 +221,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "warn-overwrite", required_argument, 0, opt_warn_overwrite }, { "inline", required_argument, 0, opt_inline }, { "polly", required_argument, 0, opt_polly }, + { "trace-compile", required_argument, 0, opt_trace_compile }, { "math-mode", required_argument, 0, opt_math_mode }, { "handle-signals", required_argument, 0, opt_handle_signals }, // hidden command line options @@ -567,6 +574,15 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_errorf("julia: invalid argument to --polly (%s)", optarg); } break; + case opt_trace_compile: + if (!strcmp(optarg,"yes")) + jl_options.trace_compile = 1; + else if (!strcmp(optarg,"no")) + jl_options.trace_compile = 0; + else { + jl_errorf("julia: invalid argument to --trace-compile (%s)", optarg); + } + break; case opt_math_mode: if (!strcmp(optarg,"ieee")) jl_options.fast_math = JL_OPTIONS_FAST_MATH_OFF; diff --git a/src/julia.h b/src/julia.h index e5b1d86e1251e0..24a99a4b143684 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1794,6 +1794,7 @@ typedef struct { int8_t warn_overwrite; int8_t can_inline; int8_t polly; + int8_t trace_compile; int8_t fast_math; int8_t worker; const char *cookie; diff --git a/stdlib/REPL/test/fake_repl.jl b/stdlib/REPL/test/fake_repl.jl new file mode 100644 index 00000000000000..ea6de10dbc1b20 --- /dev/null +++ b/stdlib/REPL/test/fake_repl.jl @@ -0,0 +1,49 @@ + +using Test + +function kill_timer(delay) + # Give ourselves a generous timer here, just to prevent + # this causing e.g. a CI hang when there's something unexpected in the output. + # This is really messy and leaves the process in an undefined state. + # the proper and correct way to do this in real code would be to destroy the + # IO handles: `close(stdout_read); close(stdin_write)` + test_task = current_task() + function kill_test(t) + # **DON'T COPY ME.** + # The correct way to handle timeouts is to close the handle: + # e.g. `close(stdout_read); close(stdin_write)` + schedule(test_task, "hard kill repl test"; error=true) + print(stderr, "WARNING: attempting hard kill of repl test after exceeding timeout\n") + end + return Timer(kill_test, delay) +end + +# REPL tests +function fake_repl(@nospecialize(f); options::REPL.Options=REPL.Options(confirm_exit=false)) + # Use pipes so we can easily do blocking reads + # In the future if we want we can add a test that the right object + # gets displayed by intercepting the display + input = Pipe() + output = Pipe() + err = Pipe() + Base.link_pipe!(input, reader_supports_async=true, writer_supports_async=true) + Base.link_pipe!(output, reader_supports_async=true, writer_supports_async=true) + Base.link_pipe!(err, reader_supports_async=true, writer_supports_async=true) + + repl = REPL.LineEditREPL(FakeTerminal(input.out, output.in, err.in), true) + repl.options = options + + hard_kill = kill_timer(900) # Your debugging session starts now. You have 15 minutes. Go. + f(input.in, output.out, repl) + t = @async begin + close(input.in) + close(output.in) + close(err.in) + end + @test read(err.out, String) == "" + #display(read(output.out, String)) + #print(read(output.out, String)) + Base._wait(t) + close(hard_kill) + nothing +end \ No newline at end of file diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 01a1d426a2b2ef..491ffc057a601f 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -15,53 +15,7 @@ include(joinpath(BASE_TEST_PATH, "testenv.jl")) include("FakeTerminals.jl") import .FakeTerminals.FakeTerminal - - -function kill_timer(delay) - # Give ourselves a generous timer here, just to prevent - # this causing e.g. a CI hang when there's something unexpected in the output. - # This is really messy and leaves the process in an undefined state. - # the proper and correct way to do this in real code would be to destroy the - # IO handles: `close(stdout_read); close(stdin_write)` - test_task = current_task() - function kill_test(t) - # **DON'T COPY ME.** - # The correct way to handle timeouts is to close the handle: - # e.g. `close(stdout_read); close(stdin_write)` - schedule(test_task, "hard kill repl test"; error=true) - print(stderr, "WARNING: attempting hard kill of repl test after exceeding timeout\n") - end - return Timer(kill_test, delay) -end - -# REPL tests -function fake_repl(@nospecialize(f); options::REPL.Options=REPL.Options(confirm_exit=false)) - # Use pipes so we can easily do blocking reads - # In the future if we want we can add a test that the right object - # gets displayed by intercepting the display - input = Pipe() - output = Pipe() - err = Pipe() - Base.link_pipe!(input, reader_supports_async=true, writer_supports_async=true) - Base.link_pipe!(output, reader_supports_async=true, writer_supports_async=true) - Base.link_pipe!(err, reader_supports_async=true, writer_supports_async=true) - - repl = REPL.LineEditREPL(FakeTerminal(input.out, output.in, err.in), true) - repl.options = options - - hard_kill = kill_timer(900) # Your debugging session starts now. You have 15 minutes. Go. - f(input.in, output.out, repl) - t = @async begin - close(input.in) - close(output.in) - close(err.in) - end - @test read(err.out, String) == "" - #display(read(output.out, String)) - Base._wait(t) - close(hard_kill) - nothing -end +include("fake_repl.jl") # Writing ^C to the repl will cause sigint, so let's not die on that ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 0)