Skip to content

Commit

Permalink
generate precompile as part of build process (#28118)
Browse files Browse the repository at this point in the history
  • Loading branch information
KristofferC authored Jul 27, 2018
1 parent 748ee1b commit b43e7ad
Show file tree
Hide file tree
Showing 15 changed files with 291 additions and 878 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ define sysimg_builder
$$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_private_libdir)/sys.ji
@$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \
if ! $$(call spawn,$3) $2 -C "$$(JULIA_CPU_TARGET)" --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \
--startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) -e nothing; then \
--startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $$(call cygpath_w,$$<); then \
echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \
false; \
fi )
Expand Down
1 change: 1 addition & 0 deletions base/options.jl
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ struct JLOptions
warn_overwrite::Int8
can_inline::Int8
polly::Int8
trace_compile::Int8
fast_math::Int8
worker::Int8
cookie::Ptr{UInt8}
Expand Down
746 changes: 0 additions & 746 deletions base/precompile.jl

This file was deleted.

7 changes: 2 additions & 5 deletions base/sysimg.jl
Original file line number Diff line number Diff line change
Expand Up @@ -522,9 +522,9 @@ let
:SuiteSparse,
:Distributed,
:SharedArrays,
:Pkg,
:Test,
:REPL,
:Pkg,
:Statistics,
]

Expand Down Expand Up @@ -922,11 +922,9 @@ Base.init_load_path() # want to be able to find external packages in userimg.jl

let
tot_time_userimg = @elapsed (Base.isfile("userimg.jl") && Base.include(Main, "userimg.jl"))
tot_time_precompile = Base.is_primary_base_module ? (@elapsed Base.include(Base, "precompile.jl")) : 0.0


tot_time_base = (Base.end_base_include - Base.start_base_include) * 10.0^(-9)
tot_time = tot_time_base + Base.tot_time_stdlib[] + tot_time_userimg + tot_time_precompile
tot_time = tot_time_base + Base.tot_time_stdlib[] + tot_time_userimg

println("Sysimage built. Summary:")
print("Total ─────── "); Base.time_print(tot_time * 10^9); print(" \n");
Expand All @@ -935,7 +933,6 @@ print("Stdlibs: ──── "); Base.time_print(Base.tot_time_stdlib[] * 10^9);
if isfile("userimg.jl")
print("Userimg: ──── "); Base.time_print(tot_time_userimg * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_userimg / tot_time) * 100); println("%")
end
print("Precompile: ─ "); Base.time_print(tot_time_precompile * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_precompile / tot_time) * 100); println("%")
end

empty!(LOAD_PATH)
Expand Down
1 change: 0 additions & 1 deletion contrib/add_license_to_files.jl
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ const excludedirs = [

const skipfiles = [
"../contrib/add_license_to_files.jl",
"../contrib/fixup_precompile.jl",
# files to check - already copyright
# see: https://github.com/JuliaLang/julia/pull/11073#issuecomment-98099389
"../base/special/trig.jl",
Expand Down
70 changes: 0 additions & 70 deletions contrib/fixup_precompile.jl

This file was deleted.

138 changes: 138 additions & 0 deletions contrib/generate_precompile.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
Base.__init__()

# Prevent this from being put into the Main namespace
let
M = Module()
@eval M begin
julia_cmd() = (julia = joinpath(Sys.BINDIR, Base.julia_exename()); `$julia`)

function generate_precompilable_package(path, pkgname)
mkpath(joinpath(path, pkgname, "src"))
write(joinpath(path, pkgname, "src", "$pkgname.jl"),
"""
__precompile__()
module $pkgname
end
""")
end

function generate_precompile_statements()
t = time()
println("Generating precompile statements...")
println("──────────────────────────────────────")

# Reset code loading vars
Base.ACTIVE_PROJECT[] = nothing
Base.HOME_PROJECT[] = nothing
empty!(LOAD_PATH)
empty!(DEPOT_PATH)
push!(LOAD_PATH, "@stdlib")

tmpd = mktempdir()
push!(DEPOT_PATH, tmpd)
push!(LOAD_PATH, tmpd)
pkgname = "__PackagePrecompilationStatementModule"
generate_precompilable_package(tmpd, pkgname)
@eval using __PackagePrecompilationStatementModule
pop!(LOAD_PATH)
pop!(DEPOT_PATH)
rm(tmpd; recursive=true)

if isempty(ARGS)
sysimg = joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys.ji")
else
sysimg = ARGS[1]
end

output = Pipe()
task = @async read(output, String)
have_repl = haskey(Base.loaded_modules,
Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL"))
if have_repl
# Have a REPL, run the repl replayer and an interactive session that we immidiately kill
setup = """
include($(repr(joinpath(@__DIR__, "precompile_replay.jl"))))
@async while true
sleep(0.01)
if isdefined(Base, :active_repl)
exit(0)
end
end
"""
# Do not redirect stdin unless it is to a tty, because that changes code paths
ok = try
run(pipeline(`$(julia_cmd()) --sysimage $sysimg --trace-compile=yes -O0
--startup-file=no --q -e $setup -i`; stderr=output))
true
catch
false
end
else
# No REPL, just record the startup
ok = try
run(pipeline(`$(julia_cmd()) --sysimage $sysimg --trace-compile=yes -O0
--startup-file=no --q -e0`; stderr=output))
true
catch
false
end
end

# Replace the FakeTerminal with a TTYTerminal and filter out everything we compiled in Main
close(output)
s = fetch(task)
if !ok
error("precompilation process failed, stderr is:\n$s")
end
new_precompiles = Set{String}()
for statement in split(s, '\n')
startswith(statement, "precompile(Tuple{") || continue
statement = replace(statement, "FakeTerminals.FakeTerminal" => "REPL.Terminals.TTYTerminal")
(occursin(r"Main.", statement) || occursin(r"FakeTerminals.", statement)) && continue
# AppVeyor CI emits a single faulty precompile statement:
# precompile(Tuple{getfield(precompile(Tuple{typeof(Base.uvfinalize), Base.PipeEndpoint})
# which lacks the correct closing brackets.
# Filter out such lines here.
for (l, r) in ('(' => ')', '{' => '}')
if count(isequal(l), statement) != count(isequal(r), statement)
continue
end
push!(new_precompiles, statement)
end
end

tmp = tempname()
write(tmp, join(sort(collect(new_precompiles)), '\n'))
# Load the precompile statements
PrecompileStagingArea = Module()
for (_pkgid, _mod) in Base.loaded_modules
if !(_pkgid.name in ("Main", "Core", "Base"))
eval(PrecompileStagingArea, :($(Symbol(_mod)) = $_mod))
end
end
Base.include(PrecompileStagingArea, tmp)
rm(tmp)

# Add a few manual things, run `julia` with `--trace-compile` to find these.
if have_repl
@eval PrecompileStagingArea begin
# Could startup with REPL banner instead but it is a bit visually noisy,
# so just precompile it manualally here.
precompile(Tuple{typeof(Base.banner), REPL.Terminals.TTYTerminal})

# This is probablably important for precompilation (0.2s precompile time)
# but doesn't seem to get caught in the script above
precompile(Tuple{typeof(Base.create_expr_cache), String, String, Array{Base.Pair{Base.PkgId, UInt64}, 1}, Base.UUID})
end
end

println("──────────────────────────────────────")
print("$(length(new_precompiles)) precompile statements generated in "), Base.time_print((time() - t) * 10^9)
println()
return
end

generate_precompile_statements()

end # @eval
end # let
69 changes: 69 additions & 0 deletions contrib/precompile_replay.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
Base.__init__()

import REPL
include(joinpath(Sys.STDLIB, "REPL", "test", "FakeTerminals.jl"))
import .FakeTerminals.FakeTerminal
include(joinpath(Sys.STDLIB, "REPL", "test", "fake_repl.jl"))

const CTRL_C = '\x03'
const UP_ARROW = "\e[A"
const DOWN_ARROW = "\e[B"

# TODO: Have a utility to generate this from a real REPL session?
precompile_script = """
2+2
println("Hello World")
@time 1+1
?reinterpret
;pwd
using Ra\t$CTRL_C
\\alpha\t$CTRL_C
\e[200~paste here ;)\e[201~"$CTRL_C
$UP_ARROW$DOWN_ARROW
123\b\b\b$CTRL_C
\b\b
f(x) = x03
f(1,2)
[][1]
cd("complet_path\t\t$CTRL_C
"""


function run_repl()
# Writing ^C to the repl will cause sigint, so let's not die on that
ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 0)

fake_repl() do stdin_write, stdout_read, repl
repl.specialdisplay = REPL.REPLDisplay(repl)
repl.history_file = false

repltask = @async begin
REPL.run_repl(repl)
end

global inc = false
global b = Condition()
global c = Condition()
mod = @__MODULE__
let cmd = "\"Hello REPL\""
write(stdin_write, "$mod.inc || wait($mod.b); r = $cmd; notify($mod.c); r\r")
end
inc = true
notify(b)
wait(c)

write(stdin_write, precompile_script)

s = readavailable(stdout_read)

# Close REPL ^D
write(stdin_write, '\x04')
Base._wait(repltask)

nothing
end
ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 1)
end

run_repl()

12 changes: 6 additions & 6 deletions src/gf.c
Original file line number Diff line number Diff line change
Expand Up @@ -1088,13 +1088,13 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt, jl_datatype
if (entry != NULL) {
jl_method_t *m = entry->func.method;
if (!jl_has_call_ambiguities((jl_value_t*)tt, m)) {
#ifdef TRACE_COMPILE
if (!jl_has_free_typevars((jl_value_t*)tt)) {
jl_printf(JL_STDERR, "precompile(");
jl_static_show(JL_STDERR, (jl_value_t*)tt);
jl_printf(JL_STDERR, ")\n");
if (jl_options.trace_compile) {
if (!jl_has_free_typevars((jl_value_t*)tt)) {
jl_printf(JL_STDERR, "precompile(");
jl_static_show(JL_STDERR, (jl_value_t*)tt);
jl_printf(JL_STDERR, ")\n");
}
}
#endif
if (!mt_cache) {
intptr_t nspec = (mt == jl_type_type_mt ? m->nargs + 1 : mt->max_args + 2);
jl_compilation_sig(tt, env, m, nspec, &newparams);
Expand Down
Loading

0 comments on commit b43e7ad

Please sign in to comment.