-
-
Notifications
You must be signed in to change notification settings - Fork 5.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
generate precompile as part of build process (#28118)
- Loading branch information
1 parent
748ee1b
commit b43e7ad
Showing
15 changed files
with
291 additions
and
878 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| Base.__init__() | ||
|
|
||
| # Prevent this from being put into the Main namespace | ||
| let | ||
| M = Module() | ||
| @eval M begin | ||
| julia_cmd() = (julia = joinpath(Sys.BINDIR, Base.julia_exename()); `$julia`) | ||
|
|
||
| function generate_precompilable_package(path, pkgname) | ||
| mkpath(joinpath(path, pkgname, "src")) | ||
| write(joinpath(path, pkgname, "src", "$pkgname.jl"), | ||
| """ | ||
| __precompile__() | ||
| module $pkgname | ||
| end | ||
| """) | ||
| end | ||
|
|
||
| function generate_precompile_statements() | ||
| t = time() | ||
| println("Generating precompile statements...") | ||
| println("──────────────────────────────────────") | ||
|
|
||
| # Reset code loading vars | ||
| Base.ACTIVE_PROJECT[] = nothing | ||
| Base.HOME_PROJECT[] = nothing | ||
| empty!(LOAD_PATH) | ||
| empty!(DEPOT_PATH) | ||
| push!(LOAD_PATH, "@stdlib") | ||
|
|
||
| tmpd = mktempdir() | ||
| push!(DEPOT_PATH, tmpd) | ||
| push!(LOAD_PATH, tmpd) | ||
| pkgname = "__PackagePrecompilationStatementModule" | ||
| generate_precompilable_package(tmpd, pkgname) | ||
| @eval using __PackagePrecompilationStatementModule | ||
| pop!(LOAD_PATH) | ||
| pop!(DEPOT_PATH) | ||
| rm(tmpd; recursive=true) | ||
|
|
||
| if isempty(ARGS) | ||
| sysimg = joinpath(dirname(Sys.BINDIR), "lib", "julia", "sys.ji") | ||
| else | ||
| sysimg = ARGS[1] | ||
| end | ||
|
|
||
| output = Pipe() | ||
| task = @async read(output, String) | ||
| have_repl = haskey(Base.loaded_modules, | ||
| Base.PkgId(Base.UUID("3fa0cd96-eef1-5676-8a61-b3b8758bbffb"), "REPL")) | ||
| if have_repl | ||
| # Have a REPL, run the repl replayer and an interactive session that we immidiately kill | ||
| setup = """ | ||
| include($(repr(joinpath(@__DIR__, "precompile_replay.jl")))) | ||
| @async while true | ||
| sleep(0.01) | ||
| if isdefined(Base, :active_repl) | ||
| exit(0) | ||
| end | ||
| end | ||
| """ | ||
| # Do not redirect stdin unless it is to a tty, because that changes code paths | ||
| ok = try | ||
| run(pipeline(`$(julia_cmd()) --sysimage $sysimg --trace-compile=yes -O0 | ||
| --startup-file=no --q -e $setup -i`; stderr=output)) | ||
| true | ||
| catch | ||
| false | ||
| end | ||
| else | ||
| # No REPL, just record the startup | ||
| ok = try | ||
| run(pipeline(`$(julia_cmd()) --sysimage $sysimg --trace-compile=yes -O0 | ||
| --startup-file=no --q -e0`; stderr=output)) | ||
| true | ||
| catch | ||
| false | ||
| end | ||
| end | ||
|
|
||
| # Replace the FakeTerminal with a TTYTerminal and filter out everything we compiled in Main | ||
| close(output) | ||
| s = fetch(task) | ||
| if !ok | ||
| error("precompilation process failed, stderr is:\n$s") | ||
| end | ||
| new_precompiles = Set{String}() | ||
| for statement in split(s, '\n') | ||
| startswith(statement, "precompile(Tuple{") || continue | ||
| statement = replace(statement, "FakeTerminals.FakeTerminal" => "REPL.Terminals.TTYTerminal") | ||
| (occursin(r"Main.", statement) || occursin(r"FakeTerminals.", statement)) && continue | ||
| # AppVeyor CI emits a single faulty precompile statement: | ||
| # precompile(Tuple{getfield(precompile(Tuple{typeof(Base.uvfinalize), Base.PipeEndpoint}) | ||
| # which lacks the correct closing brackets. | ||
| # Filter out such lines here. | ||
| for (l, r) in ('(' => ')', '{' => '}') | ||
| if count(isequal(l), statement) != count(isequal(r), statement) | ||
| continue | ||
| end | ||
| push!(new_precompiles, statement) | ||
| end | ||
| end | ||
|
|
||
| tmp = tempname() | ||
| write(tmp, join(sort(collect(new_precompiles)), '\n')) | ||
| # Load the precompile statements | ||
| PrecompileStagingArea = Module() | ||
| for (_pkgid, _mod) in Base.loaded_modules | ||
| if !(_pkgid.name in ("Main", "Core", "Base")) | ||
| eval(PrecompileStagingArea, :($(Symbol(_mod)) = $_mod)) | ||
| end | ||
| end | ||
| Base.include(PrecompileStagingArea, tmp) | ||
| rm(tmp) | ||
|
|
||
| # Add a few manual things, run `julia` with `--trace-compile` to find these. | ||
| if have_repl | ||
| @eval PrecompileStagingArea begin | ||
| # Could startup with REPL banner instead but it is a bit visually noisy, | ||
| # so just precompile it manualally here. | ||
| precompile(Tuple{typeof(Base.banner), REPL.Terminals.TTYTerminal}) | ||
|
|
||
| # This is probablably important for precompilation (0.2s precompile time) | ||
| # but doesn't seem to get caught in the script above | ||
| precompile(Tuple{typeof(Base.create_expr_cache), String, String, Array{Base.Pair{Base.PkgId, UInt64}, 1}, Base.UUID}) | ||
| end | ||
| end | ||
|
|
||
| println("──────────────────────────────────────") | ||
| print("$(length(new_precompiles)) precompile statements generated in "), Base.time_print((time() - t) * 10^9) | ||
| println() | ||
| return | ||
| end | ||
|
|
||
| generate_precompile_statements() | ||
|
|
||
| end # @eval | ||
| end # let |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| Base.__init__() | ||
|
|
||
| import REPL | ||
| include(joinpath(Sys.STDLIB, "REPL", "test", "FakeTerminals.jl")) | ||
| import .FakeTerminals.FakeTerminal | ||
| include(joinpath(Sys.STDLIB, "REPL", "test", "fake_repl.jl")) | ||
|
|
||
| const CTRL_C = '\x03' | ||
| const UP_ARROW = "\e[A" | ||
| const DOWN_ARROW = "\e[B" | ||
|
|
||
| # TODO: Have a utility to generate this from a real REPL session? | ||
| precompile_script = """ | ||
| 2+2 | ||
| println("Hello World") | ||
| @time 1+1 | ||
| ?reinterpret | ||
| ;pwd | ||
| using Ra\t$CTRL_C | ||
| \\alpha\t$CTRL_C | ||
| \e[200~paste here ;)\e[201~"$CTRL_C | ||
| $UP_ARROW$DOWN_ARROW | ||
| 123\b\b\b$CTRL_C | ||
| \b\b | ||
| f(x) = x03 | ||
| f(1,2) | ||
| [][1] | ||
| cd("complet_path\t\t$CTRL_C | ||
| """ | ||
|
|
||
|
|
||
| function run_repl() | ||
| # Writing ^C to the repl will cause sigint, so let's not die on that | ||
| ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 0) | ||
|
|
||
| fake_repl() do stdin_write, stdout_read, repl | ||
| repl.specialdisplay = REPL.REPLDisplay(repl) | ||
| repl.history_file = false | ||
|
|
||
| repltask = @async begin | ||
| REPL.run_repl(repl) | ||
| end | ||
|
|
||
| global inc = false | ||
| global b = Condition() | ||
| global c = Condition() | ||
| mod = @__MODULE__ | ||
| let cmd = "\"Hello REPL\"" | ||
| write(stdin_write, "$mod.inc || wait($mod.b); r = $cmd; notify($mod.c); r\r") | ||
| end | ||
| inc = true | ||
| notify(b) | ||
| wait(c) | ||
|
|
||
| write(stdin_write, precompile_script) | ||
|
|
||
| s = readavailable(stdout_read) | ||
|
|
||
| # Close REPL ^D | ||
| write(stdin_write, '\x04') | ||
| Base._wait(repltask) | ||
|
|
||
| nothing | ||
| end | ||
| ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 1) | ||
| end | ||
|
|
||
| run_repl() | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.