From 1f5e0a29e470b11cb02f5263a4952054d45992e5 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 9 Apr 2023 17:27:53 +0200 Subject: [PATCH] move out Distributed from the sysimage (#49258) --- base/sysimg.jl | 1 - contrib/generate_precompile.jl | 22 --------------------- pkgimage.mk | 2 +- stdlib/Distributed/src/Distributed.jl | 1 + stdlib/Distributed/src/precompile.jl | 14 +++++++++++++ stdlib/Distributed/test/distributed_exec.jl | 4 ++-- test/precompile.jl | 2 +- 7 files changed, 19 insertions(+), 27 deletions(-) create mode 100644 stdlib/Distributed/src/precompile.jl diff --git a/base/sysimg.jl b/base/sysimg.jl index ca1a4e74f7417..b0eeffa5757ba 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -56,7 +56,6 @@ let # 2-depth packages :Dates, - :Distributed, :Future, :InteractiveUtils, :LibGit2, diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index f28a0fcd3974f..f756e0bfb8fee 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -129,28 +129,6 @@ if have_repl """ end -Distributed = get(Base.loaded_modules, - Base.PkgId(Base.UUID("8ba89e20-285c-5b6f-9357-94700520ee1b"), "Distributed"), - nothing) -if Distributed !== nothing - hardcoded_precompile_statements *= """ - precompile(Tuple{typeof(Distributed.remotecall),Function,Int,Module,Vararg{Any, 100}}) - precompile(Tuple{typeof(Distributed.procs)}) - precompile(Tuple{typeof(Distributed.finalize_ref), Distributed.Future}) - """ -# This is disabled because it doesn't give much benefit -# and the code in Distributed is poorly typed causing many invalidations -#= - precompile_script *= """ - using Distributed - addprocs(2) - pmap(x->iseven(x) ? 1 : 0, 1:4) - @distributed (+) for i = 1:100 Int(rand(Bool)) end - """ -=# -end - - Artifacts = get(Base.loaded_modules, Base.PkgId(Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts"), nothing) diff --git a/pkgimage.mk b/pkgimage.mk index 554bcd5587abe..caf30a91c1d18 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -93,7 +93,7 @@ $(eval $(call pkgimg_builder,LibSSH2_jll,Artifacts Libdl MbedTLS_jll)) $(eval $(call pkgimg_builder,MPFR_jll,Artifacts Libdl GMP_jll)) $(eval $(call sysimg_builder,LinearAlgebra,Libdl libblastrampoline_jll OpenBLAS_jll)) $(eval $(call sysimg_builder,Dates,Printf)) -$(eval $(call sysimg_builder,Distributed,Random Serialization Sockets)) +$(eval $(call pkgimg_builder,Distributed,Random Serialization Sockets)) $(eval $(call sysimg_builder,Future,Random)) $(eval $(call sysimg_builder,InteractiveUtils,Markdown)) $(eval $(call sysimg_builder,LibGit2,NetworkOptions Printf SHA Base64)) diff --git a/stdlib/Distributed/src/Distributed.jl b/stdlib/Distributed/src/Distributed.jl index 65bb6224a7bcd..28933e32e8bb8 100644 --- a/stdlib/Distributed/src/Distributed.jl +++ b/stdlib/Distributed/src/Distributed.jl @@ -107,6 +107,7 @@ include("macros.jl") # @spawn and friends include("workerpool.jl") include("pmap.jl") include("managers.jl") # LocalManager and SSHManager +include("precompile.jl") function __init__() init_parallel() diff --git a/stdlib/Distributed/src/precompile.jl b/stdlib/Distributed/src/precompile.jl new file mode 100644 index 0000000000000..87380f627db7a --- /dev/null +++ b/stdlib/Distributed/src/precompile.jl @@ -0,0 +1,14 @@ +precompile(Tuple{typeof(Distributed.remotecall),Function,Int,Module,Vararg{Any, 100}}) +precompile(Tuple{typeof(Distributed.procs)}) +precompile(Tuple{typeof(Distributed.finalize_ref), Distributed.Future}) +# This is disabled because it doesn't give much benefit +# and the code in Distributed is poorly typed causing many invalidations +# TODO: Maybe reenable now that Distributed is not in sysimage. +#= + precompile_script *= """ + using Distributed + addprocs(2) + pmap(x->iseven(x) ? 1 : 0, 1:4) + @distributed (+) for i = 1:100 Int(rand(Bool)) end + """ +=# diff --git a/stdlib/Distributed/test/distributed_exec.jl b/stdlib/Distributed/test/distributed_exec.jl index 16d1e4b100bf3..44929a762b2c5 100644 --- a/stdlib/Distributed/test/distributed_exec.jl +++ b/stdlib/Distributed/test/distributed_exec.jl @@ -1874,7 +1874,7 @@ let julia = `$(Base.julia_cmd()) --startup-file=no`; mktempdir() do tmp using Distributed project = mktempdir() env = Dict( - "JULIA_LOAD_PATH" => LOAD_PATH[1], + "JULIA_LOAD_PATH" => string(LOAD_PATH[1], $(repr(pathsep)), "@stdlib"), "JULIA_DEPOT_PATH" => DEPOT_PATH[1], "TMPDIR" => ENV["TMPDIR"], ) @@ -1884,7 +1884,7 @@ let julia = `$(Base.julia_cmd()) --startup-file=no`; mktempdir() do tmp """ * setupcode * """ for w in workers() @test remotecall_fetch(depot_path, w) == [DEPOT_PATH[1]] - @test remotecall_fetch(load_path, w) == [LOAD_PATH[1]] + @test remotecall_fetch(load_path, w) == [LOAD_PATH[1], "@stdlib"] @test remotecall_fetch(active_project, w) == project @test remotecall_fetch(Base.active_project, w) == joinpath(project, "Project.toml") end diff --git a/test/precompile.jl b/test/precompile.jl index 48437fb04eca2..37498068fd39c 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -395,7 +395,7 @@ precompile_test_harness(false) do dir Base.PkgId(m) => Base.module_build_id(m) end for s in [:ArgTools, :Artifacts, :Base64, :CompilerSupportLibraries_jll, :CRC32c, :Dates, - :Distributed, :Downloads, :FileWatching, :Future, :InteractiveUtils, :libblastrampoline_jll, + :Downloads, :FileWatching, :Future, :InteractiveUtils, :libblastrampoline_jll, :LibCURL, :LibCURL_jll, :LibGit2, :Libdl, :LinearAlgebra, :Logging, :Markdown, :Mmap, :MozillaCACerts_jll, :NetworkOptions, :OpenBLAS_jll, :Pkg, :Printf, :p7zip_jll, :REPL, :Random, :SHA, :Serialization, :Sockets,