From 69ac114faaabedb2c0fc5fa93bb38710999ea574 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 19 Feb 2024 13:11:08 +0100 Subject: [PATCH] add being able to pass a `CacheFlags()` to `isprecompiled` to verify a precompile file against a set of custom flags (#53332) --- base/loading.jl | 268 ++++++++++++++++++++------------------ src/jl_exported_funcs.inc | 1 + src/staticdata.c | 2 +- src/staticdata_utils.c | 26 ++-- test/precompile.jl | 13 +- 5 files changed, 168 insertions(+), 142 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 8bcaa4b2ab5b36..f4db43d80640b7 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1495,6 +1495,126 @@ end # End extensions + +struct CacheFlags + # OOICCDDP - see jl_cache_flags + use_pkgimages::Bool + debug_level::Int + check_bounds::Int + inline::Bool + opt_level::Int +end +function CacheFlags(f::UInt8) + use_pkgimages = Bool(f & 1) + debug_level = Int((f >> 1) & 3) + check_bounds = Int((f >> 3) & 3) + inline = Bool((f >> 5) & 1) + opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in statiddata_utils + CacheFlags(use_pkgimages, debug_level, check_bounds, inline, opt_level) +end +CacheFlags(f::Int) = CacheFlags(UInt8(f)) +CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ())) + +function _cacheflag_to_uint8(cf::CacheFlags)::UInt8 + f = UInt8(0) + f |= cf.use_pkgimages << 0 + f |= cf.debug_level << 1 + f |= cf.check_bounds << 3 + f |= cf.inline << 5 + f |= cf.opt_level << 6 + return f +end + +function show(io::IO, cf::CacheFlags) + print(io, "use_pkgimages = ", cf.use_pkgimages) + print(io, ", debug_level = ", cf.debug_level) + print(io, ", check_bounds = ", cf.check_bounds) + print(io, ", inline = ", cf.inline) + print(io, ", opt_level = ", cf.opt_level) +end + +struct ImageTarget + name::String + flags::Int32 + ext_features::String + features_en::Vector{UInt8} + features_dis::Vector{UInt8} +end + +function parse_image_target(io::IO) + flags = read(io, Int32) + nfeature = read(io, Int32) + feature_en = read(io, 4*nfeature) + feature_dis = read(io, 4*nfeature) + name_len = read(io, Int32) + name = String(read(io, name_len)) + ext_features_len = read(io, Int32) + ext_features = String(read(io, ext_features_len)) + ImageTarget(name, flags, ext_features, feature_en, feature_dis) +end + +function parse_image_targets(targets::Vector{UInt8}) + io = IOBuffer(targets) + ntargets = read(io, Int32) + targets = Vector{ImageTarget}(undef, ntargets) + for i in 1:ntargets + targets[i] = parse_image_target(io) + end + return targets +end + +function current_image_targets() + targets = @ccall jl_reflect_clone_targets()::Vector{UInt8} + return parse_image_targets(targets) +end + +struct FeatureName + name::Cstring + bit::UInt32 # bit index into a `uint32_t` array; + llvmver::UInt32 # 0 if it is available on the oldest LLVM version we support +end + +function feature_names() + fnames = Ref{Ptr{FeatureName}}() + nf = Ref{Csize_t}() + @ccall jl_reflect_feature_names(fnames::Ptr{Ptr{FeatureName}}, nf::Ptr{Csize_t})::Cvoid + if fnames[] == C_NULL + @assert nf[] == 0 + return Vector{FeatureName}(undef, 0) + end + Base.unsafe_wrap(Array, fnames[], nf[], own=false) +end + +function test_feature(features::Vector{UInt8}, feat::FeatureName) + bitidx = feat.bit + u8idx = div(bitidx, 8) + 1 + bit = bitidx % 8 + return (features[u8idx] & (1 << bit)) != 0 +end + +function show(io::IO, it::ImageTarget) + print(io, it.name) + if !isempty(it.ext_features) + print(io, ",", it.ext_features) + end + print(io, "; flags=", it.flags) + print(io, "; features_en=(") + first = true + for feat in feature_names() + if test_feature(it.features_en, feat) + name = Base.unsafe_string(feat.name) + if first + first = false + print(io, name) + else + print(io, ", ", name) + end + end + end + print(io, ")") + # Is feature_dis useful? +end + # should sync with the types of arguments of `stale_cachefile` const StaleCacheKey = Tuple{Base.PkgId, UInt128, String, String} @@ -1515,11 +1635,11 @@ function isprecompiled(pkg::PkgId; ignore_loaded::Bool=false, stale_cache::Dict{StaleCacheKey,Bool}=Dict{StaleCacheKey, Bool}(), cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg), - sourcepath::Union{String,Nothing}=Base.locate_package(pkg) - ) + sourcepath::Union{String,Nothing}=Base.locate_package(pkg), + flags::CacheFlags=CacheFlags()) isnothing(sourcepath) && error("Cannot locate source for $(repr(pkg))") for path_to_try in cachepaths - staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true) + staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true, requested_flags=flags) if staledeps === true continue end @@ -1532,7 +1652,7 @@ function isprecompiled(pkg::PkgId; modpaths = find_all_in_cache_path(modkey) for modpath_to_try in modpaths::Vector{String} stale_cache_key = (modkey, modbuild_id, modpath, modpath_to_try)::StaleCacheKey - if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded) === true, + if get!(() -> stale_cachefile(stale_cache_key...; ignore_loaded, requested_flags=flags) === true, stale_cache, stale_cache_key) continue end @@ -2535,8 +2655,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: if output_o !== nothing @debug "Generating object cache file for $(repr("text/plain", pkg))" cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) - opt_level = Base.JLOptions().opt_level - opts = `-O$(opt_level) --output-o $(output_o) --output-ji $(output) --output-incremental=yes` + opts = `--output-o $(output_o) --output-ji $(output) --output-incremental=yes` else @debug "Generating cache file for $(repr("text/plain", pkg))" cpu_target = nothing @@ -2546,10 +2665,11 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing) deps = deps_eltype * "[" * join(deps_strs, ",") * "]" trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : `` - io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) $(opts) + io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) + $(flags) + $(opts) --startup-file=no --history-file=no --warn-overwrite=yes --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no") - $flags $trace -`, "OPENBLAS_NUM_THREADS" => 1, @@ -3212,116 +3332,6 @@ function check_clone_targets(clone_targets) end end -struct CacheFlags - # OOICCDDP - see jl_cache_flags - use_pkgimages::Bool - debug_level::Int - check_bounds::Int - inline::Bool - opt_level::Int - - function CacheFlags(f::UInt8) - use_pkgimages = Bool(f & 1) - debug_level = Int((f >> 1) & 3) - check_bounds = Int((f >> 3) & 3) - inline = Bool((f >> 5) & 1) - opt_level = Int((f >> 6) & 3) # define OPT_LEVEL in statiddata_utils - new(use_pkgimages, debug_level, check_bounds, inline, opt_level) - end -end -CacheFlags(f::Int) = CacheFlags(UInt8(f)) -CacheFlags() = CacheFlags(ccall(:jl_cache_flags, UInt8, ())) - -function show(io::IO, cf::CacheFlags) - print(io, "use_pkgimages = ", cf.use_pkgimages) - print(io, ", debug_level = ", cf.debug_level) - print(io, ", check_bounds = ", cf.check_bounds) - print(io, ", inline = ", cf.inline) - print(io, ", opt_level = ", cf.opt_level) -end - -struct ImageTarget - name::String - flags::Int32 - ext_features::String - features_en::Vector{UInt8} - features_dis::Vector{UInt8} -end - -function parse_image_target(io::IO) - flags = read(io, Int32) - nfeature = read(io, Int32) - feature_en = read(io, 4*nfeature) - feature_dis = read(io, 4*nfeature) - name_len = read(io, Int32) - name = String(read(io, name_len)) - ext_features_len = read(io, Int32) - ext_features = String(read(io, ext_features_len)) - ImageTarget(name, flags, ext_features, feature_en, feature_dis) -end - -function parse_image_targets(targets::Vector{UInt8}) - io = IOBuffer(targets) - ntargets = read(io, Int32) - targets = Vector{ImageTarget}(undef, ntargets) - for i in 1:ntargets - targets[i] = parse_image_target(io) - end - return targets -end - -function current_image_targets() - targets = @ccall jl_reflect_clone_targets()::Vector{UInt8} - return parse_image_targets(targets) -end - -struct FeatureName - name::Cstring - bit::UInt32 # bit index into a `uint32_t` array; - llvmver::UInt32 # 0 if it is available on the oldest LLVM version we support -end - -function feature_names() - fnames = Ref{Ptr{FeatureName}}() - nf = Ref{Csize_t}() - @ccall jl_reflect_feature_names(fnames::Ptr{Ptr{FeatureName}}, nf::Ptr{Csize_t})::Cvoid - if fnames[] == C_NULL - @assert nf[] == 0 - return Vector{FeatureName}(undef, 0) - end - Base.unsafe_wrap(Array, fnames[], nf[], own=false) -end - -function test_feature(features::Vector{UInt8}, feat::FeatureName) - bitidx = feat.bit - u8idx = div(bitidx, 8) + 1 - bit = bitidx % 8 - return (features[u8idx] & (1 << bit)) != 0 -end - -function show(io::IO, it::ImageTarget) - print(io, it.name) - if !isempty(it.ext_features) - print(io, ",", it.ext_features) - end - print(io, "; flags=", it.flags) - print(io, "; features_en=(") - first = true - for feat in feature_names() - if test_feature(it.features_en, feat) - name = Base.unsafe_string(feat.name) - if first - first = false - print(io, name) - else - print(io, ", ", name) - end - end - end - print(io, ")") - # Is feature_dis useful? -end - # Set by FileWatching.__init__() global mkpidlock_hook global trymkpidlock_hook @@ -3377,11 +3387,11 @@ list_reasons(::Nothing) = "" # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey # otherwise returns the list of dependencies to also check -@constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false, reasons=nothing) - return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded, reasons) +@constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons=nothing) + return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded, requested_flags, reasons) end @constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String; - ignore_loaded::Bool = false, reasons::Union{Dict{String,Int},Nothing}=nothing) + ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons::Union{Dict{String,Int},Nothing}=nothing) io = open(cachefile, "r") try checksum = isvalid_cache_header(io) @@ -3390,15 +3400,15 @@ end record_reason(reasons, "invalid header") return true # invalid cache file end - modules, (includes, _, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags = parse_cache_header(io, cachefile) + modules, (includes, _, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, actual_flags = parse_cache_header(io, cachefile) if isempty(modules) return true # ignore empty file end - if ccall(:jl_match_cache_flags, UInt8, (UInt8,), flags) == 0 + if @ccall(jl_match_cache_flags(_cacheflag_to_uint8(requested_flags)::UInt8, actual_flags::UInt8)::UInt8) == 0 @debug """ Rejecting cache file $cachefile for $modkey since the flags are mismatched - current session: $(CacheFlags()) - cache file: $(CacheFlags(flags)) + requested flags: $(requested_flags) [$(_cacheflag_to_uint8(requested_flags))] + cache file: $(CacheFlags(actual_flags)) [$actual_flags] """ record_reason(reasons, "mismatched flags") return true @@ -3672,7 +3682,7 @@ function precompile(@nospecialize(argt::Type), m::Method) return precompile(mi) end -precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing)) -precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) -precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), IO, IO, Cmd)) -precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), IO, IO, Cmd)) +@assert precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing)) +@assert precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) +@assert precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), Cmd, IO, IO)) +@assert precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), Cmd, IO, IO)) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 6463d30cbeae9e..4498f3132d1bc7 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -386,6 +386,7 @@ XX(jl_raise_debugger) \ XX(jl_readuntil) \ XX(jl_cache_flags) \ + XX(jl_match_cache_flags_current) \ XX(jl_match_cache_flags) \ XX(jl_read_verify_header) \ XX(jl_realloc) \ diff --git a/src/staticdata.c b/src/staticdata.c index 5172eb80c8b4a3..67f5620f7fcef2 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3566,7 +3566,7 @@ static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_ "Precompile file header verification checks failed."); } uint8_t flags = read_uint8(f); - if (pkgimage && !jl_match_cache_flags(flags)) { + if (pkgimage && !jl_match_cache_flags_current(flags)) { return jl_get_exceptionf(jl_errorexception_type, "Pkgimage flags mismatch"); } if (!pkgimage) { diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index fc7a59d9d0bf9a..9542e541826440 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -603,31 +603,35 @@ JL_DLLEXPORT uint8_t jl_cache_flags(void) return flags; } -JL_DLLEXPORT uint8_t jl_match_cache_flags(uint8_t flags) + +JL_DLLEXPORT uint8_t jl_match_cache_flags(uint8_t requested_flags, uint8_t actual_flags) { - // 1. Check which flags are relevant - uint8_t current_flags = jl_cache_flags(); - uint8_t supports_pkgimage = (current_flags & 1); - uint8_t is_pkgimage = (flags & 1); + uint8_t supports_pkgimage = (requested_flags & 1); + uint8_t is_pkgimage = (actual_flags & 1); // For .ji packages ignore other flags if (!supports_pkgimage && !is_pkgimage) { return 1; } - // If package images are optional, ignore that bit (it will be unset in current_flags) + // If package images are optional, ignore that bit (it will be unset in requested_flags) if (jl_options.use_pkgimages == JL_OPTIONS_USE_PKGIMAGES_EXISTING) { - flags &= ~1; + actual_flags &= ~1; } // 2. Check all flags, execept opt level must be exact uint8_t mask = (1 << OPT_LEVEL)-1; - if ((flags & mask) != (current_flags & mask)) + if ((actual_flags & mask) != (requested_flags & mask)) return 0; // 3. allow for higher optimization flags in cache - flags >>= OPT_LEVEL; - current_flags >>= OPT_LEVEL; - return flags >= current_flags; + actual_flags >>= OPT_LEVEL; + requested_flags >>= OPT_LEVEL; + return actual_flags >= requested_flags; +} + +JL_DLLEXPORT uint8_t jl_match_cache_flags_current(uint8_t flags) +{ + return jl_match_cache_flags(jl_cache_flags(), flags); } // return char* from String field in Base.GIT_VERSION_INFO diff --git a/test/precompile.jl b/test/precompile.jl index 015cf49f228ac8..a68d8936d1ed18 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -2072,8 +2072,16 @@ precompile_test_harness("Test flags") do load_path module TestFlags end """) + + current_flags = Base.CacheFlags() + modified_flags = Base.CacheFlags( + current_flags.use_pkgimages, + current_flags.debug_level, + 2, + current_flags.inline, + 3 + ) ji, ofile = Base.compilecache(Base.PkgId("TestFlags"); flags=`--check-bounds=no -O3`) - @show ji, ofile open(ji, "r") do io Base.isvalid_cache_header(io) _, _, _, _, _, _, _, flags = Base.parse_cache_header(io, ji) @@ -2081,6 +2089,9 @@ precompile_test_harness("Test flags") do load_path @test cacheflags.check_bounds == 2 @test cacheflags.opt_level == 3 end + id = Base.identify_package("TestFlags") + @test Base.isprecompiled(id, ;flags=modified_flags) + @test !Base.isprecompiled(id, ;flags=current_flags) end empty!(Base.DEPOT_PATH)