From 5b0d49e3890a459cac0e284146de4e7931addd34 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sun, 3 Mar 2024 16:27:41 -0500 Subject: [PATCH] Adjust to JuliaLang/julia#53219 (#547) Co-authored-by: Cody Tapscott Co-authored-by: Shuhei Kadowaki --- Project.toml | 4 +- TypedSyntax/src/node.jl | 102 +++++++++++++++++++++++---------- TypedSyntax/test/exhaustive.jl | 2 +- TypedSyntax/test/runtests.jl | 40 ++++++------- src/codeview.jl | 1 - src/interpreter.jl | 29 +++++++++- src/reflection.jl | 7 +-- test/test_Cthulhu.jl | 6 +- 8 files changed, 125 insertions(+), 66 deletions(-) diff --git a/Project.toml b/Project.toml index e12f22b3..1ed353ea 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Cthulhu" uuid = "f68482b8-f384-11e8-15f7-abe071a5a75f" authors = ["Valentin Churavy and contributors"] -version = "2.11.1" +version = "2.12.0" [deps] CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" @@ -24,7 +24,7 @@ JuliaSyntax = "0.4" PrecompileTools = "1" Preferences = "1" REPL = "1.9" -TypedSyntax = "1.2.2" +TypedSyntax = "1.3.0" UUIDs = "1.9" Unicode = "1.9" WidthLimitedIO = "1" diff --git a/TypedSyntax/src/node.jl b/TypedSyntax/src/node.jl index 0756e5aa..60c7050d 100644 --- a/TypedSyntax/src/node.jl +++ b/TypedSyntax/src/node.jl @@ -24,22 +24,23 @@ const no_default_value = NoDefaultValue() # These are TypedSyntaxNode constructor helpers # Call these directly if you want both the TypedSyntaxNode and the `mappings` list, # where `mappings[i]` corresponds to the list of nodes matching `(src::CodeInfo).code[i]`. -function tsn_and_mappings(@nospecialize(f), @nospecialize(t); kwargs...) - m = which(f, t) - src, rt = getsrc(f, t) - tsn_and_mappings(m, src, rt; kwargs...) +function tsn_and_mappings(@nospecialize(f), @nospecialize(tt=Base.default_tt(f)); kwargs...) + inferred_result = get_inferred_result(f, tt) + return tsn_and_mappings(inferred_result.mi, inferred_result.src, inferred_result.rt; kwargs...) end -function tsn_and_mappings(m::Method, src::CodeInfo, @nospecialize(rt); warn::Bool=true, strip_macros::Bool=false, kwargs...) +function tsn_and_mappings(mi::MethodInstance, src::CodeInfo, @nospecialize(rt); warn::Bool=true, strip_macros::Bool=false, kwargs...) + m = mi.def::Method def = definition(String, m) if isnothing(def) warn && @warn "couldn't retrieve source of $m" return nothing, nothing end - return tsn_and_mappings(m, src, rt, def...; warn, strip_macros, kwargs...) + return tsn_and_mappings(mi, src, rt, def...; warn, strip_macros, kwargs...) end -function tsn_and_mappings(m::Method, src::CodeInfo, @nospecialize(rt), sourcetext::AbstractString, lineno::Integer; warn::Bool=true, strip_macros::Bool=false, kwargs...) +function tsn_and_mappings(mi::MethodInstance, src::CodeInfo, @nospecialize(rt), sourcetext::AbstractString, lineno::Integer; warn::Bool=true, strip_macros::Bool=false, kwargs...) + m = mi.def::Method filename = isnothing(functionloc(m)[1]) ? string(m.file) : functionloc(m)[1] rootnode = JuliaSyntax.parsestmt(SyntaxNode, sourcetext; filename=filename, first_line=lineno, kwargs...) if strip_macros @@ -50,22 +51,26 @@ function tsn_and_mappings(m::Method, src::CodeInfo, @nospecialize(rt), sourcetex end end Δline = lineno - m.line # offset from original line number (Revise) - mappings, symtyps = map_ssas_to_source(src, rootnode, Δline) + mappings, symtyps = map_ssas_to_source(src, mi, rootnode, Δline) node = TypedSyntaxNode(rootnode, src, mappings, symtyps) node.typ = rt return node, mappings end -TypedSyntaxNode(@nospecialize(f), @nospecialize(t); kwargs...) = tsn_and_mappings(f, t; kwargs...)[1] +TypedSyntaxNode(@nospecialize(f), @nospecialize(tt=Base.default_tt(f)); kwargs...) = tsn_and_mappings(f, tt; kwargs...)[1] function TypedSyntaxNode(mi::MethodInstance; kwargs...) - m = mi.def::Method - src, rt = getsrc(mi) - tsn_and_mappings(m, src, rt; kwargs...)[1] + src, rt = code_typed1_tsn(mi) + tsn_and_mappings(mi, src, rt; kwargs...)[1] +end + +function TypedSyntaxNode(rootnode::SyntaxNode, @nospecialize(f), @nospecialize(tt=Base.default_tt(f)); kwargs...) + inferred_result = get_inferred_result(f, tt) + TypedSyntaxNode(rootnode, inferred_result.src, inferred_result.mi; kwargs...) end -TypedSyntaxNode(rootnode::SyntaxNode, src::CodeInfo, Δline::Integer=0) = - TypedSyntaxNode(rootnode, src, map_ssas_to_source(src, rootnode, Δline)...) +TypedSyntaxNode(rootnode::SyntaxNode, src::CodeInfo, mi::MethodInstance, Δline::Integer=0) = + TypedSyntaxNode(rootnode, src, map_ssas_to_source(src, mi, rootnode, Δline)...) function TypedSyntaxNode(rootnode::SyntaxNode, src::CodeInfo, mappings, symtyps) # There may be ambiguous assignments back to the source; preserve just the unambiguous ones @@ -304,17 +309,57 @@ function sparam_name(mi::MethodInstance, i::Int) return sig.var.name end -function getsrc(@nospecialize(f), @nospecialize(t)) - srcrts = code_typed(f, t; debuginfo=:source, optimize=false) - return only(srcrts) -end - -function getsrc(mi::MethodInstance) - cis = Base.code_typed_by_type(mi.specTypes; debuginfo=:source, optimize=false) - isempty(cis) && error("no applicable type-inferred code found for ", mi) - length(cis) == 1 || error("got $(length(cis)) possible type-inferred results for ", mi, - ", you may need a more specialized signature") - return cis[1]::Pair{CodeInfo} +@static if isdefined(Base, :method_instances) +using Base: method_instances +else +function method_instances(@nospecialize(f), @nospecialize(t), world::UInt) + tt = Base.signature_type(f, t) + results = Core.MethodInstance[] + # this make a better error message than the typeassert that follows + world == typemax(UInt) && error("code reflection cannot be used from generated functions") + for match in Base._methods_by_ftype(tt, -1, world)::Vector + instance = Core.Compiler.specialize_method(match) + push!(results, instance) + end + return results +end +end + +struct InferredResult + mi::MethodInstance + src::CodeInfo + rt + InferredResult(mi::MethodInstance, src::CodeInfo, @nospecialize(rt)) = new(mi, src, rt) +end +function get_inferred_result(@nospecialize(f), @nospecialize(tt=Base.default_tt(f)), + world::UInt=Base.get_world_counter()) + mis = method_instances(f, tt, world) + if isempty(mis) + sig = sprint(Base.show_tuple_as_call, Symbol(""), Base.signature_type(f, tt)) + error("no applicable type-inferred code found for ", sig) + elseif length(mis) ≠ 1 + sig = sprint(Base.show_tuple_as_call, Symbol(""), Base.signature_type(f, tt)) + error("got $(length(mis)) possible type-inferred results for ", sig, + ", you may need a more specialized signature") + end + mi = only(mis) + return InferredResult(mi, code_typed1_tsn(mi)...) +end + +code_typed1_tsn(mi::MethodInstance) = code_typed1_by_method_instance(mi; optimize=false, debuginfo=:source) + +function code_typed1_by_method_instance(mi::MethodInstance; + optimize::Bool=true, + debuginfo::Symbol=:default, + world::UInt=Base.get_world_counter(), + interp::Core.Compiler.AbstractInterpreter=Core.Compiler.NativeInterpreter(world)) + (ccall(:jl_is_in_pure_context, Bool, ()) || world == typemax(UInt)) && + error("code reflection should not be used from generated functions") + debuginfo = Base.IRShow.debuginfo(debuginfo) + code, rt = Core.Compiler.typeinf_code(interp, mi.def::Method, mi.specTypes, mi.sparam_vals, optimize) + code isa CodeInfo || error("no code is available for ", mi) + debuginfo === :none && Base.remove_linenums!(code) + return Pair{CodeInfo,Any}(code, rt) end function is_function_def(node) # this is not `Base.is_function_def` @@ -397,8 +442,7 @@ end # Main logic for mapping `src.code[i]` to node(s) in the SyntaxNode tree # Success: when we map it to a unique node # Δline is the (Revise) offset of the line number -function map_ssas_to_source(src::CodeInfo, rootnode::SyntaxNode, Δline::Int) - mi = src.parent::MethodInstance +function map_ssas_to_source(src::CodeInfo, mi::MethodInstance, rootnode::SyntaxNode, Δline::Int) slottypes = src.slottypes::Union{Nothing, Vector{Any}} have_slottypes = slottypes !== nothing ssavaluetypes = src.ssavaluetypes::Vector{Any} @@ -428,7 +472,7 @@ function map_ssas_to_source(src::CodeInfo, rootnode::SyntaxNode, Δline::Int) # (Essentially `copy!(mapped, filter(predicate, targets))`) function append_targets_for_line!(mapped#=::Vector{nodes}=#, i::Int, targets#=::Vector{nodes}=#) j = src.codelocs[i] - lt = src.linetable::Vector{Any} + lt = src.linetable::Vector start = getline(lt, j) + Δline stop = getnextline(lt, j, Δline) - 1 linerange = start : stop @@ -736,7 +780,7 @@ function map_ssas_to_source(src::CodeInfo, rootnode::SyntaxNode, Δline::Int) end return mappings, symtyps end -map_ssas_to_source(src::CodeInfo, rootnode::SyntaxNode, Δline::Integer) = map_ssas_to_source(src, rootnode, Int(Δline)) +map_ssas_to_source(src::CodeInfo, mi::MethodInstance, rootnode::SyntaxNode, Δline::Integer) = map_ssas_to_source(src, mi, rootnode, Int(Δline)) function follow_back(src, arg) # Follow SSAValue backward to see if it maps back to a slot diff --git a/TypedSyntax/test/exhaustive.jl b/TypedSyntax/test/exhaustive.jl index f81c8c3d..dfbc7a6e 100644 --- a/TypedSyntax/test/exhaustive.jl +++ b/TypedSyntax/test/exhaustive.jl @@ -26,7 +26,7 @@ const goodmis = Core.MethodInstance[] continue end try - tsn, _ = TypedSyntax.tsn_and_mappings(m, src, rt, ret...; warn=false) + tsn, _ = TypedSyntax.tsn_and_mappings(mi, src, rt, ret...; warn=false) @test isa(tsn, TypedSyntaxNode) push!(goodmis, mi) catch diff --git a/TypedSyntax/test/runtests.jl b/TypedSyntax/test/runtests.jl index afb1f776..bc0f04ac 100644 --- a/TypedSyntax/test/runtests.jl +++ b/TypedSyntax/test/runtests.jl @@ -1,5 +1,5 @@ using JuliaSyntax: JuliaSyntax, SyntaxNode, children, child, sourcetext, kind, @K_str -using TypedSyntax: TypedSyntax, TypedSyntaxNode, getsrc +using TypedSyntax: TypedSyntax, TypedSyntaxNode using Dates, InteractiveUtils, Test has_name_typ(node, name::Symbol, @nospecialize(T)) = kind(node) == K"Identifier" && node.val === name && node.typ === T @@ -15,8 +15,7 @@ include("test_module.jl") """ rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN1.jl") TSN.eval(Expr(rootnode)) - src, _ = getsrc(TSN.f, (Float32, Int, Float64)) - tsn = TypedSyntaxNode(rootnode, src) + tsn = TypedSyntaxNode(rootnode, TSN.f, (Float32, Int, Float64)) sig, body = children(tsn) @test children(sig)[2].typ === Float32 @test children(sig)[3].typ === Int @@ -33,8 +32,7 @@ include("test_module.jl") """ rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN2.jl") TSN.eval(Expr(rootnode)) - src, _ = getsrc(TSN.g, (Int16, Int16, Int32)) - tsn = TypedSyntaxNode(rootnode, src) + tsn = TypedSyntaxNode(rootnode, TSN.g, (Int16, Int16, Int32)) sig, body = children(tsn) @test length(children(sig)) == 4 @test children(body)[2].typ === Int32 @@ -46,8 +44,7 @@ include("test_module.jl") st = "math(x) = x + sin(x + π / 4)" rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN2.jl") TSN.eval(Expr(rootnode)) - src, _ = getsrc(TSN.math, (Int,)) - tsn = TypedSyntaxNode(rootnode, src) + tsn = TypedSyntaxNode(rootnode, TSN.math, (Int,)) sig, body = children(tsn) @test has_name_typ(child(body, 1), :x, Int) @test has_name_typ(child(body, 3, 2, 1), :x, Int) @@ -70,8 +67,7 @@ include("test_module.jl") st = "math2(x) = sin(x) + sin(x)" rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN2.jl") TSN.eval(Expr(rootnode)) - src, _ = getsrc(TSN.math2, (Int,)) - tsn = TypedSyntaxNode(rootnode, src) + tsn = TypedSyntaxNode(rootnode, TSN.math2, (Int,)) sig, body = children(tsn) @test body.typ === Float64 @test_broken child(body, 1).typ === Float64 @@ -91,8 +87,7 @@ include("test_module.jl") ) rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN3.jl") TSN.eval(Expr(rootnode)) - src, _ = getsrc(TSN.firstfirst, (Vector{Vector{Real}},)) - tsn = TypedSyntaxNode(rootnode, src) + tsn = TypedSyntaxNode(rootnode, TSN.firstfirst, (Vector{Vector{Real}},)) sig, body = children(tsn) @test child(body, idxsinner...).typ === nothing @test child(body, idxsouter...).typ === Vector{Real} @@ -150,8 +145,7 @@ include("test_module.jl") """ rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN4.jl") TSN.eval(Expr(rootnode)) - src, rt = getsrc(TSN.setlist!, (Vector{Vector{Float32}}, Vector{Vector{UInt8}}, Int, Int)) - tsn = TypedSyntaxNode(rootnode, src) + tsn = TypedSyntaxNode(rootnode, TSN.setlist!, (Vector{Vector{Float32}}, Vector{Vector{UInt8}}, Int, Int)) sig, body = children(tsn) nodelist = child(body, 1, 2, 1, 1) # `listget` @test sourcetext(nodelist) == "listget" && nodelist.typ === Vector{Vector{UInt8}} @@ -175,8 +169,7 @@ include("test_module.jl") """ rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN5.jl") TSN.eval(Expr(rootnode)) - src, rt = getsrc(TSN.callfindmin, (Vector{Float64},)) - tsn = TypedSyntaxNode(rootnode, src) + tsn = TypedSyntaxNode(rootnode, TSN.callfindmin, (Vector{Float64},)) sig, body = children(tsn) t = child(body, 1, 1) @test kind(t) == K"tuple" @@ -280,18 +273,18 @@ include("test_module.jl") """ rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN6.jl") TSN.eval(Expr(rootnode)) - src, rt = getsrc(TSN.avoidzero, (Int,)) + inferred_result = TypedSyntax.get_inferred_result(TSN.avoidzero, (Int,)) + src, rt, mi = inferred_result.src, inferred_result.rt, inferred_result.mi # src looks like this: # %1 = Main.TSN.:(var"#avoidzero#6")(true, #self#, x)::Float64 # return %1 # Consequently there is nothing to match, but at least we shouldn't error - tsn = TypedSyntaxNode(rootnode, src) + tsn = TypedSyntaxNode(rootnode, src, mi) @test isa(tsn, TypedSyntaxNode) @test rt === Float64 # Try the kwbodyfunc m = which(TSN.avoidzero, (Int,)) - src, rt = getsrc(Base.bodyfunction(m), (Bool, typeof(TSN.avoidzero), Int,)) - tsn = TypedSyntaxNode(rootnode, src) + tsn = TypedSyntaxNode(rootnode, Base.bodyfunction(m), (Bool, typeof(TSN.avoidzero), Int,)) sig, body = children(tsn) isz = child(body, 2, 1, 1) @test kind(isz) == K"call" && child(isz, 1).val == :iszero @@ -520,8 +513,7 @@ include("test_module.jl") @test_broken body.typ == Int # Construction from MethodInstance - src, rt = TypedSyntax.getsrc(TSN.myoftype, (Float64, Int)) - tsn = TypedSyntaxNode(src.parent) + tsn = TypedSyntaxNode(TSN.myoftype, (Float64, Int)) sig, body = children(tsn) node = child(body, 1) @test node.typ === Type{Float64} @@ -641,10 +633,10 @@ include("test_module.jl") @test isa(tsnc, TypedSyntaxNode) # issue 487 - m = which(TSN.f487, (Int,)) - src, rt = getsrc(TSN.f487, (Int,)) + inferred_result = TypedSyntax.get_inferred_result(TSN.f487, (Int,)) + src, mi = inferred_result.src, inferred_result.mi rt = Core.Const(1) - tsn, _ = TypedSyntax.tsn_and_mappings(m, src, rt) + tsn, _ = TypedSyntax.tsn_and_mappings(mi, src, rt) @test_nowarn str = sprint(tsn; context=:color=>false) do io, obj printstyled(io, obj; hide_type_stable=false) end diff --git a/src/codeview.jl b/src/codeview.jl index 45644694..03ffb008 100644 --- a/src/codeview.jl +++ b/src/codeview.jl @@ -182,7 +182,6 @@ function cthulhu_typed(io::IO, debuginfo::Symbol, # we're working on pre-optimization state, need to ignore `LimitedAccuracy` src = copy(src) src.ssavaluetypes = mapany(ignorelimited, src.ssavaluetypes::Vector{Any}) - src.rettype = ignorelimited(src.rettype) if src.slotnames !== nothing slotnames = Base.sourceinfo_slotnames(src) diff --git a/src/interpreter.jl b/src/interpreter.jl index 564cf7a1..a00fde35 100644 --- a/src/interpreter.jl +++ b/src/interpreter.jl @@ -131,12 +131,37 @@ function create_cthulhu_source(@nospecialize(opt), effects::Effects) return OptimizedSource(ir, opt.src, opt.src.inlineable, effects) end +@static if VERSION ≥ v"1.12.0-DEV.15" +function CC.transform_result_for_cache(interp::CthulhuInterpreter, + linfo::MethodInstance, valid_worlds::WorldRange, result::InferenceResult, can_discard_trees::Bool=false) + return create_cthulhu_source(result.src, result.ipo_effects) +end +else function CC.transform_result_for_cache(interp::CthulhuInterpreter, linfo::MethodInstance, valid_worlds::WorldRange, result::InferenceResult) return create_cthulhu_source(result.src, result.ipo_effects) end +end -@static if VERSION ≥ v"1.11.0-DEV.879" +@static if VERSION ≥ v"1.12.0-DEV.45" +function CC.src_inlining_policy(interp::CthulhuInterpreter, + @nospecialize(src), @nospecialize(info::CCCallInfo), stmt_flag::UInt32) + if isa(src, OptimizedSource) + if CC.is_stmt_inline(stmt_flag) || src.isinlineable + return true + end + return false + else + @assert src isa CC.IRCode || src === nothing "invalid Cthulhu code cache" + # the default inlining policy may try additional effor to find the source in a local cache + return @invoke CC.src_inlining_policy(interp::AbstractInterpreter, + src::Any, info::CCCallInfo, stmt_flag::UInt32) + end +end +CC.retrieve_ir_for_inlining(cached_result::CodeInstance, src::OptimizedSource) = CC.copy(src.ir) +CC.retrieve_ir_for_inlining(mi::Core.MethodInstance, src::OptimizedSource, preserve_local_sources::Bool) = + CC.retrieve_ir_for_inlining(mi, src.ir, preserve_local_sources) +elseif VERSION ≥ v"1.11.0-DEV.879" function CC.inlining_policy(interp::CthulhuInterpreter, @nospecialize(src), @nospecialize(info::CCCallInfo), stmt_flag::UInt32) if isa(src, OptimizedSource) @@ -181,7 +206,7 @@ function CC.IRInterpretationState(interp::CthulhuInterpreter, src = inferred.src method_info = CC.MethodInfo(src) return CC.IRInterpretationState(interp, method_info, ir, mi, argtypes, world, - src.min_world, src.max_world) + code.min_world, code.max_world) end @static if VERSION ≥ v"1.11.0-DEV.737" diff --git a/src/reflection.jl b/src/reflection.jl index 16ca9b62..a96029b7 100644 --- a/src/reflection.jl +++ b/src/reflection.jl @@ -350,13 +350,12 @@ function add_sourceline!(locs, CI, stmtidx::Int) end function get_typed_sourcetext(mi::MethodInstance, src::CodeInfo, @nospecialize(rt); warn::Bool=true) - meth = mi.def::Method - tsn, mappings = TypedSyntax.tsn_and_mappings(meth, src, rt; warn, strip_macros=true) - return truncate_if_defaultargs!(tsn, mappings, meth) + tsn, mappings = TypedSyntax.tsn_and_mappings(mi, src, rt; warn, strip_macros=true) + return truncate_if_defaultargs!(tsn, mappings, mi.def::Method) end function get_typed_sourcetext(mi::MethodInstance, ::IRCode, @nospecialize(rt); kwargs...) - src, rt = TypedSyntax.getsrc(mi) + src, rt = TypedSyntax.code_typed1_tsn(mi) return get_typed_sourcetext(mi, src, rt; kwargs...) end diff --git a/test/test_Cthulhu.jl b/test/test_Cthulhu.jl index eda37461..33202d20 100644 --- a/test/test_Cthulhu.jl +++ b/test/test_Cthulhu.jl @@ -52,7 +52,7 @@ end # Callsite handling in source-view mode: for kwarg functions, strip the body, and use "typed" callsites for m in (@which(anykwargs("animals")), @which(anykwargs("animals"; cat=1, dog=2))) mi = first_specialization(m) - src, rt = Cthulhu.TypedSyntax.getsrc(mi) + src, rt = Cthulhu.TypedSyntax.code_typed1_tsn(mi) tsn, mappings = Cthulhu.get_typed_sourcetext(mi, src, rt; warn=false) str = sprint(printstyled, tsn) @test occursin("anykwargs", str) && occursin("kwargs...", str) && !occursin("println", str) @@ -61,7 +61,7 @@ end # Likewise for methods that fill in default positional arguments m = @which hasdefaultargs(1) mi = first_specialization(m) - src, rt = Cthulhu.TypedSyntax.getsrc(mi) + src, rt = Cthulhu.TypedSyntax.code_typed1_tsn(mi) tsn, mappings = Cthulhu.get_typed_sourcetext(mi, src, rt; warn=false) str = sprint(printstyled, tsn) @test occursin("hasdefaultargs(a, b=2)", str) @@ -69,7 +69,7 @@ end @test isempty(mappings) m = @which hasdefaultargs(1, 5) mi = first_specialization(m) - src, rt = Cthulhu.TypedSyntax.getsrc(mi) + src, rt = Cthulhu.TypedSyntax.code_typed1_tsn(mi) tsn, mappings = Cthulhu.get_typed_sourcetext(mi, src, rt; warn=false) str = sprint(printstyled, tsn) @test occursin("hasdefaultargs(a, b=2)", str)