Make Compiler a stdlib

This is a further extension to #56128 to make the compiler into a proper
stdlib, useable outside of `Base` as `using Compiler` in the same way
that `JuliaSyntax` works already. There's a few remaining questions
around how loading works, but mechanically, this PR is complete.
For those remaining questions, I'll probably put up a separate PR
that would migrate JuliaSyntax to it and then simply adopt that
here once we've figured out the correct semantics.

Author: Keno

base/Base.jl

@@ -380,6 +380,7 @@ a_method_to_overwrite_in_test() = inferencebarrier(1)
 
 # Compatibility with when Compiler was in Core
 @eval Core const Compiler = Main.Base.Compiler
+@eval Compiler const fl_parse = Core.Main.Base.fl_parse
 
 # External libraries vendored into Base
 Core.println("JuliaSyntax/src/JuliaSyntax.jl")

base/Base_compiler.jl

@@ -252,15 +252,14 @@ include("namedtuple.jl")
 include("ordering.jl")
 using .Order
 
-include("compiler/compiler.jl")
+include("coreir.jl")
+
+include("../stdlib/Compiler/src/Compiler.jl")
 
 const _return_type = Compiler.return_type
 
 # Enable compiler
-Core.eval(Compiler, quote
-include("compiler/bootstrap.jl")
-ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
+Compiler.bootstrap!()
 
-include("compiler/parsing.jl")
+include("flparse.jl")
 Core._setparser!(fl_parse)
-end)

base/boot.jl

@@ -512,7 +512,6 @@ eval(Core, quote
     UpsilonNode(@nospecialize(val)) = $(Expr(:new, :UpsilonNode, :val))
     UpsilonNode() = $(Expr(:new, :UpsilonNode))
     Const(@nospecialize(v)) = $(Expr(:new, :Const, :v))
-    # NOTE the main constructor is defined within `Core.Compiler`
     _PartialStruct(@nospecialize(typ), fields::Array{Any, 1}) = $(Expr(:new, :PartialStruct, :typ, :fields))
     PartialOpaque(@nospecialize(typ), @nospecialize(env), parent::MethodInstance, source) = $(Expr(:new, :PartialOpaque, :typ, :env, :parent, :source))
     InterConditional(slot::Int, @nospecialize(thentype), @nospecialize(elsetype)) = $(Expr(:new, :InterConditional, :slot, :thentype, :elsetype))

base/compiler/bootstrap.jl

@@ -0,0 +1,54 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+Core.PhiNode() = Core.PhiNode(Int32[], Any[])
+
+"""
+    struct Const
+        val
+    end
+
+The type representing a constant value.
+"""
+Core.Const
+
+"""
+    struct PartialStruct
+        typ
+        fields::Vector{Any} # elements are other type lattice members
+    end
+
+This extended lattice element is introduced when we have information about an object's
+fields beyond what can be obtained from the object type. E.g. it represents a tuple where
+some elements are known to be constants or a struct whose `Any`-typed field is initialized
+with `Int` values.
+
+- `typ` indicates the type of the object
+- `fields` holds the lattice elements corresponding to each field of the object
+
+If `typ` is a struct, `fields` represents the fields of the struct that are guaranteed to be
+initialized. For instance, if the length of `fields` of `PartialStruct` representing a
+struct with 4 fields is 3, the 4th field may not be initialized. If the length is 4, all
+fields are guaranteed to be initialized.
+
+If `typ` is a tuple, the last element of `fields` may be `Vararg`. In this case, it is
+guaranteed that the number of elements in the tuple is at least `length(fields)-1`, but the
+exact number of elements is unknown.
+"""
+Core.PartialStruct
+
+"""
+    struct InterConditional
+        slot::Int
+        thentype
+        elsetype
+    end
+
+Similar to `Conditional`, but conveys inter-procedural constraints imposed on call arguments.
+This is separate from `Conditional` to catch logic errors: the lattice element name is `InterConditional`
+while processing a call, then `Conditional` everywhere else. Thus `InterConditional` does not appear in
+`CompilerTypes`—these type's usages are disjoint—though we define the lattice for `InterConditional`.
+"""
+Core.InterConditional
+
+InterConditional(var::SlotNumber, @nospecialize(thentype), @nospecialize(elsetype)) =
+    InterConditional(slot_id(var), thentype, elsetype)

base/coreir.jl

@@ -2856,7 +2856,7 @@ module IRShow
         Effects, ALWAYS_TRUE, ALWAYS_FALSE, DebugInfoStream, getdebugidx,
         VarState, InvalidIRError, argextype, widenconst, singleton_type,
         sptypes_from_meth_instance, EMPTY_SPTYPES
-    include("compiler/ssair/show.jl")
+    include("../stdlib/Compiler/src/ssair/show.jl")
 
     const __debuginfo = Dict{Symbol, Any}(
         # :full => src -> statementidx_lineinfo_printer(src), # and add variable slot information

base/compiler/parsing.jl renamed to base/flparse.jl

@@ -1289,6 +1289,21 @@ JL_DLLEXPORT jl_value_t *jl_prepend_cwd(jl_value_t *str)
     return jl_cstr_to_string(path);
 }
 
+JL_DLLEXPORT jl_value_t *jl_prepend_string(jl_value_t *prefix, jl_value_t *str)
+{
+    char path[1024];
+    const char *pstr = (const char*)jl_string_data(prefix);
+    size_t sz = strlen(pstr);
+    const char *fstr = (const char*)jl_string_data(str);
+    if (strlen(fstr) + sz >= sizeof(path)) {
+        jl_errorf("use a bigger buffer for jl_fullpath");
+    }
+    strcpy(path, pstr);
+    strcpy(path + sz, fstr);
+    return jl_cstr_to_string(path);
+}
+
+
 #ifdef __cplusplus
 }
 #endif

base/show.jl

@@ -0,0 +1,3 @@
+name = "Compiler"
+uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1"
+version = "0.0.1"

src/toplevel.c

@@ -12,8 +12,8 @@ import Core: print, println, show, write, unsafe_write,
              memoryref_isassigned, memoryrefnew, memoryrefoffset, memoryrefget,
              memoryrefset!, typename
 
-using ..Base
-using ..Base: Ordering, vect, EffectsOverride, BitVector, @_gc_preserve_begin, @_gc_preserve_end, RefValue,
+using Base
+using Base: Ordering, vect, EffectsOverride, BitVector, @_gc_preserve_begin, @_gc_preserve_end, RefValue,
     @nospecializeinfer, @_foldable_meta, fieldindex, is_function_def, indexed_iterate, isexpr, methods,
     get_world_counter, JLOptions, _methods_by_ftype, unwrap_unionall, cconvert, unsafe_convert,
     issingletontype, isType, rewrap_unionall, has_free_typevars, isvarargtype, hasgenerator,
@@ -29,8 +29,8 @@ using ..Base: Ordering, vect, EffectsOverride, BitVector, @_gc_preserve_begin, @
     specialize_method, hasintersect, is_nospecializeinfer, is_nospecialized,
     get_nospecializeinfer_sig, tls_world_age, uniontype_layout, kwerr,
     moduleroot, is_file_tracked, decode_effects_override
-using ..Base.Order
-import ..Base: getindex, setindex!, length, iterate, push!, isempty, first, convert, ==,
+using Base.Order
+import Base: getindex, setindex!, length, iterate, push!, isempty, first, convert, ==,
     copy, popfirst!, in, haskey, resize!, copy!, append!, last, get!, size,
     get, iterate, findall
 
@@ -46,8 +46,21 @@ ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Compiler, false)
 eval(x) = Core.eval(Compiler, x)
 eval(m, x) = Core.eval(m, x)
 
-include(x) = Base.include(Compiler, x)
-include(mod, x) = Base.include(mod, x)
+function include(x::String)
+    if !isdefined(Base, :end_base_include)
+        # During bootstrap, all includes are relative to `base/`
+        x = ccall(:jl_prepend_string, Ref{String}, (Any, Any), "../stdlib/Compiler/src/", x)
+    end
+    Base.include(Compiler, x)
+end
+
+function include(mod::Module, x::String)
+    if !isdefined(Base, :end_base_include)
+        x = ccall(:jl_prepend_string, Ref{String}, (Any, Any), "../stdlib/Compiler/src/", x)
+    end
+    Base.include(mod, x)
+end
+
 
 macro _boundscheck() Expr(:boundscheck) end
 
@@ -59,7 +72,7 @@ abstract type AbstractInterpreter end
 function return_type end # promotion.jl expects this to exist
 is_return_type(Core.@nospecialize(f)) = f === return_type
 
-include("compiler/sort.jl")
+include("sort.jl")
 
 # We don't include some.jl, but this definition is still useful.
 something(x::Nothing, y...) = something(y...)
@@ -94,30 +107,32 @@ else
     end
 end
 
-include("compiler/cicache.jl")
-include("compiler/methodtable.jl")
-include("compiler/effects.jl")
-include("compiler/types.jl")
-include("compiler/utilities.jl")
-include("compiler/validation.jl")
-
-include("compiler/ssair/basicblock.jl")
-include("compiler/ssair/domtree.jl")
-include("compiler/ssair/ir.jl")
-include("compiler/ssair/tarjan.jl")
-
-include("compiler/abstractlattice.jl")
-include("compiler/stmtinfo.jl")
-include("compiler/inferenceresult.jl")
-include("compiler/inferencestate.jl")
-
-include("compiler/typeutils.jl")
-include("compiler/typelimits.jl")
-include("compiler/typelattice.jl")
-include("compiler/tfuncs.jl")
-
-include("compiler/abstractinterpretation.jl")
-include("compiler/typeinfer.jl")
-include("compiler/optimize.jl")
+include("cicache.jl")
+include("methodtable.jl")
+include("effects.jl")
+include("types.jl")
+include("utilities.jl")
+include("validation.jl")
+
+include("ssair/basicblock.jl")
+include("ssair/domtree.jl")
+include("ssair/ir.jl")
+include("ssair/tarjan.jl")
+
+include("abstractlattice.jl")
+include("stmtinfo.jl")
+include("inferenceresult.jl")
+include("inferencestate.jl")
+
+include("typeutils.jl")
+include("typelimits.jl")
+include("typelattice.jl")
+include("tfuncs.jl")
+
+include("abstractinterpretation.jl")
+include("typeinfer.jl")
+include("optimize.jl")
+
+include("bootstrap.jl")
 
 end