Skip to content

Commit

Permalink
fix --compile=all option, and some improvements for --strip-ir (#…
Browse files Browse the repository at this point in the history 
…46935)

(cherry picked from commit 6443a46)
  • Loading branch information
@JeffBezanson
JeffBezanson committed Oct 27, 2022
1 parent d0156b5 commit 6af3b72
Show file tree
Hide file tree
Showing 5 changed files with 56 additions and 88 deletions.
5 changes: 4 additions & 1 deletion base/compiler/utilities.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,10 @@ function retrieve_code_info(linfo::MethodInstance)
end
if c === nothing && isdefined(m, :source)
src = m.source
if isa(src, Array{UInt8,1})
if src === nothing
# can happen in images built with --strip-ir
return nothing
elseif isa(src, Array{UInt8,1})
c = ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), m, C_NULL, src)
else
c = copy(src::CodeInfo)
Expand Down
4 changes: 1 addition & 3 deletions src/gf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,6 @@ jl_code_info_t *jl_type_infer(jl_method_instance_t *mi, size_t world, int force)
JL_TIMING(INFERENCE);
if (jl_typeinf_func == NULL)
return NULL;
if (jl_is_method(mi->def.method) && mi->def.method->unspecialized == mi)
return NULL; // avoid inferring the unspecialized method
static int in_inference;
if (in_inference > 2)
return NULL;
Expand Down Expand Up @@ -2062,7 +2060,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t
if (compile_option == JL_OPTIONS_COMPILE_OFF ||
compile_option == JL_OPTIONS_COMPILE_MIN) {
jl_code_info_t *src = jl_code_for_interpreter(mi);
if (!jl_code_requires_compiler(src)) {
if (!jl_code_requires_compiler(src, 0)) {
jl_code_instance_t *codeinst = jl_new_codeinst(mi,
(jl_value_t*)jl_any_type, NULL, NULL,
0, 1, ~(size_t)0, 0, 0, jl_nothing, 0);
Expand Down
2 changes: 1 addition & 1 deletion src/julia_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -530,7 +530,7 @@ jl_method_instance_t *jl_get_unspecialized(jl_method_instance_t *method JL_PROPA

JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types);
jl_code_info_t *jl_code_for_interpreter(jl_method_instance_t *lam JL_PROPAGATES_ROOT);
int jl_code_requires_compiler(jl_code_info_t *src);
int jl_code_requires_compiler(jl_code_info_t *src, int include_force_compile);
jl_code_info_t *jl_new_code_info_from_ir(jl_expr_t *ast);
JL_DLLEXPORT jl_code_info_t *jl_new_code_info_uninit(void);
void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals,
Expand Down
101 changes: 34 additions & 67 deletions src/precompile.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,13 +113,6 @@ void jl_write_compiler_output(void)
// and expanding the Union may give a leaf function
static void _compile_all_tvar_union(jl_value_t *methsig)
{
if (!jl_is_unionall(methsig) && jl_is_dispatch_tupletype(methsig)) {
// usually can create a specialized version of the function,
// if the signature is already a dispatch type
if (jl_compile_hint((jl_tupletype_t*)methsig))
return;
}

int tvarslen = jl_subtype_env_size(methsig);
jl_value_t *sigbody = methsig;
jl_value_t **roots;
Expand Down Expand Up @@ -246,74 +239,49 @@ static void _compile_all_union(jl_value_t *sig)
JL_GC_POP();
}

static void _compile_all_deq(jl_array_t *found)
{
int found_i, found_l = jl_array_len(found);
jl_printf(JL_STDERR, "found %d uncompiled methods for compile-all\n", (int)found_l);
jl_method_instance_t *mi = NULL;
jl_value_t *src = NULL;
JL_GC_PUSH2(&mi, &src);
for (found_i = 0; found_i < found_l; found_i++) {
if (found_i % (1 + found_l / 300) == 0 || found_i == found_l - 1) // show 300 progress steps, to show progress without overwhelming log files
jl_printf(JL_STDERR, " %d / %d\r", found_i + 1, found_l);
jl_typemap_entry_t *ml = (jl_typemap_entry_t*)jl_array_ptr_ref(found, found_i);
jl_method_t *m = ml->func.method;
if (m->source == NULL) // TODO: generic implementations of generated functions
continue;
mi = jl_get_unspecialized(mi);
assert(mi == m->unspecialized); // make sure we didn't get tricked by a generated function, since we can't handle those
jl_code_instance_t *ucache = jl_get_method_inferred(mi, (jl_value_t*)jl_any_type, 1, ~(size_t)0);
if (ucache->invoke != NULL)
continue;
src = m->source;
assert(src);
// TODO: we could now enable storing inferred function pointers in the `unspecialized` cache
//src = jl_type_infer(mi, jl_atomic_load_acquire(&jl_world_counter), 1);
//if (ucache->invoke != NULL)
// continue;

// first try to create leaf signatures from the signature declaration and compile those
_compile_all_union((jl_value_t*)ml->sig);
// then also compile the generic fallback
jl_generate_fptr_for_unspecialized(ucache);
}
JL_GC_POP();
jl_printf(JL_STDERR, "\n");
}

static int compile_all_enq__(jl_typemap_entry_t *ml, void *env)
static int compile_all_collect__(jl_typemap_entry_t *ml, void *env)
{
jl_array_t *found = (jl_array_t*)env;
// method definition -- compile template field
jl_array_t *allmeths = (jl_array_t*)env;
jl_method_t *m = ml->func.method;
if (m->source) {
// found a method to compile
jl_array_ptr_1d_push(found, (jl_value_t*)ml);
// method has a non-generated definition; can be compiled generically
jl_array_ptr_1d_push(allmeths, (jl_value_t*)m);
}
return 1;
}


static int compile_all_enq_(jl_methtable_t *mt, void *env)
static int compile_all_collect_(jl_methtable_t *mt, void *env)
{
jl_typemap_visitor(mt->defs, compile_all_enq__, env);
jl_typemap_visitor(mt->defs, compile_all_collect__, env);
return 1;
}

static void jl_compile_all_defs(void)
static void jl_compile_all_defs(jl_array_t *mis)
{
// this "found" array will contain
// TypeMapEntries for Methods and MethodInstances that need to be compiled
jl_array_t *m = jl_alloc_vec_any(0);
JL_GC_PUSH1(&m);
while (1) {
jl_foreach_reachable_mtable(compile_all_enq_, m);
size_t changes = jl_array_len(m);
if (!changes)
break;
_compile_all_deq(m);
jl_array_del_end(m, changes);
jl_array_t *allmeths = jl_alloc_vec_any(0);
JL_GC_PUSH1(&allmeths);

jl_foreach_reachable_mtable(compile_all_collect_, allmeths);

size_t i, l = jl_array_len(allmeths);
for (i = 0; i < l; i++) {
jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(allmeths, i);
if (jl_isa_compileable_sig((jl_tupletype_t*)m->sig, m)) {
// method has a single compileable specialization, e.g. its definition
// signature is concrete. in this case we can just hint it.
jl_compile_hint((jl_tupletype_t*)m->sig);
}
else {
// first try to create leaf signatures from the signature declaration and compile those
_compile_all_union(m->sig);

// finally, compile a fully generic fallback that can work for all arguments
jl_method_instance_t *unspec = jl_get_unspecialized(m);
if (unspec)
jl_array_ptr_1d_push(mis, (jl_value_t*)unspec);
}
}

JL_GC_POP();
}

Expand Down Expand Up @@ -371,14 +339,13 @@ static int precompile_enq_all_specializations_(jl_methtable_t *mt, void *env)

static void *jl_precompile(int all)
{
if (all)
jl_compile_all_defs();
// this "found" array will contain function
// type signatures that were inferred but haven't been compiled
// array of MethodInstances and ccallable aliases to include in the output
jl_array_t *m = jl_alloc_vec_any(0);
jl_array_t *m2 = NULL;
jl_method_instance_t *mi = NULL;
JL_GC_PUSH3(&m, &m2, &mi);
if (all)
jl_compile_all_defs(m);
jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m);
m2 = jl_alloc_vec_any(0);
for (size_t i = 0; i < jl_array_len(m); i++) {
Expand All @@ -387,7 +354,7 @@ static void *jl_precompile(int all)
mi = (jl_method_instance_t*)item;
size_t min_world = 0;
size_t max_world = ~(size_t)0;
if (!jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->def.method))
if (mi != jl_atomic_load_relaxed(&mi->def.method->unspecialized) && !jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->def.method))
mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_atomic_load_acquire(&jl_world_counter), &min_world, &max_world, 0);
if (mi)
jl_array_ptr_1d_push(m2, (jl_value_t*)mi);
Expand Down
32 changes: 16 additions & 16 deletions src/toplevel.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ JL_DLLEXPORT jl_module_t *jl_base_relative_to(jl_module_t *m)
return jl_top_module;
}

static void expr_attributes(jl_value_t *v, int *has_intrinsics, int *has_defs, int *has_opaque)
static void expr_attributes(jl_value_t *v, int *has_ccall, int *has_defs, int *has_opaque)
{
if (!jl_is_expr(v))
return;
Expand All @@ -361,11 +361,11 @@ static void expr_attributes(jl_value_t *v, int *has_intrinsics, int *has_defs, i
*has_defs = 1;
}
else if (head == jl_cfunction_sym) {
*has_intrinsics = 1;
*has_ccall = 1;
return;
}
else if (head == jl_foreigncall_sym) {
*has_intrinsics = 1;
*has_ccall = 1;
return;
}
else if (head == jl_new_opaque_closure_sym) {
Expand All @@ -389,7 +389,7 @@ static void expr_attributes(jl_value_t *v, int *has_intrinsics, int *has_defs, i
}
if (called) {
if (jl_is_intrinsic(called) && jl_unbox_int32(called) == (int)llvmcall) {
*has_intrinsics = 1;
*has_ccall = 1;
}
if (called == jl_builtin__typebody) {
*has_defs = 1;
Expand All @@ -401,28 +401,28 @@ static void expr_attributes(jl_value_t *v, int *has_intrinsics, int *has_defs, i
for (i = 0; i < jl_array_len(e->args); i++) {
jl_value_t *a = jl_exprarg(e, i);
if (jl_is_expr(a))
expr_attributes(a, has_intrinsics, has_defs, has_opaque);
expr_attributes(a, has_ccall, has_defs, has_opaque);
}
}

int jl_code_requires_compiler(jl_code_info_t *src)
int jl_code_requires_compiler(jl_code_info_t *src, int include_force_compile)
{
jl_array_t *body = src->code;
assert(jl_typeis(body, jl_array_any_type));
size_t i;
int has_intrinsics = 0, has_defs = 0, has_opaque = 0;
if (jl_has_meta(body, jl_force_compile_sym))
int has_ccall = 0, has_defs = 0, has_opaque = 0;
if (include_force_compile && jl_has_meta(body, jl_force_compile_sym))
return 1;
for(i=0; i < jl_array_len(body); i++) {
jl_value_t *stmt = jl_array_ptr_ref(body,i);
expr_attributes(stmt, &has_intrinsics, &has_defs, &has_opaque);
if (has_intrinsics)
expr_attributes(stmt, &has_ccall, &has_defs, &has_opaque);
if (has_ccall)
return 1;
}
return 0;
}

static void body_attributes(jl_array_t *body, int *has_intrinsics, int *has_defs, int *has_loops, int *has_opaque, int *forced_compile)
static void body_attributes(jl_array_t *body, int *has_ccall, int *has_defs, int *has_loops, int *has_opaque, int *forced_compile)
{
size_t i;
*has_loops = 0;
Expand All @@ -438,7 +438,7 @@ static void body_attributes(jl_array_t *body, int *has_intrinsics, int *has_defs
*has_loops = 1;
}
}
expr_attributes(stmt, has_intrinsics, has_defs, has_opaque);
expr_attributes(stmt, has_ccall, has_defs, has_opaque);
}
*forced_compile = jl_has_meta(body, jl_force_compile_sym);
}
Expand Down Expand Up @@ -868,16 +868,16 @@ jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_value_t *e, int
return (jl_value_t*)ex;
}

int has_intrinsics = 0, has_defs = 0, has_loops = 0, has_opaque = 0, forced_compile = 0;
int has_ccall = 0, has_defs = 0, has_loops = 0, has_opaque = 0, forced_compile = 0;
assert(head == jl_thunk_sym);
thk = (jl_code_info_t*)jl_exprarg(ex, 0);
assert(jl_is_code_info(thk));
assert(jl_typeis(thk->code, jl_array_any_type));
body_attributes((jl_array_t*)thk->code, &has_intrinsics, &has_defs, &has_loops, &has_opaque, &forced_compile);
body_attributes((jl_array_t*)thk->code, &has_ccall, &has_defs, &has_loops, &has_opaque, &forced_compile);

jl_value_t *result;
if (forced_compile || has_intrinsics ||
(!has_defs && fast && has_loops &&
if (has_ccall ||
((forced_compile || (!has_defs && fast && has_loops)) &&
jl_options.compile_enabled != JL_OPTIONS_COMPILE_OFF &&
jl_options.compile_enabled != JL_OPTIONS_COMPILE_MIN &&
jl_get_module_compile(m) != JL_OPTIONS_COMPILE_OFF &&
Expand Down

0 comments on commit 6af3b72

Please sign in to comment.