diff --git a/src/ast.c b/src/ast.c index 62a8905f783fa..632898f33cc3b 100644 --- a/src/ast.c +++ b/src/ast.c @@ -1032,11 +1032,11 @@ static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule margs[0] = jl_toplevel_eval(*ctx, margs[0]); jl_method_instance_t *mfunc = jl_method_lookup(jl_gf_mtable(margs[0]), margs, nargs, 1, world); if (mfunc == NULL) { - jl_method_error((jl_function_t*)margs[0], margs, nargs, world); + jl_method_error((jl_function_t*)margs[0], &margs[1], nargs - 1, world); // unreachable } *ctx = mfunc->def.method->module; - result = jl_invoke(margs[0], &margs[1], nargs - 1, mfunc); + result = jl_invoke(margs[0], &margs[1], nargs, mfunc); } JL_CATCH { if (jl_loaderror_type == NULL) { diff --git a/src/builtins.c b/src/builtins.c index 9a9e7fbf0645f..6daf3e6354a0b 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -671,18 +671,19 @@ JL_CALLABLE(jl_f__apply_latest) JL_CALLABLE(jl_f_tuple) { size_t i; - if (nargs == 0) return (jl_value_t*)jl_emptytuple; + if (nargs == 0) + return (jl_value_t*)jl_emptytuple; jl_datatype_t *tt; - if (nargs < jl_page_size/sizeof(jl_value_t*)) { - jl_value_t **types = (jl_value_t**)alloca(nargs*sizeof(jl_value_t*)); - for(i=0; i < nargs; i++) + if (nargs < jl_page_size / sizeof(jl_value_t*)) { + jl_value_t **types = (jl_value_t**)alloca(nargs * sizeof(jl_value_t*)); + for (i = 0; i < nargs; i++) types[i] = jl_typeof(args[i]); tt = jl_inst_concrete_tupletype_v(types, nargs); } else { jl_svec_t *types = jl_alloc_svec_uninit(nargs); JL_GC_PUSH1(&types); - for(i=0; i < nargs; i++) + for (i = 0; i < nargs; i++) jl_svecset(types, i, jl_typeof(args[i])); tt = jl_inst_concrete_tupletype(types); JL_GC_POP(); @@ -966,10 +967,9 @@ JL_CALLABLE(jl_f_invoke) JL_GC_PUSH1(&argtypes); if (!jl_is_tuple_type(jl_unwrap_unionall(args[1]))) jl_type_error("invoke", (jl_value_t*)jl_anytuple_type_type, args[1]); - if (!jl_tuple_isa(&args[2], nargs-2, (jl_datatype_t*)argtypes)) + if (!jl_tuple_isa(args[2], &args[3], nargs - 2, (jl_datatype_t*)argtypes)) jl_error("invoke: argument type error"); - args[1] = args[0]; // move function directly in front of arguments - jl_value_t *res = jl_gf_invoke(argtypes, &args[1], nargs-1); + jl_value_t *res = jl_gf_invoke(argtypes, args[0], &args[2], nargs - 1); JL_GC_POP(); return res; } diff --git a/src/codegen.cpp b/src/codegen.cpp index 13bb39384b1aa..f46b6505d2c71 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -280,7 +280,6 @@ static Function *jltuple_func; static Function *jlnsvec_func; static Function *jlapplygeneric_func; static Function *jlinvoke_func; -static Function *jlapply2va_func; static Function *jlgetfield_func; static Function *jlmethod_func; static Function *jlgenericfunction_func; @@ -549,11 +548,6 @@ static void allocate_gc_frame(jl_codectx_t &ctx, BasicBlock *b0); static void CreateTrap(IRBuilder<> &irbuilder); static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, jl_cgval_t *args, size_t nargs, CallingConv::ID cc); -static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF, - jl_cgval_t *args, size_t nargs) -{ - return emit_jlcall(ctx, theFptr, theF, args, nargs, theF ? JLCALL_F_CC : JLCALL_CC); -} static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static GlobalVariable *prepare_global_in(Module *M, GlobalVariable *G); @@ -2190,7 +2184,7 @@ static jl_cgval_t emit_getfield(jl_codectx_t &ctx, const jl_cgval_t &strct, jl_s strct, mark_julia_const((jl_value_t*)name) }; - Value *result = emit_jlcall(ctx, jlgetfield_func, maybe_decay_untracked(V_null), myargs_array, 2); + Value *result = emit_jlcall(ctx, jlgetfield_func, maybe_decay_untracked(V_null), myargs_array, 2, JLCALL_F_CC); return mark_julia_type(ctx, result, true, jl_any_type); } @@ -2503,7 +2497,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, nva = ctx.builder.CreateTrunc(nva, T_int32); #endif Value *theArgs = ctx.builder.CreateInBoundsGEP(ctx.argArray, ConstantInt::get(T_size, ctx.nReqArgs)); - Value *r = ctx.builder.CreateCall(prepare_call(jlapply2va_func), { theF, theArgs, nva }); + Value *r = ctx.builder.CreateCall(prepare_call(jlapplygeneric_func), { theF, theArgs, nva }); *ret = mark_julia_type(ctx, r, true, jl_any_type); return true; } @@ -3265,13 +3259,13 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) std::map::iterator it = builtin_func_map.find(jl_get_builtin_fptr(f.constant)); if (it != builtin_func_map.end()) { Value *theFptr = it->second; - Value *ret = emit_jlcall(ctx, theFptr, maybe_decay_untracked(V_null), &argv[1], nargs - 1); + Value *ret = emit_jlcall(ctx, theFptr, maybe_decay_untracked(V_null), &argv[1], nargs - 1, JLCALL_F_CC); return mark_julia_type(ctx, ret, true, rt); } } // emit function and arguments - Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs); + Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs, JLCALL_F_CC); return mark_julia_type(ctx, callval, true, rt); } @@ -4136,8 +4130,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval) assert(nargs <= jl_datatype_nfields(jl_tparam0(ty)) + 1); return emit_new_struct(ctx, jl_tparam0(ty), nargs - 1, &argv[1]); } - Value *typ = boxed(ctx, argv[0]); - Value *val = emit_jlcall(ctx, jlnew_func, typ, &argv[1], nargs - 1); + Value *val = emit_jlcall(ctx, jlnew_func, nullptr, argv, nargs, JLCALL_F_CC); // temporarily mark as `Any`, expecting `emit_ssaval_assign` to update // it to the inferred type. return mark_julia_type(ctx, val, true, (jl_value_t*)jl_any_type); @@ -4334,7 +4327,7 @@ static void emit_cfunc_invalidate( } } assert(AI == gf_thunk->arg_end()); - Value *gf_ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, myargs, nargs); + Value *gf_ret = emit_jlcall(ctx, jlapplygeneric_func, nullptr, myargs, nargs, JLCALL_F_CC); jl_cgval_t gf_retbox = mark_julia_type(ctx, gf_ret, true, jl_any_type); jl_value_t *astrt = codeinst->rettype; if (cc != jl_returninfo_t::Boxed) { @@ -4706,11 +4699,11 @@ static Function* gen_cfun_wrapper( // for jlcall, we need to pass the function object even if it is a ghost. Value *theF = boxed(ctx, inputargs[0]); assert(theF); - ret_jlcall = emit_jlcall(ctx, theFptr, theF, &inputargs[1], nargs); + ret_jlcall = emit_jlcall(ctx, theFptr, theF, &inputargs[1], nargs, JLCALL_F_CC); ctx.builder.CreateBr(b_after); ctx.builder.SetInsertPoint(b_generic); } - Value *ret = emit_jlcall(ctx, prepare_call(jlapplygeneric_func), NULL, inputargs, nargs + 1); + Value *ret = emit_jlcall(ctx, prepare_call(jlapplygeneric_func), NULL, inputargs, nargs + 1, JLCALL_F_CC); if (age_ok) { ctx.builder.CreateBr(b_after); ctx.builder.SetInsertPoint(b_after); @@ -6006,8 +5999,9 @@ static std::unique_ptr emit_function( // FIXME: this may assert since the type of vi might not be isbits here emit_varinfo_assign(ctx, vi, tuple); } else { - jl_cgval_t tuple = mark_julia_type(ctx, emit_jlcall(ctx, prepare_call(jltuple_func), maybe_decay_untracked(V_null), - vargs, ctx.nvargs), true, vi.value.typ); + Value *vtpl = emit_jlcall(ctx, prepare_call(jltuple_func), maybe_decay_untracked(V_null), + vargs, ctx.nvargs, JLCALL_F_CC); + jl_cgval_t tuple = mark_julia_type(ctx, vtpl, true, vi.value.typ); emit_varinfo_assign(ctx, vi, tuple); } } @@ -7275,12 +7269,7 @@ static void init_julia_llvm_env(Module *m) jltuple_func = builtin_func_map[jl_f_tuple]; jlgetfield_func = builtin_func_map[jl_f_getfield]; - jlapply2va_func = jlcall_func_to_llvm("jl_apply_2va", &jl_apply_2va, m); - - std::vector agargs(0); - agargs.push_back(T_pprjlvalue); - agargs.push_back(T_uint32); - jlapplygeneric_func = Function::Create(FunctionType::get(T_prjlvalue, agargs, false), + jlapplygeneric_func = Function::Create(jl_func_sig, Function::ExternalLinkage, "jl_apply_generic", m); add_return_attr(jlapplygeneric_func, Attribute::NonNull); diff --git a/src/gf.c b/src/gf.c index b06324e4f8897..637f625d0f917 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1592,7 +1592,7 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method JL_GC_POP(); } -void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args, size_t world) +static void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args, size_t world) { if (jl_methoderror_type) { jl_value_t *e = jl_new_struct_uninit(jl_methoderror_type); @@ -1619,56 +1619,59 @@ void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args, size_t void JL_NORETURN jl_method_error(jl_function_t *f, jl_value_t **args, size_t na, size_t world) { - jl_value_t *argtup = jl_f_tuple(NULL, args+1, na-1); + jl_value_t *argtup = jl_f_tuple(NULL, args, na - 1); JL_GC_PUSH1(&argtup); jl_method_error_bare(f, argtup, world); // not reached } -jl_tupletype_t *arg_type_tuple(jl_value_t **args, size_t nargs) +jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs) { jl_tupletype_t *tt; size_t i; - if (nargs * sizeof(jl_value_t*) < jl_page_size) { - jl_value_t **types; - JL_GC_PUSHARGS(types, nargs); - for (i = 0; i < nargs; i++) { - jl_value_t *ai = args[i]; - if (jl_is_type(ai)) - types[i] = (jl_value_t*)jl_wrap_Type(ai); - else - types[i] = jl_typeof(ai); - } - // if `ai` has free type vars this will not be a valid (concrete) type. - // TODO: it would be really nice to only dispatch and cache those as - // `jl_typeof(ai)`, but that will require some redesign of the caching - // logic. - tt = jl_apply_tuple_type_v(types, nargs); - JL_GC_POP(); + int onstack = (nargs * sizeof(jl_value_t*) < jl_page_size); + jl_value_t **roots; + jl_value_t **types; + JL_GC_PUSHARGS(roots, onstack ? nargs : 1); + if (onstack) { + types = roots; } else { - jl_svec_t *types = jl_alloc_svec(nargs); - JL_GC_PUSH1(&types); - for (i = 0; i < nargs; i++) { - jl_value_t *ai = args[i]; - if (jl_is_type(ai)) - jl_svecset(types, i, (jl_value_t*)jl_wrap_Type(ai)); - else - jl_svecset(types, i, jl_typeof(ai)); + roots[0] = (jl_value_t*)jl_alloc_svec(nargs); + types = jl_svec_data(roots[0]); + } + for (i = 0; i < nargs; i++) { + jl_value_t *ai = (i == 0 ? arg1 : args[i - 1]); + if (jl_is_type(ai)) { + // if `ai` has free type vars this will not be a valid (concrete) type. + // TODO: it would be really nice to only dispatch and cache those as + // `jl_typeof(ai)`, but that will require some redesign of the caching + // logic. + types[i] = (jl_value_t*)jl_wrap_Type(ai); + if (!onstack) + jl_gc_wb(roots[0], types[i]); + } + else { + types[i] = jl_typeof(ai); } - tt = jl_apply_tuple_type(types); - JL_GC_POP(); } + if (onstack) + tt = jl_apply_tuple_type_v(types, nargs); + else + tt = jl_apply_tuple_type((jl_svec_t*)roots[0]); + JL_GC_POP(); return tt; } jl_method_instance_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t nargs, int cache, size_t world) { - jl_typemap_entry_t *entry = jl_typemap_assoc_exact(mt->cache, args, nargs, jl_cachearg_offset(mt), world); + if (nargs == 0) + return NULL; + jl_typemap_entry_t *entry = jl_typemap_assoc_exact(mt->cache, args[0], &args[1], nargs, jl_cachearg_offset(mt), world); if (entry) return entry->func.linfo; JL_LOCK(&mt->writelock); - jl_tupletype_t *tt = arg_type_tuple(args, nargs); + jl_tupletype_t *tt = arg_type_tuple(args[0], &args[1], nargs); JL_GC_PUSH1(&tt); jl_method_instance_t *sf = jl_mt_assoc_by_type(mt, tt, cache, world); JL_GC_POP(); @@ -2033,35 +2036,37 @@ STATIC_INLINE jl_value_t *verify_type(jl_value_t *v) JL_NOTSAFEPOINT return v; } -STATIC_INLINE jl_value_t *_jl_invoke(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc, size_t world) +STATIC_INLINE jl_value_t *_jl_invoke(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc, size_t world) { // manually inline key parts of jl_compile_method_internal: jl_code_instance_t *codeinst = mfunc->cache; while (codeinst) { if (codeinst->min_world <= world && world <= codeinst->max_world && codeinst->invoke != NULL) { - jl_value_t *res = codeinst->invoke(f, args, nargs, codeinst); + jl_value_t *res = codeinst->invoke(F, args, nargs, codeinst); return verify_type(res); } codeinst = codeinst->next; } codeinst = jl_compile_method_internal(mfunc, world); - jl_value_t *res = codeinst->invoke(f, args, nargs, codeinst); + jl_value_t *res = codeinst->invoke(F, args, nargs, codeinst); return verify_type(res); } -JL_DLLEXPORT jl_value_t *jl_invoke(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc) +JL_DLLEXPORT jl_value_t *jl_invoke(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc) { size_t world = jl_get_ptls_states()->world_age; - return _jl_invoke(f, args, nargs, mfunc, world); + return _jl_invoke(F, args, nargs, mfunc, world); } -STATIC_INLINE int sig_match_fast(jl_value_t **args, jl_value_t **sig, size_t i, size_t n) +STATIC_INLINE int sig_match_fast(jl_value_t *arg1, jl_value_t **args, jl_value_t **sig, size_t i, size_t n) { // NOTE: This function is a huge performance hot spot!! + if (arg1 != sig[0]) + return 0; for (; i < n; i++) { jl_value_t *decl = sig[i]; - jl_value_t *a = args[i]; - if ((jl_value_t*)jl_typeof(a) != decl) { + jl_value_t *a = args[i - 1]; + if (jl_typeof(a) != decl) { /* we are only matching concrete types here, and those types are hash-consed, so pointer comparison should work. @@ -2093,7 +2098,7 @@ void call_cache_stats() } #endif -STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t **args, uint32_t nargs, +STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t **args, uint32_t nargs, uint32_t callsite, size_t world) { #ifdef JL_GF_PROFILE @@ -2102,8 +2107,9 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t **args, uint32 #ifdef JL_TRACE int traceen = trace_en; //&& ((char*)&mt < jl_stack_hi-6000000); if (traceen) - show_call(args[0], &args[1], nargs-1); + show_call(F, args, nargs); #endif + nargs++; // add F to argument count /* search order: @@ -2136,7 +2142,7 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t **args, uint32 i = _i; \ entry = call_cache[cache_idx[i]]; \ if (entry && nargs == jl_svec_len(entry->sig->parameters) && \ - sig_match_fast(args, jl_svec_data(entry->sig->parameters), 0, nargs) && \ + sig_match_fast(F, args, jl_svec_data(entry->sig->parameters), 0, nargs) && \ world >= entry->min_world && world <= entry->max_world) { \ goto have_entry; \ } \ @@ -2150,9 +2156,8 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t **args, uint32 // if no method was found in the associative cache, check the full cache if (i == 4) { JL_TIMING(METHOD_LOOKUP_FAST); - jl_value_t *F = args[0]; mt = jl_gf_mtable(F); - entry = jl_typemap_assoc_exact(mt->cache, args, nargs, jl_cachearg_offset(mt), world); + entry = jl_typemap_assoc_exact(mt->cache, F, args, nargs, jl_cachearg_offset(mt), world); if (entry && entry->isleafsig && entry->simplesig == (void*)jl_nothing && entry->guardsigs == jl_emptysvec) { // put the entry into the cache if it's valid for a leafsig lookup, // using pick_which to slightly randomize where it ends up @@ -2170,7 +2175,7 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t **args, uint32 JL_LOCK(&mt->writelock); // cache miss case JL_TIMING(METHOD_LOOKUP_SLOW); - jl_tupletype_t *tt = arg_type_tuple(args, nargs); + jl_tupletype_t *tt = arg_type_tuple(F, args, nargs); JL_GC_PUSH1(&tt); mfunc = jl_mt_assoc_by_type(mt, tt, /*cache*/1, world); JL_GC_POP(); @@ -2178,9 +2183,9 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t **args, uint32 if (mfunc == NULL) { #ifdef JL_TRACE if (error_en) - show_call(args[0], args, nargs); + show_call(F, args, nargs); #endif - jl_method_error((jl_function_t*)args[0], args, nargs, world); + jl_method_error(F, args, nargs, world); // unreachable } } @@ -2195,17 +2200,17 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t **args, uint32 jl_method_instance_t *jl_lookup_generic(jl_value_t **args, uint32_t nargs, uint32_t callsite, size_t world) { - return jl_lookup_generic_(args, nargs, callsite, world); + return jl_lookup_generic_(args[0], &args[1], nargs - 1, callsite, world); } -JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) +JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t *F, jl_value_t **args, uint32_t nargs) { size_t world = jl_get_ptls_states()->world_age; - jl_method_instance_t *mfunc = jl_lookup_generic_(args, nargs, + jl_method_instance_t *mfunc = jl_lookup_generic_(F, args, nargs, jl_int32hash_fast(jl_return_address()), world); JL_GC_PROMISE_ROOTED(mfunc); - return _jl_invoke(args[0], &args[1], nargs - 1, mfunc, world); + return _jl_invoke(F, args, nargs, mfunc, world); } JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, size_t world) @@ -2223,7 +2228,7 @@ JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROO return (jl_value_t*)entry; } -jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t **args, size_t nargs); +static jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t *gf, jl_value_t **args, size_t nargs); // invoke() // this does method dispatch with a set of types to match other than the @@ -2234,12 +2239,11 @@ jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t **args, size_ // every definition has its own private method table for this purpose. // // NOTE: assumes argument type is a subtype of the lookup type. -jl_value_t *jl_gf_invoke(jl_value_t *types0, jl_value_t **args, size_t nargs) +jl_value_t *jl_gf_invoke(jl_value_t *types0, jl_value_t *gf, jl_value_t **args, size_t nargs) { size_t world = jl_get_ptls_states()->world_age; jl_value_t *types = NULL; JL_GC_PUSH1(&types); - jl_value_t *gf = args[0]; types = jl_argtype_with_function(gf, types0); jl_typemap_entry_t *entry = (jl_typemap_entry_t*)jl_gf_invoke_lookup(types, world); @@ -2251,15 +2255,15 @@ jl_value_t *jl_gf_invoke(jl_value_t *types0, jl_value_t **args, size_t nargs) // now we have found the matching definition. // next look for or create a specialization of this definition. JL_GC_POP(); - return jl_gf_invoke_by_method(entry->func.method, args, nargs); + return jl_gf_invoke_by_method(entry->func.method, gf, args, nargs); } -jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t **args, size_t nargs) +static jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t *gf, jl_value_t **args, size_t nargs) { jl_method_instance_t *mfunc = NULL; jl_typemap_entry_t *tm = NULL; if (method->invokes != NULL) - tm = jl_typemap_assoc_exact(method->invokes, args, nargs, 1, 1); + tm = jl_typemap_assoc_exact(method->invokes, gf, args, nargs, 1, 1); if (tm) { mfunc = tm->func.linfo; } @@ -2268,7 +2272,7 @@ jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t **args, size_ jl_tupletype_t *tt = NULL; JL_GC_PUSH2(&tpenv, &tt); JL_LOCK(&method->writelock); - tt = arg_type_tuple(args, nargs); + tt = arg_type_tuple(gf, args, nargs); if (jl_is_unionall(method->sig)) { int sub = jl_subtype_matching((jl_value_t*)tt, (jl_value_t*)method->sig, &tpenv); assert(sub); (void)sub; @@ -2283,7 +2287,7 @@ jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t **args, size_ } JL_GC_PROMISE_ROOTED(mfunc); size_t world = jl_get_ptls_states()->world_age; - return _jl_invoke(args[0], &args[1], nargs - 1, mfunc, world); + return _jl_invoke(gf, args, nargs - 1, mfunc, world); } JL_DLLEXPORT jl_value_t *jl_get_invoke_lambda(jl_typemap_entry_t *entry, diff --git a/src/interpreter.c b/src/interpreter.c index 984f5ba2affe5..147bb6185cd97 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -324,7 +324,7 @@ SECT_INTERP static jl_value_t *do_call(jl_value_t **args, size_t nargs, interpre size_t i; for (i = 0; i < nargs; i++) argv[i] = eval_value(args[i], s); - jl_value_t *result = jl_apply_generic(argv, nargs); + jl_value_t *result = jl_apply(argv, nargs); JL_GC_POP(); return result; } diff --git a/src/julia.h b/src/julia.h index 55e8c9d4bca48..8b6fe3c55cf1b 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1600,14 +1600,14 @@ STATIC_INLINE int jl_vinfo_usedundef(uint8_t vi) // calling into julia --------------------------------------------------------- -JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs); -JL_DLLEXPORT jl_value_t *jl_invoke(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_method_instance_t *meth); +JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t *F, jl_value_t **args, uint32_t nargs); +JL_DLLEXPORT jl_value_t *jl_invoke(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *meth); JL_DLLEXPORT int32_t jl_invoke_api(jl_code_instance_t *linfo); STATIC_INLINE jl_value_t *jl_apply(jl_value_t **args, uint32_t nargs) { - return jl_apply_generic(args, nargs); + return jl_apply_generic(args[0], &args[1], nargs - 1); } JL_DLLEXPORT jl_value_t *jl_call(jl_function_t *f, jl_value_t **args, int32_t nargs); diff --git a/src/julia_internal.h b/src/julia_internal.h index 7c690254316d7..c06ff07542f26 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -388,7 +388,6 @@ void jl_set_t_uid_ctr(int i); uint32_t jl_get_gs_ctr(void); void jl_set_gs_ctr(uint32_t ctr); -void JL_NORETURN jl_method_error_bare(jl_function_t *f, jl_value_t *args, size_t world); void JL_NORETURN jl_method_error(jl_function_t *f, jl_value_t **args, size_t na, size_t world); jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, const char *fmt, ...); @@ -414,7 +413,7 @@ typedef struct jl_typeenv_t { struct jl_typeenv_t *prev; } jl_typeenv_t; -int jl_tuple_isa(jl_value_t **child, size_t cl, jl_datatype_t *pdt); +int jl_tuple_isa(jl_value_t *child1, jl_value_t **child, size_t cl, jl_datatype_t *pdt); int jl_has_intersect_type_not_kind(jl_value_t *t); int jl_subtype_invariant(jl_value_t *a, jl_value_t *b, int ta); @@ -463,7 +462,7 @@ void jl_linenumber_to_lineinfo(jl_code_info_t *ci, jl_value_t *name); jl_method_instance_t *jl_method_lookup(jl_methtable_t *mt JL_PROPAGATES_ROOT, jl_value_t **args, size_t nargs, int cache, size_t world); -jl_value_t *jl_gf_invoke(jl_value_t *types, jl_value_t **args, size_t nargs); +jl_value_t *jl_gf_invoke(jl_value_t *types, jl_value_t *f, jl_value_t **args, size_t nargs); jl_method_instance_t *jl_lookup_generic(jl_value_t **args, uint32_t nargs, uint32_t callsite, size_t world) JL_ALWAYS_LEAFTYPE; JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, int lim, int include_ambiguous, size_t world, size_t *min_valid, size_t *max_valid); @@ -599,7 +598,7 @@ JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *t uint32_t jl_module_next_counter(jl_module_t *m); void jl_fptr_to_llvm(void *fptr, jl_code_instance_t *codeinst, int spec_abi); -jl_tupletype_t *arg_type_tuple(jl_value_t **args, size_t nargs); +jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs); int jl_has_meta(jl_array_t *body, jl_sym_t *sym); @@ -951,20 +950,20 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( jl_typemap_t *ml_or_cache JL_PROPAGATES_ROOT, jl_value_t *types, jl_svec_t **penv, int8_t subtype, int8_t offs, size_t world, size_t max_world_mask); -jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t **args, size_t n, int8_t offs, size_t world); -jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *mn, jl_value_t **args, size_t n, size_t world); +jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t *arg1, jl_value_t **args, size_t n, int8_t offs, size_t world); +jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *mn, jl_value_t *arg1, jl_value_t **args, size_t n, size_t world); STATIC_INLINE jl_typemap_entry_t *jl_typemap_assoc_exact( jl_typemap_t *ml_or_cache JL_PROPAGATES_ROOT, - jl_value_t **args, size_t n, int8_t offs, size_t world) + jl_value_t *arg1, jl_value_t **args, size_t n, int8_t offs, size_t world) { // NOTE: This function is a huge performance hot spot!! if (jl_typeof(ml_or_cache) == (jl_value_t *)jl_typemap_entry_type) { return jl_typemap_entry_assoc_exact( - (jl_typemap_entry_t *)ml_or_cache, args, n, world); + (jl_typemap_entry_t *)ml_or_cache, arg1, args, n, world); } else if (jl_typeof(ml_or_cache) == (jl_value_t*)jl_typemap_level_type) { return jl_typemap_level_assoc_exact( - (jl_typemap_level_t *)ml_or_cache, args, n, offs, world); + (jl_typemap_level_t *)ml_or_cache, arg1, args, n, offs, world); } return NULL; } diff --git a/src/rtutils.c b/src/rtutils.c index 2275dbb7bf71b..8b991583eaff3 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -332,29 +332,6 @@ void jl_push_excstack(jl_excstack_t **stack JL_REQUIRE_ROOTED_SLOT JL_ROOTING_AR jl_excstack_raw(s)[s->top-1] = (uintptr_t)exception; } -// misc ----------------------------------------------------------------------- - -// perform f(args...) on stack -JL_DLLEXPORT jl_value_t *jl_apply_2va(jl_value_t *f, jl_value_t **args, uint32_t nargs) -{ - nargs++; - int onstack = (nargs < jl_page_size/sizeof(jl_value_t*)); - jl_value_t **newargs; - JL_GC_PUSHARGS(newargs, onstack ? nargs : 1); - jl_svec_t *arg_heap = NULL; - newargs[0] = f; // make sure f is rooted - if (!onstack) { - arg_heap = jl_alloc_svec(nargs); - newargs[0] = (jl_value_t*)arg_heap; - newargs = jl_svec_data(arg_heap); - newargs[0] = f; - } - memcpy(&newargs[1], args, (nargs-1)*sizeof(jl_value_t*)); - jl_value_t *ret = jl_apply_generic(newargs, nargs); - JL_GC_POP(); - return ret; -} - // conversion ----------------------------------------------------------------- JL_DLLEXPORT void *(jl_symbol_name)(jl_sym_t *s) diff --git a/src/subtype.c b/src/subtype.c index 84868a302022a..8f1912a3dfa27 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -1442,19 +1442,23 @@ JL_DLLEXPORT int jl_is_not_broken_subtype(jl_value_t *a, jl_value_t *b) return !jl_is_kind(b) || !jl_is_type_type(a); // || jl_is_datatype_singleton((jl_datatype_t*)jl_tparam0(a)); } -int jl_tuple_isa(jl_value_t **child, size_t cl, jl_datatype_t *pdt) +int jl_tuple_isa(jl_value_t *child1, jl_value_t **child, size_t cl, jl_datatype_t *pdt) { if (jl_is_tuple_type(pdt) && !jl_is_va_tuple(pdt)) { if (cl != jl_nparams(pdt)) return 0; size_t i; - for(i=0; i < cl; i++) { - if (!jl_isa(child[i], jl_tparam(pdt,i))) + if (cl == 0) + return 1; + if (!jl_isa(child1, jl_tparam(pdt, 0))) + return 0; + for (i = 1; i < cl; i++) { + if (!jl_isa(child[i - 1], jl_tparam(pdt, i))) return 0; } return 1; } - jl_value_t *tu = (jl_value_t*)arg_type_tuple(child, cl); + jl_value_t *tu = (jl_value_t*)arg_type_tuple(child1, child, cl); int ans; JL_GC_PUSH1(&tu); ans = jl_subtype(tu, (jl_value_t*)pdt); diff --git a/src/typemap.c b/src/typemap.c index 593e401932ed0..12df83657099b 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -95,14 +95,16 @@ static int sig_match_by_type_simple(jl_value_t **types, size_t n, jl_tupletype_t return 1; } -static inline int sig_match_leaf(jl_value_t **args, jl_value_t **sig, size_t n) +static inline int sig_match_leaf(jl_value_t *arg1, jl_value_t **args, jl_value_t **sig, size_t n) { // NOTE: This function is a huge performance hot spot!! size_t i; - for (i = 0; i < n; i++) { + if (jl_typeof(arg1) != sig[0]) + return 0; + for (i = 1; i < n; i++) { jl_value_t *decl = sig[i]; - jl_value_t *a = args[i]; - if ((jl_value_t*)jl_typeof(a) != decl) { + jl_value_t *a = args[i - 1]; + if (jl_typeof(a) != decl) { /* we are only matching concrete types here, and those types are hash-consed, so pointer comparison should work. @@ -113,16 +115,17 @@ static inline int sig_match_leaf(jl_value_t **args, jl_value_t **sig, size_t n) return 1; } -static inline int sig_match_simple(jl_value_t **args, size_t n, jl_value_t **sig, +static inline int sig_match_simple(jl_value_t *arg1, jl_value_t **args, size_t n, jl_value_t **sig, int va, size_t lensig) { // NOTE: This function is a performance hot spot!! size_t i; - if (va) lensig -= 1; + if (va) + lensig -= 1; for (i = 0; i < lensig; i++) { jl_value_t *decl = sig[i]; - jl_value_t *a = args[i]; - if (decl == (jl_value_t*)jl_any_type || ((jl_value_t*)jl_typeof(a) == decl)) { + jl_value_t *a = (i == 0 ? arg1 : args[i - 1]); + if (jl_typeof(a) == decl || decl == (jl_value_t*)jl_any_type) { /* we are only matching concrete types here, and those types are hash-consed, so pointer comparison should work. @@ -166,7 +169,8 @@ static inline int sig_match_simple(jl_value_t **args, size_t n, jl_value_t **sig } jl_value_t *t = jl_unwrap_vararg(decl); for (; i < n; i++) { - if (!jl_isa(args[i], t)) + jl_value_t *a = (i == 0 ? arg1 : args[i - 1]); + if (!jl_isa(a, t)) return 0; } return 1; @@ -754,26 +758,26 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type(jl_typemap_t *ml_or_cache, jl_value } } -jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_value_t **args, size_t n, size_t world) +jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_value_t *arg1, jl_value_t **args, size_t n, size_t world) { // some manually-unrolled common special cases while (ml->simplesig == (void*)jl_nothing && ml->guardsigs == jl_emptysvec && ml->isleafsig) { // use a tight loop for as long as possible if (world >= ml->min_world && world <= ml->max_world) { - if (n == jl_nparams(ml->sig) && jl_typeof(args[0]) == jl_tparam(ml->sig, 0)) { + if (n == jl_nparams(ml->sig) && jl_typeof(arg1) == jl_tparam(ml->sig, 0)) { if (n == 1) return ml; if (n == 2) { - if (jl_typeof(args[1]) == jl_tparam(ml->sig, 1)) + if (jl_typeof(args[0]) == jl_tparam(ml->sig, 1)) return ml; } else if (n == 3) { - if (jl_typeof(args[1]) == jl_tparam(ml->sig, 1) && - jl_typeof(args[2]) == jl_tparam(ml->sig, 2)) + if (jl_typeof(args[0]) == jl_tparam(ml->sig, 1) && + jl_typeof(args[1]) == jl_tparam(ml->sig, 2)) return ml; } else { - if (sig_match_leaf(args, jl_svec_data(ml->sig->parameters), n)) + if (sig_match_leaf(arg1, args, jl_svec_data(ml->sig->parameters), n)) return ml; } } @@ -792,7 +796,7 @@ jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_valu size_t lensimplesig = jl_nparams(ml->simplesig); int isva = lensimplesig > 0 && jl_is_vararg_type(jl_tparam(ml->simplesig, lensimplesig - 1)); if (lensig == n || (isva && lensimplesig <= n + 1)) { - if (!sig_match_simple(args, n, jl_svec_data(ml->simplesig->parameters), isva, lensimplesig)) + if (!sig_match_simple(arg1, args, n, jl_svec_data(ml->simplesig->parameters), isva, lensimplesig)) continue; } else { @@ -801,15 +805,15 @@ jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_valu } if (ml->isleafsig) { - if (!sig_match_leaf(args, jl_svec_data(ml->sig->parameters), n)) + if (!sig_match_leaf(arg1, args, jl_svec_data(ml->sig->parameters), n)) continue; } else if (ml->issimplesig) { - if (!sig_match_simple(args, n, jl_svec_data(ml->sig->parameters), ml->va, lensig)) + if (!sig_match_simple(arg1, args, n, jl_svec_data(ml->sig->parameters), ml->va, lensig)) continue; } else { - if (!jl_tuple_isa(args, n, ml->sig)) + if (!jl_tuple_isa(arg1, args, n, ml->sig)) continue; } @@ -819,7 +823,7 @@ jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_valu // checking guard entries require a more // expensive subtype check, since guard entries added for @nospecialize might be // abstract. this fixed issue #12967. - if (jl_tuple_isa(args, n, (jl_tupletype_t*)jl_svecref(ml->guardsigs, i))) { + if (jl_tuple_isa(arg1, args, n, (jl_tupletype_t*)jl_svecref(ml->guardsigs, i))) { goto nomatch; } } @@ -832,29 +836,29 @@ jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_valu return NULL; } -jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t **args, size_t n, int8_t offs, size_t world) +jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t *arg1, jl_value_t **args, size_t n, int8_t offs, size_t world) { if (n > offs) { - jl_value_t *a1 = args[offs]; - jl_value_t *ty = (jl_value_t*)jl_typeof(a1); + jl_value_t *a1 = (offs == 0 ? arg1 : args[offs - 1]); + jl_value_t *ty = jl_typeof(a1); assert(jl_is_datatype(ty)); if (ty == (jl_value_t*)jl_datatype_type && cache->targ.values != (void*)jl_nothing) { jl_typemap_t *ml_or_cache = mtcache_hash_lookup(&cache->targ, a1, 1, offs); - jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1, world); + jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, arg1, args, n, offs+1, world); if (ml) return ml; } if (cache->arg1.values != (void*)jl_nothing) { jl_typemap_t *ml_or_cache = mtcache_hash_lookup(&cache->arg1, ty, 0, offs); - jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1, world); + jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, arg1, args, n, offs+1, world); if (ml) return ml; } } if (cache->linear != (jl_typemap_entry_t*)jl_nothing) { - jl_typemap_entry_t *ml = jl_typemap_entry_assoc_exact(cache->linear, args, n, world); + jl_typemap_entry_t *ml = jl_typemap_entry_assoc_exact(cache->linear, arg1, args, n, world); if (ml) return ml; } if (cache->any != jl_nothing) - return jl_typemap_assoc_exact(cache->any, args, n, offs+1, world); + return jl_typemap_assoc_exact(cache->any, arg1, args, n, offs+1, world); return NULL; }