cleanup rebase a bit, now missing functionality

Author: vchuravy

src/aotcompile.cpp

@@ -93,7 +93,7 @@ typedef struct {
     std::vector<GlobalValue*> jl_sysimg_gvars;
     std::map<jl_code_instance_t*, std::tuple<uint32_t, uint32_t>> jl_fvar_map;
     std::vector<void*> jl_value_to_llvm;
-    std::map<std::tuple<jl_code_instance_t*, bool>, int32_t> jl_external_to_llvm; // uses 1-based indexing
+    std::vector<jl_code_instance_t*> jl_external_to_llvm;
     int32_t external_fns_begin;
 } jl_native_code_desc_t;
 
@@ -121,23 +121,12 @@ void jl_get_llvm_gvs_impl(void *native_code, arraylist_t *gvs)
 }
 
 extern "C" JL_DLLEXPORT
-void jl_iterate_llvm_external_fns_impl(void *native_code, void (*callback)(void*, int32_t, jl_code_instance_t*, uint8_t), void* ctx)
+void jl_get_llvm_external_fns_impl(void *native_code, arraylist_t *external_fns)
 {
     jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code;
-    if (data) {
-        for (std::pair<std::tuple<jl_code_instance_t*, uint8_t>, int32_t> pair : data->jl_external_to_llvm) {
-            callback(ctx, pair.second, std::get<0>(pair.first), std::get<1>(pair.first));
-        }
-    }
-}
-extern "C" JL_DLLEXPORT
-int32_t jl_get_llvm_external_fns_begin_impl(void *native_code)
-{
-    jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code;
-    if (data) {
-        return data->external_fns_begin;
-    }
-    return 0;
+    arraylist_grow(external_fns, data->jl_external_to_llvm.size());
+    memcpy(external_fns->items, data->jl_external_to_llvm.data(),
+        external_fns->len * sizeof(jl_code_instance_t*));
 }
 
 extern "C" JL_DLLEXPORT
@@ -367,16 +356,18 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
     }
     CreateNativeMethods += emitted.size();
 
-    data->external_fns_begin = gvars.size() + 1;
+    size_t offset = gvars.size();
+    data->jl_external_to_llvm.resize(params.external_fns.size());
 
     for (auto &extern_fn : params.external_fns) {
-        // jl_code_instance_t *this_code = std::get<0>(extern_fn.first);
-        // bool specsig = std::get<1>(extern_fn.first);
+        jl_code_instance_t *this_code = std::get<0>(extern_fn.first);
+        bool specsig = std::get<1>(extern_fn.first);
+        assert(specsig && "Error external_fns doesn't handle non-specsig yet");
         Function *F = extern_fn.second;
         Module *M = F->getParent();
 
         Type *T_funcp = F->getFunctionType()->getPointerTo();
-        // Can't create a GC with type FunctionType grr
+        // Can't create a GC with type FunctionType. Alias also doesn't work
         GlobalVariable *GV = new GlobalVariable(*M, T_funcp, false,
                                                 GlobalVariable::ExternalLinkage,
                                                 Constant::getNullValue(T_funcp),
@@ -399,8 +390,10 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
         GV->takeName(F);
         F->eraseFromParent();
 
+        size_t idx = gvars.size() - offset;
+        assert(idx >= 0);
+        data->jl_external_to_llvm.at(idx) = this_code;
         gvars.push_back(std::string(GV->getName()));
-        data->jl_external_to_llvm[extern_fn.first] = gvars.size();
     }
 
     // clones the contents of the module `m` to the shadow_output collector

src/codegen-stubs.c

@@ -14,9 +14,7 @@ JL_DLLEXPORT void jl_dump_native_fallback(void *native_code,
         const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname,
         const char *sysimg_data, size_t sysimg_len) UNAVAILABLE
 JL_DLLEXPORT void jl_get_llvm_gvs_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE
-JL_DLLEXPORT void jl_iterate_llvm_gv_fallback(void *native_code, void (*callback)(void*, int32_t, void*), void* ctx) UNAVAILABLE
-JL_DLLEXPORT void jl_iterate_llvm_external_fns_fallback(void *native_code, void (*callback)(void*, int32_t, jl_code_instance_t*, uint8_t), void* ctx) UNAVAILABLE
-JL_DLLEXPORT int32_t jl_get_llvm_external_fns_begin_fallback(void *native_code) UNAVAILABLE;
+JL_DLLEXPORT void jl_get_llvm_external_fns_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE
 
 JL_DLLEXPORT void jl_extern_c_fallback(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE
 JL_DLLEXPORT jl_value_t *jl_dump_method_asm_fallback(jl_method_instance_t *linfo, size_t world,

src/jl_exported_funcs.inc

@@ -535,9 +535,7 @@
     YY(jl_get_LLVM_VERSION) \
     YY(jl_dump_native) \
     YY(jl_get_llvm_gvs) \
-    YY(jl_iterate_llvm_gv) \
-    YY(jl_iterate_llvm_external_fns) \
-    YY(jl_get_llvm_external_fns_begin) \
+    YY(jl_get_llvm_external_fns) \
     YY(jl_dump_function_asm) \
     YY(jl_LLVMCreateDisasm) \
     YY(jl_LLVMDisasmInstruction) \

src/julia_internal.h

@@ -975,9 +975,7 @@ void jl_dump_native(void *native_code,
         const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname,
         const char *sysimg_data, size_t sysimg_len);
 void jl_get_llvm_gvs(void *native_code, arraylist_t *gvs);
-void jl_iterate_llvm_gv(void *native_code, void (*callback)(void*, int32_t, void*), void*) JL_NOTSAFEPOINT;
-void jl_iterate_llvm_external_fns(void *native_code, void (*callback)(void*, int32_t, jl_code_instance_t*, uint8_t), void*) JL_NOTSAFEPOINT;
-int32_t jl_get_llvm_external_fns_begin(void *native_code);
+void jl_get_llvm_external_fns(void *native_code, arraylist_t *gvs);
 JL_DLLEXPORT void jl_get_function_id(void *native_code, jl_code_instance_t *ncode,
         int32_t *func_idx, int32_t *specfunc_idx);
 

src/staticdata.c

@@ -1451,42 +1451,6 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED
     }
 }
 
-static void jl_ensure_extern(void* ctx, int32_t GV, void* v) {
-    jl_serializer_state *s = (jl_serializer_state*)ctx;
-    // externally_linked is insufficient as a check because it doesn't visit
-    // everything needed by LLVM and can't efficiently identify whether a
-    // global variable is external.
-    uintptr_t item = external_linkage(s, (jl_value_t*)v, s->link_ids_gvars);
-    if (item == 0)
-        return;
-
-    // TODO: Check that we don't override an existing entry
-    // jl_printf(JL_STDOUT, "GV: %ld, record_gvar for %lx, build_id: %lx\n", GV, item, jl_array_data(s->link_ids_gvars)[jl_array_len(s->link_ids_gvars)-1]);
-    assert(item >> RELOC_TAG_OFFSET == ExternalLinkage);
-    record_gvar(s, GV, item);
-}
-
-static void jl_ensure_extern_gv(jl_serializer_state *s) {
-    jl_iterate_llvm_gv(native_functions, jl_ensure_extern, (void*)s);
-}
-
-static void jl_ensure_extern_fn(void* ctx, int32_t GV, jl_code_instance_t* codeinst, uint8_t specsig) {
-    jl_serializer_state *s = (jl_serializer_state*)ctx;
-
-    // TODO: Record specsig in a side-table?
-    uintptr_t item = external_linkage(s, (jl_value_t*)codeinst, s->link_ids_external_fnvars);
-    jl_printf(JL_STDOUT, "External Functions: %ld, record_gvar for %lx\n", GV, item);
-    if (item == 0)
-        return;
-
-    assert(item >> RELOC_TAG_OFFSET == ExternalLinkage);
-    record_gvar(s, GV, item);
-}
-
-static void jl_ensure_extern_fns(jl_serializer_state *s) {
-    jl_iterate_llvm_external_fns(native_functions, jl_ensure_extern_fn, (void*)s);
-}
-
 // In deserialization, create Symbols and set up the
 // index for backreferencing
 static void jl_read_symbols(jl_serializer_state *s)
@@ -2265,9 +2229,13 @@ static void jl_save_system_image_to_stream(ios_t *f,
     jl_value_t **const*const tags = get_tags(); // worklist == NULL ? get_tags() : NULL;
 
     arraylist_t gvars;
+    arraylist_t external_fns;
     arraylist_new(&gvars, 0);
-    if (native_functions)
+    arraylist_new(&external_fns);
+    if (native_functions) {
         jl_get_llvm_gvs(native_functions, &gvars);
+        jl_get_llvm_external_fns(native_functions, &external_fns);
+    }
 
     if (worklist == NULL) {
         // empty!(Core.ARGS)
@@ -2333,6 +2301,7 @@ static void jl_save_system_image_to_stream(ios_t *f,
         jl_serialize_reachable(&s);
         // step 1.2: now that we have marked all bindings (badly), ensure all gvars are part of the sysimage
         record_gvars(&s, &gvars);
+        // TODO record external_fns?
         jl_serialize_reachable(&s);
         // step 1.3: prune (garbage collect) some special weak references from
         // built-in type caches
@@ -2349,12 +2318,9 @@ static void jl_save_system_image_to_stream(ios_t *f,
     { // step 2: build all the sysimg sections
         write_padding(&sysimg, sizeof(uintptr_t));
         jl_write_values(&s);
-        if (worklist) {
-            jl_ensure_extern_gv(&s);
-            jl_ensure_extern_fns(&s);
-        }
         jl_write_relocations(&s);
         write_gvars(&s, &gvars);
+        // TODO write external_fns;
     }
 
     if (sysimg.size > ((uintptr_t)1 << RELOC_TAG_OFFSET)) {