Support sticky immix (#48)

Author: qinsoon

.github/scripts/ci-build.sh

@@ -10,6 +10,8 @@ if [ $# -eq 0 ]
 fi
 # debug or release
 build_type=$1
+# plan to use
+plan=$2
 
 # helloworld.jl
 HELLO_WORLD_JL=$BINDING_PATH/.github/scripts/hello_world.jl
@@ -21,7 +23,7 @@ if [ "$build_type" == "release" ]; then
 fi
 
 cd $MMTK_JULIA_DIR/mmtk
-cargo build --features immix $build_args
+cargo build --features $plan $build_args
 
 cd $JULIA_PATH
 # Clean first

.github/scripts/ci-test-stdlib.sh

@@ -20,6 +20,11 @@ declare -a tests_to_skip=(
 declare -a tests_with_multi_workers=(
     "Pkg"
 )
+# These tests run with a single worker
+declare -a tests_with_single_worker=(
+    "SparseArrays",
+    "LinearAlgebra"
+)
 
 stdlib_path=$JULIA_PATH/usr/share/julia/stdlib
 
@@ -48,11 +53,17 @@ do
         fi
 
         if [[ "${tests_with_multi_workers[@]}" =~ "$test" ]]; then
-            echo "-> Run multi-threads"
+            echo "-> Run multi threaded"
             ci_run_jl_test $test 2
             continue
         fi
 
+        if [[ "${tests_with_single_worker[@]}" =~ "$test" ]]; then
+            echo "-> Run single threaded"
+            ci_run_jl_test $test 1
+            continue
+        fi
+
         ci_run_jl_test $test
     fi
 done

.github/workflows/binding-tests.yml

@@ -0,0 +1,56 @@
+name: Test Julia Binding
+
+on:
+  workflow_call:
+    inputs:
+      gc_plan:
+        required: true
+        type: string
+
+jobs:
+  build-debug:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup environments
+        run: |
+          ./.github/scripts/ci-checkout.sh
+          ./.github/scripts/ci-setup.sh
+      - name: Build Julia (Debug)
+        run: |
+          ./.github/scripts/ci-build.sh debug ${{ inputs.gc_plan }}
+      - name: Style check
+        run: |
+          ./.github/scripts/ci-style.sh
+
+  build-test-other:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 360
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup environments
+        run: |
+          ./.github/scripts/ci-checkout.sh
+          ./.github/scripts/ci-setup.sh
+      - name: Build Julia (Release)
+        run: |
+          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }}
+      - name: Test Julia
+        run: |
+          ./.github/scripts/ci-test-other.sh
+
+  build-test-stdlib:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 360
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup environments
+        run: |
+          ./.github/scripts/ci-checkout.sh
+          ./.github/scripts/ci-setup.sh
+      - name: Build Julia (Release)
+        run: |
+          ./.github/scripts/ci-build.sh release ${{ inputs.gc_plan }}
+      - name: Test Julia
+        run: |
+          ./.github/scripts/ci-test-stdlib.sh

.github/workflows/ci.yml

@@ -12,49 +12,11 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-debug:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup environments
-        run: |
-          ./.github/scripts/ci-checkout.sh
-          ./.github/scripts/ci-setup.sh
-      - name: Build Julia (Debug)
-        run: |
-          ./.github/scripts/ci-build.sh debug
-      - name: Style check
-        run: |
-          ./.github/scripts/ci-style.sh
-
-  build-test-other:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 360
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup environments
-        run: |
-          ./.github/scripts/ci-checkout.sh
-          ./.github/scripts/ci-setup.sh
-      - name: Build Julia (Release)
-        run: |
-          ./.github/scripts/ci-build.sh release
-      - name: Test Julia
-        run: |
-          ./.github/scripts/ci-test-other.sh
-
-  build-test-stdlib:
-    runs-on: ubuntu-22.04
-    timeout-minutes: 360
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup environments
-        run: |
-          ./.github/scripts/ci-checkout.sh
-          ./.github/scripts/ci-setup.sh
-      - name: Build Julia (Release)
-        run: |
-          ./.github/scripts/ci-build.sh release
-      - name: Test Julia
-        run: |
-          ./.github/scripts/ci-test-stdlib.sh
+  binding-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        gc_plan: [immix, stickyimmix]
+    uses: ./.github/workflows/binding-tests.yml
+    with:
+      gc_plan: ${{ matrix.gc_plan }}

julia/mmtk_julia.c

@@ -725,6 +725,8 @@ JL_DLLEXPORT void* get_stackbase(int16_t tid) {
     return ptls2->stackbase;
 }
 
+const bool PRINT_OBJ_TYPE = false;
+
 /** 
  * Corresponds to the function mark_loop in the original Julia GC. It
  * dispatches MMTk work for scanning internal pointers for the object obj.
@@ -749,6 +751,7 @@ JL_DLLEXPORT void scan_julia_obj(jl_value_t* obj, closure_pointer closure, Proce
     };
 
     if (vt == jl_simplevector_type) { // scanning a jl_simplevector_type object (inlining label `objarray_loaded` from mark_loop)
+        if (PRINT_OBJ_TYPE) { printf("scan_julia_obj %p: simple vector\n", obj); fflush(stdout); }
         size_t l = jl_svec_len(obj);
         jl_value_t **data = jl_svec_data(obj);
         jl_value_t **objary_begin = data;
@@ -757,6 +760,7 @@ JL_DLLEXPORT void scan_julia_obj(jl_value_t* obj, closure_pointer closure, Proce
             process_edge(closure, objary_begin);
         }
     } else if (vt->name == jl_array_typename) { // scanning a jl_array_typename object
+        if (PRINT_OBJ_TYPE) { printf("scan_julia_obj %p: array\n", obj); fflush(stdout); }
         jl_array_t *a = (jl_array_t*)obj;
         jl_array_flags_t flags = a->flags;
 
@@ -840,6 +844,7 @@ JL_DLLEXPORT void scan_julia_obj(jl_value_t* obj, closure_pointer closure, Proce
             return;
         }
     } else if (vt == jl_module_type) { // inlining label `module_binding` from mark_loop
+        if (PRINT_OBJ_TYPE) { printf("scan_julia_obj %p: module\n", obj); fflush(stdout); }
         jl_module_t *m = (jl_module_t*)obj;
         jl_svec_t *bindings = jl_atomic_load_relaxed(&m->bindings);
         jl_binding_t **table = (jl_binding_t**)jl_svec_data(bindings);
@@ -851,11 +856,15 @@ JL_DLLEXPORT void scan_julia_obj(jl_value_t* obj, closure_pointer closure, Proce
             if (b == (jl_binding_t*)jl_nothing)
                 continue;
 
+            if (PRINT_OBJ_TYPE) { printf(" - scan table: %p\n", begin); fflush(stdout); }
             process_edge(closure, begin);
         }
 
+        if (PRINT_OBJ_TYPE) { printf(" - scan parent: %p\n", &m->parent); fflush(stdout); }
         process_edge(closure, &m->parent);
+        if (PRINT_OBJ_TYPE) { printf(" - scan bindingkeyset: %p\n", &m->bindingkeyset); fflush(stdout); }
         process_edge(closure, &m->bindingkeyset);
+        if (PRINT_OBJ_TYPE) { printf(" - scan bindings: %p\n", &m->bindings); fflush(stdout); }
         process_edge(closure, &m->bindings);
 
         size_t nusings = m->usings.len;
@@ -865,6 +874,7 @@ JL_DLLEXPORT void scan_julia_obj(jl_value_t* obj, closure_pointer closure, Proce
 
             for (; objary_begin < objary_end; objary_begin += 1) {
                 jl_value_t *pnew_obj = *objary_begin;
+                if (PRINT_OBJ_TYPE) { printf(" - scan usings: %p\n", objary_begin); fflush(stdout); }
                 process_edge(closure, pnew_obj);
             }
         }
@@ -886,8 +896,10 @@ JL_DLLEXPORT void scan_julia_obj(jl_value_t* obj, closure_pointer closure, Proce
                 process_edge(closure, slot);
             }
     } else if (vt == jl_string_type) { // scanning a jl_string_type object
+        if (PRINT_OBJ_TYPE) { printf("scan_julia_obj %p: string\n", obj); fflush(stdout); }
         return;
     } else {  // scanning a jl_datatype object
+        if (PRINT_OBJ_TYPE) { printf("scan_julia_obj %p: datatype\n", obj); fflush(stdout); }
         if (vt == jl_weakref_type) {
             return;
         }

mmtk/Cargo.lock

@@ -10,7 +10,7 @@ edition = "2018"
 [package.metadata.julia]
 # Our CI matches the following line and extract mmtk/julia. If this line is updated, please check ci yaml files and make sure it works.
 julia_repo = "https://github.com/mmtk/julia.git"
-julia_version = "eb407eb68ea976df72c8e1cd6ec8607ff8d98fa5"
+julia_version = "e7e43f11687e7db62dc18e90f3ab9b24099539fd"
 
 [lib]
 crate-type = ["staticlib", "rlib", "dylib"]
@@ -28,8 +28,8 @@ lazy_static = "1.1"
 # These changes are safe:
 # - change branch
 # - change repo name
-# But other changes including adding/removing whitespaces in commented lines may break the CI.
-mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "e309f9351dff9cd0c8f3f590002c6edafd158c82" }
+# But other changes including adding/removing whitespaces in commented lines may break the CI
+mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "7b06ead8bffc8a33b059919d440cb0b9812359b9" }
 # Uncomment the following to build locally
 # mmtk = { path = "../repos/mmtk-core" }
 log = {version = "0.4", features = ["max_level_trace", "release_max_level_off"] }
@@ -40,13 +40,16 @@ chrono = "*"
 # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" }
 
 [features]
-default = []
+default = ["mmtk/malloc_counted_size", "scan_obj_c", "mmtk/vm_space"]
+
+# Plans
 nogc = []
-semispace = []
-gencopy = []
-pageprotect = []
-malloc_counted_size = ["mmtk/malloc_counted_size"]
-immix = ["malloc_counted_size", "mmtk/immix_non_moving", "mmtk/immix_smaller_block", "scan_obj_c", "mmtk/vm_space"]
-marksweep = ["malloc_counted_size", "scan_obj_c"]
+immix = ["non_moving_immix"]
+stickyimmix = ["non_moving_immix"]
+marksweep = []
+
+# TODO remove this when we properly support moving
+non_moving_immix = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"]
+
 # FIXME update and use rust object scanner as default for immix
 scan_obj_c = []

mmtk/Cargo.toml

@@ -56,14 +56,19 @@ extern bool is_mapped_address(void* addr);
 extern void modify_check(void* ref);
 extern int object_is_managed_by_mmtk(void* addr);
 extern void runtime_panic(void);
+extern void unreachable(void);
 
 extern void mmtk_set_vm_space(void* addr, size_t size);
+extern void mmtk_immortal_region_post_alloc(void* addr, size_t size);
 
 // Write barriers
 extern void mmtk_memory_region_copy(MMTk_Mutator mutator, void* src_obj, void* src_addr, void* dst_obj, void* dst_addr, size_t size);
 extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, const void* src, const void* target);
-extern uint8_t mmtk_needs_write_barrier(void);
-
+extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, const void* src, const void* target);
+extern const uint8_t MMTK_NEEDS_WRITE_BARRIER;
+extern const uint8_t NO_BARRIER;
+extern const uint8_t OBJECT_BARRIER;
+extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS;
 /**
  * Tracing
  */
@@ -121,7 +126,7 @@ extern void gc_init(long long min_heap_size, long long max_heap_size, Julia_Upca
 extern bool will_never_move(void* object);
 extern bool process(char* name, char* value);
 extern void scan_region(void);
-extern void handle_user_collection_request(void *tls);
+extern void handle_user_collection_request(void *tls, uint8_t collection);
 extern void initialize_collection(void* tls);
 extern void enable_collection(void);
 extern void disable_collection(void);