Merge pull request #8090 from frej/frej/destructive-tuple-update

Destructive tuple update

OTP-18972

bootstrap/lib/compiler/ebin/compiler.app

@@ -50,6 +50,7 @@
              beam_ssa_private_append,
              beam_ssa_recv,
              beam_ssa_share,
+             beam_ssa_ss,
              beam_ssa_throw,
              beam_ssa_type,
 	     beam_trim,

erts/emulator/beam/atom.names

@@ -373,6 +373,7 @@ atom info_trap
 atom inherit
 atom init
 atom initial_call
+atom inplace
 atom input
 atom integer
 atom internal

erts/emulator/beam/emu/generators.tab

@@ -1026,15 +1026,20 @@ gen.create_bin(Fail, Alloc, Live, Unit, Dst, N, Segments) {
     return op;
 }
 
-gen.update_record(Size, Src, Dst, N, Updates) {
+gen.update_record(Hint, Size, Src, Dst, N, Updates) {
     BeamOp *begin, *prev;
     Sint count, i;
 
+    ASSERT(Hint.type == TAG_a);
     ASSERT(Size.type == TAG_u && Size.val < SCRATCH_X_REG);
     ASSERT(N.type == TAG_u && !(N.val % 2) && (N.val / 2) <= Size.val);
 
     $NewBeamOp(S, begin);
-    $BeamOpNameArity(begin, i_update_record, 5);
+    if (Hint.val == am_inplace) {
+        $BeamOpNameArity(begin, i_update_record_in_place, 5);
+    } else {
+        $BeamOpNameArity(begin, i_update_record_copy, 5);
+    }
 
     begin->a[0] = Size;
     begin->a[1] = Src;
@@ -1047,6 +1052,7 @@ gen.update_record(Size, Src, Dst, N, Updates) {
 
     for (i = 2; i < count; i += 2) {
         BeamOp *next;
+        int same_reg;
 
         $NewBeamOp(S, next);
         $BeamOpNameArity(next, i_update_record_continue, 2);
@@ -1056,10 +1062,19 @@ gen.update_record(Size, Src, Dst, N, Updates) {
         next->a[0].type = TAG_u;
         next->a[0].val = (Size.val + 1) - Updates[i].val;
 
+        if (Updates[i + 1].type != Dst.type) {
+            same_reg = 0;
+        } else if (Dst.type == TAG_x || Dst.type == TAG_y) {
+            /* We must not compare the type indices (if any). */
+            same_reg = (Updates[i + 1].val & REG_MASK) == (Dst.val & REG_MASK);
+        } else {
+            same_reg = 1;
+        }
+
         /* The first instruction overwrites the destination register after
          * stashing its contents to SCRATCH_X_REG, so all updates must be
          * rewritten accordingly. */
-        if (Updates[i + 1].type == Dst.type && Updates[i + 1].val == Dst.val) {
+        if (same_reg) {
             next->a[1].type = TAG_x;
             next->a[1].val = SCRATCH_X_REG;
         } else {
@@ -1072,6 +1087,16 @@ gen.update_record(Size, Src, Dst, N, Updates) {
         prev = next;
     }
 
+    if (Hint.val == am_inplace) {
+        BeamOp *next;
+
+        $NewBeamOp(S, next);
+        $BeamOpNameArity(next, i_update_record_in_place_done, 0);
+
+        next->next = NULL;
+        prev->next = next;
+    }
+
     return begin;
 }
 

erts/emulator/beam/emu/instrs.tab

@@ -782,7 +782,7 @@ self(Dst) {
     $Dst = c_p->common.id;
 }
 
-i_update_record(Size, Src, Dst, Offset, Element) {
+i_update_record_copy(Size, Src, Dst, Offset, Element) {
     Eterm *untagged_source = tuple_val($Src);
     Uint size_on_heap = $Size + 1;
 
@@ -801,11 +801,47 @@ i_update_record(Size, Src, Dst, Offset, Element) {
     HTOP += size_on_heap;
 }
 
+i_update_record_in_place(Size, Src, Dst, Offset, Element) {
+    Eterm *untagged_source = tuple_val($Src);
+    Uint size_on_heap = $Size + 1;
+
+    if (c_p->high_water <= untagged_source && untagged_source < HTOP) {
+        /* It is safe to overwrite the old record. */
+        LIGHT_SWAPOUT;
+
+        untagged_source[$Offset] = $Element;
+
+        /* We stash the contents of the destination register in SCRATCH_X_REG in
+         * case it's used in subsequent `i_update_record_continue` instructions.
+         * The updates have been rewritten accordingly. */
+        reg[SCRATCH_X_REG] = $Dst;
+        $Dst = $Src;
+
+        HTOP = untagged_source + size_on_heap;
+    } else {
+        /* It would be unsafe to to overwrite the old record, because
+         * that could cause a term on the old generation to point to
+         * the young generation. */
+        sys_memcpy(HTOP, untagged_source, size_on_heap * sizeof(Eterm));
+        HTOP[$Offset] = $Element;
+
+        reg[SCRATCH_X_REG] = $Dst;
+        $Dst = make_tuple(HTOP);
+
+        HTOP += size_on_heap;
+        LIGHT_SWAPOUT;
+    }
+}
+
 i_update_record_continue(OffsetFromEnd, Element) {
     Sint offset = -(Sint)$OffsetFromEnd;
     HTOP[offset] = $Element;
 }
 
+i_update_record_in_place_done() {
+    LIGHT_SWAPIN;
+}
+
 set_tuple_element(Element, Tuple, Offset) {
     Eterm* p;
 

erts/emulator/beam/emu/ops.tab

@@ -1730,9 +1730,18 @@ recv_marker_use S
 #
 
 update_record Hint=a Size=u Src=s Dst=d N=u Updates=* =>
-    update_record(Size, Src, Dst, N, Updates)
+    update_record(Hint, Size, Src, Dst, N, Updates)
+
+i_update_record_copy Size=u Src=c Dst=xy Offset=u Element=s =>
+    move Src x | i_update_record_copy Size x Dst Offset Element
+
+i_update_record_in_place Size=u Src=c Dst=xy Offset=u Element=s =>
+    move Src x | i_update_record_in_place Size x Dst Offset Element
+
+i_update_record_copy t xy xy t s
+i_update_record_in_place t xy xy t s
 
-i_update_record Size=u Src=c Dst=xy Offset=u Element=s =>
-    move Src x | i_update_record Size x Dst Offset Element
-i_update_record t xy xy t s
 i_update_record_continue t s
+
+i_update_record_in_place_done
+

erts/emulator/beam/jit/arm/beam_asm.hpp

@@ -1736,8 +1736,24 @@ class BeamModuleAssembler : public BeamAssembler,
         a.cmp(gp, tmp.reg);
     }
 
+    void safe_str(a64::Gp gp, arm::Mem mem) {
+        size_t abs_offset = std::abs(mem.offset());
+        auto offset = mem.offset();
+
+        ASSERT(mem.hasBaseReg() && !mem.hasIndex());
+        ASSERT(gp.isGpX());
+
+        if (abs_offset <= sizeof(Eterm) * MAX_LDR_STR_DISPLACEMENT) {
+            a.str(gp, mem);
+        } else {
+            add(SUPER_TMP, a64::GpX(mem.baseId()), offset);
+            a.str(gp, a64::Mem(SUPER_TMP));
+        }
+    }
+
     void safe_stp(a64::Gp gp1,
                   a64::Gp gp2,
+
                   const ArgVal &Dst1,
                   const ArgVal &Dst2) {
         ASSERT(ArgVal::memory_relation(Dst1, Dst2) ==

erts/emulator/beam/jit/arm/instr_common.cpp

@@ -991,6 +991,88 @@ void BeamModuleAssembler::emit_update_record(const ArgAtom &Hint,
     flush_var(destination);
 }
 
+void BeamModuleAssembler::emit_update_record_in_place(
+        const ArgWord &TupleSize,
+        const ArgSource &Src,
+        const ArgRegister &Dst,
+        const ArgWord &UpdateCount,
+        const Span<ArgVal> &updates) {
+    bool all_safe = true;
+    ArgSource maybe_immediate = ArgNil();
+    const size_t size_on_heap = TupleSize.get() + 1;
+
+    ASSERT(UpdateCount.get() == updates.size());
+    ASSERT((UpdateCount.get() % 2) == 0);
+
+    ASSERT(size_on_heap > 2);
+
+    auto destination = init_destination(Dst, ARG1);
+    auto src = load_source(Src, ARG2);
+
+    a64::Gp untagged_src = ARG3;
+    emit_untag_ptr(untagged_src, src.reg);
+
+    for (size_t i = 0; i < updates.size(); i += 2) {
+        const auto &value = updates[i + 1].as<ArgSource>();
+        if (!(always_immediate(value) || value.isLiteral())) {
+            all_safe = false;
+            if (maybe_immediate.isNil() &&
+                always_one_of<BeamTypeId::MaybeImmediate>(value)) {
+                maybe_immediate = value;
+            } else {
+                maybe_immediate = ArgNil();
+                break;
+            }
+        }
+    }
+
+    if (all_safe) {
+        comment("skipped copy fallback because all new values are safe");
+    } else {
+        Label update = a.newLabel();
+
+        if (!maybe_immediate.isNil()) {
+            auto value = load_source(maybe_immediate, ARG5);
+            emit_is_not_boxed(update, value.reg);
+        }
+
+        a.ldr(ARG4, arm::Mem(c_p, offsetof(Process, high_water)));
+        a.cmp(untagged_src, HTOP);
+        a.ccmp(untagged_src, ARG4, imm(NZCV::kNone), imm(arm::CondCode::kLO));
+        a.b_hs(update);
+
+        emit_copy_words_increment(untagged_src, HTOP, size_on_heap);
+        sub(untagged_src, HTOP, size_on_heap * sizeof(Eterm));
+
+        a.bind(update);
+    }
+
+    for (size_t i = 0; i < updates.size(); i += 2) {
+        const auto next_index = updates[i].as<ArgWord>().get();
+        const auto &next_value = updates[i + 1].as<ArgSource>();
+        arm::Mem mem(untagged_src, next_index * sizeof(Eterm));
+
+        if (i + 2 < updates.size()) {
+            const auto adjacent_index = updates[i + 2].as<ArgWord>().get();
+            const auto &adjacent_value = updates[i + 3].as<ArgSource>();
+
+            if (adjacent_index == next_index + 1) {
+                auto [first, second] =
+                        load_sources(next_value, TMP1, adjacent_value, TMP2);
+                safe_stp(first.reg, second.reg, mem);
+                i += 2;
+                continue;
+            }
+        }
+
+        auto value = load_source(next_value, TMP1);
+        safe_str(value.reg, mem);
+    }
+
+    a.add(destination.reg, untagged_src, TAG_PRIMARY_BOXED);
+    flush_var(destination);
+}
+
 void BeamModuleAssembler::emit_set_tuple_element(const ArgSource &Element,
                                                  const ArgRegister &Tuple,
                                                  const ArgWord &Offset) {

erts/emulator/beam/jit/arm/ops.tab

@@ -1484,4 +1484,8 @@ i_lambda_trampoline F f W W
 # OTP 26
 #
 
+update_record a==am_inplace Size Src=d Dst N Updates=* =>
+    update_record_in_place Size Src Dst N Updates
+
 update_record a I s d I *
+update_record_in_place I s d I *

erts/emulator/beam/jit/x86/instr_common.cpp

@@ -984,6 +984,102 @@ void BeamModuleAssembler::emit_update_record(const ArgAtom &Hint,
     mov_arg(Dst, RET);
 }
 
+void BeamModuleAssembler::emit_update_record_in_place(
+        const ArgWord &TupleSize,
+        const ArgSource &Src,
+        const ArgRegister &Dst,
+        const ArgWord &UpdateCount,
+        const Span<ArgVal> &updates) {
+    bool all_safe = true;
+    ArgSource maybe_immediate = ArgNil();
+    const size_t size_on_heap = TupleSize.get() + 1;
+
+    ASSERT(UpdateCount.get() == updates.size());
+    ASSERT((UpdateCount.get() % 2) == 0);
+
+    ASSERT(size_on_heap > 2);
+
+    for (size_t i = 0; i < updates.size(); i += 2) {
+        const auto &value = updates[i + 1].as<ArgSource>();
+        if (!(always_immediate(value) || value.isLiteral())) {
+            all_safe = false;
+            if (maybe_immediate.isNil() &&
+                always_one_of<BeamTypeId::MaybeImmediate>(value)) {
+                maybe_immediate = value;
+            } else {
+                maybe_immediate = ArgNil();
+                break;
+            }
+        }
+    }
+
+    x86::Gp tagged_ptr = RET;
+
+    mov_arg(tagged_ptr, Src);
+
+#if defined(DEBUG) && defined(TAG_LITERAL_PTR)
+    /* The compiler guarantees that the tuple is not a literal. */
+    {
+        Label not_literal = a.newLabel();
+
+        a.test(tagged_ptr, imm(TAG_LITERAL_PTR));
+        a.short_().je(not_literal);
+        a.ud2();
+
+        a.bind(not_literal);
+    }
+#endif
+
+    if (all_safe) {
+        comment("skipped copy fallback because all new values are safe");
+    } else {
+        Label update = a.newLabel();
+
+        if (!maybe_immediate.isNil()) {
+            mov_arg(ARG4, maybe_immediate);
+            preserve_cache([&]() {
+                emit_is_not_boxed(update, ARG4, dShort);
+            });
+        }
+
+        preserve_cache(
+                [&]() {
+                    Label copy = a.newLabel();
+
+                    a.mov(ARG1, x86::Mem(c_p, offsetof(Process, high_water)));
+                    a.cmp(tagged_ptr, HTOP);
+                    a.short_().jae(copy);
+
+                    a.cmp(tagged_ptr, ARG1);
+                    a.short_().jae(update);
+
+                    a.bind(copy);
+                    emit_copy_words(emit_boxed_val(tagged_ptr, 0),
+                                    x86::qword_ptr(HTOP, 0),
+                                    size_on_heap,
+                                    ARG1);
+                    a.lea(RET, x86::qword_ptr(HTOP, TAG_PRIMARY_BOXED));
+                    a.add(HTOP, imm(size_on_heap * sizeof(Eterm)));
+
+                    a.bind(update);
+                },
+                ARG1);
+    }
+
+    for (size_t i = 0; i < updates.size(); i += 2) {
+        const auto next_index = updates[i].as<ArgWord>().get();
+        const auto &next_value = updates[i + 1].as<ArgSource>();
+
+        ASSERT(next_index > 0);
+
+        mov_arg(emit_boxed_val(RET, next_index * sizeof(Eterm)),
+                next_value,
+                ARG1);
+    }
+
+    mov_arg(Dst, RET);
+}
+
 void BeamModuleAssembler::emit_set_tuple_element(const ArgSource &Element,
                                                  const ArgRegister &Tuple,
                                                  const ArgWord &Offset) {

erts/emulator/beam/jit/x86/ops.tab

@@ -1413,4 +1413,7 @@ recv_marker_use S
 # OTP 26
 #
 
+update_record a==am_inplace Size Src=d Dst N Updates=* =>
+    update_record_in_place Size Src Dst N Updates
 update_record a I s d I *
+update_record_in_place I s d I *