Unnecessary stack writes when using write_volatile with arrays

[niluxv]

Labels: I-slow
Maybe this is an LLVM issue rather than a rustc issue, but std::ptr::write_volatile, when used with arrays, results in highly non-optimal assembly on x86_64.

For example

#[repr(align(8))]
pub struct B8 (
    [u8; 8]
);
impl B8 {
    const ZERO: Self = Self([0; 8]);
}

pub fn zeroize_b8(bytes: &mut B8) {
    let ptr: *mut B8 = bytes as *mut B8;
    unsafe {
        std::ptr::write_volatile(ptr, B8::ZERO);
    }
}

results in

playground::zeroize_b8:
	pushq	%rax
	movq	$0, (%rsp) # write 0u64 to the stack
	movq	(%rsp), %rax # read this 0u64 from the stack to rax
	movq	%rax, (%rdi) # perform the actual write operation
	popq	%rax
	retq

It first writes the value to the stack, than reads it from the stack into a register, and finially writes that register to the memory pointed at by ptr.
This takes 3 move instructions while we only need 1!

playground::zeroize_b8_handwritten_asm:
	movq	$0, (%rdi) # perform the actual write operation
	retq

Using a u64 instead of an eight byte array works fine, and only issues a single move. (In fact it emits asm identical to the handwritten asm above.)
Try it on playground

[niluxv]

It seems like the issue is on rustc's side. Looking at the generated LLVM IR we see that rust issues a volatile load from the stack, which forces LLVM to generate the assembly above.

; playground::zeroize_b8
; Function Attrs: nounwind nonlazybind uwtable
define void @_ZN10playground10zeroize_b817h04a795eb43fb6ecfE(%B8* align 8 dereferenceable(8) %bytes) unnamed_addr #0 personality i32 (i32, i32, i64, %"unwind::libunwind::_Unwind_Exception"*, %"unwind::libunwind::_Unwind_Context"*)* @rust_eh_personality {
start:
  %_5.sroa.0.i = alloca i64, align 8
  %_5.sroa.0.i.0.sroa_cast = bitcast i64* %_5.sroa.0.i to i8*
  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %_5.sroa.0.i.0.sroa_cast)
  store i64 0, i64* %_5.sroa.0.i, align 8
  %_5.sroa.0.0..sroa_cast.i = bitcast %B8* %bytes to i64*
  ; the volatile load occurs in the next line
  ; this shouldn't be volatile: the store later is volatile. That should be enough.
  %_5.sroa.0.i.0._5.sroa.0.0._5.sroa.0.0._5.sroa.0.0.copyload.i = load volatile i64, i64* %_5.sroa.0.i, align 8
  ; the store in the next line is volatile, which is what we want
  store volatile i64 %_5.sroa.0.i.0._5.sroa.0.0._5.sroa.0.0._5.sroa.0.0.copyload.i, i64* %_5.sroa.0.0..sroa_cast.i, align 8
  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %_5.sroa.0.i.0.sroa_cast)
  ret void
}

[clubby789]

Looking at the opt-pipeline, LLVM seems to be introducing the volatile on the stack load
@rustbot label +I-slow +A-llvm

[niluxv]

No, I think the volatility is there from the start (generated by the rustc codegen I guess). A debug build of the above code shows that rustc uses a volatile memcpy for core::intrinsics::volatile_store:

%B8 = type { [8 x i8] }

@0 = private unnamed_addr constant <{ [8 x i8] }> zeroinitializer, align 8

define void @_ZN4core3ptr14write_volatile17h0d514cc8a55c523bE(ptr %dst, i64 %0) unnamed_addr #0 !dbg !6 {
start:
  %1 = alloca i64, align 8
  %src = alloca %B8, align 8
  store i64 %0, ptr %1, align 8
  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %src, ptr align 8 %1, i64 8, i1 false)
  ; the memcpy below is volatile
  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 8, i1 true), !dbg !12
  ret void, !dbg !14
}

define void @_ZN7example10zeroize_b817hdf0aefd1fdb86108E(ptr align 8 %bytes) unnamed_addr #1 !dbg !15 {
start:
  %0 = alloca %B8, align 8
  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %0, ptr align 8 @0, i64 8, i1 false), !dbg !18
  %1 = load i64, ptr %0, align 8, !dbg !18
  call void @_ZN4core3ptr14write_volatile17h0d514cc8a55c523bE(ptr %bytes, i64 %1), !dbg !18
  ret void, !dbg !21
}

declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #2

attributes #0 = { inlinehint nonlazybind uwtable "probe-stack"="inline-asm" "target-cpu"="x86-64" }
attributes #1 = { nonlazybind uwtable "probe-stack"="inline-asm" "target-cpu"="x86-64" }
attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }

godbolt

But a volatile memcpy is volatile in both the load and the store. That results in the volatile load later on.

@rustbot label -A-llvm +A-codegen +A-intrinsics