Add StatementKind::CopyNonOverlapping

compiler/rustc_codegen_cranelift/src/base.rs

@@ -832,6 +832,27 @@ fn codegen_stmt<'tcx>(
             }
         }
         StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
+        StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+          src,
+          dst,
+          count,
+        }) => {
+            let dst = codegen_operand(fx, dst);
+            let pointee = dst
+              .layout()
+              .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
+              .expect("Expected pointer");
+            let dst = dst.load_scalar(fx);
+            let src = codegen_operand(fx, src).load_scalar(fx);
+            let count = codegen_operand(fx, count).load_scalar(fx);
+            let elem_size: u64 = pointee.size.bytes();
+            let bytes = if elem_size != 1 {
+               fx.bcx.ins().imul_imm(count, elem_size as i64)
+            } else {
+               count
+            };
+            fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
+        }
     }
 }
 

compiler/rustc_codegen_ssa/src/lib.rs

@@ -9,6 +9,7 @@
 #![feature(or_patterns)]
 #![feature(associated_type_bounds)]
 #![recursion_limit = "256"]
+#![feature(box_syntax)]
 
 //! This crate contains codegen code that is used by all codegen backends (LLVM and others).
 //! The backend-agnostic functions of this crate use functions defined in various traits that

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -641,67 +641,73 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             return;
         }
 
-        if intrinsic.is_some() && intrinsic != Some(sym::drop_in_place) {
-            let intrinsic = intrinsic.unwrap();
-            let dest = match ret_dest {
-                _ if fn_abi.ret.is_indirect() => llargs[0],
-                ReturnDest::Nothing => {
-                    bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret)))
-                }
-                ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval,
-                ReturnDest::DirectOperand(_) => {
-                    bug!("Cannot use direct operand with an intrinsic call")
-                }
-            };
+        match intrinsic {
+            None | Some(sym::drop_in_place) => {}
+            Some(sym::copy_nonoverlapping) => unreachable!(),
+            Some(intrinsic) => {
+                let dest = match ret_dest {
+                    _ if fn_abi.ret.is_indirect() => llargs[0],
+                    ReturnDest::Nothing => {
+                        bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret)))
+                    }
+                    ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval,
+                    ReturnDest::DirectOperand(_) => {
+                        bug!("Cannot use direct operand with an intrinsic call")
+                    }
+                };
 
-            let args: Vec<_> = args
-                .iter()
-                .enumerate()
-                .map(|(i, arg)| {
-                    // The indices passed to simd_shuffle* in the
-                    // third argument must be constant. This is
-                    // checked by const-qualification, which also
-                    // promotes any complex rvalues to constants.
-                    if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
-                        if let mir::Operand::Constant(constant) = arg {
-                            let c = self.eval_mir_constant(constant);
-                            let (llval, ty) = self.simd_shuffle_indices(
-                                &bx,
-                                constant.span,
-                                constant.literal.ty,
-                                c,
-                            );
-                            return OperandRef { val: Immediate(llval), layout: bx.layout_of(ty) };
-                        } else {
-                            span_bug!(span, "shuffle indices must be constant");
+                let args: Vec<_> = args
+                    .iter()
+                    .enumerate()
+                    .map(|(i, arg)| {
+                        // The indices passed to simd_shuffle* in the
+                        // third argument must be constant. This is
+                        // checked by const-qualification, which also
+                        // promotes any complex rvalues to constants.
+                        if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
+                            if let mir::Operand::Constant(constant) = arg {
+                                let c = self.eval_mir_constant(constant);
+                                let (llval, ty) = self.simd_shuffle_indices(
+                                    &bx,
+                                    constant.span,
+                                    constant.literal.ty,
+                                    c,
+                                );
+                                return OperandRef {
+                                    val: Immediate(llval),
+                                    layout: bx.layout_of(ty),
+                                };
+                            } else {
+                                span_bug!(span, "shuffle indices must be constant");
+                            }
                         }
-                    }
 
-                    self.codegen_operand(&mut bx, arg)
-                })
-                .collect();
+                        self.codegen_operand(&mut bx, arg)
+                    })
+                    .collect();
+
+                Self::codegen_intrinsic_call(
+                    &mut bx,
+                    *instance.as_ref().unwrap(),
+                    &fn_abi,
+                    &args,
+                    dest,
+                    span,
+                );
 
-            Self::codegen_intrinsic_call(
-                &mut bx,
-                *instance.as_ref().unwrap(),
-                &fn_abi,
-                &args,
-                dest,
-                span,
-            );
+                if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
+                    self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval);
+                }
 
-            if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
-                self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval);
-            }
+                if let Some((_, target)) = *destination {
+                    helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
+                    helper.funclet_br(self, &mut bx, target);
+                } else {
+                    bx.unreachable();
+                }
 
-            if let Some((_, target)) = *destination {
-                helper.maybe_sideeffect(self.mir, &mut bx, &[target]);
-                helper.funclet_br(self, &mut bx, target);
-            } else {
-                bx.unreachable();
+                return;
             }
-
-            return;
         }
 
         // Split the rust-call tupled arguments off.

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -125,19 +125,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let offset = args[1].immediate();
                 bx.gep(ptr, &[offset])
             }
-
-            sym::copy_nonoverlapping => {
-                copy_intrinsic(
-                    bx,
-                    false,
-                    false,
-                    substs.type_at(0),
-                    args[1].immediate(),
-                    args[0].immediate(),
-                    args[2].immediate(),
-                );
-                return;
-            }
             sym::copy => {
                 copy_intrinsic(
                     bx,

compiler/rustc_codegen_ssa/src/mir/statement.rs

@@ -115,6 +115,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.codegen_coverage(&mut bx, coverage.clone());
                 bx
             }
+            mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
+                ref src,
+                ref dst,
+                ref count,
+            }) => {
+                let dst_val = self.codegen_operand(&mut bx, dst);
+                let src_val = self.codegen_operand(&mut bx, src);
+                let count = self.codegen_operand(&mut bx, count).immediate();
+                let pointee_layout = dst_val
+                    .layout
+                    .pointee_info_at(&mut bx, rustc_target::abi::Size::ZERO)
+                    .expect("Expected pointer");
+                let bytes = bx.mul(count, bx.const_usize(pointee_layout.size.bytes()));
+
+                let align = pointee_layout.align;
+                let dst = dst_val.immediate();
+                let src = src_val.immediate();
+                bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty());
+                bx
+            }
             mir::StatementKind::FakeRead(..)
             | mir::StatementKind::Retag { .. }
             | mir::StatementKind::AscribeUserType(..)

compiler/rustc_middle/src/mir/mod.rs

@@ -1541,6 +1541,11 @@ pub enum StatementKind<'tcx> {
     /// counter varible at runtime, each time the code region is executed.
     Coverage(Box<Coverage>),
 
+    /// Denotes a call to the intrinsic function copy_overlapping, where `src_dst` denotes the
+    /// memory being read from and written to(one field to save memory), and size
+    /// indicates how many bytes are being copied over.
+    CopyNonOverlapping(Box<CopyNonOverlapping<'tcx>>),
+
     /// No-op. Useful for deleting instructions without affecting statement indices.
     Nop,
 }
@@ -1659,6 +1664,13 @@ impl Debug for Statement<'_> {
                     write!(fmt, "Coverage::{:?}", coverage.kind)
                 }
             }
+            CopyNonOverlapping(box crate::mir::CopyNonOverlapping {
+                ref src,
+                ref dst,
+                ref count,
+            }) => {
+                write!(fmt, "copy_nonoverlapping(src={:?}, dst={:?}, count={:?})", src, dst, count)
+            }
             Nop => write!(fmt, "nop"),
         }
     }
@@ -1670,6 +1682,14 @@ pub struct Coverage {
     pub code_region: Option<CodeRegion>,
 }
 
+#[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable)]
+pub struct CopyNonOverlapping<'tcx> {
+    pub src: Operand<'tcx>,
+    pub dst: Operand<'tcx>,
+    /// Number of elements to copy from src to dest, not bytes.
+    pub count: Operand<'tcx>,
+}
+
 ///////////////////////////////////////////////////////////////////////////
 // Places
 

compiler/rustc_middle/src/mir/visit.rs

@@ -436,6 +436,15 @@ macro_rules! make_mir_visitor {
                             location
                         )
                     }
+                    StatementKind::CopyNonOverlapping(box crate::mir::CopyNonOverlapping{
+                      ref $($mutability)? src,
+                      ref $($mutability)? dst,
+                      ref $($mutability)? count,
+                    }) => {
+                      self.visit_operand(src, location);
+                      self.visit_operand(dst, location);
+                      self.visit_operand(count, location)
+                    }
                     StatementKind::Nop => {}
                 }
             }

compiler/rustc_mir/src/borrow_check/invalidation.rs

@@ -92,6 +92,15 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                     self.consume_operand(location, input);
                 }
             }
+            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+                ref src,
+                ref dst,
+                ref count,
+            }) => {
+                self.consume_operand(location, src);
+                self.consume_operand(location, dst);
+                self.consume_operand(location, count);
+            }
             StatementKind::Nop
             | StatementKind::Coverage(..)
             | StatementKind::AscribeUserType(..)

compiler/rustc_mir/src/borrow_check/mod.rs

@@ -626,6 +626,15 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc
                     self.consume_operand(location, (input, span), flow_state);
                 }
             }
+
+            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+                ..
+            }) => {
+                span_bug!(
+                    span,
+                    "Unexpected CopyNonOverlapping, should only appear after lower_intrinsics",
+                )
+            }
             StatementKind::Nop
             | StatementKind::Coverage(..)
             | StatementKind::AscribeUserType(..)

compiler/rustc_mir/src/borrow_check/type_check/mod.rs

@@ -1520,6 +1520,12 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                     );
                 }
             }
+            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
+                ..
+            }) => span_bug!(
+                stmt.source_info.span,
+                "Unexpected StatementKind::CopyNonOverlapping, should only appear after lowering_intrinsics",
+            ),
             StatementKind::FakeRead(..)
             | StatementKind::StorageLive(..)
             | StatementKind::StorageDead(..)

compiler/rustc_mir/src/dataflow/impls/borrows.rs

@@ -305,6 +305,7 @@ impl<'tcx> dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
             | mir::StatementKind::Retag { .. }
             | mir::StatementKind::AscribeUserType(..)
             | mir::StatementKind::Coverage(..)
+            | mir::StatementKind::CopyNonOverlapping(..)
             | mir::StatementKind::Nop => {}
         }
     }

compiler/rustc_mir/src/dataflow/impls/storage_liveness.rs

@@ -149,6 +149,7 @@ impl<'mir, 'tcx> dataflow::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir,
             | StatementKind::FakeRead(..)
             | StatementKind::Nop
             | StatementKind::Retag(..)
+            | StatementKind::CopyNonOverlapping(..)
             | StatementKind::StorageLive(..) => {}
         }
     }

compiler/rustc_mir/src/dataflow/move_paths/builder.rs

@@ -318,6 +318,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             StatementKind::Retag { .. }
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
+            | StatementKind::CopyNonOverlapping(..)
             | StatementKind::Nop => {}
         }
     }

compiler/rustc_mir/src/interpret/intrinsics.rs

@@ -323,28 +323,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let result = Scalar::from_uint(truncated_bits, layout.size);
                 self.write_scalar(result, dest)?;
             }
-            sym::copy | sym::copy_nonoverlapping => {
-                let elem_ty = instance.substs.type_at(0);
-                let elem_layout = self.layout_of(elem_ty)?;
-                let count = self.read_scalar(&args[2])?.to_machine_usize(self)?;
-                let elem_align = elem_layout.align.abi;
-
-                let size = elem_layout.size.checked_mul(count, self).ok_or_else(|| {
-                    err_ub_format!("overflow computing total size of `{}`", intrinsic_name)
-                })?;
-                let src = self.read_scalar(&args[0])?.check_init()?;
-                let src = self.memory.check_ptr_access(src, size, elem_align)?;
-                let dest = self.read_scalar(&args[1])?.check_init()?;
-                let dest = self.memory.check_ptr_access(dest, size, elem_align)?;
-
-                if let (Some(src), Some(dest)) = (src, dest) {
-                    self.memory.copy(
-                        src,
-                        dest,
-                        size,
-                        intrinsic_name == sym::copy_nonoverlapping,
-                    )?;
-                }
+            sym::copy => {
+                self.copy(&args[0], &args[1], &args[2], /*nonoverlapping*/ false)?;
             }
             sym::offset => {
                 let ptr = self.read_scalar(&args[0])?.check_init()?;