Test interaction of unions with non-zero/niche-filling optimization

src/test/run-pass/union/union-nonzero.rs

@@ -0,0 +1,51 @@
+// run-pass
+#![allow(dead_code)]
+
+// Tests that unions aren't subject to unsafe non-zero/niche-filling optimizations.
+//
+// For example, if a union `U` can contain both a `&T` and a `*const T`, there's definitely no
+// bit-value that an `Option<U>` could reuse as `None`; this test makes sure that isn't done.
+//
+// Secondly, this tests the status quo (not a guarantee; subject to change!) to not apply such
+// optimizations to types containing unions even if they're theoretically possible. (discussion:
+// https://github.com/rust-lang/rust/issues/36394)
+//
+// Notably this nails down part of the behavior that `MaybeUninit` assumes: that a
+// `Option<MaybeUninit<&u8>>` does not take advantage of non-zero optimization, and thus is a safe
+// construct.
+
+use std::mem::{size_of, transmute};
+
+union U1<A: Copy> {
+    a: A,
+}
+
+union U2<A: Copy, B: Copy> {
+    a: A,
+    b: B,
+}
+
+// Option<E> uses a value other than 0 and 1 as None
+#[derive(Clone,Copy)]
+enum E {
+    A = 0,
+    B = 1,
+}
+
+fn main() {
+    // Unions do not participate in niche-filling/non-zero optimization...
+    assert!(size_of::<Option<U2<&u8, u8>>>() > size_of::<U2<&u8, u8>>());
+    assert!(size_of::<Option<U2<&u8, ()>>>() > size_of::<U2<&u8, ()>>());
+    assert!(size_of::<Option<U2<u8, E>>>() > size_of::<U2<u8, E>>());
+
+    // ...even when theoretically possible:
+    assert!(size_of::<Option<U1<&u8>>>() > size_of::<U1<&u8>>());
+    assert!(size_of::<Option<U2<&u8, &u8>>>() > size_of::<U2<&u8, &u8>>());
+
+    // The unused bits of the () variant can have any value.
+    let zeroed: U2<&u8, ()> = unsafe { transmute(std::ptr::null::<u8>()) };
+
+    if let None = Some(zeroed) {
+        panic!()
+    }
+}

src/test/ui/print_type_sizes/niche-filling.rs

@@ -57,6 +57,15 @@ pub enum Enum4<A, B, C, D> {
     Four(D)
 }
 
+pub union Union1<A: Copy> {
+    a: A,
+}
+
+pub union Union2<A: Copy, B: Copy> {
+    a: A,
+    b: B,
+}
+
 #[start]
 fn start(_: isize, _: *const *const u8) -> isize {
     let _x: MyOption<NonZeroU32> = Default::default();
@@ -69,5 +78,13 @@ fn start(_: isize, _: *const *const u8) -> isize {
     let _e: Enum4<(), char, (), ()> = Enum4::One(());
     let _f: Enum4<(), (), bool, ()> = Enum4::One(());
     let _g: Enum4<(), (), (), MyOption<u8>> = Enum4::One(());
+
+    // Unions do not currently participate in niche filling.
+    let _h: MyOption<Union2<NonZeroU32, u32>> = Default::default();
+
+    // ...even when theoretically possible.
+    let _i: MyOption<Union1<NonZeroU32>> = Default::default();
+    let _j: MyOption<Union2<NonZeroU32, NonZeroU32>> = Default::default();
+
     0
 }

src/test/ui/print_type_sizes/niche-filling.stdout

@@ -14,6 +14,21 @@ print-type-size         field `.post`: 2 bytes
 print-type-size         field `.pre`: 1 bytes
 print-type-size     variant `None`: 0 bytes
 print-type-size     end padding: 1 bytes
+print-type-size type: `MyOption<Union1<std::num::NonZeroU32>>`: 8 bytes, alignment: 4 bytes
+print-type-size     discriminant: 4 bytes
+print-type-size     variant `Some`: 4 bytes
+print-type-size         field `.0`: 4 bytes
+print-type-size     variant `None`: 0 bytes
+print-type-size type: `MyOption<Union2<std::num::NonZeroU32, std::num::NonZeroU32>>`: 8 bytes, alignment: 4 bytes
+print-type-size     discriminant: 4 bytes
+print-type-size     variant `Some`: 4 bytes
+print-type-size         field `.0`: 4 bytes
+print-type-size     variant `None`: 0 bytes
+print-type-size type: `MyOption<Union2<std::num::NonZeroU32, u32>>`: 8 bytes, alignment: 4 bytes
+print-type-size     discriminant: 4 bytes
+print-type-size     variant `Some`: 4 bytes
+print-type-size         field `.0`: 4 bytes
+print-type-size     variant `None`: 0 bytes
 print-type-size type: `NestedNonZero`: 8 bytes, alignment: 4 bytes
 print-type-size     field `.val`: 4 bytes
 print-type-size     field `.post`: 2 bytes
@@ -36,6 +51,17 @@ print-type-size type: `MyOption<std::num::NonZeroU32>`: 4 bytes, alignment: 4 by
 print-type-size     variant `Some`: 4 bytes
 print-type-size         field `.0`: 4 bytes
 print-type-size     variant `None`: 0 bytes
+print-type-size type: `Union1<std::num::NonZeroU32>`: 4 bytes, alignment: 4 bytes
+print-type-size     variant `Union1`: 4 bytes
+print-type-size         field `.a`: 4 bytes
+print-type-size type: `Union2<std::num::NonZeroU32, std::num::NonZeroU32>`: 4 bytes, alignment: 4 bytes
+print-type-size     variant `Union2`: 4 bytes
+print-type-size         field `.a`: 4 bytes
+print-type-size         field `.b`: 4 bytes, offset: 0 bytes, alignment: 4 bytes
+print-type-size type: `Union2<std::num::NonZeroU32, u32>`: 4 bytes, alignment: 4 bytes
+print-type-size     variant `Union2`: 4 bytes
+print-type-size         field `.a`: 4 bytes
+print-type-size         field `.b`: 4 bytes, offset: 0 bytes, alignment: 4 bytes
 print-type-size type: `std::num::NonZeroU32`: 4 bytes, alignment: 4 bytes
 print-type-size     field `.0`: 4 bytes
 print-type-size type: `Enum4<(), (), (), MyOption<u8>>`: 2 bytes, alignment: 1 bytes