diff --git a/base/genericmemory.jl b/base/genericmemory.jl
index 5f55de91ff475..f814aa4d84bdd 100644
--- a/base/genericmemory.jl
+++ b/base/genericmemory.jl
@@ -85,17 +85,15 @@ function _unsetindex!(A::MemoryRef{T}) where T
     arrayelem = datatype_arrayelem(MemT)
     elsz = datatype_layoutsize(MemT)
     isbits = 0; isboxed = 1; isunion = 2
-    arrayelem == isbits && return A
+    arrayelem == isbits && datatype_pointerfree(T::DataType) && return A
     t = @_gc_preserve_begin mem
     p = Ptr{Ptr{Cvoid}}(@inbounds pointer(A))
     if arrayelem == isboxed
         Intrinsics.atomic_pointerset(p, C_NULL, :monotonic)
     elseif arrayelem != isunion
-        if !datatype_pointerfree(T::DataType)
-            for j = 1:Core.sizeof(Ptr{Cvoid}):elsz
-                # XXX: this violates memory ordering, since it writes more than one C_NULL to each
-                Intrinsics.atomic_pointerset(p + j - 1, C_NULL, :monotonic)
-            end
+        for j = 1:Core.sizeof(Ptr{Cvoid}):elsz
+            # XXX: this violates memory ordering, since it writes more than one C_NULL to each
+            Intrinsics.atomic_pointerset(p + j - 1, C_NULL, :monotonic)
         end
     end
     @_gc_preserve_end t