More tests for Marshal.Realloc{HGlobal|CoTaskMem} (#41910)

- The corner case behavior of Marshal.Realloc method is non-standard and differs between the two realloc methods. Capture the current behavior in tests.
- Add temporary workarounds for Windows vs. Unix differences
- Disable the tests on Mono for now

Author: jkotas

src/libraries/System.Runtime.InteropServices/tests/System/Runtime/InteropServices/Marshal/ReAllocCoTaskMemTests.cs

@@ -38,5 +38,61 @@ public void ReAllocCoTaskMem_Invoke_DataCopied()
                 Marshal.FreeCoTaskMem(p2);
             }
         }
+
+        [InlineData(0)]
+        [InlineData(1)]
+        [InlineData(100)]
+        [Theory]
+        [SkipOnMono("Behavior differences on Mono")]
+        public void ReAllocCoTaskMem_PositiveSize(int size)
+        {
+            IntPtr p = Marshal.ReAllocCoTaskMem(IntPtr.Zero, size);
+            Assert.NotEqual(IntPtr.Zero, p);
+
+            IntPtr p1 = Marshal.ReAllocCoTaskMem(p, size + 1);
+            Assert.NotEqual(IntPtr.Zero, p1);
+
+            IntPtr p2 = Marshal.ReAllocCoTaskMem(p1, 0);
+
+            // TODO: Behavior differs between platforms currently
+            if (PlatformDetection.IsWindows)
+            {
+                Assert.Equal(IntPtr.Zero, p2);
+            }
+            else
+            {
+                Assert.NotEqual(IntPtr.Zero, p2);
+                Marshal.FreeCoTaskMem(p2);
+            }
+        }
+
+        [Fact]
+        [OuterLoop]
+        [SkipOnMono("Behavior differences on Mono")]
+        public void ReAllocCoTaskMem_NegativeSize_ThrowsOutOfMemoryException()
+        {
+            // -1 is treated as (uint)-1 by ReAllocCoTaskMem. The allocation may succeed on 64-bit machines.
+
+            try
+            {
+                IntPtr p1 = Marshal.ReAllocCoTaskMem(IntPtr.Zero, -1);
+                Assert.NotEqual(IntPtr.Zero, p1);
+                Marshal.FreeCoTaskMem(p1);
+            }
+            catch (OutOfMemoryException)
+            {
+            }
+
+            IntPtr p2 = Marshal.AllocCoTaskMem(1);
+            try
+            {
+                p2 = Marshal.ReAllocCoTaskMem(p2, -1);
+                Assert.NotEqual(IntPtr.Zero, p2);
+            }
+            catch (OutOfMemoryException)
+            {
+            }
+            Marshal.FreeCoTaskMem(p2);
+        }
     }
 }

src/libraries/System.Runtime.InteropServices/tests/System/Runtime/InteropServices/Marshal/ReAllocHGlobalTests.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using Microsoft.Win32.SafeHandles;
 using Xunit;
 
 namespace System.Runtime.InteropServices.Tests
@@ -38,5 +39,52 @@ public void ReAllocHGlobal_Invoke_DataCopied()
                 Marshal.FreeHGlobal(p2);
             }
         }
+
+        [InlineData(0)]
+        [InlineData(1)]
+        [InlineData(100)]
+        [Theory]
+        [SkipOnMono("Behavior differences on Mono")]
+        public void ReAllocHGlobal_PositiveSize(int size)
+        {
+            IntPtr p;
+            // TODO: Behavior differs between platforms currently
+            if (PlatformDetection.IsWindows)
+            {
+                p = Marshal.AllocHGlobal(size);
+            }
+            else
+            {
+                p = Marshal.ReAllocHGlobal(IntPtr.Zero, (IntPtr)size);
+            }
+            Assert.NotEqual(IntPtr.Zero, p);
+
+            IntPtr p1 = Marshal.ReAllocHGlobal(p, (IntPtr)(size + 1));
+            Assert.NotEqual(IntPtr.Zero, p1);
+
+            // ReAllocHGlobal never returns null, even for 0 size (different from standard C/C++ realloc)
+            IntPtr p2 = Marshal.ReAllocHGlobal(p1, IntPtr.Zero);
+            Assert.NotEqual(IntPtr.Zero, p2);
+
+            Marshal.FreeHGlobal(p2);
+        }
+
+        [Fact]
+        [SkipOnMono("Behavior differences on Mono")]
+        public void ReAllocHGlobal_NegativeSize_ThrowsOutOfMemoryException()
+        {
+            // TODO: Behavior differs between platforms currently
+            if (PlatformDetection.IsWindows)
+            {
+                // ReAllocHGlobal always throws when the original pointer is null (different from standard C/C++ realloc)
+                Assert.Throws<OutOfMemoryException>(() => Marshal.ReAllocHGlobal(IntPtr.Zero, IntPtr.Zero));
+                Assert.Throws<OutOfMemoryException>(() => Marshal.ReAllocHGlobal(IntPtr.Zero, (IntPtr)1));
+            }
+            Assert.Throws<OutOfMemoryException>(() => Marshal.ReAllocHGlobal(IntPtr.Zero, (IntPtr)(-1)));
+
+            IntPtr p = Marshal.AllocHGlobal((IntPtr)1);
+            Assert.Throws<OutOfMemoryException>(() => Marshal.ReAllocHGlobal(p, (IntPtr)(-1)));
+            Marshal.FreeHGlobal(p);
+        }
     }
 }