Make HostModel PEUtils always read/write little endian (#92392)

This makes IsPEImage, GetWindowsGraphicalUserInterfaceBit, and SetWindowsGraphicalUserInterfaceBit in PEUtils explicitly read / write in little endian (per PE format) instead of whatever the system is.

Author: elinor-fung

src/installer/managed/Microsoft.NET.HostModel/AppHost/PEUtils.cs

@@ -1,8 +1,11 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
+using System.Buffers.Binary;
 using System.IO;
 using System.IO.MemoryMappedFiles;
+using System.Reflection.PortableExecutable;
 
 namespace Microsoft.NET.HostModel.AppHost
 {
@@ -15,29 +18,13 @@ public static class PEUtils
         /// <returns>true if the accessor represents a PE image, false otherwise.</returns>
         internal static unsafe bool IsPEImage(MemoryMappedViewAccessor accessor)
         {
-            byte* pointer = null;
+            if (accessor.Capacity < PEOffsets.DosStub.PESignatureOffset + sizeof(uint))
+                return false;
 
-            try
-            {
-                accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref pointer);
-                byte* bytes = pointer + accessor.PointerOffset;
-
-                // https://en.wikipedia.org/wiki/Portable_Executable
-                // Validate that we're looking at Windows PE file
-                if (((ushort*)bytes)[0] != PEOffsets.DosImageSignature
-                    || accessor.Capacity < PEOffsets.DosStub.PESignatureOffset + sizeof(uint))
-                {
-                    return false;
-                }
-                return true;
-            }
-            finally
-            {
-                if (pointer != null)
-                {
-                    accessor.SafeMemoryMappedViewHandle.ReleasePointer();
-                }
-            }
+            // https://en.wikipedia.org/wiki/Portable_Executable
+            // Validate that we're looking at Windows PE file
+            ushort signature = AsLittleEndian(accessor.ReadUInt16(0));
+            return signature == PEOffsets.DosImageSignature;
         }
 
         public static bool IsPEImage(string filePath)
@@ -60,40 +47,15 @@ public static bool IsPEImage(string filePath)
         /// <param name="accessor">The memory accessor which has the apphost file opened.</param>
         internal static unsafe void SetWindowsGraphicalUserInterfaceBit(MemoryMappedViewAccessor accessor)
         {
-            byte* pointer = null;
-
-            try
-            {
-                accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref pointer);
-                byte* bytes = pointer + accessor.PointerOffset;
-
-                // https://en.wikipedia.org/wiki/Portable_Executable
-                uint peHeaderOffset = ((uint*)(bytes + PEOffsets.DosStub.PESignatureOffset))[0];
-
-                if (accessor.Capacity < peHeaderOffset + PEOffsets.PEHeader.Subsystem + sizeof(ushort))
-                {
-                    throw new AppHostNotPEFileException("Subsystem offset out of file range.");
-                }
-
-                ushort* subsystem = ((ushort*)(bytes + peHeaderOffset + PEOffsets.PEHeader.Subsystem));
-
-                // https://docs.microsoft.com/en-us/windows/desktop/Debug/pe-format#windows-subsystem
-                // The subsystem of the prebuilt apphost should be set to CUI
-                if (subsystem[0] != (ushort)PEOffsets.Subsystem.WindowsCui)
-                {
-                    throw new AppHostNotCUIException(subsystem[0]);
-                }
-
-                // Set the subsystem to GUI
-                subsystem[0] = (ushort)PEOffsets.Subsystem.WindowsGui;
-            }
-            finally
-            {
-                if (pointer != null)
-                {
-                    accessor.SafeMemoryMappedViewHandle.ReleasePointer();
-                }
-            }
+            // https://learn.microsoft.com/windows/win32/debug/pe-format#windows-subsystem
+            // The subsystem of the prebuilt apphost should be set to CUI
+            uint peHeaderOffset;
+            ushort subsystem = GetWindowsSubsystem(accessor, out peHeaderOffset);
+            if (subsystem != (ushort)Subsystem.WindowsCui)
+                throw new AppHostNotCUIException(subsystem);
+
+            // Set the subsystem to GUI
+            accessor.Write(peHeaderOffset + PEOffsets.PEHeader.Subsystem, AsLittleEndian((ushort)Subsystem.WindowsGui));
         }
 
         public static unsafe void SetWindowsGraphicalUserInterfaceBit(string filePath)
@@ -113,32 +75,7 @@ public static unsafe void SetWindowsGraphicalUserInterfaceBit(string filePath)
         /// <param name="accessor">The memory accessor which has the apphost file opened.</param>
         internal static unsafe ushort GetWindowsGraphicalUserInterfaceBit(MemoryMappedViewAccessor accessor)
         {
-            byte* pointer = null;
-
-            try
-            {
-                accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref pointer);
-                byte* bytes = pointer + accessor.PointerOffset;
-
-                // https://en.wikipedia.org/wiki/Portable_Executable
-                uint peHeaderOffset = ((uint*)(bytes + PEOffsets.DosStub.PESignatureOffset))[0];
-
-                if (accessor.Capacity < peHeaderOffset + PEOffsets.PEHeader.Subsystem + sizeof(ushort))
-                {
-                    throw new AppHostNotPEFileException("Subsystem offset out of file range.");
-                }
-
-                ushort* subsystem = ((ushort*)(bytes + peHeaderOffset + PEOffsets.PEHeader.Subsystem));
-
-                return subsystem[0];
-            }
-            finally
-            {
-                if (pointer != null)
-                {
-                    accessor.SafeMemoryMappedViewHandle.ReleasePointer();
-                }
-            }
+            return GetWindowsSubsystem(accessor, out _);
         }
 
         public static unsafe ushort GetWindowsGraphicalUserInterfaceBit(string filePath)
@@ -151,5 +88,25 @@ public static unsafe ushort GetWindowsGraphicalUserInterfaceBit(string filePath)
                 }
             }
         }
+
+        private static ushort GetWindowsSubsystem(MemoryMappedViewAccessor accessor, out uint peHeaderOffset)
+        {
+            // https://en.wikipedia.org/wiki/Portable_Executable
+            if (accessor.Capacity < PEOffsets.DosStub.PESignatureOffset + sizeof(uint))
+                throw new AppHostNotPEFileException("PESignature offset out of file range.");
+
+            peHeaderOffset = AsLittleEndian(accessor.ReadUInt32(PEOffsets.DosStub.PESignatureOffset));
+            if (accessor.Capacity < peHeaderOffset + PEOffsets.PEHeader.Subsystem + sizeof(ushort))
+                throw new AppHostNotPEFileException("Subsystem offset out of file range.");
+
+            // https://learn.microsoft.com/windows/win32/debug/pe-format#windows-subsystem
+            return AsLittleEndian(accessor.ReadUInt16(peHeaderOffset + PEOffsets.PEHeader.Subsystem));
+        }
+
+        private static ushort AsLittleEndian(ushort value)
+            => BitConverter.IsLittleEndian ? value : BinaryPrimitives.ReverseEndianness(value);
+
+        private static uint AsLittleEndian(uint value)
+            => BitConverter.IsLittleEndian ? value : BinaryPrimitives.ReverseEndianness(value);
     }
 }

src/installer/tests/Microsoft.NET.HostModel.Tests/Microsoft.NET.HostModel.AppHost.Tests/AppHostUpdateTests.cs

@@ -11,6 +11,7 @@
 using Microsoft.NET.HostModel.AppHost;
 using Microsoft.DotNet.CoreSetup.Test;
 using System.Diagnostics;
+using System.Reflection.PortableExecutable;
 
 namespace Microsoft.NET.HostModel.Tests
 {
@@ -111,7 +112,9 @@ public void ItCanSetWindowsGUISubsystem()
                 BitConverter
                     .ToUInt16(File.ReadAllBytes(destinationFilePath), SubsystemOffset)
                     .Should()
-                    .Be(2);
+                    .Be((ushort)Subsystem.WindowsGui);
+
+                Assert.Equal((ushort)Subsystem.WindowsGui, PEUtils.GetWindowsGraphicalUserInterfaceBit(destinationFilePath));
             }
         }
 
@@ -153,6 +156,7 @@ public void ItFailsToSetGUISubsystemWithWrongDefault()
                 string destinationFilePath = Path.Combine(testDirectory.Path, "DestinationAppHost.exe.mock");
                 string appBinaryFilePath = "Test/App/Binary/Path.dll";
 
+                Assert.Equal(42, PEUtils.GetWindowsGraphicalUserInterfaceBit(sourceAppHostMock));
                 Assert.Throws<AppHostNotCUIException>(() =>
                     HostWriter.CreateAppHost(
                         sourceAppHostMock,