Fixing native lookup (#7282)

* enable m1 tests and fixing native lookup

* Mac testing

* mac test

* macos test

* mac testing

* reverting mac changes

* testing mac temp fix

* fixed cmake

* print DYLD libraries

* osx lightgbm test

* lightgbm dyld log

* excluding osx x64 from some tests

Author: michaelgsharp

eng/helix.proj

@@ -104,7 +104,7 @@
       <HelixPreCommands Condition="$(IsPosixShell)">$(HelixPreCommands);export PATH=$HELIX_CORRELATION_PAYLOAD/$(DotNetCliDestination):$PATH</HelixPreCommands>
       <HelixPreCommands Condition="!$(IsPosixShell)">$(HelixPreCommands);set PATH=%HELIX_CORRELATION_PAYLOAD%\$(DotNetCliDestination)%3B%PATH%</HelixPreCommands> 
 
-      <HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('osx'))">$(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT</HelixPreCommands>
+      <HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('osx'))">$(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE;export DYLD_PRINT_LIBRARIES=1;otool -L $HELIX_WORKITEM_ROOT/runtimes/osx-x64/native/lib_lightgbm.dylib</HelixPreCommands>
 
       <HelixPreCommands Condition="$(HelixTargetQueues.ToLowerInvariant().Contains('armarch'))">$(HelixPreCommands);sudo apt update;sudo apt-get install libomp-dev libomp5 -y</HelixPreCommands>
 
@@ -128,7 +128,7 @@
     Command="install_name_tool -change &quot;/usr/local/opt/libomp/lib/libomp.dylib&quot; &quot;@loader_path/libomp.dylib&quot; $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/libSymSgdNative.dylib" />
 
     <Exec Condition="Exists('$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\runtimes\osx-x64\native\lib_lightgbm.dylib') AND $(HelixTargetQueues.ToLowerInvariant().Contains('osx'))" 
-    Command="install_name_tool -change &quot;/usr/local/opt/libomp/lib/libomp.dylib&quot; &quot;@loader_path/libomp.dylib&quot; $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/runtimes/osx-x64/native/lib_lightgbm.dylib" />
+    Command="install_name_tool -change &quot;/usr/local/opt/libomp/lib/libomp.dylib&quot; &quot;@rpath/libomp.dylib&quot; $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/runtimes/osx-x64/native/lib_lightgbm.dylib" />
 
     <Exec Condition="Exists('$(BUILD_SOURCESDIRECTORY)\artifacts\bin\%(ProjectsWithTargetFramework.Filename)\$(BuildConfig)\%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)\runtimes\osx-x64\native\libonnxruntime.dylib') AND $(HelixTargetQueues.ToLowerInvariant().Contains('osx'))" 
     Command="install_name_tool -change &quot;/usr/local/opt/libomp/lib/libomp.dylib&quot; &quot;@loader_path/libomp.dylib&quot; $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/runtimes/osx-x64/native/libonnxruntime.dylib" />

test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs

@@ -19,6 +19,9 @@ public NativeDependencyFactAttribute(string library) : base($"This test requires
         /// <inheritdoc />
         protected override bool IsEnvironmentSupported()
         {
+            // Starting to drop native support for X64 OSX since intel no longer makes them.
+            if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX) && System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture == System.Runtime.InteropServices.Architecture.X64)
+                return false;
             return NativeLibrary.NativeLibraryExists(_library);
         }
     }

test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs

@@ -19,6 +19,9 @@ public NativeDependencyTheoryAttribute(string library) : base($"This test requir
         /// <inheritdoc />
         protected override bool IsEnvironmentSupported()
         {
+            // Starting to drop native support for X64 OSX since intel no longer makes them.
+            if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX) && System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture == System.Runtime.InteropServices.Architecture.X64)
+                return false;
             return NativeLibrary.NativeLibraryExists(_library);
         }
     }

test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs

@@ -19,7 +19,7 @@ public static bool NativeLibraryExists(string name)
             string extension = default;
             string prefix = "lib";
 
-            if (Environment.OSVersion.Platform == PlatformID.MacOSX)
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
                 extension = ".dylib";
             else if (Environment.OSVersion.Platform == PlatformID.Unix)
                 extension = ".so";
@@ -43,7 +43,17 @@ public static bool NativeLibraryExists(string name)
                 }
                 catch
                 {
-                    return false;
+                    // If that didn't load, dispose of the first attempt and try appending lib_ in front
+                    try
+                    {
+                        nativeLibrary?.Dispose();
+                        nativeLibrary = new NativeLibrary(prefix + "_" + name + extension);
+                        return true;
+                    }
+                    catch
+                    {
+                        return false;
+                    }
                 }
             }
             finally

test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs

@@ -69,6 +69,10 @@ private bool TryLocateNativeAssetFromDeps(string name, out string appLocalNative
 
             List<string> allRIDs = new List<string>();
             allRIDs.Add(currentRID);
+            if (currentRID.Contains("arm64"))
+            {
+                allRIDs.Add("osx-arm64");
+            }
             if (!AddFallbacks(allRIDs, currentRID, defaultContext.RuntimeGraph))
             {
 #pragma warning disable MSML_ParameterLocalVarName // Parameter or local variable name not standard