The -test command for windows. Nuget packages

Directory.Build.props

@@ -13,9 +13,8 @@
 
   <PropertyGroup>
     <Copyright>$(CopyrightNetFoundation)</Copyright>
-    <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
-    <DebugType>embedded</DebugType>
+    <DebugType>portable</DebugType>
     <DebugSymbols>true</DebugSymbols>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
@@ -31,7 +30,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <PackageAssetsPath>$(ArtifactsDir)/pkgassets/</PackageAssetsPath>
+    <PackageAssetsPath>$(ArtifactsDir)pkgassets/</PackageAssetsPath>
     <PkgDir>$(RepoRoot)pkg/</PkgDir>
   </PropertyGroup>
 
@@ -49,17 +48,17 @@
   <PropertyGroup>
     <!-- Optional: Publish the repository URL in the built .nupkg (in the NuSpec <Repository> element) -->
     <PublishRepositoryUrl>true</PublishRepositoryUrl>
- 
+
     <!-- Optional: Embed source files that are not tracked by the source control manager in the PDB -->
     <EmbedUntrackedSources>true</EmbedUntrackedSources>
-  
+
     <!-- Optional: Build symbol package (.snupkg) to distribute the PDB containing Source Link -->
     <IncludeSymbols>true</IncludeSymbols>
     <SymbolPackageFormat>snupkg</SymbolPackageFormat>
   </PropertyGroup>
-  
+
   <PropertyGroup Condition="'$(DotNetBuildFromSource)' == 'true'">
-    <!--    
+    <!--
       When building using source-build the process is:
       - Newtonsoft.Json versions 9.0.1 and 12.0.2 are built by source-build
       - Version 12.0.2 is written to Version.props

Directory.Build.targets

@@ -1,4 +1,39 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project>
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
+
+  <Target Name="CopyNativeAssembiles" AfterTargets="CopyFilesToOutputDirectory">
+    <PropertyGroup>
+		<LibPrefix Condition="'$(OS)' != 'Windows_NT'">lib</LibPrefix>
+		<LibExtension Condition="'$(OS)' == 'Windows_NT'">.dll</LibExtension>
+		<LibExtension Condition="'$(OS)' != 'Windows_NT'">.so</LibExtension>
+		<LibExtension Condition="$([MSBuild]::IsOSPlatform('osx'))">.dylib</LibExtension>
+
+		<TargetArchitecture Condition="'$(Platform)' == ''">x64</TargetArchitecture>
+		<NativeTargetArchitecture Condition="'$(NativeTargetArchitecture)' == ''">$(TargetArchitecture)</NativeTargetArchitecture>
+		<BinDir Condition="'$(BinDir)'==''">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin'))</BinDir>
+		<NativeOutputPath>$(BinDir)Native\$(NativeTargetArchitecture).$(Configuration)\</NativeOutputPath>
+
+		<Platform Condition="'$(Platform)'==''">AnyCPU</Platform>
+		<PlatformConfig>$(Platform).$(Configuration)</PlatformConfig>
+		<OutputPath Condition="'$(OutputPath)'==''">$(BinDir)$(MSBuildProjectName)\Debug</OutputPath>
+	</PropertyGroup>
+
+		<ItemGroup>
+		<NativeAssemblyReference>
+			<FullAssemblyPath>$(NativeOutputPath)$(LibPrefix)%(NativeAssemblyReference.Identity)$(LibExtension)</FullAssemblyPath>
+		</NativeAssemblyReference>
+		</ItemGroup>
+
+		<Copy SourceFiles = "@(NativeAssemblyReference->'%(FullAssemblyPath)')"
+			DestinationFolder="$(OutDir)"
+			OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
+			Retries="$(CopyRetryCount)"
+			RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
+			UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
+			UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
+		<Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
+		</Copy>
+  </Target>
+
 </Project>

build.sh

@@ -10,4 +10,4 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-"$DIR/eng/common/build.sh" --restore --build --pack --warnAsError false
+"$DIR/eng/common/build.sh" --restore --build --pack --warnAsError false "$@"

docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj

@@ -7,6 +7,14 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\..\src\Microsoft.ML.AutoML\Microsoft.ML.AutoML.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.Core\Microsoft.ML.Core.csproj" >
+      <PrivateAssets>all</PrivateAssets>
+    </ProjectReference>
+
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj" >
+      <PrivateAssets>all</PrivateAssets>
+    </ProjectReference>
+
     <NativeAssemblyReference Include="MatrixFactorizationNative" />
     <NativeAssemblyReference Include="FastTreeNative" />
   </ItemGroup>

docs/samples/Microsoft.ML.Samples.GPU/Microsoft.ML.Samples.GPU.csproj

@@ -20,9 +20,13 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\..\src\Microsoft.ML.Vision\Microsoft.ML.Vision.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.LightGbm\Microsoft.ML.LightGbm.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.TensorFlow\Microsoft.ML.TensorFlow.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.SamplesUtils\Microsoft.ML.SamplesUtils.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.StandardTrainers\Microsoft.ML.StandardTrainers.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj" />
 
     <NativeAssemblyReference Include="CpuMathNative" />
     <NativeAssemblyReference Include="FastTreeNative" />

docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj

@@ -30,6 +30,9 @@
     <ProjectReference Include="..\..\..\src\Microsoft.ML.DnnImageFeaturizer.ResNet18\Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.OnnxConverter\Microsoft.ML.OnnxConverter.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.ImageAnalytics\Microsoft.ML.ImageAnalytics.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
+    <ProjectReference Include="..\..\..\src\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />
 
     <NativeAssemblyReference Include="CpuMathNative" />
     <NativeAssemblyReference Include="FastTreeNative" />

eng/Build.props

@@ -1,8 +1,17 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project>
+
+  <ItemDefinitionGroup>
+    <ProjectToBuild>
+      <RestoreInParallel>true</RestoreInParallel>
+      <BuildInParallel>false</BuildInParallel>
+    </ProjectToBuild>
+  </ItemDefinitionGroup>
+
   <ItemGroup>
     <ProjectToBuild Include="$(RepoRoot)src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj" />
     <ProjectToBuild Include="$(RepoRoot)src/Native/Native.proj" />
     <ProjectToBuild Include="$(RepoRoot)Microsoft.ML.sln" />
   </ItemGroup>
+
 </Project>

eng/Versions.props

@@ -2,9 +2,9 @@
 <Project>
   <PropertyGroup>
     <!-- This repo version -->
-    <DotNetUseShippingVersions>true</DotNetUseShippingVersions>
+    <DotNetUseShippingVersions>false</DotNetUseShippingVersions>
     <VersionPrefix>1.5.3</VersionPrefix>
-    <PreReleaseVersionLabel></PreReleaseVersionLabel>
+    <PreReleaseVersionLabel>dev</PreReleaseVersionLabel>
 
     <!--ML.NET Core dependencies-->
     <NewtonsoftJsonPackageVersion>10.0.3</NewtonsoftJsonPackageVersion>
@@ -37,7 +37,7 @@
     <ReportGeneratorVersion>4.3.6</ReportGeneratorVersion>
     <MicrosoftDotNetApiCompatPackageVersion>1.0.0-beta.19225.5</MicrosoftDotNetApiCompatPackageVersion>
     <MicrosoftSourceLinkVersion>1.1.0-beta-20206-02</MicrosoftSourceLinkVersion>
-    
+
     <!-- Test-only Dependencies -->
     <BenchmarkDotNetVersion>0.12.0</BenchmarkDotNetVersion>
     <MicrosoftCodeAnalysisTestingVersion>1.0.1-beta1.20080.1</MicrosoftCodeAnalysisTestingVersion>
@@ -77,6 +77,7 @@
     <MicrosoftExtensionsDependencyModelVersion>2.1.0</MicrosoftExtensionsDependencyModelVersion>
     <MicrosoftExtensionsFileSystemGlobbingVersion>2.0.0</MicrosoftExtensionsFileSystemGlobbingVersion>
     <MicrosoftNETCorePlatformsVersion>2.1.0</MicrosoftNETCorePlatformsVersion>
+    <MicrosoftNETCore3PlatformsVersion>3.1.0</MicrosoftNETCore3PlatformsVersion>
     <MicrosoftNetCompilersToolsetVersion>3.8.0-3.20460.2</MicrosoftNetCompilersToolsetVersion>
     <MoqVersion>4.8.3</MoqVersion>
     <MonoOptionsVersion>5.3.0.1</MonoOptionsVersion>
@@ -103,9 +104,9 @@
     <SystemThreadingTasksExtensionVersion>4.5.2</SystemThreadingTasksExtensionVersion>
     <SystemValueTupleVersion>4.4.0</SystemValueTupleVersion>
     <WindowsAzureStorageVersion>8.5.0</WindowsAzureStorageVersion>
-    <XUnitVersion>2.4.1</XUnitVersion>
+    <XUnitVersion>2.4.0</XUnitVersion>
     <XUnitAbstractionsVersion>2.0.3</XUnitAbstractionsVersion>
-    <XUnitVSRunnerVersion>2.4.1</XUnitVSRunnerVersion>
+    <XUnitVSRunnerVersion>2.4.0</XUnitVSRunnerVersion>
     <MicrosoftDotNetBuildTasksFeedVersion>5.0.0-beta.20461.7</MicrosoftDotNetBuildTasksFeedVersion>
     <MicrosoftDotNetSignToolVersion>5.0.0-beta.20461.7</MicrosoftDotNetSignToolVersion>
     <MicrosoftAzureDocumentDBVersion>1.22.0</MicrosoftAzureDocumentDBVersion>

eng/pkg/Pack.props

@@ -0,0 +1,78 @@
+<Project>
+
+  <PropertyGroup>
+    <PackageAssetsPath>$(ArtifactsDir)pkgassets/</PackageAssetsPath>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+    <IsSymbolsPackage Condition="$(MSBuildProjectName.Contains('.snupkg'))">true</IsSymbolsPackage>
+    <PackageIdFolderName>$(MSBuildProjectName.Replace('.symbols', ''))</PackageIdFolderName>
+    <IncludeBuildOutput>true</IncludeBuildOutput>
+    <IsPackable>true</IsPackable>
+
+  </PropertyGroup>
+
+  <!-- nuspec properties -->
+  <PropertyGroup>
+    <Authors>Microsoft</Authors>
+    <PackageLicenseFile>LICENSE</PackageLicenseFile>
+    <PackageProjectUrl>https://dot.net/ml</PackageProjectUrl>
+    <PackageIcon>mlnetlogo.png</PackageIcon>
+    <PackageReleaseNotes>https://aka.ms/mlnetreleasenotes</PackageReleaseNotes>
+    <!-- space separated -->
+    <PackageTags>ML.NET ML Machine Learning</PackageTags>
+
+    <LibExtension Condition="'$(OS)' == 'Windows_NT'">.dll</LibExtension>
+		<LibExtension Condition="'$(OS)' != 'Windows_NT'">.so</LibExtension>
+		<LibExtension Condition="$([MSBuild]::IsOSPlatform('osx'))">.dylib</LibExtension>
+  </PropertyGroup>
+
+  <ItemGroup>
+
+    <!-- Create symbol packages correctly by copying the library and xml to the nuget. Then the pdb is auto put in the symbols package-->
+    <Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\lib\**\*" PackagePath="lib" />
+    <Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\analyzers\**\*" PackagePath="analyzers" />
+    <Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\**\*" PackagePath="runtimes" />
+    <Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\tools\**\*" PackagePath="tools" />
+
+    <Content Include="$(RepoRoot)eng\pkg\mlnetlogo.png" Pack="true" PackagePath="" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(IncludeMLNetNotices)' != 'false'">
+    <Content Include="$(RepoRoot)\THIRD-PARTY-NOTICES.TXT" Pack="true" PackagePath="" />
+    <Content Include="$(RepoRoot)\LICENSE" Pack="true" PackagePath=""/>
+  </ItemGroup>
+
+  <!--  This check does not work corectly anymore and also removes the PDB from the symbols packages.
+        Need to figure out the correct way to do this, but since the normal packages are building correctly
+        I am leaving this as is for now. -->
+  <ItemGroup Condition="'$(IsSymbolsPackage)' != 'true'">
+    <Content Remove="$(PackageAssetsPath)$(PackageIdFolderName)\**\*.pdb" />
+    <Content Remove="$(PackageAssetsPath)$(PackageIdFolderName)\**\*.dwarf" />
+    <Content Remove="$(PackageAssetsPath)$(PackageIdFolderName)\**\*.dbg" />
+  </ItemGroup>
+
+  <ItemGroup Condition="Exists('packages.config') OR
+                        Exists('$(MSBuildProjectName).packages.config') OR
+                        Exists('packages.$(MSBuildProjectName).config')">
+    <Content Include="$(MSBuildThisFileDirectory)\..\..\runtimes\win-x64\native\*.dll"
+             Condition="'$(PlatformTarget)' == 'x64'">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <Visible>false</Visible>
+      <Link>%(Filename)%(Extension)</Link>
+    </Content>
+    <Content Include="$(MSBuildThisFileDirectory)\..\..\runtimes\win-x86\native\*.dll"
+             Condition="'$(PlatformTarget)' == 'x86'">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <Visible>false</Visible>
+      <Link>%(Filename)%(Extension)</Link>
+    </Content>
+  </ItemGroup>
+
+  <!-- Work around https://github.com/NuGet/Home/issues/6091 -->
+  <ItemDefinitionGroup>
+    <PackageReference>
+      <PrivateAssets>None</PrivateAssets>
+    </PackageReference>
+  </ItemDefinitionGroup>
+
+</Project>

global.json

@@ -1,12 +1,15 @@
 {
-  "tools": {
-    "dotnet": "5.0.100-rc.1.20452.10"
-  },
-  "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20461.7",
-    "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20461.7",
-    "Microsoft.Build.Traversal": "2.1.1",
-    "Microsoft.SourceLink.GitHub": "1.1.0-beta-20206-02",
-    "Microsoft.SourceLink.Common": "1.1.0-beta-20206-02"
+	"tools": {
+	  "dotnet": "5.0.100-rc.1.20452.10",
+	  "runtimes": {
+		"dotnet/x64": ["$(MicrosoftNETCorePlatformsVersion)", "$(MicrosoftNETCore3PlatformsVersion)" ]
+	  }
+	},
+	"msbuild-sdks": {
+	  "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20461.7",
+	  "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20461.7",
+	  "Microsoft.Build.Traversal": "2.1.1",
+	  "Microsoft.SourceLink.GitHub": "1.1.0-beta-20206-02",
+	  "Microsoft.SourceLink.Common": "1.1.0-beta-20206-02"
+	}
   }
-}