From 7780efb474db40812aa47acb3c41680f75daa7b8 Mon Sep 17 00:00:00 2001 From: Michael Sharp <51342856+michaelgsharp@users.noreply.github.com> Date: Fri, 5 May 2023 11:10:30 -0600 Subject: [PATCH] Update ML.NET to work with .NET8 (#6641) * updates for .net8 * Updates for .net 8 * fixed net6 to net6.0 * fixed approval tests --- .../Microsoft.ML.AutoML.Samples.csproj | 2 +- .../Microsoft.ML.Samples.GPU.csproj | 2 +- .../Dynamic/TensorFlow/ImageClassification.cs | 19 ++++-- .../ImageClassificationDefault.cs | 19 +++--- ...teSchedulingCifarResnetTransferLearning.cs | 19 +++--- ...esnetV2101TransferLearningEarlyStopping.cs | 19 +++--- ...snetV2101TransferLearningTrainTestSplit.cs | 19 +++--- .../Microsoft.ML.Samples.csproj | 2 +- eng/Versions.props | 1 + ...Microsoft.Data.Analysis.Interactive.csproj | 2 +- .../Microsoft.Data.Analysis.csproj | 2 +- .../Templates/Console/PredictProject.cs | 2 +- .../Templates/Console/PredictProject.tt | 2 +- .../Utilities/ResourceManagerUtils.cs | 59 ++++++------------- .../Microsoft.ML.CpuMath.csproj | 6 +- .../Microsoft.ML.DnnAnalyzer.csproj | 2 +- .../SamplesDatasetUtils.cs | 21 ++++--- src/Native/Native.proj | 2 +- ...CodeGenTest.ConsoleApp.csproj.approved.txt | 2 +- ...orTest.test.ConsoleApp.csproj.approved.txt | 2 +- ...CodeGenTest.ConsoleApp.csproj.approved.txt | 2 +- ...s.AzureImageCodeGeneratorTest.received.txt | 2 +- ...CodeGenTest.ConsoleApp.csproj.approved.txt | 2 +- ...soleAppProjectFileContentTest.approved.txt | 2 +- ...Contents_VerifyPredictProject.approved.txt | 2 +- .../Microsoft.ML.CpuMath.UnitTests.csproj | 2 +- .../BaseTestBaseline.cs | 2 +- 27 files changed, 118 insertions(+), 100 deletions(-) diff --git a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj index 215a265c26..8ed822e2dc 100644 --- a/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj +++ b/docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 false $(NoWarn);MSB3270 diff --git a/docs/samples/Microsoft.ML.Samples.GPU/Microsoft.ML.Samples.GPU.csproj b/docs/samples/Microsoft.ML.Samples.GPU/Microsoft.ML.Samples.GPU.csproj index ebe0fde3b1..8b40f06dae 100644 --- a/docs/samples/Microsoft.ML.Samples.GPU/Microsoft.ML.Samples.GPU.csproj +++ b/docs/samples/Microsoft.ML.Samples.GPU/Microsoft.ML.Samples.GPU.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net6.0 Exe false diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs index 88ec4d8d7d..5462982ddc 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs @@ -2,7 +2,9 @@ using System.IO; using System.Linq; using System.Net; +using System.Net.Http; using System.Text; +using System.Threading.Tasks; using ICSharpCode.SharpZipLib.GZip; using ICSharpCode.SharpZipLib.Tar; using Microsoft.ML; @@ -23,7 +25,9 @@ public static void Example() string modelLocation = "resnet_v2_101_299_frozen.pb"; if (!File.Exists(modelLocation)) { - modelLocation = Download(@"https://storage.googleapis.com/download.tensorflow.org/models/tflite_11_05_08/resnet_v2_101.tgz", @"resnet_v2_101_299_frozen.tgz"); + var downloadTask = Download(@"https://storage.googleapis.com/download.tensorflow.org/models/tflite_11_05_08/resnet_v2_101.tgz", @"resnet_v2_101_299_frozen.tgz"); + downloadTask.Wait(); + modelLocation = downloadTask.Result; Unzip(Path.Join(Directory.GetCurrentDirectory(), modelLocation), Directory.GetCurrentDirectory()); @@ -111,11 +115,18 @@ class OutputScores public float[] output { get; set; } } - private static string Download(string baseGitPath, string dataFile) + private static async Task Download(string baseGitPath, string dataFile) { - using (WebClient client = new WebClient()) + if (File.Exists(dataFile)) + return dataFile; + + using (HttpClient client = new HttpClient()) { - client.DownloadFile(new Uri($"{baseGitPath}"), dataFile); + var response = await client.GetStreamAsync(new Uri($"{baseGitPath}")).ConfigureAwait(false); + using (var fs = new FileStream(dataFile, FileMode.CreateNew)) + { + await response.CopyToAsync(fs).ConfigureAwait(false); + } } return dataFile; diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault.cs index c9ea3adc9a..a8844c7335 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ImageClassificationDefault.cs @@ -5,6 +5,7 @@ using System.IO.Compression; using System.Linq; using System.Net; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.ML; @@ -244,14 +245,14 @@ public static string DownloadImageSet(string imagesDownloadFolder) string fileName = "flower_photos_small_set.zip"; string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip"; - Download(url, imagesDownloadFolder, fileName); + Download(url, imagesDownloadFolder, fileName).Wait(); UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); return Path.GetFileNameWithoutExtension(fileName); } // Download file to destination directory from input URL. - public static bool Download(string url, string destDir, string destFileName) + public static async Task Download(string url, string destDir, string destFileName) { if (destFileName == null) destFileName = url.Split(Path.DirectorySeparatorChar).Last(); @@ -266,14 +267,18 @@ public static bool Download(string url, string destDir, string destFileName) return false; } - var wc = new WebClient(); Console.WriteLine($"Downloading {relativeFilePath}"); - var download = Task.Run(() => wc.DownloadFile(url, relativeFilePath)); - while (!download.IsCompleted) + + using (HttpClient client = new HttpClient()) { - Thread.Sleep(1000); - Console.Write("."); + var response = await client.GetStreamAsync(new Uri($"{url}")).ConfigureAwait(false); + + using (var fs = new FileStream(relativeFilePath, FileMode.CreateNew)) + { + await response.CopyToAsync(fs); + } } + Console.WriteLine(""); Console.WriteLine($"Downloaded {relativeFilePath}"); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/LearningRateSchedulingCifarResnetTransferLearning.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/LearningRateSchedulingCifarResnetTransferLearning.cs index c129e50314..c030d56968 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/LearningRateSchedulingCifarResnetTransferLearning.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/LearningRateSchedulingCifarResnetTransferLearning.cs @@ -5,6 +5,7 @@ using System.IO.Compression; using System.Linq; using System.Net; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.ML; @@ -275,7 +276,7 @@ public static string DownloadImageSet(string imagesDownloadFolder) // https://github.com/YoongiKim/CIFAR-10-images string url = $"https://github.com/YoongiKim/CIFAR-10-images/archive/refs/heads/master.zip"; - Download(url, imagesDownloadFolder, fileName); + Download(url, imagesDownloadFolder, fileName).Wait(); UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); @@ -283,7 +284,7 @@ public static string DownloadImageSet(string imagesDownloadFolder) } // Download file to destination directory from input URL. - public static bool Download(string url, string destDir, string destFileName) + public static async Task Download(string url, string destDir, string destFileName) { if (destFileName == null) destFileName = url.Split(Path.DirectorySeparatorChar).Last(); @@ -298,14 +299,18 @@ public static bool Download(string url, string destDir, string destFileName) return false; } - var wc = new WebClient(); Console.WriteLine($"Downloading {relativeFilePath}"); - var download = Task.Run(() => wc.DownloadFile(url, relativeFilePath)); - while (!download.IsCompleted) + + using (HttpClient client = new HttpClient()) { - Thread.Sleep(1000); - Console.Write("."); + var response = await client.GetStreamAsync(new Uri($"{url}")).ConfigureAwait(false); + + using (var fs = new FileStream(relativeFilePath, FileMode.CreateNew)) + { + await response.CopyToAsync(fs); + } } + Console.WriteLine(""); Console.WriteLine($"Downloaded {relativeFilePath}"); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs index ba6243a498..d8374c143c 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningEarlyStopping.cs @@ -5,6 +5,7 @@ using System.IO.Compression; using System.Linq; using System.Net; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.ML; @@ -232,14 +233,14 @@ public static string DownloadImageSet(string imagesDownloadFolder) string fileName = "flower_photos_small_set.zip"; string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip"; - Download(url, imagesDownloadFolder, fileName); + Download(url, imagesDownloadFolder, fileName).Wait(); UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); return Path.GetFileNameWithoutExtension(fileName); } // Download file to destination directory from input URL. - public static bool Download(string url, string destDir, string destFileName) + public static async Task Download(string url, string destDir, string destFileName) { if (destFileName == null) destFileName = url.Split(Path.DirectorySeparatorChar).Last(); @@ -254,14 +255,18 @@ public static bool Download(string url, string destDir, string destFileName) return false; } - var wc = new WebClient(); Console.WriteLine($"Downloading {relativeFilePath}"); - var download = Task.Run(() => wc.DownloadFile(url, relativeFilePath)); - while (!download.IsCompleted) + + using (HttpClient client = new HttpClient()) { - Thread.Sleep(1000); - Console.Write("."); + var response = await client.GetStreamAsync(new Uri($"{url}")).ConfigureAwait(false); + + using (var fs = new FileStream(relativeFilePath, FileMode.CreateNew)) + { + await response.CopyToAsync(fs); + } } + Console.WriteLine(""); Console.WriteLine($"Downloaded {relativeFilePath}"); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs index 6fffbe0ff4..4e4c809396 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/MulticlassClassification/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs @@ -5,6 +5,7 @@ using System.IO.Compression; using System.Linq; using System.Net; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.ML; @@ -253,14 +254,14 @@ public static string DownloadImageSet(string imagesDownloadFolder) string fileName = "flower_photos_small_set.zip"; string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip"; - Download(url, imagesDownloadFolder, fileName); + Download(url, imagesDownloadFolder, fileName).Wait(); UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder); return Path.GetFileNameWithoutExtension(fileName); } // Download file to destination directory from input URL. - public static bool Download(string url, string destDir, string destFileName) + public static async Task Download(string url, string destDir, string destFileName) { if (destFileName == null) destFileName = url.Split(Path.DirectorySeparatorChar).Last(); @@ -275,14 +276,18 @@ public static bool Download(string url, string destDir, string destFileName) return false; } - var wc = new WebClient(); Console.WriteLine($"Downloading {relativeFilePath}"); - var download = Task.Run(() => wc.DownloadFile(url, relativeFilePath)); - while (!download.IsCompleted) + + using (HttpClient client = new HttpClient()) { - Thread.Sleep(1000); - Console.Write("."); + var response = await client.GetStreamAsync(new Uri($"{url}")).ConfigureAwait(false); + + using (var fs = new FileStream(relativeFilePath, FileMode.CreateNew)) + { + await response.CopyToAsync(fs); + } } + Console.WriteLine(""); Console.WriteLine($"Downloaded {relativeFilePath}"); diff --git a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj index caba26f5cc..3fba560fd5 100644 --- a/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj +++ b/docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net6.0 Exe false diff --git a/eng/Versions.props b/eng/Versions.props index 79f2ec10a9..64c878b995 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -76,6 +76,7 @@ 5.4.7 0.12.0 6.0.9 + 8.0.0-preview.3.23174.8 5.10.2 1.1.2-beta1.22512.1 7.0.0-beta.23073.6 diff --git a/src/Microsoft.Data.Analysis.Interactive/Microsoft.Data.Analysis.Interactive.csproj b/src/Microsoft.Data.Analysis.Interactive/Microsoft.Data.Analysis.Interactive.csproj index 241a44cbca..e95ca64713 100644 --- a/src/Microsoft.Data.Analysis.Interactive/Microsoft.Data.Analysis.Interactive.csproj +++ b/src/Microsoft.Data.Analysis.Interactive/Microsoft.Data.Analysis.Interactive.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net6.0 false diff --git a/src/Microsoft.Data.Analysis/Microsoft.Data.Analysis.csproj b/src/Microsoft.Data.Analysis/Microsoft.Data.Analysis.csproj index c4e3ca036a..acfcd62199 100644 --- a/src/Microsoft.Data.Analysis/Microsoft.Data.Analysis.csproj +++ b/src/Microsoft.Data.Analysis/Microsoft.Data.Analysis.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.cs b/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.cs index 7ae7e11549..8acec761a7 100644 --- a/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.cs +++ b/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.cs @@ -27,7 +27,7 @@ internal partial class PredictProject : PredictProjectBase public virtual string TransformText() { this.Write("\r\n\r\n \r\n Exe\r\n netcoreapp3.1\r\n \r\n net6.0\r\n \r\n \r\n \r\n"); diff --git a/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.tt b/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.tt index 65a4eb8a38..a23b575ccf 100644 --- a/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.tt +++ b/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.tt @@ -8,7 +8,7 @@ Exe - netcoreapp3.1 + net6.0 diff --git a/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs b/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs index bc06467159..f50ecc9174 100644 --- a/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs +++ b/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs @@ -3,9 +3,11 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; +using System.Net.Http; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -152,19 +154,10 @@ private async Task DownloadFromUrlWithRetryAsync(IHostEnvironment env, I /// Returns the error message if an error occurred, null if download was successful. private async Task DownloadFromUrlAsync(IHostEnvironment env, IChannel ch, string url, string fileName, int timeout, string filePath) { - using (var webClient = new WebClient()) + using (var client = new HttpClient()) using (var downloadCancel = new CancellationTokenSource()) { - bool deleteNeeded = false; - EventHandler disposed = - (object sender, EventArgs e) => - { - if (File.Exists(filePath) && deleteNeeded) - TryDelete(ch, filePath); - }; - - webClient.Disposed += disposed; - var t = Task.Run(() => DownloadResource(env, ch, webClient, new Uri(url), filePath, fileName, downloadCancel.Token)); + var t = Task.Run(() => DownloadResource(env, ch, client, new Uri(url), filePath, fileName, downloadCancel.Token)); UpdateTimeout(ref timeout); var timeoutTask = Task.Delay(timeout).ContinueWith(task => default(Exception), TaskScheduler.Default); @@ -173,7 +166,8 @@ private async Task DownloadFromUrlAsync(IHostEnvironment env, IChannel c if (completedTask != t || completedTask.CompletedResult() != null) { downloadCancel.Cancel(); - deleteNeeded = true; + if (File.Exists(filePath)) + TryDelete(ch, filePath); return (await t).Message; } return null; @@ -252,7 +246,7 @@ private static string GetFilePath(IChannel ch, string fileName, string dir, out return filePath; } - private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient webClient, Uri uri, string path, string fileName, CancellationToken ct) + private async Task DownloadResource(IHostEnvironment env, IChannel ch, HttpClient httpClient, Uri uri, string path, string fileName, CancellationToken ct) { if (File.Exists(path)) return null; @@ -271,41 +265,26 @@ private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient { int blockSize = 4096; - using (var s = webClient.OpenRead(uri)) + var response = await httpClient.GetAsync(uri, ct).ConfigureAwait(false); using (var fh = env.CreateOutputFile(tempPath)) using (var ws = fh.CreateWriteStream()) { - var headers = webClient.ResponseHeaders.GetValues("Content-Length"); + response.EnsureSuccessStatusCode(); + IEnumerable headers; + var hasHeader = response.Headers.TryGetValues("content-length", out headers); if (uri.Host == "aka.ms" && IsRedirectToDefaultPage(uri.AbsoluteUri)) throw new NotSupportedException($"The provided url ({uri}) redirects to the default url ({DefaultUrl})"); - if (Utils.Size(headers) == 0 || !long.TryParse(headers[0], out var size)) + if (!hasHeader || !long.TryParse(headers.First(), out var size)) size = 10000000; - long printFreq = (long)(size / 10.0); - var buffer = new byte[blockSize]; - long total = 0; + var stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync().ConfigureAwait(false); + + await stream.CopyToAsync(ws, blockSize, ct); - // REVIEW: use a progress channel instead. - while (true) + if (ct.IsCancellationRequested) { - var task = s.ReadAsync(buffer, 0, blockSize, ct); - task.Wait(); - int count = task.Result; - - if (count <= 0) - { - break; - } - - ws.Write(buffer, 0, count); - total += count; - if ((total - (total / printFreq) * printFreq) <= blockSize) - ch.Info($"{fileName}: Downloaded {total} bytes out of {size}"); - if (ct.IsCancellationRequested) - { - ch.Error($"{fileName}: Download timed out"); - return ch.Except("Download timed out"); - } + ch.Error($"{fileName}: Download timed out"); + return ch.Except("Download timed out"); } } File.Move(tempPath, path); @@ -314,7 +293,7 @@ private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient } catch (WebException e) { - ch.Error($"{fileName}: Could not download. WebClient returned the following error: {e.Message}"); + ch.Error($"{fileName}: Could not download. HttpClient returned the following error: {e.Message}"); return e; } finally diff --git a/src/Microsoft.ML.CpuMath/Microsoft.ML.CpuMath.csproj b/src/Microsoft.ML.CpuMath/Microsoft.ML.CpuMath.csproj index a0c141a8da..5fa62087e0 100644 --- a/src/Microsoft.ML.CpuMath/Microsoft.ML.CpuMath.csproj +++ b/src/Microsoft.ML.CpuMath/Microsoft.ML.CpuMath.csproj @@ -1,7 +1,7 @@  - netstandard2.0;netcoreapp3.1 + netstandard2.0;net6.0 Microsoft.ML.CpuMath Microsoft.ML.CpuMath contains optimized math routines for ML.NET. true @@ -15,7 +15,7 @@ - + @@ -31,7 +31,7 @@ - + diff --git a/src/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer.csproj b/src/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer.csproj index c3f7e6a57a..252cce0312 100644 --- a/src/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer.csproj +++ b/src/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer/Microsoft.ML.DnnAnalyzer.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 DnnAnalyzer diff --git a/src/Microsoft.ML.SamplesUtils/SamplesDatasetUtils.cs b/src/Microsoft.ML.SamplesUtils/SamplesDatasetUtils.cs index 51ce75b3af..cd6d078a1e 100644 --- a/src/Microsoft.ML.SamplesUtils/SamplesDatasetUtils.cs +++ b/src/Microsoft.ML.SamplesUtils/SamplesDatasetUtils.cs @@ -7,6 +7,9 @@ using System.IO; using System.Linq; using System.Net; +using System.Net.Http; +using System.Runtime.InteropServices.ComTypes; +using System.Threading.Tasks; using Microsoft.ML.Data; namespace Microsoft.ML.SamplesUtils @@ -177,22 +180,26 @@ public static string DownloadTensorFlowSentimentModel() if (!Directory.Exists(varPath)) Directory.CreateDirectory(varPath); - Download(Path.Combine(remotePath, "saved_model.pb"), Path.Combine(path, "saved_model.pb")); - Download(Path.Combine(remotePath, "imdb_word_index.csv"), Path.Combine(path, "imdb_word_index.csv")); - Download(Path.Combine(remotePath, "variables", "variables.data-00000-of-00001"), Path.Combine(varPath, "variables.data-00000-of-00001")); - Download(Path.Combine(remotePath, "variables", "variables.index"), Path.Combine(varPath, "variables.index")); + Download(Path.Combine(remotePath, "saved_model.pb"), Path.Combine(path, "saved_model.pb")).Wait(); + Download(Path.Combine(remotePath, "imdb_word_index.csv"), Path.Combine(path, "imdb_word_index.csv")).Wait(); + Download(Path.Combine(remotePath, "variables", "variables.data-00000-of-00001"), Path.Combine(varPath, "variables.data-00000-of-00001")).Wait(); + Download(Path.Combine(remotePath, "variables", "variables.index"), Path.Combine(varPath, "variables.index")).Wait(); return path; } - private static string Download(string baseGitPath, string dataFile) + private static async Task Download(string baseGitPath, string dataFile) { if (File.Exists(dataFile)) return dataFile; - using (WebClient client = new WebClient()) + using (HttpClient client = new HttpClient()) { - client.DownloadFile(new Uri($"{baseGitPath}"), dataFile); + var response = await client.GetStreamAsync(new Uri($"{baseGitPath}")).ConfigureAwait(false); + using (var fs = new FileStream(dataFile, FileMode.CreateNew)) + { + await response.CopyToAsync(fs).ConfigureAwait(false); + } } return dataFile; diff --git a/src/Native/Native.proj b/src/Native/Native.proj index 8e48283b3f..25892958cf 100644 --- a/src/Native/Native.proj +++ b/src/Native/Native.proj @@ -185,7 +185,7 @@ Include="$(NativeAssetsBuiltPath)\$(NativeLibPrefix)CpuMathNative$(NativeLibExtension)" RelativePath="Microsoft.ML.CpuMath\runtimes\$(PackageRid)\nativeassets\netstandard2.0" /> + RelativePath="Microsoft.ML.CpuMath\runtimes\$(PackageRid)\nativeassets\net6.0" /> diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt index b05f146366..d56e742f3b 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.test.ConsoleApp.csproj.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.test.ConsoleApp.csproj.approved.txt index b419e51021..7e7c94d918 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.test.ConsoleApp.csproj.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.test.ConsoleApp.csproj.approved.txt @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt index 7eeab03036..12a18142d2 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.received.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.received.txt index 9bf8fac718..4637958288 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.received.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.received.txt @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureObjectDetectionCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureObjectDetectionCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt index 7eeab03036..12a18142d2 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureObjectDetectionCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureObjectDetectionCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppProjectFileContentTest.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppProjectFileContentTest.approved.txt index 97df9ea672..b6aaf2b243 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppProjectFileContentTest.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppProjectFileContentTest.approved.txt @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.Recommendation_GenerateConsoleAppProjectContents_VerifyPredictProject.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.Recommendation_GenerateConsoleAppProjectContents_VerifyPredictProject.approved.txt index e261bed0fd..2ffb1661f0 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.Recommendation_GenerateConsoleAppProjectContents_VerifyPredictProject.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.Recommendation_GenerateConsoleAppProjectContents_VerifyPredictProject.approved.txt @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net6.0 diff --git a/test/Microsoft.ML.CpuMath.UnitTests/Microsoft.ML.CpuMath.UnitTests.csproj b/test/Microsoft.ML.CpuMath.UnitTests/Microsoft.ML.CpuMath.UnitTests.csproj index 46a7f77d3f..a1fc06863d 100644 --- a/test/Microsoft.ML.CpuMath.UnitTests/Microsoft.ML.CpuMath.UnitTests.csproj +++ b/test/Microsoft.ML.CpuMath.UnitTests/Microsoft.ML.CpuMath.UnitTests.csproj @@ -5,7 +5,7 @@ - + diff --git a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs index 71f9d2c4aa..b802123206 100644 --- a/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs +++ b/test/Microsoft.ML.TestFramework/BaseTestBaseline.cs @@ -118,7 +118,7 @@ private IEnumerable GetConfigurationDirs() // on x64 vs x86 and dotnet core 3.1 vs others, so we have 4 combination: // x64-netcore3.1, x86-netcore3.1, x64-rest, x86-rest. In some cases x64 vs x86 // have different results, in some cases netcore 3.1 vs rest have different results, - // the most complicate situation is 12 combinations (x64 vs x86, netcoreapp3.1 vs rest, + // the most complicate situation is 12 combinations (x64 vs x86, net6.0 vs rest, // win vs linux vs osx) have different results. // So use list of string to return different configurations and test will try to search // through this list and use the one file first found, make sure we don't have baseline file