dotnet · mstfbl · Mar 25, 2020 · Mar 13, 2020 · Mar 16, 2020 · Mar 16, 2020
diff --git a/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs b/src/Microsoft.ML.Core/Utilities/ResourceManagerUtils.cs
@@ -113,8 +113,7 @@ public async Task<ResourceDownloadResults> EnsureResourceAsync(IHostEnvironment
                 await DownloadFromUrlWithRetryAsync(env, ch, absoluteUrl.AbsoluteUri, fileName, timeout, filePath), absoluteUrl.AbsoluteUri);
         }
 
-        private async Task<string> DownloadFromUrlWithRetryAsync(IHostEnvironment env, IChannel ch, string url, string fileName,
-            int timeout, string filePath, int retryTimes = 5)
+        private async Task<string> DownloadFromUrlWithRetryAsync(IHostEnvironment env, IChannel ch, string url, string fileName, int timeout, string filePath, int retryTimes = 5)
         {
             var downloadResult = "";
 
@@ -125,8 +124,12 @@ private async Task<string> DownloadFromUrlWithRetryAsync(IHostEnvironment env, I
                 if (string.IsNullOrEmpty(thisDownloadResult))
                     return thisDownloadResult;
                 else
+                {
                     downloadResult += thisDownloadResult + @"\n";
-
+                    // Do not retry if the URL does not exist
+                    if (thisDownloadResult.Contains("does not exist"))
+                        return downloadResult;
+                }
                 await Task.Delay(10 * 1000);
             }
 
@@ -136,7 +139,7 @@ private async Task<string> DownloadFromUrlWithRetryAsync(IHostEnvironment env, I
         /// <returns>Returns the error message if an error occurred, null if download was successful.</returns>
         private async Task<string> DownloadFromUrlAsync(IHostEnvironment env, IChannel ch, string url, string fileName, int timeout, string filePath)
         {
-            using (var webClient = new WebClient())
+            using (var webClient = new WebClientResponse())
             using (var downloadCancel = new CancellationTokenSource())
             {
                 bool deleteNeeded = false;
@@ -160,27 +163,8 @@ private async Task<string> DownloadFromUrlAsync(IHostEnvironment env, IChannel c
                     deleteNeeded = true;
                     return (await t).Message;
                 }
-
-                return CheckValidDownload(ch, filePath, url, ref deleteNeeded);
-            }
-        }
-
-        private static string CheckValidDownload(IChannel ch, string filePath, string url, ref bool deleteNeeded)
-        {
-            // If the relative url does not exist, aka.ms redirects to www.microsoft.com. Make sure this did not happen.
-            // If the file is big then it is definitely not the redirect.
-            var info = new FileInfo(filePath);
-            if (info.Length > 4096)
                 return null;
-            string error = null;
-            using (var r = new StreamReader(filePath))
-            {
-                var text = r.ReadToEnd();
-                if (text.Contains("<head>") && text.Contains("<body>") && text.Contains("microsoft.com"))
-                    error = $"The url '{url}' does not exist. Url was redirected to www.microsoft.com.";
             }
-            deleteNeeded = error != null;
-            return error;
         }
 
         private static void TryDelete(IChannel ch, string filePath, bool warn = true)
@@ -252,7 +236,7 @@ private static string GetFilePath(IChannel ch, string fileName, string dir, out
             return filePath;
         }
 
-        private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient webClient, Uri uri, string path, string fileName, CancellationToken ct)
+        private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClientResponse webClient, Uri uri, string path, string fileName, CancellationToken ct)
         {
             if (File.Exists(path))
                 return null;
@@ -274,6 +258,8 @@ private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient
                 using (var ws = fh.CreateWriteStream())
                 {
                     var headers = webClient.ResponseHeaders.GetValues("Content-Length");
+                    if (IsRedirectToDefaultPage(uri))
+                        return ch.Except($"Invalid aka.ms url: {uri}");
                     if (Utils.Size(headers) == 0 || !long.TryParse(headers[0], out var size))
                         size = 10000000;
 
@@ -311,6 +297,28 @@ private Exception DownloadResource(IHostEnvironment env, IChannel ch, WebClient
             }
         }
 
+        public bool IsRedirectToDefaultPage(Uri uri)
+        {
+            try
+            {
+                HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(uri.AbsoluteUri);
+                myHttpWebRequest.AllowAutoRedirect = false;
+                HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();
+            }
+            catch (WebException e)
+            {
+                // Redirects to https://www.microsoft.com/en-us/?ref=aka
+                if (e.Message == "The remote server returned an error: (302) Moved Temporarily.")
+                    return true;
+                // Redirects to another url
+                else if (e.Message == "The remote server returned an error: (301) Moved Permanently.")
+                    return false;
+                else
+                    return false;
+            }
+            return false;
+        }
+
         public static ResourceDownloadResults GetErrorMessage(out string errorMessage, params ResourceDownloadResults[] result)
         {
             var errorResult = result.FirstOrDefault(res => !string.IsNullOrEmpty(res.ErrorMessage));
@@ -328,4 +336,15 @@ public static ResourceDownloadResults GetErrorMessage(out string errorMessage, p
         private static extern int chmod(string pathname, int mode);
 #pragma warning restore IDE1006
     }
+
+    public class WebClientResponse : WebClient
+    {
+        public WebResponse Response { get; private set; }
+
+        protected override WebResponse GetWebResponse(WebRequest request)
+        {
+            Response = base.GetWebResponse(request);
+            return Response;
+        }
+    }
 }
diff --git a/src/Microsoft.ML.Transforms/Text/WordEmbeddingsExtractor.cs b/src/Microsoft.ML.Transforms/Text/WordEmbeddingsExtractor.cs
@@ -628,7 +628,7 @@ private string EnsureModelFile(IHostEnvironment env, out int linesToSkip, WordEm
                 {
                     string dir = kind == WordEmbeddingEstimator.PretrainedModelKind.SentimentSpecificWordEmbedding ? Path.Combine("Text", "Sswe") : "WordVectors";
                     var url = $"{dir}/{modelFileName}";
-                    var ensureModel = ResourceManagerUtils.Instance.EnsureResourceAsync(Host, ch, url, modelFileName, dir, Timeout);
+                    var ensureModel = ResourceManagerUtils.Instance.EnsureResourceAsync(env, ch, url, modelFileName, dir, Timeout);
                     ensureModel.Wait();
                     var errorResult = ResourceManagerUtils.GetErrorMessage(out var errorMessage, ensureModel.Result);
                     if (errorResult != null)