Use direct URLs for test resources

test/Microsoft.ML.AutoML.Tests/DatasetUtil.cs

@@ -185,7 +185,7 @@ public static IEnumerable<ImageData> LoadImagesFromDirectory(string folder)
         public static string DownloadImageSet(string imagesDownloadFolder)
         {
             string fileName = "flower_photos_tiny_set_for_unit_tests.zip";
-            string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_tiny_set_for_unit_test.zip";
+            string url = "https://mlpublicassets.blob.core.windows.net/assets/datasets/flower_photos_tiny_set_for_unit_tests.zip";
 
             Download(url, imagesDownloadFolder, fileName).Wait();
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);

test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs

@@ -4,8 +4,6 @@
 
 using System;
 using System.IO;
-using Microsoft.ML.Internal.Utilities;
-using Microsoft.ML.Runtime;
 using Microsoft.ML.TestFrameworkCommon;
 
 namespace Microsoft.ML.PerformanceTests
@@ -29,37 +27,29 @@ static BenchmarkBase()
         // is running is it sometime cause process hanging when the constructor trying 
         // to load MKL, this is related to below issue:
         // https://github.com/dotnet/machinelearning/issues/1073
-        public static string GetBenchmarkDataPathAndEnsureData(string name, string path = "")
+        public static string GetLocalBenchmarkDataPath(string name)
         {
             if (string.IsNullOrWhiteSpace(name))
                 return null;
 
-            var filePath = path == "" ?
-                Path.GetFullPath(Path.Combine(DataDir, name)) :
-                Path.GetFullPath(Path.Combine(DataDir, path, name));
+            return Path.GetFullPath(Path.Combine(DataDir, name));
+        }
+
+        public static string GetBenchmarkDataPathAndEnsureData(string name, string path)
+        {
+            if (string.IsNullOrWhiteSpace(name))
+                return null;
 
+            var filePath = Path.GetFullPath(Path.Combine(DataDir, path, name));
             if (File.Exists(filePath))
                 return filePath;
 
-            var mlContext = new MLContext(1);
-            int timeout = 10 * 60 * 1000;
-            string url = $"benchmarks/{name}";
-            var localPath = path == "" ?
-                Path.GetFullPath(DataDir) :
-                Path.GetFullPath(Path.Combine(DataDir, path));
-
-            using (var ch = (mlContext as IHostEnvironment).Start("Ensuring dataset files are present."))
-            {
-                var ensureModel = ResourceManagerUtils.Instance.EnsureResourceAsync(
-                    mlContext, ch, url, name, localPath, timeout);
-                ensureModel.Wait();
-                var errorResult = ResourceManagerUtils.GetErrorMessage(out var errorMessage, ensureModel.Result);
-                if (errorResult != null)
-                {
-                    throw ch.Except($"{errorMessage}\n{name} could not be downloaded!");
-                }
-            }
+            var blobName = Path.GetFileName(name);
+            if (blobName == "WikiDetoxAnnotated160kRows.tsv")
+                blobName = "wikiDetoxAnnotated160kRows.tsv";
+            string url = $"https://mlpublicassets.blob.core.windows.net/assets/benchmarks/{blobName}";
 
+            TestDownloadUtils.DownloadFile(url, filePath, TimeSpan.FromMinutes(10));
             return filePath;
         }
     }

test/Microsoft.ML.PerformanceTests/HashBench.cs

@@ -100,7 +100,7 @@ private void InitMapMurmurHashV1<T>(T val, DataViewType type, ValueGetter<T> get
                 getter = (ref T dst) => dst = val;
             _inRow = RowImpl.Create(type, getter);
 
-            var modelPath = GetBenchmarkDataPathAndEnsureData("backcompat/MurmurHashV1.zip");
+            var modelPath = GetLocalBenchmarkDataPath("backcompat/MurmurHashV1.zip");
             var estimator = _env.Model.Load(modelPath, out var schema);
             var mapper = ((ITransformer)estimator).GetRowToRowMapper(_inRow.Schema);
             var column = mapper.OutputSchema["Bar"];

test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs

@@ -139,7 +139,7 @@ public static string DownloadImageSet(string imagesDownloadFolder)
 
             //SINGLE SMALL FLOWERS IMAGESET (200 files)
             string fileName = "flower_photos_small_set.zip";
-            string url = $"https://aka.ms/mlnet-resources/datasets/flower_photos_small_set.zip/";
+            string url = "https://mlpublicassets.blob.core.windows.net/assets/datasets/flower_photos_small_set.zip";
 
             Download(url, imagesDownloadFolder, fileName);
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);

test/Microsoft.ML.PerformanceTests/KMeansAndLogisticRegressionBench.cs

@@ -13,7 +13,7 @@ namespace Microsoft.ML.PerformanceTests
     [CIBenchmark]
     public class KMeansAndLogisticRegressionBench : BenchmarkBase
     {
-        private readonly string _dataPath = GetBenchmarkDataPathAndEnsureData("adult.with-schema.txt");
+        private readonly string _dataPath = GetLocalBenchmarkDataPath("adult.with-schema.txt");
 
         [Benchmark]
         public CalibratedModelParametersBase<LinearBinaryModelParameters, PlattCalibrator> TrainKMeansAndLR()

test/Microsoft.ML.PerformanceTests/PredictionEngineBench.cs

@@ -33,7 +33,7 @@ public void SetupIrisPipeline()
                 PetalWidth = 5.1f,
             };
 
-            string irisDataPath = GetBenchmarkDataPathAndEnsureData("iris.txt");
+            string irisDataPath = GetLocalBenchmarkDataPath("iris.txt");
 
             var env = new MLContext(seed: 1);
 
@@ -72,7 +72,7 @@ public void SetupSentimentPipeline()
                 SentimentText = "Not a big fan of this."
             };
 
-            string sentimentDataPath = GetBenchmarkDataPathAndEnsureData("wikipedia-detox-250-line-data.tsv");
+            string sentimentDataPath = GetLocalBenchmarkDataPath("wikipedia-detox-250-line-data.tsv");
 
             var mlContext = new MLContext(seed: 1);
 
@@ -107,7 +107,7 @@ public void SetupBreastCancerPipeline()
                 Features = new[] { 5f, 1f, 1f, 1f, 2f, 1f, 3f, 1f, 1f }
             };
 
-            string breastCancerDataPath = GetBenchmarkDataPathAndEnsureData("breast-cancer.txt");
+            string breastCancerDataPath = GetLocalBenchmarkDataPath("breast-cancer.txt");
 
             var env = new MLContext(seed: 1);
 

test/Microsoft.ML.PerformanceTests/StochasticDualCoordinateAscentClassifierBench.cs

@@ -17,8 +17,8 @@ namespace Microsoft.ML.PerformanceTests
     [CIBenchmark]
     public class StochasticDualCoordinateAscentClassifierBench : WithExtraMetrics
     {
-        private readonly string _dataPath = GetBenchmarkDataPathAndEnsureData("iris.txt");
-        private readonly string _sentimentDataPath = GetBenchmarkDataPathAndEnsureData("wikipedia-detox-250-line-data.tsv");
+        private readonly string _dataPath = GetLocalBenchmarkDataPath("iris.txt");
+        private readonly string _sentimentDataPath = GetLocalBenchmarkDataPath("wikipedia-detox-250-line-data.tsv");
         private readonly Consumer _consumer = new Consumer(); // BenchmarkDotNet utility type used to prevent dead code elimination
 
         private readonly MLContext _mlContext = new MLContext(seed: 1);

test/Microsoft.ML.PerformanceTests/TextPredictionEngineCreation.cs

@@ -22,7 +22,7 @@ public void Setup()
         {
             _context = new MLContext(1);
             var data = _context.Data.LoadFromTextFile<SentimentData>(
-                GetBenchmarkDataPathAndEnsureData("wikipedia-detox-250-line-data.tsv"), hasHeader: true);
+                GetLocalBenchmarkDataPath("wikipedia-detox-250-line-data.tsv"), hasHeader: true);
 
             // Pipeline.
             var pipeline = _context.Transforms.Text.FeaturizeText("Features", "SentimentText")

test/Microsoft.ML.TensorFlow.Tests/TensorflowTests.cs

@@ -8,8 +8,6 @@
 using System.IO.Compression;
 using System.Linq;
 using Microsoft.ML.Data;
-using Microsoft.ML.Internal.Utilities;
-using Microsoft.ML.Runtime;
 using Microsoft.ML.TensorFlow;
 using Microsoft.ML.TestFramework;
 using Microsoft.ML.TestFramework.Attributes;
@@ -61,7 +59,6 @@ public sealed class TensorFlowScenariosTests : BaseTestClass, IClassFixture<Tens
     {
         private readonly string _fullImagesetFolderPath;
         private readonly string _finalImagesFolderName;
-        private string _timeOutOldValue;
         private MLContext _mlContext = new MLContext(seed: 1);
 
         public TensorFlowScenariosTests(ITestOutputHelper output) : base(output)
@@ -77,19 +74,6 @@ public TensorFlowScenariosTests(ITestOutputHelper output) : base(output)
                 imagesDownloadFolderPath, _finalImagesFolderName);
         }
 
-        protected override void Initialize()
-        {
-            // set timeout to 3 minutes, download sometimes will stuck so set smaller timeout to fail fast and retry download
-            _timeOutOldValue = Environment.GetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable);
-            Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, (3 * 60 * 1000).ToString());
-        }
-
-        protected override void Cleanup()
-        {
-            // set back timeout value
-            Environment.SetEnvironmentVariable(ResourceManagerUtils.TimeoutEnvVariable, _timeOutOldValue);
-        }
-
         private class TestData
         {
             [VectorType(4)]
@@ -1952,7 +1936,7 @@ public string DownloadImageSet(string imagesDownloadFolder)
         {
             string fileName = "flower_photos_tiny_set_for_unit_tests.zip";
             string filenameAlias = "FPTSUT"; // FPTSUT = flower photos tiny set for unit tests
-            string url = "datasets/flower_photos_tiny_set_for_unit_test.zip";
+            string url = "https://mlpublicassets.blob.core.windows.net/assets/datasets/flower_photos_tiny_set_for_unit_tests.zip";
 
             Download(url, imagesDownloadFolder, fileName);
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
@@ -1965,7 +1949,7 @@ public string DownloadImageSet(string imagesDownloadFolder)
         public string DownloadBadImageSet(string imagesDownloadFolder)
         {
             string fileName = "CatsVsDogs_tiny_for_unit_tests.zip";
-            string url = "datasets/CatsVsDogs_tiny_for_unit_tests.zip";
+            string url = "https://mlpublicassets.blob.core.windows.net/assets/datasets/CatsVsDogs_tiny_for_unit_tests.zip";
 
             Download(url, imagesDownloadFolder, fileName);
             UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
@@ -1985,20 +1969,7 @@ private bool Download(string url, string destDir, string destFileName)
             if (File.Exists(relativeFilePath))
                 return false;
 
-            int timeout = 10 * 60 * 1000;
-            using (var ch = (_mlContext as IHostEnvironment).Start("Ensuring image files are present."))
-            {
-                var ensureModel = ResourceManagerUtils.Instance.EnsureResourceAsync(_mlContext, ch, url, destFileName, destDir, timeout);
-                ensureModel.Wait();
-                var errorResult = ResourceManagerUtils.GetErrorMessage(out var errorMessage, ensureModel.Result);
-                if (errorResult != null)
-                {
-                    var directory = Path.GetDirectoryName(errorResult.FileName);
-                    var name = Path.GetFileName(errorResult.FileName);
-                    throw ch.Except($"{errorMessage}\nImage file could not be downloaded!");
-                }
-            }
-
+            TestDownloadUtils.DownloadFile(url, relativeFilePath, TimeSpan.FromMinutes(3));
             return true;
         }
 

test/Microsoft.ML.TestFrameworkCommon/Microsoft.ML.TestFrameworkCommon.csproj

@@ -2,6 +2,10 @@
 
   <Import Project="..\TestFrameworkDependency.props" />
 
+  <PropertyGroup>
+    <Nullable>annotations</Nullable>
+  </PropertyGroup>
+
   <ItemGroup>
     <None Update="xunit.runner.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -13,4 +17,8 @@
     <PackageReference Include="Microsoft.DotNet.PlatformAbstractions" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Compile Include="..\..\src\Common\tests\RetryHelper.cs" Link="RetryHelper.cs" />
+  </ItemGroup>
+
 </Project>

test/Microsoft.ML.TestFrameworkCommon/TestDownloadUtils.cs

@@ -0,0 +1,63 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading;
+
+namespace Microsoft.ML.TestFrameworkCommon
+{
+    public static class TestDownloadUtils
+    {
+        public static void DownloadFile(string url, string filePath, TimeSpan timeout)
+        {
+            var directory = Path.GetDirectoryName(filePath);
+            if (!string.IsNullOrEmpty(directory))
+                Directory.CreateDirectory(directory);
+
+            string temporaryFilePath = $"{filePath}.{Guid.NewGuid():N}.download";
+            using (var client = new HttpClient { Timeout = Timeout.InfiniteTimeSpan })
+            {
+                RetryHelper.Execute(() =>
+                {
+                    try
+                    {
+                        if (File.Exists(filePath))
+                            return;
+
+                        using (var cancellationSource = new CancellationTokenSource(timeout))
+                        using (var response = client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationSource.Token).GetAwaiter().GetResult())
+                        {
+                            response.EnsureSuccessStatusCode();
+                            long? expectedLength = response.Content.Headers.ContentLength;
+                            using (var contentStream = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult())
+                            using (var fileStream = new FileStream(temporaryFilePath, FileMode.Create, FileAccess.Write, FileShare.None))
+                            {
+                                contentStream.CopyToAsync(fileStream, 81920, cancellationSource.Token).GetAwaiter().GetResult();
+                                if (expectedLength.HasValue && fileStream.Length != expectedLength.Value)
+                                    throw new IOException($"Expected {expectedLength.Value} bytes from '{url}', but downloaded {fileStream.Length} bytes.");
+                            }
+                        }
+
+                        try
+                        {
+                            File.Move(temporaryFilePath, filePath);
+                        }
+                        catch (IOException) when (File.Exists(filePath))
+                        {
+                            // Another caller completed the same download first.
+                        }
+                    }
+                    finally
+                    {
+                        if (File.Exists(temporaryFilePath))
+                            File.Delete(temporaryFilePath);
+                    }
+                }, backoffFunc: attempt => 10_000,
+                    retryWhen: ex => ex is HttpRequestException || ex is OperationCanceledException || ex is IOException);
+            }
+        }
+    }
+}