Fixed & Reactivated AutoFitImageClassificationTrainTest hanging by freeing old Tensor objects

src/Microsoft.ML.AutoML/API/ExperimentResults/CrossValidationExperimentResult.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System;
 using System.Collections.Generic;
 using Microsoft.ML.Data;
 
@@ -11,7 +12,7 @@ namespace Microsoft.ML.AutoML
     /// Result of an AutoML experiment that includes cross validation details.
     /// </summary>
     /// <typeparam name="TMetrics">Metrics type for the experiment (like <see cref="BinaryClassificationMetrics"/>).</typeparam>
-    public class CrossValidationExperimentResult<TMetrics>
+    public class CrossValidationExperimentResult<TMetrics> : IDisposable
     {
         /// <summary>
         /// Details of the cross validation runs in this experiment.
@@ -35,6 +36,46 @@ internal CrossValidationExperimentResult(IEnumerable<CrossValidationRunDetail<TM
         {
             RunDetails = runDetails;
             BestRun = bestRun;
+            _disposed = false;
+            _disposedRunDetails = false;
         }
+
+        #region IDisposable Support
+        private bool _disposed;
+        private bool _disposedRunDetails;
+
+        /// <summary>
+        /// Releases unmanaged Tensor objects in models stored in RunDetail and BestRun instances
+        /// </summary>
+        /// <remarks>
+        /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and
+        /// avoid a memory leak
+        /// </remarks>
+        public void Dispose()
+        {
+            if (_disposed)
+                return;
+            if (!_disposedRunDetails)
+                (RunDetails as IDisposable)?.Dispose();
+            (BestRun as IDisposable)?.Dispose();
+            _disposed = true;
+            _disposedRunDetails = true;
+        }
+
+        /// <summary>
+        /// Releases unmanaged Tensor objects in models stored in RunDetail instances
+        /// </summary>
+        /// <remarks>
+        /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and
+        /// avoid a memory leak
+        /// </remarks>
+        public void DisposeRunDetails()
+        {
+            if (_disposedRunDetails || _disposed)
+                return;
+            (RunDetails as IDisposable)?.Dispose();
+            _disposedRunDetails = true;
+        }
+        #endregion
     }
 }

src/Microsoft.ML.AutoML/API/ExperimentResults/ExperimentResult.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System;
 using System.Collections.Generic;
 using Microsoft.ML.Data;
 
@@ -11,7 +12,7 @@ namespace Microsoft.ML.AutoML
     /// Result of an AutoML experiment.
     /// </summary>
     /// <typeparam name="TMetrics">Metrics type for the experiment (like <see cref="BinaryClassificationMetrics"/>).</typeparam>
-    public class ExperimentResult<TMetrics>
+    public class ExperimentResult<TMetrics> : IDisposable
     {
         /// <summary>
         /// Details of the runs in this experiment.
@@ -35,6 +36,46 @@ internal ExperimentResult(IEnumerable<RunDetail<TMetrics>> runDetails,
         {
             RunDetails = runDetails;
             BestRun = bestRun;
+            _disposed = false;
+            _disposedRunDetails = false;
         }
+
+        #region IDisposable Support
+        private bool _disposed;
+        private bool _disposedRunDetails;
+
+        /// <summary>
+        /// Releases unmanaged Tensor objects in models stored in RunDetail and BestRun instances
+        /// </summary>
+        /// <remarks>
+        /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and
+        /// avoid a memory leak
+        /// </remarks>
+        public void Dispose()
+        {
+            if (_disposed)
+                return;
+            if (!_disposedRunDetails)
+                (RunDetails as IDisposable)?.Dispose();
+            (BestRun as IDisposable)?.Dispose();
+            _disposed = true;
+            _disposedRunDetails = true;
+        }
+
+        /// <summary>
+        /// Releases unmanaged Tensor objects in models stored in RunDetail instances
+        /// </summary>
+        /// <remarks>
+        /// Invocation of Dispose() is necessary to clean up remaining C library Tensor objects and
+        /// avoid a memory leak
+        /// </remarks>
+        public void DisposeRunDetails()
+        {
+            if (_disposedRunDetails || _disposed)
+                return;
+            (RunDetails as IDisposable)?.Dispose();
+            _disposedRunDetails = true;
+        }
+        #endregion
     }
 }

src/Microsoft.ML.AutoML/Experiment/ModelContainer.cs

@@ -10,13 +10,6 @@ internal class ModelContainer
     {
         private readonly MLContext _mlContext;
         private readonly FileInfo _fileInfo;
-        private readonly ITransformer _model;
-
-        internal ModelContainer(MLContext mlContext, ITransformer model)
-        {
-            _mlContext = mlContext;
-            _model = model;
-        }
 
         internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer model, DataViewSchema modelInputSchema)
         {
@@ -32,12 +25,6 @@ internal ModelContainer(MLContext mlContext, FileInfo fileInfo, ITransformer mod
 
         public ITransformer GetModel()
         {
-            // If model stored in memory, return it
-            if (_model != null)
-            {
-                return _model;
-            }
-
             // Load model from disk
             ITransformer model;
             using (var stream = new FileStream(_fileInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.Read))

src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs

@@ -53,7 +53,7 @@ public CrossValRunner(MLContext context,
                 var modelFileInfo = RunnerUtil.GetModelFileInfo(modelDirectory, iterationNum, i + 1);
                 var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainDatasets[i], _validDatasets[i],
                     _labelColumn, _metricsAgent, _preprocessorTransforms?[i], modelFileInfo, _modelInputSchema, _logger);
-                trainResults.Add(new SuggestedPipelineTrainResult<TMetrics>(trainResult.model, trainResult.metrics, trainResult.exception, trainResult.score));
+                trainResults.Add(new SuggestedPipelineTrainResult<TMetrics>(trainResult.modelContainer, trainResult.metrics, trainResult.exception, trainResult.score));
             }
 
             var avgScore = CalcAverageScore(trainResults.Select(r => r.Score));

src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs

@@ -10,7 +10,7 @@ namespace Microsoft.ML.AutoML
 {
     internal static class RunnerUtil
     {
-        public static (ModelContainer model, TMetrics metrics, Exception exception, double score)
+        public static (ModelContainer modelContainer, TMetrics metrics, Exception exception, double score)
             TrainAndScorePipeline<TMetrics>(MLContext context,
             SuggestedPipeline pipeline,
             IDataView trainData,
@@ -37,9 +37,7 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub
                 }
 
                 // Build container for model
-                var modelContainer = modelFileInfo == null ?
-                    new ModelContainer(context, model) :
-                    new ModelContainer(context, modelFileInfo, model, modelInputSchema);
+                var modelContainer = new ModelContainer(context, modelFileInfo, model, modelInputSchema);
 
                 return (modelContainer, metrics, null, score);
             }
@@ -53,7 +51,7 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub
         public static FileInfo GetModelFileInfo(DirectoryInfo modelDirectory, int iterationNum, int foldNum)
         {
             return modelDirectory == null ?
-                null :
+                new FileInfo(Path.Combine(Path.GetTempPath(), $"Model{iterationNum}_{foldNum}.zip")) :
                 new FileInfo(Path.Combine(modelDirectory.FullName, $"Model{iterationNum}_{foldNum}.zip"));
         }
     }

src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs

@@ -50,7 +50,7 @@ public TrainValidateRunner(MLContext context,
                 trainResult.score,
                 trainResult.exception == null,
                 trainResult.metrics,
-                trainResult.model,
+                trainResult.modelContainer,
                 trainResult.exception);
             var runDetail = suggestedPipelineRunDetail.ToIterationResult(_preFeaturizer);
             return (suggestedPipelineRunDetail, runDetail);
@@ -59,7 +59,7 @@ public TrainValidateRunner(MLContext context,
         private static FileInfo GetModelFileInfo(DirectoryInfo modelDirectory, int iterationNum)
         {
             return modelDirectory == null ?
-                null :
+                new FileInfo(Path.Combine(Path.GetTempPath(), $"Model{iterationNum}.zip")) :
                 new FileInfo(Path.Combine(modelDirectory.FullName, $"Model{iterationNum}.zip"));
         }
     }

test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs

@@ -36,6 +36,7 @@ public void AutoFitBinaryTest()
             Assert.NotNull(result.BestRun.Estimator);
             Assert.NotNull(result.BestRun.Model);
             Assert.NotNull(result.BestRun.TrainerName);
+            result.Dispose();
         }
 
         [Fact]
@@ -51,11 +52,10 @@ public void AutoFitMultiTest()
             Assert.True(result.BestRun.Results.First().ValidationMetrics.MicroAccuracy >= 0.7);
             var scoredData = result.BestRun.Results.First().Model.Transform(trainData);
             Assert.Equal(NumberDataViewType.Single, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type);
+            result.Dispose();
         }
 
         [TensorFlowFact]
-        //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined
-        [Trait("Category", "SkipInCI")]
         public void AutoFitImageClassificationTrainTest()
         {
             var context = new MLContext(seed: 1);
@@ -75,6 +75,7 @@ public void AutoFitImageClassificationTrainTest()
 
             var scoredData = result.BestRun.Model.Transform(trainData);
             Assert.Equal(TextDataViewType.Instance, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type);
+            result.Dispose();
         }
 
         [Fact(Skip ="Takes too much time, ~10 minutes.")]
@@ -96,6 +97,7 @@ public void AutoFitImageClassification()
             Assert.InRange(result.BestRun.ValidationMetrics.MicroAccuracy, 0.80, 0.9);
             var scoredData = result.BestRun.Model.Transform(trainData);
             Assert.Equal(TextDataViewType.Instance, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type);
+            result.Dispose();
         }
 
         private void Context_Log(object sender, LoggingEventArgs e)
@@ -119,6 +121,7 @@ public void AutoFitRegressionTest()
                     new ColumnInformation() { LabelColumnName = DatasetUtil.MlNetGeneratedRegressionLabel });
 
             Assert.True(result.RunDetails.Max(i => i.ValidationMetrics.RSquared > 0.9));
+            result.Dispose();
         }
 
         [Fact]
@@ -165,6 +168,7 @@ public void AutoFitRecommendationTest()
 
             var metrices = mlContext.Recommendation().Evaluate(testDataViewWithBestScore, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName);
             Assert.NotEqual(0, metrices.MeanSquaredError);
+            experimentResult.Dispose();
         }
 
         private TextLoader.Options GetLoaderArgs(string labelColumnName, string userIdColumnName, string itemIdColumnName)