Description
I am not sure whether this is write channel but I need quick help
We have observed that gradual increase in memory of Host object of module Microsoft.ML.Data.
We have .net core worker service running as windows service.
It does prediction for 50 ML models in parallel.
-
We are loading all models into memory then creating 50 threads for 50 models which would be running continuously and predicting values.
-
For testing purpose we stop all threads ever 1hour and dispose it. Then calling GC.Collect().
-
Also taking memory snapshot every 1 hour using VS performance profiler. We have observed that
gradual increase in memory of Host object of module Microsoft.ML.Data. -
If we remove all loded models from memory and call GC. the the memory consumed by Host object of module Microsoft.ML.Data gets freed.
Could you please suggest anything if we want to resolve gradual increase of memory without removing models from memory.
private List<PredictionResult> ExecuteModel(ModelTransformer modelTransformer, IDataView dataView, DateTime requestDateTime)
{
List<PredictionResult> predictionResults = null;
try
{
if (modelTransformer.Transformer != null)
{
string modelConfigId = modelTransformer.ModelConfigId;
string actualValueColumnName = modelTransformer.TextLoaderOutputColumn;
string tagNameForPredictedValue = modelTransformer.PredictedTagName;
string tagNameForActualValue = modelTransformer.TargetVariable;
predictionResults = new List<PredictionResult>();
IDataView predictions = modelTransformer.Transformer.Transform(dataView);
Logger.log.InfoFormat("[Request at {0} Model Id: {1}] prediction value extraction", requestDateTime, modelConfigId);
IEnumerable<float> listScore = predictions.GetColumn<float>("Score");
Logger.log.InfoFormat("[Request at {0} Model Id: {1}] score extraction completed", requestDateTime, modelConfigId);
IEnumerable<string> listTime = predictions.GetColumn<string>(preModelExecutionDetails.TextLoaderColumn[0].Name);
Logger.log.InfoFormat("[Request at {0} Model Id: {1}] Time extraction completed", requestDateTime, modelConfigId);
IEnumerable<float> listActualValues = predictions.GetColumn<float>(actualValueColumnName);
predictions = null;
Logger.log.InfoFormat("[Request at {0} Model Id: {1}] actual value extraction completed", requestDateTime, modelConfigId);
predictionResults.AddRange(
listTime.Zip(listScore, (time, score) => new PredictionResult
{
TagName = tagNameForPredictedValue,
Timestamp = Convert.ToDateTime(time),
Confidence = ConfigParameters.TagValueDefaultConfidence,
Value = score.ToString(),
ValueType = Models.ValueType.SCORE.ToString()
}).ToList());
predictionResults.AddRange(
listTime.Zip(listActualValues, (time, actualVal) => new PredictionResult
{
TagName = tagNameForActualValue,
Timestamp = Convert.ToDateTime(time),
Confidence = ConfigParameters.TagValueDefaultConfidence,
Value = actualVal.ToString(),
ValueType = Models.ValueType.ACTUAL.ToString()
}).ToList());
}
}
finally
{
modelTransformer = null;
}
return predictionResults;
}