BaseTestPredictorsMaml.cs

// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using Microsoft.ML.Runtime;
using Microsoft.ML.TestFrameworkCommon;

namespace Microsoft.ML.RunTests
{
    using ResultProcessor = ResultProcessor.ResultProcessor;

    /// <summary>
    /// This is a base test class designed to support running trainings and related
    /// commands, and comparing the results against baselines.
    /// </summary>
    public abstract partial class BaseTestPredictors : TestDmCommandBase
    {
        public enum Cmd
        {
            TrainTest,
            Train,
            Test,
            CV
        }

        /// <summary>
        /// A generic class for a test run.
        /// </summary>
        protected sealed class RunContext : RunContextBase
        {
            public readonly Cmd Command;
            public readonly PredictorAndArgs Predictor;
            public readonly TestDataset Dataset;

            public readonly string[] ExtraArgs;
            public readonly string ExtraTag;

            public readonly bool ExpectedToFail;
            public readonly bool Summary;
            public readonly bool SaveAsIni;

            public readonly OutputPath ModelOverride;

            public override bool NoComparisons { get { return true; } }

            public RunContext(TestCommandBase test, Cmd cmd, PredictorAndArgs predictor, TestDataset dataset,
                string[] extraArgs = null, string extraTag = "",
                bool expectFailure = false, OutputPath modelOverride = null, bool summary = false, bool saveAsIni = false)
                : base(test, predictor.Trainer.Kind, GetNamePrefix(cmd.ToString(), predictor, dataset, extraTag), predictor.BaselineProgress)
            {
                Command = cmd;
                Predictor = predictor;
                Dataset = dataset;

                ExtraArgs = extraArgs;
                ExtraTag = extraTag;

                ExpectedToFail = expectFailure;
                Summary = summary;

                ModelOverride = modelOverride;
                SaveAsIni = saveAsIni;
            }

            public override OutputPath ModelPath()
            {
                return ModelOverride ?? base.ModelPath();
            }

            public RunContextBase TestCtx()
            {
                return new TestImpl(this);
            }

            private sealed class TestImpl : RunContextBase
            {
                public override bool NoComparisons { get { return true; } }

                public TestImpl(RunContextBase ctx) :
                    base(ctx.Test, ctx.BaselineDir, ctx.BaselineNamePrefix + "-test", ctx.BaselineProgress)
                {
                }
            }
        }

        public delegate bool Equal<T1, T2>(ref T1 a, ref T2 b, out int nonEqualIdx);

        /// <summary>
        /// Run the predictor with given args and check if it adds up
        /// </summary>
        protected void Run(RunContext ctx, int digitsOfPrecision = DigitsOfPrecision, NumberParseOption parseOption = NumberParseOption.Default)
        {
            Contracts.Assert(IsActive);
            List<string> args = new List<string>();
            if (ctx.Command != Cmd.Test)
                AddIfNotEmpty(args, ctx.Predictor.Trainer, "tr");
            string dataName = ctx.Command == Cmd.Test ? ctx.Dataset.testFilename : ctx.Dataset.trainFilename;
            AddIfNotEmpty(args, GetDataPath(dataName), "data");
            AddIfNotEmpty(args, 1, "seed");
            //AddIfNotEmpty(args, false, "threads");

            Log("Running '{0}' on '{1}'", ctx.Predictor.Trainer.Kind, ctx.Dataset.name);

            string dir = ctx.BaselineDir;
            if (ctx.Command == Cmd.TrainTest)
                AddIfNotEmpty(args, GetDataPath(ctx.Dataset.testFilename), "test");
            if (ctx.Command == Cmd.TrainTest || ctx.Command == Cmd.Train)
                AddIfNotEmpty(args, GetDataPath(ctx.Dataset.validFilename), "valid");

            // Add in the loader args, and keep a location so we can backtrack and remove it later.
            int loaderArgIndex = -1;
            string loaderArgs = GetLoaderTransformSettings(ctx.Dataset);
            if (!string.IsNullOrWhiteSpace(loaderArgs))
            {
                loaderArgIndex = args.Count;
                args.Add(loaderArgs);
            }
            // Add in the dataset transforms. These need to come before the predictor imposed transforms.
            if (ctx.Dataset.mamlExtraSettings != null)
                args.AddRange(ctx.Dataset.mamlExtraSettings);

            // Model file output, used only for train/traintest.
            var modelPath = ctx.Command == Cmd.Train || ctx.Command == Cmd.TrainTest ? ctx.ModelPath() : null;
            AddIfNotEmpty(args, modelPath, "out");

            string basePrefix = ctx.BaselineNamePrefix;

            // Predictions output, for all types of commands except train.
            OutputPath predOutPath = ctx.Command == Cmd.Train ? null : ctx.InitPath(".txt");
            AddIfNotEmpty(args, predOutPath, "dout");

            if (ctx.Predictor.MamlArgs != null)
                args.AddRange(ctx.Predictor.MamlArgs);

            // If CV, do not run the CV in multiple threads.
            if (ctx.Command == Cmd.CV)
                args.Add("threads-");

            if (ctx.ExtraArgs != null)
            {
                foreach (string arg in ctx.ExtraArgs)
                    args.Add(arg);
            }

            AddIfNotEmpty(args, ctx.Predictor.Scorer, "scorer");
            if (ctx.Command != Cmd.Test)
                AddIfNotEmpty(args, ctx.Predictor.Tester, "eval");
            else
                AddIfNotEmpty(args, ctx.ModelOverride.Path, "in");

            string runcmd = string.Join(" ", args.Where(a => !string.IsNullOrWhiteSpace(a)));
            Log("  Running as: {0} {1}", ctx.Command, runcmd);

            int res;
            if (basePrefix == null)
            {
                // Not capturing into a specific log.
                Log("*** Start raw predictor output");
                res = MainForTest(_env, LogWriter, string.Join(" ", ctx.Command, runcmd), ctx.BaselineProgress);
                Log("*** End raw predictor output, return={0}", res);
                return;
            }
            var consOutPath = ctx.StdoutPath();
            TestCore(ctx, ctx.Command.ToString(), runcmd, digitsOfPrecision: digitsOfPrecision, parseOption: parseOption);
            bool matched = consOutPath.CheckEqualityNormalized(digitsOfPrecision, parseOption: parseOption);

            if (modelPath != null && (ctx.Summary || ctx.SaveAsIni))
            {
                // Save the predictor summary and compare it to baseline.
                string str = string.Format("SavePredictorAs in={{{0}}}", modelPath.Path);
                List<string> files = new List<string>();
                if (ctx.Summary)
                {
                    var summaryName = basePrefix + "-summary.txt";
                    files.Add(summaryName);
                    var summaryPath = DeleteOutputPath(dir, summaryName);
                    str += string.Format(" sum={{{0}}}", summaryPath);
                    Log("  Saving summary with: {0}", str);
                }

                if (ctx.SaveAsIni)
                {
                    var iniName = basePrefix + ".ini";
                    files.Add(iniName);
                    var iniPath = DeleteOutputPath(dir, iniName);
                    str += string.Format(" ini={{{0}}}", iniPath);
                    Log("  Saving ini file: {0}", str);
                }

                MainForTest(_env, LogWriter, str);
                files.ForEach(file => CheckEqualityNormalized(dir, file, digitsOfPrecision: digitsOfPrecision, parseOption: parseOption));
            }

            if (ctx.Command == Cmd.Train || ctx.Command == Cmd.Test || ctx.ExpectedToFail)
                return;

            // ResultProcessor output
            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // -rp.txt files are not getting generated for Non-Windows Os
            {
                string rpName = basePrefix + "-rp.txt";
                string rpOutPath = DeleteOutputPath(dir, rpName);

                string[] rpArgs = null;
                if (ctx.Command == Cmd.CV && ctx.ExtraArgs != null && ctx.ExtraArgs.Any(arg => arg.Contains("opf+")))
                    rpArgs = new string[] { "opf+" };

                // Run result processor on the console output.
                RunResultProcessorTest(new string[] { consOutPath.Path }, rpOutPath, rpArgs);
                CheckEqualityNormalized(dir, rpName, digitsOfPrecision:digitsOfPrecision, parseOption: parseOption);
            }

            // Check the prediction output against its baseline.
            Contracts.Assert(predOutPath != null);
            predOutPath.CheckEquality(digitsOfPrecision: digitsOfPrecision, parseOption: parseOption);

            if (ctx.Command == Cmd.TrainTest)
            {
                // Adjust the args so that we no longer have the loader and transform
                // arguments in there.
                if (loaderArgIndex >= 0)
                    args.RemoveAt(loaderArgIndex);
                bool foundOut = false;
                List<int> toRemove = new List<int>();
                HashSet<string> removeArgs = new HashSet<string>();
                removeArgs.Add("tr=");
                removeArgs.Add("data=");
                removeArgs.Add("valid=");
                removeArgs.Add("norm=");
                removeArgs.Add("cali=");
                removeArgs.Add("numcali=");
                removeArgs.Add("xf=");
                removeArgs.Add("cache-");
                removeArgs.Add("sf=");
                removeArgs.Add("loader=");

                for (int i = 0; i < args.Count; ++i)
                {
                    if (string.IsNullOrWhiteSpace(args[i]))
                        continue;
                    if (removeArgs.Any(x => args[i].StartsWith(x)))
                        toRemove.Add(i);
                    if (args[i].StartsWith("out="))
                        args[i] = string.Format("in={0}", args[i].Substring(4));
                    if (args[i].StartsWith("test="))
                        args[i] = string.Format("data={0}", args[i].Substring(5));
                    foundOut = true;
                }
                Contracts.Assert(foundOut);
                toRemove.Reverse();
                foreach (int i in toRemove)
                    args.RemoveAt(i);
                runcmd = string.Join(" ", args.Where(a => !string.IsNullOrWhiteSpace(a)));

                // Redirect output to the individual log and run the test.
                var ctx2 = ctx.TestCtx();
                OutputPath consOutPath2 = ctx2.StdoutPath();
                TestCore(ctx2, "Test", runcmd, digitsOfPrecision, parseOption);

                if (CheckTestOutputMatchesTrainTest(consOutPath.Path, consOutPath2.Path, 1))
                    File.Delete(consOutPath2.Path);
                else if (matched)
                {
                    // The TrainTest output matched the baseline, but the SaveLoadTest output did not, so
                    // append some stuff to the .txt output so comparing output to baselines in BeyondCompare
                    // will show the issue.
                    using (var writer = OpenWriter(consOutPath.Path, true))
                    {
                        writer.WriteLine("*** Unit Test Failure! ***");
                        writer.WriteLine("Loaded predictor test results differ! Compare baseline with {0}", consOutPath2.Path);
                        writer.WriteLine("*** Unit Test Failure! ***");
                    }
                }
                // REVIEW: There is nothing analogous to the old predictor output comparison here.
                // The MAML command does not "export" the result of its training programmatically, that would
                // allow us to compare it to the loaded model. To verify that the result of the trained model
                // is the same as its programmatic 
            }
        }

        protected void RunResultProcessorTest(string[] dataFiles, string outPath, string[] extraArgs)
        {
            Contracts.Assert(IsActive);

            File.Delete(outPath);

            List<string> args = new List<string>();
            for (int i = 0; i < dataFiles.Length; i++)
            {
                args.Add("\"" + dataFiles[i] + "\"");
            }
            args.Add("/o");
            args.Add(outPath);
            args.Add("/calledFromUnitTestSuite+");

            if (extraArgs != null)
                args.AddRange(extraArgs);
            ResultProcessor.Main(Env, args.ToArray());
        }

        private static string GetNamePrefix(string testType, PredictorAndArgs predictor, TestDataset dataset, string extraTag = "")
        {
            // REVIEW: Once we finish the TL->MAML conversion effort, please make the output/baseline
            // names take some form that someone could actually tell what test generated that file.

            string datasetSuffix = dataset.name;
            if (!string.IsNullOrEmpty(extraTag))
            {
                if (char.IsLetterOrDigit(extraTag[0]))
                    datasetSuffix += "." + extraTag;
                else
                    datasetSuffix += extraTag;
            }
            string filePrefix = (string.IsNullOrEmpty(predictor.Tag) ? predictor.Trainer.Kind : predictor.Tag);
            return filePrefix + "-" + testType + "-" + datasetSuffix;
        }

        /// <summary>
        /// Create a string for specifying the loader and transform.
        /// </summary>
        public string GetLoaderTransformSettings(TestDataset dataset)
        {
            List<string> settings = new List<string>();

            Contracts.Check(dataset.testSettings == null, "Separate test loader pipeline is not supported");

            if (!string.IsNullOrEmpty(dataset.loaderSettings))
                settings.Add(dataset.loaderSettings);
            if (!string.IsNullOrEmpty(dataset.labelFilename))
                settings.Add(string.Format("xf=lookup{{col=Label data={{{0}}}}}", GetDataPath(dataset.labelFilename)));

            return settings.Count > 0 ? string.Join(" ", settings) : null;
        }

        /// <summary>
        /// Run TrainTest and CV for a set of predictors on a set of datasets.
        /// </summary>
        protected void RunAllTests(
            IList<PredictorAndArgs> predictors, IList<TestDataset> datasets,
            string[] extraSettings = null, string extraTag = "", bool summary = false,
            int digitsOfPrecision = DigitsOfPrecision, NumberParseOption parseOption = NumberParseOption.Default)
        {
            Contracts.Assert(IsActive);
            foreach (TestDataset dataset in datasets)
            {
                foreach (PredictorAndArgs predictor in predictors)
                    RunOneAllTests(predictor, dataset, extraSettings, extraTag, summary, digitsOfPrecision, parseOption);
            }
        }

        /// <summary>
        /// Run TrainTest, CV, and TrainSaveTest for a single predictor on a single dataset.
        /// </summary>
        protected void RunOneAllTests(PredictorAndArgs predictor, TestDataset dataset,
            string[] extraSettings = null, string extraTag = "", bool summary = false,
            int digitsOfPrecision = DigitsOfPrecision, NumberParseOption parseOption = NumberParseOption.Default)
        {
            Contracts.Assert(IsActive);
            Run_TrainTest(predictor, dataset, extraSettings, extraTag, summary: summary, digitsOfPrecision: digitsOfPrecision, parseOption: parseOption);
            Run_CV(predictor, dataset, extraSettings, extraTag, useTest: true, digitsOfPrecision: digitsOfPrecision, parseOption: parseOption);
        }

        /// <summary>
        /// Run Train for a single predictor on a single dataset.
        /// </summary>
        protected RunContext RunOneTrain(PredictorAndArgs predictor, TestDataset dataset,
            string[] extraSettings = null, string extraTag = "")
        {
            Contracts.Assert(IsActive);
            return Run_Train(predictor, dataset, extraSettings, extraTag);
        }

        /// <summary>
        /// Run a train unit test
        /// </summary>
        protected RunContext Run_Train(PredictorAndArgs predictor, TestDataset dataset,
            string[] extraSettings = null, string extraTag = "")
        {
            RunContext ctx = new RunContext(this, Cmd.Train, predictor, dataset, extraSettings, extraTag);
            Run(ctx);
            return ctx;
        }

        /// <summary>
        /// Run a train-test unit test
        /// </summary>
        protected void Run_TrainTest(PredictorAndArgs predictor, TestDataset dataset,
            string[] extraSettings = null, string extraTag = "", bool expectFailure = false, bool summary = false,
            bool saveAsIni = false, int digitsOfPrecision = DigitsOfPrecision,
             NumberParseOption parseOption = NumberParseOption.Default)
        {
            RunContext ctx = new RunContext(this, Cmd.TrainTest, predictor, dataset, extraSettings, extraTag, expectFailure: expectFailure, summary: summary, saveAsIni: saveAsIni);
            Run(ctx, digitsOfPrecision, parseOption);
        }

        // REVIEW: Remove TrainSaveTest and supporting code.

        /// <summary>
        /// Run a unit test which does training, saves the model, and then tests
        /// after loading the model
        /// </summary>
        protected void Run_TrainSaveTest(PredictorAndArgs predictor, TestDataset dataset,
            string[] extraSettings = null, string extraTag = "")
        {
            // Train and save the model.
            RunContext trainCtx = new RunContext(this, Cmd.Train, predictor, dataset, extraSettings, extraTag);
            Run(trainCtx);
            // Load the model and test.
            RunContext testCtx = new RunContext(this, Cmd.Test, predictor, dataset, extraSettings, extraTag,
                modelOverride: trainCtx.ModelPath());
            Run(testCtx);
        }

        protected void Run_Test(PredictorAndArgs predictor, TestDataset dataset, string modelPath,
            string[] extraSettings = null, string extraTag = "")
        {
            OutputPath path = new OutputPath(modelPath);
            RunContext testCtx = new RunContext(this, Cmd.Test, predictor, dataset,
                extraSettings, extraTag, modelOverride: path);
            Run(testCtx);
        }

        /// <summary>
        /// Run a cross-validation unit test, over the training set, unless
        /// <paramref name="useTest"/> is set.
        /// </summary>
        protected void Run_CV(PredictorAndArgs predictor, TestDataset dataset,
            string[] extraSettings = null, string extraTag = "", bool useTest = false,
            int digitsOfPrecision = DigitsOfPrecision, NumberParseOption parseOption = NumberParseOption.Default)
        {
            if (useTest)
            {
                // REVIEW: It is very strange to use the *test* set in
                // cross validation. Should this just be deprecated outright?
                dataset = dataset.Clone();
                dataset.trainFilename = dataset.testFilename;
            }
            RunContext cvCtx = new RunContext(this, Cmd.CV, predictor, dataset, extraSettings, extraTag);
            Run(cvCtx, digitsOfPrecision, parseOption);
        }

        /// <summary>
        /// Add a /arg value pair if value is not null/empty
        /// </summary>
        private static void AddIfNotEmpty(List<string> list, OutputPath val, string name)
        {
            if (val != null && !string.IsNullOrWhiteSpace(val.Path))
                list.Add(val.ArgStr(name));
        }

        /// <summary>
        /// Add a /arg value pair if value is not null/empty
        /// </summary>
        private static void AddIfNotEmpty(List<string> list, object val, string name)
        {
            string sval = val as string;
            if (!string.IsNullOrWhiteSpace(sval) || ((sval == null) != (val == null)))
                list.Add(string.Format("{0}={1}", name, val));
        }

        /// <summary>
        /// Combine all sets of options
        /// </summary>
        public static string[] JoinOptions(params string[][] options)
        {
            List<string> optionsList = new List<string>();
            foreach (string[] o in options)
                if (o != null)
                    optionsList.AddRange(o);
            return optionsList.ToArray();
        }
    }
}