test: base changes to PR-1114

com.unity.netcode.gameobjects/Runtime/Messaging/MessageQueue/MessageQueueContainer.cs

@@ -812,8 +812,7 @@ public MessageQueueContainer(NetworkManager networkManager, uint maxFrameHistory
             Initialize(maxFrameHistory);
         }
 
-
-#if UNITY_EDITOR || DEVELOPMENT_BUILD
+#if UNITY_INCLUDE_TESTS
         /// <summary>
         /// Enables testing of the MessageQueueContainer
         /// </summary>

testproject/Assets/Tests/Runtime/MultiprocessRuntime/BaseMultiprocessTests.cs

@@ -17,68 +17,116 @@ public MultiprocessTestsAttribute() : base(MultiprocessCategoryName) { }
     [MultiprocessTests]
     public abstract class BaseMultiprocessTests
     {
-        protected virtual bool IsPerformanceTest => true;
-
-        private const string k_GlobalEmptySceneName = "EmptyScene";
+        // TODO: Remove UTR check once we have Multiprocess tests fully working
+        protected bool IgnoreMultiprocessTests => MultiprocessOrchestration.ShouldIgnoreUTRTests();
 
-        private bool m_SceneHasLoaded;
 
-        protected bool ShouldIgnoreTests => IsPerformanceTest && Application.isEditor || MultiprocessOrchestration.IsUsingUTR(); // todo remove UTR check once we have proper automation
+        protected virtual bool IsPerformanceTest => true;
 
         /// <summary>
         /// Implement this to specify the amount of workers to spawn from your main test runner
-        /// TODO there's a good chance this will be refactored with something fancier once we start integrating with bokken
+        /// TODO there's a good chance this will be re-factored with something fancier once we start integrating with bokken
         /// </summary>
         protected abstract int WorkerCount { get; }
 
+        private const string k_FirstPartOfTestRunnerSceneName = "InitTestScene";
+
+        // Since we want to additively load our BuildMultiprocessTestPlayer.MainSceneName
+        // We want to keep a reference to the
+        private Scene m_OriginalActiveScene;
+
         [OneTimeSetUp]
         public virtual void SetupTestSuite()
         {
-            if (ShouldIgnoreTests)
+            if (IgnoreMultiprocessTests)
+            {
+                Assert.Ignore("Ignoring tests under UTR. For testing, include the \"-bypassIgnoreUTR\" command line parameter.");
+            }
+
+            if (IsPerformanceTest)
             {
-                Assert.Ignore("Ignoring tests that shouldn't run from unity editor. Performance tests should be run from remote test execution on device (this can be ran using the \"run selected tests (your platform)\" button");
+                Assert.Ignore("Performance tests should be run from remote test execution on device (this can be ran using the \"run selected tests (your platform)\" button");
             }
 
-            SceneManager.LoadScene(BuildMultiprocessTestPlayer.MainSceneName, LoadSceneMode.Single);
+            var currentlyActiveScene = SceneManager.GetActiveScene();
+
+            // Just adding a sanity check here to help with debugging in the event that SetupTestSuite is
+            // being invoked and the TestRunner scene has not been set to the active scene yet.
+            // This could mean that TeardownSuite wasn't called or SceneManager is not finished unloading
+            // or could not unload the BuildMultiprocessTestPlayer.MainSceneName.
+            if (!currentlyActiveScene.name.StartsWith(k_FirstPartOfTestRunnerSceneName))
+            {
+                Debug.LogError($"Expected the currently active scene to begin with ({k_FirstPartOfTestRunnerSceneName}) but currently active scene is {currentlyActiveScene.name}");
+            }
+            m_OriginalActiveScene = currentlyActiveScene;
+
             SceneManager.sceneLoaded += OnSceneLoaded;
+            SceneManager.LoadScene(BuildMultiprocessTestPlayer.MainSceneName, LoadSceneMode.Additive);
         }
 
         private void OnSceneLoaded(Scene scene, LoadSceneMode mode)
         {
             SceneManager.sceneLoaded -= OnSceneLoaded;
+            if (scene.name == BuildMultiprocessTestPlayer.MainSceneName)
+            {
+                SceneManager.SetActiveScene(scene);
+            }
 
             NetworkManager.Singleton.StartHost();
-            for (int i = 0; i < WorkerCount; i++)
+
+            // Use scene verification to make sure we don't try to get clients to synchronize the TestRunner scene
+            NetworkManager.Singleton.SceneManager.VerifySceneBeforeLoading = VerifySceneIsValidForClientsToLoad;
+        }
+
+        /// <summary>
+        /// We want to exclude the TestRunner scene on the host-server side so it won't try to tell clients to
+        /// synchronize to this scene when they connect (host-server side only for multiprocess)
+        /// </summary>
+        /// <returns>true - scene is fine to synchronize/inform clients to load and false - scene should not be loaded by clients</returns>
+        private bool VerifySceneIsValidForClientsToLoad(int sceneIndex, string sceneName, LoadSceneMode loadSceneMode)
+        {
+            if (sceneName.StartsWith(k_FirstPartOfTestRunnerSceneName))
             {
-                MultiprocessOrchestration.StartWorkerNode(); // will automatically start built player as clients
+                return false;
             }
-
-            m_SceneHasLoaded = true;
+            return true;
         }
 
         [UnitySetUp]
         public virtual IEnumerator Setup()
         {
-            yield return new WaitUntil(() => NetworkManager.Singleton != null && NetworkManager.Singleton.IsServer && m_SceneHasLoaded);
-
+            yield return new WaitUntil(() => NetworkManager.Singleton != null && NetworkManager.Singleton.IsServer && NetworkManager.Singleton.IsListening);
             var startTime = Time.time;
+
+            // Moved this out of OnSceneLoaded as OnSceneLoaded is a callback from the SceneManager and just wanted to avoid creating
+            // processes from within the same callstack/context as the SceneManager.  This will instantiate up to the WorkerCount and
+            // then any subsequent calls to Setup if there are already workers it will skip this step
+            if (MultiprocessOrchestration.Processes.Count < WorkerCount)
+            {
+                var numProcessesToCreate = WorkerCount - MultiprocessOrchestration.Processes.Count;
+                for (int i = 0; i < numProcessesToCreate; i++)
+                {
+                    MultiprocessOrchestration.StartWorkerNode(); // will automatically start built player as clients
+                }
+            }
+
+            var timeOutTime = Time.realtimeSinceStartup + TestCoordinator.MaxWaitTimeoutSec;
             while (NetworkManager.Singleton.ConnectedClients.Count <= WorkerCount)
             {
                 yield return new WaitForSeconds(0.2f);
 
-                if (Time.time - startTime > TestCoordinator.MaxWaitTimeoutSec)
+                if (Time.realtimeSinceStartup > timeOutTime)
                 {
                     throw new Exception($"waiting too long to see clients to connect, got {NetworkManager.Singleton.ConnectedClients.Count - 1} clients, but was expecting {WorkerCount}, failing");
                 }
             }
-
             TestCoordinator.Instance.KeepAliveClientRpc();
         }
 
         [TearDown]
         public virtual void Teardown()
         {
-            if (!ShouldIgnoreTests)
+            if (!IgnoreMultiprocessTests)
             {
                 TestCoordinator.Instance.TestRunTeardown();
             }
@@ -87,12 +135,16 @@ public virtual void Teardown()
         [OneTimeTearDown]
         public virtual void TeardownSuite()
         {
-            if (!ShouldIgnoreTests)
+            if (!IgnoreMultiprocessTests)
             {
-                TestCoordinator.Instance.CloseRemoteClientRpc();
+                MultiprocessOrchestration.ShutdownAllProcesses();
                 NetworkManager.Singleton.Shutdown();
                 Object.Destroy(NetworkManager.Singleton.gameObject); // making sure we clear everything before reloading our scene
-                SceneManager.LoadScene(k_GlobalEmptySceneName); // using empty scene to clear our state
+                if (m_OriginalActiveScene.IsValid())
+                {
+                    SceneManager.SetActiveScene(m_OriginalActiveScene);
+                }
+                SceneManager.UnloadSceneAsync(BuildMultiprocessTestPlayer.MainSceneName);
             }
         }
     }

testproject/Assets/Tests/Runtime/MultiprocessRuntime/Helpers/BuildMultiprocessTestPlayer.cs

@@ -87,10 +87,10 @@ private static BuildReport BuildPlayer(bool isDebug = false)
             }
 
             Debug.Log($"Starting multiprocess player build using path {buildPathToUse}");
-
+            // Include all EditorBuildSettings.scenes with clients so they are in alignment with the server's scenes in build list indices
             buildOptions &= ~BuildOptions.AutoRunPlayer;
             var buildReport = BuildPipeline.BuildPlayer(
-                new[] { $"Assets/Scenes/{MainSceneName}.unity" },
+                EditorBuildSettings.scenes,
                 buildPathToUse,
                 EditorUserBuildSettings.activeBuildTarget,
                 buildOptions);

testproject/Assets/Tests/Runtime/MultiprocessRuntime/Helpers/MultiprocessOrchestration.cs

@@ -1,37 +1,85 @@
 using System;
+using System.Linq;
+using System.Collections.Generic;
 using System.ComponentModel;
 using System.Diagnostics;
 using System.IO;
+using System.Runtime.InteropServices;
 using Unity.Netcode.MultiprocessRuntimeTests;
-using System.Linq;
 using UnityEngine;
 using Debug = UnityEngine.Debug;
 
 public class MultiprocessOrchestration
 {
     public const string IsWorkerArg = "-isWorker";
+    private static DirectoryInfo s_MultiprocessDirInfo;
+    public static List<Process> Processes = new List<Process>();
+
+    /// <summary>
+    /// This is to detect if we should ignore Multiprocess tests
+    /// For testing, include the -bypassIgnoreUTR command line parameter when running UTR.
+    /// </summary>
+    public static bool ShouldIgnoreUTRTests()
+    {
+        return Environment.GetCommandLineArgs().Contains("-automated") && !Environment.GetCommandLineArgs().Contains("-bypassIgnoreUTR");
+    }
 
     public static void StartWorkerNode()
     {
+        if (Processes == null)
+        {
+            Processes = new List<Process>();
+        }
+
+        string userprofile = "";
+
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+        {
+            userprofile = Environment.GetEnvironmentVariable("USERPROFILE");
+        }
+        else
+        {
+            userprofile = Environment.GetEnvironmentVariable("HOME");
+        }
+        // Debug.Log($"userprofile is {userprofile}");
+        s_MultiprocessDirInfo = new DirectoryInfo(Path.Combine(userprofile, ".multiprocess"));
+
         var workerProcess = new Process();
+        if (Processes.Count > 0)
+        {
+            string message = "";
+            foreach (var p in Processes)
+            {
+                message += $" {p.Id} {p.HasExited} {p.StartTime} ";
+            }
+            Debug.Log($"Current process count {Processes.Count} with data {message}");
+        }
+        Processes.Add(workerProcess);
 
         //TODO this should be replaced eventually by proper orchestration for all supported platforms
         // Starting new local processes is a solution to help run perf tests locally. CI should have multi machine orchestration to
         // run performance tests with more realistic conditions.
         string buildInstructions = $"You probably didn't generate your build. Please make sure you build a player using the '{BuildMultiprocessTestPlayer.BuildAndExecuteMenuName}' menu";
+        string extraArgs = "";
         try
         {
-
             var buildPath = BuildMultiprocessTestPlayer.ReadBuildInfo().BuildPath;
             switch (Application.platform)
             {
                 case RuntimePlatform.OSXPlayer:
                 case RuntimePlatform.OSXEditor:
                     workerProcess.StartInfo.FileName = $"{buildPath}.app/Contents/MacOS/testproject";
+                    extraArgs += "-popupwindow -screen-width 100 -screen-height 100";
                     break;
                 case RuntimePlatform.WindowsPlayer:
                 case RuntimePlatform.WindowsEditor:
                     workerProcess.StartInfo.FileName = $"{buildPath}.exe";
+                    extraArgs += "-popupwindow -screen-width 100 -screen-height 100";
+                    break;
+                case RuntimePlatform.LinuxPlayer:
+                case RuntimePlatform.LinuxEditor:
+                    workerProcess.StartInfo.FileName = $"{buildPath}";
+                    extraArgs += "-nographics";
                     break;
                 default:
                     throw new NotImplementedException($"{nameof(StartWorkerNode)}: Current platform is not supported");
@@ -43,13 +91,18 @@ public static void StartWorkerNode()
             throw;
         }
 
+        string logPath = Path.Combine(s_MultiprocessDirInfo.FullName, $"logfile-mp{Processes.Count}");
+
+
         workerProcess.StartInfo.UseShellExecute = false;
         workerProcess.StartInfo.RedirectStandardError = true;
         workerProcess.StartInfo.RedirectStandardOutput = true;
-        workerProcess.StartInfo.Arguments = $"{IsWorkerArg} -popupwindow -screen-width 100 -screen-height 100";
-        // workerNode.StartInfo.Arguments += " -deepprofiling"; // enable for deep profiling
+
+        workerProcess.StartInfo.Arguments = $"{IsWorkerArg} {extraArgs} -logFile {logPath} -s {BuildMultiprocessTestPlayer.MainSceneName}";
+
         try
         {
+            Debug.Log($"Attempting to start new process, current process count: {Processes.Count}");
             var newProcessStarted = workerProcess.Start();
             if (!newProcessStarted)
             {
@@ -63,9 +116,29 @@ public static void StartWorkerNode()
         }
     }
 
-    // todo remove this once we have proper automation
-    public static bool IsUsingUTR()
+    public static void ShutdownAllProcesses()
     {
-        return Environment.GetCommandLineArgs().Contains("-automated");
+        Debug.Log("Shutting down all processes..");
+        foreach (var process in Processes)
+        {
+            Debug.Log($"Shutting down process {process.Id} with state {process.HasExited}");
+            try
+            {
+                if (!process.HasExited)
+                {
+                    // Close process by sending a close message to its main window.
+                    process.CloseMainWindow();
+
+                    // Free resources associated with process.
+                    process.Close();
+                }
+            }
+            catch (Exception ex)
+            {
+                Debug.LogException(ex);
+            }
+        }
+
+        Processes.Clear();
     }
 }