Merge branch 'main' into exp-petting-registry

Author: maryamhonari

.yamato/training-backcompat-tests.yml

@@ -1,19 +1,19 @@
 
-test_mac_backcompat_2020.1:
-  {% capture editor_version %}2020.1{% endcapture %}
+test_mac_backcompat_2019.4:
+  {% capture editor_version %}2019.4{% endcapture %}
   {% capture csharp_backcompat_version %}1.0.0{% endcapture %}
   # This test has to run on mac because it requires the custom build of tensorflow without AVX
   # Test against 2020.1 because 2020.2 has to run against package version 1.2.0
   name: Test Mac Backcompat Training {{ editor_version }}
   agent:
     type: Unity::VM::osx
-    image: ml-agents/ml-agents-bokken-mac:0.1.4-492264
+    image: ml-agents/ml-agents-bokken-mac:0.1.5-853758
     flavor: b1.small
   variables:
     UNITY_VERSION: {{ editor_version }}
   commands:
     - |
-      python3 -m venv venv && source venv/bin/activate
+      python -m venv venv && source venv/bin/activate
       python -m pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
       python -m pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple --upgrade
       unity-downloader-cli -u {{ editor_version }} -c editor --wait --fast
@@ -23,9 +23,9 @@ test_mac_backcompat_2020.1:
       python -u -m ml-agents.tests.yamato.standalone_build_tests --build-target=mac
       python -u -m ml-agents.tests.yamato.training_int_tests --csharp {{ csharp_backcompat_version }}
     - |
-      python3 -m venv venv_old && source venv_old/bin/activate
+      python -m venv venv_old && source venv_old/bin/activate
       python -m pip install pyyaml --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple
-      python -u -m ml-agents.tests.yamato.training_int_tests --python 0.16.0
+      python -u -m ml-agents.tests.yamato.training_int_tests --python 0.24.0
   triggers:
     cancel_old_ci: true
     recurring:

README.md

@@ -178,7 +178,7 @@ package by replying to
 Your opinion matters a great deal to us. Only by hearing your thoughts on the
 Unity ML-Agents Toolkit can we continue to improve and grow. Please take a few
 minutes to
-[let us know about it](https://github.com/Unity-Technologies/ml-agents/issues/1454).
+[let us know about it](https://unitysoftware.co1.qualtrics.com/jfe/form/SV_55pQKCZ578t0kbc).
 
 For any other questions or feedback, connect directly with the ML-Agents team at
 ml-agents@unity3d.com.

com.unity.ml-agents.extensions/Tests/Runtime/Input/InputActuatorComponentTests.cs

@@ -49,6 +49,11 @@ public override void Setup()
         public override void TearDown()
         {
             m_ActuatorComponent.CleanupActionAsset();
+            var objects = GameObject.FindObjectsOfType<GameObject>();
+            foreach (var o in objects)
+            {
+                UnityEngine.Object.DestroyImmediate(o);
+            }
             base.TearDown();
         }
 

com.unity.ml-agents/CHANGELOG.md

@@ -8,29 +8,45 @@ and this project adheres to
 
 ## [Unreleased]
 ### Major Changes
+
 #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#)
+
 #### ml-agents / ml-agents-envs / gym-unity (Python)
+
 ### Minor Changes
+
 #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#)
+- Added the capacity to initialize behaviors from any checkpoint and not just the latest one (#5525)
+
 #### ml-agents / ml-agents-envs / gym-unity (Python)
+- Set gym version in gym-unity to gym release 0.20.0
+- Added support for having `beta`, `epsilon`, and `learning rate` on separate schedules (affects only PPO and POCA). (#5538)
+
 ### Bug Fixes
+
 #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#)
+
 #### ml-agents / ml-agents-envs / gym-unity (Python)
 - Fixed a bug in multi-agent cooperative training where agents might not receive all of the states of
 terminated teammates. (#5441)
 - Fixed wrong attribute name in argparser for torch device option (#5433)(#5467)
 - Fixed conflicting CLI and yaml options regarding resume & initialize_from (#5495)
+- Fixed failing tests for gym-unity due to gym 0.20.0 release (#5540)
+- Fixed a bug in VAIL where the variational bottleneck was not properly passing gradients (#5546)
 - Added minimal analytics collection to LL-API (#5511)
+
 ## [2.1.0-exp.1] - 2021-06-09
 ### Minor Changes
 #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#)
 - update Barracuda to 2.0.0-pre.3. (#5385)
 - Fixed NullReferenceException when adding Behavior Parameters with no Agent. (#5382)
 - Add stacking option in Editor for `VectorSensorComponent`. (#5376)
+
 #### ml-agents / ml-agents-envs / gym-unity (Python)
 - Lock cattrs dependency version to 1.6. (#5397)
 - Added a fully connected visual encoder for environments with very small image inputs. (#5351)
 - Colab notebooks illustrating the use of the Python API are now part of the repository. (#5399)
+
 ### Bug Fixes
 #### com.unity.ml-agents / com.unity.ml-agents.extensions (C#)
 - RigidBodySensorComponent now displays a warning if it's used in a way that won't generate useful observations. (#5387)

com.unity.ml-agents/Plugins/ProtoBuffer/runtimes/osx/native/libgrpc_csharp_ext.x64.bundle

@@ -1,19 +1,45 @@
 using System;
+using System.Text;
+using System.Security.Cryptography;
 using UnityEngine;
 
 namespace Unity.MLAgents.Analytics
 {
+
     internal static class AnalyticsUtils
     {
+        /// <summary>
+        /// Conversion function from byte array to hex string
+        /// </summary>
+        /// <param name="array"></param>
+        /// <returns>A byte array to be hex encoded.</returns>
+        private static string ToHexString(byte[] array)
+        {
+            StringBuilder hex = new StringBuilder(array.Length * 2);
+            foreach (byte b in array)
+            {
+                hex.AppendFormat("{0:x2}", b);
+            }
+            return hex.ToString();
+        }
+
         /// <summary>
         /// Hash a string to remove PII or secret info before sending to analytics
         /// </summary>
-        /// <param name="s"></param>
-        /// <returns>A string containing the Hash128 of the input string.</returns>
-        public static string Hash(string s)
+        /// <param name="key"></param>
+        /// <returns>A string containing the key to be used for HMAC encoding.</returns>
+        /// <param name="value"></param>
+        /// <returns>A string containing the value to be encoded.</returns>
+        public static string Hash(string key, string value)
         {
-            var behaviorNameHash = Hash128.Compute(s);
-            return behaviorNameHash.ToString();
+            string hash;
+            UTF8Encoding encoder = new UTF8Encoding();
+            using (HMACSHA256 hmac = new HMACSHA256(encoder.GetBytes(key)))
+            {
+                Byte[] hmBytes = hmac.ComputeHash(encoder.GetBytes(value));
+                hash = ToHexString(hmBytes);
+            }
+            return hash;
         }
 
         internal static bool s_SendEditorAnalytics = true;

com.unity.ml-agents/Runtime/Analytics/AnalyticsUtils.cs

@@ -156,6 +156,7 @@ internal struct TrainingEnvironmentInitializedEvent
         public string TorchDeviceType;
         public int NumEnvironments;
         public int NumEnvironmentParameters;
+        public string RunOptions;
     }
 
     [Flags]
@@ -188,5 +189,6 @@ internal struct TrainingBehaviorInitializedEvent
         public string VisualEncoder;
         public int NumNetworkLayers;
         public int NumNetworkHiddenUnits;
+        public string Config;
     }
 }

com.unity.ml-agents/Runtime/Analytics/Events.cs

@@ -156,7 +156,7 @@ IList<IActuator> actuators
             var inferenceEvent = new InferenceEvent();
 
             // Hash the behavior name so that there's no concern about PII or "secret" data being leaked.
-            inferenceEvent.BehaviorName = AnalyticsUtils.Hash(behaviorName);
+            inferenceEvent.BehaviorName = AnalyticsUtils.Hash(k_VendorKey, behaviorName);
 
             inferenceEvent.BarracudaModelSource = barracudaModel.IrSource;
             inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion;

com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs

@@ -192,8 +192,21 @@ internal static string ParseBehaviorName(string fullyQualifiedBehaviorName)
             return fullyQualifiedBehaviorName.Substring(0, lastQuestionIndex);
         }
 
+        internal static TrainingBehaviorInitializedEvent SanitizeTrainingBehaviorInitializedEvent(TrainingBehaviorInitializedEvent tbiEvent)
+        {
+            // Hash the behavior name if the message version is from an older version of ml-agents that doesn't do trainer-side hashing.
+            // We'll also, for extra safety, verify that the BehaviorName is the size of the expected SHA256 hash.
+            // Context: The config field was added at the same time as trainer side hashing, so messages including it should already be hashed.
+            if (tbiEvent.Config.Length == 0 || tbiEvent.BehaviorName.Length != 64)
+            {
+                tbiEvent.BehaviorName = AnalyticsUtils.Hash(k_VendorKey, tbiEvent.BehaviorName);
+            }
+
+            return tbiEvent;
+        }
+
         [Conditional("MLA_UNITY_ANALYTICS_MODULE")]
-        public static void TrainingBehaviorInitialized(TrainingBehaviorInitializedEvent tbiEvent)
+        public static void TrainingBehaviorInitialized(TrainingBehaviorInitializedEvent rawTbiEvent)
         {
 #if UNITY_EDITOR && MLA_UNITY_ANALYTICS_MODULE
             if (!IsAnalyticsEnabled())
@@ -202,6 +215,7 @@ public static void TrainingBehaviorInitialized(TrainingBehaviorInitializedEvent
             if (!EnableAnalytics())
                 return;
 
+            var tbiEvent = SanitizeTrainingBehaviorInitializedEvent(rawTbiEvent);
             var behaviorName = tbiEvent.BehaviorName;
             var added = s_SentTrainingBehaviorInitialized.Add(behaviorName);
 
@@ -211,9 +225,7 @@ public static void TrainingBehaviorInitialized(TrainingBehaviorInitializedEvent
                 return;
             }
 
-            // Hash the behavior name so that there's no concern about PII or "secret" data being leaked.
             tbiEvent.TrainingSessionGuid = s_TrainingSessionGuid.ToString();
-            tbiEvent.BehaviorName = AnalyticsUtils.Hash(tbiEvent.BehaviorName);
 
             // Note - to debug, use JsonUtility.ToJson on the event.
             // Debug.Log(
@@ -236,7 +248,7 @@ IList<IActuator> actuators
             var remotePolicyEvent = new RemotePolicyInitializedEvent();
 
             // Hash the behavior name so that there's no concern about PII or "secret" data being leaked.
-            remotePolicyEvent.BehaviorName = AnalyticsUtils.Hash(behaviorName);
+            remotePolicyEvent.BehaviorName = AnalyticsUtils.Hash(k_VendorKey, behaviorName);
 
             remotePolicyEvent.TrainingSessionGuid = s_TrainingSessionGuid.ToString();
             remotePolicyEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec);

com.unity.ml-agents/Runtime/Analytics/TrainingAnalytics.cs

@@ -501,6 +501,7 @@ internal static TrainingEnvironmentInitializedEvent ToTrainingEnvironmentInitial
                 TorchDeviceType = inputProto.TorchDeviceType,
                 NumEnvironments = inputProto.NumEnvs,
                 NumEnvironmentParameters = inputProto.NumEnvironmentParameters,
+                RunOptions = inputProto.RunOptions,
             };
         }
 
@@ -530,6 +531,7 @@ internal static TrainingBehaviorInitializedEvent ToTrainingBehaviorInitializedEv
                 VisualEncoder = inputProto.VisualEncoder,
                 NumNetworkLayers = inputProto.NumNetworkLayers,
                 NumNetworkHiddenUnits = inputProto.NumNetworkHiddenUnits,
+                Config = inputProto.Config,
             };
         }
 

com.unity.ml-agents/Runtime/Communicator/GrpcExtensions.cs

@@ -2,7 +2,7 @@
 #define MLA_SUPPORTED_TRAINING_PLATFORM
 #endif
 
-# if MLA_SUPPORTED_TRAINING_PLATFORM
+#if MLA_SUPPORTED_TRAINING_PLATFORM
 using Grpc.Core;
 #if UNITY_EDITOR
 using UnityEditor;
@@ -50,6 +50,7 @@ internal class RpcCommunicator : ICommunicator
 
         /// The Unity to External client.
         UnityToExternalProto.UnityToExternalProtoClient m_Client;
+        Channel m_Channel;
 
         /// <summary>
         /// Initializes a new instance of the RPCCommunicator class.
@@ -140,6 +141,7 @@ out input
                     Debug.Log($"Unexpected exception when trying to initialize communication: {ex}");
                 }
                 initParametersOut = new UnityRLInitParameters();
+                NotifyQuitAndShutDownChannel();
                 return false;
             }
 
@@ -181,6 +183,8 @@ out input
 
             UpdateEnvironmentWithInput(input.RlInput);
             initParametersOut = initializationInput.RlInitializationInput.ToUnityRLInitParameters();
+            // Be sure to shut down the grpc channel when the application is quitting.
+            Application.quitting += NotifyQuitAndShutDownChannel;
             return true;
 #else
             initParametersOut = new UnityRLInitParameters();
@@ -217,9 +221,9 @@ void UpdateEnvironmentWithInput(UnityRLInputProto rlInput)
         UnityInputProto Initialize(int port, UnityOutputProto unityOutput, out UnityInputProto unityInput)
         {
             m_IsOpen = true;
-            var channel = new Channel($"localhost:{port}", ChannelCredentials.Insecure);
+            m_Channel = new Channel($"localhost:{port}", ChannelCredentials.Insecure);
 
-            m_Client = new UnityToExternalProto.UnityToExternalProtoClient(channel);
+            m_Client = new UnityToExternalProto.UnityToExternalProtoClient(m_Channel);
             var result = m_Client.Exchange(WrapMessage(unityOutput, 200));
             var inputMessage = m_Client.Exchange(WrapMessage(null, 200));
             unityInput = inputMessage.UnityInput;
@@ -229,11 +233,24 @@ UnityInputProto Initialize(int port, UnityOutputProto unityOutput, out UnityInpu
             if (result.Header.Status != 200 || inputMessage.Header.Status != 200)
             {
                 m_IsOpen = false;
-                QuitCommandReceived?.Invoke();
+                NotifyQuitAndShutDownChannel();
             }
             return result.UnityInput;
         }
 
+        void NotifyQuitAndShutDownChannel()
+        {
+            QuitCommandReceived?.Invoke();
+            try
+            {
+                m_Channel.ShutdownAsync().Wait();
+            }
+            catch (Exception)
+            {
+                // do nothing
+            }
+        }
+
 #endregion
 
 #region Destruction
@@ -269,7 +286,7 @@ void SendCommandEvent(CommandProto command)
             {
                 case CommandProto.Quit:
                     {
-                        QuitCommandReceived?.Invoke();
+                        NotifyQuitAndShutDownChannel();
                         return;
                     }
                 case CommandProto.Reset:
@@ -456,7 +473,7 @@ UnityInputProto Exchange(UnityOutputProto unityOutput)
                 // Not sure if the quit command is actually sent when a
                 // non 200 message is received.  Notify that we are indeed
                 // quitting.
-                QuitCommandReceived?.Invoke();
+                NotifyQuitAndShutDownChannel();
                 return message.UnityInput;
             }
             catch (Exception ex)
@@ -488,7 +505,7 @@ UnityInputProto Exchange(UnityOutputProto unityOutput)
                 }
 
                 m_IsOpen = false;
-                QuitCommandReceived?.Invoke();
+                NotifyQuitAndShutDownChannel();
                 return null;
             }
         }