In-Editor Analytics for inference

com.unity.ml-agents/Runtime/Analytics/Events.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Sensors;
+
+namespace Unity.MLAgents.Analytics
+{
+    internal struct InferenceEvent
+    {
+        public string BehaviorName;
+        public string BarracudaModelSource;
+        public string BarracudaModelVersion;
+        public string BarracudaModelProducer;
+        public string BarracudaPackageVersion;
+        public int InferenceDevice;
+        public List<EventObservationSpec> ObservationSpecs;
+        public EventActionSpec ActionSpec;
+        public int MemorySize;
+        public string ModelHash;
+    }
+
+    /// <summary>
+    /// Simplified version of ActionSpec struct for use in analytics
+    /// </summary>
+    internal struct EventActionSpec
+    {
+        public int NumContinuousActions;
+        public int NumDiscreteActions;
+        public int[] BranchSizes;
+
+        public static EventActionSpec FromActionSpec(ActionSpec actionSpec)
+        {
+            var branchSizes = actionSpec.BranchSizes ?? Array.Empty<int>();
+            return new EventActionSpec
+            {
+                NumContinuousActions = actionSpec.NumContinuousActions,
+                NumDiscreteActions = actionSpec.NumDiscreteActions,
+                BranchSizes = branchSizes,
+            };
+        }
+    }
+
+    /// <summary>
+    /// Simplified summary of Agent observations for use in analytics
+    /// </summary>
+    internal struct EventObservationSpec
+    {
+        public string SensorName;
+        public int[] ObservationShape;
+
+        public static EventObservationSpec FromSensor(ISensor sensor)
+        {
+            return new EventObservationSpec
+            {
+                SensorName = sensor.GetName(),
+                ObservationShape = sensor.GetObservationShape(),
+            };
+        }
+    }
+}

com.unity.ml-agents/Runtime/Analytics/InferenceAnalytics.cs

@@ -0,0 +1,180 @@
+using System;
+using System.Collections.Generic;
+using Unity.Barracuda;
+using Unity.MLAgents.Actuators;
+using Unity.MLAgents.Inference;
+using Unity.MLAgents.Policies;
+using Unity.MLAgents.Sensors;
+using UnityEditor;
+using UnityEditor.Analytics;
+using UnityEngine;
+using UnityEngine.Analytics;
+
+namespace Unity.MLAgents.Analytics
+{
+    internal class InferenceAnalytics
+    {
+        static bool s_EventRegistered = false;
+        const int k_MaxEventsPerHour = 1000;
+        const int k_MaxNumberOfElements = 1000;
+        const string k_VendorKey = "unity.ml-agents";
+        const string k_EventName = "InferenceModelSet";
+
+        private static HashSet<NNModel> s_SentModels;
+
+        static bool EnableAnalytics()
+        {
+            if (s_EventRegistered)
+            {
+                return true;
+            }
+
+            if (s_SentModels == null)
+            {
+                s_SentModels = new HashSet<NNModel>();
+            }
+
+            AnalyticsResult result = EditorAnalytics.RegisterEventWithLimit(k_EventName, k_MaxEventsPerHour, k_MaxNumberOfElements, k_VendorKey);
+            if (result == AnalyticsResult.Ok)
+            {
+                s_EventRegistered = true;
+            }
+
+            return s_EventRegistered;
+        }
+
+        public static void InferenceModelSet(
+            NNModel nnModel,
+            string behaviorName,
+            InferenceDevice inferenceDevice,
+            IList<ISensor> sensors,
+            ActionSpec actionSpec
+        )
+        {
+            // The event shouldn't be able to report if this is disabled but if we know we're not going to report
+            // Lets early out and not waste time gathering all the data
+            if (!EditorAnalytics.enabled)
+                return;
+
+            if (!EnableAnalytics())
+                return;
+
+            var added = s_SentModels.Add(nnModel);
+
+            if (!added)
+            {
+                // We previously added this model. Exit so we don't resend.
+                return;
+            }
+
+            var data = GetEventForModel(nnModel, behaviorName, inferenceDevice, sensors, actionSpec);
+            //EditorAnalytics.SendEventWithLimit(k_EventName, data);
+        }
+
+        static InferenceEvent GetEventForModel(
+            NNModel nnModel,
+            string behaviorName,
+            InferenceDevice inferenceDevice,
+            IList<ISensor> sensors,
+            ActionSpec actionSpec
+        )
+        {
+            var barracudaModel = ModelLoader.Load(nnModel);
+            var inferenceEvent = new InferenceEvent();
+            inferenceEvent.BehaviorName = behaviorName;
+            inferenceEvent.BarracudaModelSource = barracudaModel.IrSource;
+            inferenceEvent.BarracudaModelVersion = barracudaModel.IrVersion;
+            inferenceEvent.BarracudaModelProducer = barracudaModel.ProducerName;
+            inferenceEvent.MemorySize = (int)barracudaModel.GetTensorByName(TensorNames.MemorySize)[0];
+            inferenceEvent.InferenceDevice = (int)inferenceDevice;
+
+            if (barracudaModel.ProducerName == "Script")
+            {
+                // .nn files don't have these fields set correctly. Assign some placeholder values.
+                inferenceEvent.BarracudaModelSource = "NN";
+                inferenceEvent.BarracudaModelProducer = "tf2bc.py";
+            }
+
+#if UNITY_2019_3_OR_NEWER
+            var barracudaPackageInfo = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Tensor).Assembly);
+            inferenceEvent.BarracudaPackageVersion = barracudaPackageInfo.version;
+#else
+            inferenceEvent.BarracudaPackageVersion = "unknown";
+#endif
+
+            inferenceEvent.ActionSpec = EventActionSpec.FromActionSpec(actionSpec);
+            inferenceEvent.ObservationSpecs = new List<EventObservationSpec>(sensors.Count);
+            foreach (var sensor in sensors)
+            {
+                inferenceEvent.ObservationSpecs.Add(EventObservationSpec.FromSensor(sensor));
+            }
+
+            inferenceEvent.ModelHash = GetModelHash(barracudaModel);
+            return inferenceEvent;
+        }
+
+        internal class FNVHash
+        {
+            const ulong kFNV_prime = 1099511628211;
+            const ulong kFNV_offset_basis = 14695981039346656037;
+            private const int kMaxBytes = 1024;
+
+            public ulong hash;
+
+            public FNVHash()
+            {
+                hash = kFNV_offset_basis;
+            }
+
+            public void Append(float[] values)
+            {
+                // Limit the max number of float bytes that we hash for performance.
+                // This increases the chance of a collision, but this should still be extremely rare.
+                var bytesToHash = Mathf.Min(kMaxBytes, Buffer.ByteLength(values));
+                for (var i = 0; i < bytesToHash; i++)
+                {
+                    var b = Buffer.GetByte(values, i);
+                    Update(b);
+                }
+            }
+
+            public void Append(string value)
+            {
+                foreach (var c in value)
+                {
+                    Update((byte)c);
+                }
+            }
+
+            private void Update(byte b)
+            {
+                hash *= kFNV_prime;
+                hash ^= b;
+            }
+
+            public override string ToString()
+            {
+                return hash.ToString();
+            }
+        }
+
+        static string GetModelHash(Model barracudaModel)
+        {
+            // Pre-2020 versions of Unity don't have Hash128.Append() (can only hash strings)
+            // For these versions, we'll use a simple FNV-1 hash.
+            // https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
+#if UNITY_2020_1_OR_NEWER
+            var hash = new Hash128();
+#else
+            var hash = new FNVHash();
+#endif
+            foreach (var layer in barracudaModel.layers)
+            {
+                hash.Append(layer.name);
+                hash.Append(layer.weights);
+            }
+
+            return hash.ToString();
+        }
+    }
+}

com.unity.ml-agents/Runtime/Inference/ModelRunner.cs

@@ -86,6 +86,16 @@ public ModelRunner(
                 actionSpec, seed, m_TensorAllocator, m_Memories, barracudaModel);
         }
 
+        public InferenceDevice InferenceDevice
+        {
+            get { return m_InferenceDevice; }
+        }
+
+        public NNModel Model
+        {
+            get { return m_Model; }
+        }
+
         static Dictionary<string, Tensor> PrepareBarracudaInputs(IEnumerable<TensorProxy> infInputs)
         {
             var inputs = new Dictionary<string, Tensor>();

com.unity.ml-agents/Runtime/Policies/BarracudaPolicy.cs

@@ -41,21 +41,46 @@ internal class BarracudaPolicy : IPolicy
         List<int[]> m_SensorShapes;
         SpaceType m_SpaceType;
 
+        private ActionSpec m_ActionSpec;
+
+        private string m_BehaviorName;
+
+        /// <summary>
+        /// Whether or not we've tried to send analytics for this model. We only ever try to send once per policy,
+        /// and do additional deduplication in the analytics code.
+        /// </summary>
+        private bool m_AnalyticsSent;
+
         /// <inheritdoc />
         public BarracudaPolicy(
             ActionSpec actionSpec,
             NNModel model,
-            InferenceDevice inferenceDevice)
+            InferenceDevice inferenceDevice,
+            string behaviorName
+        )
         {
             var modelRunner = Academy.Instance.GetOrCreateModelRunner(model, actionSpec, inferenceDevice);
             m_ModelRunner = modelRunner;
             actionSpec.CheckNotHybrid();
             m_SpaceType = actionSpec.NumContinuousActions > 0 ? SpaceType.Continuous : SpaceType.Discrete;
+            m_BehaviorName = behaviorName;
+            m_ActionSpec = actionSpec;
         }
 
         /// <inheritdoc />
         public void RequestDecision(AgentInfo info, List<ISensor> sensors)
         {
+            if (!m_AnalyticsSent)
+            {
+                m_AnalyticsSent = true;
+                Analytics.InferenceAnalytics.InferenceModelSet(
+                    m_ModelRunner.Model,
+                    m_BehaviorName,
+                    m_ModelRunner.InferenceDevice,
+                    sensors,
+                    m_ActionSpec
+                );
+            }
             m_AgentId = info.episodeId;
             m_ModelRunner?.PutObservations(info, sensors);
         }

com.unity.ml-agents/Runtime/Policies/BehaviorParameters.cs

@@ -212,7 +212,7 @@ internal IPolicy GeneratePolicy(ActionSpec actionSpec, HeuristicPolicy.ActionGen
                                 "Either assign a model, or change to a different Behavior Type."
                             );
                         }
-                        return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice);
+                        return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice, m_BehaviorName);
                     }
                 case BehaviorType.Default:
                     if (Academy.Instance.IsCommunicatorOn)
@@ -221,7 +221,7 @@ internal IPolicy GeneratePolicy(ActionSpec actionSpec, HeuristicPolicy.ActionGen
                     }
                     if (m_Model != null)
                     {
-                        return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice);
+                        return new BarracudaPolicy(actionSpec, m_Model, m_InferenceDevice, m_BehaviorName);
                     }
                     else
                     {
@@ -241,5 +241,6 @@ internal void UpdateAgentPolicy()
             }
             agent.ReloadPolicy();
         }
+
     }
 }