[WIP] ISensor interface and use for visual observations (#2731)

* ISensor and SensorBase

* camera and rendertex first pass

* use isensors for visual obs

* Update gridworld with CameraSensors

* compressed obs for reals

* Remove AgentInfo.visualObservations

* better separation of train and inference sensor calls

* compressed obs proto - need CI to generate code

* int32

* get proto name right

* run protoc locally for new fiels

* apply generated proto patch (pyi files were weird)

* don't repeat bytes

* hook up compressedobs

* dont send BrainParameters until there's an AgentInfo

* python BrainParameters now needs an AgentInfo to create

* remove last (I hope) dependency on camerares

* remove CameraResolutions and AgentInfo.visual_observations

* update mypy-protobuf version

* cleanup todos

* python cleanup

* more unit test fixes

* more unit test fix

* camera sensors for VisualFood collector, record demo

* SensorComponent

* timers, rename Sensor fields, GetName interface

* sort sensors, add test

* remove empty test

* revert ProjectVersion.txt change

* fix demo recording

* remove AgentParameters cameras and textures

* PR feedback, use RenderTexture sensor in GridWorld

* update scenes to use Camera sensors

* Add comments on future of GenerateSensorData

* make SensorComponent and subclasses public

* #if DEBUG around checks

Author: Chris Elion

UnitySDK/Assets/ML-Agents/Editor/AgentEditor.cs

@@ -25,60 +25,9 @@ public override void OnInspectorGUI()
                 "agentParameters.resetOnDone");
             var isOdd = serializedAgent.FindProperty(
                 "agentParameters.onDemandDecision");
-            var cameras = serializedAgent.FindProperty(
-                "agentParameters.agentCameras");
-            var renderTextures = serializedAgent.FindProperty(
-                "agentParameters.agentRenderTextures");
 
             EditorGUILayout.PropertyField(brain);
 
-            if (cameras.arraySize > 0 && renderTextures.arraySize > 0)
-            {
-                EditorGUILayout.HelpBox("Brain visual observations created by first getting all cameras then all render textures.", MessageType.Info);
-            }
-
-            EditorGUILayout.LabelField("Agent Cameras");
-            for (var i = 0; i < cameras.arraySize; i++)
-            {
-                EditorGUILayout.PropertyField(
-                    cameras.GetArrayElementAtIndex(i),
-                    new GUIContent("Camera " + (i + 1) + ": "));
-            }
-
-            EditorGUILayout.BeginHorizontal();
-            if (GUILayout.Button("Add Camera", EditorStyles.miniButton))
-            {
-                cameras.arraySize++;
-            }
-
-            if (GUILayout.Button("Remove Camera", EditorStyles.miniButton))
-            {
-                cameras.arraySize--;
-            }
-
-            EditorGUILayout.EndHorizontal();
-
-            EditorGUILayout.LabelField("Agent RenderTextures");
-            for (var i = 0; i < renderTextures.arraySize; i++)
-            {
-                EditorGUILayout.PropertyField(
-                    renderTextures.GetArrayElementAtIndex(i),
-                    new GUIContent("RenderTexture " + (i + 1) + ": "));
-            }
-
-            EditorGUILayout.BeginHorizontal();
-            if (GUILayout.Button("Add RenderTextures", EditorStyles.miniButton))
-            {
-                renderTextures.arraySize++;
-            }
-
-            if (GUILayout.Button("Remove RenderTextures", EditorStyles.miniButton))
-            {
-                renderTextures.arraySize--;
-            }
-
-            EditorGUILayout.EndHorizontal();
-
 
             EditorGUILayout.PropertyField(
                 maxSteps,

UnitySDK/Assets/ML-Agents/Editor/BrainParametersDrawer.cs

@@ -13,18 +13,11 @@ public class BrainParametersDrawer : PropertyDrawer
         // The height of a line in the Unity Inspectors
         private const float k_LineHeight = 17f;
         private const int k_VecObsNumLine = 3;
-        private const string k_CamResPropName = "cameraResolutions";
         private const string k_ActionSizePropName = "vectorActionSize";
         private const string k_ActionTypePropName = "vectorActionSpaceType";
         private const string k_ActionDescriptionPropName = "vectorActionDescriptions";
         private const string k_VecObsPropName = "vectorObservationSize";
         private const string k_NumVecObsPropName = "numStackedVectorObservations";
-        private const string k_CamWidthPropName = "width";
-        private const string k_CamHeightPropName = "height";
-        private const string k_CamGrayPropName = "blackAndWhite";
-        private const int k_DefaultCameraWidth = 84;
-        private const int k_DefaultCameraHeight = 84;
-        private const bool k_DefaultCameraGray = false;
 
         /// <inheritdoc />
         public override float GetPropertyHeight(SerializedProperty property, GUIContent label)
@@ -33,7 +26,6 @@ public override float GetPropertyHeight(SerializedProperty property, GUIContent
             {
                 return k_LineHeight +
                     GetHeightDrawVectorObservation() +
-                    GetHeightDrawVisualObservation(property) +
                     GetHeightDrawVectorAction(property) +
                     GetHeightDrawVectorActionDescriptions(property);
             }
@@ -57,10 +49,6 @@ public override void OnGUI(Rect position, SerializedProperty property, GUIConten
                 DrawVectorObservation(position, property);
                 position.y += GetHeightDrawVectorObservation();
 
-                //Visual Observations
-                DrawVisualObservations(position, property);
-                position.y += GetHeightDrawVisualObservation(property);
-
                 // Vector Action
                 DrawVectorAction(position, property);
                 position.y += GetHeightDrawVectorAction(property);
@@ -111,107 +99,6 @@ private static float GetHeightDrawVectorObservation()
             return k_VecObsNumLine * k_LineHeight;
         }
 
-        /// <summary>
-        /// Draws the Visual Observations parameters for the Brain Parameters
-        /// </summary>
-        /// <param name="position">Rectangle on the screen to use for the property GUI.</param>
-        /// <param name="property">The SerializedProperty of the BrainParameters
-        /// to make the custom GUI for.</param>
-        private static void DrawVisualObservations(Rect position, SerializedProperty property)
-        {
-            EditorGUI.LabelField(position, "Visual Observations");
-            position.y += k_LineHeight;
-            var quarter = position.width / 4;
-            var resolutions = property.FindPropertyRelative(k_CamResPropName);
-            DrawVisualObsButtons(position, resolutions);
-            position.y += k_LineHeight;
-
-            // Display the labels for the columns : Index, Width, Height and Gray
-            var indexRect = new Rect(position.x, position.y, quarter, position.height);
-            var widthRect = new Rect(position.x + quarter, position.y, quarter, position.height);
-            var heightRect = new Rect(position.x + 2 * quarter, position.y, quarter, position.height);
-            var bwRect = new Rect(position.x + 3 * quarter, position.y, quarter, position.height);
-            EditorGUI.indentLevel++;
-            if (resolutions.arraySize > 0)
-            {
-                EditorGUI.LabelField(indexRect, "Index");
-                indexRect.y += k_LineHeight;
-                EditorGUI.LabelField(widthRect, "Width");
-                widthRect.y += k_LineHeight;
-                EditorGUI.LabelField(heightRect, "Height");
-                heightRect.y += k_LineHeight;
-                EditorGUI.LabelField(bwRect, "Gray");
-                bwRect.y += k_LineHeight;
-            }
-
-            // Iterate over the resolutions
-            for (var i = 0; i < resolutions.arraySize; i++)
-            {
-                EditorGUI.LabelField(indexRect, "Obs " + i);
-                indexRect.y += k_LineHeight;
-                var res = resolutions.GetArrayElementAtIndex(i);
-                var w = res.FindPropertyRelative("width");
-                w.intValue = EditorGUI.IntField(widthRect, w.intValue);
-                widthRect.y += k_LineHeight;
-                var h = res.FindPropertyRelative("height");
-                h.intValue = EditorGUI.IntField(heightRect, h.intValue);
-                heightRect.y += k_LineHeight;
-                var bw = res.FindPropertyRelative("blackAndWhite");
-                bw.boolValue = EditorGUI.Toggle(bwRect, bw.boolValue);
-                bwRect.y += k_LineHeight;
-            }
-            EditorGUI.indentLevel--;
-        }
-
-        /// <summary>
-        /// Draws the buttons to add and remove the visual observations parameters
-        /// </summary>
-        /// <param name="position">Rectangle on the screen to use for the property GUI.</param>
-        /// <param name="resolutions">The SerializedProperty of the resolution array
-        /// to make the custom GUI for.</param>
-        private static void DrawVisualObsButtons(Rect position, SerializedProperty resolutions)
-        {
-            var widthEighth = position.width / 8;
-            var addButtonRect = new Rect(position.x + widthEighth, position.y,
-                3 * widthEighth, position.height);
-            var removeButtonRect = new Rect(position.x + 4 * widthEighth, position.y,
-                3 * widthEighth, position.height);
-            if (resolutions.arraySize == 0)
-            {
-                addButtonRect.width *= 2;
-            }
-            // Display the buttons
-            if (GUI.Button(addButtonRect, "Add New", EditorStyles.miniButton))
-            {
-                resolutions.arraySize += 1;
-                var newRes = resolutions.GetArrayElementAtIndex(resolutions.arraySize - 1);
-                newRes.FindPropertyRelative(k_CamWidthPropName).intValue = k_DefaultCameraWidth;
-                newRes.FindPropertyRelative(k_CamHeightPropName).intValue = k_DefaultCameraHeight;
-                newRes.FindPropertyRelative(k_CamGrayPropName).boolValue = k_DefaultCameraGray;
-            }
-            if (resolutions.arraySize > 0)
-            {
-                if (GUI.Button(removeButtonRect, "Remove Last", EditorStyles.miniButton))
-                {
-                    resolutions.arraySize -= 1;
-                }
-            }
-        }
-
-        /// <summary>
-        /// The Height required to draw the Visual Observations parameters
-        /// </summary>
-        /// <returns>The height of the drawer of the Visual Observations </returns>
-        private static float GetHeightDrawVisualObservation(SerializedProperty property)
-        {
-            var visObsSize = property.FindPropertyRelative(k_CamResPropName).arraySize + 2;
-            if (property.FindPropertyRelative(k_CamResPropName).arraySize > 0)
-            {
-                visObsSize += 1;
-            }
-            return k_LineHeight * visObsSize;
-        }
-
         /// <summary>
         /// Draws the Vector Actions parameters for the Brain Parameters
         /// </summary>

UnitySDK/Assets/ML-Agents/Editor/DemonstrationDrawer.cs

@@ -59,33 +59,6 @@ static string BuildActionArrayLabel(SerializedProperty actionSizeProperty)
         return actionLabel.ToString();
     }
 
-    /// <summary>
-    /// Constructs complex label for each CameraResolution object.
-    /// An example of this could be `[ 84 X 84 ]`
-    /// for a single camera with 84 pixels height and width.
-    /// </summary>
-    private static string BuildCameraResolutionLabel(SerializedProperty cameraArray)
-    {
-        var numCameras = cameraArray.arraySize;
-        var cameraLabel = new StringBuilder("[ ");
-        for (var i = 0; i < numCameras; i++)
-        {
-            var camHeightPropName =
-                cameraArray.GetArrayElementAtIndex(i).FindPropertyRelative("height");
-            cameraLabel.Append(camHeightPropName.intValue);
-            cameraLabel.Append(" X ");
-            var camWidthPropName =
-                cameraArray.GetArrayElementAtIndex(i).FindPropertyRelative("width");
-            cameraLabel.Append(camWidthPropName.intValue);
-            if (i < numCameras - 1)
-            {
-                cameraLabel.Append(", ");
-            }
-        }
-
-        cameraLabel.Append(" ]");
-        return cameraLabel.ToString();
-    }
 
     /// <summary>
     /// Renders Inspector UI for Brain Parameters of Demonstration.
@@ -95,21 +68,18 @@ void MakeBrainParametersProperty(SerializedProperty property)
         var vecObsSizeProp = property.FindPropertyRelative("vectorObservationSize");
         var numStackedProp = property.FindPropertyRelative("numStackedVectorObservations");
         var actSizeProperty = property.FindPropertyRelative("vectorActionSize");
-        var camResProp = property.FindPropertyRelative("cameraResolutions");
         var actSpaceTypeProp = property.FindPropertyRelative("vectorActionSpaceType");
 
         var vecObsSizeLabel = vecObsSizeProp.displayName + ": " + vecObsSizeProp.intValue;
         var numStackedLabel = numStackedProp.displayName + ": " + numStackedProp.intValue;
         var vecActSizeLabel =
             actSizeProperty.displayName + ": " + BuildActionArrayLabel(actSizeProperty);
-        var camResLabel = camResProp.displayName + ": " + BuildCameraResolutionLabel(camResProp);
         var actSpaceTypeLabel = actSpaceTypeProp.displayName + ": " +
             (SpaceType)actSpaceTypeProp.enumValueIndex;
 
         EditorGUILayout.LabelField(vecObsSizeLabel);
         EditorGUILayout.LabelField(numStackedLabel);
         EditorGUILayout.LabelField(vecActSizeLabel);
-        EditorGUILayout.LabelField(camResLabel);
         EditorGUILayout.LabelField(actSpaceTypeLabel);
     }
 

UnitySDK/Assets/ML-Agents/Editor/Tests/DemonstrationTests.cs

@@ -33,7 +33,6 @@ public void TestStoreInitalize()
             {
                 vectorObservationSize = 3,
                 numStackedVectorObservations = 2,
-                cameraResolutions = new[] {new Resolution()},
                 vectorActionDescriptions = new[] {"TestActionA", "TestActionB"},
                 vectorActionSize = new[] {2, 2},
                 vectorActionSpaceType = SpaceType.Discrete
@@ -47,7 +46,6 @@ public void TestStoreInitalize()
             var agentInfo = new AgentInfo
             {
                 reward = 1f,
-                visualObservations = new List<Texture2D>(),
                 actionMasks = new[] {false, true},
                 done = true,
                 id = 5,

UnitySDK/Assets/ML-Agents/Editor/Tests/MLAgentsEditModeTest.cs

@@ -1,6 +1,8 @@
 using UnityEngine;
 using NUnit.Framework;
 using System.Reflection;
+using MLAgents.Sensor;
+using MLAgents.InferenceBrain;
 
 namespace MLAgents.Tests
 {
@@ -33,6 +35,13 @@ public class TestAgent : Agent
         public override void InitializeAgent()
         {
             initializeAgentCalls += 1;
+
+            // Add in some custom sensors so we can confirm they get sorted as expected.
+            var sensor1 = new TestSensor("testsensor1");
+            var sensor2 = new TestSensor("testsensor2");
+
+            m_Sensors.Add(sensor2);
+            m_Sensors.Add(sensor1);
         }
 
         public override void CollectObservations()
@@ -57,6 +66,37 @@ public override void AgentOnDone()
         }
     }
 
+    public class TestSensor : ISensor
+    {
+        public string sensorName;
+
+        public TestSensor(string n)
+        {
+            sensorName = n;
+        }
+
+        public int[] GetFloatObservationShape() {
+            return new[] {1};
+        }
+
+        public void WriteToTensor(TensorProxy tensorProxy, int agentIndex) { }
+
+        public byte[] GetCompressedObservation()
+        {
+            return null;
+        }
+
+        public CompressionType GetCompressionType()
+        {
+            return CompressionType.None;
+        }
+
+        public string GetName()
+        {
+            return sensorName;
+        }
+    }
+
     // This is an empty class for testing the behavior of agents and academy
     // It is left empty because we are not testing any brain behavior
     public class TestBrain : Brain
@@ -177,6 +217,10 @@ public void TestAgent()
             Assert.AreEqual(1, agent2.initializeAgentCalls);
             Assert.AreEqual(0, agent1.agentActionCalls);
             Assert.AreEqual(0, agent2.agentActionCalls);
+
+            // Make sure the sensors were sorted
+            Assert.AreEqual(agent1.m_Sensors[0].GetName(), "testsensor1");
+            Assert.AreEqual(agent1.m_Sensors[1].GetName(), "testsensor2");
         }
     }
 
@@ -244,7 +288,6 @@ public void TestAgent()
             agent2.agentParameters.onDemandDecision = true;
             // agent2 will request decisions only when RequestDecision is called
             brain.brainParameters.vectorObservationSize = 0;
-            brain.brainParameters.cameraResolutions = new Resolution[0];
             agent1.GiveBrain(brain);
             agent2.GiveBrain(brain);
 
@@ -370,7 +413,6 @@ public void TestAgent()
             agent2.agentParameters.onDemandDecision = true;
             // agent2 will request decisions only when RequestDecision is called
             brain.brainParameters.vectorObservationSize = 0;
-            brain.brainParameters.cameraResolutions = new Resolution[0];
             agent1.GiveBrain(brain);
             agent2.GiveBrain(brain);
 
@@ -487,7 +529,6 @@ public void TestResetOnDone()
             agent1.agentParameters.resetOnDone = false;
             agent2.agentParameters.resetOnDone = false;
             brain.brainParameters.vectorObservationSize = 0;
-            brain.brainParameters.cameraResolutions = new Resolution[0];
             agent1.GiveBrain(brain);
             agent2.GiveBrain(brain);
 
@@ -567,7 +608,6 @@ public void TestCumulativeReward()
             // agent2 will request decisions only when RequestDecision is called
             agent1.agentParameters.maxStep = 20;
             brain.brainParameters.vectorObservationSize = 0;
-            brain.brainParameters.cameraResolutions = new Resolution[0];
             agent1.GiveBrain(brain);
             agent2.GiveBrain(brain);