enforce min coverage % (#3642)

Author: Chris Elion

.yamato/com.unity.ml-agents-test.yml

@@ -2,10 +2,13 @@ test_editors:
   - version: 2018.4
     # 2018.4 doesn't support code-coverage
     coverageOptions:
+    minCoveragePct: 0
   - version: 2019.3
     coverageOptions: --enable-code-coverage --code-coverage-options 'generateHtmlReport;assemblyFilters:+Unity.ML-Agents'
+    minCoveragePct: 72
   - version: 2020.1
     coverageOptions: --enable-code-coverage --code-coverage-options 'generateHtmlReport;assemblyFilters:+Unity.ML-Agents'
+    minCoveragePct: 72
 test_platforms:
   - name: win
     type: Unity::VM
@@ -32,6 +35,7 @@ test_{{ platform.name }}_{{ editor.version }}:
   commands:
     - npm install upm-ci-utils@stable -g --registry https://api.bintray.com/npm/unity/unity-npm
     - upm-ci package test -u {{ editor.version }} --package-path com.unity.ml-agents {{ editor.coverageOptions }}
+    - python ml-agents/tests/yamato/check_coverage_percent.py upm-ci~/test-results/ {{ editor.minCoveragePct }}
   artifacts:
     logs:
       paths:
@@ -42,6 +46,7 @@ test_{{ platform.name }}_{{ editor.version }}:
     changes:
       only:
         - "com.unity.ml-agents/**"
+        - "ml-agents/tests/yamato/**"
         - ".yamato/com.unity.ml-agents-test.yml"
 
   {% endfor %}

com.unity.ml-agents/Runtime/Sensors/StackingSensor.cs

@@ -64,7 +64,7 @@ public StackingSensor(ISensor wrapped, int numStackedObservations)
 
         public int Write(WriteAdapter adapter)
         {
-            // First, call the wrapped sensor's write method. Make sure to use our own adapater, not the passed one.
+            // First, call the wrapped sensor's write method. Make sure to use our own adapter, not the passed one.
             var wrappedShape = m_WrappedSensor.GetObservationShape();
             m_LocalAdapter.SetTarget(m_StackedObservations[m_CurrentIndex], wrappedShape, 0);
             m_WrappedSensor.Write(m_LocalAdapter);

ml-agents/tests/yamato/check_coverage_percent.py

@@ -0,0 +1,61 @@
+from __future__ import print_function
+import sys
+import os
+
+SUMMARY_XML_FILENAME = "Summary.xml"
+
+# Note that this is python2 compatible, since that's currently what's installed on most CI images.
+
+
+def check_coverage(root_dir, min_percentage):
+    # Walk the root directory looking for the summary file that
+    # is output by ther code coverage checks. It's possible that
+    # we'll need to refine this later in case there are multiple
+    # such files.
+    summary_xml = None
+    for dirpath, _, filenames in os.walk(root_dir):
+        if SUMMARY_XML_FILENAME in filenames:
+            summary_xml = os.path.join(dirpath, SUMMARY_XML_FILENAME)
+            break
+    if not summary_xml:
+        print("Couldn't find {} in root directory".format(SUMMARY_XML_FILENAME))
+        sys.exit(1)
+
+    with open(summary_xml) as f:
+        # Rather than try to parse the XML, just look for a line of the form
+        # <Linecoverage>73.9</Linecoverage>
+        lines = f.readlines()
+        for l in lines:
+            if "Linecoverage" in l:
+                pct = l.replace("<Linecoverage>", "").replace("</Linecoverage>", "")
+                pct = float(pct)
+                if pct < min_percentage:
+                    print(
+                        "Coverage {} is below the min percentage of {}.".format(
+                            pct, min_percentage
+                        )
+                    )
+                    sys.exit(1)
+                else:
+                    print(
+                        "Coverage {} is above the min percentage of {}.".format(
+                            pct, min_percentage
+                        )
+                    )
+                    sys.exit(0)
+
+    # Couldn't find the results in the file.
+    print("Couldn't find Linecoverage in summary file")
+    sys.exit(1)
+
+
+def main():
+    root_dir = sys.argv[1]
+    min_percent = float(sys.argv[2])
+    if min_percent > 0:
+        # This allows us to set 0% coverage on 2018.4
+        check_coverage(root_dir, min_percent)
+
+
+if __name__ == "__main__":
+    main()