Use CMake and separate runner for QEMU tests for esp32 (#13095)

* Start adding a tests target to qemu * Fix extra compatibility test * Prepare qemu builds for the test driver using cmake * Ensure sdkconfig can be ignored for esp32 test driver: this file is re-written by the cmake build * In theory we compile and link at least one test * Tests now build for a few sub-items * Tests now build for a few sub-items * move the cmake logic for esp tests into a separate cmakefile * bin is also built by the cmake system now * Start being able to run test - for now remove the qemu sh scripts * Remove obsolete makefiles * Switch unit test execution to a python script * Some comments and cleanup * Restyle files * Upate build rules and logic - we detect failing and passing tests now * Add a verbose test argument * Restyle files * Fix flashbundle logic * Correct placement of verbose argument * Correct placement of verbose argument * Run test runner with python3 since that is available (just python will not work in our build image) * Update build example py unit tests
project-chip · Jun 9, 2022 · 2519988 · 2519988
1 parent 7b91b64
commit 2519988
Show file tree

Hide file tree

Showing 23 changed files with 420 additions and 337 deletions.
diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml
@@ -54,12 +54,20 @@ jobs:
                   path: |
                    .environment/gn_out/.ninja_log
                    .environment/pigweed-venv/*.log
-            - name: Build example All Clusters App
-              timeout-minutes: 25
-              run: scripts/examples/esp_example.sh all-clusters-app
-            - name: Build ESP32 QEMU and Run Tests
-              timeout-minutes: 35
-              run: scripts/tests/esp32_qemu_tests.sh /tmp/test_logs
+            - name: Build ESP32 QEMU test images
+              timeout-minutes: 20
+              run: |
+                  scripts/run_in_build_env.sh "         \
+                      ./scripts/build/build_examples.py \
+                      --target esp32-qemu-tests         \
+                      build                             \
+                  "
+            - name: Run all tests
+              timeout-minutes: 30
+              run: |
+                  src/test_driver/esp32/run_qemu_image.py \
+                    --verbose                             \
+                    --file-image-list ./out/esp32-qemu-tests/test_images.txt
             - name: Uploading Logs
               uses: actions/upload-artifact@v2
               if: ${{ !env.ACT }}

diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py
@@ -220,6 +220,8 @@ def Esp32Targets():
     yield devkitc.Extend('bridge', app=Esp32App.BRIDGE)
     yield devkitc.Extend('temperature-measurement', app=Esp32App.TEMPERATURE_MEASUREMENT)
 
+    yield esp32_target.Extend('qemu-tests', board=Esp32Board.QEMU, app=Esp32App.TESTS)
+
 
 def Efr32Targets():
     efr_target = Target('efr32', Efr32Builder)

diff --git a/scripts/build/builders/esp32.py b/scripts/build/builders/esp32.py
@@ -25,6 +25,7 @@ class Esp32Board(Enum):
     DevKitC = auto()
     M5Stack = auto()
     C3DevKit = auto()
+    QEMU = auto()
 
 
 class Esp32App(Enum):
@@ -33,19 +34,22 @@ class Esp32App(Enum):
     SHELL = auto()
     BRIDGE = auto()
     TEMPERATURE_MEASUREMENT = auto()
+    TESTS = auto()
 
     @property
-    def ExampleName(self):
+    def ExamplePath(self):
         if self == Esp32App.ALL_CLUSTERS:
-            return 'all-clusters-app'
+            return 'examples/all-clusters-app'
         elif self == Esp32App.LOCK:
-            return 'lock-app'
+            return 'examples/lock-app'
         elif self == Esp32App.SHELL:
-            return 'shell'
+            return 'examples/shell'
         elif self == Esp32App.BRIDGE:
-            return 'bridge-app'
+            return 'examples/bridge-app'
         elif self == Esp32App.TEMPERATURE_MEASUREMENT:
-            return 'temperature-measurement-app'
+            return 'examples/temperature-measurement-app'
+        elif self == Esp32App.TESTS:
+            return 'src/test_driver'
         else:
             raise Exception('Unknown app type: %r' % self)
 
@@ -61,15 +65,33 @@ def AppNamePrefix(self):
             return 'chip-bridge-app'
         elif self == Esp32App.TEMPERATURE_MEASUREMENT:
             return 'chip-temperature-measurement-app'
+        elif self == Esp32App.TESTS:
+            return None
         else:
             raise Exception('Unknown app type: %r' % self)
 
+    @property
     def FlashBundleName(self):
+        if not self.AppNamePrefix:
+            return None
+
         return self.AppNamePrefix + '.flashbundle.txt'
 
+    def IsCompatible(self, board: Esp32Board):
+        if board == Esp32Board.QEMU:
+            return self == Esp32App.TESTS
+        elif board == Esp32Board.M5Stack:
+            return self == Esp32App.ALL_CLUSTERS
+        elif board == Esp32Board.C3DevKit:
+            return self == Esp32App.ALL_CLUSTERS
+        else:
+            return (board == Esp32Board.DevKitC) and (self != Esp32App.TESTS)
+
 
 def DefaultsFileName(board: Esp32Board, app: Esp32App, enable_rpcs: bool):
-    if app != Esp32App.ALL_CLUSTERS:
+    if app == Esp32App.TESTS:
+        return 'sdkconfig_qemu.defaults'
+    elif app != Esp32App.ALL_CLUSTERS:
         return 'sdkconfig.defaults'
 
     rpc = "_rpc" if enable_rpcs else ""
@@ -99,6 +121,10 @@ def __init__(self,
         self.enable_rpcs = enable_rpcs
         self.enable_ipv4 = enable_ipv4
 
+        if not app.IsCompatible(board):
+            raise Exception(
+                "Incompatible app/board combination: %r and %r", app, board)
+
     def _IdfEnvExecute(self, cmd, title=None):
         # Run activate.sh after export.sh to ensure using the chip environment.
         self._Execute(
@@ -107,7 +133,7 @@ def _IdfEnvExecute(self, cmd, title=None):
 
     @property
     def ExamplePath(self):
-        return os.path.join('examples', self.app.ExampleName, 'esp32')
+        return os.path.join(self.app.ExamplePath, 'esp32')
 
     def generate(self):
         if os.path.exists(os.path.join(self.output_dir, 'build.ninja')):
@@ -162,6 +188,16 @@ def _build(self):
         self._IdfEnvExecute(cmd, title='Building ' + self.identifier)
 
     def build_outputs(self):
+        if self.app == Esp32App.TESTS:
+            # Include the runnable image names as artifacts
+            result = dict()
+            with open(os.path.join(self.output_dir, 'test_images.txt'), 'rt') as f:
+                for name in f.readlines():
+                    name = name.strip()
+                    result[name] = os.path.join(self.output_dir, name)
+
+            return result
+
         return {
             self.app.AppNamePrefix + '.elf':
                 os.path.join(self.output_dir, self.app.AppNamePrefix + '.elf'),
@@ -170,7 +206,10 @@ def build_outputs(self):
         }
 
     def flashbundle(self):
-        with open(os.path.join(self.output_dir, self.app.FlashBundleName()), 'r') as fp:
+        if not self.app.FlashBundleName:
+            return {}
+
+        with open(os.path.join(self.output_dir, self.app.FlashBundleName), 'r') as fp:
             return {
                 l.strip(): os.path.join(self.output_dir, l.strip()) for l in fp.readlines() if l.strip()
             }
diff --git a/scripts/build/testdata/all_targets_except_host.txt b/scripts/build/testdata/all_targets_except_host.txt
@@ -70,6 +70,7 @@ esp32-m5stack-all-clusters
 esp32-m5stack-all-clusters-ipv6only
 esp32-m5stack-all-clusters-rpc
 esp32-m5stack-all-clusters-rpc-ipv6only
+esp32-qemu-tests
 infineon-p6-all-clusters
 infineon-p6-light
 infineon-p6-lock

diff --git a/scripts/build/testdata/build_all_except_host.txt b/scripts/build/testdata/build_all_except_host.txt
@@ -410,6 +410,17 @@ bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh;
 export SDKCONFIG_DEFAULTS={out}/esp32-m5stack-all-clusters-rpc-ipv6only/sdkconfig.defaults
 idf.py -C examples/all-clusters-app/esp32 -B {out}/esp32-m5stack-all-clusters-rpc-ipv6only reconfigure'
 
+# Generating esp32-qemu-tests
+mkdir -p {out}/esp32-qemu-tests
+
+cp src/test_driver/esp32/sdkconfig_qemu.defaults {out}/esp32-qemu-tests/sdkconfig.defaults
+
+rm -f src/test_driver/esp32/sdkconfig
+
+bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh; 
+export SDKCONFIG_DEFAULTS={out}/esp32-qemu-tests/sdkconfig.defaults
+idf.py -C src/test_driver/esp32 -B {out}/esp32-qemu-tests reconfigure'
+
 # Generating infineon-p6-all-clusters
 gn gen --check --fail-on-unused-args --export-compile-commands --root={root}/examples/all-clusters-app/p6 '--args=p6_board="CY8CKIT-062S2-43012"' {out}/infineon-p6-all-clusters
 
@@ -1002,6 +1013,13 @@ bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh;
 export SDKCONFIG_DEFAULTS={out}/esp32-m5stack-all-clusters-rpc-ipv6only/sdkconfig.defaults
 idf.py -C examples/all-clusters-app/esp32 -B {out}/esp32-m5stack-all-clusters-rpc-ipv6only build'
 
+rm -f src/test_driver/esp32/sdkconfig
+
+# Building esp32-qemu-tests
+bash -c 'source $IDF_PATH/export.sh; source scripts/activate.sh; 
+export SDKCONFIG_DEFAULTS={out}/esp32-qemu-tests/sdkconfig.defaults
+idf.py -C src/test_driver/esp32 -B {out}/esp32-qemu-tests build'
+
 # Building infineon-p6-all-clusters
 ninja -C {out}/infineon-p6-all-clusters
 

diff --git a/scripts/build/testdata/glob_star_targets_except_host.txt b/scripts/build/testdata/glob_star_targets_except_host.txt
@@ -28,6 +28,7 @@ esp32-m5stack-all-clusters
 esp32-m5stack-all-clusters-ipv6only
 esp32-m5stack-all-clusters-rpc
 esp32-m5stack-all-clusters-rpc-ipv6only
+esp32-qemu-tests
 infineon-p6-all-clusters
 infineon-p6-light
 infineon-p6-lock

diff --git a/scripts/tests/esp32_qemu_tests.sh b/scripts/tests/esp32_qemu_tests.sh
diff --git a/scripts/tools/esp32_qemu_run.sh b/scripts/tools/esp32_qemu_run.sh