Better management of targets in build_examples.py, support -rpc when …

…building all (#10133) * Initial version to convert to build targets - scripts only for now * Update cloudbuild to use the new target format * Update documentation a bit * Correctly name output dirs for linux * Add a sample glob matcher as a convenience for builds * Update documentation and make compilation based on targetts work * Update list of available targets for building in the vscode interface * Remove non-json comments * Fix glob target in vscode * Fix globbing prefix start, update vscode tasks * Make unit tests pass (but not host yet) * More unit tests: this validates linux builds as well * Restyle and fix gn args * Make tizen build be a standard gn build * Remove rpc command line argument * Add rpc targets to vscode * Fix unit test: had duplicated method name preventing testing * Fix github workflows with the new target syntax * Typo: target_glob to target-glob * Update android to run unit tests * Update infineon output dir * Update infineon dir...again * Code review updates * Remove empty whitespace from non-md file to see if that triggers CI * Minor python file change (doc text update) to try to trigger CI run
project-chip · Nov 24, 2021 · 1405440 · 1405440
1 parent 8e4a69a
commit 1405440
Show file tree

Hide file tree

Showing 32 changed files with 786 additions and 909 deletions.
diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml
@@ -57,8 +57,8 @@ jobs:
             - name: Build android examples
               run: |
                 ./scripts/run_in_build_env.sh \
-                  "./scripts/build/build_examples.py --no-log-timestamps --platform android build"
+                  "./scripts/build/build_examples.py --no-log-timestamps --target-glob 'android-*' build"
             - name: Run Android build rule tests
               run: |
                 ./scripts/run_in_build_env.sh \
-                  "ninja -C out/android-arm64-chip_tool build/chip/java/tests:java_build_test.tests"
+                  "ninja -C out/android-arm64-chip-tool build/chip/java/tests:java_build_test.tests"
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -22,7 +22,7 @@ on:
 concurrency:
     group: ${{ github.ref }}-${{ github.workflow }}-${{ (github.event_name == 'pull_request' && github.event.number) || (github.event_name == 'workflow_dispatch' && github.run_number) || github.sha }}
     cancel-in-progress: true
-  
+
 jobs:
     build_linux_gcc_debug:
         name: Build on Linux (gcc_debug)

diff --git a/.github/workflows/examples-infineon.yaml b/.github/workflows/examples-infineon.yaml
@@ -21,7 +21,7 @@ on:
 
 concurrency:
     group: ${{ github.ref }}-${{ github.workflow }}-${{ (github.event_name == 'pull_request' && github.event.number) || (github.event_name == 'workflow_dispatch' && github.run_number) || github.sha }}
-    cancel-in-progress: true    
+    cancel-in-progress: true
 
 jobs:
     infineon:
@@ -59,15 +59,15 @@ jobs:
               timeout-minutes: 10
               run: |
                 scripts/run_in_build_env.sh \
-                  "scripts/build/build_examples.py --no-log-timestamps --platform infineon --app lock build"
+                  "scripts/build/build_examples.py --no-log-timestamps --target-glob 'infineon-*' build"
                 .environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py \
                     p6 default lock-app \
-                    out/infineon-p6board-lock/chip-p6-lock-example.out
+                    out/infineon-p6-lock/chip-p6-lock-example.out
 
             - name: Uploading Size Reports
               uses: actions/upload-artifact@v2
               if: ${{ !env.ACT }}
               with:
                   name: Size,P6-Examples,${{ env.GH_EVENT_PR }},${{ env.GH_EVENT_HASH }},${{ env.GH_EVENT_PARENT }}
                   path: |
-                    out/infineon-p6board-lock/p6-default-lock-app-sizes.json
+                    out/infineon-p6-lock/p6-default-lock-app-sizes.json
diff --git a/.github/workflows/examples-telink.yaml b/.github/workflows/examples-telink.yaml
@@ -47,7 +47,7 @@ jobs:
             - name: Build example Telink Lighting App
               run: |
                 ./scripts/run_in_build_env.sh \
-                  "./scripts/build/build_examples.py --no-log-timestamps --platform telink --app light build"
+                  "./scripts/build/build_examples.py --no-log-timestamps --target-glob 'telink-*' build"
                 .environment/pigweed-venv/bin/python3 scripts/tools/memory/gh_sizes.py \
                   telink tlsr9518adk80d lighting-app \
                   out/telink-tlsr9518adk80d-light/zephyr/zephyr.elf \

diff --git a/.github/workflows/examples-tizen.yaml b/.github/workflows/examples-tizen.yaml
@@ -41,4 +41,4 @@ jobs:
                   submodules: true
             - name: Build example Tizen lighting app
               run:
-                  scripts/run_in_build_env.sh "./scripts/build/build_examples.py --platform tizen build"
+                  scripts/run_in_build_env.sh "./scripts/build/build_examples.py --target-glob 'tizen-*' build"
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -2,35 +2,23 @@
     "version": "2.0.0",
     "tasks": [
         {
-            "label": "Build examples for platform",
+            "label": "Build specific example",
             "type": "shell",
             "command": "scripts/run_in_build_env.sh",
             "args": [
-                "./scripts/build/build_examples.py --platform ${input:examplePlatform} build"
+                "./scripts/build/build_examples.py --target ${input:exampleTarget} build"
             ],
             "group": "build",
             "problemMatcher": {
                 "base": "$gcc"
             }
         },
         {
-            "label": "Build specific example (all compatible platforms)",
+            "label": "Build examples (glob)",
             "type": "shell",
             "command": "scripts/run_in_build_env.sh",
             "args": [
-                "./scripts/build/build_examples.py --app ${input:exampleApp} build"
-            ],
-            "group": "build",
-            "problemMatcher": {
-                "base": "$gcc"
-            }
-        },
-        {
-            "label": "Build single example (specific board/example)",
-            "type": "shell",
-            "command": "scripts/run_in_build_env.sh",
-            "args": [
-                "./scripts/build/build_examples.py --board ${input:exampleBoard} --app ${input:exampleApp} build"
+                "./scripts/build/build_examples.py --target-glob \"${input:exampleGlob}\" build"
             ],
             "group": "build",
             "problemMatcher": {
@@ -114,29 +102,6 @@
             "group": "none",
             "problemMatcher": ["$gcc"]
         },
-        {
-            "label": "Build Shell",
-            "type": "shell",
-            "command": "scripts/examples/gn_build_example.sh examples/shell/standalone out/shell",
-            "group": "build",
-            "problemMatcher": {
-                "base": "$gcc",
-                "fileLocation": ["relative", "${workspaceFolder}/out/shell/"]
-            }
-        },
-        {
-            "label": "Build Lighting App (Linux)",
-            "type": "shell",
-            "command": "scripts/examples/gn_build_example.sh examples/lighting-app/linux out/lighting-app",
-            "group": "build",
-            "problemMatcher": {
-                "base": "$gcc",
-                "fileLocation": [
-                    "relative",
-                    "${workspaceFolder}/out/lighting-app/"
-                ]
-            }
-        },
         {
             "label": "Build Minimal MDNS Example",
             "type": "shell",
@@ -169,19 +134,6 @@
                 ]
             }
         },
-        {
-            "label": "Build Telink Lighting Example (TLSR9518ADK80D)",
-            "type": "shell",
-            "command": "source scripts/activate.sh && scripts/examples/telink_example.sh lighting-app tlsr9518adk80d",
-            "group": "build",
-            "problemMatcher": {
-                "base": "$gcc",
-                "fileLocation": [
-                    "relative",
-                    "${workspaceFolder}/examples/lighting-app/telink/build/tlsr9518adk80d"
-                ]
-            }
-        },
         {
             "label": "Run Mbed Application",
             "type": "shell",
@@ -250,77 +202,53 @@
             "default": "develop"
         },
         {
-            "id": "efr32_board",
-            "type": "pickString",
-            "description": "What's the type of your EFR32 board?",
-            "options": [
-                "BRD4304A",
-                "BRD4161A",
-                "BRD4163A",
-                "BRD4164A",
-                "BRD4166A",
-                "BRD4170A",
-                "BRD4186A",
-                "BRD4187A"
-            ],
-            "default": "BRD4166A"
+            "type": "promptString",
+            "id": "exampleGlob",
+            "description": "What applications to build (e.g. '*m5stack*' or 'esp32-*')"
         },
         {
             "type": "pickString",
-            "id": "examplePlatform",
-            "description": "Which platform would you like to build",
+            "id": "exampleTarget",
+            "description": "What target to build",
             "options": [
-                "all",
-                "android",
-                "efr32",
-                "esp32",
-                "host",
-                "nrf",
-                "qpg",
-                "infineon",
-                "telink",
-                "tizen"
-            ],
-            "default": "build"
-        },
-        {
-            "type": "pickString",
-            "id": "exampleApp",
-            "description": "Which example application would you like to build",
-            "options": [
-                "all-clusters",
-                "bridge",
-                "chip-tool",
-                "light",
-                "lock",
-                "shell",
-                "temperature-measurement",
-                "thermostat",
-                "window-covering",
-                "pump",
-                "pump-controller"
-            ],
-            "default": "build"
-        },
-        {
-            "type": "pickString",
-            "id": "exampleBoard",
-            "description": "Which example board to build for",
-            "options": [
-                "arm",
-                "arm64",
-                "brd4161a",
-                "c3devkit",
-                "devkitc",
-                "m5stack",
-                "native",
-                "nrf52840",
-                "nrf5340",
-                "qpg6100",
-                "x64",
-                "p6board"
-            ],
-            "default": "build"
+                "android-arm-chip-tool",
+                "android-arm64-chip-tool",
+                "android-x64-chip-tool",
+                "android-x86-chip-tool",
+                "efr32-brd4161a-light-light-rpc",
+                "efr32-brd4161a-light-light",
+                "efr32-brd4161a-light-lock-rpc",
+                "efr32-brd4161a-light-lock",
+                "efr32-brd4161a-light-window-covering-rpc",
+                "efr32-brd4161a-light-window-covering",
+                "esp32-c3devkit-all-clusters",
+                "esp32-devkitc-all-clusters",
+                "esp32-devkitc-bridge",
+                "esp32-devkitc-lock",
+                "esp32-devkitc-shell",
+                "esp32-devkitc-temperature-measurement",
+                "esp32-m5stack-all-clusters",
+                "infineon-p6-lock",
+                "linux-arm64-all-clusters",
+                "linux-arm64-chip-tool",
+                "linux-arm64-thermostat",
+                "linux-x64-all-clusters",
+                "linux-x64-chip-tool",
+                "linux-x64-thermostat",
+                "nrf-nrf52840-light",
+                "nrf-nrf52840-lock",
+                "nrf-nrf52840-pump",
+                "nrf-nrf52840-pump-controller",
+                "nrf-nrf52840-shell",
+                "nrf-nrf5340-light",
+                "nrf-nrf5340-lock",
+                "nrf-nrf5340-pump",
+                "nrf-nrf5340-pump-controller",
+                "nrf-nrf5340-shell",
+                "qpg-qpg6100-lock",
+                "telink-tlsr9518adk80d-light",
+                "tizen-arm-light"
+            ]
         }
     ]
 }
diff --git a/examples/lighting-app/telink/Readme.md b/examples/lighting-app/telink/Readme.md
@@ -26,7 +26,7 @@
 1. Run build script:
 
     ```
-    ./scripts/build/build_examples.py --app light --platform telink build
+    ./scripts/build/build_examples.py --target telink-tlsr9518adk80d-light build
     ```
 
 1. Exit docker container and collect build artifacts. Firmware binary would be

diff --git a/integrations/cloudbuild/build-all.yaml b/integrations/cloudbuild/build-all.yaml
@@ -5,8 +5,8 @@ steps:
       entrypoint: "./scripts/run_in_build_env.sh"
       args:
           [
-              "./scripts/build/build_examples.py --platform all
-              --enable-flashbundle build --create-archives /workspace/artifacts/",
+              "./scripts/build/build_examples.py --enable-flashbundle build
+              --create-archives /workspace/artifacts/",
           ]
       timeout: 7200s
 

diff --git a/scripts/build/BUILD.gn b/scripts/build/BUILD.gn
@@ -22,24 +22,33 @@ pw_python_package("build_examples") {
   setup = [ "setup.py" ]
   inputs = [
     # Dependency for the unit test
-    "expected_all_platform_commands.txt",
+    "testdata/all_targets_except_host.txt",
+    "testdata/build_all_except_host.txt",
+    "testdata/build_linux_on_x64.txt",
   ]
 
   sources = [
     "build/__init__.py",
-    "build/factory.py",
     "build/targets.py",
     "build_examples.py",
     "builders/__init__.py",
     "builders/builder.py",
     "builders/efr32.py",
     "builders/esp32.py",
+    "builders/gn.py",
     "builders/host.py",
     "builders/infineon.py",
+    "builders/nrf.py",
     "builders/qpg.py",
+    "builders/telink.py",
+    "builders/tizen.py",
     "runner/__init__.py",
     "runner/printonly.py",
     "runner/shell.py",
   ]
-  tests = [ "test.py" ]
+
+  tests = [
+    "test.py",
+    "test_glob_matcher.py",
+  ]
 }
diff --git a/scripts/build/README.md b/scripts/build/README.md
@@ -32,34 +32,39 @@ requirement.
 
 Usage examples:
 
-1. Compiles the Lock app on all supported platforms
+1. Compiles all targets
 
     ```
-    ./scripts/build/build_examples.py --app lock build
+    ./scripts/build/build_examples.py --target all build
     ```
 
 2. Compile the all clusters app for a ESP32 DevKitC
 
     ```
-    ./scripts/build/build_examples.py --app all_clusters_app --board devkitc build
+    ./scripts/build/build_examples.py --target esp32-devkitc-all-clusters build
     ```
 
-3. Generate all the build rules (but do not compile) all native apps
+3. Generate all the makefiles (but do not compile) using a specific output root
 
     ```
-    ./scripts/build/build_examples.py --platform native generate
+    ./scripts/build/build_examples.py --target linux-x64-chip-tool --out-prefix ./mydir gen
     ```
 
-4. Generate all the makefiles (but do not compile) using a specific output root
+4. Compile the qpg lock app and copy the output in a 'artifact' folder. Note the
+   argument order (artifact copying is an argument for the build command)
 
     ```
-    ./scripts/build/build_examples.py --platform native generate --out-prefix ./mydir
+    ./scripts/build/build_examples.py --target qpg-qpg6100-lock build --copy-artifacts-to /tmp/artifacts
     ```
 
-5. Compile the qpg lock app and copy the output in a 'artifact' folder. Note the
-   argument order (artifact copying is an argument for the build command)
+5. Find out all possible targets for compiling the 'light' app:
+
+    ```
+    ./scripts/build/build_examples.py --target-glob '*light' --log-level fatal targets
+    ```
+
+6. Compile everything except linux or darwin:
 
     ```
-    ./scripts/build/build_examples.py --board qpg6100 --app lock build \
-       --copy-artifacts-to /tmp/artifacts
+    ./scripts/build/build_examples.py --skip-target-glob '{darwin,linux}-*' --log-level fatal build
     ```