Add a glob blacklist option for the build system - some variants are …

…either not buildable (bugs) or want to restrict variants to build for build performance (#11200)

integrations/cloudbuild/build-all.yaml

@@ -17,8 +17,8 @@ steps:
           - PW_ENVIRONMENT_ROOT=/pwenv
       args:
           - >-
-              ./scripts/build/build_examples.py --enable-flashbundle build
-              --create-archives /workspace/artifacts/
+              ./scripts/build/build_examples.py --enable-flashbundle
+              --target-glob '*' build --create-archives /workspace/artifacts/
       id: CompileAll
       waitFor:
           - Bootstrap

integrations/cloudbuild/smoke-test.yaml

@@ -86,8 +86,8 @@ steps:
       args:
           - >-
               ./scripts/build/build_examples.py --enable-flashbundle
-              --target-glob 'android-arm64-chip-tool' build
-              --create-archives /workspace/artifacts/
+              --target-glob 'android-arm64-chip-tool' build --create-archives
+              /workspace/artifacts/
       waitFor:
           - Bootstrap
           - Linux

scripts/build/build/targets.py

@@ -37,17 +37,27 @@ class Target:
     def __init__(self, name, builder_class, **kwargs):
         self.name = name
         self.builder_class = builder_class
+        self.glob_blacklist_reason = None
+
         self.create_kw_args = kwargs
 
+    def Clone(self):
+        """Creates a clone of self."""
+
+        clone = Target(self.name, self.builder_class,
+                       **self.create_kw_args.copy())
+        clone.glob_blacklist_reason = self.glob_blacklist_reason
+
+        return clone
+
     def Extend(self, suffix, **kargs):
         """Creates a clone of the current object extending its build parameters.
 
         Arguments:
            suffix: appended with a "-" as separator to the clone name
            **kargs: arguments needed to produce the new build variant
         """
-        clone = Target(self.name, self.builder_class,
-                       **self.create_kw_args.copy())
+        clone = self.Clone()
         clone.name += "-" + suffix
         clone.create_kw_args.update(kargs)
         return clone
@@ -56,12 +66,31 @@ def Create(self, runner, repository_path: str, output_prefix: str, enable_flashb
         builder = self.builder_class(
             repository_path, runner=runner, **self.create_kw_args)
 
+        builder.target = self
         builder.identifier = self.name
         builder.output_dir = os.path.join(output_prefix, self.name)
         builder.enable_flashbundle(enable_flashbundle)
 
         return builder
 
+    def GlobBlacklist(self, reason):
+        clone = self.Clone()
+        if clone.glob_blacklist_reason:
+            clone.glob_blacklist_reason += ", "
+            clone.glob_blacklist_reason += reason
+        else:
+            clone.glob_blacklist_reason = reason
+
+        return clone
+
+    @property
+    def IsGlobBlacklisted(self):
+        return self.glob_blacklist_reason is not None
+
+    @property
+    def GlobBlacklistReason(self):
+        return self.glob_blacklist_reason
+
 
 def HostTargets():
     target = Target(HostBoard.NATIVE.PlatformName(), HostBuilder)
@@ -135,7 +164,7 @@ def NrfTargets():
     for target in targets:
         yield target.Extend('lock', app=NrfApp.LOCK)
         yield target.Extend('light', app=NrfApp.LIGHT)
-        yield target.Extend('light-rpc', app=NrfApp.LIGHT, enable_rpcs=True)
+        yield target.Extend('light-rpc', app=NrfApp.LIGHT, enable_rpcs=True).GlobBlacklist('Compile failure due to pw_build args not forwarded to proto compiler. https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/66760')
         yield target.Extend('shell', app=NrfApp.SHELL)
         yield target.Extend('pump', app=NrfApp.PUMP)
         yield target.Extend('pump-controller', app=NrfApp.PUMP_CONTROLLER)

scripts/build/build_examples.py

@@ -68,7 +68,7 @@ def ValidateRepoPath(context, parameter, value):
     type=click.Choice(
         ['all'] + [t.name for t in build.ALL_TARGETS], case_sensitive=False),
     multiple=True,
-    help='Build target(s)'
+    help='Build target(s). Note that "all" includes glob blacklisted targets'
 )
 @click.option(
     '--target-glob',
@@ -139,6 +139,8 @@ def main(context, log_level, target, target_glob, skip_target_glob, repo, out_pr
         runner = ShellRunner(root=repo)
 
     if 'all' in target:
+        # NOTE: ALL includes things that are glob blacklisted (so that 'targets' works and
+        # displays all)
         targets = build.ALL_TARGETS
     else:
         requested_targets = set([t.lower for t in target])
@@ -152,7 +154,8 @@ def main(context, log_level, target, target_glob, skip_target_glob, repo, out_pr
 
     if target_glob:
         matcher = GlobMatcher(target_glob)
-        targets = [t for t in targets if matcher.matches(t.name)]
+        targets = [t for t in targets if matcher.matches(
+            t.name) and not t.IsGlobBlacklisted]
 
     if skip_target_glob:
         matcher = GlobMatcher(skip_target_glob)
@@ -184,7 +187,11 @@ def cmd_generate(context):
 @click.pass_context
 def cmd_generate(context):
     for builder in context.obj.builders:
-        print(builder.identifier)
+        if builder.target.IsGlobBlacklisted:
+            print("%s (NOGLOB: %s)" %
+                  (builder.target.name, builder.target.GlobBlacklistReason))
+        else:
+            print(builder.target.name)
 
 
 @main.command('build', help='generate and run ninja/make as needed to compile')

scripts/build/test.py

@@ -106,6 +106,13 @@ def test_targets(self):
             '--skip-target-glob {linux,darwin}-* targets'.split(' ')
         )
 
+    def test_glob_targets(self):
+        self.assertCommandOutput(
+            os.path.join('testdata', 'glob_star_targets_except_host.txt'),
+            '--target-glob * --skip-target-glob {linux,darwin}-* targets'.split(
+                ' ')
+        )
+
     @unittest.skipUnless(sys.platform == 'linux', 'Build on linux test')
     @unittest.skipUnless(os.uname().machine == 'x86_64', 'Validation x64 and crosscompile, requires linux x64')
     def test_linux_build(self):

scripts/build/testdata/all_targets_except_host.txt

@@ -25,13 +25,13 @@ esp32-m5stack-all-clusters-rpc
 esp32-m5stack-all-clusters-rpc-ipv6only
 infineon-p6-lock
 nrf-nrf52840-light
-nrf-nrf52840-light-rpc
+nrf-nrf52840-light-rpc (NOGLOB: Compile failure due to pw_build args not forwarded to proto compiler. https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/66760)
 nrf-nrf52840-lock
 nrf-nrf52840-pump
 nrf-nrf52840-pump-controller
 nrf-nrf52840-shell
 nrf-nrf5340-light
-nrf-nrf5340-light-rpc
+nrf-nrf5340-light-rpc (NOGLOB: Compile failure due to pw_build args not forwarded to proto compiler. https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/66760)
 nrf-nrf5340-lock
 nrf-nrf5340-pump
 nrf-nrf5340-pump-controller

scripts/build/testdata/glob_star_targets_except_host.txt

@@ -0,0 +1,39 @@
+ameba-amebad-all-clusters
+android-androidstudio-arm-chip-tool
+android-androidstudio-arm64-chip-tool
+android-androidstudio-x64-chip-tool
+android-androidstudio-x86-chip-tool
+android-arm-chip-tool
+android-arm64-chip-test
+android-arm64-chip-tool
+android-x64-chip-tool
+android-x86-chip-tool
+efr32-brd4161a-light
+efr32-brd4161a-light-rpc
+efr32-brd4161a-lock
+efr32-brd4161a-window-covering
+esp32-c3devkit-all-clusters
+esp32-devkitc-all-clusters
+esp32-devkitc-all-clusters-ipv6only
+esp32-devkitc-bridge
+esp32-devkitc-lock
+esp32-devkitc-shell
+esp32-devkitc-temperature-measurement
+esp32-m5stack-all-clusters
+esp32-m5stack-all-clusters-ipv6only
+esp32-m5stack-all-clusters-rpc
+esp32-m5stack-all-clusters-rpc-ipv6only
+infineon-p6-lock
+nrf-nrf52840-light
+nrf-nrf52840-lock
+nrf-nrf52840-pump
+nrf-nrf52840-pump-controller
+nrf-nrf52840-shell
+nrf-nrf5340-light
+nrf-nrf5340-lock
+nrf-nrf5340-pump
+nrf-nrf5340-pump-controller
+nrf-nrf5340-shell
+qpg-qpg6100-lock
+telink-tlsr9518adk80d-light
+tizen-arm-light