Unified build script for chip - out of source build for all applicati…

…ons/targets/boards (#8221)

* Imported a general chip builder script, currently covering a few platforms/boards/apps as a start to have a single build entrypoint

* Move build requirements into global script/requirements.txt so that they get picked up by the bootstrap script

* Update script to assume and require bootstrapping

* Code review comments

* Support building the lock app for ESP32

It turns out that ESP32 lock app is ONLY for devkitc, so
I added application matching logic to include board restrictions.

Tested: lock app compiles and only for devkitc if esp32 platform is
used.

* Remove obsolete todo

* Fix the duplicated accept for efr32 lock app

* Address some code review comments

* Rename chipbuild to build_examples

* Fix names

* Restyle

scripts/build/README.md

@@ -0,0 +1,65 @@
+# CHIP automated builds scripts
+
+CHIP compilation is generally split into two steps
+
+1. Generate ninja/makefile for out-of-source builds
+2. Compilation using ninja/makefiles
+
+## Building manually
+
+Manual building is generally platform-dependent. All build steps would require a
+bootstrapped environment (loads a pigweed build environment) and will then be
+followed by platform-specific instructions.
+
+The file BUILDING.md describes general requirements and examples. Typical usage
+is:
+
+```
+source scripts/activate
+gn gen out/host
+ninja -C out/host
+```
+
+## Unified build script
+
+The script `build_examples.py` provides a single entry point for generating and
+executing the build.
+
+Build environment _MUST_ be properly configured for build_examples to succeed.
+For example ESP32 builds requite IDF_PATH to be set. Building in the
+corresponding build image or the chip vscode image satisfy the build environment
+requirement.
+
+Usage examples:
+
+1. Compiles the Lock app on all supported platforms
+
+    ```
+    ./scripts/build/build_examples.py --app lock build
+    ```
+
+2. Compile the all clusters app for a ESP32 DevKitC
+
+    ```
+    ./scripts/build/build_examples.py --app all_clusters_app --board devkitc build
+    ```
+
+3. Generate all the build rules (but do not compile) all native apps
+
+    ```
+    ./scripts/build/build_examples.py --platform native generate
+    ```
+
+4. Generate all the makefiles (but do not compile) using a specific output root
+
+    ```
+    ./scripts/build/build_examples.py --platform native generate --out-prefix ./mydir
+    ```
+
+5. Compile the qpg lock app and copy the output in a 'artifact' folder. Note the
+   argument order (artifact copying is an argument for the build command)
+
+    ```
+    ./scripts/build/build_examples.py --board qpg6100 --app lock build \
+       --copy-artifacts-to /tmp/artifacts
+    ```

scripts/build/build/__init__.py

@@ -0,0 +1,153 @@
+import logging
+import os
+import shutil
+
+from enum import Enum, auto
+from typing import Sequence
+
+from .targets import Platform, Board, Application
+from .factory import BuilderFactory, TargetRelations
+
+
+def CommaSeparate(items) -> str:
+  return ', '.join([x.ArgName for x in items])
+
+
+# Supported platforms/boards/apps for generation/compilation
+# note that not all combinations are supported, however command-line will
+# accept all combinations
+PLATFORMS = [x.ArgName for x in Platform]
+BOARDS = [x.ArgName for x in Board]
+APPLICATIONS = [x.ArgName for x in Application]
+
+
+class BuildSteps(Enum):
+  GENERATED = auto()
+
+
+class Context:
+  """Represents a grouped list of platform/board/app builders to use
+
+       to generate make/ninja instructions and to compile.
+    """
+
+  def __init__(self, repository_path:str, output_prefix:str):
+    self.builders = []
+    self.repository_path = repository_path
+    self.output_prefix = output_prefix
+    self.completed_steps = set()
+
+  def SetupBuilders(self, platforms: Sequence[Platform],
+                    boards: Sequence[Board],
+                    applications: Sequence[Application]):
+    """Configures internal builders for the given platform/board/app combination.
+
+        Handles smart default selection, so that users only need to specify
+        part of platform/board/application information and the method tries
+        to automatically deduce the rest of the arguments.
+        """
+    if not platforms and not boards:
+      if applications:
+        platforms = set().union(*[
+            TargetRelations.PlatformsForApplication(app) for app in applications
+        ])
+      else:
+        # when nothing is specified, start with a default host build
+        # TODO: this is only for linux. Should be moved to 'HOST' as a platform
+        # to also support building on MacOS
+        platforms = [Platform.LINUX]
+
+    # at this point, at least one of 'platforms' or 'boards' is non-empty
+    if not boards:
+      boards = set().union(*[
+          TargetRelations.BoardsForPlatform(platform) for platform in platforms
+      ])
+    elif not platforms:
+      platforms = set().union(
+          *[TargetRelations.PlatformsForBoard(board) for board in boards])
+
+    if not applications:
+      applications = set().union(*[
+          TargetRelations.ApplicationsForPlatform(platform)
+          for platform in platforms
+      ])
+
+    platforms = set(platforms)
+    boards = set(boards)
+    applications = set(applications)
+
+    logging.info('Platforms being built: %s', CommaSeparate(platforms))
+    logging.info('Boards being built: %s', CommaSeparate(boards))
+    logging.info('Applications being built: %s', CommaSeparate(applications))
+
+    # Sanity check: ensure all input arguments generate at least an output
+    platforms_with_builders = set()
+    boards_with_builders = set()
+    applications_with_builders = set()
+
+    factory = BuilderFactory(self.repository_path, self.output_prefix)
+
+    for platform in platforms:
+      for board in boards:
+        for application in applications:
+          builder = factory.Create(platform, board, application)
+          if not builder:
+            logging.debug('Builder not supported for tuple %s/%s/%s', platform,
+                          board, application)
+            continue
+
+          self.builders.append(builder)
+          platforms_with_builders.add(platform)
+          boards_with_builders.add(board)
+          applications_with_builders.add(application)
+
+    if platforms != platforms_with_builders:
+      logging.warn('Platforms without build output: %s',
+                   CommaSeparate(platforms.difference(platforms_with_builders)))
+
+    if boards != boards_with_builders:
+      logging.warn('Boards without build output: %s',
+                   CommaSeparate(boards.difference(boards_with_builders)))
+
+    if applications != applications_with_builders:
+      logging.warn(
+          'Applications without build output: %s',
+          CommaSeparate(applications.difference(applications_with_builders)))
+
+    # whenever builders change, assume generation is required again
+    self.completed_steps.discard(BuildSteps.GENERATED)
+
+  def Generate(self):
+    """Performs a build generation IFF code generation has not yet been performed."""
+    if BuildSteps.GENERATED in self.completed_steps:
+      return
+
+    for builder in self.builders:
+      logging.info('Generating %s', builder.output_dir)
+      builder.generate()
+
+    self.completed_steps.add(BuildSteps.GENERATED)
+
+  def Build(self):
+    self.Generate()
+
+    for builder in self.builders:
+      logging.info('Building %s', builder.output_dir)
+      builder.build()
+
+  def CleanOutputDirectories(self):
+    for builder in self.builders:
+      logging.warn('Cleaning %s', builder.output_dir)
+      shutil.rmtree(builder.output_dir)
+
+    # any generated output was cleaned
+    self.completed_steps.discard(BuildSteps.GENERATED)
+
+  def CopyArtifactsTo(self, path: str):
+    logging.info('Copying build artifacts to %s', path)
+    if not os.path.exists(path):
+      os.makedirs(path)
+
+    for builder in self.builders:
+      # FIXME: builder subdir...
+      builder.CopyArtifacts(os.path.join(path, builder.identifier))

scripts/build/build/factory.py

@@ -0,0 +1,154 @@
+import os
+
+from typing import Set
+
+from builders.builder import Builder
+from builders.linux import LinuxBuilder
+from builders.qpg import QpgBuilder
+from builders.esp32 import Esp32Builder, Esp32Board, Esp32App
+from builders.efr32 import Efr32Builder, Efr32App, Efr32Board
+
+from .targets import Application, Board, Platform
+
+
+class MatchApplication:
+
+  def __init__(self, app, board=None):
+    self.app = app
+    self.board = board
+
+  def Match(self, board: Board, app: Application):
+    if app != self.app:
+      return False
+    return self.board is None or board == self.board
+
+
+class Matcher():
+  """Figures out if a proper builder can be created for a platform/board/app combination."""
+
+  def __init__(self, builder_class):
+    self.builder_class = builder_class
+    self.app_arguments = {}
+    self.board_arguments = {}
+
+  def AcceptApplication(self, __app_key: Application, **kargs):
+    self.app_arguments[MatchApplication(__app_key)] = kargs
+
+  def AcceptApplicationForBoard(self, __app_key: Application, __board: Board,
+                                **kargs):
+    self.app_arguments[MatchApplication(__app_key, __board)] = kargs
+
+  def AcceptBoard(self, __board_key: Board, **kargs):
+    self.board_arguments[__board_key] = kargs
+
+  def Create(self, __board_key: Board, __app_key: Application, repo_path: str,
+             **kargs):
+    """Creates a new builder for the given board/app. """
+    if not __board_key in self.board_arguments:
+      return None
+
+    extra_app_args = None
+    for key, value in self.app_arguments.items():
+      if key.Match(__board_key, __app_key):
+        extra_app_args = value
+        break
+
+    if extra_app_args is None:
+      return None
+
+    kargs.update(self.board_arguments[__board_key])
+    kargs.update(extra_app_args)
+
+    return self.builder_class(repo_path, **kargs)
+
+
+# Builds a list of acceptable application/board combination for every platform
+_MATCHERS = {
+    Platform.LINUX: Matcher(LinuxBuilder),
+    Platform.ESP32: Matcher(Esp32Builder),
+    Platform.QPG: Matcher(QpgBuilder),
+    Platform.EFR32: Matcher(Efr32Builder),
+}
+
+# Matrix of what can be compiled and what build options are required
+# by such compilation
+_MATCHERS[Platform.LINUX].AcceptApplication(Application.ALL_CLUSTERS)
+_MATCHERS[Platform.LINUX].AcceptBoard(Board.NATIVE)
+
+_MATCHERS[Platform.ESP32].AcceptBoard(Board.DEVKITC, board=Esp32Board.DevKitC)
+_MATCHERS[Platform.ESP32].AcceptBoard(Board.M5STACK, board=Esp32Board.M5Stack)
+_MATCHERS[Platform.ESP32].AcceptApplication(
+    Application.ALL_CLUSTERS, app=Esp32App.ALL_CLUSTERS)
+_MATCHERS[Platform.ESP32].AcceptApplicationForBoard(
+    Application.LOCK, Board.DEVKITC, app=Esp32App.LOCK)
+
+_MATCHERS[Platform.QPG].AcceptApplication(Application.LOCK)
+_MATCHERS[Platform.QPG].AcceptBoard(Board.QPG6100)
+
+_MATCHERS[Platform.EFR32].AcceptBoard(Board.BRD4161A, board=Efr32Board.BRD4161A)
+_MATCHERS[Platform.EFR32].AcceptApplication(
+    Application.LIGHT, app=Efr32App.LIGHT)
+_MATCHERS[Platform.EFR32].AcceptApplication(Application.LOCK, app=Efr32App.LOCK)
+_MATCHERS[Platform.EFR32].AcceptApplication(
+    Application.WINDOW_COVERING, app=Efr32App.WINDOW_COVERING)
+
+
+class BuilderFactory:
+  """Creates application builders."""
+
+  def __init__(self, repository_path: str, output_prefix: str):
+    self.repository_path = repository_path
+    self.output_prefix = output_prefix
+
+  def Create(self, platform: Platform, board: Board, app: Application):
+    """Creates a builder object for the specified arguments. """
+
+    identifier = '%s-%s-%s' % (platform.name.lower(), board.name.lower(),
+                               app.name.lower())
+
+    output_directory = os.path.join(self.output_prefix, identifier)
+    builder = _MATCHERS[platform].Create(
+        board, app, self.repository_path, output_dir=output_directory)
+
+    if builder:
+      builder.identifier = identifier
+
+    return builder
+
+
+class TargetRelations:
+  """Figures out valid combinations of boards/platforms/applications."""
+
+  @staticmethod
+  def BoardsForPlatform(platform: Platform) -> Set[Board]:
+    global _MATCHERS
+    return set(_MATCHERS[platform].board_arguments.keys())
+
+  @staticmethod
+  def PlatformsForBoard(board: Board) -> Set[Platform]:
+    """Return the platforms that are using the specified board."""
+    global _MATCHERS
+    platforms = set()
+    for platform, matcher in _MATCHERS.items():
+      if board in matcher.board_arguments:
+        platforms.add(platform)
+    return platforms
+
+  @staticmethod
+  def ApplicationsForPlatform(platform: Platform) -> Set[Application]:
+    """What applications are buildable for a specific platform."""
+    global _MATCHERS
+    return set(
+        [matcher.app for matcher in _MATCHERS[platform].app_arguments.keys()])
+
+  @staticmethod
+  def PlatformsForApplication(application: Application) -> Set[Platform]:
+    """For what platforms can the given application be compiled."""
+    global _MATCHERS
+    platforms = set()
+    for platform, matcher in _MATCHERS.items():
+      for app_matcher in matcher.app_arguments:
+        if application == app_matcher.app:
+          platforms.add(platform)
+          break
+    return platforms