Add --enable-flashbundle flag for generating flashbundle files. (proj…

…ect-chip#8832)

This commit contains flashbundle generation for nrf platform, the
flashbundle for other platforms will be added later.

integrations/cloudbuild/build-all.yaml

@@ -5,8 +5,8 @@ steps:
       entrypoint: "./scripts/run_in_build_env.sh"
       args:
           [
-              "./scripts/build/build_examples.py --platform all build
-              --create-archives /workspace/artifacts/",
+              "./scripts/build/build_examples.py --platform all
+              --enable-flashbundle build --create-archives /workspace/artifacts/",
           ]
       timeout: 7200s
 

scripts/build/build/__init__.py

@@ -39,7 +39,8 @@ def __init__(self, runner, repository_path:str, output_prefix:str):
 
   def SetupBuilders(self, platforms: Sequence[Platform],
                     boards: Sequence[Board],
-                    applications: Sequence[Application]):
+                    applications: Sequence[Application],
+                    enable_flashbundle: bool):
     """Configures internal builders for the given platform/board/app combination.
 
         Handles smart default selection, so that users only need to specify
@@ -86,7 +87,7 @@ def SetupBuilders(self, platforms: Sequence[Platform],
     for platform in sorted(platforms):
       for board in sorted(boards):
         for application in sorted(applications):
-          builder = self.builder_factory.Create(platform, board, application)
+          builder = self.builder_factory.Create(platform, board, application, enable_flashbundle=enable_flashbundle)
           if not builder:
             logging.debug('Builder not supported for tuple %s/%s/%s', platform,
                           board, application)

scripts/build/build/factory.py

@@ -129,7 +129,7 @@ def __init__(self, runner, repository_path: str, output_prefix: str):
     self.repository_path = repository_path
     self.output_prefix = output_prefix
 
-  def Create(self, platform: Platform, board: Board, app: Application):
+  def Create(self, platform: Platform, board: Board, app: Application, enable_flashbundle: bool = False):
     """Creates a builder object for the specified arguments. """
 
     builder = _MATCHERS[platform].Create(
@@ -141,6 +141,7 @@ def Create(self, platform: Platform, board: Board, app: Application):
 
     if builder:
       builder.SetIdentifier(platform.name.lower(), board.name.lower(), app.name.lower())
+      builder.enable_flashbundle(enable_flashbundle)
 
     return builder
 

scripts/build/build_examples.py

@@ -77,6 +77,12 @@ def ValidateRepoPath(context, parameter, value):
     multiple=True,
     help='What example application to build. Empty will find suitable applications.'
 )
+@click.option(
+    '--enable-flashbundle',
+    default=False,
+    is_flag=True,
+    help='Also generate the flashbundles for the app.'
+)
 @click.option(
     '--repo',
     default='.',
@@ -104,7 +110,7 @@ def ValidateRepoPath(context, parameter, value):
     help='Where to write the dry run output')
 @click.pass_context
 def main(context, log_level, platform, board, app, repo, out_prefix, clean,
-         dry_run, dry_run_output):
+         dry_run, dry_run_output, enable_flashbundle):
   # Ensures somewhat pretty logging of what is going on
   coloredlogs.install(
       level=__LOG_LEVELS__[log_level],
@@ -132,7 +138,8 @@ def main(context, log_level, platform, board, app, repo, out_prefix, clean,
   context.obj.SetupBuilders(
       platforms=[build.Platform.FromArgName(name) for name in platform],
       boards=[build.Board.FromArgName(name) for name in board],
-      applications=[build.Application.FromArgName(name) for name in app])
+      applications=[build.Application.FromArgName(name) for name in app],
+      enable_flashbundle=enable_flashbundle)
 
   if clean:
     context.obj.CleanOutputDirectories()

scripts/build/builders/builder.py

@@ -27,34 +27,70 @@ class Builder(ABC):
 
   """
 
-  def __init__(self, root, runner, output_prefix: str ='out'):
+  def __init__(self, root, runner, output_prefix: str = 'out'):
     self.root = os.path.abspath(root)
     self._runner = runner
     self.output_prefix = output_prefix
+    self._enable_flashbundle = False
 
     # Set post-init once actual build target is known
     self.identifier = None
     self.output_dir = None
 
+  def enable_flashbundle(self, enable_flashbundle: bool):
+    self._enable_flashbundle = enable_flashbundle
+
   @abstractmethod
   def generate(self):
     """Generate the build files - generally the ninja/makefiles"""
     raise NotImplementedError()
 
   @abstractmethod
-  def build(self):
+  def _build(self):
     """Perform an actual build"""
     raise NotImplementedError()
 
+  def _generate_flashbundle(self):
+    """Perform an actual generating of flashbundle
+
+       May do nothing (and builder can choose not to implement this) if the
+       app does not need special steps for generating flashbundle. (e.g. the
+       example apps on Linux platform can run the ELF files directly.)
+    """
+    pass
+
   @abstractmethod
-  def outputs(self):
+  def build_outputs(self):
     """Return a list of relevant output files after a build.
 
        May use build output data (e.g. manifests), so this should be invoked
        only after a build has succeeded.
     """
     raise NotImplementedError()
 
+  def flashbundle(self):
+    """Return the files in flashbundle.
+
+       Return an empty dict (and builder can choose not to implement this) if the
+       app does not need special files as flashbundle. (e.g. the example apps on
+       Linux platform can run the ELF files directly.)
+
+       May use data from do_generate_flashbundle, so this should be invoked only
+       after do_generate_flashbundle has succeeded.
+    """
+    return {}
+
+  def outputs(self):
+    artifacts = self.build_outputs()
+    if self._enable_flashbundle:
+      artifacts.update(self.flashbundle())
+    return artifacts
+
+  def build(self):
+    self._build()
+    if self._enable_flashbundle:
+      self._generate_flashbundle()
+
   def _Execute(self, cmdarray, cwd=None, title=None):
     self._runner.Run(cmdarray, cwd=cwd, title=title)
 

scripts/build/builders/efr32.py

@@ -79,7 +79,7 @@ def __init__(self,
     self.app = app
     self.gn_build_args = ['efr32_board="%s"' % board.GnArgName()]
 
-  def outputs(self):
+  def build_outputs(self):
     items = {
         '%s.out' % self.app.AppNamePrefix():
             os.path.join(self.output_dir, '%s.out' % self.app.AppNamePrefix()),

scripts/build/builders/esp32.py

@@ -110,13 +110,13 @@ def generate(self):
     self._IdfEnvExecute(
         cmd, cwd=self.root, title='Generating ' + self.identifier)
 
-  def build(self):
+  def _build(self):
     logging.info('Compiling Esp32 at %s', self.output_dir)
 
     self._IdfEnvExecute(
         "ninja -C '%s'" % self.output_dir, title='Building ' + self.identifier)
 
-  def outputs(self):
+  def build_outputs(self):
     return {
         self.app.AppNamePrefix + '.elf':
             os.path.join(self.output_dir, self.app.AppNamePrefix + '.elf'),

scripts/build/builders/gn.py

@@ -46,6 +46,6 @@ def generate(self):
 
       self._Execute(cmd, title='Generating ' + self.identifier)
 
-  def build(self):
+  def _build(self):
     self._Execute(['ninja', '-C', self.output_dir],
                   title='Building ' + self.identifier)

scripts/build/builders/host.py

@@ -54,11 +54,11 @@ def __init__(self, root, runner, output_prefix: str, app: HostApp):
     self.app_name = app.BinaryName()
     self.map_name = self.app_name + '.map'
 
-  def outputs(self):
+  def build_outputs(self):
     return {
         self.app_name: os.path.join(self.output_dir, self.app_name),
         self.map_name : os.path.join(self.output_dir, self.map_name)
     }
 
   def SetIdentifier(self, platform: str, board: str, app: str):
-    super(HostBuilder, self).SetIdentifier(ConcretePlatformName(), board, app)
+    super(HostBuilder, self).SetIdentifier(ConcretePlatformName(), board, app)

scripts/build/builders/nrf.py

@@ -46,6 +46,19 @@ def AppNamePrefix(self):
     else:
       raise Exception('Unknown app type: %r' % self)
 
+  def _FlashBundlePrefix(self):
+    if self == NrfApp.LIGHT:
+      return 'chip-nrfconnect-lighting-example'
+    elif self == NrfApp.LOCK:
+      return 'chip-nrfconnect-lock-example'
+    elif self == NrfApp.SHELL:
+      return 'chip-nrfconnect-shell-example'
+    else:
+      raise Exception('Unknown app type: %r' % self)
+
+  def FlashBundleName(self):
+    '''Nrf build script will generate a file naming <project_name>.flashbundle.txt, go through the output dir to find the file and return it.'''
+    return self._FlashBundlePrefix() + '.flashbundle.txt'
 
 class NrfBoard(Enum):
   NRF52840 = auto()
@@ -109,13 +122,24 @@ def generate(self):
         self._Execute(['bash', '-c', cmd], title='Generating ' + self.identifier)
 
 
-  def build(self):
+  def _build(self):
     logging.info('Compiling NrfConnect at %s', self.output_dir)
 
     self._Execute(['ninja', '-C', self.output_dir], title='Building ' + self.identifier)
 
-  def outputs(self):
+  def _generate_flashbundle(self):
+    logging.info(f'Generating flashbundle at {self.output_dir}')
+
+    self._Execute(['ninja', '-C', self.output_dir, 'flashing_script'], title='Generating flashable files of ' + self.identifier)
+
+  def build_outputs(self):
     return {
         '%s.elf' % self.app.AppNamePrefix(): os.path.join(self.output_dir, 'zephyr', 'zephyr.elf'),
         '%s.map' % self.app.AppNamePrefix(): os.path.join(self.output_dir, 'zephyr', 'zephyr.map'),
     }
+
+  def flashbundle(self):
+    with open(os.path.join(self.output_dir, self.app.FlashBundleName()), 'r') as fp:
+      return {
+        l.strip(): os.path.join(self.output_dir, l.strip()) for l in fp.readlines() if l.strip()
+      }

scripts/build/builders/qpg.py

@@ -26,7 +26,7 @@ def __init__(self, root, runner, output_prefix):
         runner=runner,
         output_prefix=output_prefix)
 
-  def outputs(self):
+  def build_outputs(self):
     return {
         'chip-qpg-lock-example.out':
             os.path.join(self.output_dir, 'chip-qpg6100-lock-example.out'),