-
Notifications
You must be signed in to change notification settings - Fork 2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Unified build script for chip - out of source build for all applicati…
…ons/targets/boards (#8221) * Imported a general chip builder script, currently covering a few platforms/boards/apps as a start to have a single build entrypoint * Move build requirements into global script/requirements.txt so that they get picked up by the bootstrap script * Update script to assume and require bootstrapping * Code review comments * Support building the lock app for ESP32 It turns out that ESP32 lock app is ONLY for devkitc, so I added application matching logic to include board restrictions. Tested: lock app compiles and only for devkitc if esp32 platform is used. * Remove obsolete todo * Fix the duplicated accept for efr32 lock app * Address some code review comments * Rename chipbuild to build_examples * Fix names * Restyle
- Loading branch information
Showing
13 changed files
with
929 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
# CHIP automated builds scripts | ||
|
||
CHIP compilation is generally split into two steps | ||
|
||
1. Generate ninja/makefile for out-of-source builds | ||
2. Compilation using ninja/makefiles | ||
|
||
## Building manually | ||
|
||
Manual building is generally platform-dependent. All build steps would require a | ||
bootstrapped environment (loads a pigweed build environment) and will then be | ||
followed by platform-specific instructions. | ||
|
||
The file BUILDING.md describes general requirements and examples. Typical usage | ||
is: | ||
|
||
``` | ||
source scripts/activate | ||
gn gen out/host | ||
ninja -C out/host | ||
``` | ||
|
||
## Unified build script | ||
|
||
The script `build_examples.py` provides a single entry point for generating and | ||
executing the build. | ||
|
||
Build environment _MUST_ be properly configured for build_examples to succeed. | ||
For example ESP32 builds requite IDF_PATH to be set. Building in the | ||
corresponding build image or the chip vscode image satisfy the build environment | ||
requirement. | ||
|
||
Usage examples: | ||
|
||
1. Compiles the Lock app on all supported platforms | ||
|
||
``` | ||
./scripts/build/build_examples.py --app lock build | ||
``` | ||
|
||
2. Compile the all clusters app for a ESP32 DevKitC | ||
|
||
``` | ||
./scripts/build/build_examples.py --app all_clusters_app --board devkitc build | ||
``` | ||
|
||
3. Generate all the build rules (but do not compile) all native apps | ||
|
||
``` | ||
./scripts/build/build_examples.py --platform native generate | ||
``` | ||
|
||
4. Generate all the makefiles (but do not compile) using a specific output root | ||
|
||
``` | ||
./scripts/build/build_examples.py --platform native generate --out-prefix ./mydir | ||
``` | ||
|
||
5. Compile the qpg lock app and copy the output in a 'artifact' folder. Note the | ||
argument order (artifact copying is an argument for the build command) | ||
|
||
``` | ||
./scripts/build/build_examples.py --board qpg6100 --app lock build \ | ||
--copy-artifacts-to /tmp/artifacts | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,153 @@ | ||
import logging | ||
import os | ||
import shutil | ||
|
||
from enum import Enum, auto | ||
from typing import Sequence | ||
|
||
from .targets import Platform, Board, Application | ||
from .factory import BuilderFactory, TargetRelations | ||
|
||
|
||
def CommaSeparate(items) -> str: | ||
return ', '.join([x.ArgName for x in items]) | ||
|
||
|
||
# Supported platforms/boards/apps for generation/compilation | ||
# note that not all combinations are supported, however command-line will | ||
# accept all combinations | ||
PLATFORMS = [x.ArgName for x in Platform] | ||
BOARDS = [x.ArgName for x in Board] | ||
APPLICATIONS = [x.ArgName for x in Application] | ||
|
||
|
||
class BuildSteps(Enum): | ||
GENERATED = auto() | ||
|
||
|
||
class Context: | ||
"""Represents a grouped list of platform/board/app builders to use | ||
to generate make/ninja instructions and to compile. | ||
""" | ||
|
||
def __init__(self, repository_path:str, output_prefix:str): | ||
self.builders = [] | ||
self.repository_path = repository_path | ||
self.output_prefix = output_prefix | ||
self.completed_steps = set() | ||
|
||
def SetupBuilders(self, platforms: Sequence[Platform], | ||
boards: Sequence[Board], | ||
applications: Sequence[Application]): | ||
"""Configures internal builders for the given platform/board/app combination. | ||
Handles smart default selection, so that users only need to specify | ||
part of platform/board/application information and the method tries | ||
to automatically deduce the rest of the arguments. | ||
""" | ||
if not platforms and not boards: | ||
if applications: | ||
platforms = set().union(*[ | ||
TargetRelations.PlatformsForApplication(app) for app in applications | ||
]) | ||
else: | ||
# when nothing is specified, start with a default host build | ||
# TODO: this is only for linux. Should be moved to 'HOST' as a platform | ||
# to also support building on MacOS | ||
platforms = [Platform.LINUX] | ||
|
||
# at this point, at least one of 'platforms' or 'boards' is non-empty | ||
if not boards: | ||
boards = set().union(*[ | ||
TargetRelations.BoardsForPlatform(platform) for platform in platforms | ||
]) | ||
elif not platforms: | ||
platforms = set().union( | ||
*[TargetRelations.PlatformsForBoard(board) for board in boards]) | ||
|
||
if not applications: | ||
applications = set().union(*[ | ||
TargetRelations.ApplicationsForPlatform(platform) | ||
for platform in platforms | ||
]) | ||
|
||
platforms = set(platforms) | ||
boards = set(boards) | ||
applications = set(applications) | ||
|
||
logging.info('Platforms being built: %s', CommaSeparate(platforms)) | ||
logging.info('Boards being built: %s', CommaSeparate(boards)) | ||
logging.info('Applications being built: %s', CommaSeparate(applications)) | ||
|
||
# Sanity check: ensure all input arguments generate at least an output | ||
platforms_with_builders = set() | ||
boards_with_builders = set() | ||
applications_with_builders = set() | ||
|
||
factory = BuilderFactory(self.repository_path, self.output_prefix) | ||
|
||
for platform in platforms: | ||
for board in boards: | ||
for application in applications: | ||
builder = factory.Create(platform, board, application) | ||
if not builder: | ||
logging.debug('Builder not supported for tuple %s/%s/%s', platform, | ||
board, application) | ||
continue | ||
|
||
self.builders.append(builder) | ||
platforms_with_builders.add(platform) | ||
boards_with_builders.add(board) | ||
applications_with_builders.add(application) | ||
|
||
if platforms != platforms_with_builders: | ||
logging.warn('Platforms without build output: %s', | ||
CommaSeparate(platforms.difference(platforms_with_builders))) | ||
|
||
if boards != boards_with_builders: | ||
logging.warn('Boards without build output: %s', | ||
CommaSeparate(boards.difference(boards_with_builders))) | ||
|
||
if applications != applications_with_builders: | ||
logging.warn( | ||
'Applications without build output: %s', | ||
CommaSeparate(applications.difference(applications_with_builders))) | ||
|
||
# whenever builders change, assume generation is required again | ||
self.completed_steps.discard(BuildSteps.GENERATED) | ||
|
||
def Generate(self): | ||
"""Performs a build generation IFF code generation has not yet been performed.""" | ||
if BuildSteps.GENERATED in self.completed_steps: | ||
return | ||
|
||
for builder in self.builders: | ||
logging.info('Generating %s', builder.output_dir) | ||
builder.generate() | ||
|
||
self.completed_steps.add(BuildSteps.GENERATED) | ||
|
||
def Build(self): | ||
self.Generate() | ||
|
||
for builder in self.builders: | ||
logging.info('Building %s', builder.output_dir) | ||
builder.build() | ||
|
||
def CleanOutputDirectories(self): | ||
for builder in self.builders: | ||
logging.warn('Cleaning %s', builder.output_dir) | ||
shutil.rmtree(builder.output_dir) | ||
|
||
# any generated output was cleaned | ||
self.completed_steps.discard(BuildSteps.GENERATED) | ||
|
||
def CopyArtifactsTo(self, path: str): | ||
logging.info('Copying build artifacts to %s', path) | ||
if not os.path.exists(path): | ||
os.makedirs(path) | ||
|
||
for builder in self.builders: | ||
# FIXME: builder subdir... | ||
builder.CopyArtifacts(os.path.join(path, builder.identifier)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,154 @@ | ||
import os | ||
|
||
from typing import Set | ||
|
||
from builders.builder import Builder | ||
from builders.linux import LinuxBuilder | ||
from builders.qpg import QpgBuilder | ||
from builders.esp32 import Esp32Builder, Esp32Board, Esp32App | ||
from builders.efr32 import Efr32Builder, Efr32App, Efr32Board | ||
|
||
from .targets import Application, Board, Platform | ||
|
||
|
||
class MatchApplication: | ||
|
||
def __init__(self, app, board=None): | ||
self.app = app | ||
self.board = board | ||
|
||
def Match(self, board: Board, app: Application): | ||
if app != self.app: | ||
return False | ||
return self.board is None or board == self.board | ||
|
||
|
||
class Matcher(): | ||
"""Figures out if a proper builder can be created for a platform/board/app combination.""" | ||
|
||
def __init__(self, builder_class): | ||
self.builder_class = builder_class | ||
self.app_arguments = {} | ||
self.board_arguments = {} | ||
|
||
def AcceptApplication(self, __app_key: Application, **kargs): | ||
self.app_arguments[MatchApplication(__app_key)] = kargs | ||
|
||
def AcceptApplicationForBoard(self, __app_key: Application, __board: Board, | ||
**kargs): | ||
self.app_arguments[MatchApplication(__app_key, __board)] = kargs | ||
|
||
def AcceptBoard(self, __board_key: Board, **kargs): | ||
self.board_arguments[__board_key] = kargs | ||
|
||
def Create(self, __board_key: Board, __app_key: Application, repo_path: str, | ||
**kargs): | ||
"""Creates a new builder for the given board/app. """ | ||
if not __board_key in self.board_arguments: | ||
return None | ||
|
||
extra_app_args = None | ||
for key, value in self.app_arguments.items(): | ||
if key.Match(__board_key, __app_key): | ||
extra_app_args = value | ||
break | ||
|
||
if extra_app_args is None: | ||
return None | ||
|
||
kargs.update(self.board_arguments[__board_key]) | ||
kargs.update(extra_app_args) | ||
|
||
return self.builder_class(repo_path, **kargs) | ||
|
||
|
||
# Builds a list of acceptable application/board combination for every platform | ||
_MATCHERS = { | ||
Platform.LINUX: Matcher(LinuxBuilder), | ||
Platform.ESP32: Matcher(Esp32Builder), | ||
Platform.QPG: Matcher(QpgBuilder), | ||
Platform.EFR32: Matcher(Efr32Builder), | ||
} | ||
|
||
# Matrix of what can be compiled and what build options are required | ||
# by such compilation | ||
_MATCHERS[Platform.LINUX].AcceptApplication(Application.ALL_CLUSTERS) | ||
_MATCHERS[Platform.LINUX].AcceptBoard(Board.NATIVE) | ||
|
||
_MATCHERS[Platform.ESP32].AcceptBoard(Board.DEVKITC, board=Esp32Board.DevKitC) | ||
_MATCHERS[Platform.ESP32].AcceptBoard(Board.M5STACK, board=Esp32Board.M5Stack) | ||
_MATCHERS[Platform.ESP32].AcceptApplication( | ||
Application.ALL_CLUSTERS, app=Esp32App.ALL_CLUSTERS) | ||
_MATCHERS[Platform.ESP32].AcceptApplicationForBoard( | ||
Application.LOCK, Board.DEVKITC, app=Esp32App.LOCK) | ||
|
||
_MATCHERS[Platform.QPG].AcceptApplication(Application.LOCK) | ||
_MATCHERS[Platform.QPG].AcceptBoard(Board.QPG6100) | ||
|
||
_MATCHERS[Platform.EFR32].AcceptBoard(Board.BRD4161A, board=Efr32Board.BRD4161A) | ||
_MATCHERS[Platform.EFR32].AcceptApplication( | ||
Application.LIGHT, app=Efr32App.LIGHT) | ||
_MATCHERS[Platform.EFR32].AcceptApplication(Application.LOCK, app=Efr32App.LOCK) | ||
_MATCHERS[Platform.EFR32].AcceptApplication( | ||
Application.WINDOW_COVERING, app=Efr32App.WINDOW_COVERING) | ||
|
||
|
||
class BuilderFactory: | ||
"""Creates application builders.""" | ||
|
||
def __init__(self, repository_path: str, output_prefix: str): | ||
self.repository_path = repository_path | ||
self.output_prefix = output_prefix | ||
|
||
def Create(self, platform: Platform, board: Board, app: Application): | ||
"""Creates a builder object for the specified arguments. """ | ||
|
||
identifier = '%s-%s-%s' % (platform.name.lower(), board.name.lower(), | ||
app.name.lower()) | ||
|
||
output_directory = os.path.join(self.output_prefix, identifier) | ||
builder = _MATCHERS[platform].Create( | ||
board, app, self.repository_path, output_dir=output_directory) | ||
|
||
if builder: | ||
builder.identifier = identifier | ||
|
||
return builder | ||
|
||
|
||
class TargetRelations: | ||
"""Figures out valid combinations of boards/platforms/applications.""" | ||
|
||
@staticmethod | ||
def BoardsForPlatform(platform: Platform) -> Set[Board]: | ||
global _MATCHERS | ||
return set(_MATCHERS[platform].board_arguments.keys()) | ||
|
||
@staticmethod | ||
def PlatformsForBoard(board: Board) -> Set[Platform]: | ||
"""Return the platforms that are using the specified board.""" | ||
global _MATCHERS | ||
platforms = set() | ||
for platform, matcher in _MATCHERS.items(): | ||
if board in matcher.board_arguments: | ||
platforms.add(platform) | ||
return platforms | ||
|
||
@staticmethod | ||
def ApplicationsForPlatform(platform: Platform) -> Set[Application]: | ||
"""What applications are buildable for a specific platform.""" | ||
global _MATCHERS | ||
return set( | ||
[matcher.app for matcher in _MATCHERS[platform].app_arguments.keys()]) | ||
|
||
@staticmethod | ||
def PlatformsForApplication(application: Application) -> Set[Platform]: | ||
"""For what platforms can the given application be compiled.""" | ||
global _MATCHERS | ||
platforms = set() | ||
for platform, matcher in _MATCHERS.items(): | ||
for app_matcher in matcher.app_arguments: | ||
if application == app_matcher.app: | ||
platforms.add(platform) | ||
break | ||
return platforms |
Oops, something went wrong.