From 3e58a0eb64670291a1ed337f32984d96d56466fb Mon Sep 17 00:00:00 2001 From: cpagravel Date: Tue, 10 May 2022 06:11:33 -0700 Subject: [PATCH] Chef Refactor (#18009) * Chef - Add stateful_shell.py and constants.py Change-Id: Ia59a3d2a71204e4af2c41d6192d47f048a58c3b4 * Chef - Print splash text Change-Id: I9b162cabab700e87f12806668b2953bf57348449 * Chef - Use textwrap.dedent on textblocks Change-Id: I861f24ac20d440cf4a3e605809421a766ecdd476 * Chef - Remove use of commandQueue Change-Id: I75d5ae6fea25c6b933cb9823a29e9c2224b8188d * Chef - Replace shell writes with Python open() calls Change-Id: I0147fa6b96d3a15bf95352acbd136e8f1e648b8f * Chef - Make paths into global constants Change-Id: Ib177d584bcd463aaafe10f4f338a90d6b234c3f1 * Chef - Remove HexInputToInt Change-Id: Iebe5e76c169951e842262530d3feba5ac899a33c * Chef - Remove unused statements Change-Id: I56670e31722348fb47a1e59d2651cdcf3f983e28 * Chef - Convert variables to snake case per PEP8 Change-Id: I52520eec1dfb8093c39ce4570aa99cdd0b0d854a * Chef - Add function type hinting Change-Id: Ib82f0bf4752819b65d2c76058429931c6986bd83 --- examples/chef/chef.py | 458 ++++++++++++++------------------ examples/chef/constants.py | 19 ++ examples/chef/stateful_shell.py | 94 +++++++ 3 files changed, 312 insertions(+), 259 deletions(-) create mode 100644 examples/chef/constants.py create mode 100644 examples/chef/stateful_shell.py diff --git a/examples/chef/chef.py b/examples/chef/chef.py index 05cf43fedf08a3..899c3005201c0d 100755 --- a/examples/chef/chef.py +++ b/examples/chef/chef.py @@ -17,45 +17,47 @@ import sys import optparse import os -import subprocess -from pathlib import Path -from sys import platform +import sys +import textwrap +from typing import Sequence import yaml -global commandQueue -commandQueue = "" +import constants +import stateful_shell + +TermColors = constants.TermColors -class TermColors: - STRBOLD = '\033[1m' - STRRESET = '\033[0m' - STRCYAN = '\033[1;36m' - STRYELLOW = '\033[1;33m' +shell = stateful_shell.StatefulShell() +_CHEF_SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) +_REPO_BASE_PATH = os.path.join(_CHEF_SCRIPT_PATH, "../../") +_DEVICE_FOLDER = os.path.join(_CHEF_SCRIPT_PATH, "devices") +_DEVICE_LIST = [file[:-4] for file in os.listdir(_DEVICE_FOLDER) if file.endswith(".zap")] -def splash(): - splashText = TermColors.STRBOLD + TermColors.STRYELLOW + '''\ - ______ __ __ _______ _______ - / || | | | | ____|| ____| -| ,----'| |__| | | |__ | |__ -| | | __ | | __| | __| -| `----.| | | | | |____ | | - \\______||__| |__| |_______||__|\ -''' + TermColors.STRRESET - return splash +gen_dir = "" # Filled in after sample app type is read from args. -def printc(strInput): - color = TermColors.STRCYAN - print(color + strInput + TermColors.STRRESET) +def splash() -> None: + splashText = textwrap.dedent( + f"""\ + {TermColors.STRBOLD}{TermColors.STRYELLOW} + ______ __ __ _______ _______ + / || | | | | ____|| ____| + | ,----'| |__| | | |__ | |__ + | | | __ | | __| | __| + | `----.| | | | | |____ | | + \\______||__| |__| |_______||__|{TermColors.STRRESET} + """) + print(splashText) -def loadConfig(paths): +def load_config() -> None: config = dict() config["nrfconnect"] = dict() config["esp32"] = dict() - configFile = paths["scriptFolder"] + "/config.yaml" + configFile = f"{_CHEF_SCRIPT_PATH}/config.yaml" if (os.path.exists(configFile)): configStream = open(configFile, 'r') config = yaml.load(configStream, Loader=yaml.SafeLoader) @@ -76,79 +78,22 @@ def loadConfig(paths): return config -def definePaths(): - paths = dict() - paths["scriptFolder"] = os.path.abspath(os.path.dirname(__file__)) - paths["matterFolder"] = paths["scriptFolder"] + "/../../" - paths["rootSampleFolder"] = paths["scriptFolder"] - paths["devices"] = [] - - for filepath in Path(f"{paths['rootSampleFolder']}/devices").rglob('*.zap'): - paths["devices"].append( - str(os.path.splitext(os.path.basename(filepath))[0])) - return paths - - -def checkPythonVersion(): +def check_python_version() -> None: if sys.version_info[0] < 3: print('Must use Python 3. Current version is ' + str(sys.version_info[0])) exit(1) -def queuePrint(str): - global commandQueue - if (len(commandQueue) > 0): - commandQueue = commandQueue + "; " - commandQueue = commandQueue + "echo -e " + str - - -def queueCommand(command): - global commandQueue - if (len(commandQueue) > 0): - commandQueue = commandQueue + "; " - commandQueue = commandQueue + command - - -def execQueue(): - global commandQueue - subprocess.run(commandQueue, shell=True, executable=shellApp) - commandQueue = "" - - -def hexInputToInt(valIn): - '''Parses inputs as hexadecimal numbers, takes into account optional 0x - prefix - ''' - if (type(valIn) is str): - if (valIn[0:2].lower() == "0x"): - valOut = valIn[2:] - else: - valOut = valIn - valOut = int(valOut, 16) - else: - valOut = valIn - return valOut - - -def main(argv): - checkPythonVersion() - paths = definePaths() - config = loadConfig(paths) - - global myEnv - myEnv = os.environ.copy() +def main(argv: Sequence[str]) -> None: + check_python_version() + config = load_config() # # Build environment switches # - global shellApp - if platform == "linux" or platform == "linux2": - shellApp = '/bin/bash' - elif platform == "darwin": - shellApp = '/bin/zsh' - elif platform == "win32": + if sys.platform == "win32": print('Windows is currently not supported. Use Linux or MacOS platforms') exit(1) @@ -156,60 +101,61 @@ def main(argv): # Arguments parser # - deviceTypes = "\n ".join(paths["devices"]) + deviceTypes = "\n ".join(_DEVICE_LIST) - usage = f'''usage: chef.py [options] + usage = textwrap.dedent(f"""\ + usage: chef.py [options] -Platforms: - nrfconnect - esp32 - linux + Platforms: + nrfconnect + esp32 + linux -Device Types: - {deviceTypes} + Device Types: + {deviceTypes} -Notes: -- Whenever you change a device type, make sure to also use options -zbe -- Be careful if you have more than one device connected. The script assumes you have only one device connected and might flash the wrong one\ -''' + Notes: + - Whenever you change a device type, make sure to also use options -zbe + - Be careful if you have more than one device connected. The script assumes you have only one device connected and might flash the wrong one\ + """) parser = optparse.OptionParser(usage=usage) parser.add_option("-u", "--update_toolchain", help="updates toolchain & installs zap", - action="store_true", dest="doUpdateToolchain") + action="store_true", dest="do_update_toolchain") parser.add_option("-b", "--build", help="builds", - action="store_true", dest="doBuild") + action="store_true", dest="do_build") parser.add_option("-c", "--clean", help="clean build. Only valid if also building", - action="store_true", dest="doClean") + action="store_true", dest="do_clean") parser.add_option("-f", "--flash", help="flashes device", - action="store_true", dest="doFlash") + action="store_true", dest="do_flash") parser.add_option("-e", "--erase", help="erases flash before flashing. Only valid if also flashing", - action="store_true", dest="doErase") + action="store_true", dest="do_erase") parser.add_option("-i", "--terminal", help="opens terminal to interact with with device", - action="store_true", dest="doInteract") + action="store_true", dest="do_interact") parser.add_option("-m", "--menuconfig", help="runs menuconfig on platforms that support it", - action="store_true", dest="doMenuconfig") + action="store_true", dest="do_menuconfig") parser.add_option("", "--bootstrap_zap", help="installs zap dependencies", - action="store_true", dest="doBootstrapZap") + action="store_true", dest="do_bootstrap_zap") parser.add_option("-z", "--zap", help="runs zap to generate data model & interaction model artifacts", - action="store_true", dest="doRunZap") + action="store_true", dest="do_run_zap") parser.add_option("-g", "--zapgui", help="runs zap GUI display to allow editing of data model", - action="store_true", dest="doRunGui") - parser.add_option("-d", "--device", dest="sampleDeviceTypeName", + action="store_true", dest="do_run_gui") + parser.add_option("-d", "--device", dest="sample_device_type_name", help="specifies device type. Default is lighting. See info above for supported device types", metavar="TARGET", default="lighting") parser.add_option("-t", "--target", type='choice', action='store', - dest="buildTarget", + dest="build_target", help="specifies target platform. Default is esp32. See info below for currently supported target platforms", choices=['nrfconnect', 'esp32', 'linux', ], metavar="TARGET", default="esp32") - parser.add_option("-r", "--rpc", help="enables Pigweed RPC interface. Enabling RPC disables the shell interface. Your sdkconfig configurations will be reverted to default. Default is PW RPC off. When enabling or disabling this flag, on the first build force a clean build with -c", action="store_true", dest="doRPC") - parser.add_option("-v", "--vid", dest="vid", + parser.add_option("-r", "--rpc", help="enables Pigweed RPC interface. Enabling RPC disables the shell interface. Your sdkconfig configurations will be reverted to default. Default is PW RPC off. When enabling or disabling this flag, on the first build force a clean build with -c", action="store_true", dest="do_rpc") + parser.add_option("-v", "--vid", dest="vid", type=int, help="specifies the Vendor ID. Default is 0xFFF1", metavar="VID", default=0xFFF1) - parser.add_option("-p", "--pid", dest="pid", + parser.add_option("-p", "--pid", dest="pid", type=int, help="specifies the Product ID. Default is 0x8000", metavar="PID", default=0x8000) parser.add_option("", "--rpc_console", help="Opens PW RPC Console", - action="store_true", dest="doRPC_CONSOLE") + action="store_true", dest="do_rpc_console") parser.add_option("-y", "--tty", help="Enumerated USB tty/serial interface enumerated for your physical device. E.g.: /dev/ACM0", dest="tty", metavar="TTY", default=None) @@ -221,161 +167,157 @@ def main(argv): # Platform Folder # - queuePrint(f"Target is set to {options.sampleDeviceTypeName}") - paths["genFolder"] = paths["rootSampleFolder"] + f"/out/{options.sampleDeviceTypeName}/zap-generated/" + print(f"Target is set to {options.sample_device_type_name}") + global gen_dir + gen_dir = ( + f"{_CHEF_SCRIPT_PATH}/out/{options.sample_device_type_name}/zap-generated/") - queuePrint("Setting up environment...") - if options.buildTarget == "esp32": + print("Setting up environment...") + if options.build_target == "esp32": if config['esp32']['IDF_PATH'] is None: print('Path for esp32 SDK was not found. Make sure esp32.IDF_PATH is set on your config.yaml file') exit(1) - paths["platFolder"] = os.path.normpath( - paths["rootSampleFolder"] + "/esp32") - queueCommand(f'source {config["esp32"]["IDF_PATH"]}/export.sh') - elif options.buildTarget == "nrfconnect": + plat_folder = os.path.normpath(f"{_CHEF_SCRIPT_PATH}/esp32") + shell.run_cmd(f'source {config["esp32"]["IDF_PATH"]}/export.sh') + elif options.build_target == "nrfconnect": if config['nrfconnect']['ZEPHYR_BASE'] is None: print('Path for nrfconnect SDK was not found. Make sure nrfconnect.ZEPHYR_BASE is set on your config.yaml file') exit(1) - paths["platFolder"] = os.path.normpath( - paths["rootSampleFolder"] + "/nrfconnect") - queueCommand(f'source {config["nrfconnect"]["ZEPHYR_BASE"]}/zephyr-env.sh') - queueCommand("export ZEPHYR_TOOLCHAIN_VARIANT=gnuarmemb") - elif options.buildTarget == "linux": + plat_folder = os.path.normpath(f"{_CHEF_SCRIPT_PATH}/nrfconnect") + shell.run_cmd(f'source {config["nrfconnect"]["ZEPHYR_BASE"]}/zephyr-env.sh') + shell.run_cmd("export ZEPHYR_TOOLCHAIN_VARIANT=gnuarmemb") + elif options.build_target == "linux": pass else: - print(f"Target {options.buildTarget} not supported") + print(f"Target {options.build_target} not supported") - queueCommand(f"source {paths['matterFolder']}/scripts/activate.sh") + shell.run_cmd(f"source {_REPO_BASE_PATH}/scripts/activate.sh") # # Toolchain update # - if options.doUpdateToolchain: - if options.buildTarget == "esp32": - queuePrint("ESP32 toolchain update not supported. Skipping") - elif options.buildTarget == "nrfconnect": - queuePrint("Updating toolchain") - queueCommand( - f"cd {paths['matterFolder']} && python3 scripts/setup/nrfconnect/update_ncs.py --update") - elif options.buildTarget == "linux": - queuePrint("Linux toolchain update not supported. Skipping") + if options.do_update_toolchain: + if options.build_target == "esp32": + print("ESP32 toolchain update not supported. Skipping") + elif options.build_target == "nrfconnect": + print("Updating toolchain") + shell.run_cmd( + f"cd {_REPO_BASE_PATH} && python3 scripts/setup/nrfconnect/update_ncs.py --update") + elif options.build_target == "linux": + print("Linux toolchain update not supported. Skipping") # # ZAP bootstrapping # - if options.doBootstrapZap: - if platform == "linux" or platform == "linux2": - queuePrint("Installing ZAP OS package dependencies") - queueCommand(f"sudo apt-get install sudo apt-get install node node-yargs npm libpixman-1-dev libcairo2-dev libpango1.0-dev node-pre-gyp libjpeg9-dev libgif-dev node-typescript") - if platform == "darwin": - queuePrint("Installation of ZAP OS packages not supported on MacOS") - if platform == "win32": - queuePrint( + if options.do_bootstrap_zap: + if sys.platform == "linux" or sys.platform == "linux2": + print("Installing ZAP OS package dependencies") + shell.run_cmd( + f"sudo apt-get install sudo apt-get install node node-yargs npm libpixman-1-dev libcairo2-dev libpango1.0-dev node-pre-gyp libjpeg9-dev libgif-dev node-typescript") + if sys.platform == "darwin": + print("Installation of ZAP OS packages not supported on MacOS") + if sys.platform == "win32": + print( "Installation of ZAP OS packages not supported on Windows") - queuePrint("Running NPM to install ZAP Node.JS dependencies") - queueCommand( - f"cd {paths['matterFolder']}/third_party/zap/repo/ && npm install") + print("Running NPM to install ZAP Node.JS dependencies") + shell.run_cmd( + f"cd {_REPO_BASE_PATH}/third_party/zap/repo/ && npm install") # # Cluster customization # - if options.doRunGui: - queuePrint("Starting ZAP GUI editor") - queueCommand(f"cd {paths['rootSampleFolder']}/devices") - queueCommand( - f"{paths['matterFolder']}/scripts/tools/zap/run_zaptool.sh {options.sampleDeviceTypeName}.zap") - - if options.doRunZap: - queuePrint("Running ZAP script to generate artifacts") - queueCommand(f"mkdir -p {paths['genFolder']}/") - queueCommand(f"rm {paths['genFolder']}/*") - queueCommand( - f"{paths['matterFolder']}/scripts/tools/zap/generate.py {paths['rootSampleFolder']}/devices/{options.sampleDeviceTypeName}.zap -o {paths['genFolder']}") + if options.do_run_gui: + print("Starting ZAP GUI editor") + shell.run_cmd(f"cd {_CHEF_SCRIPT_PATH}/devices") + shell.run_cmd( + f"{_REPO_BASE_PATH}/scripts/tools/zap/run_zaptool.sh {options.sample_device_type_name}.zap") + + if options.do_run_zap: + print("Running ZAP script to generate artifacts") + shell.run_cmd(f"mkdir -p {gen_dir}/") + shell.run_cmd(f"rm {gen_dir}/*") + shell.run_cmd( + f"{_REPO_BASE_PATH}/scripts/tools/zap/generate.py {_CHEF_SCRIPT_PATH}/devices/{options.sample_device_type_name}.zap -o {gen_dir}") # af-gen-event.h is not generated - queueCommand(f"touch {paths['genFolder']}/af-gen-event.h") + shell.run_cmd(f"touch {gen_dir}/af-gen-event.h") # # Menuconfig # - if options.doMenuconfig: - if options.buildTarget == "esp32": - queueCommand(f"cd {paths['rootSampleFolder']}/esp32") - queueCommand("idf.py menuconfig") - elif options.buildTarget == "nrfconnect": - queueCommand(f"cd {paths['rootSampleFolder']}/nrfconnect") - queueCommand("west build -t menuconfig") - elif options.buildTarget == "linux": - queuePrint("Menuconfig not available on Linux target. Skipping") + if options.do_menuconfig: + if options.build_target == "esp32": + shell.run_cmd(f"cd {_CHEF_SCRIPT_PATH}/esp32") + shell.run_cmd("idf.py menuconfig") + elif options.build_target == "nrfconnect": + shell.run_cmd(f"cd {_CHEF_SCRIPT_PATH}/nrfconnect") + shell.run_cmd("west build -t menuconfig") + elif options.build_target == "linux": + print("Menuconfig not available on Linux target. Skipping") # # Build # - if options.doBuild: - queuePrint("Building...") - if options.doRPC: - queuePrint("RPC PW enabled") - queueCommand( - f"export SDKCONFIG_DEFAULTS={paths['rootSampleFolder']}/esp32/sdkconfig_rpc.defaults") + if options.do_build: + print("Building...") + if options.do_rpc: + print("RPC PW enabled") + shell.run_cmd( + f"export SDKCONFIG_DEFAULTS={_CHEF_SCRIPT_PATH}/esp32/sdkconfig_rpc.defaults") else: - queuePrint("RPC PW disabled") - queueCommand( - f"export SDKCONFIG_DEFAULTS={paths['rootSampleFolder']}/esp32/sdkconfig.defaults") - options.vid = hexInputToInt(options.vid) - options.pid = hexInputToInt(options.pid) - queuePrint( + print("RPC PW disabled") + shell.run_cmd( + f"export SDKCONFIG_DEFAULTS={_CHEF_SCRIPT_PATH}/esp32/sdkconfig.defaults") + print( f"Product ID 0x{options.pid:02X} / Vendor ID 0x{options.vid:02X}") - queueCommand(f"cd {paths['rootSampleFolder']}") - - if (options.buildTarget == "esp32") or (options.buildTarget == "nrfconnect"): - queueCommand(f''' -cat > project_include.cmake < args.gni < sample.gni < None: + if sys.platform == "linux" or sys.platform == "linux2": + self.shell_app = '/bin/bash' + elif sys.platform == "darwin": + self.shell_app = '/bin/zsh' + elif sys.platform == "win32": + print('Windows is currently not supported. Use Linux or MacOS platforms') + exit(1) + + self.env: Dict[str, str] = os.environ.copy() + self.cwd: str = self.env["PWD"] + + # This file holds the env after running a command. This is a better approach + # than writing to stdout because commands could redirect the stdout. + self.envfile_path: str = os.path.join(tempfile.gettempdir(), "envfile") + + def print_env(self) -> None: + """Print environment variables in commandline friendly format for export. + + The purpose of this function is to output the env variables in such a way + that a user can copy the env variables and paste them in their terminal to + quickly recreate the environment state. + """ + for env_var in self.env: + quoted_value = shlex.quote(self.env[env_var]) + if env_var: + print(f"export {env_var}={quoted_value}") + + def run_cmd(self, cmd: str, *, raise_on_returncode=False) -> None: + """Runs a command and updates environment. + + Args: + cmd: Command to execute. + raise_on_returncode: Whether to raise an error if the return code is nonzero. + + Raises: + RuntimeError: If raise_on_returncode is set and nonzero return code is given. + """ + env_dict = {} + + # Set OLDPWD at beginning because opening the shell clears this. This handles 'cd -'. + # env -0 prints the env variables separated by null characters for easy parsing. + command_with_state = f"OLDPWD={self.env.get('OLDPWD', '')}; {cmd}; env -0 > {self.envfile_path}" + with subprocess.Popen( + [command_with_state], + env=self.env, cwd=self.cwd, + shell=True, executable=self.shell_app + ) as proc: + returncode = proc.wait() + + # Load env state from envfile. + with open(self.envfile_path) as f: + # Split on null char because we use env -0. + env_entries = f.read().split("\0") + for entry in env_entries: + parts = entry.split("=") + # Handle case where an env variable contains text with '='. + env_dict[parts[0]] = "=".join(parts[1:]) + self.env = env_dict + self.cwd = self.env["PWD"] + + if raise_on_returncode and returncode != 0: + raise RuntimeError( + f"Error. Return code is not 0. It is: {returncode}")