Refactor

examples/chef/chef.py

@@ -27,6 +27,7 @@
 
 import constants
 import stateful_shell
+import chef_util
 
 from typing import Sequence
 
@@ -172,6 +173,7 @@ def main(argv: Sequence[str]) -> None:
                       dest="generate_zzz", action="store_true")
     parser.add_option("", "--validate_zzz", help="Checks if cached ZAP output needs to be regenrated, for use in CI. If this flag is set, all other arguments are ignored.", dest="validate_zzz", action="store_true")
     parser.add_option("", "--use_zzz", help="Use pre generated output from the ZAP tool found in the zzz_generated folder. Used to decrease execution time of CI jobs", dest="use_zzz", action="store_true")
+    parser.add_option("", "--build_all", help="Builds all chef examples across all platforms and collates artifacts. Chef exits after completion.", dest="build_all", action="store_true")
 
     options, _ = parser.parse_args(argv)
 
@@ -180,32 +182,13 @@ def main(argv: Sequence[str]) -> None:
     chef_zzz_root = os.path.join(_CHEF_SCRIPT_PATH, "zzz_generated")
     ci_manifest_file_name = os.path.join(_CHEF_SCRIPT_PATH, "cimanifest.json")
     chef_devices_dir = os.path.join(_CHEF_SCRIPT_PATH, "devices")
-    ci_allowlist = ['lighting-app.zap']
 
     #
     # Validate zzz_generated
     #
 
     if options.validate_zzz:
-        ci_manifest = {}
-        print(f"Validating {chef_zzz_root}")
-        for device_dir_item in os.listdir(chef_devices_dir):
-            target_file_ext = ".zap"
-            if device_dir_item.endswith(target_file_ext) and device_dir_item in ci_allowlist:
-                device_name = device_dir_item[:-len(target_file_ext)]
-                device_file_path = os.path.join(chef_devices_dir, device_dir_item)
-                with open(device_file_path, "rb") as device_file:
-                    device_file_data = device_file.read()
-                    device_file_md5 = hashlib.md5(device_file_data).hexdigest()
-                    ci_manifest[device_name] = device_file_md5
-                    print(f"Manifest for {device_name} : {device_file_md5}")
-        # git ls-tree fails in CI
-        # git_cmd = ["git", "ls-tree", "master", "third_party/zap/repo"]
-        # zap_commit = str(subprocess.check_output(git_cmd, cwd=_REPO_BASE_PATH))
-        # zap_commit = zap_commit.split(" ")[2]
-        # zap_commit = zap_commit[:zap_commit.index("\\")]
-        # print(f"zap commit: {zap_commit}")
-        # ci_manifest["zap_commit"] = zap_commit
+        ci_manifest = chef_util.generate_device_manifest(chef_devices_dir)
         with open(ci_manifest_file_name, "r", encoding="utf-8") as ci_manifest_file:
             cached_manifest = json.loads(ci_manifest_file.read())
             for device in ci_manifest:
@@ -218,6 +201,7 @@ def main(argv: Sequence[str]) -> None:
                     if cached_manifest[device] != ci_manifest[device]:
                         print("ZAP updated in master, please pull master and run chef with the flag --generate_zzz and commit")
                         exit(1)
+        print("Cached ZAP output is up to date!")
         exit(0)
 
     #
@@ -246,7 +230,6 @@ def main(argv: Sequence[str]) -> None:
     #
 
     if options.generate_zzz:
-        ci_manifest = {}
         print(f"Cleaning {chef_zzz_root}")
         if not os.path.exists(chef_zzz_root):
             print(f"{chef_zzz_root} doesn't exist; creating")
@@ -258,15 +241,8 @@ def main(argv: Sequence[str]) -> None:
         print(f"Generating files in {chef_zzz_root} for all devices")
         for device_dir_item in os.listdir(chef_devices_dir):
             target_file_ext = ".zap"
-            if device_dir_item.endswith(target_file_ext) and device_dir_item in ci_allowlist:
-                # Filter to one example
+            if device_dir_item.endswith(target_file_ext) and device_dir_item in chef_util.ci_allowlist:
                 device_name = device_dir_item[:-len(target_file_ext)]
-                device_file_path = os.path.join(chef_devices_dir, device_dir_item)
-                with open(device_file_path, "rb") as device_file:
-                    device_file_data = device_file.read()
-                    device_file_md5 = hashlib.md5(device_file_data).hexdigest()
-                    ci_manifest[device_name] = device_file_md5
-                    print(f"Manifest for {device_name} : {device_file_md5}")
                 print(f"Generating files for {device_name}")
                 device_out_dir = os.path.join(chef_zzz_root, device_name)
                 os.mkdir(device_out_dir)
@@ -275,14 +251,13 @@ def main(argv: Sequence[str]) -> None:
                 shell.run_cmd(f"{_REPO_BASE_PATH}/scripts/tools/zap/generate.py\
  {_CHEF_SCRIPT_PATH}/devices/{device_name}.zap -o {device_out_dir}")
                 shell.run_cmd(f"touch {device_out_dir}/af-gen-event.h")
-        with open(ci_manifest_file_name, "w+", encoding="utf-8") as ci_manifest_file:
-            git_cmd = ["git", "ls-tree", "master", "third_party/zap/repo"]
-            zap_commit = str(subprocess.check_output(git_cmd, cwd=_REPO_BASE_PATH))
-            zap_commit = zap_commit.split(" ")[2]
-            zap_commit = zap_commit[:zap_commit.index("\\")]
-            print(f"zap commit: {zap_commit}")
-            ci_manifest["zap_commit"] = zap_commit
-            ci_manifest_file.write(json.dumps(ci_manifest, indent=4))
+        chef_util.generate_device_manifest(chef_devices_dir, include_zap_submod=True, write_manifest_file = True, ci_manifest_file_name = ci_manifest_file_name, repo_base_path=_REPO_BASE_PATH)
+        exit(0)
+    #
+    # Build all
+    #
+
+    if options.build_all:
         exit(0)
 
     #

examples/chef/chef_util.py

@@ -0,0 +1,40 @@
+import subprocess
+import json
+import os
+import hashlib
+
+ci_allowlist = ['lighting-app.zap']
+cd_output_dirs = {
+    'linux': 'linux/out',
+    'esp32': 'esp32/build',
+    'nrfconnect': 'nrfconnect/build',
+}
+
+def check_zap_master(repo_base_path: str) -> str:
+    """Produces hash of ZAP submodule in branch master"""
+    git_cmd = ["git", "ls-tree", "master", "third_party/zap/repo"]
+    zap_commit = str(subprocess.check_output(git_cmd, cwd=repo_base_path))
+    zap_commit = zap_commit.split(" ")[2]
+    zap_commit = zap_commit[:zap_commit.index("\\")]
+    print(f"zap commit: {zap_commit}")
+    return zap_commit
+
+def generate_device_manifest(chef_devices_dir: str, include_zap_submod: bool = False, write_manifest_file: bool = False, ci_manifest_file_name: str = '', repo_base_path: str = '') -> dict:
+    """Produces dictionary containing md5 of device dir zap files"""
+    ci_manifest = {}
+    for device_dir_item in os.listdir(chef_devices_dir):
+        target_file_ext = ".zap"
+        if device_dir_item.endswith(target_file_ext) and device_dir_item in ci_allowlist:
+            device_name = device_dir_item[:-len(target_file_ext)]
+            device_file_path = os.path.join(chef_devices_dir, device_dir_item)
+    with open(device_file_path, "rb") as device_file:
+        device_file_data = device_file.read()
+        device_file_md5 = hashlib.md5(device_file_data).hexdigest()
+        ci_manifest[device_name] = device_file_md5
+        print(f"Manifest for {device_name} : {device_file_md5}")
+    if include_zap_submod:
+        ci_manifest["zap_commit"] = check_zap_master(repo_base_path)
+    if write_manifest_file:
+        with open(ci_manifest_file_name, "w+", encoding="utf-8") as ci_manifest_file:
+            ci_manifest_file.write(json.dumps(ci_manifest, indent=4))
+    return ci_manifest