[scripts] created bootstrap script for ZAP (#22757)

Extracted bootstrap code from run_zaptool.sh and reuse it
in other Python scripts. This allows to run ZAP convert/generate
tools out of the box without necessity to run zaptool beforehand.

Signed-off-by: Marcin Kajor <marcin.kajor@nordicsemi.no>

Signed-off-by: Marcin Kajor <marcin.kajor@nordicsemi.no>

scripts/tools/zap/convert.py

@@ -60,11 +60,13 @@ def runArgumentsParser():
     parser = argparse.ArgumentParser(
         description='Convert .zap files to the current zap version')
     parser.add_argument('zap', help='Path to the application .zap file')
+    parser.add_argument('--run-bootstrap', default=None, action='store_true',
+                        help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')
     args = parser.parse_args()
 
     zap_file = getFilePath(args.zap)
 
-    return zap_file
+    return zap_file, args.run_bootstrap
 
 
 def detectZclFile(zapFile):
@@ -96,11 +98,17 @@ def runConversion(zap_file):
                           '-z', zcl_file, '-g', templates_file, '-o', zap_file, zap_file])
 
 
+def runBootstrap():
+    subprocess.check_call(getFilePath("scripts/tools/zap/zap_bootstrap.sh"), shell=True)
+
+
 def main():
     checkPythonVersion()
+    zap_file, run_bootstrap = runArgumentsParser()
+    if run_bootstrap:
+        runBootstrap()
     os.chdir(CHIP_ROOT_DIR)
 
-    zap_file = runArgumentsParser()
     runConversion(zap_file)
 
 

scripts/tools/zap/generate.py

@@ -22,6 +22,17 @@
 import subprocess
 import sys
 import urllib.request
+from dataclasses import dataclass
+
+
+@dataclass
+class CmdLineArgs:
+    zapFile: str
+    zclFile: str
+    templateFile: str
+    outputDir: str
+    runBootstrap: bool
+
 
 CHIP_ROOT_DIR = os.path.realpath(
     os.path.join(os.path.dirname(__file__), '../../..'))
@@ -77,7 +88,7 @@ def detectZclFile(zapFile):
     return getFilePath(path)
 
 
-def runArgumentsParser():
+def runArgumentsParser() -> CmdLineArgs:
     default_templates = 'src/app/zap-templates/app-templates.json'
     default_output_dir = 'zap-generated/'
 
@@ -90,6 +101,8 @@ def runArgumentsParser():
                         help='Path to the zcl templates records to use for generating artifacts (default: autodetect read from zap file)')
     parser.add_argument('-o', '--output-dir', default=None,
                         help='Output directory for the generated files (default: automatically selected)')
+    parser.add_argument('--run-bootstrap', default=None, action='store_true',
+                        help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')
     args = parser.parse_args()
 
     # By default, this script assumes that the global CHIP template is used with
@@ -113,7 +126,7 @@ def runArgumentsParser():
     templates_file = getFilePath(args.templates)
     output_dir = getDirPath(output_dir)
 
-    return (zap_file, zcl_file, templates_file, output_dir)
+    return CmdLineArgs(zap_file, zcl_file, templates_file, output_dir, args.run_bootstrap)
 
 
 def extractGeneratedIdl(output_dir, zap_config_path):
@@ -209,21 +222,26 @@ def runJavaPrettifier(templates_file, output_dir):
         print('google-java-format error:', err)
 
 
+def runBootstrap():
+    subprocess.check_call(getFilePath("scripts/tools/zap/zap_bootstrap.sh"), shell=True)
+
+
 def main():
     checkPythonVersion()
-
-    # The maximum meory usage is over 4GB (#15620)
+    cmdLineArgs = runArgumentsParser()
+    if cmdLineArgs.runBootstrap:
+        runBootstrap()
+    # The maximum memory usage is over 4GB (#15620)
     os.environ["NODE_OPTIONS"] = "--max-old-space-size=8192"
-    zap_file, zcl_file, templates_file, output_dir = runArgumentsParser()
-    runGeneration(zap_file, zcl_file, templates_file, output_dir)
+    runGeneration(cmdLineArgs.zapFile, cmdLineArgs.zclFile, cmdLineArgs.templateFile, cmdLineArgs.outputDir)
 
     prettifiers = [
         runClangPrettifier,
         runJavaPrettifier,
     ]
 
     for prettifier in prettifiers:
-        prettifier(templates_file, output_dir)
+        prettifier(cmdLineArgs.templateFile, cmdLineArgs.outputDir)
 
 
 if __name__ == '__main__':

scripts/tools/zap/run_zaptool.sh

@@ -33,17 +33,9 @@ CHIP_ROOT="${SCRIPT_PATH%/scripts/tools/zap/run_zaptool.sh}"
 
 (
 
-    cd "$CHIP_ROOT" &&
-        git submodule update --init third_party/zap/repo
+    "$CHIP_ROOT"/scripts/tools/zap/zap_bootstrap.sh
 
-    cd "third_party/zap/repo"
-    if ! npm list installed-check &>/dev/null; then
-        npm install installed-check
-    fi
-
-    if ! ./node_modules/.bin/installed-check -c &>/dev/null; then
-        npm install
-    fi
+    cd "$CHIP_ROOT/third_party/zap/repo"
 
     echo "ARGS: ${ZAP_ARGS[@]}"
 

scripts/tools/zap/zap_bootstrap.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+#
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function _get_fullpath() {
+    cd "$(dirname "$1")" && echo "$PWD/$(basename "$1")"
+}
+
+function _usage() {
+    cat <<EOF
+Invalid arguments passed.
+Usage:
+    zap_bootstrap.sh -> install and update required packages
+    zap_bootstrap.sh -c -> run a clean bootstrap, install packages from scratch
+EOF
+    exit 1
+}
+
+set -e
+
+SCRIPT_PATH="$(_get_fullpath "$0")"
+CHIP_ROOT="${SCRIPT_PATH%/scripts/tools/zap/zap_bootstrap.sh}"
+
+(
+    cd "$CHIP_ROOT" &&
+        git submodule update --init third_party/zap/repo
+
+    cd "third_party/zap/repo"
+
+    if [ $# -eq 0 ]; then
+        echo "Running ZAP bootstrap"
+        if ! npm list installed-check &>/dev/null; then
+            npm install installed-check
+        fi
+
+        if ! ./node_modules/.bin/installed-check -c &>/dev/null; then
+            npm install
+        fi
+    elif [ $# -eq 1 ] && [ "$1" = "-c" ]; then
+        echo "Running clean ZAP bootstrap"
+        npm ci
+        npm run version-stamp
+        npm rebuild canvas --update-binary
+        npm run build-spa
+    else
+        _usage
+    fi
+)
+
+echo "ZAP bootstrap done!"

scripts/tools/zap_convert_all.py

@@ -19,6 +19,7 @@
 from pathlib import Path
 import sys
 import subprocess
+import argparse
 
 CHIP_ROOT_DIR = os.path.realpath(
     os.path.join(os.path.dirname(__file__), '../..'))
@@ -53,8 +54,23 @@ def getTargets():
     return targets
 
 
+def runArgumentsParser():
+    parser = argparse.ArgumentParser(
+        description='Convert all .zap files to the current zap version')
+    parser.add_argument('--run-bootstrap', default=None, action='store_true',
+                        help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')
+    return parser.parse_args()
+
+
+def runBootstrap():
+    subprocess.check_call(os.path.join(CHIP_ROOT_DIR, "scripts/tools/zap/zap_bootstrap.sh"), shell=True)
+
+
 def main():
+    args = runArgumentsParser()
     checkPythonVersion()
+    if args.run_bootstrap:
+        runBootstrap()
     os.chdir(CHIP_ROOT_DIR)
 
     targets = getTargets()

scripts/tools/zap_regen_all.py

@@ -93,7 +93,9 @@ def setupArgumentsParser():
     parser.add_argument('--tests', default='all', choices=['all', 'chip-tool', 'darwin-framework-tool', 'app1', 'app2'],
                         help='When generating tests only target, Choose which tests to generate (default: all)')
     parser.add_argument('--dry-run', default=False, action='store_true',
-                        help="Don't do any generationl just log what targets would be generated (default: False)")
+                        help="Don't do any generation, just log what targets would be generated (default: False)")
+    parser.add_argument('--run-bootstrap', default=None, action='store_true',
+                        help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')
     return parser.parse_args()
 
 
@@ -224,6 +226,10 @@ def getTargets(type, test_target):
     return targets
 
 
+def runBootstrap():
+    subprocess.check_call(os.path.join(CHIP_ROOT_DIR, "scripts/tools/zap/zap_bootstrap.sh"), shell=True)
+
+
 def main():
     logging.basicConfig(
         level=logging.INFO,
@@ -236,6 +242,8 @@ def main():
     targets = getTargets(args.type, args.tests)
 
     if (not args.dry_run):
+        if (args.run_bootstrap):
+            runBootstrap()
         for target in targets:
             target.generate()