Skip to content

Commit

Permalink
Parallelize zap_convert_all.py.
Browse files Browse the repository at this point in the history
It's 4-5x faster for me this way.

Also sets ZAP_SKIP_REAL_VERSION so that once
project-chip/zap#876 merges we stop messing with the ZAP
repo just to run it.
  • Loading branch information
bzbarsky-apple committed Dec 22, 2022
1 parent 4088a77 commit 8f203e1
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 2 deletions.
2 changes: 2 additions & 0 deletions scripts/tools/zap/convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ def runConversion(zap_file):
if 'ZAP_DEVELOPMENT_PATH' in os.environ:
convert_cmd = ['node', 'src-script/zap-start.js', 'convert']
working_directory = os.environ['ZAP_DEVELOPMENT_PATH']
# Make sure we don't try to munge the package.json in the ZAP repo.
os.environ['ZAP_SKIP_REAL_VERSION'] = '1'
elif 'ZAP_INSTALL_PATH' in os.environ:
convert_cmd = [os.path.join(os.environ['ZAP_INSTALL_PATH'], 'zap-cli'), 'convert']
working_directory = None
Expand Down
2 changes: 2 additions & 0 deletions scripts/tools/zap/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ def runGeneration(zap_file, zcl_file, templates_file, output_dir):
if 'ZAP_DEVELOPMENT_PATH' in os.environ:
generate_cmd = ['node', 'src-script/zap-start.js', 'generate']
working_directory = os.environ['ZAP_DEVELOPMENT_PATH']
# Make sure we don't try to munge the package.json in the ZAP repo.
os.environ['ZAP_SKIP_REAL_VERSION'] = '1'
elif 'ZAP_INSTALL_PATH' in os.environ:
generate_cmd = [os.path.join(os.environ['ZAP_INSTALL_PATH'], 'zap-cli'), 'generate']
working_directory = None
Expand Down
2 changes: 2 additions & 0 deletions scripts/tools/zap/run_zaptool.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ CHIP_ROOT="${SCRIPT_PATH%/scripts/tools/zap/run_zaptool.sh}"
if [ ! -z "$ZAP_DEVELOPMENT_PATH" ]; then
WORKING_DIR=$ZAP_DEVELOPMENT_PATH
ZAP_CMD="node src-script/zap-start.js"
# Make sure we don't try to munge the package.json in the ZAP repo.
export ZAP_SKIP_REAL_VERSION=1

"$CHIP_ROOT"/scripts/tools/zap/zap_bootstrap.sh
elif [ ! -z "$ZAP_INSTALL_PATH" ]; then
Expand Down
24 changes: 22 additions & 2 deletions scripts/tools/zap_convert_all.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import sys
import subprocess
import argparse
import multiprocessing

CHIP_ROOT_DIR = os.path.realpath(
os.path.join(os.path.dirname(__file__), '../..'))
Expand Down Expand Up @@ -59,9 +60,20 @@ def runArgumentsParser():
description='Convert all .zap files to the current zap version')
parser.add_argument('--run-bootstrap', default=None, action='store_true',
help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')
parser.add_argument('--parallel', action='store_true')
parser.add_argument('--no-parallel', action='store_false', dest='parallel')
parser.set_defaults(parallel=True)

return parser.parse_args()


def convertOne(target):
"""
Helper method that may be run in parallel to convert a single target.
"""
subprocess.check_call(['./scripts/tools/zap/convert.py'] + target)


def main():
args = runArgumentsParser()
checkPythonVersion()
Expand All @@ -72,8 +84,16 @@ def main():
os.chdir(CHIP_ROOT_DIR)

targets = getTargets()
for target in targets:
subprocess.check_call(['./scripts/tools/zap/convert.py'] + target)

if args.parallel:
# Ensure each zap run is independent
os.environ['ZAP_TEMPSTATE'] = '1'
with multiprocessing.Pool() as pool:
for _ in pool.imap_unordered(convertOne, targets):
pass
else:
for target in targets:
generateOne(target)


if __name__ == '__main__':
Expand Down

0 comments on commit 8f203e1

Please sign in to comment.