Skip to content

Commit

Permalink
Run zap in parallel by default for zap_regen_all.py (#23766)
Browse files Browse the repository at this point in the history
* Run zap in parallel on regen all. This makes my local zap run in 88 seconds instead of about 222

* Restyle

* Fix arguments for generate.py
  • Loading branch information
andy31415 authored and pull[bot] committed Dec 2, 2022
1 parent 9a9a18b commit 1012060
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 2 deletions.
9 changes: 9 additions & 0 deletions scripts/tools/zap/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ class CmdLineArgs:
templateFile: str
outputDir: str
runBootstrap: bool
parallel: bool = True


CHIP_ROOT_DIR = os.path.realpath(
Expand Down Expand Up @@ -103,6 +104,9 @@ def runArgumentsParser() -> CmdLineArgs:
help='Output directory for the generated files (default: automatically selected)')
parser.add_argument('--run-bootstrap', default=None, action='store_true',
help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')
parser.add_argument('--parallel', action='store_true')
parser.add_argument('--no-parallel', action='store_false', dest='parallel')
parser.set_defaults(parallel=True)
args = parser.parse_args()

# By default, this script assumes that the global CHIP template is used with
Expand Down Expand Up @@ -259,6 +263,11 @@ def main():

# The maximum memory usage is over 4GB (#15620)
os.environ["NODE_OPTIONS"] = "--max-old-space-size=8192"

if cmdLineArgs.parallel:
# Parallel-compatible runs will need separate state
os.environ["ZAP_TEMPSTATE"] = "1"

runGeneration(cmdLineArgs.zapFile, cmdLineArgs.zclFile, cmdLineArgs.templateFile, cmdLineArgs.outputDir)

prettifiers = [
Expand Down
26 changes: 24 additions & 2 deletions scripts/tools/zap_regen_all.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import sys
import subprocess
import logging
import multiprocessing

from dataclasses import dataclass

CHIP_ROOT_DIR = os.path.realpath(
Expand Down Expand Up @@ -113,6 +115,11 @@ def setupArgumentsParser():
help="Don't do any generation, just log what targets would be generated (default: False)")
parser.add_argument('--run-bootstrap', default=None, action='store_true',
help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')

parser.add_argument('--parallel', action='store_true')
parser.add_argument('--no-parallel', action='store_false', dest='parallel')
parser.set_defaults(parallel=True)

return parser.parse_args()


Expand Down Expand Up @@ -253,6 +260,14 @@ def getTargets(type, test_target):
return targets


def _ParallelGenerateOne(target):
"""
Helper method to be passed to multiprocessing parallel generation of
items.
"""
target.generate()


def main():
logging.basicConfig(
level=logging.INFO,
Expand All @@ -269,8 +284,15 @@ def main():
if args.run_bootstrap:
subprocess.check_call(os.path.join(CHIP_ROOT_DIR, "scripts/tools/zap/zap_bootstrap.sh"), shell=True)

for target in targets:
target.generate()
if args.parallel:
# Ensure each zap run is independent
os.environ['ZAP_TEMPSTATE'] = '1'
with multiprocessing.Pool() as pool:
for _ in pool.imap_unordered(_ParallelGenerateOne, targets):
pass
else:
for target in targets:
target.generate()


if __name__ == '__main__':
Expand Down

0 comments on commit 1012060

Please sign in to comment.