ci: build only the Zephyr boards that have the changed module

ci_set_matrix.py cannot ask make which Zephyr boards compile a module,
so a change to any module built all 29 of them, and a frozen library
update too although the port has no frozen modules.

The module table of a Zephyr board is only known to its build, which
writes it to autogen_board_info.toml. Keep it: build_release_files.py
copies the file the push build generated to bin/zephyr-modules/<ref>/
<board>.toml, from where the existing bin/ upload puts it on S3 next to
mpy-cross and the stubs. The scheduler fetches it for the PR's base ref
and builds a board when the table says the module is enabled, or when
the table is missing, unreadable or does not know the module. frozen/
changes build no Zephyr boards; supervisor/ changes still build all.

Until the first push build after this lands there is no table and
every Zephyr board is built, as today. ZEPHYR_MODULES_URL overrides the
location for testing.

Author: lynt-smitka

tools/build_release_files.py

@@ -32,6 +32,22 @@
 
 build_all = os.environ.get("GITHUB_EVENT_NAME") != "pull_request"
 
+
+def save_zephyr_board_info(board, board_dir):
+    """Keep the module table a Zephyr build generated, under bin/ so it is uploaded.
+
+    tools/ci_set_matrix.py reads it from S3 to decide which Zephyr boards a change
+    to a module concerns.
+    """
+    ref = os.environ.get("GITHUB_REF_NAME")
+    board_info = board_dir / "autogen_board_info.toml"
+    if not ref or not board_info.exists():
+        return
+    destination = pathlib.Path("../bin/zephyr-modules") / ref
+    destination.mkdir(parents=True, exist_ok=True)
+    shutil.copyfile(board_info, destination / f"{board}.toml")
+
+
 LANGUAGE_FIRST = "en_US"
 LANGUAGE_THRESHOLD = 10 * 1024
 
@@ -61,7 +77,9 @@
         board_settings = {"CLEAN_REBUILD_LANGUAGES": []}
         with cp_toml.open("rb") as f:
             board_settings.update(tomllib.load(f))
+        zephyr_board_dir = cp_toml.parent
     else:
+        zephyr_board_dir = None
         board_settings = get_settings_from_makefile("../ports/" + board_info["port"], board)
         board_settings["CIRCUITPY_BUILD_EXTENSIONS"] = [
             extension.strip()
@@ -141,6 +159,9 @@
                     if exit_status == 0:
                         exit_status = 1
 
+        if zephyr_board_dir and language == LANGUAGE_FIRST and make_result.returncode == 0:
+            save_zephyr_board_info(board, zephyr_board_dir)
+
         print(
             "Build {board} for {language}{clean_build} took {build_duration:.2f}s and {success}".format(
                 board=board,

tools/ci_set_matrix.py

@@ -28,6 +28,8 @@
 import json
 import pathlib
 import subprocess
+import tomllib
+import urllib.request
 from concurrent.futures import ThreadPoolExecutor
 
 tools_dir = pathlib.Path(__file__).resolve().parent
@@ -71,6 +73,51 @@
 
 GITHUB_MATRIX_LIMIT = 256
 
+# Zephyr boards don't use make, so their module tables can't be computed here. Each push
+# build uploads the board's autogen_board_info.toml with the firmware
+# (tools/build_release_files.py); a board whose table is missing, unreadable or does not
+# know a module is built.
+ZEPHYR_MODULES_URL = os.environ.get(
+    "ZEPHYR_MODULES_URL",
+    "https://adafruit-circuit-python.s3.amazonaws.com/bin/zephyr-modules/{ref}/{board}.toml",
+)
+zephyr_modules = {}
+
+
+def fetch_zephyr_modules(boards):
+    ref = os.environ.get("GITHUB_BASE_REF") or os.environ.get("GITHUB_REF_NAME") or "main"
+
+    def fetch(board):
+        url = ZEPHYR_MODULES_URL.format(board=board, ref=ref)
+        try:
+            with urllib.request.urlopen(url, timeout=10) as response:
+                return board, tomllib.loads(response.read().decode("utf-8"))["modules"]
+        except Exception as e:  # noqa: BLE001 -- whatever went wrong, the board gets built
+            print(f"  {board}: no module table ({e})")
+            return board, None
+
+    need = [board for board in boards if board not in zephyr_modules]
+    with ThreadPoolExecutor(max_workers=os.cpu_count()) as ex:
+        zephyr_modules.update(ex.map(fetch, need))
+
+
+def zephyr_boards_for(file, module, boards):
+    """The Zephyr boards a change to `file` concerns, out of `boards`."""
+    if file.startswith("frozen"):
+        # The port has no frozen modules.
+        return []
+    if module is None:
+        return boards
+    fetch_zephyr_modules(boards)
+    selected = [
+        board
+        for board in boards
+        if zephyr_modules.get(board) is None or zephyr_modules[board].get(module, True)
+    ]
+    print(f"Zephyr boards with {module}: {len(selected)} of {len(boards)}")
+    return selected
+
+
 PATTERN_WINDOWS = {
     ".github/",
     "extmod/",
@@ -205,10 +252,10 @@ def get_settings(board):
                 # the logic to build all boards breaks.
                 boards = set(port_to_board[port] if port else all_board_ids)
 
-                # Zephyr boards don't use make, so build them and don't compute their settings.
-                for board in port_to_board["zephyr-cp"]:
-                    if board in boards:
-                        boards_to_build.add(board)
+                zephyr_boards = sorted(boards & port_to_board["zephyr-cp"])
+                module = module_matches.group(2) if module_matches else None
+                boards_to_build.update(zephyr_boards_for(file, module, zephyr_boards))
+                boards -= port_to_board["zephyr-cp"]
 
                 for board in boards_to_build:
                     if board in boards: