python: New //:python_wheels target (#25024)

- //BUILD.gn - Cleanup some existing python related groups - Add group("python_wheels") which is used by //scripts/build_python.sh - //scripts/build_python.sh - Install matter_yamltests wheel into the venv if --include_yamltests
project-chip · Nov 8, 2023 · d9ef480 · d9ef480
1 parent 458a0a3
commit d9ef480
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 101 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -196,7 +196,7 @@ jobs:
             - name: Build Apps
               timeout-minutes: 60
               run: |
-                  scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env'
+                  scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env --include_yamltests'
                   ./scripts/run_in_build_env.sh \
                      "./scripts/build/build_examples.py \
                         --target linux-x64-chip-tool${CHIP_TOOL_VARIANT}-${BUILD_VARIANT} \

diff --git a/BUILD.gn b/BUILD.gn
@@ -58,56 +58,12 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
     }
   }
 
-  # Pigweed Python packages expected to be used in the :matter_build_venv
-  # target. If all packages are needed this list should match
-  # _pigweed_python_deps in:
-  # https://cs.opensource.google/pigweed/pigweed/+/master:pw_env_setup/BUILD.gn?q=_pigweed_python_deps
-  _pigweed_python_packages = [
-    "$dir_pw_allocator/py",
-    "$dir_pw_arduino_build/py",
-    "$dir_pw_bloat/py",
-    "$dir_pw_build/py",
-    "$dir_pw_build_info/py",
-    "$dir_pw_build_mcuxpresso/py",
-    "$dir_pw_cli/py",
-    "$dir_pw_console/py",
-    "$dir_pw_cpu_exception_cortex_m/py",
-    "$dir_pw_docgen/py",
-    "$dir_pw_doctor/py",
-    "$dir_pw_env_setup/py",
-    "$dir_pw_hdlc/py",
-    "$dir_pw_log:protos.python",
-    "$dir_pw_log_tokenized/py",
-    "$dir_pw_metric/py",
-    "$dir_pw_module/py",
-    "$dir_pw_package/py",
-    "$dir_pw_presubmit/py",
-    "$dir_pw_protobuf/py",
-    "$dir_pw_protobuf_compiler/py",
-    "$dir_pw_rpc/py",
-    "$dir_pw_snapshot/py:pw_snapshot",
-    "$dir_pw_snapshot/py:pw_snapshot_metadata",
-    "$dir_pw_software_update/py",
-    "$dir_pw_status/py",
-    "$dir_pw_stm32cube_build/py",
-    "$dir_pw_symbolizer/py",
-    "$dir_pw_system/py",
-    "$dir_pw_thread/py",
-    "$dir_pw_tls_client/py",
-    "$dir_pw_tokenizer/py",
-    "$dir_pw_toolchain/py",
-    "$dir_pw_trace/py",
-    "$dir_pw_trace_tokenized/py",
-    "$dir_pw_transfer/py",
-    "$dir_pw_unit_test/py",
-    "$dir_pw_watch/py",
-  ]
-
   # Matter's in-tree pw_python_package or pw_python_distribution targets.
   _matter_python_packages = [
-    "//integrations/mobly:chip_mobly",
     "//examples/chef",
     "//examples/common/pigweed/rpc_console/py:chip_rpc",
+    "//integrations/mobly:chip_mobly",
+    "//scripts/py_matter_yamltests:matter_yamltests",
   ]
 
   pw_python_venv("matter_build_venv") {
@@ -116,7 +72,8 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
     requirements = pw_build_PIP_REQUIREMENTS
 
     # Packages available to import within GN's build venv.
-    source_packages = _matter_python_packages + _pigweed_python_packages
+    source_packages = _matter_python_packages +
+                      [ "$dir_pw_env_setup:core_pigweed_python_packages" ]
   }
 
   pw_python_pip_install("pip_install_matter_packages") {
@@ -141,6 +98,19 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
     editable = true
   }
 
+  # Python wheel targets, used by //scripts/build_python.sh
+  group("python_wheels") {
+    if (chip_enable_python_modules) {
+      deps = [
+        "${chip_root}/scripts:matter_yamltests_distribution.wheel",
+        "${chip_root}/src/controller/python:chip-repl",
+      ]
+      if (enable_pylib) {
+        deps += [ "${chip_root}/src/pybindings/pycontroller" ]
+      }
+    }
+  }
+
   # This is a real toolchain. Build CHIP.
   group("default") {
     deps = [
@@ -201,10 +171,7 @@ if (current_toolchain != "${dir_pw_toolchain}/default:default") {
         deps += [ "${chip_root}/src/tools/chip-cert" ]
       }
       if (chip_enable_python_modules) {
-        if (enable_pylib) {
-          deps += [ "${chip_root}/src/pybindings/pycontroller" ]
-        }
-        deps += [ "${chip_root}/src/controller/python:chip-repl" ]
+        deps += [ ":python_wheels" ]
       }
     }
 

diff --git a/examples/common/pigweed/rpc_console/BUILD.gn b/examples/common/pigweed/rpc_console/BUILD.gn
@@ -32,10 +32,11 @@ group("default") {
 pw_python_distribution("chip_rpc_distribution") {
   packages = [ "py:chip_rpc" ]
   generate_setup_cfg = {
-    common_config_file = "common_setup.cfg"
+    name = "chip_rpc"
+    version = "0.0.1"
+    include_default_pyproject_file = true
     append_date_to_version = true
   }
-  extra_files = [ "py/pyproject.toml > pyproject.toml" ]
 }
 
 pw_internal_pip_install("pip_install_editable_chip_rpc") {

diff --git a/examples/common/pigweed/rpc_console/common_setup.cfg b/examples/common/pigweed/rpc_console/common_setup.cfg
diff --git a/scripts/BUILD.gn b/scripts/BUILD.gn
@@ -25,7 +25,8 @@ import("$dir_pw_build/python_dist.gni")
 pw_python_distribution("matter_yamltests_distribution") {
   packages = [ "${chip_root}/scripts/py_matter_yamltests:matter_yamltests" ]
   generate_setup_cfg = {
-    common_config_file = "common_setup.cfg"
+    name = "matter_yamltests"
+    version = "0.0.1"
     include_default_pyproject_file = true
     append_date_to_version = true
   }

diff --git a/scripts/build_python.sh b/scripts/build_python.sh
@@ -63,6 +63,7 @@ Input Options:
                                                             no: Do not install
                                                             build-env: install to virtual env for build matter
                                                             separate: install to another virtual env (out/python_env)
+  --include_yamltests                                       Whether to install the matter_yamltests wheel.
   -z --pregen_dir DIRECTORY                                 Directory where generated zap files have been pre-generated.
 "
 }
@@ -95,6 +96,9 @@ while (($#)); do
             install_wheel=$2
             shift
             ;;
+        --include_yamltests)
+            include_yamltests="yes"
+            ;;
         --pregen_dir | -z)
             pregen_dir=$2
             shift
@@ -121,20 +125,41 @@ source "$CHIP_ROOT/scripts/activate.sh"
 
 gn --root="$CHIP_ROOT" gen "$OUTPUT_ROOT" --args="chip_detail_logging=$chip_detail_logging enable_pylib=$enable_pybindings enable_rtti=$enable_pybindings chip_project_config_include_dirs=[\"//config/python\"] $chip_mdns_arg $chip_case_retry_arg $pregen_dir_arg"
 
-# Compiles python files
-# Check pybindings was requested
-if [ "$enable_pybindings" == true ]; then
-    ninja -C "$OUTPUT_ROOT" pycontroller
-else
-    ninja -C "$OUTPUT_ROOT" chip-repl
-fi
+function ninja_target() {
+    # Print the ninja target required to build a gn label.
+    local GN_LABEL="$1"
+    local NINJA_TARGET="$(gn ls "$OUTPUT_ROOT" --as=output "$GN_LABEL")"
+    echo "$NINJA_TARGET"
+}
 
+function wheel_output_dir() {
+    # Print the wheel output directory for a pw_python_package or
+    # pw_python_distribution. The label must end in "._build_wheel".
+    local GN_LABEL="$1"
+    local NINJA_TARGET="$(ninja_target "$GN_LABEL")"
+    local WHEEL_DIR="$OUTPUT_ROOT"/"$(dirname "$NINJA_TARGET")/$(basename -s .stamp "$NINJA_TARGET")"
+    echo "$WHEEL_DIR"
+}
+
+# Compile Python wheels
+ninja -C "$OUTPUT_ROOT" python_wheels
+
+# Add wheels from chip_python_wheel_action templates.
 if [ "$enable_pybindings" == true ]; then
     WHEEL=("$OUTPUT_ROOT"/pybindings/pycontroller/pychip-*.whl)
 else
     WHEEL=("$OUTPUT_ROOT"/controller/python/chip*.whl)
 fi
 
+if [ -n "$include_yamltests" ]; then
+    YAMLTESTS_GN_LABEL="//scripts:matter_yamltests_distribution._build_wheel"
+
+    # Add wheels from pw_python_package or pw_python_distribution templates.
+    WHEEL+=(
+        "$(ls -tr "$(wheel_output_dir "$YAMLTESTS_GN_LABEL")"/*.whl | head -n 1)"
+    )
+fi
+
 if [ "$install_wheel" = "no" ]; then
     exit 0
 elif [ "$install_wheel" = "separate" ]; then
@@ -143,7 +168,7 @@ elif [ "$install_wheel" = "separate" ]; then
 
     source "$ENVIRONMENT_ROOT"/bin/activate
     "$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade pip
-    "$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall --no-cache-dir "${WHEEL[@]}"
+    "$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall "${WHEEL[@]}"
 
     echo ""
     echo_green "Compilation completed and WHL package installed in: "

diff --git a/scripts/common_setup.cfg b/scripts/common_setup.cfg