diff --git a/.gn b/.gn index 604bc15b848d69..742fdc4273d666 100644 --- a/.gn +++ b/.gn @@ -27,7 +27,7 @@ default_args = { pw_unit_test_AUTOMATIC_RUNNER = "$dir_pigweed/targets/host/run_test" pw_build_PIP_CONSTRAINTS = [ "//scripts/setup/constraints.txt" ] - pw_build_PIP_REQUIREMENTS = [ "//scripts/setup/requirements.txt" ] + pw_build_PIP_REQUIREMENTS = [ "//scripts/setup/requirements.build.txt" ] # GN target to use for the default Python build venv. pw_build_PYTHON_BUILD_VENV = "//:matter_build_venv" diff --git a/config/esp32/.gn b/config/esp32/.gn index 37684f9211c628..9f5db65cdad79b 100644 --- a/config/esp32/.gn +++ b/config/esp32/.gn @@ -26,8 +26,10 @@ default_args = { pw_build_PIP_CONSTRAINTS = [ "//third_party/connectedhomeip/scripts/setup/constraints.txt" ] - pw_build_PIP_REQUIREMENTS = - [ "//third_party/connectedhomeip/scripts/setup/requirements.txt" ] + pw_build_PIP_REQUIREMENTS = [ + "//third_party/connectedhomeip/scripts/setup/requirements.build.txt", + "//third_party/connectedhomeip/scripts/setup/requirements.esp32.txt", + ] import("//args.gni") } diff --git a/config/standalone/args.gni b/config/standalone/args.gni index 8ce29968cdd4a8..2bc0d7519c410b 100644 --- a/config/standalone/args.gni +++ b/config/standalone/args.gni @@ -19,4 +19,5 @@ import("//build_overrides/chip.gni") chip_build_tests = false pw_build_PIP_CONSTRAINTS = [ "${chip_root}/scripts/setup/constraints.txt" ] -pw_build_PIP_REQUIREMENTS = [ "${chip_root}/scripts/setup/requirements.txt" ] +pw_build_PIP_REQUIREMENTS = + [ "${chip_root}/scripts/setup/requirements.build.txt" ] diff --git a/examples/common/pigweed/rpc_console/.gn b/examples/common/pigweed/rpc_console/.gn index 963ca108a40545..ead10c338ecd07 100644 --- a/examples/common/pigweed/rpc_console/.gn +++ b/examples/common/pigweed/rpc_console/.gn @@ -23,7 +23,7 @@ default_args = { pw_build_PIP_CONSTRAINTS = [ "//third_party/connectedhomeip/scripts/setup/constraints.txt" ] pw_build_PIP_REQUIREMENTS = - [ "//third_party/connectedhomeip/scripts/setup/requirements.txt" ] + [ "//third_party/connectedhomeip/scripts/setup/requirements.build.txt" ] pw_build_USE_NEW_PYTHON_BUILD = true } diff --git a/scripts/setup/bootstrap.sh b/scripts/setup/bootstrap.sh index 3d0a1ad7e3d2c1..af00b719fcfd5d 100644 --- a/scripts/setup/bootstrap.sh +++ b/scripts/setup/bootstrap.sh @@ -14,6 +14,38 @@ # limitations under the License. # +_install_additional_pip_requirements() { + _SETUP_PLATFORM=$1 + shift + + # figure out additional pip install items + while [[ $# -gt 0 ]]; do + case $1 in + -p | --platform) + _SETUP_PLATFORM=$2 + shift # argument + shift # value + ;; + *) + shift + ;; + esac + done + + if ! [ -z "$_SETUP_PLATFORM" ]; then + IFS="," read -r -a _PLATFORMS <<<"$_SETUP_PLATFORM" + for platform in "${_PLATFORMS[@]}"; do + # Allow none as an alias of nothing extra installed (like -p none) + if [ "$platform" != "none" ]; then + echo "Installing pip requirements for $platform..." + pip install -q \ + -r "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" \ + -c "$_CHIP_ROOT/scripts/setup/constraints.txt" + fi + done + fi +} + _bootstrap_or_activate() { if [ -n "$BASH" ]; then local _BOOTSTRAP_PATH="${BASH_SOURCE[0]}" @@ -24,7 +56,7 @@ _bootstrap_or_activate() { local _BOOTSTRAP_NAME="${_BOOTSTRAP_PATH##*/}" local _BOOTSTRAP_DIR="${_BOOTSTRAP_PATH%/*}" # Strip off the 'scripts[/setup]' directory, leaving the root of the repo. - local _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." && pwd)" + _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." && pwd)" local _CONFIG_FILE="scripts/setup/environment.json" @@ -91,21 +123,42 @@ EOF --config-file "$_CHIP_ROOT/$_CONFIG_FILE" \ --virtualenv-gn-out-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT/gn_out" pw_finalize bootstrap "$_SETUP_SH" + _ACTION_TAKEN="bootstrap" else pw_activate pw_finalize activate "$_SETUP_SH" + _ACTION_TAKEN="activate" fi } +# remember PW_ENVIRONMENT_ROOT so that things like another +# bootstrap or run_in_build_env.sh can be executed in a build env +_ORIGINAL_PW_ENVIRONMENT_ROOT="$PW_ENVIRONMENT_ROOT" + _bootstrap_or_activate "$0" + +if [ "$_ACTION_TAKEN" = "bootstrap" ]; then + # By default, install all extra pip dependencies even if slow. -p/--platform + # arguments may override this default. + _install_additional_pip_requirements "all" "$@" +else + _install_additional_pip_requirements "none" "$@" +fi + unset -f _bootstrap_or_activate +unset -f _install_additional_pip_requirements pw_cleanup +unset _ACTION_TAKEN +unset _CHIP_ROOT unset PW_CIPD_INSTALL_DIR -unset CIPD_CACHE_DIR unset _PW_BANNER_FUNC unset _PW_TEXT unset PW_DOCTOR_SKIP_CIPD_CHECKS unset -f _chip_bootstrap_banner + +if ! [ -z "$_ORIGINAL_PW_ENVIRONMENT_ROOT" ]; then + export PW_ENVIRONMENT_ROOT="$_ORIGINAL_PW_ENVIRONMENT_ROOT" +fi diff --git a/scripts/setup/constraints.txt b/scripts/setup/constraints.txt index 33cb08901a93e4..1fa12ca4a45c83 100644 --- a/scripts/setup/constraints.txt +++ b/scripts/setup/constraints.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --output-file=constraints.txt --resolver=backtracking --strip-extras requirements.txt +# pip-compile --output-file=constraints.txt --resolver=backtracking --strip-extras requirements.all.txt # anytree==2.8.0 # via -r requirements.txt diff --git a/scripts/setup/environment.json b/scripts/setup/environment.json index ebe77d9db6dbc8..7ded85c3b43b0a 100644 --- a/scripts/setup/environment.json +++ b/scripts/setup/environment.json @@ -8,7 +8,7 @@ "virtualenv": { "gn_root": ".", "gn_targets": [":python_packages.install"], - "requirements": ["scripts/setup/requirements.txt"], + "requirements": ["scripts/setup/requirements.build.txt"], "constraints": ["scripts/setup/constraints.txt"], "gn_args": ["chip_crypto=\"boringssl\""] }, diff --git a/scripts/setup/environment_no_cipd.json b/scripts/setup/environment_no_cipd.json index b2a9075fc56fed..014a7cfaa09dcb 100644 --- a/scripts/setup/environment_no_cipd.json +++ b/scripts/setup/environment_no_cipd.json @@ -2,7 +2,7 @@ "virtualenv": { "gn_root": ".", "gn_targets": [":python_packages.install"], - "requirements": ["scripts/setup/requirements.txt"], + "requirements": ["scripts/setup/requirements.build.txt"], "constraints": ["scripts/setup/constraints.txt"], "gn_args": ["chip_crypto=\"boringssl\""] }, diff --git a/scripts/setup/requirements.txt b/scripts/setup/requirements.all.txt similarity index 68% rename from scripts/setup/requirements.txt rename to scripts/setup/requirements.all.txt index 2a5c57c1eef574..eca54173203d46 100644 --- a/scripts/setup/requirements.txt +++ b/scripts/setup/requirements.all.txt @@ -8,26 +8,16 @@ virtualenv -c constraints.esp32.txt -r requirements.esp32.txt -# mbed-os -r requirements.mbed.txt - -# bouffalolab -r requirements.bouffalolab.txt - -# openiotsdk -r requirements.openiotsdk.txt - -# Infineon -r requirements.infineon.txt - -# TI -r requirements.ti.txt - -# Telink -r requirements.telink.txt - -# cirque tests -requests>=2.24.0 +-r requirements.zephyr.txt +-r requirements.cirque.txt +-r requirements.memory.txt +-r requirements.yaml_tests.txt # device controller wheel package wheel; sys_platform == 'linux' @@ -40,17 +30,6 @@ pyobjc-framework-corebluetooth; sys_platform == 'darwin' portpicker mobly -# zephyr -west>=0.12.0 -intelhex -cbor -cbor2 -jsonschema - -# happy tests -lockfile -psutil >= 5.7.3 - # pigweed ipython appnope @@ -63,13 +42,6 @@ mypy-protobuf==3.2.0 protobuf==3.20.1 types-protobuf==3.19.22 -# scripts/tools/memory -anytree -cxxfilt -ghapi -pandas ; platform_machine != 'aarch64' and platform_machine != 'arm64' -tabulate - cryptography # python unit tests @@ -78,6 +50,3 @@ colorama # update tornado for pw_watch tornado -# YAML test harness -diskcache -websockets diff --git a/scripts/setup/requirements.cirque.txt b/scripts/setup/requirements.cirque.txt new file mode 100644 index 00000000000000..f1e8ee5f9418ce --- /dev/null +++ b/scripts/setup/requirements.cirque.txt @@ -0,0 +1 @@ +requests>=2.24.0 diff --git a/scripts/setup/requirements.memory.txt b/scripts/setup/requirements.memory.txt new file mode 100644 index 00000000000000..3aeee8b36eadb7 --- /dev/null +++ b/scripts/setup/requirements.memory.txt @@ -0,0 +1,6 @@ +# scripts/tools/memory +anytree +cxxfilt +ghapi +pandas ; platform_machine != 'aarch64' and platform_machine != 'arm64' +tabulate diff --git a/scripts/setup/requirements.yaml_tests.txt b/scripts/setup/requirements.yaml_tests.txt new file mode 100644 index 00000000000000..7fc0c5f653c194 --- /dev/null +++ b/scripts/setup/requirements.yaml_tests.txt @@ -0,0 +1,2 @@ +diskcache +websockets diff --git a/scripts/setup/requirements.zephyr.txt b/scripts/setup/requirements.zephyr.txt new file mode 100644 index 00000000000000..5c871eef856d19 --- /dev/null +++ b/scripts/setup/requirements.zephyr.txt @@ -0,0 +1,5 @@ +west>=0.12.0 +intelhex +cbor +cbor2 +jsonschema diff --git a/scripts/tools/telink/readme.md b/scripts/tools/telink/readme.md index 4bb4a8a32a9d1c..1ee492c2ffc695 100644 --- a/scripts/tools/telink/readme.md +++ b/scripts/tools/telink/readme.md @@ -35,7 +35,8 @@ tool. ```shell cd path/to/connectedhomeip/scripts/tools/telink/ -python3 -m pip install -r requirements.txt +python3 -m pip install -r requirements.build.txt +python3 -m pip install -r requirements.telink.txt ``` ## Usage diff --git a/src/test_driver/efr32/.gn b/src/test_driver/efr32/.gn index f7a835ede9461e..9082b04c220ef7 100644 --- a/src/test_driver/efr32/.gn +++ b/src/test_driver/efr32/.gn @@ -27,7 +27,7 @@ default_args = { pw_build_PIP_CONSTRAINTS = [ "//third_party/connectedhomeip/scripts/setup/constraints.txt" ] pw_build_PIP_REQUIREMENTS = - [ "//third_party/connectedhomeip/scripts/setup/requirements.txt" ] + [ "//third_party/connectedhomeip/scripts/setup/requirements.build.txt" ] import("//args.gni") }