Skip to content

Commit

Permalink
Make main build python env not depend on all platform build requireme…
Browse files Browse the repository at this point in the history
…nts. (#27267)

* Attempt to separate out requirements, so that bootstrap/activate can selectively add more things

* A few more requirements updates

* Also update standalone build requirements

* Add some intermediate requirements text files

* Updated more references to requirements.txt

* Preserve environment root

* Fix memory txt, make platform bootstrap configurable

* Make sure constraints are being used for pip install requirements for various platforms

* Fix activate not installing additional stuff unless asked for

* Restyled by gn

* Restyled by prettier-json

* Restyled by shellharden

* Restyled by shfmt

* Update comments

* Preserve CIPD_CACHE_DIR ...we may want to cache this instead of the entire .environment at some point

---------

Co-authored-by: Andrei Litvin <andreilitvin@google.com>
Co-authored-by: Restyled.io <commits@restyled.io>
  • Loading branch information
3 people authored and pull[bot] committed Feb 16, 2024
1 parent bbaed2e commit 1002892
Show file tree
Hide file tree
Showing 15 changed files with 87 additions and 47 deletions.
2 changes: 1 addition & 1 deletion .gn
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ default_args = {
pw_unit_test_AUTOMATIC_RUNNER = "$dir_pigweed/targets/host/run_test"

pw_build_PIP_CONSTRAINTS = [ "//scripts/setup/constraints.txt" ]
pw_build_PIP_REQUIREMENTS = [ "//scripts/setup/requirements.txt" ]
pw_build_PIP_REQUIREMENTS = [ "//scripts/setup/requirements.build.txt" ]

# GN target to use for the default Python build venv.
pw_build_PYTHON_BUILD_VENV = "//:matter_build_venv"
Expand Down
6 changes: 4 additions & 2 deletions config/esp32/.gn
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@ default_args = {

pw_build_PIP_CONSTRAINTS =
[ "//third_party/connectedhomeip/scripts/setup/constraints.txt" ]
pw_build_PIP_REQUIREMENTS =
[ "//third_party/connectedhomeip/scripts/setup/requirements.txt" ]
pw_build_PIP_REQUIREMENTS = [
"//third_party/connectedhomeip/scripts/setup/requirements.build.txt",
"//third_party/connectedhomeip/scripts/setup/requirements.esp32.txt",
]

import("//args.gni")
}
3 changes: 2 additions & 1 deletion config/standalone/args.gni
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ import("//build_overrides/chip.gni")
chip_build_tests = false

pw_build_PIP_CONSTRAINTS = [ "${chip_root}/scripts/setup/constraints.txt" ]
pw_build_PIP_REQUIREMENTS = [ "${chip_root}/scripts/setup/requirements.txt" ]
pw_build_PIP_REQUIREMENTS =
[ "${chip_root}/scripts/setup/requirements.build.txt" ]
2 changes: 1 addition & 1 deletion examples/common/pigweed/rpc_console/.gn
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ default_args = {
pw_build_PIP_CONSTRAINTS =
[ "//third_party/connectedhomeip/scripts/setup/constraints.txt" ]
pw_build_PIP_REQUIREMENTS =
[ "//third_party/connectedhomeip/scripts/setup/requirements.txt" ]
[ "//third_party/connectedhomeip/scripts/setup/requirements.build.txt" ]

pw_build_USE_NEW_PYTHON_BUILD = true
}
57 changes: 55 additions & 2 deletions scripts/setup/bootstrap.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,38 @@
# limitations under the License.
#

_install_additional_pip_requirements() {
_SETUP_PLATFORM=$1
shift

# figure out additional pip install items
while [[ $# -gt 0 ]]; do
case $1 in
-p | --platform)
_SETUP_PLATFORM=$2
shift # argument
shift # value
;;
*)
shift
;;
esac
done

if ! [ -z "$_SETUP_PLATFORM" ]; then
IFS="," read -r -a _PLATFORMS <<<"$_SETUP_PLATFORM"
for platform in "${_PLATFORMS[@]}"; do
# Allow none as an alias of nothing extra installed (like -p none)
if [ "$platform" != "none" ]; then
echo "Installing pip requirements for $platform..."
pip install -q \
-r "$_CHIP_ROOT/scripts/setup/requirements.$platform.txt" \
-c "$_CHIP_ROOT/scripts/setup/constraints.txt"
fi
done
fi
}

_bootstrap_or_activate() {
if [ -n "$BASH" ]; then
local _BOOTSTRAP_PATH="${BASH_SOURCE[0]}"
Expand All @@ -24,7 +56,7 @@ _bootstrap_or_activate() {
local _BOOTSTRAP_NAME="${_BOOTSTRAP_PATH##*/}"
local _BOOTSTRAP_DIR="${_BOOTSTRAP_PATH%/*}"
# Strip off the 'scripts[/setup]' directory, leaving the root of the repo.
local _CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." && pwd)"
_CHIP_ROOT="$(cd "${_BOOTSTRAP_DIR%/setup}/.." && pwd)"

local _CONFIG_FILE="scripts/setup/environment.json"

Expand Down Expand Up @@ -91,21 +123,42 @@ EOF
--config-file "$_CHIP_ROOT/$_CONFIG_FILE" \
--virtualenv-gn-out-dir "$_PW_ACTUAL_ENVIRONMENT_ROOT/gn_out"
pw_finalize bootstrap "$_SETUP_SH"
_ACTION_TAKEN="bootstrap"
else
pw_activate
pw_finalize activate "$_SETUP_SH"
_ACTION_TAKEN="activate"
fi
}

# remember PW_ENVIRONMENT_ROOT so that things like another
# bootstrap or run_in_build_env.sh can be executed in a build env
_ORIGINAL_PW_ENVIRONMENT_ROOT="$PW_ENVIRONMENT_ROOT"

_bootstrap_or_activate "$0"

if [ "$_ACTION_TAKEN" = "bootstrap" ]; then
# By default, install all extra pip dependencies even if slow. -p/--platform
# arguments may override this default.
_install_additional_pip_requirements "all" "$@"
else
_install_additional_pip_requirements "none" "$@"
fi

unset -f _bootstrap_or_activate
unset -f _install_additional_pip_requirements

pw_cleanup

unset _ACTION_TAKEN
unset _CHIP_ROOT
unset PW_CIPD_INSTALL_DIR
unset CIPD_CACHE_DIR
unset _PW_BANNER_FUNC
unset _PW_TEXT
unset PW_DOCTOR_SKIP_CIPD_CHECKS

unset -f _chip_bootstrap_banner

if ! [ -z "$_ORIGINAL_PW_ENVIRONMENT_ROOT" ]; then
export PW_ENVIRONMENT_ROOT="$_ORIGINAL_PW_ENVIRONMENT_ROOT"
fi
2 changes: 1 addition & 1 deletion scripts/setup/constraints.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# This file is autogenerated by pip-compile with Python 3.10
# by the following command:
#
# pip-compile --output-file=constraints.txt --resolver=backtracking --strip-extras requirements.txt
# pip-compile --output-file=constraints.txt --resolver=backtracking --strip-extras requirements.all.txt
#
anytree==2.8.0
# via -r requirements.txt
Expand Down
2 changes: 1 addition & 1 deletion scripts/setup/environment.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"virtualenv": {
"gn_root": ".",
"gn_targets": [":python_packages.install"],
"requirements": ["scripts/setup/requirements.txt"],
"requirements": ["scripts/setup/requirements.build.txt"],
"constraints": ["scripts/setup/constraints.txt"],
"gn_args": ["chip_crypto=\"boringssl\""]
},
Expand Down
2 changes: 1 addition & 1 deletion scripts/setup/environment_no_cipd.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"virtualenv": {
"gn_root": ".",
"gn_targets": [":python_packages.install"],
"requirements": ["scripts/setup/requirements.txt"],
"requirements": ["scripts/setup/requirements.build.txt"],
"constraints": ["scripts/setup/constraints.txt"],
"gn_args": ["chip_crypto=\"boringssl\""]
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,16 @@ virtualenv
-c constraints.esp32.txt
-r requirements.esp32.txt

# mbed-os
-r requirements.mbed.txt

# bouffalolab
-r requirements.bouffalolab.txt

# openiotsdk
-r requirements.openiotsdk.txt

# Infineon
-r requirements.infineon.txt

# TI
-r requirements.ti.txt

# Telink
-r requirements.telink.txt

# cirque tests
requests>=2.24.0
-r requirements.zephyr.txt
-r requirements.cirque.txt
-r requirements.memory.txt
-r requirements.yaml_tests.txt

# device controller wheel package
wheel; sys_platform == 'linux'
Expand All @@ -40,17 +30,6 @@ pyobjc-framework-corebluetooth; sys_platform == 'darwin'
portpicker
mobly

# zephyr
west>=0.12.0
intelhex
cbor
cbor2
jsonschema

# happy tests
lockfile
psutil >= 5.7.3

# pigweed
ipython
appnope
Expand All @@ -63,13 +42,6 @@ mypy-protobuf==3.2.0
protobuf==3.20.1
types-protobuf==3.19.22

# scripts/tools/memory
anytree
cxxfilt
ghapi
pandas ; platform_machine != 'aarch64' and platform_machine != 'arm64'
tabulate

cryptography

# python unit tests
Expand All @@ -78,6 +50,3 @@ colorama
# update tornado for pw_watch
tornado

# YAML test harness
diskcache
websockets
1 change: 1 addition & 0 deletions scripts/setup/requirements.cirque.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
requests>=2.24.0
6 changes: 6 additions & 0 deletions scripts/setup/requirements.memory.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# scripts/tools/memory
anytree
cxxfilt
ghapi
pandas ; platform_machine != 'aarch64' and platform_machine != 'arm64'
tabulate
2 changes: 2 additions & 0 deletions scripts/setup/requirements.yaml_tests.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
diskcache
websockets
5 changes: 5 additions & 0 deletions scripts/setup/requirements.zephyr.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
west>=0.12.0
intelhex
cbor
cbor2
jsonschema
3 changes: 2 additions & 1 deletion scripts/tools/telink/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ tool.

```shell
cd path/to/connectedhomeip/scripts/tools/telink/
python3 -m pip install -r requirements.txt
python3 -m pip install -r requirements.build.txt
python3 -m pip install -r requirements.telink.txt
```

## Usage
Expand Down
2 changes: 1 addition & 1 deletion src/test_driver/efr32/.gn
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ default_args = {
pw_build_PIP_CONSTRAINTS =
[ "//third_party/connectedhomeip/scripts/setup/constraints.txt" ]
pw_build_PIP_REQUIREMENTS =
[ "//third_party/connectedhomeip/scripts/setup/requirements.txt" ]
[ "//third_party/connectedhomeip/scripts/setup/requirements.build.txt" ]

import("//args.gni")
}

0 comments on commit 1002892

Please sign in to comment.