From 1634332da1a6635b83fd92e71593d6b57acde3fa Mon Sep 17 00:00:00 2001 From: Michael Spang Date: Tue, 15 Jun 2021 16:36:35 -0400 Subject: [PATCH] Remove costly python requirements on Raspberry Pi (#7635) It takes up to 1 hour to boostrap on Raspberry Pi, largely due to adding "pandas" which is expensive to build and doesn't have a linux aarch64 wheel on pypi. Remove packages with no wheels from the requirements on aarch64 to save time (pandas, gdbgui). This switches the compiled requirements.txt to a "constraints.txt" which is like a requirements file but only contains version constraints and does not automatically install its contents. That way, we can specify what version to use for packages on amd64 without forcing their installation on the Pi. See pip help install for more details on this feature. This reduces initial Pi 4 Python bootstrap down to ~4m from ~50m. --- .gn | 1 + scripts/constraints.txt | 205 +++++++++++++++++++++++++++ scripts/requirements.esp32.txt | 12 ++ scripts/requirements.in | 52 ------- scripts/requirements.txt | 244 ++++++--------------------------- 5 files changed, 258 insertions(+), 256 deletions(-) create mode 100644 scripts/constraints.txt create mode 100644 scripts/requirements.esp32.txt delete mode 100644 scripts/requirements.in diff --git a/.gn b/.gn index e3a67f12cd948b..6df7f3cf536489 100644 --- a/.gn +++ b/.gn @@ -25,4 +25,5 @@ script_executable = "python3" default_args = { pw_unit_test_AUTOMATIC_RUNNER = "$dir_pigweed/targets/host/run_test" + pw_build_PIP_CONSTRAINTS = ["//scripts/constraints.txt"] } diff --git a/scripts/constraints.txt b/scripts/constraints.txt new file mode 100644 index 00000000000000..5509c399317bbe --- /dev/null +++ b/scripts/constraints.txt @@ -0,0 +1,205 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=constraints.txt requirements.txt +# +anytree==2.8.0 + # via -r requirements.txt +appdirs==1.4.4 + # via + # -r requirements.txt + # virtualenv +appnope==0.1.2 + # via -r requirements.txt +backcall==0.2.0 + # via ipython +bidict==0.21.2 + # via python-socketio +bitstring==3.1.7 + # via -r requirements.esp32.txt +brotli==1.0.9 + # via flask-compress +certifi==2021.5.30 + # via requests +cffi==1.14.5 + # via cryptography +chardet==4.0.0 + # via requests +click==8.0.1 + # via + # -r requirements.esp32.txt + # flask + # pip-tools +colorama==0.4.4 + # via west +coloredlogs==15.0.1 + # via -r requirements.txt +cryptography==3.4.7 + # via -r requirements.esp32.txt +cxxfilt==0.2.2 + # via -r requirements.txt +dbus-python==1.2.16 ; sys_platform == "linux" + # via -r requirements.txt +decorator==5.0.9 + # via ipython +distlib==0.3.2 + # via virtualenv +docopt==0.6.2 + # via pykwalify +ecdsa==0.17.0 + # via -r requirements.esp32.txt +filelock==3.0.12 + # via virtualenv +flask-compress==1.10.0 + # via gdbgui +flask-socketio==2.9.6 + # via gdbgui +flask==0.12.5 + # via + # flask-compress + # flask-socketio + # gdbgui +future==0.18.2 + # via + # -r requirements.esp32.txt + # mobly +gdbgui==0.13.2.0 ; platform_machine != "aarch64" + # via -r requirements.esp32.txt +gevent==1.5.0 + # via gdbgui +greenlet==1.1.0 + # via gevent +humanfriendly==9.2 + # via coloredlogs +idna==2.10 + # via requests +intelhex==2.3.0 + # via -r requirements.txt +ipython-genutils==0.2.0 + # via traitlets +ipython==7.24.1 + # via -r requirements.txt +itsdangerous==2.0.1 + # via flask +jedi==0.18.0 + # via ipython +jinja2==3.0.1 + # via flask +lockfile==0.12.2 + # via -r requirements.txt +markupsafe==2.0.1 + # via jinja2 +matplotlib-inline==0.1.2 + # via ipython +mobly==1.10.1 + # via -r requirements.txt +numpy==1.20.3 + # via pandas +packaging==20.9 + # via west +pandas==1.2.4 ; platform_machine != "aarch64" + # via -r requirements.txt +parso==0.8.2 + # via jedi +pep517==0.10.0 + # via pip-tools +pexpect==4.8.0 + # via ipython +pgi==0.0.11.2 ; sys_platform == "linux" + # via -r requirements.txt +pickleshare==0.7.5 + # via ipython +pip-tools==6.1.0 + # via -r requirements.txt +portpicker==1.4.0 + # via + # -r requirements.txt + # mobly +prompt-toolkit==3.0.18 + # via ipython +protobuf==3.17.3 + # via -r requirements.txt +psutil==5.8.0 + # via + # -r requirements.txt + # mobly +ptyprocess==0.7.0 + # via pexpect +pycparser==2.20 + # via cffi +pyelftools==0.27 + # via -r requirements.esp32.txt +pygdbmi==0.9.0.2 + # via + # -r requirements.esp32.txt + # gdbgui +pygments==2.9.0 + # via + # gdbgui + # ipython +pykwalify==1.8.0 + # via west +pyparsing==2.3.1 + # via + # -r requirements.esp32.txt + # packaging +pyserial==3.5 + # via + # -r requirements.esp32.txt + # mobly +python-dateutil==2.8.1 + # via + # pandas + # pykwalify +python-engineio==4.2.0 + # via python-socketio +python-socketio==5.3.0 + # via flask-socketio +pytz==2021.1 + # via pandas +pyyaml==5.4.1 + # via + # mobly + # west +reedsolo==1.5.4 + # via -r requirements.esp32.txt +requests==2.25.1 + # via -r requirements.txt +ruamel.yaml.clib==0.2.2 + # via ruamel.yaml +ruamel.yaml==0.17.9 + # via pykwalify +six==1.16.0 + # via + # anytree + # ecdsa + # protobuf + # python-dateutil + # virtualenv +timeout-decorator==0.5.0 + # via mobly +toml==0.10.2 + # via pep517 +traitlets==5.0.5 + # via + # ipython + # matplotlib-inline +urllib3==1.26.5 + # via requests +virtualenv==20.4.7 + # via -r requirements.txt +watchdog==2.1.2 + # via -r requirements.txt +wcwidth==0.2.5 + # via prompt-toolkit +werkzeug==0.16.1 + # via flask +west==0.11.0 + # via -r requirements.txt +wheel==0.36.2 + # via -r requirements.txt + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/scripts/requirements.esp32.txt b/scripts/requirements.esp32.txt new file mode 100644 index 00000000000000..086fcc35550cb8 --- /dev/null +++ b/scripts/requirements.esp32.txt @@ -0,0 +1,12 @@ +setuptools>=21 +click>=5.0 +pyserial>=3.0 +future>=0.15.2 +cryptography>=2.1.4 +pyparsing>=2.0.3,<2.4.0 +pyelftools>=0.22 +gdbgui==0.13.2.0 ; platform_machine != 'aarch64' +pygdbmi<=0.9.0.2 +reedsolo>=1.5.3,<=1.5.4 +bitstring>=3.1.6 +ecdsa>=0.16.0 diff --git a/scripts/requirements.in b/scripts/requirements.in deleted file mode 100644 index 8f362abf52853a..00000000000000 --- a/scripts/requirements.in +++ /dev/null @@ -1,52 +0,0 @@ -pip-tools -virtualenv - -# esp-idf -setuptools>=21 -click>=5.0 -pyserial>=3.0 -future>=0.15.2 -cryptography>=2.1.4 -pyparsing>=2.0.3,<2.4.0 -pyelftools>=0.22 -gdbgui==0.13.2.0 -pygdbmi<=0.9.0.2 -reedsolo>=1.5.3,<=1.5.4 -bitstring>=3.1.6 -ecdsa>=0.16.0 - -# cirque tests -requests>=2.24.0 - -# device controller wheel package -wheel -dbus-python; sys_platform == 'linux' -pgi; sys_platform == 'linux' -pyobjc-core; sys_platform == 'darwin' -pyobjc-framework-cocoa; sys_platform == 'darwin' -pyobjc-framework-corebluetooth; sys_platform == 'darwin' - -# mobly tests -portpicker -mobly - -# zephyr -west>=0.11.0 -intelhex - -# happy tests -lockfile -psutil >= 5.7.3 - -# pigweed -ipython -appnope -appdirs -coloredlogs -watchdog -protobuf - -# scripts/tools/memory -anytree -cxxfilt -pandas diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 694e4e83d845c6..97ce36131da5f6 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,205 +1,41 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile requirements.in -# -anytree==2.8.0 - # via -r requirements.in -appdirs==1.4.4 - # via - # -r requirements.in - # virtualenv -appnope==0.1.2 - # via -r requirements.in -backcall==0.2.0 - # via ipython -bidict==0.21.2 - # via python-socketio -bitstring==3.1.7 - # via -r requirements.in -brotli==1.0.9 - # via flask-compress -certifi==2021.5.30 - # via requests -cffi==1.14.5 - # via cryptography -chardet==4.0.0 - # via requests -click==8.0.1 - # via - # -r requirements.in - # flask - # pip-tools -colorama==0.4.4 - # via west -coloredlogs==15.0.1 - # via -r requirements.in -cryptography==3.4.7 - # via -r requirements.in -cxxfilt==0.2.2 - # via -r requirements.in -dbus-python==1.2.16 ; sys_platform == "linux" - # via -r requirements.in -decorator==5.0.9 - # via ipython -distlib==0.3.2 - # via virtualenv -docopt==0.6.2 - # via pykwalify -ecdsa==0.17.0 - # via -r requirements.in -filelock==3.0.12 - # via virtualenv -flask-compress==1.9.0 - # via gdbgui -flask-socketio==2.9.6 - # via gdbgui -flask==0.12.5 - # via - # flask-compress - # flask-socketio - # gdbgui -future==0.18.2 - # via - # -r requirements.in - # mobly -gdbgui==0.13.2.0 - # via -r requirements.in -gevent==1.5.0 - # via gdbgui -greenlet==1.1.0 - # via gevent -humanfriendly==9.2 - # via coloredlogs -idna==2.10 - # via requests -intelhex==2.3.0 - # via -r requirements.in -ipython-genutils==0.2.0 - # via traitlets -ipython==7.24.1 - # via -r requirements.in -itsdangerous==2.0.1 - # via flask -jedi==0.18.0 - # via ipython -jinja2==3.0.1 - # via flask -lockfile==0.12.2 - # via -r requirements.in -markupsafe==2.0.1 - # via jinja2 -matplotlib-inline==0.1.2 - # via ipython -mobly==1.10.1 - # via -r requirements.in -numpy==1.20.3 - # via pandas -packaging==20.9 - # via west -pandas==1.2.4 - # via -r requirements.in -parso==0.8.2 - # via jedi -pep517==0.10.0 - # via pip-tools -pexpect==4.8.0 - # via ipython -pgi==0.0.11.2 ; sys_platform == "linux" - # via -r requirements.in -pickleshare==0.7.5 - # via ipython -pip-tools==6.1.0 - # via -r requirements.in -portpicker==1.4.0 - # via - # -r requirements.in - # mobly -prompt-toolkit==3.0.18 - # via ipython -protobuf==3.17.3 - # via -r requirements.in -psutil==5.8.0 - # via - # -r requirements.in - # mobly -ptyprocess==0.7.0 - # via pexpect -pycparser==2.20 - # via cffi -pyelftools==0.27 - # via -r requirements.in -pygdbmi==0.9.0.2 - # via - # -r requirements.in - # gdbgui -pygments==2.9.0 - # via - # gdbgui - # ipython -pykwalify==1.8.0 - # via west -pyparsing==2.3.1 - # via - # -r requirements.in - # packaging -pyserial==3.5 - # via - # -r requirements.in - # mobly -python-dateutil==2.8.1 - # via - # pandas - # pykwalify -python-engineio==4.2.0 - # via python-socketio -python-socketio==5.3.0 - # via flask-socketio -pytz==2021.1 - # via pandas -pyyaml==5.4.1 - # via - # mobly - # west -reedsolo==1.5.4 - # via -r requirements.in -requests==2.25.1 - # via -r requirements.in -ruamel.yaml.clib==0.2.2 - # via ruamel.yaml -ruamel.yaml==0.17.9 - # via pykwalify -six==1.16.0 - # via - # anytree - # ecdsa - # protobuf - # python-dateutil - # virtualenv -timeout-decorator==0.5.0 - # via mobly -toml==0.10.2 - # via pep517 -traitlets==5.0.5 - # via - # ipython - # matplotlib-inline -urllib3==1.26.5 - # via requests -virtualenv==20.4.7 - # via -r requirements.in -watchdog==2.1.2 - # via -r requirements.in -wcwidth==0.2.5 - # via prompt-toolkit -werkzeug==0.16.1 - # via flask -west==0.11.0 - # via -r requirements.in -wheel==0.36.2 - # via -r requirements.in +pip-tools +virtualenv -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools +# esp-idf +-r requirements.esp32.txt ; platform_machine != 'aarch64' + +# cirque tests +requests>=2.24.0 + +# device controller wheel package +wheel +dbus-python; sys_platform == 'linux' +pgi; sys_platform == 'linux' +pyobjc-core; sys_platform == 'darwin' +pyobjc-framework-cocoa; sys_platform == 'darwin' +pyobjc-framework-corebluetooth; sys_platform == 'darwin' + +# mobly tests +portpicker +mobly + +# zephyr +west>=0.11.0 +intelhex + +# happy tests +lockfile +psutil >= 5.7.3 + +# pigweed +ipython +appnope +appdirs +coloredlogs +watchdog +protobuf + +# scripts/tools/memory +anytree +cxxfilt +pandas ; platform_machine != 'aarch64'