Skip to content

Commit

Permalink
Remove costly python requirements on Raspberry Pi (#7635)
Browse files Browse the repository at this point in the history
It takes up to 1 hour to boostrap on Raspberry Pi, largely due to adding
"pandas" which is expensive to build and doesn't have a linux aarch64
wheel on pypi. Remove packages with no wheels from the requirements on
aarch64 to save time (pandas, gdbgui).

This switches the compiled requirements.txt to a "constraints.txt" which
is like a requirements file but only contains version constraints and
does not automatically install its contents. That way, we can specify
what version to use for packages on amd64 without forcing their
installation on the Pi. See pip help install for more details on this
feature.

This reduces initial Pi 4 Python bootstrap down to ~4m from ~50m.
  • Loading branch information
mspang authored and pull[bot] committed Jun 28, 2021
1 parent 6370469 commit 1634332
Show file tree
Hide file tree
Showing 5 changed files with 258 additions and 256 deletions.
1 change: 1 addition & 0 deletions .gn
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,5 @@ script_executable = "python3"

default_args = {
pw_unit_test_AUTOMATIC_RUNNER = "$dir_pigweed/targets/host/run_test"
pw_build_PIP_CONSTRAINTS = ["//scripts/constraints.txt"]
}
205 changes: 205 additions & 0 deletions scripts/constraints.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# pip-compile --output-file=constraints.txt requirements.txt
#
anytree==2.8.0
# via -r requirements.txt
appdirs==1.4.4
# via
# -r requirements.txt
# virtualenv
appnope==0.1.2
# via -r requirements.txt
backcall==0.2.0
# via ipython
bidict==0.21.2
# via python-socketio
bitstring==3.1.7
# via -r requirements.esp32.txt
brotli==1.0.9
# via flask-compress
certifi==2021.5.30
# via requests
cffi==1.14.5
# via cryptography
chardet==4.0.0
# via requests
click==8.0.1
# via
# -r requirements.esp32.txt
# flask
# pip-tools
colorama==0.4.4
# via west
coloredlogs==15.0.1
# via -r requirements.txt
cryptography==3.4.7
# via -r requirements.esp32.txt
cxxfilt==0.2.2
# via -r requirements.txt
dbus-python==1.2.16 ; sys_platform == "linux"
# via -r requirements.txt
decorator==5.0.9
# via ipython
distlib==0.3.2
# via virtualenv
docopt==0.6.2
# via pykwalify
ecdsa==0.17.0
# via -r requirements.esp32.txt
filelock==3.0.12
# via virtualenv
flask-compress==1.10.0
# via gdbgui
flask-socketio==2.9.6
# via gdbgui
flask==0.12.5
# via
# flask-compress
# flask-socketio
# gdbgui
future==0.18.2
# via
# -r requirements.esp32.txt
# mobly
gdbgui==0.13.2.0 ; platform_machine != "aarch64"
# via -r requirements.esp32.txt
gevent==1.5.0
# via gdbgui
greenlet==1.1.0
# via gevent
humanfriendly==9.2
# via coloredlogs
idna==2.10
# via requests
intelhex==2.3.0
# via -r requirements.txt
ipython-genutils==0.2.0
# via traitlets
ipython==7.24.1
# via -r requirements.txt
itsdangerous==2.0.1
# via flask
jedi==0.18.0
# via ipython
jinja2==3.0.1
# via flask
lockfile==0.12.2
# via -r requirements.txt
markupsafe==2.0.1
# via jinja2
matplotlib-inline==0.1.2
# via ipython
mobly==1.10.1
# via -r requirements.txt
numpy==1.20.3
# via pandas
packaging==20.9
# via west
pandas==1.2.4 ; platform_machine != "aarch64"
# via -r requirements.txt
parso==0.8.2
# via jedi
pep517==0.10.0
# via pip-tools
pexpect==4.8.0
# via ipython
pgi==0.0.11.2 ; sys_platform == "linux"
# via -r requirements.txt
pickleshare==0.7.5
# via ipython
pip-tools==6.1.0
# via -r requirements.txt
portpicker==1.4.0
# via
# -r requirements.txt
# mobly
prompt-toolkit==3.0.18
# via ipython
protobuf==3.17.3
# via -r requirements.txt
psutil==5.8.0
# via
# -r requirements.txt
# mobly
ptyprocess==0.7.0
# via pexpect
pycparser==2.20
# via cffi
pyelftools==0.27
# via -r requirements.esp32.txt
pygdbmi==0.9.0.2
# via
# -r requirements.esp32.txt
# gdbgui
pygments==2.9.0
# via
# gdbgui
# ipython
pykwalify==1.8.0
# via west
pyparsing==2.3.1
# via
# -r requirements.esp32.txt
# packaging
pyserial==3.5
# via
# -r requirements.esp32.txt
# mobly
python-dateutil==2.8.1
# via
# pandas
# pykwalify
python-engineio==4.2.0
# via python-socketio
python-socketio==5.3.0
# via flask-socketio
pytz==2021.1
# via pandas
pyyaml==5.4.1
# via
# mobly
# west
reedsolo==1.5.4
# via -r requirements.esp32.txt
requests==2.25.1
# via -r requirements.txt
ruamel.yaml.clib==0.2.2
# via ruamel.yaml
ruamel.yaml==0.17.9
# via pykwalify
six==1.16.0
# via
# anytree
# ecdsa
# protobuf
# python-dateutil
# virtualenv
timeout-decorator==0.5.0
# via mobly
toml==0.10.2
# via pep517
traitlets==5.0.5
# via
# ipython
# matplotlib-inline
urllib3==1.26.5
# via requests
virtualenv==20.4.7
# via -r requirements.txt
watchdog==2.1.2
# via -r requirements.txt
wcwidth==0.2.5
# via prompt-toolkit
werkzeug==0.16.1
# via flask
west==0.11.0
# via -r requirements.txt
wheel==0.36.2
# via -r requirements.txt

# The following packages are considered to be unsafe in a requirements file:
# pip
# setuptools
12 changes: 12 additions & 0 deletions scripts/requirements.esp32.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
setuptools>=21
click>=5.0
pyserial>=3.0
future>=0.15.2
cryptography>=2.1.4
pyparsing>=2.0.3,<2.4.0
pyelftools>=0.22
gdbgui==0.13.2.0 ; platform_machine != 'aarch64'
pygdbmi<=0.9.0.2
reedsolo>=1.5.3,<=1.5.4
bitstring>=3.1.6
ecdsa>=0.16.0
52 changes: 0 additions & 52 deletions scripts/requirements.in

This file was deleted.

Loading

0 comments on commit 1634332

Please sign in to comment.