-
Notifications
You must be signed in to change notification settings - Fork 0
/
mk_pyme_env.py
274 lines (232 loc) · 13.2 KB
/
mk_pyme_env.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
import condacmds as cmds
from condacmds import download_pyme_extra, download_pyme, build_pyme_extra, build_pyme, pyme_extra_install_plugins
from pathlib import Path
import logging
# central package list
Packages = {
'with_pyme_depends' : {
# the initial matplotlib pinning should ensure we do not get a too recent version
'packages' : 'matplotlib<=3.6 pyme-depends'.split()
},
'no_pyme_depends' : {
'packagelists_mac' : {
'conda' : [
['setuptools<=73'], # setuptools 74.x triggers issue https://github.com/numpy/numpy/issues/27405 on mac
# start off with numpy/scipy
# the "libblas=*=*accelerate" arguments according to a number of sites, e.g.
# - https://github.com/joblib/threadpoolctl/issues/135
# - https://github.com/conda-forge/numpy-feedstock/issues/303
# note docs on blas selection: https://conda-forge.org/docs/maintainer/knowledge_base.html#switching-blas-implementation
# possible options
# conda install "libblas=*=*mkl"
# conda install "libblas=*=*openblas"
# conda install "libblas=*=*blis"
# conda install "libblas=*=*accelerate"
# conda install "libblas=*=*netlib"
'scipy numpy "libblas=*=*accelerate"'.split(),
# next the main other dependecies
('$matplotlib$ pytables pyopengl jinja2 cython pip requests pyyaml' +
' psutil pandas scikit-image scikit-learn sphinx toposort pybind11').split(),
'traits traitsui pyface'.split(),
'pyfftw zeroconf python.app'.split(),
['ujson'], # ujson for ClusterOfOne
['wxpython'],
],
#'pip': ['wxpython'] # recently (Sep 24), pip installs of wx on mac seem to be broken; by contrast, conda-forge builds seem ok
'pip': ['pymecompress'] # IO of certain h5's seems to require pymecompress; to build with recent python needs pip install
},
'packagelists_win' : {
'conda': [
['"setuptools<=73"'], # setuptools 74.x triggers issue https://github.com/numpy/numpy/issues/27405 on win, too!
'scipy numpy'.split(), # here we should have some suitably fast installation by default but may want to check
'"$matplotlib$" pytables pyopengl jinja2 cython pip requests pyyaml'.split(),
'psutil pandas scikit-image scikit-learn sphinx toposort pybind11'.split(),
'traits traitsui pyface'.split(),
'pyfftw zeroconf pywin32'.split(),
['pymecompress','ujson'], # IO of certain h5's seems to require pymecompress; ujson for ClusterOfOne
# pyserial?
['wxpython'], # let's check if wxpython installs ok from conda-forge
],
#'pip': ['wxpython']
'pip': []
}
}
}
# 0. some basic setup/parameter choices via command line arguments
import sys
commandline = " ".join(sys.argv)
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--python',default='3.9',
help='specify the python version for the new environment')
parser.add_argument('--buildstem',default='build-test',
help='stem for the name of the build directory')
parser.add_argument('--suffix',default=None,
help='suffix appended to default environment and build_dir')
parser.add_argument('-c','--condacmd',default='conda',choices=['conda','mamba'],
help='conda command, should be one of conda or mamba')
parser.add_argument('-e','--environment',default=None,
help='name for the new environment, autogenerated from other params by default')
parser.add_argument('--recipes',action="store_true",
help='install the included customrecipes into the PYME config directory')
parser.add_argument('--pyme-repo',default='python-microscopy/python-microscopy',
help='github repository name of python-microscopy; defaults to python-microscopy/python-microscopy')
parser.add_argument('--pyme-branch',default='master',
help='branch of pyme to use in build; defaults to master')
parser.add_argument('--pymex-repo',default='csoeller/PYME-extra',
help='github repository name of PYME-extra; defaults to csoeller/PYME-extra')
parser.add_argument('--pymex-branch',default='master',
help='branch of PYME-extra to use in build; defaults to master')
parser.add_argument('--no-pymex',action="store_true",
help='omit downloading and installing PYME-extra')
parser.add_argument('--no-pyme-depends',action="store_true",
help='install from package list rather than using pyme-depends')
parser.add_argument('--use-git',action="store_true",
help='clone git repo locally rather than just downloading snapshot')
parser.add_argument('--no-strict-channel',action="store_true",
help='enforce strict adherance to conda-forge channel')
parser.add_argument('--dry-run',action="store_true",
help='just process options but do not run any commands')
parser.add_argument('-x','--xtra-packages', action="extend", nargs="+", type=str,
help='extra packages to install into the new environment')
### Note
### we may want to add, will enforce from conda-forge over default channel (I think)
# conda config --set channel_priority strict
# we should be able to provide this as an option, according to https://docs.conda.io/projects/conda/en/latest/commands/install.html:
# --strict-channel-priority
### Check how to check, verify and unset
# other possible arguments to enable
# --pyme-branch: pick a branch other than the default 'master' branch
# --pyme-extra-branch: pick a branch other than the default 'master' branch
# --build_dir: specify full build directory name (e.g. for a more permanent install)
# potentially add jupyter notebook option
# potentially add R option (+R notebooks?)
# potentially allow cloning the whole github repos with a command line switch
args = parser.parse_args()
cmds.check_condaenv('base') # check we are running in the base environment
cmds.check_yaml_installed() # check yaml is available
# # this makes methods/attributes for the standard parameters available
# # also does any setup stuff, e.g. create build_dir, setup logging etc
pbld = cmds.PymeBuild(pythonver=args.python,
build_dir=args.buildstem,
condacmd=args.condacmd,
environment=args.environment,
with_pyme_depends=not args.no_pyme_depends,
with_pymex=not args.no_pymex,
with_recipes=args.recipes,
pyme_repo=args.pyme_repo, pyme_branch=args.pyme_branch,
pymex_repo=args.pymex_repo, pymex_branch=args.pymex_branch,
use_git=args.use_git,suffix=args.suffix,
strict_conda_forge_channel=not args.no_strict_channel,
dry_run=args.dry_run,xtra_packages=args.xtra_packages,
)
environment = pbld.env
build_dir = pbld.build_dir
logging.info("Command called as\n")
logging.info(commandline + "\n")
if args.dry_run:
logging.info("dry run, aborting...")
import sys
sys.exit(0)
# some checks if some of the required packages are available
try:
# should be available in the base install; otherwise we may need "# conda/pip install packaging"
from packaging import version
except ImportError:
raise RuntimeError("need to install module 'packaging'; e.g. 'conda/pip install packaging'")
if pbld.use_git:
try:
import git
except ImportError:
raise RuntimeError("requesting git install but git (from GitPython) could not be imported; check if GitPython is installed (including git executable)")
# 1. make test environment
envs = cmds.conda_envs()
if environment not in envs:
cc = cmds.conda_create(environment, pbld.pythonver, channels=['conda-forge'])
logging.info(cc)
if pbld.strict_conda_forge_channel:
cc = cmds.run_cmd_in_environment('conda config --env --set channel_priority strict',environment,check=True)
logging.info(cc)
cc = cmds.run_cmd_in_environment('conda config --env --add channels conda-forge',environment,check=True)
logging.info(cc)
cc = cmds.run_cmd_in_environment('conda config --env --add channels david_baddeley',environment,check=True)
logging.info(cc)
else:
print('environment %s already exists' % environment)
answer = input("Continue?")
if answer.lower() not in ["y","yes"]:
print("aborting...")
import sys
sys.exit(0)
# just a quick check that we get the expected python version and can invoke it ok
cc = cmds.run_cmd_in_environment('python -V',environment,check=True)
logging.info("got python version info: %s" % cc)
# 2. build/install pyme and dependencies
import platform
prepy3_10 = version.parse(pbld.pythonver) < version.parse("3.10")
postpy3_10 = version.parse(pbld.pythonver) > version.parse("3.10")
if not prepy3_10: # we are now testing if latest PYME can deal with more recent matplotlib
matplotlib = 'matplotlib<=3.8'
else:
matplotlib = 'matplotlib<=3.6'
if platform.machine() != 'arm64' and prepy3_10 and pbld.with_pyme_depends:
packages = Packages['with_pyme_depends']['packages']
result = cmds.conda_install(environment, packages, channels = ['conda-forge','david_baddeley'])
logging.info(result)
else:
# install required packages from conda-forge and/or via pip
# currently we only check for mac and otherwise assume windows; this clearly ignores linux - should be extended when needed
if platform.system() == 'Darwin': # now selected for all macs
package_sets = Packages['no_pyme_depends']['packagelists_mac']['conda']
else:
package_sets = Packages['no_pyme_depends']['packagelists_win']['conda']
for packages in package_sets:
packages_expanded = [pack.replace('$matplotlib$',matplotlib) for pack in packages]
result = cmds.conda_install(environment, packages_expanded, channels = ['conda-forge'])
logging.info(result)
if platform.system() == 'Darwin': # now selected for all macs
# next the main other dependecies
pip_package_set = Packages['no_pyme_depends']['packagelists_mac']['pip']
else:
pip_package_set = Packages['no_pyme_depends']['packagelists_win']['pip']
# now pip install packages that seem to need a pip install (always subject to checks if really required)
if len(pip_package_set)>0:
result = cmds.pip_install(environment, pip_package_set)
logging.info(result)
######################################
# note that we can get a windows error:
# error: could not create 'build\temp.win-amd64-cpython-310\Release\Users\soeller\src\PYME-test-env\b-py3.10-mamba\python-microscopy-python-310-compat\PYME\Analysis\points\traveling_salesperson':
# The filename or extension is too long
# silly solution: move PYME-test-env repo close to root of disk and abbreviate build_directory name
#### There must be a better solution!!
if pbld.use_git:
download_mode = 'git'
else:
download_mode = 'snapshot'
download_pyme(build_dir=build_dir,repo=args.pyme_repo,branch=args.pyme_branch,mode=download_mode)
build_pyme(environment,build_dir=build_dir,repo=args.pyme_repo,branch=args.pyme_branch)
# this should fail if our PYME install failed
result = cmds.run_cmd_in_environment('python -c "import PYME.version; print(PYME.version.version)"',environment,check=True)
logging.info("Got PYME version %s" % result)
# 3. build/install pyme-extra
if pbld.with_pymex:
# pyme-extra dependencies
pymex_conda_packages = 'statsmodels roifile colorcet'.split()
# circle-fit is not available in a recent enough version via conda-forge
pymex_pip_packages = 'circle-fit'.split()
result = cmds.conda_install(environment, pymex_conda_packages, channels = ['conda-forge'])
logging.info(result)
result = cmds.pip_install(environment, pymex_pip_packages)
logging.info(result)
download_pyme_extra(build_dir=build_dir,repo=args.pymex_repo,branch=args.pymex_branch,mode=download_mode)
build_pyme_extra(environment,build_dir=build_dir,repo=args.pymex_repo,branch=args.pymex_branch)
pyme_extra_install_plugins(environment,build_dir=build_dir,repo=args.pymex_repo,branch=args.pymex_branch)
# 4. some custom recipes for some ease in a testing environment
if pbld.with_recipes:
output = cmds.run_cmd_in_environment('python install_config_files.py',environment)
logging.info(output)
# 5. any extra packages although in future this will be preferable via mk_extra_stuff
if pbld.xtra_packages is not None and len(pbld.xtra_packages) > 0:
result = cmds.conda_install(environment, pbld.xtra_packages, channels = ['conda-forge'])
logging.info(result)
# potentially here: test for succesfull pyme-extra install