BLD: Setup meson builds #49115
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| # This file is adapted from https://github.com/scipy/scipy/blob/main/meson.build | ||
| project( | ||
| 'pandas', | ||
| 'c', 'cpp', 'cython', | ||
| version: '1.6.0.dev0', | ||
| license: 'BSD-3', | ||
| # TODO: bump when meson 0.64.0 comes out, | ||
| # we are relying on 0.64.0 features | ||
| meson_version: '>=0.63', | ||
| default_options: [ | ||
| 'buildtype=release', | ||
| 'c_std=c99' | ||
| ] | ||
| ) | ||
|
|
||
| py_mod = import('python') | ||
| fs = import('fs') | ||
| py = py_mod.find_installation('python') | ||
| py_dep = py.dependency() | ||
| tempita = files('generate_pxi.py') | ||
| versioneer = files('generate_version.py') | ||
|
|
||
| if fs.exists('_version_meson.py') | ||
| py.install_sources('_version_meson.py', subdir: 'pandas') | ||
| else | ||
| custom_target('write_version_file', | ||
| output: '_version_meson.py', | ||
| command: [ | ||
| py, versioneer, '-o', '@OUTPUT@' | ||
| ], | ||
| build_by_default: true, | ||
| build_always_stale: true, | ||
| install: true, | ||
| install_dir: py.get_install_dir(pure: false) / 'pandas' | ||
| ) | ||
| meson.add_dist_script(py, versioneer, '-o', '_version_meson.py') | ||
| endif | ||
|
|
||
| subdir('pandas') | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,165 @@ | ||
| _algos_take_helper = custom_target('algos_take_helper_pxi', | ||
| output: 'algos_take_helper.pxi', | ||
| input: 'algos_take_helper.pxi.in', | ||
| command: [ | ||
| py, tempita, '@INPUT@', '-o', '@OUTDIR@' | ||
| ], | ||
| # TODO: remove these two below lines | ||
| # Weird bug in meson that only repros on my potato computer | ||
| # (possibly b/c of low number of threads)? | ||
| # The first custom_target is never built for some reason | ||
| # so algos.pyx will error out later in the build. | ||
| build_by_default: true, | ||
| build_always_stale: true | ||
| ) | ||
| _algos_common_helper = custom_target('algos_common_helper_pxi', | ||
| output: 'algos_common_helper.pxi', | ||
| input: 'algos_common_helper.pxi.in', | ||
| command: [ | ||
| py, tempita, '@INPUT@', '-o', '@OUTDIR@' | ||
| ] | ||
| ) | ||
| _khash_primitive_helper = custom_target('khash_primitive_helper_pxi', | ||
| output: 'khash_for_primitive_helper.pxi', | ||
| input: 'khash_for_primitive_helper.pxi.in', | ||
| command: [ | ||
| py, tempita, '@INPUT@', '-o', '@OUTDIR@' | ||
| ] | ||
| ) | ||
| _hashtable_class_helper = custom_target('hashtable_class_helper_pxi', | ||
| output: 'hashtable_class_helper.pxi', | ||
| input: 'hashtable_class_helper.pxi.in', | ||
| command: [ | ||
| py, tempita, '@INPUT@', '-o', '@OUTDIR@' | ||
| ] | ||
| ) | ||
| _hashtable_func_helper = custom_target('hashtable_func_helper_pxi', | ||
| output: 'hashtable_func_helper.pxi', | ||
| input: 'hashtable_func_helper.pxi.in', | ||
| command: [ | ||
| py, tempita, '@INPUT@', '-o', '@OUTDIR@' | ||
| ] | ||
| ) | ||
| _index_class_helper = custom_target('index_class_helper_pxi', | ||
| output: 'index_class_helper.pxi', | ||
| input: 'index_class_helper.pxi.in', | ||
| command: [ | ||
| py, tempita, '@INPUT@', '-o', '@OUTDIR@' | ||
| ] | ||
| ) | ||
| _sparse_op_helper = custom_target('sparse_op_helper_pxi', | ||
| output: 'sparse_op_helper.pxi', | ||
| input: 'sparse_op_helper.pxi.in', | ||
| command: [ | ||
| py, tempita, '@INPUT@', '-o', '@OUTDIR@' | ||
| ] | ||
| ) | ||
| _intervaltree_helper = custom_target('intervaltree_helper_pxi', | ||
| output: 'intervaltree.pxi', | ||
| input: 'intervaltree.pxi.in', | ||
| command: [ | ||
| py, tempita, '@INPUT@', '-o', '@OUTDIR@' | ||
| ] | ||
| ) | ||
| _khash_primitive_helper_dep = declare_dependency(sources: _khash_primitive_helper) | ||
| # TODO: can this be removed, I wish meson copied .pyx source to the build dir automatically | ||
| # The reason we can't build the pyx files inplace and copy to build dir is because | ||
| # the generated pxi files cannot be written to the source directory. | ||
| # (Meson only supports out of tree builds) | ||
| cython_sources_list = [ | ||
| # List of cython sources e.g. .pyx, .pxd & __init__.py | ||
| # Does NOT include .pxi.in | ||
| '__init__.py', | ||
| 'algos.pxd', | ||
| 'algos.pyx', | ||
| 'arrays.pxd', | ||
| 'dtypes.pxd', | ||
| 'hashtable.pxd', | ||
| 'hashtable.pyx', | ||
| 'index.pyx', | ||
| 'indexing.pyx', | ||
| 'internals.pyx', | ||
| 'interval.pyx', | ||
| 'join.pyx', | ||
| 'khash.pxd', | ||
| 'lib.pxd', | ||
| 'missing.pxd', | ||
| 'parsers.pyx', | ||
| 'sparse.pyx', | ||
| 'testing.pyx', | ||
| 'tslib.pyx', | ||
| 'util.pxd', | ||
| ] | ||
| cython_sources = {} | ||
|
|
||
| foreach source: cython_sources_list | ||
| source_pyx = fs.copyfile(source) | ||
| cython_sources += {source: source_pyx} | ||
| endforeach | ||
|
|
||
| subdir('tslibs') | ||
|
|
||
| libs_sources = { | ||
| # Dict of extension name -> dict of {sources, include_dirs, and deps} | ||
| # numpy include dir is implicitly included | ||
| 'algos': {'sources': [cython_sources['algos.pyx'], _algos_common_helper, _algos_take_helper, _khash_primitive_helper], | ||
| 'include_dirs': klib_include}, | ||
WillAyd marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| 'arrays': {'sources': ['arrays.pyx']}, | ||
| 'groupby': {'sources': ['groupby.pyx']}, | ||
| 'hashing': {'sources': ['hashing.pyx']}, | ||
| 'hashtable': {'sources': [cython_sources['hashtable.pyx'], _khash_primitive_helper, _hashtable_class_helper, _hashtable_func_helper], | ||
| 'include_dirs': klib_include}, | ||
| 'index': {'sources': [cython_sources['index.pyx'], _index_class_helper], | ||
| 'include_dirs': [klib_include, 'tslibs']}, | ||
| 'indexing': {'sources': ['indexing.pyx']}, | ||
| 'internals': {'sources': ['internals.pyx']}, | ||
| 'interval': {'sources': [cython_sources['interval.pyx'], _intervaltree_helper], | ||
| 'include_dirs': [klib_include, 'tslibs']}, | ||
| 'join': {'sources': [cython_sources['join.pyx'], _khash_primitive_helper], | ||
| 'include_dirs': klib_include, | ||
| 'deps': _khash_primitive_helper_dep}, | ||
| 'lib': {'sources': ['lib.pyx', 'src/parser/tokenizer.c'], | ||
| 'include_dirs': [klib_include, inc_datetime]}, | ||
| 'missing': {'sources': ['missing.pyx'], | ||
| 'include_dirs': [inc_datetime]}, | ||
| 'parsers': {'sources': [cython_sources['parsers.pyx'], 'src/parser/tokenizer.c', 'src/parser/io.c'], | ||
| 'include_dirs': [klib_include, 'src'], | ||
| 'deps': _khash_primitive_helper_dep}, | ||
| 'json': {'sources': ['src/ujson/python/ujson.c', | ||
| 'src/ujson/python/objToJSON.c', | ||
| 'src/ujson/python/date_conversions.c', | ||
| 'src/ujson/python/JSONtoObj.c', | ||
| 'src/ujson/lib/ultrajsonenc.c', | ||
| 'src/ujson/lib/ultrajsondec.c', | ||
| 'tslibs/src/datetime/np_datetime.c', | ||
| 'tslibs/src/datetime/np_datetime_strings.c'], | ||
| 'include_dirs': [inc_datetime, 'src/ujson/lib', 'src/ujson/python']}, | ||
| 'reduction': {'sources': ['reduction.pyx']}, | ||
| 'ops': {'sources': ['ops.pyx']}, | ||
| 'ops_dispatch': {'sources': ['ops_dispatch.pyx']}, | ||
| 'properties': {'sources': ['properties.pyx']}, | ||
| 'reshape': {'sources': ['reshape.pyx']}, | ||
| 'sparse': {'sources': [cython_sources['sparse.pyx'], _sparse_op_helper]}, | ||
| 'tslib': {'sources': ['tslib.pyx', 'tslibs/src/datetime/np_datetime.c'], | ||
| 'include_dirs': inc_datetime}, | ||
| 'testing': {'sources': ['testing.pyx']}, | ||
| 'writers': {'sources': ['writers.pyx']} | ||
| } | ||
|
|
||
|
|
||
| foreach ext_name, ext_dict : libs_sources | ||
| py.extension_module( | ||
| ext_name, | ||
| ext_dict.get('sources'), | ||
| include_directories: [inc_np] + ext_dict.get('include_dirs', ''), | ||
| dependencies: ext_dict.get('deps', ''), | ||
| subdir: 'pandas/_libs', | ||
| install: true | ||
| ) | ||
| endforeach | ||
|
|
||
| py.install_sources('__init__.py', | ||
| pure: false, | ||
| subdir: 'pandas/_libs') | ||
|
|
||
| subdir('window') | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| # TODO: can this be removed, I wish meson copied .pyx source to the build dir automatically | ||
|
There was a problem hiding this comment. Is this copy actually required? Seems to somewhat blur the lines of the purpose of an in source vs out of source build if we have to copy the source over There was a problem hiding this comment. AFAICT the ones that are passed to the extension_module() definitions below are the original sources, so it's not clear to me either, what this is doing. In general I would not expect this to be needed unless somehow cython is attempting to automatically figure out information traversing multiple files. For example includes. But there do not seem to be any generated sources in this directory anyway? There was a problem hiding this comment. The .pxi files are generated in the build directory, but the other sources are in the source directory. I don't think cython has an option to pass an include directory for .pxi files, so we have to copy the source files that depend on the .pxi files over. If a source doesn't have a dependency on a .pxi file, we don't copy it. There was a problem hiding this comment. What confused me mostly is that you don't seem to be using the copied .pxd anyway? Meson is getting told to run the There was a problem hiding this comment.
Right, and this is another thing that may make sense to add to cython upstream, there's a couple other options that already got added while SciPy was porting over. EDIT: Hold on, this should already exist... There was a problem hiding this comment.
cython_sources is a dict mapping source file -> configure_file output(build directory location). The copied pxd would be used if we did something like There was a problem hiding this comment. Not sure if this is too crazy but would it make sense to just have any imports of the generated templates assume those templates come from the build folder rather than the source? There was a problem hiding this comment. This would mean patching Cython unfortunately. I don't think the include dir command line option works, for pxi files. There was a problem hiding this comment. Yeah I was afraid of that... I wonder why not though. Is this a cython bug? If not, it's definitely a cython feature request. |
||
| tslibs_pxd_sources_list = [ | ||
| # List of cython sources e.g. .pyx, .pxd & __init__.py | ||
| # Does NOT include .pxi.in | ||
| '__init__.py', | ||
| 'base.pxd', | ||
| 'ccalendar.pxd', | ||
| 'conversion.pxd', | ||
| 'dtypes.pxd', | ||
| 'nattype.pxd', | ||
| 'np_datetime.pxd', | ||
| 'offsets.pxd', | ||
| 'parsing.pxd', | ||
| 'period.pxd', | ||
| 'timedeltas.pxd', | ||
| 'timestamps.pxd', | ||
| 'timezones.pxd', | ||
| 'tzconversion.pxd', | ||
| 'util.pxd', | ||
| ] | ||
|
|
||
| foreach source: tslibs_pxd_sources_list | ||
| source_pxd = configure_file( | ||
| input: source, | ||
| output: source, | ||
| copy: true | ||
| ) | ||
| endforeach | ||
|
|
||
| tslibs_sources = { | ||
| # Dict of extension name -> dict of {sources, include_dirs, and deps} | ||
| # numpy include dir is implicitly included | ||
| 'base': {'sources': ['base.pyx']}, | ||
| 'ccalendar': {'sources': ['ccalendar.pyx']}, | ||
| 'dtypes': {'sources': ['dtypes.pyx']}, | ||
| 'conversion': {'sources': ['conversion.pyx', 'src/datetime/np_datetime.c'], | ||
| 'include_dirs': inc_datetime}, | ||
| 'fields': {'sources': ['fields.pyx', 'src/datetime/np_datetime.c']}, | ||
| 'nattype': {'sources': ['nattype.pyx']}, | ||
| 'np_datetime': {'sources': ['np_datetime.pyx', 'src/datetime/np_datetime.c', 'src/datetime/np_datetime_strings.c'], | ||
| 'include_dirs': inc_datetime}, | ||
| 'offsets': {'sources': ['offsets.pyx', 'src/datetime/np_datetime.c'], | ||
| 'include_dirs': inc_datetime}, | ||
| 'parsing': {'sources': ['parsing.pyx', '../src/parser/tokenizer.c'], | ||
| 'include_dirs': klib_include}, | ||
| 'period': {'sources': ['period.pyx', 'src/datetime/np_datetime.c'], | ||
| 'include_dirs': inc_datetime}, | ||
| 'strptime': {'sources': ['strptime.pyx', 'src/datetime/np_datetime.c'], | ||
| 'include_dirs': inc_datetime}, | ||
| 'timedeltas': {'sources': ['timedeltas.pyx', 'src/datetime/np_datetime.c'], | ||
| 'include_dirs': inc_datetime}, | ||
| 'timestamps': {'sources': ['timestamps.pyx', 'src/datetime/np_datetime.c'], | ||
| 'include_dirs': inc_datetime}, | ||
| 'timezones': {'sources': ['timezones.pyx', 'src/datetime/np_datetime.c'], | ||
| 'include_dirs': inc_datetime}, | ||
| 'tzconversion': {'sources': ['tzconversion.pyx', 'src/datetime/np_datetime.c'], | ||
| 'include_dirs': inc_datetime}, | ||
| 'vectorized': {'sources': ['vectorized.pyx', 'src/datetime/np_datetime.c'], | ||
| 'include_dirs': inc_datetime} | ||
| } | ||
|
|
||
| foreach ext_name, ext_dict : tslibs_sources | ||
| py.extension_module( | ||
| ext_name, | ||
| ext_dict.get('sources'), | ||
| include_directories: [inc_np] + ext_dict.get('include_dirs', ''), | ||
| dependencies: ext_dict.get('deps', ''), | ||
| subdir: 'pandas/_libs/tslibs', | ||
| install: true | ||
| ) | ||
| endforeach | ||
|
|
||
| py.install_sources('__init__.py', | ||
| pure: false, | ||
| subdir: 'pandas/_libs/tslibs') | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| py.extension_module( | ||
| 'aggregations', | ||
| ['aggregations.pyx'], | ||
| include_directories: [inc_np, '../src'], | ||
WillAyd marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| dependencies: [py_dep], | ||
| subdir: 'pandas/_libs/window', | ||
| override_options : ['cython_language=cpp'], | ||
| install: true | ||
| ) | ||
|
|
||
| py.extension_module( | ||
| 'indexers', | ||
| ['indexers.pyx'], | ||
| include_directories: [inc_np], | ||
| dependencies: [py_dep], | ||
| subdir: 'pandas/_libs/window', | ||
| install: true | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| subdirs_list = [ | ||
| # exclude sas, since it contains extension modules | ||
| # and has its own meson.build | ||
| 'clipboard', | ||
| 'excel', | ||
| 'formats', | ||
| 'json', | ||
| 'parsers' | ||
| ] | ||
| foreach subdir: subdirs_list | ||
| install_subdir(subdir, install_dir: py.get_install_dir(pure: false) / 'pandas/io') | ||
| endforeach | ||
| top_level_py_list = [ | ||
| '__init__.py', | ||
| 'api.py', | ||
| 'clipboards.py', | ||
| 'common.py', | ||
| 'date_converters.py', | ||
| 'feather_format.py', | ||
| 'gbq.py', | ||
| 'html.py', | ||
| 'orc.py', | ||
| 'parquet.py', | ||
| 'pickle.py', | ||
| 'pytables.py', | ||
| 'spss.py', | ||
| 'sql.py', | ||
| 'stata.py', | ||
| 'xml.py' | ||
| ] | ||
| foreach file: top_level_py_list | ||
|
There was a problem hiding this comment. Why do we need to do this for python modules? There was a problem hiding this comment. The So we have to install each of the top-level files under io separately (since we are not using Ideally, this file disappears when the sas Cython files are moved to _libs. I can do that in a follow-up if you'd like? There was a problem hiding this comment. Instead of a for loop you could probably just pass the entire array as a variadic argument to py.install_sources |
||
| py.install_sources(file, | ||
| pure: false, | ||
| subdir: 'pandas/io') | ||
| endforeach | ||
| subdir('sas') | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
How does this version interact with some of the other work you've done to get versioneer to work?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
versioneer determines the
__version__attribute. The version here is currently only used be meson-python(incorrectly) to determine the wheel filename(we are working on getting meson-python to use the__version__tag as well).In the future, this'll probably only be a fallback version, if everything fails to detect the pandas version properly.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I am still a bit wary of using
meson-pythoncoming from the angle that it is relatively immature. Sorry if I lost the answer to this question but what is the advantage of using meson-python versus just creating a simple setuptools hook to invoke meson as needed? The latter wouldn't require any changes to versioneer or other setuptools-based tooling @rgommersThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
From lithomas1#19 (comment):
@WillAyd it is possible, but I think that opens a whole new can of worms for no real reason. For example, you build the sdist with setuptools while you're building the wheel with a mix Meson and setuptools, leading to possible inconsistencies. And you'll be making cross-compiling worse rather than better.
If you want a gradual transition, it'd be better to merge this PR first, but still default to
setuptoolsinpyproject.toml. That allows devs to start using Meson, without all contributors and users who are pip-installing from GitHub getting it straight away. And then it's a patch of a few lines to switch that default later.In addition, I think it's optimistic to think that you can write a hook like that which won't have bugs or surprising behavior.
That's understandable. I'm sure there's a couple of bugs left to discover. That said, it's now been in SciPy
mainsince last December, and in the SciPy 1.9.x release series. So hopefully the most glaring defects have been found by now. Andmesonandmeson-pythonare quite actively maintained, so any painful bugs can hopefully be remedied quickly.You also have a number of months till the next release, right? Can always just try it, and in case you're not happy right before the next release, it's literally a one-line patch to revert back to setuptools for one more release cycle.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm fine with sticking to meson-python right now.
IMO, meson-python is pretty close to being feature-complete for our use case. We're just missing FFY00/meson-python#177 and the versioning issue(we can always rename the wheel by hand in the meantime).