diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 027e4a47e3..6dfb5b5672 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,7 +3,37 @@ For more detailed information, please see the git log. These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. -The latest version of easybuild-easyblocks provides 251 software-specific easyblocks and 42 generic easyblocks. +The latest version of easybuild-easyblocks provides 254 software-specific easyblocks and 43 generic easyblocks. + + +v4.9.0 (30 December 2023) +------------------------- + +feature release + +- add generic `CargoPythonBundle` easyblock (#2964) +- 3 new software-specific easyblocks: flook (#3034), HPCC (#3009), PALM (#3020) +- minor enhancements and updates, including: + - add custom easyconfig parameter `cmake_options` to SuiteSparse easyblock (#3031) + - update custom intel-compilers easyblock for versions >= 2024 (#3037) + - update custom easyblock for Intel MPI easyblock for v2021.11 (#3039) + - update numpy easyblock for v1.26+ (#3041) + - update custom easyblock for Intel MKL for v2024.x (#3042) + - update Ferret easyblock to be compatible with v7.6.0 (#3052) +- various bug fixes, including: + - add support for allowing version mismatch + consider versionsuffix when creating `.modulerc` in `ModuleRC` easyblock (#3028) + - update error detection for PyTorch tests (#3033) + - disable LLVM build downloads from CI in Rust (#3038) + - add requirement for EULA acceptance to CUDA easyblock (#3045) + - make various fixes and enhancements to NWChem easyblock (#3049) + - add binutils symlinks when building TensorFlow with `--rpath` (#3054, #3058) + - fix specifying path to SuiteSparse header files and libraries in numpy, Trilinos, PETSc easyblocks (#3056) + - fix `det_pylibdir` provided by `PythonPackage` easyblock for Python 3.12+ (#3057) + - fix nvptx sanity check for Clang >= 14.x (#3059) +- other changes: + - update SuiteSparse easyblock to only install SuiteSparse libraries with `make install` (#3004) + - also consider `$EB_COMSOL_LICENSE_FILE` environment variable in custom easyblock for COMSOL (#3044) + - import `LooseVersion` from `easybuild.tools` instead of `distutils.version` in easyblocks (#3048) v4.8.2 (29 October 2023) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 7d1118ed84..80b10f5187 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.2') +VERSION = LooseVersion('4.9.0') UNKNOWN = 'UNKNOWN' diff --git a/easybuild/easyblocks/a/abaqus.py b/easybuild/easyblocks/a/abaqus.py index 50404d75db..371a1b2b42 100644 --- a/easybuild/easyblocks/a/abaqus.py +++ b/easybuild/easyblocks/a/abaqus.py @@ -32,9 +32,9 @@ @author: Jens Timmerman (Ghent University) @author: Simon Branford (University of Birmingham) """ -from distutils.version import LooseVersion import glob import os +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/a/acml.py b/easybuild/easyblocks/a/acml.py index b20704a53e..5d82316dc5 100644 --- a/easybuild/easyblocks/a/acml.py +++ b/easybuild/easyblocks/a/acml.py @@ -33,7 +33,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/a/advisor.py b/easybuild/easyblocks/a/advisor.py index 54246495fa..45bced4c0a 100644 --- a/easybuild/easyblocks/a/advisor.py +++ b/easybuild/easyblocks/a/advisor.py @@ -31,7 +31,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase diff --git a/easybuild/easyblocks/a/amber.py b/easybuild/easyblocks/a/amber.py index 0140ac8d10..ca626883db 100644 --- a/easybuild/easyblocks/a/amber.py +++ b/easybuild/easyblocks/a/amber.py @@ -31,7 +31,7 @@ Enhanced/cleaned up by Kenneth Hoste (HPC-UGent) CMake support (Amber 20) added by James Carpenter and Simon Branford (University of Birmingham) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os import easybuild.tools.environment as env diff --git a/easybuild/easyblocks/a/ansys.py b/easybuild/easyblocks/a/ansys.py index 0c4b834846..6e68fad40b 100644 --- a/easybuild/easyblocks/a/ansys.py +++ b/easybuild/easyblocks/a/ansys.py @@ -31,7 +31,7 @@ import os import re import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index 8ccdaf1d86..aff71ee1b0 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -34,7 +34,7 @@ import os import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/a/armadillo.py b/easybuild/easyblocks/a/armadillo.py index a2faa04317..27415f3441 100644 --- a/easybuild/easyblocks/a/armadillo.py +++ b/easybuild/easyblocks/a/armadillo.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root diff --git a/easybuild/easyblocks/a/atlas.py b/easybuild/easyblocks/a/atlas.py index da8fdefbc9..2956e30636 100644 --- a/easybuild/easyblocks/a/atlas.py +++ b/easybuild/easyblocks/a/atlas.py @@ -36,7 +36,7 @@ import re import os import sys -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/b/bamtools.py b/easybuild/easyblocks/b/bamtools.py index 0a11f2e4cb..1938ce4c6a 100644 --- a/easybuild/easyblocks/b/bamtools.py +++ b/easybuild/easyblocks/b/bamtools.py @@ -28,7 +28,7 @@ @author: Andreas Panteli (The Cyprus Institute) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.easyblocks.generic.makecp import MakeCp from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/b/bazel.py b/easybuild/easyblocks/b/bazel.py index e626457001..97a013c8fb 100644 --- a/easybuild/easyblocks/b/bazel.py +++ b/easybuild/easyblocks/b/bazel.py @@ -25,7 +25,7 @@ """ EasyBuild support for building and installing Bazel, implemented as an easyblock """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import tempfile diff --git a/easybuild/easyblocks/b/berkeleygw.py b/easybuild/easyblocks/b/berkeleygw.py index e9a9b87f8e..b5b2e1f28b 100644 --- a/easybuild/easyblocks/b/berkeleygw.py +++ b/easybuild/easyblocks/b/berkeleygw.py @@ -28,7 +28,7 @@ @author: Miguel Dias Costa (National University of Singapore) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/b/binutils.py b/easybuild/easyblocks/b/binutils.py index 306a535d99..3985595288 100644 --- a/easybuild/easyblocks/b/binutils.py +++ b/easybuild/easyblocks/b/binutils.py @@ -30,7 +30,7 @@ import glob import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/b/boost.py b/easybuild/easyblocks/b/boost.py index 5110105cd8..7cb4bd7f03 100644 --- a/easybuild/easyblocks/b/boost.py +++ b/easybuild/easyblocks/b/boost.py @@ -38,7 +38,7 @@ @author: Michele Dolfi (ETH Zurich) @author: Simon Branford (University of Birmingham) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import fileinput import glob import os diff --git a/easybuild/easyblocks/b/bowtie.py b/easybuild/easyblocks/b/bowtie.py index 4daa8512ba..899019b35b 100644 --- a/easybuild/easyblocks/b/bowtie.py +++ b/easybuild/easyblocks/b/bowtie.py @@ -30,7 +30,7 @@ @author: Kenneth Hoste (Ghent University) @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import shutil diff --git a/easybuild/easyblocks/b/bowtie2.py b/easybuild/easyblocks/b/bowtie2.py index 285f9b98db..5d0821a6ed 100644 --- a/easybuild/easyblocks/b/bowtie2.py +++ b/easybuild/easyblocks/b/bowtie2.py @@ -16,7 +16,7 @@ @author: Fotis Georgatos (Uni.Lu) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.makecp import MakeCp diff --git a/easybuild/easyblocks/b/bwa.py b/easybuild/easyblocks/b/bwa.py index 646cc85e4e..587dd03125 100644 --- a/easybuild/easyblocks/b/bwa.py +++ b/easybuild/easyblocks/b/bwa.py @@ -20,7 +20,7 @@ """ import os import glob -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/c/cgal.py b/easybuild/easyblocks/c/cgal.py index 6cb2937cd8..0d69d0543d 100644 --- a/easybuild/easyblocks/c/cgal.py +++ b/easybuild/easyblocks/c/cgal.py @@ -31,7 +31,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index 3fb1b006f5..3423e1c96b 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -38,7 +38,7 @@ import glob import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.framework.easyconfig import CUSTOM @@ -622,9 +622,11 @@ def sanity_check_step(self): custom_commands = ['clang --help', 'clang++ --help', 'llvm-config --cxxflags'] shlib_ext = get_shared_lib_ext() + version = LooseVersion(self.version) + # Clang v16+ only use the major version number for the resource dir resdir_version = self.version - if LooseVersion(self.version) >= LooseVersion('16'): + if version >= '16': resdir_version = self.version.split('.')[0] # Detect OpenMP support for CPU architecture @@ -640,7 +642,7 @@ def sanity_check_step(self): else: print_warning("Unknown CPU architecture (%s) for OpenMP and runtime libraries check!" % arch) - if LooseVersion(self.version) >= LooseVersion('14'): + if version >= '14': glob_pattern = os.path.join(self.installdir, 'lib', '%s-*' % arch) matches = glob.glob(glob_pattern) if matches: @@ -663,7 +665,7 @@ def sanity_check_step(self): if self.cfg['static_analyzer']: custom_paths['files'].extend(["bin/scan-build", "bin/scan-view"]) - if 'clang-tools-extra' in self.cfg['llvm_projects'] and LooseVersion(self.version) >= LooseVersion('3.4'): + if 'clang-tools-extra' in self.cfg['llvm_projects'] and version >= '3.4': custom_paths['files'].extend(["bin/clang-tidy"]) if 'polly' in self.cfg['llvm_projects']: @@ -685,15 +687,15 @@ def sanity_check_step(self): if 'libcxxabi' in self.cfg['llvm_runtimes']: custom_paths['files'].extend([os.path.join(self.runtime_lib_path, "libc++abi.%s" % shlib_ext)]) - if 'flang' in self.cfg['llvm_projects'] and LooseVersion(self.version) >= LooseVersion('15'): + if 'flang' in self.cfg['llvm_projects'] and version >= '15': flang_compiler = 'flang-new' custom_paths['files'].extend(["bin/%s" % flang_compiler]) custom_commands.extend(["%s --help" % flang_compiler]) - if LooseVersion(self.version) >= LooseVersion('3.8'): + if version >= '3.8': custom_paths['files'].extend(["lib/libomp.%s" % shlib_ext, "lib/clang/%s/include/omp.h" % resdir_version]) - if LooseVersion(self.version) >= LooseVersion('12'): + if version >= '12': omp_target_libs = ["lib/libomptarget.%s" % shlib_ext, "lib/libomptarget.rtl.%s.%s" % (arch, shlib_ext)] else: omp_target_libs = ["lib/libomptarget.%s" % shlib_ext] @@ -703,24 +705,26 @@ def sanity_check_step(self): if 'NVPTX' in self.cfg['build_targets']: custom_paths['files'].append("lib/libomptarget.rtl.cuda.%s" % shlib_ext) # The static 'nvptx.a' library is not built from version 12 onwards - if LooseVersion(self.version) < LooseVersion('12.0'): + if version < '12.0': custom_paths['files'].append("lib/libomptarget-nvptx.a") ec_cuda_cc = self.cfg['cuda_compute_capabilities'] cfg_cuda_cc = build_option('cuda_compute_capabilities') cuda_cc = cfg_cuda_cc or ec_cuda_cc or [] # We need the CUDA capability in the form of '75' and not '7.5' cuda_cc = [cc.replace('.', '') for cc in cuda_cc] - if LooseVersion('12.0') < LooseVersion(self.version) < LooseVersion('13.0'): + if '12.0' < version < '13.0': custom_paths['files'].extend(["lib/libomptarget-nvptx-cuda_%s-sm_%s.bc" % (x, y) for x in CUDA_TOOLKIT_SUPPORT for y in cuda_cc]) - else: + # libomptarget-nvptx-sm*.bc is not there for Clang 14.x; + elif version < '14.0' or version >= '15.0': custom_paths['files'].extend(["lib/libomptarget-nvptx-sm_%s.bc" % cc for cc in cuda_cc]) # From version 13, and hopefully onwards, the naming of the CUDA # '.bc' files became a bit simpler and now we don't need to take # into account the CUDA version Clang was compiled with, making it - # easier to check for the bitcode files we expect - if LooseVersion(self.version) >= LooseVersion('13.0'): + # easier to check for the bitcode files we expect; + # libomptarget-new-nvptx-sm*.bc is only there in Clang 13.x and 14.x; + if version >= '13.0' and version < '15.0': custom_paths['files'].extend(["lib/libomptarget-new-nvptx-sm_%s.bc" % cc for cc in cuda_cc]) # If building for AMDGPU check that OpenMP target library was created @@ -729,12 +733,12 @@ def sanity_check_step(self): # OpenMP offloading support to AMDGPU was not added until version # 13, however, building for the AMDGPU target predates this and so # doesn't necessarily mean that the AMDGPU target failed - if LooseVersion(self.version) >= LooseVersion('13.0'): + if version >= '13.0': custom_paths['files'].append("lib/libomptarget.rtl.amdgpu.%s" % shlib_ext) custom_paths['files'].extend(["lib/libomptarget-amdgcn-%s.bc" % gfx for gfx in self.cfg['amd_gfx_list']]) custom_paths['files'].append("bin/amdgpu-arch") - if LooseVersion(self.version) >= LooseVersion('14.0'): + if version >= '14.0': custom_paths['files'].extend(["lib/libomptarget-new-amdgpu-%s.bc" % gfx for gfx in self.cfg['amd_gfx_list']]) diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index 0104ab64f9..3a1a2081cf 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -31,7 +31,7 @@ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.clang import DEFAULT_TARGETS_MAP as LLVM_ARCH_MAP from easybuild.easyblocks.generic.bundle import Bundle from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/c/comsol.py b/easybuild/easyblocks/c/comsol.py index 38d778a623..dd69c2dac3 100644 --- a/easybuild/easyblocks/c/comsol.py +++ b/easybuild/easyblocks/c/comsol.py @@ -59,14 +59,14 @@ def extract_step(self): def configure_step(self): """Configure COMSOL installation: create license file.""" - default_lic_env_var = 'LMCOMSOL_LICENSE_FILE' - lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=[default_lic_env_var], + comsol_lic_env_vars = ['EB_COMSOL_LICENSE_FILE', 'LMCOMSOL_LICENSE_FILE'] + lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=comsol_lic_env_vars, lic_specs=[self.cfg['license_file']]) if lic_specs: if self.license_env_var is None: self.log.info("Using COMSOL license specifications from 'license_file': %s", lic_specs) - self.license_env_var = default_lic_env_var + self.license_env_var = comsol_lic_env_vars[0] else: self.log.info("Using COMSOL license specifications from $%s: %s", self.license_env_var, lic_specs) @@ -74,7 +74,7 @@ def configure_step(self): env.setvar(self.license_env_var, self.license_file) else: msg = "No viable license specifications found; " - msg += "specify 'license_file', or define $%s" % default_lic_env_var + msg += "specify 'license_file', or define %s" % (', '.join('$%s' % x for x in comsol_lic_env_vars)) raise EasyBuildError(msg) copy_file(os.path.join(self.start_dir, 'setupconfig.ini'), self.configfile) diff --git a/easybuild/easyblocks/c/cp2k.py b/easybuild/easyblocks/c/cp2k.py index a921523e07..aa2ad4a93f 100644 --- a/easybuild/easyblocks/c/cp2k.py +++ b/easybuild/easyblocks/c/cp2k.py @@ -42,7 +42,7 @@ import re import os import sys -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/c/cplex.py b/easybuild/easyblocks/c/cplex.py index cc6ec0e52f..9c4a894a4e 100644 --- a/easybuild/easyblocks/c/cplex.py +++ b/easybuild/easyblocks/c/cplex.py @@ -31,7 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import stat diff --git a/easybuild/easyblocks/c/cryptography.py b/easybuild/easyblocks/c/cryptography.py index bfb6688358..ac0d4c68d1 100644 --- a/easybuild/easyblocks/c/cryptography.py +++ b/easybuild/easyblocks/c/cryptography.py @@ -27,7 +27,7 @@ @author: Alexander Grund """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.run import run_cmd diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index 32629aab40..dbc30cf4a7 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -38,7 +38,7 @@ import re import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM @@ -92,6 +92,16 @@ def __init__(self, *args, **kwargs): self.cfg.template_values['cudaarch'] = cudaarch self.cfg.generate_template_values() + def fetch_step(self, *args, **kwargs): + """Check for EULA acceptance prior to getting sources.""" + # EULA for CUDA must be accepted via --accept-eula-for EasyBuild configuration option, + # or via 'accept_eula = True' in easyconfig file + self.check_accepted_eula( + name='CUDA', + more_info='https://docs.nvidia.com/cuda/eula/index.html' + ) + return super(EB_CUDA, self).fetch_step(*args, **kwargs) + def extract_step(self): """Extract installer to have more control, e.g. options, patching Perl scripts, etc.""" execpath = self.src[0]['path'] diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py index e97db8e2c0..ef72014175 100644 --- a/easybuild/easyblocks/c/cudacompat.py +++ b/easybuild/easyblocks/c/cudacompat.py @@ -31,7 +31,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM, MANDATORY diff --git a/easybuild/easyblocks/c/cudnn.py b/easybuild/easyblocks/c/cudnn.py index 39e04fe8d5..d22834985d 100644 --- a/easybuild/easyblocks/c/cudnn.py +++ b/easybuild/easyblocks/c/cudnn.py @@ -28,7 +28,7 @@ @author: Simon Branford (University of Birmingham) @author: Robert Mijakovic (LuxProvide) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.tarball import Tarball from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/d/dolfin.py b/easybuild/easyblocks/d/dolfin.py index 6cb323636c..086fdff170 100644 --- a/easybuild/easyblocks/d/dolfin.py +++ b/easybuild/easyblocks/d/dolfin.py @@ -32,7 +32,7 @@ import os import re import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/d/doxygen.py b/easybuild/easyblocks/d/doxygen.py index 7a8a16365d..498e4f920b 100644 --- a/easybuild/easyblocks/d/doxygen.py +++ b/easybuild/easyblocks/d/doxygen.py @@ -33,7 +33,7 @@ @author: Balazs Hajgato (Free University Brussels (VUB)) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.tools.run import run_cmd from easybuild.easyblocks.generic.cmakemake import CMakeMake diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index f4b52ff921..73e1115fae 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -31,7 +31,7 @@ import os import re import sys -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pip_version from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/e/eigen.py b/easybuild/easyblocks/e/eigen.py index 70dc6a4e36..70205c5ee0 100644 --- a/easybuild/easyblocks/e/eigen.py +++ b/easybuild/easyblocks/e/eigen.py @@ -19,7 +19,7 @@ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.tools.filetools import copy_dir, copy_file, mkdir, apply_regex_substitutions diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 41b81e15e8..44acaf507b 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -30,7 +30,7 @@ @author: Kenneth Hoste (Ghent University) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index 03ff400619..676be44e44 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -30,7 +30,7 @@ @author: Maxime Boissonneault (Digital Research Alliance of Canada) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/f/ferret.py b/easybuild/easyblocks/f/ferret.py index 37e38e9a0b..1b692b485f 100644 --- a/easybuild/easyblocks/f/ferret.py +++ b/easybuild/easyblocks/f/ferret.py @@ -36,7 +36,7 @@ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError @@ -172,6 +172,14 @@ def configure_step(self): regex_subs.append((r"^(\s*%s\s*)=.*" % key, r"\1 = %s" % os.getenv(value))) if LooseVersion(self.version) >= LooseVersion("7.3"): + flag_vars = { + "CFLAGS": "CFLAGS", + "FFLAGS": "FFLAGS", + "PPLUS_FFLAGS": "FFLAGS", + } + for key, value in flag_vars.items(): + regex_subs.append((r"^(\s*%s\s*=).*-m64 (.*)" % key, r"\1%s \2" % os.getenv(value))) + regex_subs.extend([ (r"^(\s*LDFLAGS\s*=).*", r"\1 -fPIC %s -lnetcdff -lnetcdf -lhdf5_hl -lhdf5" % os.getenv("LDFLAGS")), (r"^(\s*)CDFLIB", r"\1NONEED"), @@ -182,8 +190,6 @@ def configure_step(self): for x in ["CFLAGS", "FFLAGS"]: regex_subs.append((r"^(\s*%s\s*=\s*\$\(CPP_FLAGS\)).*\\" % x, r"\1 %s \\" % os.getenv(x))) if LooseVersion(self.version) >= LooseVersion("7.3"): - for x in ["CFLAGS", "FFLAGS"]: - regex_subs.append((r"^(\s*%s\s*=).*-m64 (.*)" % x, r"\1%s \2" % os.getenv(x))) regex_subs.extend(sorted(gfort2ifort.items())) regex_subs.append((r"^(\s*MYDEFINES\s*=.*)\\", r"\1-DF90_SYSTEM_ERROR_CALLS \\")) @@ -208,6 +214,9 @@ def sanity_check_step(self): """Custom sanity check for Ferret.""" major_minor_version = '.'.join(self.version.split('.')[:2]) + if LooseVersion(self.version) >= LooseVersion("7.6"): + major_minor_version += self.version.split('.')[2] + custom_paths = { 'files': ["bin/ferret_v%s" % major_minor_version], 'dirs': [], diff --git a/easybuild/easyblocks/f/fftw.py b/easybuild/easyblocks/f/fftw.py index 6f10c711c9..aa30f3bf06 100644 --- a/easybuild/easyblocks/f/fftw.py +++ b/easybuild/easyblocks/f/fftw.py @@ -27,7 +27,7 @@ @author: Kenneth Hoste (HPC-UGent) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/f/flex.py b/easybuild/easyblocks/f/flex.py index a7fd414f8d..8e17da560b 100644 --- a/easybuild/easyblocks/f/flex.py +++ b/easybuild/easyblocks/f/flex.py @@ -27,7 +27,7 @@ @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/f/flook.py b/easybuild/easyblocks/f/flook.py new file mode 100644 index 0000000000..635fa99854 --- /dev/null +++ b/easybuild/easyblocks/f/flook.py @@ -0,0 +1,75 @@ +## +# Copyright 2023 Utrecht University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing flook, implemented as an easyblock + +@author: Arnold Kole (Utrecht University) +""" + +from easybuild.easyblocks.generic.configuremake import ConfigureMake + + +class EB_flook(ConfigureMake): + """Support for building/installing flook.""" + + def __init__(self, *args, **kwargs): + # call out to original constructor first, so 'self' (i.e. the class instance) is initialised + super(EB_flook, self).__init__(*args, **kwargs) + + # Determine vendor + vendor = None + if self.toolchain.COMPILER_FAMILY == 'Clang': + vendor = 'clang' + elif self.toolchain.COMPILER_FAMILY == 'GCC': + vendor = 'gnu' + elif self.toolchain.COMPILER_FAMILY == 'Intel': + vendor = 'intel' + elif self.toolchain.COMPILER_FAMILY == 'PGI': + vendor = 'pgi' + + # Set some default options + if vendor is not None: + local_comp_flags = 'VENDOR="%s" FFLAGS="$FFLAGS" CFLAGS="$CFLAGS"' % vendor + else: + local_comp_flags = 'FFLAGS="$FFLAGS" CFLAGS="$CFLAGS"' + self.cfg.update('buildopts', 'liball %s' % local_comp_flags) + self.cfg['parallel'] = 1 + + def configure_step(self): + # flook has no configure step + pass + + def install_step(self): + self.cfg.update('install_cmd', 'PREFIX=%s' % self.installdir) + super(EB_flook, self).install_step() + + def sanity_check_step(self): + custom_paths = { + 'files': ['include/flook.mod', 'lib/libflook.a', 'lib/libflookall.a', 'lib/pkgconfig/flook.pc'], + 'dirs': [], + } + + # call out to parent to do the actual sanity checking, pass through custom paths + super(EB_flook, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/f/fluent.py b/easybuild/easyblocks/f/fluent.py index 5976f8ea4f..5b2a479ce4 100644 --- a/easybuild/easyblocks/f/fluent.py +++ b/easybuild/easyblocks/f/fluent.py @@ -29,7 +29,7 @@ """ import os import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/f/freesurfer.py b/easybuild/easyblocks/f/freesurfer.py index cccb0fbd2b..e547c49453 100644 --- a/easybuild/easyblocks/f/freesurfer.py +++ b/easybuild/easyblocks/f/freesurfer.py @@ -30,7 +30,7 @@ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.tarball import Tarball from easybuild.framework.easyconfig import MANDATORY diff --git a/easybuild/easyblocks/f/fsl.py b/easybuild/easyblocks/f/fsl.py index e8ebaa9b21..dbd95edce5 100644 --- a/easybuild/easyblocks/f/fsl.py +++ b/easybuild/easyblocks/f/fsl.py @@ -31,7 +31,7 @@ import difflib import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/g/g2clib.py b/easybuild/easyblocks/g/g2clib.py index 15ad20f57b..591280cd0f 100644 --- a/easybuild/easyblocks/g/g2clib.py +++ b/easybuild/easyblocks/g/g2clib.py @@ -40,7 +40,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_g2clib(ConfigureMake): diff --git a/easybuild/easyblocks/g/gate.py b/easybuild/easyblocks/g/gate.py index 572204b836..223f49eefe 100644 --- a/easybuild/easyblocks/g/gate.py +++ b/easybuild/easyblocks/g/gate.py @@ -34,7 +34,7 @@ """ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.cmakemake import CMakeMake diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index 7ee469763e..589bdb5386 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -39,7 +39,7 @@ import re import shutil from copy import copy -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.clang import DEFAULT_TARGETS_MAP as LLVM_ARCH_MAP diff --git a/easybuild/easyblocks/g/gctf.py b/easybuild/easyblocks/g/gctf.py index 9e2ca40b86..17397a08c9 100644 --- a/easybuild/easyblocks/g/gctf.py +++ b/easybuild/easyblocks/g/gctf.py @@ -36,7 +36,7 @@ from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir from easybuild.tools.filetools import symlink, write_file from easybuild.tools.modules import get_software_root -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_Gctf(EasyBlock): diff --git a/easybuild/easyblocks/g/geant4.py b/easybuild/easyblocks/g/geant4.py index ca431ca946..6c94076980 100644 --- a/easybuild/easyblocks/g/geant4.py +++ b/easybuild/easyblocks/g/geant4.py @@ -35,7 +35,7 @@ import os import shutil import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/g/ghc.py b/easybuild/easyblocks/g/ghc.py index 20c6edcd75..af2ef6382f 100644 --- a/easybuild/easyblocks/g/ghc.py +++ b/easybuild/easyblocks/g/ghc.py @@ -27,7 +27,7 @@ @author: Andy Georges (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/g/go.py b/easybuild/easyblocks/g/go.py index 932df72f3e..97f0d79db7 100644 --- a/easybuild/easyblocks/g/go.py +++ b/easybuild/easyblocks/g/go.py @@ -31,7 +31,7 @@ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/g/gromacs.py b/easybuild/easyblocks/g/gromacs.py index 973b3f323a..74d129626c 100644 --- a/easybuild/easyblocks/g/gromacs.py +++ b/easybuild/easyblocks/g/gromacs.py @@ -37,7 +37,7 @@ import os import re import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 61cc7125fd..97e982405a 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -77,11 +77,13 @@ def __init__(self, *args, **kwargs): # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] - # list of sources for bundle itself *must* be empty - if self.cfg['sources']: - raise EasyBuildError("List of sources for bundle itself must be empty, found %s", self.cfg['sources']) - if self.cfg['patches']: - raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches']) + check_for_sources = getattr(self, 'check_for_sources', True) + # list of sources for bundle itself *must* be empty (unless overridden by subclass) + if check_for_sources: + if self.cfg['sources']: + raise EasyBuildError("List of sources for bundle itself must be empty, found %s", self.cfg['sources']) + if self.cfg['patches']: + raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches']) # disable templating to avoid premature resolving of template values self.cfg.enable_templating = False diff --git a/easybuild/easyblocks/generic/cargopythonbundle.py b/easybuild/easyblocks/generic/cargopythonbundle.py new file mode 100644 index 0000000000..4745b8a91e --- /dev/null +++ b/easybuild/easyblocks/generic/cargopythonbundle.py @@ -0,0 +1,58 @@ +## +# Copyright 2018-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for installing a bundle of Python packages, where some are built with Rust + +@author: Mikael Oehman (Chalmers University of Technology) +""" + +from easybuild.easyblocks.generic.cargo import Cargo +from easybuild.easyblocks.generic.pythonbundle import PythonBundle + + +class CargoPythonBundle(PythonBundle, Cargo): # PythonBundle must come first to take precedence + """ + Builds just like PythonBundle with setup for Rust and crates from Cargo easyblock + + The cargo init step will set up the environment variables for rustc and vendor sources + but all the build steps are triggered like normal. + """ + + @staticmethod + def extra_options(extra_vars=None): + """Define extra easyconfig parameters specific to Cargo""" + extra_vars = PythonBundle.extra_options(extra_vars) + extra_vars = Cargo.extra_options(extra_vars) # not all extra options here will used here + + return extra_vars + + def __init__(self, *args, **kwargs): + """Constructor for CargoPythonBundle easyblock.""" + self.check_for_sources = False # make Bundle allow sources (as crates are treated as sources) + super(CargoPythonBundle, self).__init__(*args, **kwargs) + + def extract_step(self): + """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums.""" + return Cargo.extract_step(self) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 0401f872e4..0a326a8e5a 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -36,7 +36,7 @@ import glob import re import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import BUILD, CUSTOM diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index 5c2d96bdc9..8a3b591402 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -28,7 +28,7 @@ @author: Pavel Grochal (INUITS) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index cab94b770c..1e918605dc 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -40,7 +40,7 @@ import shutil import stat import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock @@ -111,6 +111,26 @@ def __init__(self, *args, **kwargs): self.home_subdir_local = os.path.join(common_tmp_dir, os.environ.get('USER', 'nouser'), 'easybuild_intel') self.install_components = None + # dictionary to keep track of "latest" directory symlinks + # the target may only have major.minor, and tbb may have a lower version number than the compiler + # for example compiler 2021.1.2 has tbb 2021.1.1, 2024.0.0 has directory name 2024.0 + self._latest_subdir = {} + + def get_versioned_subdir(self, subdir): + """Return versioned directory that the 'latest' symlink points to in subdir""" + if subdir not in self._latest_subdir: + if os.path.islink(os.path.join(self.installdir, subdir, 'latest')): + version = os.readlink(os.path.join(self.installdir, subdir, 'latest')) + else: + version = 'latest' + latest_subdir = os.path.join(subdir, version) + self._latest_subdir[subdir] = latest_subdir + self.log.debug('Determined versioned directory for %s: %s', subdir, version) + return self._latest_subdir[subdir] + + def set_versioned_subdir(self, subdir, path): + """Set version-specific path for specified subdirectory.""" + self._latest_subdir[subdir] = path def get_guesses_tools(self): """Find reasonable paths for a subset of Intel tools, ignoring CPATH, LD_LIBRARY_PATH and LIBRARY_PATH""" diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 6e535d7f44..2ff1e49e89 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -30,7 +30,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/generic/mesonninja.py b/easybuild/easyblocks/generic/mesonninja.py index 8ce6b530e4..1fafaed53d 100644 --- a/easybuild/easyblocks/generic/mesonninja.py +++ b/easybuild/easyblocks/generic/mesonninja.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/generic/modulerc.py b/easybuild/easyblocks/generic/modulerc.py index aa67e90fe3..c6bd4b988c 100644 --- a/easybuild/easyblocks/generic/modulerc.py +++ b/easybuild/easyblocks/generic/modulerc.py @@ -30,6 +30,7 @@ import os from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.easyconfig import ActiveMNS from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import install_path @@ -41,6 +42,16 @@ class ModuleRC(EasyBlock): Generic easyblock to create a software-specific .modulerc file """ + @staticmethod + def extra_options(extra_vars=None): + """Define extra easyconfig parameters specific to ModuleRC""" + if extra_vars is None: + extra_vars = {} + extra_vars.update({ + 'check_version': [True, "Check version is prefix of dependency", CUSTOM], + }) + return EasyBlock.extra_options(extra_vars) + def configure_step(self): """Do nothing.""" pass @@ -67,7 +78,8 @@ def make_module_step(self, fake=False): raise EasyBuildError("Name does not match dependency name: %s vs %s", self.name, deps[0]['name']) # ensure version to alias to is a prefix of the version of the dependency - if not deps[0]['version'].startswith(self.version) and not self.version == "default": + if self.cfg['check_version'] and \ + not deps[0]['version'].startswith(self.version) and not self.version == "default": raise EasyBuildError("Version is not 'default' and not a prefix of dependency version: %s vs %s", self.version, deps[0]['version']) @@ -85,7 +97,7 @@ def make_module_step(self, fake=False): module_version_specs = { 'modname': alias_modname, - 'sym_version': self.version, + 'sym_version': self.version + self.cfg['versionsuffix'], 'version': deps[0]['version'], } self.module_generator.modulerc(module_version=module_version_specs, filepath=modulerc) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 29c7a8d803..952666220c 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -37,7 +37,7 @@ import re import sys import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from distutils.sysconfig import get_config_vars import easybuild.tools.environment as env @@ -172,9 +172,15 @@ def det_pylibdir(plat_specific=False, python_cmd=None): # determine Python lib dir via distutils # use run_cmd, we can to talk to the active Python, not the system Python running EasyBuild prefix = '/tmp/' - args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix) - pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args - cmd = "%s -c '%s'" % (python_cmd, pycode) + if LooseVersion(det_python_version(python_cmd)) >= LooseVersion('3.12'): + # Python 3.12 removed distutils but has a core sysconfig module which is similar + pathname = 'platlib' if plat_specific else 'purelib' + vars = {'platbase': prefix, 'base': prefix} + pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars) + else: + args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix) + pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args + cmd = "%s -c '%s'" % (python_cmd, pycode.replace("'", '"')) log.debug("Determining Python library directory using command '%s'", cmd) diff --git a/easybuild/easyblocks/generic/rpm.py b/easybuild/easyblocks/generic/rpm.py index 0a3f284c7f..1ab63bfff1 100644 --- a/easybuild/easyblocks/generic/rpm.py +++ b/easybuild/easyblocks/generic/rpm.py @@ -37,7 +37,7 @@ import os import re import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from os.path import expanduser import easybuild.tools.environment as env diff --git a/easybuild/easyblocks/generic/systemcompiler.py b/easybuild/easyblocks/generic/systemcompiler.py index def79c8543..0b01c30036 100644 --- a/easybuild/easyblocks/generic/systemcompiler.py +++ b/easybuild/easyblocks/generic/systemcompiler.py @@ -31,7 +31,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.base import fancylogger from easybuild.easyblocks.generic.bundle import Bundle diff --git a/easybuild/easyblocks/h/healpix.py b/easybuild/easyblocks/h/healpix.py index badc564abb..3f3bd08b2d 100644 --- a/easybuild/easyblocks/h/healpix.py +++ b/easybuild/easyblocks/h/healpix.py @@ -29,7 +29,7 @@ @author: Josef Dvoracek (Institute of Physics, Czech Academy of Sciences) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/h/hpcc.py b/easybuild/easyblocks/h/hpcc.py new file mode 100644 index 0000000000..57893788b2 --- /dev/null +++ b/easybuild/easyblocks/h/hpcc.py @@ -0,0 +1,81 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing HPCC, implemented as an easyblock + +@author: Samuel Moors (Vrije Universiteit Brussel) +""" + +import os + +from easybuild.easyblocks.hpl import EB_HPL +from easybuild.tools.filetools import copy_file, mkdir + + +class EB_HPCC(EB_HPL): + """ + Support for building HPCC (HPC Challenge) + - create Make.UNKNOWN + - build with make and install + """ + + def configure_step(self): + """ + Create Make.UNKNOWN file to build from + """ + # the build script file should be created in the hpl subdir + super(EB_HPCC, self).configure_step(subdir='hpl') + + def build_step(self): + """ + Build with make and correct make options + """ + # TOPdir should always be ../../.. regardless of what it was in the HPL build script file + super(EB_HPCC, self).build_step(topdir='../../..') + + def install_step(self): + """ + Install by copying files to install dir + """ + srcdir = self.cfg['start_dir'] + destdir = os.path.join(self.installdir, 'bin') + mkdir(destdir) + for filename in ["hpcc", "_hpccinf.txt"]: + srcfile = os.path.join(srcdir, filename) + copy_file(srcfile, destdir) + + def sanity_check_step(self): + """ + Custom sanity check for HPL + """ + + custom_paths = { + 'files': ['bin/hpcc', 'bin/_hpccinf.txt'], + 'dirs': [] + } + + custom_commands = ['hpcc'] + + super(EB_HPL, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) diff --git a/easybuild/easyblocks/h/hpcg.py b/easybuild/easyblocks/h/hpcg.py index 8d68fa34ea..13d9fe0621 100644 --- a/easybuild/easyblocks/h/hpcg.py +++ b/easybuild/easyblocks/h/hpcg.py @@ -37,7 +37,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import mkdir from easybuild.tools.run import run_cmd -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_HPCG(ConfigureMake): diff --git a/easybuild/easyblocks/h/hpl.py b/easybuild/easyblocks/h/hpl.py index d408a7b05e..3599b3004a 100644 --- a/easybuild/easyblocks/h/hpl.py +++ b/easybuild/easyblocks/h/hpl.py @@ -33,10 +33,10 @@ """ import os -import shutil from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import change_dir, copy_file, mkdir, remove_file, symlink from easybuild.tools.run import run_cmd @@ -61,26 +61,22 @@ def configure_step(self, subdir=None): makeincfile = os.path.join(basedir, 'Make.UNKNOWN') setupdir = os.path.join(basedir, 'setup') - try: - os.chdir(setupdir) - except OSError as err: - raise EasyBuildError("Failed to change to to dir %s: %s", setupdir, err) + change_dir(setupdir) cmd = "/bin/bash make_generic" run_cmd(cmd, log_all=True, simple=True, log_output=True) - try: - os.symlink(os.path.join(setupdir, 'Make.UNKNOWN'), os.path.join(makeincfile)) - except OSError as err: - raise EasyBuildError("Failed to symlink Make.UNKNOWN from %s to %s: %s", setupdir, makeincfile, err) + remove_file(makeincfile) + symlink(os.path.join(setupdir, 'Make.UNKNOWN'), makeincfile) # go back - os.chdir(self.cfg['start_dir']) + change_dir(self.cfg['start_dir']) - def build_step(self): + def build_step(self, topdir=None): """ Build with make and correct make options + - provide topdir argument so this can be reused in HPCC easyblock """ for envvar in ['MPICC', 'LIBLAPACK_MT', 'CPPFLAGS', 'LDFLAGS', 'CFLAGS']: @@ -89,7 +85,9 @@ def build_step(self): raise EasyBuildError("Required environment variable %s not found (no toolchain used?).", envvar) # build dir - extra_makeopts = 'TOPdir="%s" ' % self.cfg['start_dir'] + if not topdir: + topdir = self.cfg['start_dir'] + extra_makeopts = 'TOPdir="%s" ' % topdir # compilers extra_makeopts += 'CC="%(mpicc)s" MPICC="%(mpicc)s" LINKER="%(mpicc)s" ' % {'mpicc': os.getenv('MPICC')} @@ -116,14 +114,10 @@ def install_step(self): """ srcdir = os.path.join(self.cfg['start_dir'], 'bin', 'UNKNOWN') destdir = os.path.join(self.installdir, 'bin') - srcfile = None - try: - os.makedirs(destdir) - for filename in ["xhpl", "HPL.dat"]: - srcfile = os.path.join(srcdir, filename) - shutil.copy2(srcfile, destdir) - except OSError as err: - raise EasyBuildError("Copying %s to installation dir %s failed: %s", srcfile, destdir, err) + mkdir(destdir) + for filename in ["xhpl", "HPL.dat"]: + srcfile = os.path.join(srcdir, filename) + copy_file(srcfile, destdir) def sanity_check_step(self): """ diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index 37651f9e30..f827dd4679 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -36,7 +36,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, COMP_ALL from easybuild.easyblocks.generic.intelbase import LICENSE_FILE_NAME_2012 diff --git a/easybuild/easyblocks/i/ifort.py b/easybuild/easyblocks/i/ifort.py index d47013a510..5b3356c84d 100644 --- a/easybuild/easyblocks/i/ifort.py +++ b/easybuild/easyblocks/i/ifort.py @@ -34,7 +34,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.easyblocks.icc import EB_icc # @UnresolvedImport diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index a342a4686c..3901a22cf3 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -39,7 +39,7 @@ import os import shutil import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain @@ -79,14 +79,26 @@ def __init__(self, *args, **kwargs): self.cdftlibs = [] self.mpi_spec = None + if self.cfg['flexiblas'] is None: + self.cfg['flexiblas'] = LooseVersion(self.version) >= LooseVersion('2021') + + if LooseVersion(self.version) >= LooseVersion('2024'): + self.examples_subdir = os.path.join('share', 'doc', 'mkl', 'examples') + self.compiler_libdir = 'lib' + else: + self.examples_subdir = 'examples' + self.compiler_libdir = os.path.join('linux', 'compiler', 'lib', 'intel64_lin') + + @property + def mkl_basedir(self): if LooseVersion(self.version) >= LooseVersion('2021'): - self.mkl_basedir = os.path.join('mkl', self.version) - if self.cfg['flexiblas'] is None: - self.cfg['flexiblas'] = True + return self.get_versioned_subdir('mkl') else: - self.mkl_basedir = 'mkl' - if self.cfg['flexiblas'] is None: - self.cfg['flexiblas'] = False + return 'mkl' + + @mkl_basedir.setter + def mkl_basedir(self, path): + self.set_versioned_subdir('mkl', path) def prepare_step(self, *args, **kwargs): """Prepare build environment.""" @@ -162,7 +174,10 @@ def build_mkl_fftw_interfaces(self, libdir): loosever = LooseVersion(self.version) if loosever >= LooseVersion('10.3'): - intsubdir = os.path.join(self.mkl_basedir, 'interfaces') + intsubdir = self.mkl_basedir + if loosever >= LooseVersion('2024'): + intsubdir = os.path.join(intsubdir, 'share', 'mkl') + intsubdir = os.path.join(intsubdir, 'interfaces') inttarget = 'libintel64' else: intsubdir = 'interfaces' @@ -306,8 +321,10 @@ def build_mkl_flexiblas(self, flexiblasdir): and libflexiblas_imkl_sequential.so. They can be used as FlexiBLAS backends via FLEXIBLAS_LIBRARY_PATH. """ - builderdir = os.path.join(self.installdir, self.mkl_basedir, 'tools', 'builder') - change_dir(builderdir) + builder_subdir = os.path.join('tools', 'builder') + if LooseVersion(self.version) >= LooseVersion('2024'): + builder_subdir = os.path.join('share', 'mkl', builder_subdir) + change_dir(os.path.join(self.installdir, self.mkl_basedir, builder_subdir)) mkdir(flexiblasdir, parents=True) # concatenate lists of all BLAS, CBLAS and LAPACK functions @@ -318,8 +335,7 @@ def build_mkl_flexiblas(self, flexiblasdir): with open(lst + "_example_list") as src: shutil.copyfileobj(src, dst) - compilerdir = os.path.join('..', '..', '..', '..', 'compiler', self.version, - 'linux', 'compiler', 'lib', 'intel64_lin') + compilerdir = os.path.join(self.installdir, self.get_versioned_subdir('compiler'), self.compiler_libdir) # IFACE_COMP_PART=gf gives the gfortran calling convention that FlexiBLAS expects cmds = ["make libintel64 IFACE_COMP_PART=gf export=%s name=%s" % ( listfilename, os.path.join(flexiblasdir, 'libflexiblas_imkl_')) + @@ -339,7 +355,7 @@ def post_install_step(self): super(EB_imkl, self).post_install_step() # extract examples - examples_subdir = os.path.join(self.installdir, self.mkl_basedir, 'examples') + examples_subdir = os.path.join(self.installdir, self.mkl_basedir, self.examples_subdir) if os.path.exists(examples_subdir): cwd = change_dir(examples_subdir) for examples_tarball in glob.glob('examples_*.tgz'): @@ -515,7 +531,7 @@ def make_module_req_guess(self): raise EasyBuildError("32-bit not supported yet for IMKL v%s (>= 10.3)", self.version) else: if LooseVersion(self.version) >= LooseVersion('2021'): - compiler_subdir = os.path.join('compiler', self.version, 'linux', 'compiler', 'lib', 'intel64_lin') + compiler_subdir = os.path.join(self.get_versioned_subdir('compiler'), self.compiler_libdir) pkg_config_path = [os.path.join(self.mkl_basedir, 'tools', 'pkgconfig'), os.path.join(self.mkl_basedir, 'lib', 'pkgconfig')] else: @@ -580,16 +596,12 @@ def make_module_extra(self): if 'MKL_EXAMPLES' not in self.cfg['modextravars']: self.cfg.update('modextravars', { - 'MKL_EXAMPLES': os.path.join(self.installdir, self.mkl_basedir, 'examples'), + 'MKL_EXAMPLES': os.path.join(self.installdir, self.mkl_basedir, self.examples_subdir), }) txt = super(EB_imkl, self).make_module_extra() - if LooseVersion(self.version) >= LooseVersion('2021'): - mklroot = os.path.join(self.installdir, 'mkl', self.version) - else: - mklroot = os.path.join(self.installdir, 'mkl') - + mklroot = os.path.join(self.installdir, self.mkl_basedir) txt += self.module_generator.set_environment('MKLROOT', mklroot) return txt diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index 7eee0df099..9b646ce9be 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -34,7 +34,7 @@ @author: Alex Domingo (Vrije Universiteit Brussel) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 @@ -42,7 +42,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions, change_dir, extract_file, mkdir, write_file -from easybuild.tools.modules import get_software_root +from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template @@ -208,7 +208,7 @@ def sanity_check_step(self): mpi_mods.extend(['mpi_base.mod', 'mpi_constants.mod', 'mpi_sizeofs.mod']) if impi_ver >= LooseVersion('2021'): - mpi_subdir = os.path.join('mpi', self.version) + mpi_subdir = self.get_versioned_subdir('mpi') bin_dir = os.path.join(mpi_subdir, 'bin') include_dir = os.path.join(mpi_subdir, 'include') lib_dir = os.path.join(mpi_subdir, 'lib', 'release') @@ -223,11 +223,15 @@ def sanity_check_step(self): lib_dir = 'lib%s' % suff mpi_mods.extend(['i_malloc.h']) + mpi_mods_dir = include_dir + if impi_ver >= LooseVersion('2021.11'): + mpi_mods_dir = os.path.join(mpi_mods_dir, 'mpi') + shlib_ext = get_shared_lib_ext() custom_paths = { 'files': [os.path.join(bin_dir, 'mpi%s' % x) for x in ['icc', 'icpc', 'ifort']] + [os.path.join(include_dir, 'mpi%s.h' % x) for x in ['cxx', 'f', '', 'o', 'of']] + - [os.path.join(include_dir, x) for x in mpi_mods] + + [os.path.join(mpi_mods_dir, x) for x in mpi_mods] + [os.path.join(lib_dir, 'libmpi.%s' % shlib_ext)] + [os.path.join(lib_dir, 'libmpi.a')], 'dirs': [], @@ -239,7 +243,10 @@ def sanity_check_step(self): if impi_ver >= LooseVersion('2017'): # Add minimal test program to sanity checks if impi_ver >= LooseVersion('2021'): - impi_testsrc = os.path.join(self.installdir, 'mpi', self.version, 'test', 'test.c') + impi_testsrc = os.path.join(self.installdir, self.get_versioned_subdir('mpi')) + if impi_ver >= LooseVersion('2021.11'): + impi_testsrc = os.path.join(impi_testsrc, 'opt', 'mpi') + impi_testsrc = os.path.join(impi_testsrc, 'test', 'test.c') else: impi_testsrc = os.path.join(self.installdir, 'test', 'test.c') @@ -247,12 +254,7 @@ def sanity_check_step(self): self.log.info("Adding minimal MPI test program to sanity checks: %s", impi_testsrc) # Build test program with appropriate compiler from current toolchain - comp_fam = self.toolchain.comp_family() - if comp_fam == toolchain.INTELCOMP: - build_comp = 'mpiicc' - else: - build_comp = 'mpicc' - build_cmd = "%s %s -o %s" % (build_comp, impi_testsrc, impi_testexe) + build_cmd = "mpicc -cc=%s %s -o %s" % (os.getenv('CC'), impi_testsrc, impi_testexe) # Execute test program with appropriate MPI executable for target toolchain params = {'nr_ranks': self.cfg['parallel'], 'cmd': impi_testexe} @@ -283,7 +285,7 @@ def make_module_req_guess(self): impi_ver = LooseVersion(self.version) if impi_ver >= LooseVersion('2021'): - mpi_subdir = os.path.join('mpi', self.version) + mpi_subdir = self.get_versioned_subdir('mpi') lib_dirs = [ os.path.join(mpi_subdir, 'lib'), os.path.join(mpi_subdir, 'lib', 'release'), @@ -294,10 +296,13 @@ def make_module_req_guess(self): os.path.join(mpi_subdir, 'bin'), os.path.join(mpi_subdir, 'libfabric', 'bin'), ] - manpath = os.path.join(mpi_subdir, 'man') + if impi_ver >= LooseVersion('2021.11'): + manpath = os.path.join(mpi_subdir, 'share', 'man') + else: + manpath = os.path.join(mpi_subdir, 'man') if self.cfg['ofi_internal']: - libfabric_dir = os.path.join('mpi', self.version, 'libfabric') + libfabric_dir = os.path.join(mpi_subdir, 'libfabric') lib_dirs.append(os.path.join(libfabric_dir, 'lib')) path_dirs.append(os.path.join(libfabric_dir, 'bin')) guesses['FI_PROVIDER_PATH'] = [os.path.join(libfabric_dir, 'lib', 'prov')] @@ -332,7 +337,7 @@ def make_module_extra(self, *args, **kwargs): """Overwritten from Application to add extra txt""" if LooseVersion(self.version) >= LooseVersion('2021'): - mpiroot = os.path.join(self.installdir, 'mpi', self.version) + mpiroot = os.path.join(self.installdir, self.get_versioned_subdir('mpi')) else: mpiroot = self.installdir @@ -361,11 +366,21 @@ def make_module_extra(self, *args, **kwargs): if self.cfg['set_mpi_wrapper_aliases_intel'] or self.cfg['set_mpi_wrappers_all']: # do the same for mpiicc/mpiipc/mpiifort to be consistent, even if they may not exist - txt += self.module_generator.set_alias('mpiicc', 'mpiicc -cc=icc') - txt += self.module_generator.set_alias('mpiicpc', 'mpiicpc -cxx=icpc') + if (get_software_root('intel-compilers') and + LooseVersion(get_software_version('intel-compilers')) >= LooseVersion('2024')): + txt += self.module_generator.set_alias('mpiicc', 'mpiicc -cc=icx') + txt += self.module_generator.set_alias('mpiicpc', 'mpiicpc -cxx=icpx') + else: + txt += self.module_generator.set_alias('mpiicc', 'mpiicc -cc=icc') + txt += self.module_generator.set_alias('mpiicpc', 'mpiicpc -cxx=icpc') # -fc also works, but -f90 takes precedence txt += self.module_generator.set_alias('mpiifort', 'mpiifort -f90=ifort') + if LooseVersion(self.version) >= LooseVersion('2021.11'): + txt += self.module_generator.set_alias('mpiicx', 'mpiicx -cc=icx') + txt += self.module_generator.set_alias('mpiicpx', 'mpiicpx -cxx=icpx') + txt += self.module_generator.set_alias('mpiifx', 'mpiifx -f90=ifx') + # set environment variable UCX_TLS to 'all', this works in all hardware configurations # needed with UCX regardless of the transports available (even without a Mellanox HCA) # more information in easybuilders/easybuild-easyblocks#2253 diff --git a/easybuild/easyblocks/i/inspector.py b/easybuild/easyblocks/i/inspector.py index e3ce7653b1..a0d595c2e8 100644 --- a/easybuild/easyblocks/i/inspector.py +++ b/easybuild/easyblocks/i/inspector.py @@ -29,7 +29,7 @@ @author: Damian Alvarez (Forschungzentrum Juelich GmbH) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 diff --git a/easybuild/easyblocks/i/intel_compilers.py b/easybuild/easyblocks/i/intel_compilers.py index 3d6d9b030e..d1839a7a7b 100644 --- a/easybuild/easyblocks/i/intel_compilers.py +++ b/easybuild/easyblocks/i/intel_compilers.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.easyblocks.t.tbb import get_tbb_gccprefix @@ -51,10 +51,16 @@ def __init__(self, *args, **kwargs): if LooseVersion(self.version) < LooseVersion('2021'): raise EasyBuildError("Invalid version %s, should be >= 2021.x" % self.version) - self.compilers_subdir = os.path.join('compiler', self.version, 'linux') - # note that tbb may have a lower version number than the compiler, so use 'latest' symlink - # for example compiler 2021.1.2 has tbb 2021.1.1. - self.tbb_subdir = os.path.join('tbb', 'latest') + @property + def compilers_subdir(self): + compilers_subdir = self.get_versioned_subdir('compiler') + if LooseVersion(self.version) < LooseVersion('2024'): + compilers_subdir = os.path.join(compilers_subdir, 'linux') + return compilers_subdir + + @property + def tbb_subdir(self): + return self.get_versioned_subdir('tbb') def prepare_step(self, *args, **kwargs): """ @@ -92,7 +98,6 @@ def sanity_check_step(self): Custom sanity check for Intel compilers. """ - classic_compiler_cmds = ['icc', 'icpc', 'ifort'] oneapi_compiler_cmds = [ 'dpcpp', # Intel oneAPI Data Parallel C++ compiler 'icx', # oneAPI Intel C compiler @@ -100,8 +105,14 @@ def sanity_check_step(self): 'ifx', # oneAPI Intel Fortran compiler ] bindir = os.path.join(self.compilers_subdir, 'bin') - classic_compiler_paths = [os.path.join(bindir, x) for x in oneapi_compiler_cmds] - oneapi_compiler_paths = [os.path.join(bindir, 'intel64', x) for x in classic_compiler_cmds] + oneapi_compiler_paths = [os.path.join(bindir, x) for x in oneapi_compiler_cmds] + if LooseVersion(self.version) >= LooseVersion('2024'): + classic_compiler_cmds = ['ifort'] + classic_bindir = bindir + else: + classic_compiler_cmds = ['icc', 'icpc', 'ifort'] + classic_bindir = os.path.join(bindir, 'intel64') + classic_compiler_paths = [os.path.join(classic_bindir, x) for x in classic_compiler_cmds] custom_paths = { 'files': classic_compiler_paths + oneapi_compiler_paths, @@ -131,12 +142,7 @@ def make_module_req_guess(self): os.path.join('compiler', 'lib', 'intel64_lin'), ] libdirs = [os.path.join(self.compilers_subdir, x) for x in libdirs] - # resolve 'latest' symlink for tbb (if module guess is run with install in place) - if os.path.islink(os.path.join(self.installdir, self.tbb_subdir)): - tbb_version = os.readlink(os.path.join(self.installdir, self.tbb_subdir)) - else: - tbb_version = 'latest' - tbb_subdir = os.path.join('tbb', tbb_version) + tbb_subdir = self.tbb_subdir tbb_libsubdir = os.path.join(tbb_subdir, 'lib', 'intel64') libdirs.append(os.path.join(tbb_libsubdir, get_tbb_gccprefix(os.path.join(self.installdir, tbb_libsubdir)))) @@ -148,10 +154,12 @@ def make_module_req_guess(self): 'LD_LIBRARY_PATH': libdirs, 'LIBRARY_PATH': libdirs, 'MANPATH': [ - os.path.join('compiler', self.version, 'documentation', 'en', 'man', 'common'), + os.path.join(os.path.dirname(self.compilers_subdir), 'documentation', 'en', 'man', 'common'), + os.path.join(self.compilers_subdir, 'share', 'man'), ], 'OCL_ICD_FILENAMES': [ os.path.join(self.compilers_subdir, 'lib', 'x64', 'libintelocl.so'), + os.path.join(self.compilers_subdir, 'lib', 'libintelocl.so'), ], 'CPATH': [ os.path.join(tbb_subdir, 'include'), diff --git a/easybuild/easyblocks/i/ipp.py b/easybuild/easyblocks/i/ipp.py index 789a44d71c..3b70620b0d 100644 --- a/easybuild/easyblocks/i/ipp.py +++ b/easybuild/easyblocks/i/ipp.py @@ -34,7 +34,7 @@ @author: Damian Alvarez (Forschungszentrum Juelich GmbH) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 diff --git a/easybuild/easyblocks/i/itac.py b/easybuild/easyblocks/i/itac.py index 1f37b8ab01..542b1f5e37 100644 --- a/easybuild/easyblocks/i/itac.py +++ b/easybuild/easyblocks/i/itac.py @@ -34,7 +34,7 @@ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index 9eca14cf72..caef94fabe 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -32,7 +32,7 @@ import os import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option diff --git a/easybuild/easyblocks/j/jaxlib.py b/easybuild/easyblocks/j/jaxlib.py index dac39bf16c..ad6fa016f5 100644 --- a/easybuild/easyblocks/j/jaxlib.py +++ b/easybuild/easyblocks/j/jaxlib.py @@ -33,7 +33,7 @@ import os import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.pythonpackage import PythonPackage diff --git a/easybuild/easyblocks/l/lammps.py b/easybuild/easyblocks/l/lammps.py index 8b4b4dbe2e..2769014eab 100644 --- a/easybuild/easyblocks/l/lammps.py +++ b/easybuild/easyblocks/l/lammps.py @@ -34,7 +34,7 @@ import os import re import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/l/lapack.py b/easybuild/easyblocks/l/lapack.py index 112875a8fa..b0322030b7 100644 --- a/easybuild/easyblocks/l/lapack.py +++ b/easybuild/easyblocks/l/lapack.py @@ -31,7 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os diff --git a/easybuild/easyblocks/l/libint.py b/easybuild/easyblocks/l/libint.py index aa76bb6020..a5ff58df41 100644 --- a/easybuild/easyblocks/l/libint.py +++ b/easybuild/easyblocks/l/libint.py @@ -30,7 +30,7 @@ """ import os.path -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import DEFAULT_CONFIGURE_CMD, ConfigureMake from easybuild.easyblocks.generic.cmakemake import CMakeMake diff --git a/easybuild/easyblocks/l/libqglviewer.py b/easybuild/easyblocks/l/libqglviewer.py index 55d6a5d0a6..b126068ead 100644 --- a/easybuild/easyblocks/l/libqglviewer.py +++ b/easybuild/easyblocks/l/libqglviewer.py @@ -33,7 +33,7 @@ from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.modules import get_software_root from easybuild.tools.build_log import EasyBuildError -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_libQGLViewer(ConfigureMake): diff --git a/easybuild/easyblocks/l/libsmm.py b/easybuild/easyblocks/l/libsmm.py index 19a9f6c145..58d73cbb45 100644 --- a/easybuild/easyblocks/l/libsmm.py +++ b/easybuild/easyblocks/l/libsmm.py @@ -33,7 +33,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/l/libxml2.py b/easybuild/easyblocks/l/libxml2.py index 73e83cc804..10544c2db9 100644 --- a/easybuild/easyblocks/l/libxml2.py +++ b/easybuild/easyblocks/l/libxml2.py @@ -30,7 +30,7 @@ @author: Alan O'Cais (Juelich Supercomputing Centre) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os import easybuild.tools.environment as env diff --git a/easybuild/easyblocks/l/llvm.py b/easybuild/easyblocks/l/llvm.py index 5bced9a4c2..7b63296045 100644 --- a/easybuild/easyblocks/l/llvm.py +++ b/easybuild/easyblocks/l/llvm.py @@ -37,7 +37,7 @@ from easybuild.tools.filetools import move_file from easybuild.tools.modules import get_software_root from easybuild.tools.systemtools import get_cpu_architecture -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_LLVM(CMakeMake): diff --git a/easybuild/easyblocks/l/lua.py b/easybuild/easyblocks/l/lua.py index 16fc4065e1..e77c44775c 100644 --- a/easybuild/easyblocks/l/lua.py +++ b/easybuild/easyblocks/l/lua.py @@ -28,7 +28,7 @@ @author: Ruben Di Battista (Ecole Polytechnique) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/m/mathematica.py b/easybuild/easyblocks/m/mathematica.py index 9f1fb78827..6f18474490 100644 --- a/easybuild/easyblocks/m/mathematica.py +++ b/easybuild/easyblocks/m/mathematica.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os diff --git a/easybuild/easyblocks/m/matlab.py b/easybuild/easyblocks/m/matlab.py index ec2f43fa39..7e5a7e54be 100644 --- a/easybuild/easyblocks/m/matlab.py +++ b/easybuild/easyblocks/m/matlab.py @@ -37,7 +37,7 @@ import stat import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/m/mcr.py b/easybuild/easyblocks/m/mcr.py index 49cb3b36f4..7a420bd660 100644 --- a/easybuild/easyblocks/m/mcr.py +++ b/easybuild/easyblocks/m/mcr.py @@ -38,7 +38,7 @@ import re import shutil import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/m/mesa.py b/easybuild/easyblocks/m/mesa.py index 897faacbaa..b6d127361f 100644 --- a/easybuild/easyblocks/m/mesa.py +++ b/easybuild/easyblocks/m/mesa.py @@ -31,7 +31,7 @@ @author: Alexander Grund (TU Dresden) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.mesonninja import MesonNinja from easybuild.tools.filetools import copy_dir diff --git a/easybuild/easyblocks/m/metis.py b/easybuild/easyblocks/m/metis.py index b8582b7842..451ca67f84 100644 --- a/easybuild/easyblocks/m/metis.py +++ b/easybuild/easyblocks/m/metis.py @@ -33,7 +33,7 @@ """ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/m/molpro.py b/easybuild/easyblocks/m/molpro.py index 86358f0dcd..6cfdfa069d 100644 --- a/easybuild/easyblocks/m/molpro.py +++ b/easybuild/easyblocks/m/molpro.py @@ -30,7 +30,7 @@ import os import shutil import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/m/mono.py b/easybuild/easyblocks/m/mono.py index b23088ba65..11b04cce53 100644 --- a/easybuild/easyblocks/m/mono.py +++ b/easybuild/easyblocks/m/mono.py @@ -31,7 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os import shutil diff --git a/easybuild/easyblocks/m/mothur.py b/easybuild/easyblocks/m/mothur.py index 78f5f0f832..bc14dd4cef 100644 --- a/easybuild/easyblocks/m/mothur.py +++ b/easybuild/easyblocks/m/mothur.py @@ -30,7 +30,7 @@ import glob import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/m/motioncor2.py b/easybuild/easyblocks/m/motioncor2.py index 9224e89d9b..7b0c086a04 100644 --- a/easybuild/easyblocks/m/motioncor2.py +++ b/easybuild/easyblocks/m/motioncor2.py @@ -32,7 +32,7 @@ import os import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, write_file diff --git a/easybuild/easyblocks/m/mpich.py b/easybuild/easyblocks/m/mpich.py index df97d7265b..05057f4ff2 100644 --- a/easybuild/easyblocks/m/mpich.py +++ b/easybuild/easyblocks/m/mpich.py @@ -34,7 +34,7 @@ @author: Xavier Besseron (University of Luxembourg) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/m/mrbayes.py b/easybuild/easyblocks/m/mrbayes.py index 8fc399b5d3..ea7da301b5 100644 --- a/easybuild/easyblocks/m/mrbayes.py +++ b/easybuild/easyblocks/m/mrbayes.py @@ -36,7 +36,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/m/mrtrix.py b/easybuild/easyblocks/m/mrtrix.py index 7b148131a8..a676719c65 100644 --- a/easybuild/easyblocks/m/mrtrix.py +++ b/easybuild/easyblocks/m/mrtrix.py @@ -27,7 +27,7 @@ """ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/m/mumps.py b/easybuild/easyblocks/m/mumps.py index 26c43adbf9..d701f82de6 100644 --- a/easybuild/easyblocks/m/mumps.py +++ b/easybuild/easyblocks/m/mumps.py @@ -34,7 +34,7 @@ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools import toolchain diff --git a/easybuild/easyblocks/m/mvapich2.py b/easybuild/easyblocks/m/mvapich2.py index 3aecd40103..5810f227ac 100644 --- a/easybuild/easyblocks/m/mvapich2.py +++ b/easybuild/easyblocks/m/mvapich2.py @@ -34,7 +34,7 @@ @author: Xavier Besseron (University of Luxembourg) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.mpich import EB_MPICH from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/m/mxnet.py b/easybuild/easyblocks/m/mxnet.py index 10bf26cc5d..8e52f3635d 100644 --- a/easybuild/easyblocks/m/mxnet.py +++ b/easybuild/easyblocks/m/mxnet.py @@ -30,7 +30,7 @@ import glob import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.makecp import MakeCp diff --git a/easybuild/easyblocks/m/mymedialite.py b/easybuild/easyblocks/m/mymedialite.py index d4876e2d31..181ef9f051 100644 --- a/easybuild/easyblocks/m/mymedialite.py +++ b/easybuild/easyblocks/m/mymedialite.py @@ -32,7 +32,7 @@ @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.run import run_cmd diff --git a/easybuild/easyblocks/n/namd.py b/easybuild/easyblocks/n/namd.py index a0f4c82b95..301dfb5028 100644 --- a/easybuild/easyblocks/n/namd.py +++ b/easybuild/easyblocks/n/namd.py @@ -17,7 +17,7 @@ import os import re import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.makecp import MakeCp diff --git a/easybuild/easyblocks/n/netcdf.py b/easybuild/easyblocks/n/netcdf.py index 45017748bb..b1b0a41ed8 100644 --- a/easybuild/easyblocks/n/netcdf.py +++ b/easybuild/easyblocks/n/netcdf.py @@ -33,7 +33,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/n/neuron.py b/easybuild/easyblocks/n/neuron.py index a094670be6..cdad4158e5 100644 --- a/easybuild/easyblocks/n/neuron.py +++ b/easybuild/easyblocks/n/neuron.py @@ -41,7 +41,7 @@ from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_NEURON(CMakeMake): diff --git a/easybuild/easyblocks/n/numexpr.py b/easybuild/easyblocks/n/numexpr.py index 1c62b4502c..e5d192fff2 100644 --- a/easybuild/easyblocks/n/numexpr.py +++ b/easybuild/easyblocks/n/numexpr.py @@ -26,7 +26,7 @@ EasyBuild support for building and installing numexpr, implemented as an easyblock """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.filetools import write_file diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index f42b914e55..889faf9d2d 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -45,7 +45,7 @@ from easybuild.tools.filetools import change_dir, mkdir, read_file, remove_dir from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_numpy(FortranPythonPackage): @@ -183,23 +183,17 @@ def get_libs_for_mkl(varname): suitesparseroot = get_software_root('SuiteSparse') if suitesparseroot: - amddir = os.path.join(suitesparseroot, 'AMD') - umfpackdir = os.path.join(suitesparseroot, 'UMFPACK') - if not os.path.exists(amddir) or not os.path.exists(umfpackdir): - raise EasyBuildError("Expected SuiteSparse subdirectories are not both there: %s, %s", - amddir, umfpackdir) - else: - extrasiteconfig += '\n'.join([ - "[amd]", - "library_dirs = %s" % os.path.join(amddir, 'Lib'), - "include_dirs = %s" % os.path.join(amddir, 'Include'), - "amd_libs = amd", - "[umfpack]", - "library_dirs = %s" % os.path.join(umfpackdir, 'Lib'), - "include_dirs = %s" % os.path.join(umfpackdir, 'Include'), - "umfpack_libs = umfpack", - ]) + extrasiteconfig += '\n'.join([ + "[amd]", + "library_dirs = %s" % os.path.join(suitesparseroot, 'lib'), + "include_dirs = %s" % os.path.join(suitesparseroot, 'include'), + "amd_libs = amd", + "[umfpack]", + "library_dirs = %s" % os.path.join(suitesparseroot, 'lib'), + "include_dirs = %s" % os.path.join(suitesparseroot, 'include'), + "umfpack_libs = umfpack", + ]) self.sitecfg = '\n'.join([self.sitecfg, extrasiteconfig]) @@ -217,6 +211,31 @@ def get_libs_for_mkl(varname): cmd = "%s setup.py config" % self.python_cmd run_cmd(cmd, log_all=True, simple=True) + if LooseVersion(self.version) >= LooseVersion('1.26'): + # control BLAS/LAPACK library being used + # see https://github.com/numpy/numpy/blob/v1.26.2/doc/source/release/1.26.1-notes.rst#build-system-changes + # and 'blas-order' in https://github.com/numpy/numpy/blob/v1.26.2/meson_options.txt + blas_lapack_names = { + toolchain.BLIS: 'blis', + toolchain.FLEXIBLAS: 'flexiblas', + toolchain.LAPACK: 'lapack', + toolchain.INTELMKL: 'mkl', + toolchain.OPENBLAS: 'openblas', + } + blas_family = self.toolchain.blas_family() + if blas_family in blas_lapack_names: + self.cfg.update('installopts', "-Csetup-args=-Dblas=" + blas_lapack_names[blas_family]) + else: + raise EasyBuildError("Unknown BLAS library for numpy %s: %s", self.version, blas_family) + + lapack_family = self.toolchain.lapack_family() + if lapack_family in blas_lapack_names: + self.cfg.update('installopts', "-Csetup-args=-Dlapack=" + blas_lapack_names[lapack_family]) + else: + raise EasyBuildError("Unknown LAPACK library for numpy %s: %s", self.version, lapack_family) + + self.cfg.update('installopts', "-Csetup-args=-Dallow-noblas=false") + def test_step(self): """Run available numpy unit tests, and more.""" @@ -332,7 +351,30 @@ def sanity_check_step(self, *args, **kwargs): custom_commands = [] - if LooseVersion(self.version) >= LooseVersion("1.10"): + if LooseVersion(self.version) >= LooseVersion('1.26'): + # make sure BLAS library was found + blas_check_pytxt = '; '.join([ + "import numpy", + "numpy_config = numpy.show_config(mode='dicts')", + "numpy_build_deps = numpy_config['Build Dependencies']", + "blas_found = numpy_build_deps['blas']['found']", + "assert blas_found", + ]) + custom_commands.append('python -c "%s"' % blas_check_pytxt) + + # if FlexiBLAS is used, make sure we are linking to it + # (rather than directly to a backend library like OpenBLAS or Intel MKL) + if self.toolchain.blas_family() == toolchain.FLEXIBLAS: + blas_check_pytxt = '; '.join([ + "import numpy", + "numpy_config = numpy.show_config(mode='dicts')", + "numpy_build_deps = numpy_config['Build Dependencies']", + "blas_name = numpy_build_deps['blas']['name']", + "assert blas_name == 'flexiblas', 'BLAS library should be flexiblas, found %s' % blas_name", + ]) + custom_commands.append('python -c "%s"' % blas_check_pytxt) + + elif LooseVersion(self.version) >= LooseVersion('1.10'): # generic check to see whether numpy v1.10.x and up was built against a CBLAS-enabled library # cfr. https://github.com/numpy/numpy/issues/6675#issuecomment-162601149 blas_check_pytxt = '; '.join([ diff --git a/easybuild/easyblocks/n/nvhpc.py b/easybuild/easyblocks/n/nvhpc.py index b9b27cb009..6efc018a93 100644 --- a/easybuild/easyblocks/n/nvhpc.py +++ b/easybuild/easyblocks/n/nvhpc.py @@ -40,7 +40,7 @@ import sys import platform -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.filetools import adjust_permissions, write_file diff --git a/easybuild/easyblocks/n/nwchem.py b/easybuild/easyblocks/n/nwchem.py index 667b0bbd1e..d60d019855 100644 --- a/easybuild/easyblocks/n/nwchem.py +++ b/easybuild/easyblocks/n/nwchem.py @@ -29,18 +29,18 @@ """ import os import re -import shutil import stat import tempfile import easybuild.tools.config as config import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, remove_file, symlink, write_file +from easybuild.tools.filetools import (adjust_permissions, change_dir, copy_dir, copy_file, mkdir, remove_file, + remove_dir, symlink, write_file) from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version from easybuild.tools.run import run_cmd @@ -325,16 +325,19 @@ def install_step(self): # binary bindir = os.path.join(self.installdir, 'bin') mkdir(bindir) - shutil.copy(os.path.join(self.cfg['start_dir'], 'bin', self.cfg['target'], 'nwchem'), - bindir) + copy_file(os.path.join(self.cfg['start_dir'], 'bin', self.cfg['target'], 'nwchem'), + os.path.join(bindir, 'nwchem')) # data - shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'data'), - os.path.join(self.installdir, 'data')) - shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'basis', 'libraries'), - os.path.join(self.installdir, 'data', 'libraries')) - shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'nwpw', 'libraryps'), - os.path.join(self.installdir, 'data', 'libraryps')) + copy_dir(os.path.join(self.cfg['start_dir'], 'src', 'data'), + os.path.join(self.installdir, 'data')) + copy_dir(os.path.join(self.cfg['start_dir'], 'src', 'basis', 'libraries'), + os.path.join(self.installdir, 'data', 'libraries')) + copy_dir(os.path.join(self.cfg['start_dir'], 'src', 'nwpw', 'libraryps'), + os.path.join(self.installdir, 'data', 'libraryps')) + # examples (needed for the test_cases_step) + copy_dir(os.path.join(self.cfg['start_dir'], 'examples'), + os.path.join(self.installdir, 'examples')) except OSError as err: raise EasyBuildError("Failed to install NWChem: %s", err) @@ -388,23 +391,6 @@ def make_module_extra(self): return txt - def cleanup_step(self): - """Copy stuff from build directory we still need, if any.""" - - try: - exs_dir = os.path.join(self.cfg['start_dir'], 'examples') - - self.examples_dir = os.path.join(tempfile.mkdtemp(), 'examples') - - shutil.copytree(exs_dir, self.examples_dir) - - self.log.info("Copied %s to %s." % (exs_dir, self.examples_dir)) - - except OSError as err: - raise EasyBuildError("Failed to copy examples: %s", err) - - super(EB_NWChem, self).cleanup_step() - def test_cases_step(self): """Run provided list of test cases, or provided examples is no test cases were specified.""" @@ -438,7 +424,7 @@ def test_cases_step(self): ('md/benzene', ['benzene.nw']) ] - self.cfg['tests'] = [(os.path.join(self.examples_dir, d), l) for (d, l) in examples] + self.cfg['tests'] = [(os.path.join(self.installdir, 'examples', d), l) for (d, l) in examples] self.log.info("List of examples to be run as test cases: %s" % self.cfg['tests']) try: @@ -452,7 +438,7 @@ def test_cases_step(self): local_nwchemrc_dir = os.path.dirname(self.local_nwchemrc) if not os.path.exists(local_nwchemrc_dir): os.makedirs(local_nwchemrc_dir) - shutil.copy2(default_nwchemrc, self.local_nwchemrc) + copy_file(default_nwchemrc, self.local_nwchemrc) # only try to create symlink if it's not there yet # we've verified earlier that the symlink is what we expect it to be if it's there @@ -483,11 +469,13 @@ def test_cases_step(self): test_file = os.path.join(testdir, item) if os.path.isfile(test_file): self.log.debug("Copying %s to %s" % (test_file, tmpdir)) - shutil.copy2(test_file, tmpdir) + copy_file(test_file, tmpdir) # run tests for testx in tests: - cmd = "nwchem %s" % testx + # some test cases hang with more than 1 MPI rank on some architectures + n_mpi_ranks = 1 + cmd = '%s %s' % (self.toolchain.mpi_cmd_for('nwchem', n_mpi_ranks), testx) msg = "Running test '%s' (from %s) in %s..." % (cmd, testdir, tmpdir) self.log.info(msg) test_cases_log.write("\n%s\n" % msg) @@ -516,7 +504,7 @@ def test_cases_step(self): # go back change_dir(cwd) - shutil.rmtree(tmpdir) + remove_dir(tmpdir) fail_ratio = fail / tot fail_pcnt = fail_ratio * 100 @@ -534,8 +522,7 @@ def test_cases_step(self): # cleanup try: - shutil.rmtree(self.examples_dir) - shutil.rmtree(local_nwchemrc_dir) + remove_dir(local_nwchemrc_dir) except OSError as err: raise EasyBuildError("Cleanup failed: %s", err) diff --git a/easybuild/easyblocks/o/ocaml.py b/easybuild/easyblocks/o/ocaml.py index fb43648798..0a8d9c8172 100644 --- a/easybuild/easyblocks/o/ocaml.py +++ b/easybuild/easyblocks/o/ocaml.py @@ -29,7 +29,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/o/openbabel.py b/easybuild/easyblocks/o/openbabel.py index b054cc56b0..61e54e2cc5 100644 --- a/easybuild/easyblocks/o/openbabel.py +++ b/easybuild/easyblocks/o/openbabel.py @@ -36,7 +36,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.systemtools import get_shared_lib_ext -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_OpenBabel(CMakeMake): diff --git a/easybuild/easyblocks/o/openblas.py b/easybuild/easyblocks/o/openblas.py index 996dd280e1..e4b5e61bac 100644 --- a/easybuild/easyblocks/o/openblas.py +++ b/easybuild/easyblocks/o/openblas.py @@ -7,7 +7,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.systemtools import POWER, get_cpu_architecture, get_shared_lib_ext diff --git a/easybuild/easyblocks/o/opencv.py b/easybuild/easyblocks/o/opencv.py index 69a97a903d..538d7a7e29 100644 --- a/easybuild/easyblocks/o/opencv.py +++ b/easybuild/easyblocks/o/opencv.py @@ -30,7 +30,7 @@ """ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.easyblocks.generic.pythonpackage import det_pylibdir diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index 68589af669..a13368c0ef 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -41,7 +41,7 @@ import shutil import stat import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/o/openmpi.py b/easybuild/easyblocks/o/openmpi.py index e76dd131bf..b60c01142c 100644 --- a/easybuild/easyblocks/o/openmpi.py +++ b/easybuild/easyblocks/o/openmpi.py @@ -30,7 +30,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/o/openssl.py b/easybuild/easyblocks/o/openssl.py index 8238772899..c84a6e065e 100644 --- a/easybuild/easyblocks/o/openssl.py +++ b/easybuild/easyblocks/o/openssl.py @@ -33,7 +33,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index f73a5e5295..aa4a5dd91c 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -30,7 +30,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.bundle import Bundle from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/o/orca.py b/easybuild/easyblocks/o/orca.py index e6066a28eb..3f08570d81 100644 --- a/easybuild/easyblocks/o/orca.py +++ b/easybuild/easyblocks/o/orca.py @@ -30,7 +30,7 @@ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.makecp import MakeCp from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/p/palm.py b/easybuild/easyblocks/p/palm.py new file mode 100644 index 0000000000..6674be5c7a --- /dev/null +++ b/easybuild/easyblocks/p/palm.py @@ -0,0 +1,76 @@ +## +# Copyright 2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing PALM, implemented as an easyblock + +@author: Viktor Rehnberg (Chalmers University of Technology) +""" +import os + +from easybuild.framework.easyblock import EasyBlock +from easybuild.tools.filetools import find_glob_pattern +from easybuild.tools.run import run_cmd + + +class EB_PALM(EasyBlock): + """Support for building/installing PALM.""" + + def __init__(self, *args, **kwargs): + """Initialise PALM easyblock.""" + super().__init__(*args, **kwargs) + + def configure_step(self): + """No configuration procedure for PALM.""" + pass + + def build_step(self): + """No build procedure for PALM.""" + pass + + def install_step(self): + """Custom install procedure for PALM.""" + + install_script_pattern = "install" + if self.dry_run: + install_script = install_script_pattern + else: + install_script = find_glob_pattern(install_script_pattern) + + cmd = ' '.join([ + self.cfg['preinstallopts'], + "bash", + install_script, + "-p %s" % self.installdir, + self.cfg['installopts'], + ]) + run_cmd(cmd, log_all=True, simple=True) + + def sanity_check_step(self): + """Custom sanity check for PALM.""" + custom_paths = { + 'files': [os.path.join(self.installdir, 'bin', 'palmrun')], + 'dirs': [], + } + super().sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/p/paraver.py b/easybuild/easyblocks/p/paraver.py index 181d660707..08288d2721 100644 --- a/easybuild/easyblocks/p/paraver.py +++ b/easybuild/easyblocks/p/paraver.py @@ -31,7 +31,7 @@ """ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError, print_msg diff --git a/easybuild/easyblocks/p/parmetis.py b/easybuild/easyblocks/p/parmetis.py index a9885b1c75..807ba8bd53 100644 --- a/easybuild/easyblocks/p/parmetis.py +++ b/easybuild/easyblocks/p/parmetis.py @@ -34,7 +34,7 @@ """ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.framework.easyblock import EasyBlock from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/p/perl.py b/easybuild/easyblocks/p/perl.py index 11542473cf..93cc07d6ef 100644 --- a/easybuild/easyblocks/p/perl.py +++ b/easybuild/easyblocks/p/perl.py @@ -28,7 +28,7 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import stat diff --git a/easybuild/easyblocks/p/petsc.py b/easybuild/easyblocks/p/petsc.py index 23c18c169c..d5081d528a 100644 --- a/easybuild/easyblocks/p/petsc.py +++ b/easybuild/easyblocks/p/petsc.py @@ -29,12 +29,12 @@ """ import os import re -from distutils.version import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import BUILD, CUSTOM +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import symlink, apply_regex_substitutions from easybuild.tools.modules import get_software_root, get_software_version @@ -254,26 +254,21 @@ def configure_step(self): ss_libs = ["UMFPACK", "KLU", "CHOLMOD", "BTF", "CCOLAMD", "COLAMD", "CAMD", "AMD"] # More libraries added after version 3.17 if LooseVersion(self.version) >= LooseVersion("3.17"): - # specified order of libs matters! ss_libs = ["UMFPACK", "KLU", "SPQR", "CHOLMOD", "BTF", "CCOLAMD", - "COLAMD", "CSparse", "CXSparse", "LDL", "RBio", - "SLIP_LU", "CAMD", "AMD"] + "COLAMD", "CXSparse", "LDL", "RBio", "SLIP_LU", "CAMD", "AMD"] - suitesparse_inc = [os.path.join(suitesparse, x, "Include") - for x in ss_libs] - suitesparse_inc.append(os.path.join(suitesparse, "SuiteSparse_config")) - inc_spec = "-include=[%s]" % ','.join(suitesparse_inc) + suitesparse_inc = os.path.join(suitesparse, "include") + inc_spec = "-include=[%s]" % suitesparse_inc - suitesparse_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.replace("_", "").lower()) + suitesparse_libs = [os.path.join(suitesparse, "lib", "lib%s.so" % x.replace("_", "").lower()) for x in ss_libs] - suitesparse_libs.append(os.path.join(suitesparse, "SuiteSparse_config", "libsuitesparseconfig.a")) lib_spec = "-lib=[%s]" % ','.join(suitesparse_libs) else: # CHOLMOD and UMFPACK are part of SuiteSparse (PETSc < 3.5) withdep = "--with-umfpack" - inc_spec = "-include=%s" % os.path.join(suitesparse, "UMFPACK", "Include") + inc_spec = "-include=%s" % os.path.join(suitesparse, "include") # specified order of libs matters! - umfpack_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.lower()) + umfpack_libs = [os.path.join(suitesparse, "lib", "lib%s.a" % x.lower()) for x in ["UMFPACK", "CHOLMOD", "COLAMD", "AMD"]] lib_spec = "-lib=[%s]" % ','.join(umfpack_libs) diff --git a/easybuild/easyblocks/p/pgi.py b/easybuild/easyblocks/p/pgi.py index 8d34ae7b06..38e8076756 100644 --- a/easybuild/easyblocks/p/pgi.py +++ b/easybuild/easyblocks/p/pgi.py @@ -39,7 +39,7 @@ import sys import easybuild.tools.environment as env -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.types import ensure_iterable_license_specs diff --git a/easybuild/easyblocks/p/picard.py b/easybuild/easyblocks/p/picard.py index b9f2d1a9b1..e49bc58ca0 100644 --- a/easybuild/easyblocks/p/picard.py +++ b/easybuild/easyblocks/p/picard.py @@ -35,7 +35,7 @@ import os import re import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.tarball import Tarball from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/p/psi.py b/easybuild/easyblocks/p/psi.py index af400c3cec..14324a2711 100644 --- a/easybuild/easyblocks/p/psi.py +++ b/easybuild/easyblocks/p/psi.py @@ -29,7 +29,7 @@ @author: Ward Poelmans (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import shutil diff --git a/easybuild/easyblocks/p/psmpi.py b/easybuild/easyblocks/p/psmpi.py index 755cc1f809..f12b6aa5d9 100644 --- a/easybuild/easyblocks/p/psmpi.py +++ b/easybuild/easyblocks/p/psmpi.py @@ -30,7 +30,7 @@ import easybuild.tools.toolchain as toolchain -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.mpich import EB_MPICH from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 975935cdfb..d01cf50ae8 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -38,7 +38,7 @@ import fileinput import sys import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 58a5e9bdb8..ec9b40a5ee 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -32,7 +32,7 @@ import re import tempfile import easybuild.tools.environment as env -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError, print_warning @@ -288,7 +288,9 @@ def test_step(self): failed_test_cases = re.findall(regex, tests_out, re.M) # And patterns like: # FAILED test_ops_gradients.py::TestGradientsCPU::test_fn_grad_linalg_det_singular_cpu_complex128 - [snip] - regex = r"^(FAILED) \w+\.py.*::(test_.*?) - " + # FAILED [22.8699s] test_sparse_csr.py::TestSparseCompressedCPU::test_invalid_input_csr_large_cpu - [snip] + # FAILED [0.0623s] dynamo/test_dynamic_shapes.py::DynamicShapesExportTests::test_predispatch - [snip] + regex = r"^(FAILED) (?:\[.*?\] )?(?:\w|/)+\.py.*::(test_.*?) - " failed_test_cases.extend(re.findall(regex, tests_out, re.M)) if failed_test_cases: errored_test_cases = sorted(m[1] for m in failed_test_cases if m[0] == 'ERROR') @@ -350,11 +352,13 @@ def get_count_for_pattern(regex, text): # OR: # ===================== 2 failed, 128 passed, 2 skipped, 2 warnings in 63.43s (01:03:43) ========= # FINISHED PRINTING LOG FILE + # # test_quantization failed! regex = ( r"^=+ (?P.*) in [0-9]+\.*[0-9]*[a-zA-Z]* (\([0-9]+:[0-9]+:[0-9]+\) )?=+$\n" - r"(?:^(?:(?!failed!).)*$\n){0,5}" + r"(?:.*FINISHED PRINTING LOG FILE.*\n)?" + r"(?:^\s*\n)*" r"(?P.*) failed!$" ) @@ -414,7 +418,23 @@ def get_count_for_pattern(regex, text): # Calculate total number of unsuccesful and total tests failed_test_cnt = failure_cnt + error_cnt - test_cnt = sum(int(hit) for hit in re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M)) + # Pattern for tests ran with unittest like: + # Ran 3 tests in 0.387s + regex = r"^Ran (?P[0-9]+) tests in" + test_cnt = sum(int(hit) for hit in re.findall(regex, tests_out, re.M)) + # Pattern for tests ran with pytest like: + # ============ 286 passed, 18 skipped, 2 xfailed in 38.71s ============ + regex = r"=+ (?P.*) in \d+.* =+\n" + count_patterns = [re.compile(r"([0-9]+) " + reason) for reason in [ + "failed", + "passed", + "skipped", + "deselected", + "xfailed", + "xpassed", + ]] + for m in re.finditer(regex, tests_out, re.M): + test_cnt += sum(get_count_for_pattern(p, m.group("summary")) for p in count_patterns) if failed_test_cnt > 0: max_failed_tests = self.cfg['max_failed_tests'] diff --git a/easybuild/easyblocks/q/qscintilla.py b/easybuild/easyblocks/q/qscintilla.py index 3a64d6c607..ef98de5df6 100644 --- a/easybuild/easyblocks/q/qscintilla.py +++ b/easybuild/easyblocks/q/qscintilla.py @@ -29,7 +29,7 @@ author: Maxime Boissonneault (Compute Canada) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.pythonpackage import det_pylibdir diff --git a/easybuild/easyblocks/q/qt.py b/easybuild/easyblocks/q/qt.py index cc23f27a99..2cace72e0c 100644 --- a/easybuild/easyblocks/q/qt.py +++ b/easybuild/easyblocks/q/qt.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/q/quantumespresso.py b/easybuild/easyblocks/q/quantumespresso.py index b5a232817b..15d00f53d0 100644 --- a/easybuild/easyblocks/q/quantumespresso.py +++ b/easybuild/easyblocks/q/quantumespresso.py @@ -33,7 +33,7 @@ import re import shutil import sys -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/r/r.py b/easybuild/easyblocks/r/r.py index 5f73952873..27d5d2ac32 100644 --- a/easybuild/easyblocks/r/r.py +++ b/easybuild/easyblocks/r/r.py @@ -30,7 +30,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/r/repeatmasker.py b/easybuild/easyblocks/r/repeatmasker.py index 0a4d10329b..e0a50b1a1d 100644 --- a/easybuild/easyblocks/r/repeatmasker.py +++ b/easybuild/easyblocks/r/repeatmasker.py @@ -25,7 +25,7 @@ """ EasyBuild support for building and installing RepeatMasker, implemented as an easyblock """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.tarball import Tarball diff --git a/easybuild/easyblocks/r/rmpi.py b/easybuild/easyblocks/r/rmpi.py index c9a1528acd..8bca1893be 100644 --- a/easybuild/easyblocks/r/rmpi.py +++ b/easybuild/easyblocks/r/rmpi.py @@ -33,7 +33,7 @@ @author: Balazs Hajgato (Vrije Universiteit Brussel) """ import easybuild.tools.toolchain as toolchain -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.rpackage import RPackage diff --git a/easybuild/easyblocks/r/root.py b/easybuild/easyblocks/r/root.py index 872671eb4b..2a10a86d57 100644 --- a/easybuild/easyblocks/r/root.py +++ b/easybuild/easyblocks/r/root.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/r/rust.py b/easybuild/easyblocks/r/rust.py index ee66f6854e..777d962391 100644 --- a/easybuild/easyblocks/r/rust.py +++ b/easybuild/easyblocks/r/rust.py @@ -59,6 +59,9 @@ def configure_step(self): self.cfg.update('configopts', "--sysconfdir=%s" % os.path.join(self.installdir, 'etc')) + # old llvm builds from CI get deleted after a certain time + self.cfg.update('configopts', "--set=llvm.download-ci-llvm=false") + # don't use Ninja if it is not listed as a build dependency; # may be because Ninja requires Python, and Rust is a build dependency for cryptography # which may be included as an extension with Python diff --git a/easybuild/easyblocks/s/samtools.py b/easybuild/easyblocks/s/samtools.py index 6d297c2283..4be92850f0 100644 --- a/easybuild/easyblocks/s/samtools.py +++ b/easybuild/easyblocks/s/samtools.py @@ -19,7 +19,7 @@ @author: Fotis Georgatos (Uni.Lu) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import stat diff --git a/easybuild/easyblocks/s/scalapack.py b/easybuild/easyblocks/s/scalapack.py index 466fef57c3..11d285e4ae 100644 --- a/easybuild/easyblocks/s/scalapack.py +++ b/easybuild/easyblocks/s/scalapack.py @@ -34,7 +34,7 @@ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.blacs import det_interface # @UnresolvedImport diff --git a/easybuild/easyblocks/s/scalasca1.py b/easybuild/easyblocks/s/scalasca1.py index a78398cc9d..d991af9a31 100644 --- a/easybuild/easyblocks/s/scalasca1.py +++ b/easybuild/easyblocks/s/scalasca1.py @@ -29,7 +29,7 @@ @author: Bernd Mohr (Juelich Supercomputing Centre) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/s/scipion.py b/easybuild/easyblocks/s/scipion.py index 86d7460a33..42d21a981b 100644 --- a/easybuild/easyblocks/s/scipion.py +++ b/easybuild/easyblocks/s/scipion.py @@ -30,7 +30,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy, mkdir, symlink diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 094a0259b5..b0551a2b74 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -35,7 +35,7 @@ """ import os import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/s/scotch.py b/easybuild/easyblocks/s/scotch.py index 89c728f7c9..b09d3cdc0d 100644 --- a/easybuild/easyblocks/s/scotch.py +++ b/easybuild/easyblocks/s/scotch.py @@ -32,7 +32,7 @@ @author: Jens Timmerman (Ghent University) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/s/siesta.py b/easybuild/easyblocks/s/siesta.py index 81b132236e..69a911fa80 100644 --- a/easybuild/easyblocks/s/siesta.py +++ b/easybuild/easyblocks/s/siesta.py @@ -32,7 +32,7 @@ import stat import easybuild.tools.toolchain as toolchain -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/s/slepc.py b/easybuild/easyblocks/s/slepc.py index 0254f195ad..43ecff7a4c 100644 --- a/easybuild/easyblocks/s/slepc.py +++ b/easybuild/easyblocks/s/slepc.py @@ -30,7 +30,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/s/snphylo.py b/easybuild/easyblocks/s/snphylo.py index 88eb26f7ae..7fed0ace86 100644 --- a/easybuild/easyblocks/s/snphylo.py +++ b/easybuild/easyblocks/s/snphylo.py @@ -31,7 +31,7 @@ import os import re import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.framework.easyblock import EasyBlock from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index f25fc66785..c581b3c7a1 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -35,14 +35,14 @@ import fileinput import re import os -import shutil import sys -import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import toolchain from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import mkdir, write_file, adjust_permissions, copy_dir +from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root from easybuild.tools.modules import get_software_libdir from easybuild.tools.systemtools import get_shared_lib_ext @@ -51,6 +51,14 @@ class EB_SuiteSparse(ConfigureMake): """Support for building SuiteSparse.""" + @staticmethod + def extra_options(extra_vars=None): + """Define extra easyconfig parameters""" + extra_vars = { + 'cmake_options': ['', "CMAKE_OPTIONS used by SuiteSparse since v6.0", CUSTOM], + } + return ConfigureMake.extra_options(extra_vars) + def __init__(self, *args, **kwargs): """Custom constructor for SuiteSparse easyblock, initialize custom class parameters.""" super(EB_SuiteSparse, self).__init__(*args, **kwargs) @@ -61,23 +69,26 @@ def configure_step(self): if LooseVersion(self.version) < LooseVersion('4.0'): self.config_name = 'UFconfig' - else: + elif LooseVersion(self.version) < LooseVersion('6.0.0'): self.config_name = 'SuiteSparse_config' + else: + # config file is removed after v6.0.0 + self.config_name = '' cfgvars = { - 'CC': os.getenv('MPICC'), + 'CC': os.getenv('CC'), 'CFLAGS': os.getenv('CFLAGS'), - 'CXX': os.getenv('MPICXX'), - 'F77': os.getenv('MPIF77'), + 'CXX': os.getenv('CXX'), + 'F77': os.getenv('F77'), 'F77FLAGS': os.getenv('F77FLAGS'), + 'BLAS': os.getenv('LIBBLAS_MT'), + 'LAPACK': os.getenv('LIBLAPACK_MT'), } - # avoid that (system) Intel compilers are always considered - self.cfg.update('buildopts', 'AUTOCC=no') - - # Set BLAS and LAPACK libraries as specified in SuiteSparse README.txt - self.cfg.update('buildopts', 'BLAS="%s"' % os.getenv('LIBBLAS_MT')) - self.cfg.update('buildopts', 'LAPACK="%s"' % os.getenv('LIBLAPACK_MT')) + cmake = get_software_root('CMake') + if not cmake and LooseVersion(self.version) >= LooseVersion('5.1.2'): + # graphblas exists from v5.1.2, needs cmake + raise EasyBuildError("CMake module is not loaded") # Get CUDA and set it up appropriately cuda = get_software_root('CUDA') @@ -91,125 +102,117 @@ def configure_step(self): # Get METIS or ParMETIS settings metis = get_software_root('METIS') parmetis = get_software_root('ParMETIS') - if parmetis: - metis_path = parmetis - metis_include = os.path.join(parmetis, 'include') - metis_libs = os.path.join(parmetis, get_software_libdir('ParMETIS'), 'libmetis.a') - - elif metis: - metis_path = metis - metis_include = os.path.join(metis, 'include') - metis_libs = os.path.join(metis, get_software_libdir('METIS'), 'libmetis.a') + if parmetis or metis: + if parmetis: + metis_name = 'ParMETIS' + else: + metis_name = 'METIS' + metis_path = get_software_root(metis_name) + metis_include = os.path.join(metis_path, 'include') + metis_libs = os.path.join(metis_path, get_software_libdir(metis_name), 'libmetis.a') else: - raise EasyBuildError("Neither METIS or ParMETIS module loaded.") + self.log.info("Use METIS built in SuiteSparse") + # raise EasyBuildError("Neither METIS or ParMETIS module loaded.") - if LooseVersion(self.version) >= LooseVersion('4.5.1'): + # config file can catch environment variables after v4.5.0 + if LooseVersion(self.version) < LooseVersion('4.5.0'): cfgvars.update({ - 'MY_METIS_LIB': metis_libs, - 'MY_METIS_INC': metis_include, - }) - else: - cfgvars.update({ - 'METIS_PATH': metis_path, - 'METIS': metis_libs, + 'INSTALL_LIB': os.path.join(self.installdir, 'lib'), + 'INSTALL_INCLUDE': os.path.join(self.installdir, 'include'), }) + if parmetis or metis: + cfgvars.update({ + 'METIS_PATH': metis_path, + 'METIS': metis_libs, + }) - # patch file - fp = os.path.join(self.cfg['start_dir'], self.config_name, '%s.mk' % self.config_name) - - try: - for line in fileinput.input(fp, inplace=1, backup='.orig'): - for (var, val) in list(cfgvars.items()): - # Let's overwrite NVCCFLAGS at the end, since the line breaks and the fact that it appears multiple - # times makes it tricky to handle it properly - if var != 'NVCCFLAGS': - orig_line = line - # for variables in cfgvars, substiture lines assignment - # in the file, whatever they are, by assignments to the - # values in cfgvars - line = re.sub(r"^\s*(%s\s*=\s*).*\n$" % var, - r"\1 %s # patched by EasyBuild\n" % val, - line) - if line != orig_line: - cfgvars.pop(var) - sys.stdout.write(line) - except IOError as err: - raise EasyBuildError("Failed to patch %s in: %s", fp, err) - - # add remaining entries at the end - if cfgvars: - cfgtxt = '# lines below added automatically by EasyBuild\n' - cfgtxt += '\n'.join(["%s = %s" % (var, val) for (var, val) in cfgvars.items()]) - write_file(fp, cfgtxt, append=True) + # patch file + fp = os.path.join(self.cfg['start_dir'], self.config_name, '%s.mk' % self.config_name) - def install_step(self): - """Install by copying the contents of the builddir to the installdir (preserving permissions)""" - for x in os.listdir(self.cfg['start_dir']): - src = os.path.join(self.cfg['start_dir'], x) - dst = os.path.join(self.installdir, x) try: - if os.path.isdir(src): - # symlink points to CUDA folder that is - # not created for non GPU nodes. shutil - # throws an error in this case. - copy_dir(src, dst, symlinks=True) - # symlink - # - dst/Lib to dst/lib - # - dst/Include to dst/include - for c in ['Lib', 'Include']: - nsrc = os.path.join(dst, c) - ndst = os.path.join(dst, c.lower()) - if os.path.exists(nsrc): - os.symlink(nsrc, ndst) - # enable r-x permissions for group/others - perms = stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH - adjust_permissions(dst, perms, add=True, recursive=True, onlydirs=True) + for line in fileinput.input(fp, inplace=1, backup='.orig'): + for (var, val) in list(cfgvars.items()): + # Let's overwrite NVCCFLAGS at the end, since the line breaks and + # the fact that it appears multiple times makes it tricky to handle it properly + # path variables are also moved to the end + if var not in ['NVCCFLAGS', 'INSTALL_LIB', 'INSTALL_INCLUDE', 'METIS_PATH']: + orig_line = line + # for variables in cfgvars, substiture lines assignment + # in the file, whatever they are, by assignments to the + # values in cfgvars + line = re.sub(r"^\s*(%s\s*=\s*).*\n$" % var, + r"\1 %s # patched by EasyBuild\n" % val, + line) + if line != orig_line: + cfgvars.pop(var) + sys.stdout.write(line) + except IOError as err: + raise EasyBuildError("Failed to patch %s in: %s", fp, err) + + # add remaining entries at the end + if cfgvars: + cfgtxt = '# lines below added automatically by EasyBuild\n' + cfgtxt += '\n'.join(["%s = %s" % (var, val) for (var, val) in cfgvars.items()]) + write_file(fp, cfgtxt, append=True) + + elif LooseVersion(self.version) < LooseVersion('6.0.0'): + # avoid that (system) Intel compilers are always considered + self.cfg.update('prebuildopts', 'AUTOCC=no') + + # Set BLAS and LAPACK libraries as specified in SuiteSparse README.txt + self.cfg.update('buildopts', 'BLAS="%s"' % cfgvars.get('BLAS')) + self.cfg.update('buildopts', 'LAPACK="%s"' % cfgvars.get('LAPACK')) + + self.cfg.update('installopts', 'INSTALL="%s"' % self.installdir) + self.cfg.update('installopts', 'BLAS="%s"' % cfgvars.get('BLAS')) + self.cfg.update('installopts', 'LAPACK="%s"' % cfgvars.get('LAPACK')) + + if LooseVersion(self.version) >= LooseVersion('5.1.2'): + # v5.0.0 until v5.1.2 has no CMAKE_OPTIONS to set, patches are needed + self.cfg.update('preinstallopts', 'CMAKE_OPTIONS="-DCMAKE_INSTALL_PREFIX=%s"' % self.installdir) + + # set METIS library + if parmetis or metis: + if LooseVersion(self.version) == LooseVersion('4.5.0'): + self.cfg.update('buildopts', 'METIS_PATH="%s"' % metis_path) + self.cfg.update('installopts', 'METIS_PATH="%s"' % metis_path) else: - shutil.copy2(src, dst) - except OSError as err: - raise EasyBuildError("Copying src %s to dst %s failed: %s", src, dst, err) - - # some extra symlinks are necessary for UMFPACK to work. - paths = [ - os.path.join('AMD', 'include', 'amd.h'), - os.path.join('AMD', 'include', 'amd_internal.h'), - os.path.join(self.config_name, '%s.h' % self.config_name), - os.path.join('AMD', 'lib', 'libamd.a') - ] - for path in paths: - src = os.path.join(self.installdir, path) - dn = path.split(os.path.sep)[-2] - fn = path.split(os.path.sep)[-1] - dstdir = os.path.join(self.installdir, 'UMFPACK', dn) - mkdir(dstdir) - if os.path.exists(src): - try: - os.symlink(src, os.path.join(dstdir, fn)) - except OSError as err: - raise EasyBuildError("Failed to make symbolic link from %s to %s: %s", src, dst, err) - - def make_module_req_guess(self): - """ - Extra path to consider for module file: - * add config dir and include to $CPATH so include files are found - * add UMFPACK and AMD library, and lib dirs to $LD_LIBRARY_PATH - """ - - guesses = super(EB_SuiteSparse, self).make_module_req_guess() - - # Previous versions of SuiteSparse used specific directories for includes and libraries - if LooseVersion(self.version) < LooseVersion('4.5'): - include_dirs = [self.config_name] - ld_library_path = ['AMD/lib', 'BTF/lib', 'CAMD/lib', 'CCOLAMD/lib', 'CHOLAMD/lib', 'CHOLMOD/lib', - 'COLAMD/lib/', 'CSparse/lib', 'CXSparse/lib', 'KLU/lib', 'LDL/lib', 'RBio/lib', - 'UMFPACK/lib', self.config_name] + self.cfg.update('buildopts', 'MY_METIS_LIB="%s"' % metis_libs) + self.cfg.update('buildopts', 'MY_METIS_INC="%s"' % metis_include) + self.cfg.update('installopts', 'MY_METIS_LIB="%s"' % metis_libs) + self.cfg.update('installopts', 'MY_METIS_INC="%s"' % metis_include) - guesses['CPATH'].extend(include_dirs) - guesses['LD_LIBRARY_PATH'].extend(ld_library_path) - guesses['LIBRARY_PATH'].extend(ld_library_path) + else: + # after v6.0.0, no option for metis, its own metis is used anyway + # set CMAKE_OPTIONS if it is not specified in easyconfigs + # CMAKE_INSTALL_PREFIX is managed by easybuild + cmake_options = '-DCMAKE_INSTALL_PREFIX=%s' % self.installdir + + lapack_lib = self.toolchain.lapack_family() + if '-DBLA_VENDOR=' in self.cfg['cmake_options']: + blas_lapack = '' + elif lapack_lib == toolchain.FLEXIBLAS: + blas_lapack = '-DBLA_VENDOR=FlexiBLAS' + elif lapack_lib == toolchain.INTELMKL: + blas_lapack = '-DBLA_VENDOR=Intel' + elif lapack_lib == toolchain.OPENBLAS: + blas_lapack = '-DBLA_VENDOR=OpenBLAS' + else: + raise EasyBuildError("BLA_VENDOR is not assigned and FlexiBLAS/MKL/OpenBLAS are not found. " + "Please assign BLA_VENDOR in cmake_options in easyconfigs") + + cmake_options = ' '.join([cmake_options, blas_lapack, self.cfg['cmake_options']]) + self.cfg.update('prebuildopts', 'CMAKE_OPTIONS="%s"' % cmake_options) + + def install_step(self): + """Install by copying the contents of the builddir to the installdir (preserving permissions)""" + + if LooseVersion(self.version) < LooseVersion('4.5.0'): + mkdir(os.path.join(self.installdir, 'lib')) + mkdir(os.path.join(self.installdir, 'include')) - return guesses + super(EB_SuiteSparse, self).install_step() def sanity_check_step(self): """Custom sanity check for SuiteSparse.""" @@ -219,24 +222,17 @@ def sanity_check_step(self): raise EasyBuildError("SuiteSparse has compiled its own Metis. This will conflict with the Metis build." " The SuiteSparse EasyBlock need to be updated!") + shlib_ext = get_shared_lib_ext() libnames = ['AMD', 'BTF', 'CAMD', 'CCOLAMD', 'CHOLMOD', 'COLAMD', 'CXSparse', 'KLU', 'LDL', 'RBio', 'SPQR', 'UMFPACK'] - libs = [os.path.join(x, 'lib', 'lib%s.a' % x.lower()) for x in libnames] - - if LooseVersion(self.version) < LooseVersion('4.0'): - csparse_dir = 'CSparse3' + if LooseVersion(self.version) < LooseVersion('4.5'): + libs = [os.path.join('lib', 'lib%s.a' % x.lower()) for x in libnames] else: - csparse_dir = 'CSparse' - libs.append(os.path.join(csparse_dir, 'lib', 'libcsparse.a')) - - # Latest version of SuiteSparse also compiles shared library and put them in 'lib' - shlib_ext = get_shared_lib_ext() - if LooseVersion(self.version) >= LooseVersion('4.5.1'): - libs += [os.path.join('lib', 'lib%s.%s' % (x.lower(), shlib_ext)) for x in libnames] + libs = [os.path.join('lib', 'lib%s.%s' % (x.lower(), shlib_ext)) for x in libnames] custom_paths = { 'files': libs, - 'dirs': ['MATLAB_Tools'], + 'dirs': [], } super(EB_SuiteSparse, self).sanity_check_step(custom_paths=custom_paths) diff --git a/easybuild/easyblocks/s/superlu.py b/easybuild/easyblocks/s/superlu.py index eafa3c1e3e..9e85160587 100644 --- a/easybuild/easyblocks/s/superlu.py +++ b/easybuild/easyblocks/s/superlu.py @@ -30,7 +30,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/t/tbb.py b/easybuild/easyblocks/t/tbb.py index e3f34e1ab8..995bfa854a 100644 --- a/easybuild/easyblocks/t/tbb.py +++ b/easybuild/easyblocks/t/tbb.py @@ -37,7 +37,7 @@ import glob import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.intelbase import INSTALL_MODE_NAME_2015, INSTALL_MODE_2015 diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 926a1920cf..174a5e38fe 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -35,6 +35,7 @@ import re import stat import tempfile +from itertools import chain import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain @@ -43,12 +44,13 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools import run, LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning -from easybuild.tools.config import build_option, IGNORE +from easybuild.tools.config import build_option, IGNORE, WARN, ERROR from easybuild.tools.filetools import adjust_permissions, apply_regex_substitutions, copy_file, mkdir, resolve_path -from easybuild.tools.filetools import is_readable, read_file, which, write_file, remove_file +from easybuild.tools.filetools import is_readable, read_file, symlink, which, write_file, remove_file from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import AARCH64, X86_64, get_cpu_architecture, get_os_name, get_os_version +from easybuild.tools.toolchain.toolchain import RPATH_WRAPPERS_SUBDIR CPU_DEVICE = 'cpu' @@ -76,6 +78,8 @@ %(compiler_path)s "$@" """ +KNOWN_BINUTILS = ('ar', 'as', 'dwp', 'ld', 'ld.bfd', 'ld.gold', 'nm', 'objcopy', 'objdump', 'strip') + def split_tf_libs_txt(valid_libs_txt): """Split the VALID_LIBS entry from the TF file into single names""" @@ -299,7 +303,7 @@ def handle_jemalloc(self): def write_wrapper(self, wrapper_dir, compiler, i_mpi_root): """Helper function to write a compiler wrapper.""" wrapper_txt = INTEL_COMPILER_WRAPPER % { - 'compiler_path': which(compiler), + 'compiler_path': which(compiler, on_error=IGNORE if self.dry_run else ERROR), 'intel_mpi_root': i_mpi_root, 'cpath': os.getenv('CPATH'), 'intel_license_file': os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')), @@ -445,10 +449,13 @@ def setup_build_dirs(self): self.output_user_root_dir = os.path.join(parent_dir, 'bazel-root') # Folder where wrapper binaries can be placed, where required. TODO: Replace by --action_env cmds self.wrapper_dir = os.path.join(parent_dir, 'wrapper_bin') + mkdir(self.wrapper_dir) def configure_step(self): """Custom configuration procedure for TensorFlow.""" + self.setup_build_dirs() + # Bazel seems to not be able to handle a large amount of parallel jobs, e.g. 176 on some Power machines, # and will hang forever building the TensorFlow package. # So limit to something high but still reasonable while allowing ECs to overwrite it @@ -459,8 +466,34 @@ def configure_step(self): # determine location where binutils' ld command is installed # note that this may be an RPATH wrapper script (when EasyBuild is configured with --rpath) - ld_path = which('ld') + ld_path = which('ld', on_error=ERROR) self.binutils_bin_path = os.path.dirname(ld_path) + if self.toolchain.is_rpath_wrapper(ld_path): + # TF expects all binutils binaries in a single path but newer EB puts each in its own subfolder + # This new layout is: /RPATH_WRAPPERS_SUBDIR/_folder/ + rpath_wrapper_root = os.path.dirname(os.path.dirname(ld_path)) + if os.path.basename(rpath_wrapper_root) == RPATH_WRAPPERS_SUBDIR: + # Add symlinks to each binutils binary into a single folder + new_rpath_wrapper_dir = os.path.join(self.wrapper_dir, RPATH_WRAPPERS_SUBDIR) + binutils_root = get_software_root('binutils') + if binutils_root: + self.log.debug("Using binutils dependency at %s to gather binutils files.", binutils_root) + binutils_files = next(os.walk(os.path.join(binutils_root, 'bin')))[2] + else: + # binutils might be filtered (--filter-deps), so recursively gather files in the rpath wrapper dir + binutils_files = {f for (_, _, files) in os.walk(rpath_wrapper_root) for f in files} + # And add known ones + binutils_files.update(KNOWN_BINUTILS) + self.log.info("Found %s to be an rpath wrapper. Adding symlinks for binutils (%s) to %s.", + ld_path, ', '.join(binutils_files), new_rpath_wrapper_dir) + mkdir(new_rpath_wrapper_dir) + for file in binutils_files: + # use `which` to take rpath wrappers where available + # Ignore missing ones if binutils was filtered (in which case we used a heuristic) + path = which(file, on_error=ERROR if binutils_root else WARN) + if path: + symlink(path, os.path.join(new_rpath_wrapper_dir, file)) + self.binutils_bin_path = new_rpath_wrapper_dir # filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that # might conflict with dependencies on the system and/or installed with EB. For example: protobuf @@ -473,8 +506,6 @@ def configure_step(self): filtered_path = os.pathsep.join([p for fil in path_filter for p in path if fil not in p]) env.setvar(var, filtered_path) - self.setup_build_dirs() - use_wrapper = False if self.toolchain.comp_family() == toolchain.INTELCOMP: # put wrappers for Intel C/C++ compilers in place (required to make sure license server is found) @@ -571,9 +602,9 @@ def configure_step(self): # $GCC_HOST_COMPILER_PATH should be set to path of the actual compiler (not the MPI compiler wrapper) if use_mpi: - compiler_path = which(os.getenv('CC_SEQ')) + compiler_path = which(os.getenv('CC_SEQ'), on_error=ERROR) else: - compiler_path = which(os.getenv('CC')) + compiler_path = which(os.getenv('CC'), on_error=ERROR) # list of CUDA compute capabilities to use can be specifed in two ways (where (2) overrules (1)): # (1) in the easyconfig file, via the custom cuda_compute_capabilities; @@ -749,12 +780,11 @@ def patch_crosstool_files(self): (r'(cxx_builtin_include_directory:).*', ''), (r'^toolchain {', 'toolchain {\n' + '\n'.join(cxx_inc_dirs)), ] - for tool in ['ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy', 'objdump', 'strip']: - path = which(tool) + required_tools = {'ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy', 'objdump', 'strip'} + for tool in set(chain(required_tools, KNOWN_BINUTILS)): + path = which(tool, on_error=ERROR if tool in required_tools else WARN) if path: regex_subs.append((os.path.join('/usr', 'bin', tool), path)) - else: - raise EasyBuildError("Failed to determine path to '%s'", tool) # -fPIE/-pie and -fPIC are not compatible, so patch out hardcoded occurences of -fPIE/-pie if -fPIC is used if self.toolchain.options.get('pic', None): diff --git a/easybuild/easyblocks/t/tensorrt.py b/easybuild/easyblocks/t/tensorrt.py index 204302e7d6..8c818b959c 100644 --- a/easybuild/easyblocks/t/tensorrt.py +++ b/easybuild/easyblocks/t/tensorrt.py @@ -30,7 +30,7 @@ """ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.easyblocks.generic.pythonpackage import PythonPackage, PIP_INSTALL_CMD diff --git a/easybuild/easyblocks/t/tinker.py b/easybuild/easyblocks/t/tinker.py index 4892873b0d..78ecd57312 100644 --- a/easybuild/easyblocks/t/tinker.py +++ b/easybuild/easyblocks/t/tinker.py @@ -31,7 +31,7 @@ import os import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/t/tkinter.py b/easybuild/easyblocks/t/tkinter.py index 8f4312f4b5..eb792c876c 100644 --- a/easybuild/easyblocks/t/tkinter.py +++ b/easybuild/easyblocks/t/tkinter.py @@ -33,7 +33,7 @@ import glob import os import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.pythonpackage import det_pylibdir diff --git a/easybuild/easyblocks/t/trilinos.py b/easybuild/easyblocks/t/trilinos.py index aa4b911473..51ed37e9cc 100644 --- a/easybuild/easyblocks/t/trilinos.py +++ b/easybuild/easyblocks/t/trilinos.py @@ -31,11 +31,10 @@ import random import re -from distutils.version import LooseVersion - import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_path from easybuild.tools.filetools import mkdir, remove_dir, symlink @@ -144,14 +143,12 @@ def configure_step(self): if suitesparse: self.cfg.update('configopts', "-DTPL_ENABLE_UMFPACK:BOOL=ON") self.cfg.update('configopts', "-DTPL_ENABLE_Cholmod:BOOL=ON") - incdirs, libdirs, libnames = [], [], [] - for lib in ["UMFPACK", "CHOLMOD", "COLAMD", "AMD", "CCOLAMD", "CAMD"]: - incdirs.append(os.path.join(suitesparse, lib, "Include")) - libdirs.append(os.path.join(suitesparse, lib, "Lib")) - libnames.append(lib.lower()) + incdir = os.path.join(suitesparse, "include") + libdir = os.path.join(suitesparse, "lib") + libs = ["UMFPACK", "CHOLMOD", "COLAMD", "AMD", "CCOLAMD", "CAMD"] + libnames = [lib.lower() for lib in libs] # add SuiteSparse config lib, it is in recent versions of suitesparse - libdirs.append(os.path.join(suitesparse, 'SuiteSparse_config')) libnames.append('suitesparseconfig') # because of "SuiteSparse_config.c:function SuiteSparse_tic: error: undefined reference to 'clock_gettime'" libnames.append('rt') @@ -162,11 +159,11 @@ def configure_step(self): # see https://answers.launchpad.net/dorsal/+question/223167 libnames.append('libmetis.a') - self.cfg.update('configopts', '-DUMFPACK_INCLUDE_DIRS:PATH="%s"' % ';'.join(incdirs)) - self.cfg.update('configopts', '-DUMFPACK_LIBRARY_DIRS:PATH="%s"' % ';'.join(libdirs)) + self.cfg.update('configopts', '-DUMFPACK_INCLUDE_DIRS:PATH="%s"' % incdir) + self.cfg.update('configopts', '-DUMFPACK_LIBRARY_DIRS:PATH="%s"' % libdir) self.cfg.update('configopts', '-DUMFPACK_LIBRARY_NAMES:STRING="%s"' % ';'.join(libnames)) - self.cfg.update('configopts', '-DCholmod_INCLUDE_DIRS:PATH="%s"' % ';'.join(incdirs)) - self.cfg.update('configopts', '-DCholmod_LIBRARY_DIRS:PATH="%s"' % ';'.join(libdirs)) + self.cfg.update('configopts', '-DCholmod_INCLUDE_DIRS:PATH="%s"' % incdir) + self.cfg.update('configopts', '-DCholmod_LIBRARY_DIRS:PATH="%s"' % libdir) self.cfg.update('configopts', '-DCholmod_LIBRARY_NAMES:STRING="%s"' % ';'.join(libnames)) # BLACS diff --git a/easybuild/easyblocks/t/trinity.py b/easybuild/easyblocks/t/trinity.py index 7eeedf6e09..72cf9e454b 100644 --- a/easybuild/easyblocks/t/trinity.py +++ b/easybuild/easyblocks/t/trinity.py @@ -36,7 +36,7 @@ import glob import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/u/ufc.py b/easybuild/easyblocks/u/ufc.py index f77cf7ea24..361409dff7 100644 --- a/easybuild/easyblocks/u/ufc.py +++ b/easybuild/easyblocks/u/ufc.py @@ -27,7 +27,7 @@ @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakepythonpackage import CMakePythonPackage from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/v/vmd.py b/easybuild/easyblocks/v/vmd.py index c7273ab5c6..3c122bd2e0 100644 --- a/easybuild/easyblocks/v/vmd.py +++ b/easybuild/easyblocks/v/vmd.py @@ -31,7 +31,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.pythonpackage import det_pylibdir from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/v/vtune.py b/easybuild/easyblocks/v/vtune.py index 40253f57af..5459f38b00 100644 --- a/easybuild/easyblocks/v/vtune.py +++ b/easybuild/easyblocks/v/vtune.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) @author: Damian Alvarez (Forschungzentrum Juelich GmbH) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 diff --git a/easybuild/easyblocks/w/wien2k.py b/easybuild/easyblocks/w/wien2k.py index 07c32ad40c..32430c4f21 100644 --- a/easybuild/easyblocks/w/wien2k.py +++ b/easybuild/easyblocks/w/wien2k.py @@ -39,7 +39,7 @@ import shutil import sys import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/w/wps.py b/easybuild/easyblocks/w/wps.py index d6c6cca849..4dba12ac02 100644 --- a/easybuild/easyblocks/w/wps.py +++ b/easybuild/easyblocks/w/wps.py @@ -35,7 +35,7 @@ import os import re import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index ddd6387fd8..a21eabd12c 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -35,7 +35,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/w/wxpython.py b/easybuild/easyblocks/w/wxpython.py index 939e2718a1..524b90a6a8 100644 --- a/easybuild/easyblocks/w/wxpython.py +++ b/easybuild/easyblocks/w/wxpython.py @@ -32,7 +32,7 @@ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_python_version from easybuild.tools.filetools import change_dir, symlink from easybuild.tools.modules import get_software_root diff --git a/easybuild/easyblocks/x/xmipp.py b/easybuild/easyblocks/x/xmipp.py index 95230cd03e..3043a4c93a 100644 --- a/easybuild/easyblocks/x/xmipp.py +++ b/easybuild/easyblocks/x/xmipp.py @@ -34,7 +34,7 @@ import os import easybuild.tools.environment as env -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.scons import SCons from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import apply_regex_substitutions, change_dir diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index db44834031..1fac194147 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -46,6 +46,7 @@ from easybuild.easyblocks.generic.juliapackage import JuliaPackage from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.easyblocks.generic.pythonbundle import PythonBundle +from easybuild.easyblocks.generic.cargopythonbundle import CargoPythonBundle from easybuild.easyblocks.gcc import EB_GCC from easybuild.easyblocks.imod import EB_IMOD from easybuild.easyblocks.fftwmpi import EB_FFTW_period_MPI @@ -279,7 +280,7 @@ def template_module_only_test(self, easyblock, name, version='1.3.2', extra_txt= # $JAVA_HOME must be set for IMOD os.environ['JAVA_HOME'] = tmpdir - elif app_class == PythonBundle: + elif app_class == PythonBundle or app_class == CargoPythonBundle: # $EBROOTPYTHON must be set for PythonBundle easyblock os.environ['EBROOTPYTHON'] = '/fake/install/prefix/Python/2.7.14-foss-2018a'