Download MKL deps from PiPy (#1432)

.ci/pipeline/ci.yml

@@ -77,8 +77,6 @@ jobs:
                            openjdk-11-jdk \
                            $(BAZEL_VERSION)
 
-      .ci/env/oneapi.sh mkl
-
       # Create .bazelrc and set cache directory
       # Minimal CPU instruction set in Azure is AVX2
       echo "build --disk_cache=$(BAZEL_CACHE_DIR) --cpu=avx2" > ~/.bazelrc
@@ -119,7 +117,6 @@ jobs:
     displayName: 'cpp-examples-thread-release-dynamic'
 
   - script: |
-      export MKLROOT=/opt/intel/oneapi/mkl/latest
       bazel test //cpp/oneapi/dal:tests \
                  --config=host \
                  --test_link_mode=dev \

WORKSPACE

@@ -58,6 +58,18 @@ load("@onedal//dev/bazel/deps:mkl.bzl", "mkl_repo")
 mkl_repo(
     name = "mkl",
     root_env_var = "MKLROOT",
+    urls = [
+        "https://files.pythonhosted.org/packages/bd/74/556cd5efce782ebee2832bd29a49426e88caf2e3cfae38e1d23c0abd41d7/mkl_static-2021.1.1-py2.py3-none-manylinux1_x86_64.whl",
+        "https://files.pythonhosted.org/packages/7d/70/86d0db59598c34a5c1d334ba996271dddad89108127b743c84beb6354afd/mkl_include-2021.1.1-py2.py3-none-manylinux1_x86_64.whl",
+    ],
+    sha256s = [
+        "1db75a53f58cad32935bfbd63206124f6218d71193eea1e75a16aadb9c078370",
+        "865c884473b0a76da201fe972e68c3b2591e6580753485548acc32169db3ffe7",
+    ],
+    strip_prefixes = [
+        "mkl_static-2021.1.1.data/data",
+        "mkl_include-2021.1.1.data/data",
+    ],
 )
 
 load("@onedal//dev/bazel/deps:onedal.bzl", "onedal_repo")

dev/bazel/deps/mkl.bzl

@@ -26,4 +26,11 @@ mkl_repo = repos.prebuilt_libs_repo_rule(
         "lib/intel64/libmkl_intel_ilp64.a",
     ],
     build_template = "@onedal//dev/bazel/deps:mkl.tpl.BUILD",
+    download_mapping = {
+        # Required directory layout and layout in the downloaded
+        # archives may be different. Mapping helps to setup relations
+        # between required layout (LHS) and downloaded (RHS).
+        # In this case, files from `lib/*` will be copied to `lib/intel64/*`.
+        "lib/intel64": "lib",
+    },
 )

dev/bazel/repos.bzl

@@ -16,28 +16,86 @@
 
 load("@onedal//dev/bazel:utils.bzl", "utils", "paths")
 
-def _create_symlinks(repo_ctx, root, entries, substitutions={}):
+def _download_and_extract(repo_ctx, url, sha256, output, strip_prefix):
+    # Workaround Python wheel extraction. Bazel cannot determine file
+    # type automatically as does not support wheels out-of-the-box.
+    archive_type = ""
+    if url.endswith(".whl"):
+        archive_type = "zip"
+    repo_ctx.download_and_extract(
+        url = url,
+        sha256 = sha256,
+        output = output,
+        stripPrefix = strip_prefix,
+        type = archive_type,
+    )
+
+def _create_download_info(repo_ctx):
+    if repo_ctx.attr.url and repo_ctx.attr.urls:
+        fail("Either `url` or `urls` attribute must be set")
+    if repo_ctx.attr.sha256 and repo_ctx.attr.sha256s:
+        fail("Either `sha256` or `sha256s` attribute must be set")
+    if repo_ctx.attr.strip_prefix and repo_ctx.attr.strip_prefixes:
+        fail("Either `strip_prefix` or `strip_prefixes` attribute must be set")
+    if repo_ctx.attr.url:
+        return struct(
+            urls = [repo_ctx.attr.url],
+            sha256s = [repo_ctx.attr.sha256],
+            strip_prefixes = [repo_ctx.attr.strip_prefix],
+        )
+    else:
+        return struct(
+            urls = repo_ctx.attr.urls,
+            sha256s = repo_ctx.attr.sha256s,
+            strip_prefixes = (
+                repo_ctx.attr.strip_prefixes if repo_ctx.attr.strip_prefixes else
+                len(repo_ctx.attr.urls) * [repo_ctx.attr.strip_prefix]
+            ),
+        )
+
+def _normalize_download_info(repo_ctx):
+    info = _create_download_info(repo_ctx)
+    expected_len = len(info.urls)
+    if len(info.sha256s) != expected_len:
+        fail("sha256 hashes count does not match URLs count")
+    if len(info.strip_prefixes) != expected_len:
+        fail("strip_prefixes count does not match URLs count")
+    result = []
+    for url, sha256, strip_prefix in zip(info.urls, info.sha256s, info.strip_prefixes):
+        result.append(struct(
+            url = url,
+            sha256 = sha256,
+            strip_prefix = strip_prefix
+        ))
+    return result
+
+def _create_symlinks(repo_ctx, root, entries, substitutions={}, mapping={}):
     for entry in entries:
         entry_fmt = utils.substitude(entry, substitutions)
-        src_entry_path = paths.join(root, entry_fmt)
+        src_entry_path = utils.substitude(paths.join(root, entry_fmt), mapping)
         dst_entry_path = entry_fmt
         repo_ctx.symlink(src_entry_path, dst_entry_path)
 
 def _download(repo_ctx):
     output = repo_ctx.path("archive")
-    repo_ctx.download_and_extract(
-        url = repo_ctx.attr.url,
-        sha256 = repo_ctx.attr.sha256,
-        output = output,
-        stripPrefix = repo_ctx.attr.strip_prefix,
-    )
+    info_entries = _normalize_download_info(repo_ctx)
+    for info in info_entries:
+        _download_and_extract(
+            repo_ctx,
+            url = info.url,
+            sha256 = info.sha256,
+            output = output,
+            strip_prefix = info.strip_prefix,
+        )
     return str(output)
 
 def _prebuilt_libs_repo_impl(repo_ctx):
+    mapping = repo_ctx.attr._local_mapping
     root = repo_ctx.os.environ.get(repo_ctx.attr.root_env_var)
     if not root:
-        if repo_ctx.attr.url:
+        if repo_ctx.attr.url or repo_ctx.attr.urls:
             root = _download(repo_ctx)
+            mapping = repo_ctx.attr._download_mapping
         elif repo_ctx.attr.fallback_root:
             root = repo_ctx.attr.fallback_root
         else:
@@ -46,8 +104,8 @@ def _prebuilt_libs_repo_impl(repo_ctx):
         # TODO: Detect OS
         "%{os}": "lnx",
     }
-    _create_symlinks(repo_ctx, root, repo_ctx.attr.includes, substitutions)
-    _create_symlinks(repo_ctx, root, repo_ctx.attr.libs, substitutions)
+    _create_symlinks(repo_ctx, root, repo_ctx.attr.includes, substitutions, mapping)
+    _create_symlinks(repo_ctx, root, repo_ctx.attr.libs, substitutions, mapping)
     repo_ctx.template(
         "BUILD",
         repo_ctx.attr.build_template,
@@ -56,7 +114,8 @@ def _prebuilt_libs_repo_impl(repo_ctx):
 
 def _prebuilt_libs_repo_rule(includes, libs, build_template,
                              root_env_var="", fallback_root="",
-                             url="", sha256="", strip_prefix=""):
+                             url="", sha256="", strip_prefix="",
+                             local_mapping={}, download_mapping={}):
     return repository_rule(
         implementation = _prebuilt_libs_repo_impl,
         environ = [
@@ -68,12 +127,17 @@ def _prebuilt_libs_repo_rule(includes, libs, build_template,
             "root_env_var": attr.string(default=root_env_var),
             "fallback_root": attr.string(default=fallback_root),
             "url": attr.string(default=url),
+            "urls": attr.string_list(default=[]),
             "sha256": attr.string(default=sha256),
+            "sha256s": attr.string_list(default=[]),
             "strip_prefix": attr.string(default=strip_prefix),
+            "strip_prefixes": attr.string_list(default=[]),
             "includes": attr.string_list(default=includes),
             "libs": attr.string_list(default=libs),
             "build_template": attr.label(allow_files=True,
                                          default=Label(build_template)),
+            "_local_mapping": attr.string_dict(default=local_mapping),
+            "_download_mapping": attr.string_dict(default=download_mapping),
         }
     )