diff --git a/.bazelignore b/.bazelignore index 37fa09a7af..2a58041707 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,2 +1,3 @@ docs examples +crate_universe/private/bootstrap diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 0000000000..5410c6ec86 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,3 @@ +# When building in CI, a bazelrc file may be generated for overriding +# the binary used by crate_universe +try-import %workspace%/crate_universe.bazelrc diff --git a/.github/workflows/crate_universe.yaml b/.github/workflows/crate_universe.yaml new file mode 100644 index 0000000000..a7e3aafcca --- /dev/null +++ b/.github/workflows/crate_universe.yaml @@ -0,0 +1,214 @@ +--- +name: Crate Universe CI+CD +on: + push: + branches: + - main + paths: + # Only run these jobs if rust code has changed since they're time consuming + - ".github/workflows/crate_universe.yaml" + - "crate_universe/**" + +defaults: + run: + shell: bash + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + # Create a job for each target triple + include: + - os: macos-10.15 + env: + TARGET: "aarch64-apple-darwin" + EXTENSION: "" + - os: ubuntu-20.04 + env: + TARGET: "aarch64-unknown-linux-gnu" + EXTENSION: "" + - os: macos-10.15 + env: + TARGET: "x86_64-apple-darwin" + EXTENSION: "" + - os: ubuntu-20.04 + env: + TARGET: "x86_64-pc-windows-gnu" + EXTENSION: ".exe" + - os: ubuntu-20.04 + env: + TARGET: "x86_64-unknown-linux-gnu" + EXTENSION: "" + steps: + - uses: actions/checkout@v2 + - run: | + # Install cross + if [[ "${RUNNER_OS}" == "macOS" ]]; then + curl --fail -Lo ~/cross.tar.gz https://github.com/rust-embedded/cross/releases/download/v0.2.1/cross-v0.2.1-x86_64-apple-darwin.tar.gz + else + curl --fail -Lo ~/cross.tar.gz https://github.com/rust-embedded/cross/releases/download/v0.2.1/cross-v0.2.1-x86_64-unknown-linux-gnu.tar.gz + fi + sudo tar -xf ~/cross.tar.gz -C /usr/local/bin/ + sudo chmod +x /usr/local/bin/cross + if: matrix.os != 'windows-2019' + - run: | + # Install rust toolchains for host + + # Detect the current version of rust + version="$(grep 'DEFAULT_RUST_VERSION =' ./rust/repositories.bzl | sed 's/DEFAULT_RUST_VERSION = "//' | sed 's/"//')" + + rustup override set "${version}" + rustup update stable && rustup default stable + - run: | + # Setup macos build tooling + sudo xcode-select -s /Applications/Xcode_12.4.app/Contents/Developer/ + + # Set SDK environment variables + echo "SDKROOT=$(xcrun -sdk macosx11.1 --show-sdk-path)" >> $GITHUB_ENV + echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx11.1 --show-sdk-platform-version)" >> $GITHUB_ENV + if: matrix.os == 'macOS-10.15' + - run: | + # Build binaries + ./crate_universe/private/bootstrap/build.sh + env: + TARGET: "${{ matrix.env.TARGET }}" + - uses: actions/upload-artifact@v2 + with: + name: "${{ matrix.env.TARGET }}" + path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/${{ matrix.env.TARGET }} + if-no-files-found: error + release: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # Save the git credentials so we can commit back to the repo + persist-credentials: true + - uses: actions/download-artifact@v2 + with: + path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin + - run: | + # Ensure all files are executable + chmod +x ${{ github.workspace }}/crate_universe/private/bootstrap/bin/*/release/* + - uses: actions/create-release@v1 + id: crate_universe_release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + prerelease: true + tag_name: crate_universe-${{ github.run_number }} + release_name: crate_universe-${{ github.run_number }} + body: | + From commit: ${{ github.sha }} + # There must be a upload action for each platform triple we create + - name: "Upload aarch64-apple-darwin" + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.crate_universe_release.outputs.upload_url }} + asset_name: crate_universe_resolver-aarch64-apple-darwin + asset_path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/aarch64-apple-darwin/release/crate_universe_resolver + asset_content_type: application/octet-stream + - name: "Upload aarch64-unknown-linux-gnu" + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.crate_universe_release.outputs.upload_url }} + asset_name: crate_universe_resolver-aarch64-unknown-linux-gnu + asset_path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/aarch64-unknown-linux-gnu/release/crate_universe_resolver + asset_content_type: application/octet-stream + - name: "Upload x86_64-apple-darwin" + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.crate_universe_release.outputs.upload_url }} + asset_name: crate_universe_resolver-x86_64-apple-darwin + asset_path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/x86_64-apple-darwin/release/crate_universe_resolver + asset_content_type: application/octet-stream + - name: "Upload x86_64-pc-windows-gnu" + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.crate_universe_release.outputs.upload_url }} + asset_name: crate_universe_resolver-x86_64-pc-windows-gnu.exe + asset_path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/x86_64-pc-windows-gnu/release/crate_universe_resolver.exe + asset_content_type: application/octet-stream + - name: "Upload x86_64-unknown-linux-gnu" + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.crate_universe_release.outputs.upload_url }} + asset_name: crate_universe_resolver-x86_64-unknown-linux-gnu + asset_path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/x86_64-unknown-linux-gnu/release/crate_universe_resolver + asset_content_type: application/octet-stream + - run: | + # Write new defaults.bzl file + # Copy the new template + cp ${{ github.workspace }}/crate_universe/private/defaults.bzl.template ${{ github.workspace }}/crate_universe/private/defaults.bzl + + # Replace the url + url="$(echo $URL | sed 's|releases/tag/|releases/download/|')/{bin}" + sed -i "s|{DEFAULT_URL_TEMPLATE}|${url}|" ${{ github.workspace }}/crate_universe/private/defaults.bzl + + # Populate all sha256 values + TARGETS=( + aarch64-apple-darwin + aarch64-unknown-linux-gnu + x86_64-apple-darwin + x86_64-pc-windows-gnu + x86_64-unknown-linux-gnu + ) + for triple in ${TARGETS[@]}; do + if [[ "${triple}" == *"windows"* ]]; then + bin_name=crate_universe_resolver.exe + else + bin_name=crate_universe_resolver + fi + sha256="$(shasum --algorithm 256 ${{ github.workspace }}/crate_universe/private/bootstrap/bin/${triple}/release/${bin_name} | awk '{ print $1 }')" + sed -i "s|{${triple}--sha256}|${sha256}|" ${{ github.workspace }}/crate_universe/private/defaults.bzl + done + env: + URL: ${{ steps.crate_universe_release.outputs.html_url }} + - run: | + # Commit and push changes back to the repo + if [[ -z "$(git status --porcelain)" ]]; then + echo "No change to any binaries" + exit 0 + fi + + git stage ${GITHUB_WORKSPACE}/crate_universe/private/defaults.bzl && git status + + # Setup git and commit back to the repo + git config user.email "rules_rust@googlegroups.com" + git config user.name "github-actions" + + # Cache the current info + commit="$(git rev-parse HEAD)" + branch="$(git branch --show-current)" + + # commit the change + git commit -m "github-actions: update crate_universe binaries from ${commit}" + + # Attempt to push but compensate for potential issues where another PR would + # have been merged before this pipeline gets to this point. + for i in 1 2 3 4 5 ; do + { + git push origin "${branch}" && break + } || { + sleep 10 + git pull --rebase + } + done + + # Ensure we've pushed our commit + if [[ -n "$(git diff "${branch}..HEAD")" ]]; then + echo "Failed to push changes" + exit 1 + fi diff --git a/.gitignore b/.gitignore index 2f8bfb1f1d..25f75fe373 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ /examples/bazel-* /examples/crate_universe/*/bazel-* /docs/bazel-* +crate_universe.bazelrc # rustfmt *.rs.bk diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 67c7f5117c..25950379e2 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -4,6 +4,10 @@ load("@rules_rust//rust:repositories.bzl", "rust_repositories") rust_repositories() +load("@rules_rust//crate_universe:defs.bzl", "crate_universe_deps") + +crate_universe_deps() + load("@rules_rust//proto:repositories.bzl", "rust_proto_repositories") rust_proto_repositories() diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 0000000000..166a956fea --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -euo pipefail + +####################################################################################################################### +# Crate Universe +####################################################################################################################### + +# Find the location of the script +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +BOOTSTRAP_DIR="${SCRIPT_DIR}/crate_universe/private/bootstrap" + +# Go to root of rules_rust +pushd "${SCRIPT_DIR}" &> /dev/null +has_changes="$(git log origin/main.. -- crate_universe)" +popd &> /dev/null + +# Only bootstrap the binary if there are changes +if [[ -n "${has_changes}" ]]; then + pushd "${BOOTSTRAP_DIR}" &> /dev/null + bazel run //:build && bazel run //:install && bazel clean + popd &> /dev/null + + # Generate a bazelrc file for ensuring we always use the resolver binary for the current host + cat << EOF > "${SCRIPT_DIR}/crate_universe.bazelrc" +common --announce_rc +common --override_repository="rules_rust_crate_universe__aarch64-apple-darwin=${BOOTSTRAP_DIR}" +common --override_repository="rules_rust_crate_universe__aarch64-unknown-linux-gnu=${BOOTSTRAP_DIR}" +common --override_repository="rules_rust_crate_universe__x86_64-apple-darwin=${BOOTSTRAP_DIR}" +common --override_repository="rules_rust_crate_universe__x86_64-pc-windows-gnu=${BOOTSTRAP_DIR}" +common --override_repository="rules_rust_crate_universe__x86_64-unknown-linux-gnu=${BOOTSTRAP_DIR}" +EOF + + cp ${SCRIPT_DIR}/crate_universe.bazelrc ${SCRIPT_DIR}/examples/crate_universe/crate_universe.bazelrc +else + echo "No changes to crate_universe. Nothing to do" +fi + +####################################################################################################################### diff --git a/crate_universe/CONTRIBUTING.md b/crate_universe/CONTRIBUTING.md index 1f95f9b44f..42b686dc29 100644 --- a/crate_universe/CONTRIBUTING.md +++ b/crate_universe/CONTRIBUTING.md @@ -18,21 +18,20 @@ Some areas have unit testing, there are a few (very brittle) integration tests, ## How to test local changes -To use a local version, first build it: +To use a local version, first bootstrap it: ```console -crate_universe $ cargo build +rules_rust $ ./bootstrap.sh ``` -Then run a bazel build with this environment variable: +This will build `crate_universe_resolver` and generate a `crate_universe.bazelrc` file which contains +[override_repository](https://docs.bazel.build/versions/master/command-line-reference.html#flag--override_repository) +definitions fo the newly built binary. For more details on the build process, see +[crate_universe/private/bootstrap/README.md](./private/bootstrap/README.md). -```console -RULES_RUST_RESOLVER_URL_OVERRIDE=file:///path/to/resolver/target/debug/resolver bazel build //whatever -``` - -To get verbose logging, edit `defs.bzl` to set `RUST_LOG` to `debug` or `trace` instead of `info`. In particular, that will print out the generated Cargo.toml, and the path to the generated workspace file. +To get verbose logging, edit `defs.bzl` to set `RUST_LOG` to `debug` or `trace` instead of `info`. In particular, that will print out the generated `Cargo.toml`, and the path to the generated workspace file. -Note that you may need to `bazel shutdown` between bazel commands. Hopefully not, but if you're seeing stale results, try it out. This only affects local development. +Note that you may need to `bazel shutdown` between Bazel commands. Hopefully not, but if you're seeing stale results, try it out. This only affects local development. ## Testing diff --git a/crate_universe/Cargo.toml b/crate_universe/Cargo.toml index c001404582..d9ec5e8c0e 100644 --- a/crate_universe/Cargo.toml +++ b/crate_universe/Cargo.toml @@ -2,9 +2,9 @@ name = "crate_universe_resolver" version = "0.1.0-experimental" authors = [ - "Daniel Wagner-Hall ", - "Romain Chossart ", - "Gibson Fahnestock ", + "Daniel Wagner-Hall ", + "Romain Chossart ", + "Gibson Fahnestock ", ] edition = "2018" @@ -14,9 +14,9 @@ edition = "2018" # Several of these dependencies need to be compatible with the version cargo-raze is using. anyhow = "1" +cargo_metadata = "0.13.1" # Until https://github.com/google/cargo-raze/pull/407 is merged: cargo-raze = { git = "https://github.com/UebelAndre/cargo-raze.git", rev = "caaf8345046eee830ea89fc04771ee117bd835eb", default-features = false } -cargo_metadata = "0.13.1" cfg-expr = "0.7.4" env_logger = "0.8" hex = "0.4" diff --git a/crate_universe/Cross.toml b/crate_universe/Cross.toml new file mode 100644 index 0000000000..0a73031bda --- /dev/null +++ b/crate_universe/Cross.toml @@ -0,0 +1,18 @@ +# Darwin targets build on the host +# https://github.com/rust-embedded/cross/issues/223 +# [target.aarch64-apple-darwin] +# image = "rustembedded/cross:aarch64-apple-darwin" + +[target.aarch64-unknown-linux-gnu] +image = "rustembedded/cross:aarch64-unknown-linux-gnu@sha256:c970d529baa1bab953f9a0f2c2294e51a12a2e710599ab91de69c45a1785ba46" + +# Darwin targets build on the host +# https://github.com/rust-embedded/cross/issues/223 +# [target.x86_64-apple-darwin] +# image = "rustembedded/cross:x86_64-apple-darwin" + +[target.x86_64-pc-windows-gnu] +image = "rustembedded/cross:x86_64-pc-windows-gnu@sha256:5005553ddf4e82d4703ee7f458a24802a014f03884f87c6277e9c718765d3517" + +[target.x86_64-unknown-linux-gnu] +image = "rustembedded/cross:x86_64-unknown-linux-gnu@sha256:a7f681882d23eae287cff48a93aff93f71a84859a6670c2a4fe2b82949ebb91c" diff --git a/crate_universe/defs.bzl b/crate_universe/defs.bzl index 11bf2deeb6..e33efc5ea4 100644 --- a/crate_universe/defs.bzl +++ b/crate_universe/defs.bzl @@ -1,5 +1,7 @@ """A module defining the `crate_universe` rule""" +load(":deps.bzl", _crate_universe_deps = "crate_universe_deps") + DEFAULT_REPOSITORY_TEMPLATE = "https://crates.io/api/v1/crates/{crate}/{version}/download" _INPUT_CONTENT_TEMPLATE = """{{ @@ -20,6 +22,96 @@ _INPUT_CONTENT_TEMPLATE = """{{ "cargo": "{cargo}" }}""" +_CPU_ARCH_ERROR_MSG = """\ +Command failed with exit code '{code}': {args} +----------stdout: +{stdout} +----------stderr: +{stderr} +""" + +def _query_cpu_architecture(repository_ctx, expected_archs, is_windows = False): + """Detect the host CPU architecture + + Args: + repository_ctx (repository_ctx): The repository rule's context object + expected_archs (list): A list of expected architecture strings + is_windows (bool, optional): If true, the cpu lookup will use the windows method (`wmic` vs `uname`) + + Returns: + str: The host's CPU architecture + """ + if is_windows: + arguments = ["wmic", "os", "get", "osarchitecture"] + else: + arguments = ["uname", "-m"] + + result = repository_ctx.execute(arguments) + + if result.return_code: + fail(_CPU_ARCH_ERROR_MSG.format( + code = result.return_code, + args = arguments, + stdout = result.stdout, + stderr = result.stderr, + )) + + if is_windows: + # Example output: + # OSArchitecture + # 64-bit + lines = result.stdout.split("\n") + arch = lines[1] + else: + arch = result.stdout.strip("\n") + + if not arch in expected_archs: + fail("{} is not a expected cpu architecture. {}".format( + arch, + expected_archs, + )) + + return arch + +def _get_host_info(repository_ctx): + """Query host information for the appropriate triple and toolchain repo name + + Args: + repository_ctx (repository_ctx): The rule's repository_ctx + + Returns: + tuple: A tuple containing a triple (str) and repository name (str) + """ + + # Detect the host's cpu architecture + + supported_architectures = { + "linux": ["aarch64", "x86_64"], + "macos": ["aarch64", "x86_64"], + "windows": ["x86_64"], + } + + # The expected file extension of crate resolver binaries + extension = "" + + if "linux" in repository_ctx.os.name: + cpu = _query_cpu_architecture(repository_ctx, supported_architectures["linux"]) + resolver_triple = "{}-unknown-linux-gnu".format(cpu) + toolchain_repo = "@rust_linux_{}".format(cpu) + elif "mac" in repository_ctx.os.name: + cpu = _query_cpu_architecture(repository_ctx, supported_architectures["macos"]) + resolver_triple = "{}-apple-darwin".format(cpu) + toolchain_repo = "@rust_darwin_{}".format(cpu) + elif "win" in repository_ctx.os.name: + cpu = _query_cpu_architecture(repository_ctx, supported_architectures["windows"], True) + resolver_triple = "{}-pc-windows-gnu".format(cpu) + toolchain_repo = "@rust_windows_{}".format(cpu) + extension = ".exe" + else: + fail("Could not locate resolver for OS " + repository_ctx.os.name) + + return (resolver_triple, toolchain_repo, extension) + def _crate_universe_resolve_impl(repository_ctx): """Entry-point repository to manage rust dependencies. @@ -34,15 +126,34 @@ def _crate_universe_resolve_impl(repository_ctx): RULES_RUST_CRATE_UNIVERSE_RESOLVER_URL_OVERRIDE: Override URL to use to download resolver binary - for local paths use a file:// URL. """ - if repository_ctx.os.name == "linux": - toolchain_repo = "@rust_linux_x86_64" - elif repository_ctx.os.name == "mac os x": - toolchain_repo = "@rust_darwin_x86_64" - else: - fail("Could not locate resolver for OS " + repository_ctx.os.name) + # Get info about the current host's tool locations + resolver_triple, toolchain_repo, extension = _get_host_info(repository_ctx) + cargo_label = Label(toolchain_repo + "//:bin/cargo") rustc_label = Label(toolchain_repo + "//:bin/rustc") + # Allow for an an override environment variable to define a url to a binary + resolver_url = repository_ctx.os.environ.get("RULES_RUST_CRATE_UNIVERSE_RESOLVER_URL_OVERRIDE", None) + resolver_sha = repository_ctx.os.environ.get("RULES_RUST_CRATE_UNIVERSE_RESOLVER_URL_OVERRIDE_SHA256", None) + if resolver_url: + if resolver_url.startswith("file://"): + sha256_result = repository_ctx.execute(["sha256sum", resolver_url[7:]]) + resolver_sha = sha256_result.stdout[:64] + + resolver_path = repository_ctx.path("resolver") + repository_ctx.download( + url = resolver_url, + sha256 = resolver_sha, + output = resolver_path, + executable = True, + ) + else: + resolver_label = Label("@rules_rust_crate_universe__{}//file:resolver{}".format( + resolver_triple, + extension, + )) + resolver_path = repository_ctx.path(resolver_label) + lockfile_path = None if repository_ctx.attr.lockfile: lockfile_path = repository_ctx.path(repository_ctx.attr.lockfile) @@ -59,30 +170,6 @@ def _crate_universe_resolve_impl(repository_ctx): ) input_path = "_{name}.json".format(name = repository_ctx.attr.name) - - # For debugging or working on changes to the resolver, you can set something like: - # export RULES_RUST_RESOLVER_URL_OVERRIDE=file:///path/to/rules_rust/cargo/crate_universe_resolver/target/release/resolver - resolver_url = repository_ctx.os.environ.get("RULES_RUST_CRATE_UNIVERSE_RESOLVER_URL_OVERRIDE", None) - if resolver_url and resolver_url.startswith("file://"): - sha256_result = repository_ctx.execute(["sha256sum", resolver_url[7:]]) - resolver_sha = sha256_result.stdout[:64] - - resolver_path = repository_ctx.path("resolver") - repository_ctx.download( - url = resolver_url, - sha256 = resolver_sha, - output = resolver_path, - executable = True, - ) - else: - # TODO: Avoid downloading both the linux and darwin one each time. - if repository_ctx.os.name == "linux": - resolver_path = repository_ctx.path(repository_ctx.attr._resolver_script_linux) - elif repository_ctx.os.name == "mac os x": - resolver_path = repository_ctx.path(repository_ctx.attr._resolver_script_darwin) - else: - fail("Could not locate resolver for OS " + repository_ctx.os.name) - repository_ctx.file(input_path, content = input_content) args = [ @@ -122,16 +209,10 @@ def _crate_universe_resolve_impl(repository_ctx): repository_ctx.file("BUILD.bazel") _crate_universe_resolve = repository_rule( - implementation = _crate_universe_resolve_impl, - attrs = { - "cargo_toml_files": attr.label_list(), - "lockfile": attr.label( - allow_single_file = True, - mandatory = False, - ), - "overrides": attr.string_dict(doc = """Mapping of crate name to crate.override(...) entries) + doc = """\ +A rule for downloading Rust dependencies (crates). -Example: +`override` Example: load("@rules_rust//crate_universe:defs.bzl", "rust_library", "crate") @@ -167,22 +248,46 @@ Example: ), }, ) - -"""), - "packages": attr.string_list(allow_empty = True), - "repository_template": attr.string(), - "supported_targets": attr.string_list(allow_empty = False), - "_resolver_script_darwin": attr.label( - default = "@crate_universe_resolver_darwin//file:downloaded", - executable = True, - cfg = "host", +""", + implementation = _crate_universe_resolve_impl, + attrs = { + "cargo_toml_files": attr.label_list( + doc = "", ), - "_resolver_script_linux": attr.label( - default = "@crate_universe_resolver_linux//file:downloaded", - executable = True, - cfg = "host", + "lockfile": attr.label( + doc = "", + allow_single_file = True, + mandatory = False, + ), + "overrides": attr.string_dict( + doc = "Mapping of crate name to `crate.override(...)` entries)", + ), + "packages": attr.string_list( + doc = "", + allow_empty = True, + ), + "repository_template": attr.string( + doc = "", + ), + "supported_targets": attr.string_list( + doc = "", + allow_empty = False, + ), + "_resolvers": attr.label_list( + doc = "A list of resolver binaries for various platforms", + default = [ + "@rules_rust_crate_universe__aarch64-apple-darwin//:resolver", + "@rules_rust_crate_universe__aarch64-unknown-linux-gnu//:resolver", + "@rules_rust_crate_universe__x86_64-apple-darwin//:resolver", + "@rules_rust_crate_universe__x86_64-pc-windows-gnu//:resolver.exe", + "@rules_rust_crate_universe__x86_64-unknown-linux-gnu//:resolver", + ], ), }, + environ = [ + "RULES_RUST_CRATE_UNIVERSE_RESOLVER_URL_OVERRIDE", + "RULES_RUST_CRATE_UNIVERSE_RESOLVER_URL_OVERRIDE_SHA256", + ], ) def crate_universe( @@ -264,3 +369,6 @@ crate = struct( spec = _spec, override = _override, ) + +# Reexport the dependencies macro +crate_universe_deps = _crate_universe_deps diff --git a/crate_universe/deps.bzl b/crate_universe/deps.bzl new file mode 100644 index 0000000000..3617b32038 --- /dev/null +++ b/crate_universe/deps.bzl @@ -0,0 +1,39 @@ +"""A module defining the all dependencies of the crate_universe repository rule""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("//crate_universe/private:defaults.bzl", "DEFAULT_SHA256_CHECKSUMS", "DEFAULT_URL_TEMPLATE") + +def crate_universe_bins(url_template = DEFAULT_URL_TEMPLATE, sha256s = DEFAULT_SHA256_CHECKSUMS): + """Defines repositories for crate universe binaries + + Args: + url_template (str, optional): A template url for downloading binaries. + This must contain a `{bin}` key. + sha256s (dict, optional): A dict of sha256 values where the key is the + platform triple of the associated binary. + """ + + # If a repository declaration is added or removed from there, the same + # should occur in `defaults.bzl` and `create_universe.yaml`. + triples = { + "aarch64-apple-darwin": "", + "aarch64-unknown-linux-gnu": "", + "x86_64-apple-darwin": "", + "x86_64-pc-windows-gnu": ".exe", + "x86_64-unknown-linux-gnu": "", + } + + for (triple, extension) in triples.items(): + maybe( + http_file, + name = "rules_rust_crate_universe__{}".format(triple), + downloaded_file_path = "resolver{}".format(extension), + executable = True, + sha256 = sha256s.get(triple), + urls = [url_template.format(bin = "crate_universe_resolver-{}{}".format(triple, extension))], + ) + +def crate_universe_deps(): + """Define all dependencies for the crate_universe repository rule""" + crate_universe_bins() diff --git a/crate_universe/private/BUILD.bazel b/crate_universe/private/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crate_universe/private/bootstrap/.bazelversion b/crate_universe/private/bootstrap/.bazelversion new file mode 100644 index 0000000000..0c89fc927e --- /dev/null +++ b/crate_universe/private/bootstrap/.bazelversion @@ -0,0 +1 @@ +4.0.0 \ No newline at end of file diff --git a/crate_universe/private/bootstrap/.gitignore b/crate_universe/private/bootstrap/.gitignore new file mode 100644 index 0000000000..8a1e0c90cf --- /dev/null +++ b/crate_universe/private/bootstrap/.gitignore @@ -0,0 +1,8 @@ +# Bootstrapped binaries will go in `./bin` +bin/ + +# And to use the binaries as http_file repositories, we create +file/ + +# Bazel outputs +bazel-* diff --git a/crate_universe/private/bootstrap/BUILD.bazel b/crate_universe/private/bootstrap/BUILD.bazel new file mode 100644 index 0000000000..9138c38e23 --- /dev/null +++ b/crate_universe/private/bootstrap/BUILD.bazel @@ -0,0 +1,72 @@ +alias( + name = "cargo", + actual = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": "@rust_darwin_aarch64//:cargo", + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": "@rust_linux_aarch64//:cargo", + "@rules_rust//rust/platform:x86_64-apple-darwin": "@rust_darwin_x86_64//:cargo", + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": "@rust_windows_x86_64//:cargo", + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": "@rust_linux_x86_64//:cargo", + }), +) + +alias( + name = "rustc", + actual = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": "@rust_darwin_aarch64//:rustc", + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": "@rust_linux_aarch64//:rustc", + "@rules_rust//rust/platform:x86_64-apple-darwin": "@rust_darwin_x86_64//:rustc", + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": "@rust_windows_x86_64//:rustc", + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": "@rust_linux_x86_64//:rustc", + }), +) + +_COMMON_ENV = { + "CARGO": "$${PWD}/$(execpath :cargo)", + "DETECT_CHANGES": "true", + "RUSTC": "$${PWD}/$(execpath :rustc)", +} + +_ENV = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": dict(_COMMON_ENV.items() + { + "IS_WINDOWS": "false", + "TARGET": "aarch64-apple-darwin", + }.items()), + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": dict(_COMMON_ENV.items() + { + "IS_WINDOWS": "false", + "TARGET": "aarch64-unknown-linux-gnu", + }.items()), + "@rules_rust//rust/platform:x86_64-apple-darwin": dict(_COMMON_ENV.items() + { + "IS_WINDOWS": "false", + "TARGET": "x86_64-apple-darwin", + }.items()), + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": dict(_COMMON_ENV.items() + { + "IS_WINDOWS": "true", + "TARGET": "x86_64-pc-windows-gnu", + }.items()), + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": dict(_COMMON_ENV.items() + { + "IS_WINDOWS": "false", + "TARGET": "x86_64-unknown-linux-gnu", + }.items()), +}) + +# Bootstrap the crate_universe resolver for the current exec platform +sh_binary( + name = "build", + srcs = ["build.sh"], + data = [ + ":cargo", + ":rustc", + ], + env = _ENV, + tags = ["requires-network"], +) + +sh_binary( + name = "install", + srcs = ["install.sh"], + data = [ + ":cargo", + ":rustc", + ], + env = _ENV, +) diff --git a/crate_universe/private/bootstrap/README.md b/crate_universe/private/bootstrap/README.md new file mode 100644 index 0000000000..9a5f9e72bb --- /dev/null +++ b/crate_universe/private/bootstrap/README.md @@ -0,0 +1,26 @@ +# Crate Universe Bootstrap + +This workspace contains tools for bootstrapping `crate_universe` binaries. + +## Build + +Users can use Bazel to build a binary for the current host by running `bazel run //:build`. + +If a user is looking to build binaries for all supported platforms, they should simply run +`./build.sh` directly. + +### Dependencies + +When running `./build.sh` directly, the script expects [Cargo](https://doc.rust-lang.org/cargo/) to be +installed on the host and will attempt to find or install [cross](https://github.com/rust-embedded/cross) +which depends on [Docker](https://www.docker.com/). + +#### Installing Dependencies + +- `Cargo`: use [rustup](https://rustup.rs/). +- `Cross`: run `cargo install cross` +- `Docker`: Follow [this guide](https://docs.docker.com/engine/install/) + +### Artifacts + +Artifacts can be found in `./bin` once `./build.sh` is run. diff --git a/crate_universe/private/bootstrap/WORKSPACE.bazel b/crate_universe/private/bootstrap/WORKSPACE.bazel new file mode 100644 index 0000000000..380a1d41d1 --- /dev/null +++ b/crate_universe/private/bootstrap/WORKSPACE.bazel @@ -0,0 +1,10 @@ +workspace(name = "rules_rust_crate_universe_bootstrap") + +local_repository( + name = "rules_rust", + path = "../../..", +) + +load("@rules_rust//rust:repositories.bzl", "rust_repositories") + +rust_repositories() diff --git a/crate_universe/private/bootstrap/build.sh b/crate_universe/private/bootstrap/build.sh new file mode 100755 index 0000000000..e1dd773ad9 --- /dev/null +++ b/crate_universe/private/bootstrap/build.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +set -euo pipefail + +# Find the location of the script +if [[ -n "${BUILD_WORKSPACE_DIRECTORY:-}" ]]; then + SCRIPT_DIR="${BUILD_WORKSPACE_DIRECTORY}" +else + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +fi + +# All supported targets +if [[ $# -gt 0 ]]; then + TARGETS=("$@") +elif [[ -n "${TARGET:-}" ]]; then + TARGETS=("${TARGET}") +else + TARGETS=( + "aarch64-apple-darwin" + "aarch64-unknown-linux-gnu" + "x86_64-apple-darwin" + "x86_64-pc-windows-gnu" + "x86_64-unknown-linux-gnu" + ) +fi + +echo "TARGETS=${TARGETS[@]}" + +# Specify the path to the cargo manifest +MANIFEST="${SCRIPT_DIR}/../../Cargo.toml" + +if [[ "${IS_WINDOWS:-}" == "true" ]]; then + WIN_PWD="$(type -t cygpath > /dev/null && cygpath $(pwd) -w || pwd -W)" + # Resolve absolute paths that could potentially be in the cargo and rustc vars + CARGO="$(echo ${CARGO-cargo} | sed "s^\${PWD}^${WIN_PWD}^")" + RUSTC="$(echo ${RUSTC-rustc} | sed "s^\${PWD}^${WIN_PWD}^")" +else + # Resolve absolute paths that could potentially be in the cargo and rustc vars + CARGO="$(echo ${CARGO-cargo} | sed "s^\${PWD}^${PWD}^")" + RUSTC="$(echo ${RUSTC-rustc} | sed "s^\${PWD}^${PWD}^")" +fi + +# If there are multiple targets or we're in github CI, ensure `cross` is installed +if [[ "${#TARGETS[@]}" != 1 || -n "${GITHUB_WORKFLOW:-}" ]]; then + + # Ensure we have an aboslute path to the cargo binary + ${CARGO} version + + # Ensure cross is installed which is used for bootstrapping on all platforms + if [[ -z "$(cross --version || echo '')" ]]; then + ${CARGO} install cross + fi + + BUILD_TOOL=cross +else + # Ensure rustc is set when using cargo + BUILD_TOOL="env RUSTC=${RUSTC} ${CARGO}" +fi + +# Fetch cargo dependencies in advance to streamline the build process +echo "Fetch cargo dependencies" +${CARGO} fetch --manifest-path=${MANIFEST} + +if [[ -z "${OUT_DIR:-}" ]]; then + OUT_DIR="${SCRIPT_DIR}/bin" +fi + +# Because --target-dir does not work, we change directories and move built binaries after the fact +# https://github.com/rust-embedded/cross/issues/272 +pushd "$(dirname "${MANIFEST}")" + +# Build all binaries +for target in ${TARGETS[@]}; do + echo "Building for ${target}" + + if [[ "${target}" == *"windows"* ]]; then + bin_name=crate_universe_resolver.exe + else + bin_name=crate_universe_resolver + fi + + # This clean avoids linker issues + # https://github.com/rust-embedded/cross/issues/455 + ${CARGO} clean + + # Build the binary for the current target + ${BUILD_TOOL} build --release --locked --target="${target}" + + # Install it into the rules_rust repository + install_path="${OUT_DIR}/${target}/release/${bin_name}" + mkdir -p "$(dirname "${install_path}")" + cp -p "./target/${target}/release/${bin_name}" "${install_path}" +done +popd diff --git a/crate_universe/private/bootstrap/install.sh b/crate_universe/private/bootstrap/install.sh new file mode 100755 index 0000000000..0ddee8b796 --- /dev/null +++ b/crate_universe/private/bootstrap/install.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -euo pipefail + +# Find the location of the script +if [[ -n "${BUILD_WORKSPACE_DIRECTORY:-}" ]]; then + SCRIPT_DIR="${BUILD_WORKSPACE_DIRECTORY}" +else + SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +fi + +if [[ "${IS_WINDOWS:-}" == "true" ]]; then + bin_name=crate_universe_resolver.exe + install_name=resolver.exe +else + bin_name=crate_universe_resolver + install_name=resolver +fi + +mkdir -p "${SCRIPT_DIR}/file" +touch "${SCRIPT_DIR}/file/BUILD.bazel" +cp "${SCRIPT_DIR}/bin/${TARGET}/release/${bin_name}" "${SCRIPT_DIR}/file/${install_name}" diff --git a/crate_universe/private/defaults.bzl b/crate_universe/private/defaults.bzl new file mode 100644 index 0000000000..f2f8b283dd --- /dev/null +++ b/crate_universe/private/defaults.bzl @@ -0,0 +1,14 @@ +"""A helper module defining generated information about crate_universe dependencies""" + +# This global should match the current release of `crate_unvierse`. +DEFAULT_URL_TEMPLATE = "{bin}" + +# Note that if any additional platforms are added here, the pipeline defined +# by `create_universe.yaml` should also be updated +DEFAULT_SHA256_CHECKSUMS = { + "aarch64-apple-darwin": "{aarch64-apple-darwin--sha256}", + "aarch64-unknown-linux-gnu": "{aarch64-unknown-linux-gnu--sha256}", + "x86_64-apple-darwin": "{x86_64-apple-darwin--sha256}", + "x86_64-pc-windows-gnu": "{x86_64-pc-windows-gnu--sha256}", + "x86_64-unknown-linux-gnu": "{x86_64-unknown-linux-gnu--sha256}", +} diff --git a/crate_universe/private/defaults.bzl.template b/crate_universe/private/defaults.bzl.template new file mode 100644 index 0000000000..372ed2bfae --- /dev/null +++ b/crate_universe/private/defaults.bzl.template @@ -0,0 +1,14 @@ +"""A helper module defining generated information about crate_universe dependencies""" + +# This global should match the current release of `crate_unvierse`. +DEFAULT_URL_TEMPLATE = "{DEFAULT_URL_TEMPLATE}" + +# Note that if any additional platforms are added here, the pipeline defined +# by `create_universe.yaml` should also be updated +DEFAULT_SHA256_CHECKSUMS = { + "aarch64-apple-darwin": "{aarch64-apple-darwin--sha256}", + "aarch64-unknown-linux-gnu": "{aarch64-unknown-linux-gnu--sha256}", + "x86_64-apple-darwin": "{x86_64-apple-darwin--sha256}", + "x86_64-pc-windows-gnu": "{x86_64-pc-windows-gnu--sha256}", + "x86_64-unknown-linux-gnu": "{x86_64-unknown-linux-gnu--sha256}", +} diff --git a/examples/crate_universe/.bazelrc b/examples/crate_universe/.bazelrc new file mode 100644 index 0000000000..5410c6ec86 --- /dev/null +++ b/examples/crate_universe/.bazelrc @@ -0,0 +1,3 @@ +# When building in CI, a bazelrc file may be generated for overriding +# the binary used by crate_universe +try-import %workspace%/crate_universe.bazelrc diff --git a/examples/crate_universe/.gitignore b/examples/crate_universe/.gitignore index ac51a054d2..d63036d5b8 100644 --- a/examples/crate_universe/.gitignore +++ b/examples/crate_universe/.gitignore @@ -1 +1,2 @@ bazel-* +crate_universe.bazelrc diff --git a/examples/crate_universe/WORKSPACE.bazel b/examples/crate_universe/WORKSPACE.bazel index ba80818850..d116458bc8 100644 --- a/examples/crate_universe/WORKSPACE.bazel +++ b/examples/crate_universe/WORKSPACE.bazel @@ -9,6 +9,10 @@ load("@rules_rust//rust:repositories.bzl", "rust_repositories") rust_repositories() +load("@rules_rust//crate_universe:defs.bzl", "crate_universe_deps") + +crate_universe_deps() + load("//basic:workspace.bzl", basic_deps = "deps") basic_deps() diff --git a/rust/repositories.bzl b/rust/repositories.bzl index fdf7549957..a368c51696 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -15,9 +15,12 @@ load(":known_shas.bzl", "FILE_KEY_TO_SHA") DEFAULT_TOOLCHAIN_NAME_PREFIX = "toolchain_for" DEFAULT_STATIC_RUST_URL_TEMPLATES = ["https://static.rust-lang.org/dist/{}.tar.gz"] +# Note: Code in `.github/workflows/crate_universe.yaml` looks for this line, if you remove it or change its format, you will also need to update that code. +DEFAULT_RUST_VERSION = "1.51.0" + # buildifier: disable=unnamed-macro def rust_repositories( - version = "1.51.0", + version = DEFAULT_RUST_VERSION, iso_date = None, rustfmt_version = None, edition = None,