Added bootstrapping for crate_universe (bazelbuild#663)

* Added bootstrap script for crate_universe and github action for auto-releasing new binaries

* Addressed PR feedback

Co-authored-by: Daniel Wagner-Hall <dawagner@gmail.com>

* Updated local dev documentation

* Add announce_rc to crate_universe.bazelrc

* Addressed buildifier defect

* Addressed additional user feedback

* Apply suggestions from code review

Co-authored-by: Daniel Wagner-Hall <dawagner@gmail.com>

* Fixed missing extension for windows.

Co-authored-by: Daniel Wagner-Hall <dawagner@gmail.com>

.bazelignore

@@ -1,2 +1,3 @@
 docs
 examples
+crate_universe/private/bootstrap

.bazelrc

@@ -0,0 +1,3 @@
+# When building in CI, a bazelrc file may be generated for overriding
+# the binary used by crate_universe
+try-import %workspace%/crate_universe.bazelrc

.github/workflows/crate_universe.yaml

@@ -0,0 +1,214 @@
+---
+name: Crate Universe CI+CD
+on:
+  push:
+    branches:
+      - main
+    paths:
+      # Only run these jobs if rust code has changed since they're time consuming
+      - ".github/workflows/crate_universe.yaml"
+      - "crate_universe/**"
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # Create a job for each target triple
+        include:
+          - os: macos-10.15
+            env:
+              TARGET: "aarch64-apple-darwin"
+              EXTENSION: ""
+          - os: ubuntu-20.04
+            env:
+              TARGET: "aarch64-unknown-linux-gnu"
+              EXTENSION: ""
+          - os: macos-10.15
+            env:
+              TARGET: "x86_64-apple-darwin"
+              EXTENSION: ""
+          - os: ubuntu-20.04
+            env:
+              TARGET: "x86_64-pc-windows-gnu"
+              EXTENSION: ".exe"
+          - os: ubuntu-20.04
+            env:
+              TARGET: "x86_64-unknown-linux-gnu"
+              EXTENSION: ""
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          # Install cross
+          if [[ "${RUNNER_OS}" == "macOS" ]]; then
+            curl --fail -Lo ~/cross.tar.gz https://github.com/rust-embedded/cross/releases/download/v0.2.1/cross-v0.2.1-x86_64-apple-darwin.tar.gz
+          else
+            curl --fail -Lo ~/cross.tar.gz https://github.com/rust-embedded/cross/releases/download/v0.2.1/cross-v0.2.1-x86_64-unknown-linux-gnu.tar.gz
+          fi
+          sudo tar -xf ~/cross.tar.gz -C /usr/local/bin/
+          sudo chmod +x /usr/local/bin/cross
+        if: matrix.os != 'windows-2019'
+      - run: |
+          # Install rust toolchains for host
+
+          # Detect the current version of rust
+          version="$(grep 'DEFAULT_RUST_VERSION =' ./rust/repositories.bzl | sed 's/DEFAULT_RUST_VERSION = "//' | sed 's/"//')"
+
+          rustup override set "${version}"
+          rustup update stable && rustup default stable
+      - run: |
+          # Setup macos build tooling
+          sudo xcode-select -s /Applications/Xcode_12.4.app/Contents/Developer/
+
+          # Set SDK environment variables
+          echo "SDKROOT=$(xcrun -sdk macosx11.1 --show-sdk-path)" >> $GITHUB_ENV
+          echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx11.1 --show-sdk-platform-version)" >> $GITHUB_ENV
+        if: matrix.os == 'macOS-10.15'
+      - run: |
+          # Build binaries
+          ./crate_universe/private/bootstrap/build.sh
+        env:
+          TARGET: "${{ matrix.env.TARGET }}"
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "${{ matrix.env.TARGET }}"
+          path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/${{ matrix.env.TARGET }}
+          if-no-files-found: error
+  release:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # Save the git credentials so we can commit back to the repo
+          persist-credentials: true
+      - uses: actions/download-artifact@v2
+        with:
+          path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin
+      - run: |
+          # Ensure all files are executable
+          chmod +x ${{ github.workspace }}/crate_universe/private/bootstrap/bin/*/release/*
+      - uses: actions/create-release@v1
+        id: crate_universe_release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          prerelease: true
+          tag_name: crate_universe-${{ github.run_number }}
+          release_name: crate_universe-${{ github.run_number }}
+          body: |
+            From commit: ${{ github.sha }}
+      # There must be a upload action for each platform triple we create
+      - name: "Upload aarch64-apple-darwin"
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.crate_universe_release.outputs.upload_url }}
+          asset_name: crate_universe_resolver-aarch64-apple-darwin
+          asset_path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/aarch64-apple-darwin/release/crate_universe_resolver
+          asset_content_type: application/octet-stream
+      - name: "Upload aarch64-unknown-linux-gnu"
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.crate_universe_release.outputs.upload_url }}
+          asset_name: crate_universe_resolver-aarch64-unknown-linux-gnu
+          asset_path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/aarch64-unknown-linux-gnu/release/crate_universe_resolver
+          asset_content_type: application/octet-stream
+      - name: "Upload x86_64-apple-darwin"
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.crate_universe_release.outputs.upload_url }}
+          asset_name: crate_universe_resolver-x86_64-apple-darwin
+          asset_path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/x86_64-apple-darwin/release/crate_universe_resolver
+          asset_content_type: application/octet-stream
+      - name: "Upload x86_64-pc-windows-gnu"
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.crate_universe_release.outputs.upload_url }}
+          asset_name: crate_universe_resolver-x86_64-pc-windows-gnu.exe
+          asset_path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/x86_64-pc-windows-gnu/release/crate_universe_resolver.exe
+          asset_content_type: application/octet-stream
+      - name: "Upload x86_64-unknown-linux-gnu"
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.crate_universe_release.outputs.upload_url }}
+          asset_name: crate_universe_resolver-x86_64-unknown-linux-gnu
+          asset_path: ${{ github.workspace }}/crate_universe/private/bootstrap/bin/x86_64-unknown-linux-gnu/release/crate_universe_resolver
+          asset_content_type: application/octet-stream
+      - run: |
+          # Write new defaults.bzl file
+          # Copy the new template
+          cp ${{ github.workspace }}/crate_universe/private/defaults.bzl.template ${{ github.workspace }}/crate_universe/private/defaults.bzl
+          
+          # Replace the url
+          url="$(echo $URL | sed 's|releases/tag/|releases/download/|')/{bin}"
+          sed -i "s|{DEFAULT_URL_TEMPLATE}|${url}|" ${{ github.workspace }}/crate_universe/private/defaults.bzl
+
+          # Populate all sha256 values
+          TARGETS=(
+            aarch64-apple-darwin
+            aarch64-unknown-linux-gnu
+            x86_64-apple-darwin
+            x86_64-pc-windows-gnu
+            x86_64-unknown-linux-gnu
+          )
+          for triple in ${TARGETS[@]}; do
+            if [[ "${triple}" == *"windows"* ]]; then
+                bin_name=crate_universe_resolver.exe
+            else
+                bin_name=crate_universe_resolver
+            fi
+            sha256="$(shasum --algorithm 256 ${{ github.workspace }}/crate_universe/private/bootstrap/bin/${triple}/release/${bin_name} | awk '{ print $1 }')"
+            sed -i "s|{${triple}--sha256}|${sha256}|" ${{ github.workspace }}/crate_universe/private/defaults.bzl
+          done
+        env:
+          URL: ${{ steps.crate_universe_release.outputs.html_url }}
+      - run: |
+          # Commit and push changes back to the repo
+          if [[ -z "$(git status --porcelain)" ]]; then 
+            echo "No change to any binaries" 
+            exit 0 
+          fi
+          
+          git stage ${GITHUB_WORKSPACE}/crate_universe/private/defaults.bzl && git status
+
+          # Setup git and commit back to the repo
+          git config user.email "rules_rust@googlegroups.com"
+          git config user.name "github-actions"
+
+          # Cache the current info
+          commit="$(git rev-parse HEAD)"
+          branch="$(git branch --show-current)"
+
+          # commit the change
+          git commit -m "github-actions: update crate_universe binaries from ${commit}"
+
+          # Attempt to push but compensate for potential issues where another PR would
+          # have been merged before this pipeline gets to this point.
+          for i in 1 2 3 4 5 ; do
+            {
+              git push origin "${branch}" && break
+            } || {
+              sleep 10
+              git pull --rebase
+            }
+          done
+
+          # Ensure we've pushed our commit
+          if [[ -n "$(git diff "${branch}..HEAD")" ]]; then
+            echo "Failed to push changes"
+            exit 1
+          fi

.gitignore

@@ -8,6 +8,7 @@
 /examples/bazel-*
 /examples/crate_universe/*/bazel-*
 /docs/bazel-*
+crate_universe.bazelrc
 
 # rustfmt
 *.rs.bk

WORKSPACE.bazel

@@ -4,6 +4,10 @@ load("@rules_rust//rust:repositories.bzl", "rust_repositories")
 
 rust_repositories()
 
+load("@rules_rust//crate_universe:defs.bzl", "crate_universe_deps")
+
+crate_universe_deps()
+
 load("@rules_rust//proto:repositories.bzl", "rust_proto_repositories")
 
 rust_proto_repositories()

bootstrap.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -euo pipefail
+
+#######################################################################################################################
+# Crate Universe
+#######################################################################################################################
+
+# Find the location of the script
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+BOOTSTRAP_DIR="${SCRIPT_DIR}/crate_universe/private/bootstrap"
+
+# Go to root of rules_rust
+pushd "${SCRIPT_DIR}" &> /dev/null
+has_changes="$(git log origin/main.. -- crate_universe)"
+popd &> /dev/null
+
+# Only bootstrap the binary if there are changes
+if [[ -n "${has_changes}" ]]; then
+    pushd "${BOOTSTRAP_DIR}" &> /dev/null
+    bazel run //:build && bazel run //:install && bazel clean
+    popd &> /dev/null
+
+    # Generate a bazelrc file for ensuring we always use the resolver binary for the current host
+    cat << EOF > "${SCRIPT_DIR}/crate_universe.bazelrc"
+common --announce_rc
+common --override_repository="rules_rust_crate_universe__aarch64-apple-darwin=${BOOTSTRAP_DIR}"
+common --override_repository="rules_rust_crate_universe__aarch64-unknown-linux-gnu=${BOOTSTRAP_DIR}"
+common --override_repository="rules_rust_crate_universe__x86_64-apple-darwin=${BOOTSTRAP_DIR}"
+common --override_repository="rules_rust_crate_universe__x86_64-pc-windows-gnu=${BOOTSTRAP_DIR}"
+common --override_repository="rules_rust_crate_universe__x86_64-unknown-linux-gnu=${BOOTSTRAP_DIR}"
+EOF
+
+    cp ${SCRIPT_DIR}/crate_universe.bazelrc ${SCRIPT_DIR}/examples/crate_universe/crate_universe.bazelrc
+else
+    echo "No changes to crate_universe. Nothing to do"
+fi
+
+#######################################################################################################################

crate_universe/CONTRIBUTING.md

@@ -18,21 +18,20 @@ Some areas have unit testing, there are a few (very brittle) integration tests,
 
 ## How to test local changes
 
-To use a local version, first build it:
+To use a local version, first bootstrap it:
 
 ```console
-crate_universe $ cargo build
+rules_rust $ ./bootstrap.sh
 ```
 
-Then run a bazel build with this environment variable:
+This will build `crate_universe_resolver` and generate a `crate_universe.bazelrc` file which contains
+[override_repository](https://docs.bazel.build/versions/master/command-line-reference.html#flag--override_repository)
+definitions fo the newly built binary. For more details on the build process, see
+[crate_universe/private/bootstrap/README.md](./private/bootstrap/README.md).
 
-```console
-RULES_RUST_RESOLVER_URL_OVERRIDE=file:///path/to/resolver/target/debug/resolver bazel build //whatever
-```
-
-To get verbose logging, edit `defs.bzl` to set `RUST_LOG` to `debug` or `trace` instead of `info`. In particular, that will print out the generated Cargo.toml, and the path to the generated workspace file.
+To get verbose logging, edit `defs.bzl` to set `RUST_LOG` to `debug` or `trace` instead of `info`. In particular, that will print out the generated `Cargo.toml`, and the path to the generated workspace file.
 
-Note that you may need to `bazel shutdown` between bazel commands. Hopefully not, but if you're seeing stale results, try it out. This only affects local development.
+Note that you may need to `bazel shutdown` between Bazel commands. Hopefully not, but if you're seeing stale results, try it out. This only affects local development.
 
 ## Testing
 

crate_universe/Cargo.toml

@@ -2,9 +2,9 @@
 name = "crate_universe_resolver"
 version = "0.1.0-experimental"
 authors = [
-  "Daniel Wagner-Hall <dwagnerhall@apple.com>",
-  "Romain Chossart <romainchossart@gmail.com>",
-  "Gibson Fahnestock <gib@apple.com>",
+    "Daniel Wagner-Hall <dwagnerhall@apple.com>",
+    "Romain Chossart <romainchossart@gmail.com>",
+    "Gibson Fahnestock <gib@apple.com>",
 ]
 edition = "2018"
 
@@ -14,9 +14,9 @@ edition = "2018"
 # Several of these dependencies need to be compatible with the version cargo-raze is using.
 
 anyhow = "1"
+cargo_metadata = "0.13.1"
 # Until https://github.com/google/cargo-raze/pull/407 is merged:
 cargo-raze = { git = "https://github.com/UebelAndre/cargo-raze.git", rev = "caaf8345046eee830ea89fc04771ee117bd835eb", default-features = false }
-cargo_metadata = "0.13.1"
 cfg-expr = "0.7.4"
 env_logger = "0.8"
 hex = "0.4"

crate_universe/Cross.toml

@@ -0,0 +1,18 @@
+# Darwin targets build on the host
+# https://github.com/rust-embedded/cross/issues/223
+# [target.aarch64-apple-darwin]
+# image = "rustembedded/cross:aarch64-apple-darwin"
+
+[target.aarch64-unknown-linux-gnu]
+image = "rustembedded/cross:aarch64-unknown-linux-gnu@sha256:c970d529baa1bab953f9a0f2c2294e51a12a2e710599ab91de69c45a1785ba46"
+
+# Darwin targets build on the host
+# https://github.com/rust-embedded/cross/issues/223
+# [target.x86_64-apple-darwin]
+# image = "rustembedded/cross:x86_64-apple-darwin"
+
+[target.x86_64-pc-windows-gnu]
+image = "rustembedded/cross:x86_64-pc-windows-gnu@sha256:5005553ddf4e82d4703ee7f458a24802a014f03884f87c6277e9c718765d3517"
+
+[target.x86_64-unknown-linux-gnu]
+image = "rustembedded/cross:x86_64-unknown-linux-gnu@sha256:a7f681882d23eae287cff48a93aff93f71a84859a6670c2a4fe2b82949ebb91c"