From 4d6679906b902d0e11847c1b0c7e2a55aca28829 Mon Sep 17 00:00:00 2001 From: Michael Vines Date: Thu, 30 May 2019 11:53:41 -0700 Subject: [PATCH] Clean up crates.io publishing (#4478) * Clean up crates.io publishing * Cargo.lock --- Cargo.lock | 2 - ci/order-crates-for-publishing.py | 65 +++++++++++++++++++++++++++++++ ci/publish-crate.sh | 47 ++++------------------ ci/test-checks.sh | 1 + 4 files changed, 74 insertions(+), 41 deletions(-) create mode 100755 ci/order-crates-for-publishing.py diff --git a/Cargo.lock b/Cargo.lock index 65efa8d1127181..0b0c1d7ba72822 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2586,7 +2586,6 @@ version = "0.16.0" dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "solana-logger 0.16.0", - "solana-runtime 0.16.0", "solana-sdk 0.16.0", ] @@ -2625,7 +2624,6 @@ dependencies = [ "solana-sdk 0.16.0", "solana-stake-api 0.16.0", "solana-stake-program 0.16.0", - "solana-storage-program 0.16.0", "solana-vote-api 0.16.0", "solana-vote-program 0.16.0", "sys-info 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ci/order-crates-for-publishing.py b/ci/order-crates-for-publishing.py new file mode 100755 index 00000000000000..5a6bc2f7a55da4 --- /dev/null +++ b/ci/order-crates-for-publishing.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# +# This script figures the order in which workspace crates must be published to +# crates.io. Along the way it also ensures there are no circular dependencies +# that would cause a |cargo publish| to fail. +# +# On success an ordered list of Cargo.toml files is written to stdout +# + +import os +import json +import subprocess +import sys; + +def load_metadata(): + return json.loads(subprocess.Popen( + 'cargo metadata --no-deps --format-version=1', + shell=True, stdout=subprocess.PIPE).communicate()[0]) + +def get_packages(): + metadata = load_metadata() + + manifest_path = dict() + + # Build dictionary of packages and their immediate solana-only dependencies + dependency_graph = dict() + for pkg in metadata['packages']: + manifest_path[pkg['name']] = pkg['manifest_path']; + dependency_graph[pkg['name']] = [x['name'] for x in pkg['dependencies'] if x['name'].startswith('solana-')]; + + # Check for direct circular dependencies + circular_dependencies = set() + for package, dependencies in dependency_graph.items(): + for dependency in dependencies: + if dependency in dependency_graph and package in dependency_graph[dependency]: + circular_dependencies.add(' <--> '.join(sorted([package, dependency]))) + + for dependency in circular_dependencies: + sys.stderr.write('Error: Circular dependency: {}\n'.format(dependency)) + + if len(circular_dependencies) != 0: + sys.exit(1) + + # Order dependencies + sorted_dependency_graph = [] + max_iterations = pow(len(dependency_graph),2) + while dependency_graph: + if max_iterations == 0: + # TODO: Be more helpful and find the actual cycle for the user + sys.exit('Error: Circular dependency suspected between these packages: {}\n'.format(' '.join(dependency_graph.keys()))) + + max_iterations -= 1 + for package, dependencies in dependency_graph.items(): + for dependency in dependencies: + if dependency in dependency_graph: + break + else: + del dependency_graph[package] + sorted_dependency_graph.append((package, manifest_path[package])) + + + return sorted_dependency_graph + +for package, manifest in get_packages(): + print os.path.relpath(manifest) diff --git a/ci/publish-crate.sh b/ci/publish-crate.sh index d9c5ea28ced9c0..6426f8df67c32b 100755 --- a/ci/publish-crate.sh +++ b/ci/publish-crate.sh @@ -3,36 +3,6 @@ set -e cd "$(dirname "$0")/.." source ci/semver_bash/semver.sh -# List of internal crates to publish -# -# IMPORTANT: the order of the CRATES *is* significant. Crates must be published -# before the crates that depend on them. Note that this information is already -# expressed in the various Cargo.toml files, and ideally would not be duplicated -# here. (TODO: figure the crate ordering dynamically) -# -CRATES=( - kvstore - logger - netutil - sdk - keygen - metrics - client - drone - programs/{budget_api,config_api,stake_api,storage_api,token_api,vote_api,exchange_api} - programs/{vote_program,budget_program,bpf_loader,config_program,exchange_program,failure_program} - programs/{noop_program,stake_program,storage_program,token_program} - runtime - vote-signer - core - validator - genesis - gossip - ledger-tool - wallet - install -) - # Only package/publish if this is a tagged release [[ -n $TRIGGERED_BUILDKITE_TAG ]] || { echo TRIGGERED_BUILDKITE_TAG unset, skipped @@ -49,25 +19,24 @@ expectedCrateVersion="$MAJOR.$MINOR.$PATCH$SPECIAL" cargoCommand="cargo publish --token $CRATES_IO_TOKEN" -for crate in "${CRATES[@]}"; do - if [[ ! -r $crate/Cargo.toml ]]; then - echo "Error: $crate/Cargo.toml does not exist" - exit 1 - fi - echo "-- $crate" - grep -q "^version = \"$expectedCrateVersion\"$" "$crate"/Cargo.toml || { - echo "Error: $crate/Cargo.toml version is not $expectedCrateVersion" +Cargo_tomls=$(ci/order-crates-for-publishing.py) + +for Cargo_toml in "${Cargo_tomls[@]}"; do + echo "-- $Cargo_toml" + grep -q "^version = \"$expectedCrateVersion\"$" "$Cargo_toml" || { + echo "Error: $Cargo_toml version is not $expectedCrateVersion" exit 1 } ( set -x + crate=$(dirname "$Cargo_toml") # TODO: the rocksdb package does not build with the stock rust docker image, # so use the solana rust docker image until this is resolved upstream source ci/rust-version.sh ci/docker-run.sh "$rust_stable_docker_image" bash -exc "cd $crate; $cargoCommand" #ci/docker-run.sh rust bash -exc "cd $crate; $cargoCommand" - ) + ) || true # <-- Don't fail. We want to be able to retry the job in cases when a publish fails halfway due to network/cloud issues done exit 0 diff --git a/ci/test-checks.sh b/ci/test-checks.sh index ddc9e05457959b..0e63e9d83cf9bf 100755 --- a/ci/test-checks.sh +++ b/ci/test-checks.sh @@ -14,6 +14,7 @@ _ cargo +"$rust_stable" clippy --all -- --version _ cargo +"$rust_stable" clippy --all -- --deny=warnings _ cargo +"$rust_stable" audit _ ci/nits.sh +_ ci/order-crates-for-publishing.py _ book/build.sh echo --- ok