Clean up crates.io publishing (#4478)

* Clean up crates.io publishing

* Cargo.lock

ci/order-crates-for-publishing.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+#
+# This script figures the order in which workspace crates must be published to
+# crates.io.  Along the way it also ensures there are no circular dependencies
+# that would cause a |cargo publish| to fail.
+#
+# On success an ordered list of Cargo.toml files is written to stdout
+#
+
+import os
+import json
+import subprocess
+import sys;
+
+def load_metadata():
+    return json.loads(subprocess.Popen(
+        'cargo metadata --no-deps --format-version=1',
+        shell=True, stdout=subprocess.PIPE).communicate()[0])
+
+def get_packages():
+    metadata = load_metadata()
+
+    manifest_path = dict()
+
+    # Build dictionary of packages and their immediate solana-only dependencies
+    dependency_graph = dict()
+    for pkg in metadata['packages']:
+        manifest_path[pkg['name']] = pkg['manifest_path'];
+        dependency_graph[pkg['name']] = [x['name'] for x in pkg['dependencies'] if x['name'].startswith('solana-')];
+
+    # Check for direct circular dependencies
+    circular_dependencies = set()
+    for package, dependencies in dependency_graph.items():
+        for dependency in dependencies:
+            if dependency in dependency_graph and package in dependency_graph[dependency]:
+                circular_dependencies.add(' <--> '.join(sorted([package, dependency])))
+
+    for dependency in circular_dependencies:
+        sys.stderr.write('Error: Circular dependency: {}\n'.format(dependency))
+
+    if len(circular_dependencies) != 0:
+        sys.exit(1)
+
+    # Order dependencies
+    sorted_dependency_graph = []
+    max_iterations = pow(len(dependency_graph),2)
+    while dependency_graph:
+        if max_iterations == 0:
+            # TODO: Be more helpful and find the actual cycle for the user
+            sys.exit('Error: Circular dependency suspected between these packages: {}\n'.format(' '.join(dependency_graph.keys())))
+
+        max_iterations -= 1
+        for package, dependencies in dependency_graph.items():
+            for dependency in dependencies:
+                if dependency in dependency_graph:
+                    break
+            else:
+                del dependency_graph[package]
+                sorted_dependency_graph.append((package, manifest_path[package]))
+
+
+    return sorted_dependency_graph
+
+for package, manifest in get_packages():
+    print os.path.relpath(manifest)

ci/publish-crate.sh

@@ -3,36 +3,6 @@ set -e
 cd "$(dirname "$0")/.."
 source ci/semver_bash/semver.sh
 
-# List of internal crates to publish
-#
-# IMPORTANT: the order of the CRATES *is* significant.  Crates must be published
-# before the crates that depend on them.  Note that this information is already
-# expressed in the various Cargo.toml files, and ideally would not be duplicated
-# here. (TODO: figure the crate ordering dynamically)
-#
-CRATES=(
-  kvstore
-  logger
-  netutil
-  sdk
-  keygen
-  metrics
-  client
-  drone
-  programs/{budget_api,config_api,stake_api,storage_api,token_api,vote_api,exchange_api}
-  programs/{vote_program,budget_program,bpf_loader,config_program,exchange_program,failure_program}
-  programs/{noop_program,stake_program,storage_program,token_program}
-  runtime
-  vote-signer
-  core
-  validator
-  genesis
-  gossip
-  ledger-tool
-  wallet
-  install
-)
-
 # Only package/publish if this is a tagged release
 [[ -n $TRIGGERED_BUILDKITE_TAG ]] || {
   echo TRIGGERED_BUILDKITE_TAG unset, skipped
@@ -49,25 +19,24 @@ expectedCrateVersion="$MAJOR.$MINOR.$PATCH$SPECIAL"
 
 cargoCommand="cargo publish --token $CRATES_IO_TOKEN"
 
-for crate in "${CRATES[@]}"; do
-  if [[ ! -r $crate/Cargo.toml ]]; then
-    echo "Error: $crate/Cargo.toml does not exist"
-    exit 1
-  fi
-  echo "-- $crate"
-  grep -q "^version = \"$expectedCrateVersion\"$" "$crate"/Cargo.toml || {
-    echo "Error: $crate/Cargo.toml version is not $expectedCrateVersion"
+Cargo_tomls=$(ci/order-crates-for-publishing.py)
+
+for Cargo_toml in "${Cargo_tomls[@]}"; do
+  echo "-- $Cargo_toml"
+  grep -q "^version = \"$expectedCrateVersion\"$" "$Cargo_toml" || {
+    echo "Error: $Cargo_toml version is not $expectedCrateVersion"
     exit 1
   }
 
   (
     set -x
+    crate=$(dirname "$Cargo_toml")
     # TODO: the rocksdb package does not build with the stock rust docker image,
     # so use the solana rust docker image until this is resolved upstream
     source ci/rust-version.sh
     ci/docker-run.sh "$rust_stable_docker_image" bash -exc "cd $crate; $cargoCommand"
     #ci/docker-run.sh rust bash -exc "cd $crate; $cargoCommand"
-  )
+  ) || true # <-- Don't fail.  We want to be able to retry the job in cases when a publish fails halfway due to network/cloud issues
 done
 
 exit 0

ci/test-checks.sh

@@ -14,6 +14,7 @@ _ cargo +"$rust_stable" clippy --all -- --version
 _ cargo +"$rust_stable" clippy --all -- --deny=warnings
 _ cargo +"$rust_stable" audit
 _ ci/nits.sh
+_ ci/order-crates-for-publishing.py
 _ book/build.sh
 
 echo --- ok