Reduce wasm binary size with cargo-xbuild & removing rlib crate-type

.gitlab-ci.yml

@@ -17,6 +17,8 @@ variables:
   CARGO_TARGET_DIR:                "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}"
   CI_SERVER_NAME:                  "GitLab CI"
   REGISTRY:                        registry.parity.io/parity/infrastructure/scripts
+  RUSTUP_TOOLCHAIN:                nightly
+  RUST_LIB_BACKTRACE:              0
 
 .collect-artifacts:                &collect-artifacts
   artifacts:

CHANGELOG.md

@@ -0,0 +1,8 @@
+# Version 0.4.0 (2020-02-26)
+
+- Minimize contract wasm binary size:
+  - Uses [`cargo-xbuild`](https://github.com/rust-osdev/cargo-xbuild) to build custom sysroot crates without panic string
+bloat.
+  - Automatically removes the `rlib` crate type from `Cargo.toml`, removing redundant metadata.
+- Removes requirement for linker args specified in `.cargo/config`.
+- Added `--verbose` and `--quiet` flags for `build` and `generate-metadata` commands.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cargo-contract"
-version = "0.3.0"
+version = "0.4.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 build = "build.rs"
 edition = "2018"
@@ -27,14 +27,19 @@ cargo_metadata = "0.9.1"
 codec = { package = "parity-scale-codec", version = "1.1" }
 which = "3.1.0"
 colored = "1.9"
+toml = "0.5.4"
+cargo-xbuild = "0.5.26"
+rustc_version = "0.2.3"
+serde_json = "1.0"
+tempfile = "3.1.0"
 
 # dependencies for optional extrinsics feature
 async-std = { version = "1.5.0", optional = true }
 sp-core = { git = "https://github.com/paritytech/substrate/", package = "sp-core", optional = true }
 subxt = { git = "https://github.com/paritytech/substrate-subxt/", rev = "b7565ff435b9499ca1ff4dad519009e94a13111c", package = "substrate-subxt", optional = true }
 futures = { version = "0.3.2", optional = true }
-url = { version = "2.1.1", optional = true }
 hex = { version = "0.4.0", optional = true }
+url = { version = "2.1.1", optional = true }
 
 [build-dependencies]
 anyhow = "1.0.26"
@@ -43,10 +48,9 @@ walkdir = "2.3.1"
 
 [dev-dependencies]
 assert_matches = "1.3.0"
-tempfile = "3.1.0"
 wabt = "0.9.2"
 
 [features]
 default = []
-extrinsics = ["sp-core", "subxt", "async-std", "futures", "url", "hex"]
+extrinsics = ["sp-core", "subxt", "async-std", "futures", "hex", "url"]
 test-ci-only = []

README.md

@@ -8,6 +8,11 @@ A small CLI tool for helping setting up and managing WebAssembly smart contracts
 
 ## Installation
 
+### Prerequisites
+
+- **rust-src**: `rustup component add rust-src`
+- **wasm-opt**: https://github.com/WebAssembly/binaryen#tools
+
 `cargo install --git https://github.com/paritytech/cargo-contract cargo-contract --force`
 
 Use the --force to ensure you are updated to the most recent cargo-contract version.
@@ -35,6 +40,13 @@ SUBCOMMANDS:
     help                 Prints this message or the help of the given subcommand(s)
 ```
 
+## `build` requires the `nightly` toolchain
+
+`cargo contract build` must be run using the `nightly` toolchain. If you have 
+[`rustup`](https://github.com/rust-lang/rustup) installed, the simplest way to do so is `cargo +nightly contract build`.
+To avoid having to add `+nightly` you can also create a `rust-toolchain` file in your local directory containing 
+`nightly`. Read more about how to [specify the rustup toolchain](https://github.com/rust-lang/rustup#override-precedence).
+
 ## Features
 
 The `deploy` and `instantiate` subcommands are **disabled by default**, since they are not fully stable yet and increase the build time.

src/cmd/build.rs

@@ -17,45 +17,52 @@
 use std::{
     fs::metadata,
     io::{self, Write},
-    path::PathBuf,
+    path::{Path, PathBuf},
     process::Command,
 };
 
+use crate::{
+    util,
+    workspace::{ManifestPath, Workspace},
+    Verbosity,
+};
 use anyhow::{Context, Result};
-use cargo_metadata::MetadataCommand;
+use cargo_metadata::Package;
 use colored::Colorize;
 use parity_wasm::elements::{External, MemoryType, Module, Section};
 
 /// This is the maximum number of pages available for a contract to allocate.
 const MAX_MEMORY_PAGES: u32 = 16;
 
 /// Relevant metadata obtained from Cargo.toml.
+#[derive(Debug)]
 pub struct CrateMetadata {
+    manifest_path: ManifestPath,
+    cargo_meta: cargo_metadata::Metadata,
     package_name: String,
+    root_package: Package,
     original_wasm: PathBuf,
     pub dest_wasm: PathBuf,
 }
 
-/// Parses the contract manifest and returns relevant metadata.
-pub fn collect_crate_metadata(working_dir: Option<&PathBuf>) -> Result<CrateMetadata> {
-    let mut cmd = MetadataCommand::new();
-    if let Some(dir) = working_dir {
-        cmd.current_dir(dir);
+impl CrateMetadata {
+    pub fn target_dir(&self) -> &Path {
+        self.cargo_meta.target_directory.as_path()
     }
-    let metadata = cmd.exec()?;
+}
 
-    let root_package_id = metadata
-        .resolve
-        .and_then(|resolve| resolve.root)
-        .context("Cannot infer the root project id")?;
+/// Parses the contract manifest and returns relevant metadata.
+pub fn collect_crate_metadata(manifest_path: &ManifestPath) -> Result<CrateMetadata> {
+    let (metadata, root_package_id) = crate::util::get_cargo_metadata(manifest_path)?;
 
     // Find the root package by id in the list of packages. It is logical error if the root
     // package is not found in the list.
     let root_package = metadata
         .packages
         .iter()
         .find(|package| package.id == root_package_id)
-        .expect("The package is not found in the `cargo metadata` output");
+        .expect("The package is not found in the `cargo metadata` output")
+        .clone();
 
     // Normalize the package name.
     let package_name = root_package.name.replace("-", "_");
@@ -72,27 +79,69 @@ pub fn collect_crate_metadata(working_dir: Option<&PathBuf>) -> Result<CrateMeta
     dest_wasm.push(package_name.clone());
     dest_wasm.set_extension("wasm");
 
-    Ok(CrateMetadata {
+    let crate_metadata = CrateMetadata {
+        manifest_path: manifest_path.clone(),
+        cargo_meta: metadata,
+        root_package: root_package.clone(),
         package_name,
         original_wasm,
         dest_wasm,
-    })
+    };
+    Ok(crate_metadata)
 }
 
-/// Invokes `cargo build` in the specified directory, defaults to the current directory.
+/// Builds the project in the specified directory, defaults to the current directory.
 ///
-/// Currently it assumes that user wants to use `+nightly`.
-fn build_cargo_project(working_dir: Option<&PathBuf>) -> Result<()> {
-    super::exec_cargo(
-        "build",
-        &[
+/// Uses [`cargo-xbuild`](https://github.com/rust-osdev/cargo-xbuild) for maximum optimization of
+/// the resulting Wasm binary.
+fn build_cargo_project(crate_metadata: &CrateMetadata, verbosity: Option<Verbosity>) -> Result<()> {
+    util::assert_channel()?;
+
+    // set RUSTFLAGS, read from environment var by cargo-xbuild
+    std::env::set_var(
+        "RUSTFLAGS",
+        "-C link-arg=-z -C link-arg=stack-size=65536 -C link-arg=--import-memory",
+    );
+
+    let verbosity = verbosity.map(|v| match v {
+        Verbosity::Verbose => xargo_lib::Verbosity::Verbose,
+        Verbosity::Quiet => xargo_lib::Verbosity::Quiet,
+    });
+
+    let xbuild = |manifest_path: &ManifestPath| {
+        let manifest_path = Some(manifest_path);
+        let target = Some("wasm32-unknown-unknown");
+        let target_dir = crate_metadata.target_dir();
+        let other_args = [
             "--no-default-features",
             "--release",
-            "--target=wasm32-unknown-unknown",
-            "--verbose",
-        ],
-        working_dir,
-    )
+            &format!("--target-dir={}", target_dir.to_string_lossy()),
+        ];
+        let args = xargo_lib::Args::new(target, manifest_path, verbosity, &other_args)
+            .map_err(|e| anyhow::anyhow!("{}", e))
+            .context("Creating xargo args")?;
+
+        let config = xargo_lib::Config {
+            sysroot_path: target_dir.join("sysroot"),
+            memcpy: false,
+            panic_immediate_abort: true,
+        };
+
+        let exit_status = xargo_lib::build(args, "build", Some(config))
+            .map_err(|e| anyhow::anyhow!("{}", e))
+            .context("Building with xbuild")?;
+        log::debug!("xargo exit status: {:?}", exit_status);
+        Ok(())
+    };
+
+    Workspace::new(&crate_metadata.cargo_meta, &crate_metadata.root_package.id)?
+        .with_root_package_manifest(|manifest| {
+            manifest.with_removed_crate_type("rlib")?;
+            Ok(())
+        })?
+        .using_temp(xbuild)?;
+
+    Ok(())
 }
 
 /// Ensures the wasm memory import of a given module has the maximum number of pages.
@@ -145,7 +194,11 @@ fn strip_custom_sections(module: &mut Module) {
 /// Performs required post-processing steps on the wasm artifact.
 fn post_process_wasm(crate_metadata: &CrateMetadata) -> Result<()> {
     // Deserialize wasm module from a file.
-    let mut module = parity_wasm::deserialize_file(&crate_metadata.original_wasm)?;
+    let mut module =
+        parity_wasm::deserialize_file(&crate_metadata.original_wasm).context(format!(
+            "Loading original wasm file '{}'",
+            crate_metadata.original_wasm.display()
+        ))?;
 
     // Perform optimization.
     //
@@ -198,10 +251,10 @@ fn optimize_wasm(crate_metadata: &CrateMetadata) -> Result<()> {
         anyhow::bail!("wasm-opt optimization failed");
     }
 
-    let original_size = metadata(&crate_metadata.dest_wasm)?.len() / 1000;
-    let optimized_size = metadata(&optimized)?.len() / 1000;
+    let original_size = metadata(&crate_metadata.dest_wasm)?.len() as f64 / 1000.0;
+    let optimized_size = metadata(&optimized)?.len() as f64 / 1000.0;
     println!(
-        " Original wasm size: {}K, Optimized: {}K",
+        " Original wasm size: {:.1}K, Optimized: {:.1}K",
         original_size, optimized_size
     );
 
@@ -213,19 +266,22 @@ fn optimize_wasm(crate_metadata: &CrateMetadata) -> Result<()> {
 /// Executes build of the smart-contract which produces a wasm binary that is ready for deploying.
 ///
 /// It does so by invoking build by cargo and then post processing the final binary.
-pub(crate) fn execute_build(working_dir: Option<&PathBuf>) -> Result<String> {
+pub(crate) fn execute_build(
+    manifest_path: ManifestPath,
+    verbosity: Option<Verbosity>,
+) -> Result<String> {
     println!(
         " {} {}",
         "[1/4]".bold(),
         "Collecting crate metadata".bright_green().bold()
     );
-    let crate_metadata = collect_crate_metadata(working_dir)?;
+    let crate_metadata = collect_crate_metadata(&manifest_path)?;
     println!(
         " {} {}",
         "[2/4]".bold(),
         "Building cargo project".bright_green().bold()
     );
-    build_cargo_project(working_dir)?;
+    build_cargo_project(&crate_metadata, verbosity)?;
     println!(
         " {} {}",
         "[3/4]".bold(),
@@ -248,13 +304,15 @@ pub(crate) fn execute_build(working_dir: Option<&PathBuf>) -> Result<String> {
 #[cfg(feature = "test-ci-only")]
 #[cfg(test)]
 mod tests {
-    use crate::cmd::{execute_new, tests::with_tmp_dir};
+    use crate::{cmd::execute_new, util::tests::with_tmp_dir, workspace::ManifestPath};
 
     #[test]
     fn build_template() {
         with_tmp_dir(|path| {
             execute_new("new_project", Some(path)).expect("new project creation failed");
-            super::execute_build(Some(&path.join("new_project"))).expect("build failed");
+            let manifest_path =
+                ManifestPath::new(&path.join("new_project").join("Cargo.toml")).unwrap();
+            super::execute_build(manifest_path, None).expect("build failed");
         });
     }
 }

src/cmd/deploy.rs

@@ -28,7 +28,7 @@ use crate::{cmd::build, ExtrinsicOpts};
 fn load_contract_code(path: Option<&PathBuf>) -> Result<Vec<u8>> {
     let contract_wasm_path = match path {
         Some(path) => path.clone(),
-        None => build::collect_crate_metadata(path)?.dest_wasm,
+        None => build::collect_crate_metadata(&Default::default())?.dest_wasm,
     };
     log::info!("Contract code path: {}", contract_wasm_path.display());
     let mut data = Vec::new();
@@ -61,10 +61,7 @@ pub(crate) fn execute_deploy(
 mod tests {
     use std::{fs, io::Write};
 
-    use crate::{
-        cmd::{deploy::execute_deploy, tests::with_tmp_dir},
-        ExtrinsicOpts,
-    };
+    use crate::{cmd::deploy::execute_deploy, util::tests::with_tmp_dir, ExtrinsicOpts};
     use assert_matches::assert_matches;
 
     const CONTRACT: &str = r#"

src/cmd/instantiate.rs

@@ -40,10 +40,7 @@ pub(crate) fn execute_instantiate(
 mod tests {
     use std::{fs, io::Write};
 
-    use crate::{
-        cmd::{deploy::execute_deploy, tests::with_tmp_dir},
-        ExtrinsicOpts, HexData,
-    };
+    use crate::{cmd::deploy::execute_deploy, util::tests::with_tmp_dir, ExtrinsicOpts, HexData};
     use assert_matches::assert_matches;
 
     const CONTRACT: &str = r#"