Auto merge of rust-lang#78486 - pietroalbini:manifest-artifacts, r=Mark-Simulacrum

Include non-rustup artifacts in the manifest

This PR fixes rust-lang/promote-release#22 by including all the files we ship in the generated manifests, even the ones that are not installable through rustup. In practice this adds the following "artifacts":

* `source-code`: the tarball containing the full source code used to build the release (`rustc-{channel}-src.tar.xz`)
* `installer-msi`: the MSI installer for Windows systems (`rust-{channel}-{target}.msi`)
* `installer-pkg`: the PKG installer for macOS systems (`rust-{channel}-{target}.pkg`)

These files are included in a new `artifacts` table of the manifest, like so:

```toml
[[artifacts.installer-msi.target.aarch64-pc-windows-msvc]]
url = "https://example.com/2020-10-28/rust-nightly-aarch64-pc-windows-msvc.msi"
hash-sha256 = "6b41d5b829d20834c5d93628d008ec618f8914ee79303363bd13a86fd5f305dd"

[[artifacts.installer-msi.target.i686-pc-windows-gnu]]
url = "https://example.com/2020-10-28/rust-nightly-i686-pc-windows-gnu.msi"
hash-sha256 = "83f020de6e180c155add9fce1cea2ac6e5f744edbd6dc1581e24de8f56b2ca7a"

[[artifacts.installer-msi.target.i686-pc-windows-msvc]]
url = "https://example.com/2020-10-28/rust-nightly-i686-pc-windows-msvc.msi"
hash-sha256 = "dbc80c24e9d5df01616c6f216114b4351f51a94218e2368b5cebe4165b270702"

[[artifacts.installer-msi.target.x86_64-pc-windows-gnu]]
url = "https://example.com/2020-10-28/rust-nightly-x86_64-pc-windows-gnu.msi"
hash-sha256 = "8196eca3f02d72d4c8776ad4fcc72897125e2cf6404ae933e31c07e197e3c9fa"

[[artifacts.installer-msi.target.x86_64-pc-windows-msvc]]
url = "https://example.com/2020-10-28/rust-nightly-x86_64-pc-windows-msvc.msi"
hash-sha256 = "b2e7fd6463790732fcf9c726b9448068712341943199cb40fc11d1138b8a207b"

[[artifacts.installer-pkg.target.aarch64-apple-darwin]]
url = "https://example.com/2020-10-28/rust-nightly-aarch64-apple-darwin.pkg"
hash-sha256 = "70421c191752fb33886f8033b029e634bcc993b72308cef52a38405840e91f5c"

[[artifacts.installer-pkg.target.x86_64-apple-darwin]]
url = "https://example.com/2020-10-28/rust-nightly-x86_64-apple-darwin.pkg"
hash-sha256 = "ebd7a5acb61e82d85e855146cc9bd856f32228ee7f40dd94c659b00614ed4f1f"

[[artifacts.source-code.target."*"]]
url = "https://example.com/2020-10-28/rustc-nightly-src.tar.gz"
hash-sha256 = "5fcc487ee4c15c689de8ddf7daac7ff6a65c80498197b9aea58622dc2b3bca10"

[[artifacts.source-code.target."*"]]
url = "https://example.com/2020-10-28/rustc-nightly-src.tar.xz"
hash-sha256 = "0c618ef0ec5f64da1801e9d0df6c755f6ed1a8780ec5c8ee75e55614be51d42c"

```

Each artifact can be available for multiple targets, and each target can have multiple versions of the same file (for example, a `gz`-compressed one and a `xz`-compressed one). In the future rustup might add functionality to let users retrieve the artifacts, but that's not needed to land this PR, and whether to do the implementation is up to the rustup maintainers.

r? `@kinnison`
cc `@Mark-Simulacrum`

Author: bors

src/tools/build-manifest/src/main.rs

@@ -168,6 +168,16 @@ static DOCS_TARGETS: &[&str] = &[
     "x86_64-unknown-linux-musl",
 ];
 
+static MSI_INSTALLERS: &[&str] = &[
+    "aarch64-pc-windows-msvc",
+    "i686-pc-windows-gnu",
+    "i686-pc-windows-msvc",
+    "x86_64-pc-windows-gnu",
+    "x86_64-pc-windows-msvc",
+];
+
+static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"];
+
 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 
 static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview", "rust-analyzer-preview"];
@@ -314,10 +324,12 @@ impl Builder {
             manifest_version: "2".to_string(),
             date: self.date.to_string(),
             pkg: BTreeMap::new(),
+            artifacts: BTreeMap::new(),
             renames: BTreeMap::new(),
             profiles: BTreeMap::new(),
         };
         self.add_packages_to(&mut manifest);
+        self.add_artifacts_to(&mut manifest);
         self.add_profiles_to(&mut manifest);
         self.add_renames_to(&mut manifest);
         manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
@@ -346,6 +358,27 @@ impl Builder {
         package("llvm-tools-preview", TARGETS);
     }
 
+    fn add_artifacts_to(&mut self, manifest: &mut Manifest) {
+        manifest.add_artifact("source-code", |artifact| {
+            let tarball = self.versions.tarball_name(&PkgType::Rustc, "src").unwrap();
+            artifact.add_tarball(self, "*", &tarball);
+        });
+
+        manifest.add_artifact("installer-msi", |artifact| {
+            for target in MSI_INSTALLERS {
+                let msi = self.versions.archive_name(&PkgType::Rust, target, "msi").unwrap();
+                artifact.add_file(self, target, &msi);
+            }
+        });
+
+        manifest.add_artifact("installer-pkg", |artifact| {
+            for target in PKG_INSTALLERS {
+                let pkg = self.versions.archive_name(&PkgType::Rust, target, "pkg").unwrap();
+                artifact.add_file(self, target, &pkg);
+            }
+        });
+    }
+
     fn add_profiles_to(&mut self, manifest: &mut Manifest) {
         let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
         profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);

src/tools/build-manifest/src/manifest.rs

@@ -9,10 +9,19 @@ pub(crate) struct Manifest {
     pub(crate) manifest_version: String,
     pub(crate) date: String,
     pub(crate) pkg: BTreeMap<String, Package>,
+    pub(crate) artifacts: BTreeMap<String, Artifact>,
     pub(crate) renames: BTreeMap<String, Rename>,
     pub(crate) profiles: BTreeMap<String, Vec<String>>,
 }
 
+impl Manifest {
+    pub(crate) fn add_artifact(&mut self, name: &str, f: impl FnOnce(&mut Artifact)) {
+        let mut artifact = Artifact { target: BTreeMap::new() };
+        f(&mut artifact);
+        self.artifacts.insert(name.to_string(), artifact);
+    }
+}
+
 #[derive(Serialize)]
 pub(crate) struct Package {
     pub(crate) version: String,
@@ -25,6 +34,42 @@ pub(crate) struct Rename {
     pub(crate) to: String,
 }
 
+#[derive(Serialize)]
+pub(crate) struct Artifact {
+    pub(crate) target: BTreeMap<String, Vec<ArtifactFile>>,
+}
+
+impl Artifact {
+    pub(crate) fn add_file(&mut self, builder: &mut Builder, target: &str, path: &str) {
+        if let Some(path) = record_shipped_file(builder, builder.input.join(path)) {
+            self.target.entry(target.into()).or_insert_with(Vec::new).push(ArtifactFile {
+                url: builder.url(&path),
+                hash_sha256: FileHash::Missing(path),
+            });
+        }
+    }
+
+    pub(crate) fn add_tarball(&mut self, builder: &mut Builder, target: &str, base_path: &str) {
+        let files = self.target.entry(target.into()).or_insert_with(Vec::new);
+        let base_path = builder.input.join(base_path);
+        for compression in &["gz", "xz"] {
+            if let Some(tarball) = tarball_variant(builder, &base_path, compression) {
+                files.push(ArtifactFile {
+                    url: builder.url(&tarball),
+                    hash_sha256: FileHash::Missing(tarball),
+                });
+            }
+        }
+    }
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) struct ArtifactFile {
+    pub(crate) url: String,
+    pub(crate) hash_sha256: FileHash,
+}
+
 #[derive(Serialize, Default)]
 pub(crate) struct Target {
     pub(crate) available: bool,
@@ -39,8 +84,8 @@ pub(crate) struct Target {
 impl Target {
     pub(crate) fn from_compressed_tar(builder: &mut Builder, base_path: &str) -> Self {
         let base_path = builder.input.join(base_path);
-        let gz = Self::tarball_variant(builder, &base_path, "gz");
-        let xz = Self::tarball_variant(builder, &base_path, "xz");
+        let gz = tarball_variant(builder, &base_path, "gz");
+        let xz = tarball_variant(builder, &base_path, "xz");
 
         if gz.is_none() {
             return Self::unavailable();
@@ -59,23 +104,6 @@ impl Target {
         }
     }
 
-    fn tarball_variant(builder: &mut Builder, base: &Path, ext: &str) -> Option<PathBuf> {
-        let mut path = base.to_path_buf();
-        path.set_extension(ext);
-        if path.is_file() {
-            builder.shipped_files.insert(
-                path.file_name()
-                    .expect("missing filename")
-                    .to_str()
-                    .expect("non-utf-8 filename")
-                    .to_string(),
-            );
-            Some(path)
-        } else {
-            None
-        }
-    }
-
     pub(crate) fn unavailable() -> Self {
         Self::default()
     }
@@ -111,6 +139,27 @@ impl Serialize for FileHash {
     }
 }
 
+fn tarball_variant(builder: &mut Builder, base: &Path, ext: &str) -> Option<PathBuf> {
+    let mut path = base.to_path_buf();
+    path.set_extension(ext);
+    record_shipped_file(builder, path)
+}
+
+fn record_shipped_file(builder: &mut Builder, path: PathBuf) -> Option<PathBuf> {
+    if path.is_file() {
+        builder.shipped_files.insert(
+            path.file_name()
+                .expect("missing filename")
+                .to_str()
+                .expect("non-utf-8 filename")
+                .to_string(),
+        );
+        Some(path)
+    } else {
+        None
+    }
+}
+
 pub(crate) fn visit_file_hashes(manifest: &mut Manifest, mut f: impl FnMut(&mut FileHash)) {
     for pkg in manifest.pkg.values_mut() {
         for target in pkg.target.values_mut() {
@@ -122,4 +171,12 @@ pub(crate) fn visit_file_hashes(manifest: &mut Manifest, mut f: impl FnMut(&mut
             }
         }
     }
+
+    for artifact in manifest.artifacts.values_mut() {
+        for target in artifact.target.values_mut() {
+            for file in target {
+                f(&mut file.hash_sha256);
+            }
+        }
+    }
 }

src/tools/build-manifest/src/versions.rs

@@ -13,6 +13,7 @@ const RUSTC_VERSION: &str = include_str!("../../../version");
 pub(crate) enum PkgType {
     Rust,
     RustSrc,
+    Rustc,
     Cargo,
     Rls,
     RustAnalyzer,
@@ -28,6 +29,7 @@ impl PkgType {
         match component {
             "rust" => PkgType::Rust,
             "rust-src" => PkgType::RustSrc,
+            "rustc" => PkgType::Rustc,
             "cargo" => PkgType::Cargo,
             "rls" | "rls-preview" => PkgType::Rls,
             "rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer,
@@ -44,6 +46,7 @@ impl PkgType {
         match self {
             PkgType::Rust => "rust",
             PkgType::RustSrc => "rust-src",
+            PkgType::Rustc => "rustc",
             PkgType::Cargo => "cargo",
             PkgType::Rls => "rls",
             PkgType::RustAnalyzer => "rust-analyzer",
@@ -69,6 +72,7 @@ impl PkgType {
 
             PkgType::Rust => true,
             PkgType::RustSrc => true,
+            PkgType::Rustc => true,
             PkgType::Other(_) => true,
         }
     }
@@ -165,10 +169,11 @@ impl Versions {
         }
     }
 
-    pub(crate) fn tarball_name(
+    pub(crate) fn archive_name(
         &mut self,
         package: &PkgType,
         target: &str,
+        extension: &str,
     ) -> Result<String, Error> {
         let component_name = package.tarball_component_name();
         let version = match self.channel.as_str() {
@@ -179,12 +184,20 @@ impl Versions {
         };
 
         if package.target_independent() {
-            Ok(format!("{}-{}.tar.gz", component_name, version))
+            Ok(format!("{}-{}.{}", component_name, version, extension))
         } else {
-            Ok(format!("{}-{}-{}.tar.gz", component_name, version, target))
+            Ok(format!("{}-{}-{}.{}", component_name, version, target, extension))
         }
     }
 
+    pub(crate) fn tarball_name(
+        &mut self,
+        package: &PkgType,
+        target: &str,
+    ) -> Result<String, Error> {
+        self.archive_name(package, target, "tar.gz")
+    }
+
     pub(crate) fn rustc_version(&self) -> &str {
         RUSTC_VERSION
     }