Auto merge of #14367 - epage:vendor, r=weihanglo

fix(vendor): Strip excluded build targets

### What does this PR try to resolve?

Fixes #14348

### How should we test and review this PR?

The commits are setup to be able to show, through the tests, how the `cargo package` and `cargo vendor` behavior diverge and then how `cargo vendor`s behavior changes over time.

This is a **very** hacky solution, duplicating the minimum of what
`prepare_for_publish` does to fix this one issue and in the least
intrusive way to the vendor code.

The intention is to keep this low risk for backporting to beta and
stable.
We need to revisit this, refactoring the `cargo package` code so that we
can call into that for each vendored dependency.

### Additional information

Author: bors

src/cargo/ops/vendor.rs

@@ -221,7 +221,7 @@ fn sync(
         let pathsource = PathSource::new(src, id.source_id(), gctx);
         let paths = pathsource.list_files(pkg)?;
         let mut map = BTreeMap::new();
-        cp_sources(pkg, src, &paths, &dst, &mut map, &mut tmp_buf)
+        cp_sources(pkg, src, &paths, &dst, &mut map, &mut tmp_buf, gctx)
             .with_context(|| format!("failed to copy over vendored sources for: {}", id))?;
 
         // Finally, emit the metadata about this package
@@ -317,6 +317,7 @@ fn cp_sources(
     dst: &Path,
     cksums: &mut BTreeMap<String, String>,
     tmp_buf: &mut [u8],
+    gctx: &GlobalContext,
 ) -> CargoResult<()> {
     for p in paths {
         let relative = p.strip_prefix(&src).unwrap();
@@ -360,7 +361,12 @@ fn cp_sources(
         let cksum = if dst.file_name() == Some(OsStr::new("Cargo.toml"))
             && pkg.package_id().source_id().is_git()
         {
-            let contents = pkg.manifest().to_normalized_contents()?;
+            let packaged_files = paths
+                .iter()
+                .map(|p| p.strip_prefix(src).unwrap().to_owned())
+                .collect::<Vec<_>>();
+            let vendored_pkg = prepare_for_vendor(pkg, &packaged_files, gctx)?;
+            let contents = vendored_pkg.manifest().to_normalized_contents()?;
             copy_and_checksum(
                 &dst,
                 &mut dst_opts,
@@ -392,6 +398,119 @@ fn cp_sources(
     Ok(())
 }
 
+/// HACK: Perform the bare minimum of `prepare_for_publish` needed for #14348.
+///
+/// There are parts of `prepare_for_publish` that could be directly useful (e.g. stripping
+/// `[workspace]`) while other parts that require other filesystem operations (moving the README
+/// file) and ideally we'd reuse `cargo package` code to take care of all of this for us.
+fn prepare_for_vendor(
+    me: &Package,
+    packaged_files: &[PathBuf],
+    gctx: &GlobalContext,
+) -> CargoResult<Package> {
+    let contents = me.manifest().contents();
+    let document = me.manifest().document();
+    let original_toml = prepare_toml_for_vendor(
+        me.manifest().normalized_toml().clone(),
+        packaged_files,
+        gctx,
+    )?;
+    let normalized_toml = original_toml.clone();
+    let features = me.manifest().unstable_features().clone();
+    let workspace_config = me.manifest().workspace_config().clone();
+    let source_id = me.package_id().source_id();
+    let mut warnings = Default::default();
+    let mut errors = Default::default();
+    let manifest = crate::util::toml::to_real_manifest(
+        contents.to_owned(),
+        document.clone(),
+        original_toml,
+        normalized_toml,
+        features,
+        workspace_config,
+        source_id,
+        me.manifest_path(),
+        gctx,
+        &mut warnings,
+        &mut errors,
+    )?;
+    let new_pkg = Package::new(manifest, me.manifest_path());
+    Ok(new_pkg)
+}
+
+fn prepare_toml_for_vendor(
+    mut me: cargo_util_schemas::manifest::TomlManifest,
+    packaged_files: &[PathBuf],
+    gctx: &GlobalContext,
+) -> CargoResult<cargo_util_schemas::manifest::TomlManifest> {
+    let package = me
+        .package
+        .as_mut()
+        .expect("venedored manifests must have packages");
+    if let Some(cargo_util_schemas::manifest::StringOrBool::String(path)) = &package.build {
+        let path = paths::normalize_path(Path::new(path));
+        let included = packaged_files.contains(&path);
+        let build = if included {
+            let path = path
+                .into_os_string()
+                .into_string()
+                .map_err(|_err| anyhow::format_err!("non-UTF8 `package.build`"))?;
+            let path = crate::util::toml::normalize_path_string_sep(path);
+            cargo_util_schemas::manifest::StringOrBool::String(path)
+        } else {
+            gctx.shell().warn(format!(
+                "ignoring `package.build` as `{}` is not included in the published package",
+                path.display()
+            ))?;
+            cargo_util_schemas::manifest::StringOrBool::Bool(false)
+        };
+        package.build = Some(build);
+    }
+
+    let lib = if let Some(target) = &me.lib {
+        crate::util::toml::prepare_target_for_publish(
+            target,
+            Some(packaged_files),
+            "library",
+            gctx,
+        )?
+    } else {
+        None
+    };
+    let bin = crate::util::toml::prepare_targets_for_publish(
+        me.bin.as_ref(),
+        Some(packaged_files),
+        "binary",
+        gctx,
+    )?;
+    let example = crate::util::toml::prepare_targets_for_publish(
+        me.example.as_ref(),
+        Some(packaged_files),
+        "example",
+        gctx,
+    )?;
+    let test = crate::util::toml::prepare_targets_for_publish(
+        me.test.as_ref(),
+        Some(packaged_files),
+        "test",
+        gctx,
+    )?;
+    let bench = crate::util::toml::prepare_targets_for_publish(
+        me.bench.as_ref(),
+        Some(packaged_files),
+        "benchmark",
+        gctx,
+    )?;
+
+    me.lib = lib;
+    me.bin = bin;
+    me.example = example;
+    me.test = test;
+    me.bench = bench;
+
+    Ok(me)
+}
+
 fn copy_and_checksum<T: Read>(
     dst_path: &Path,
     dst_opts: &mut OpenOptions,

src/cargo/util/toml/mod.rs

@@ -1086,7 +1086,7 @@ fn deprecated_ws_default_features(
 }
 
 #[tracing::instrument(skip_all)]
-fn to_real_manifest(
+pub fn to_real_manifest(
     contents: String,
     document: toml_edit::ImDocument<String>,
     original_toml: manifest::TomlManifest,
@@ -2889,7 +2889,7 @@ fn prepare_toml_for_publish(
     }
 }
 
-fn prepare_targets_for_publish(
+pub fn prepare_targets_for_publish(
     targets: Option<&Vec<manifest::TomlTarget>>,
     packaged_files: Option<&[PathBuf]>,
     context: &str,
@@ -2915,7 +2915,7 @@ fn prepare_targets_for_publish(
     }
 }
 
-fn prepare_target_for_publish(
+pub fn prepare_target_for_publish(
     target: &manifest::TomlTarget,
     packaged_files: Option<&[PathBuf]>,
     context: &str,
@@ -2950,7 +2950,7 @@ fn normalize_path_sep(path: PathBuf, context: &str) -> CargoResult<PathBuf> {
     Ok(path.into())
 }
 
-fn normalize_path_string_sep(path: String) -> String {
+pub fn normalize_path_string_sep(path: String) -> String {
     if std::path::MAIN_SEPARATOR != '/' {
         path.replace(std::path::MAIN_SEPARATOR, "/")
     } else {