Verify that all crate sources are in sync

Cargo.lock

@@ -3367,7 +3367,6 @@ dependencies = [
  "rustc_type_ir",
  "serde_json",
  "smallvec",
- "snap",
  "tempfile",
  "thorin-dwp",
  "tracing",

compiler/rustc_codegen_ssa/Cargo.toml

@@ -17,7 +17,6 @@ tempfile = "3.2"
 thorin-dwp = "0.6"
 pathdiff = "0.2.0"
 serde_json = "1.0.59"
-snap = "1"
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 regex = "1.4"
 

compiler/rustc_codegen_ssa/src/back/metadata.rs

@@ -10,8 +10,6 @@ use object::{
     ObjectSymbol, SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope,
 };
 
-use snap::write::FrameEncoder;
-
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::owned_slice::{try_slice_owned, OwnedSlice};
 use rustc_metadata::fs::METADATA_FILENAME;
@@ -481,19 +479,15 @@ pub fn create_compressed_metadata_file(
     metadata: &EncodedMetadata,
     symbol_name: &str,
 ) -> Vec<u8> {
-    let mut compressed = rustc_metadata::METADATA_HEADER.to_vec();
-    // Our length will be backfilled once we're done writing
-    compressed.write_all(&[0; 4]).unwrap();
-    FrameEncoder::new(&mut compressed).write_all(metadata.raw_data()).unwrap();
-    let meta_len = rustc_metadata::METADATA_HEADER.len();
-    let data_len = (compressed.len() - meta_len - 4) as u32;
-    compressed[meta_len..meta_len + 4].copy_from_slice(&data_len.to_be_bytes());
+    let mut packed_metadata = rustc_metadata::METADATA_HEADER.to_vec();
+    packed_metadata.write_all(&(metadata.raw_data().len() as u32).to_be_bytes()).unwrap();
+    packed_metadata.extend(metadata.raw_data());
 
     let Some(mut file) = create_object_file(sess) else {
-        return compressed.to_vec();
+        return packed_metadata.to_vec();
     };
     if file.format() == BinaryFormat::Xcoff {
-        return create_compressed_metadata_file_for_xcoff(file, &compressed, symbol_name);
+        return create_compressed_metadata_file_for_xcoff(file, &packed_metadata, symbol_name);
     }
     let section = file.add_section(
         file.segment_name(StandardSegment::Data).to_vec(),
@@ -507,14 +501,14 @@ pub fn create_compressed_metadata_file(
         }
         _ => {}
     };
-    let offset = file.append_section_data(section, &compressed, 1);
+    let offset = file.append_section_data(section, &packed_metadata, 1);
 
     // For MachO and probably PE this is necessary to prevent the linker from throwing away the
     // .rustc section. For ELF this isn't necessary, but it also doesn't harm.
     file.add_symbol(Symbol {
         name: symbol_name.as_bytes().to_vec(),
         value: offset,
-        size: compressed.len() as u64,
+        size: packed_metadata.len() as u64,
         kind: SymbolKind::Data,
         scope: SymbolScope::Dynamic,
         weak: false,

compiler/rustc_metadata/src/locator.rs

@@ -511,7 +511,7 @@ impl<'a> CrateLocator<'a> {
             rlib: self.extract_one(rlibs, CrateFlavor::Rlib, &mut slot)?,
             dylib: self.extract_one(dylibs, CrateFlavor::Dylib, &mut slot)?,
         };
-        Ok(slot.map(|(svh, metadata)| (svh, Library { source, metadata })))
+        Ok(slot.map(|(svh, metadata, _)| (svh, Library { source, metadata })))
     }
 
     fn needs_crate_flavor(&self, flavor: CrateFlavor) -> bool {
@@ -535,11 +535,13 @@ impl<'a> CrateLocator<'a> {
     // read the metadata from it if `*slot` is `None`. If the metadata couldn't
     // be read, it is assumed that the file isn't a valid rust library (no
     // errors are emitted).
+    //
+    // The `PathBuf` in `slot` will only be used for diagnostic purposes.
     fn extract_one(
         &mut self,
         m: FxHashMap<PathBuf, PathKind>,
         flavor: CrateFlavor,
-        slot: &mut Option<(Svh, MetadataBlob)>,
+        slot: &mut Option<(Svh, MetadataBlob, PathBuf)>,
     ) -> Result<Option<(PathBuf, PathKind)>, CrateError> {
         // If we are producing an rlib, and we've already loaded metadata, then
         // we should not attempt to discover further crate sources (unless we're
@@ -550,16 +552,9 @@ impl<'a> CrateLocator<'a> {
         //
         // See also #68149 which provides more detail on why emitting the
         // dependency on the rlib is a bad thing.
-        //
-        // We currently do not verify that these other sources are even in sync,
-        // and this is arguably a bug (see #10786), but because reading metadata
-        // is quite slow (especially from dylibs) we currently do not read it
-        // from the other crate sources.
         if slot.is_some() {
             if m.is_empty() || !self.needs_crate_flavor(flavor) {
                 return Ok(None);
-            } else if m.len() == 1 {
-                return Ok(Some(m.into_iter().next().unwrap()));
             }
         }
 
@@ -610,8 +605,7 @@ impl<'a> CrateLocator<'a> {
                         candidates,
                     ));
                 }
-                err_data = Some(vec![ret.as_ref().unwrap().0.clone()]);
-                *slot = None;
+                err_data = Some(vec![slot.take().unwrap().2]);
             }
             if let Some(candidates) = &mut err_data {
                 candidates.push(lib);
@@ -644,7 +638,7 @@ impl<'a> CrateLocator<'a> {
                     continue;
                 }
             }
-            *slot = Some((hash, metadata));
+            *slot = Some((hash, metadata, lib.clone()));
             ret = Some((lib, kind));
         }
 
@@ -814,19 +808,26 @@ fn get_metadata_section<'p>(
             let compressed_len = u32::from_be_bytes(len_bytes) as usize;
 
             // Header is okay -> inflate the actual metadata
-            let compressed_bytes = &buf[data_start..(data_start + compressed_len)];
-            debug!("inflating {} bytes of compressed metadata", compressed_bytes.len());
-            // Assume the decompressed data will be at least the size of the compressed data, so we
-            // don't have to grow the buffer as much.
-            let mut inflated = Vec::with_capacity(compressed_bytes.len());
-            FrameDecoder::new(compressed_bytes).read_to_end(&mut inflated).map_err(|_| {
-                MetadataError::LoadFailure(format!(
-                    "failed to decompress metadata: {}",
-                    filename.display()
-                ))
-            })?;
+            let compressed_bytes = buf.slice(|buf| &buf[data_start..(data_start + compressed_len)]);
+            if &compressed_bytes[..cmp::min(METADATA_HEADER.len(), compressed_bytes.len())]
+                == METADATA_HEADER
+            {
+                // The metadata was not actually compressed.
+                compressed_bytes
+            } else {
+                debug!("inflating {} bytes of compressed metadata", compressed_bytes.len());
+                // Assume the decompressed data will be at least the size of the compressed data, so we
+                // don't have to grow the buffer as much.
+                let mut inflated = Vec::with_capacity(compressed_bytes.len());
+                FrameDecoder::new(&*compressed_bytes).read_to_end(&mut inflated).map_err(|_| {
+                    MetadataError::LoadFailure(format!(
+                        "failed to decompress metadata: {}",
+                        filename.display()
+                    ))
+                })?;
 
-            slice_owned(inflated, Deref::deref)
+                slice_owned(inflated, Deref::deref)
+            }
         }
         CrateFlavor::Rmeta => {
             // mmap the file, because only a small fraction of it is read.

tests/run-make/extern-flag-pathless/Makefile

@@ -3,17 +3,32 @@ include ../tools.mk
 
 # Test mixing pathless --extern with paths.
 
+# Test for static linking by checking that the binary runs if the dylib
+# is removed and test for dynamic linking by checking that the binary
+# fails to run if the dylib is removed.
+
 all:
-	$(RUSTC) bar-static.rs --crate-name=bar --crate-type=rlib
-	$(RUSTC) bar-dynamic.rs --crate-name=bar --crate-type=dylib -C prefer-dynamic
+	$(RUSTC) bar.rs --crate-type=rlib --crate-type=dylib -Cprefer-dynamic
+
 	# rlib preferred over dylib
 	$(RUSTC) foo.rs --extern bar
-	$(call RUN,foo) | $(CGREP) 'static'
+	mv $(call DYLIB,bar) $(TMPDIR)/bar.tmp
+	$(call RUN,foo)
+	mv $(TMPDIR)/bar.tmp $(call DYLIB,bar)
+
 	$(RUSTC) foo.rs --extern bar=$(TMPDIR)/libbar.rlib --extern bar
-	$(call RUN,foo) | $(CGREP) 'static'
+	mv $(call DYLIB,bar) $(TMPDIR)/bar.tmp
+	$(call RUN,foo)
+	mv $(TMPDIR)/bar.tmp $(call DYLIB,bar)
+
 	# explicit --extern overrides pathless
 	$(RUSTC) foo.rs --extern bar=$(call DYLIB,bar) --extern bar
-	$(call RUN,foo) | $(CGREP) 'dynamic'
+	mv $(call DYLIB,bar) $(TMPDIR)/bar.tmp
+	$(call FAIL,foo)
+	mv $(TMPDIR)/bar.tmp $(call DYLIB,bar)
+
 	# prefer-dynamic does what it says
 	$(RUSTC) foo.rs --extern bar -C prefer-dynamic
-	$(call RUN,foo) | $(CGREP) 'dynamic'
+	mv $(call DYLIB,bar) $(TMPDIR)/bar.tmp
+	$(call FAIL,foo)
+	mv $(TMPDIR)/bar.tmp $(call DYLIB,bar)

tests/run-make/extern-flag-pathless/bar.rs

@@ -0,0 +1 @@
+pub fn f() {}

tests/run-make/mixing-libs/Makefile

@@ -2,9 +2,7 @@
 include ../tools.mk
 
 all:
-	$(RUSTC) rlib.rs
-	$(RUSTC) dylib.rs
-	$(RUSTC) rlib.rs --crate-type=dylib
-	$(RUSTC) dylib.rs
-	$(call REMOVE_DYLIBS,rlib)
+	$(RUSTC) rlib.rs --crate-type=rlib --crate-type=dylib
+	$(RUSTC) dylib.rs # no -Cprefer-dynamic so statically linking librlib.rlib
+	$(call REMOVE_DYLIBS,rlib) # remove librlib.so to test that prog.rs doesn't get confused about the removed dylib version of librlib
 	$(RUSTC) prog.rs && exit 1 || exit 0

tests/run-make/no-cdylib-as-rdylib/Makefile

@@ -0,0 +1,16 @@
+# ignore-cross-compile
+include ../tools.mk
+
+# Test that rustc will not attempt to link against a cdylib as if
+# it is a rust dylib when an rlib for the same crate is available.
+# Previously rustc didn't actually check if any further formats of
+# a crate which has been loaded are of the same version and if
+# they are actually valid. This caused a cdylib to be interpreted
+# as rust dylib as soon as the corresponding rlib was loaded. As
+# cdylibs don't export any rust symbols, linking would fail if
+# rustc decides to link against the cdylib rather than the rlib.
+
+all:
+	$(RUSTC) bar.rs --crate-type=rlib --crate-type=cdylib
+	$(RUSTC) foo.rs -C prefer-dynamic
+	$(call RUN,foo)

tests/run-make/no-cdylib-as-rdylib/bar.rs

@@ -0,0 +1 @@
+pub fn bar() {}

tests/run-make/no-cdylib-as-rdylib/foo.rs

@@ -0,0 +1,5 @@
+extern crate bar;
+
+fn main() {
+    bar::bar();
+}

tests/run-make/rmeta-preferred/Makefile

@@ -0,0 +1,16 @@
+# ignore-cross-compile
+include ../tools.mk
+
+# Test that using rlibs and rmeta dep crates work together. Specifically, that
+# there can be both an rmeta and an rlib file and rustc will prefer the rmeta
+# file.
+#
+# This behavior is simply making sure this doesn't accidentally change; in this
+# case we want to make sure that the rlib isn't being used as that would cause
+# bugs in -Zbinary-dep-depinfo (see #68298).
+
+all:
+	$(RUSTC) rmeta_aux.rs --crate-type=rlib --emit link,metadata
+	$(RUSTC) lib.rs --crate-type=rlib --emit dep-info -Zbinary-dep-depinfo
+	$(CGREP) "librmeta_aux.rmeta" < $(TMPDIR)/lib.d
+	$(CGREP) -v "librmeta_aux.rlib" < $(TMPDIR)/lib.d

tests/ui/rmeta/rmeta-rpass.rs → tests/run-make/rmeta-preferred/lib.rs

@@ -1,4 +1,3 @@
-// run-pass
 // Test that using rlibs and rmeta dep crates work together. Specifically, that
 // there can be both an rmeta and an rlib file and rustc will prefer the rmeta
 // file.
@@ -7,12 +6,9 @@
 // case we want to make sure that the rlib isn't being used as that would cause
 // bugs in -Zbinary-dep-depinfo (see #68298).
 
-// aux-build:rmeta-rmeta.rs
-// aux-build:rmeta-rlib-rpass.rs
-
 extern crate rmeta_aux;
 use rmeta_aux::Foo;
 
-pub fn main() {
-    let _ = Foo { field2: 42 };
+pub fn foo() {
+    let _ = Foo { field: 42 };
 }

tests/run-make/rmeta-preferred/rmeta_aux.rs

@@ -0,0 +1,3 @@
+pub struct Foo {
+    pub field: i32,
+}