rustc crashes with segmentation fault in 1.73.0

[amircodota]

When I compile a project with rust 1.73.0 on linux, I get a segmentation fault error.
When I compile it with 1.72.1 all is well.

How to reproduce

on linux

git clone https://github.com/qdrant/qdrant.git
cd qdrant/
cargo +1.73.0 build --release --no-default-features --features parking_lot --bin qdrant 

If reproduced, you should get this error

/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/librustc_driver-453cf35e1dd187fa.so(+0x3121c13)[0x7fd7b4f21c13]
/lib/x86_64-linux-gnu/libc.so.6(+0x42520)[0x7fd7b1a42520]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.73.0-stable.so(_ZN4llvm12DemandedBits15performAnalysisEv+0x12fa)[0x7fd7ae19017a]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.73.0-stable.so(_ZN4llvm8BDCEPass3runERNS_8FunctionERNS_15AnalysisManagerIS1_JEEE+0x73c)[0x7fd7ae18e5fc]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.73.0-stable.so(+0x5f8dea1)[0x7fd7ae18dea1]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.73.0-stable.so(_ZN4llvm11PassManagerINS_8FunctionENS_15AnalysisManagerIS1_JEEEJEE3runERS1_RS3_+0xeab)[0x7fd7ae54bbbb]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.73.0-stable.so(_ZN4llvm27ModuleToFunctionPassAdaptor3runERNS_6ModuleERNS_15AnalysisManagerIS1_JEEE+0x68f)[0x7fd7ae54a665]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.73.0-stable.so(+0x6349fd1)[0x7fd7ae549fd1]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/libLLVM-17-rust-1.73.0-stable.so(_ZN4llvm11PassManagerINS_6ModuleENS_15AnalysisManagerIS1_JEEEJEE3runERS1_RS3_+0xb9)[0x7fd7ae236587]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/librustc_driver-453cf35e1dd187fa.so(+0x26d723c)[0x7fd7b44d723c]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/librustc_driver-453cf35e1dd187fa.so(+0x26cf1da)[0x7fd7b44cf1da]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/librustc_driver-453cf35e1dd187fa.so(+0x27c73f6)[0x7fd7b45c73f6]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/librustc_driver-453cf35e1dd187fa.so(_RNvXs1_Cs1HXOGTbQ6wr_18rustc_codegen_llvmNtB5_18LlvmCodegenBackendNtNtNtCs1yZWZnyBLnz_17rustc_codegen_ssa6traits5write19WriteBackendMethods12optimize_fat+0x30)[0x7fd7b4dc5fa0]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/librustc_driver-453cf35e1dd187fa.so(+0x27c5e7f)[0x7fd7b45c5e7f]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/librustc_driver-453cf35e1dd187fa.so(+0x26ca808)[0x7fd7b44ca808]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/librustc_driver-453cf35e1dd187fa.so(+0x2659256)[0x7fd7b4459256]
/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/lib/libstd-6498d8891e016dca.so(rust_metadata_std_3759e478f3a6c4f2+0x10b295)[0x7fd7b1d71295]
/lib/x86_64-linux-gnu/libc.so.6(+0x94ac3)[0x7fd7b1a94ac3]
/lib/x86_64-linux-gnu/libc.so.6(+0x126a40)[0x7fd7b1b26a40]
warning: `qdrant` (bin "qdrant") generated 4 warnings
error: could not compile `qdrant` (bin "qdrant"); 4 warnings emitted

Caused by:
  process didn't exit successfully: `/home/amir/.rustup/toolchains/1.73.0-x86_64-unknown-linux-gnu/bin/rustc --crate-name qdrant --edition=2021 src/main.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --diagnostic-width=204 --crate-type bin --emit=dep-info,link -C opt-level=3 -C lto=fat --cfg 'feature="parking_lot"' -C metadata=678c84f13770865d -C extra-filename=-678c84f13770865d --out-dir /home/amir/work/qdrant/target/release/deps -L dependency=/home/amir/work/qdrant/target/release/deps --extern actix_cors=/home/amir/work/qdrant/target/release/deps/libactix_cors-5d753288e4e6e1d9.rlib --extern actix_files=/home/amir/work/qdrant/target/release/deps/libactix_files-d206554be5000416.rlib --extern actix_multipart=/home/amir/work/qdrant/target/release/deps/libactix_multipart-e4119f55be86e64c.rlib --extern actix_web_validator=/home/amir/work/qdrant/target/release/deps/libactix_web_validator-f128b23484ab37de.rlib --extern anyhow=/home/amir/work/qdrant/target/release/deps/libanyhow-1376d6f5954dcf8f.rlib --extern api=/home/amir/work/qdrant/target/release/deps/libapi-74988dce99cf8d16.rlib --extern chrono=/home/amir/work/qdrant/target/release/deps/libchrono-7a0f10592547f130.rlib --extern clap=/home/amir/work/qdrant/target/release/deps/libclap-be247841c8445007.rlib --extern collection=/home/amir/work/qdrant/target/release/deps/libcollection-12b7be4776cdde4f.rlib --extern colored=/home/amir/work/qdrant/target/release/deps/libcolored-af2403cb46be9aa8.rlib --extern common=/home/amir/work/qdrant/target/release/deps/libcommon-7b887a942e9ae78b.rlib --extern config=/home/amir/work/qdrant/target/release/deps/libconfig-4c8ba52845fcc7ae.rlib --extern constant_time_eq=/home/amir/work/qdrant/target/release/deps/libconstant_time_eq-05a34683a9500646.rlib --extern futures=/home/amir/work/qdrant/target/release/deps/libfutures-7a65cb071a0e33a5.rlib --extern futures_util=/home/amir/work/qdrant/target/release/deps/libfutures_util-ed9527f9c672721f.rlib --extern itertools=/home/amir/work/qdrant/target/release/deps/libitertools-d8d087d1b77ebb89.rlib --extern log=/home/amir/work/qdrant/target/release/deps/liblog-e9330644bc5cecfd.rlib --extern memory=/home/amir/work/qdrant/target/release/deps/libmemory-c205ba57935be5a0.rlib --extern num_traits=/home/amir/work/qdrant/target/release/deps/libnum_traits-6e6b42730eba894d.rlib --extern num_cpus=/home/amir/work/qdrant/target/release/deps/libnum_cpus-9c5009be1fd05028.rlib --extern parking_lot=/home/amir/work/qdrant/target/release/deps/libparking_lot-c669684e36361cf8.rlib --extern prometheus=/home/amir/work/qdrant/target/release/deps/libprometheus-72772a37e76487a9.rlib --extern prost=/home/amir/work/qdrant/target/release/deps/libprost-183387812257cc8f.rlib --extern raft=/home/amir/work/qdrant/target/release/deps/libraft-89453d70fa8145be.rlib --extern raft_proto=/home/amir/work/qdrant/target/release/deps/libraft_proto-44bdfb8c4851b3da.rlib --extern reqwest=/home/amir/work/qdrant/target/release/deps/libreqwest-d83626a88d6bbd7c.rlib --extern rustls=/home/amir/work/qdrant/target/release/deps/librustls-dd3c7f94cdd3e204.rlib --extern rustls_pemfile=/home/amir/work/qdrant/target/release/deps/librustls_pemfile-fee18894ea33de01.rlib --extern schemars=/home/amir/work/qdrant/target/release/deps/libschemars-e151af419579664c.rlib --extern segment=/home/amir/work/qdrant/target/release/deps/libsegment-6ccbfec029f00e9d.rlib --extern serde=/home/amir/work/qdrant/target/release/deps/libserde-7250163179c417ba.rlib --extern serde_cbor=/home/amir/work/qdrant/target/release/deps/libserde_cbor-b1da93ac39a02281.rlib --extern serde_json=/home/amir/work/qdrant/target/release/deps/libserde_json-07e099448f80c968.rlib --extern slog=/home/amir/work/qdrant/target/release/deps/libslog-89efcb485691e0df.rlib --extern slog_stdlog=/home/amir/work/qdrant/target/release/deps/libslog_stdlog-40019af6d9aaba15.rlib --extern storage=/home/amir/work/qdrant/target/release/deps/libstorage-cd01e47fad08cf03.rlib --extern sys_info=/home/amir/work/qdrant/target/release/deps/libsys_info-504e7c1041746bf1.rlib --extern tar=/home/amir/work/qdrant/target/release/deps/libtar-93ebe77f2e73ae60.rlib --extern thiserror=/home/amir/work/qdrant/target/release/deps/libthiserror-05462fe689457c5f.rlib --extern tikv_jemallocator=/home/amir/work/qdrant/target/release/deps/libtikv_jemallocator-2f70f552465631e0.rlib --extern tokio=/home/amir/work/qdrant/target/release/deps/libtokio-f94a48abeb349ab1.rlib --extern tonic=/home/amir/work/qdrant/target/release/deps/libtonic-aab94c1a615270a6.rlib --extern tonic_reflection=/home/amir/work/qdrant/target/release/deps/libtonic_reflection-650ef766ecd687af.rlib --extern tower=/home/amir/work/qdrant/target/release/deps/libtower-a4b4ca25b067cb0f.rlib --extern tower_layer=/home/amir/work/qdrant/target/release/deps/libtower_layer-764c2b33180e9a63.rlib --extern tracing=/home/amir/work/qdrant/target/release/deps/libtracing-162e7d8fba50553e.rlib --extern tracing_log=/home/amir/work/qdrant/target/release/deps/libtracing_log-50e78ce7e71a2b46.rlib --extern tracing_subscriber=/home/amir/work/qdrant/target/release/deps/libtracing_subscriber-47bc2e85f74ba27b.rlib --extern uuid=/home/amir/work/qdrant/target/release/deps/libuuid-baf1bcf0d0cc6ebf.rlib --extern validator=/home/amir/work/qdrant/target/release/deps/libvalidator-b0cfc16999d19c5d.rlib --extern wal=/home/amir/work/qdrant/target/release/deps/libwal-f80d42582edca8ce.rlib -L native=/home/amir/work/qdrant/target/release/build/quantization-6c07ea8a878b1467/out -L native=/home/amir/work/qdrant/target/release/build/librocksdb-sys-88a70bdc670d21fe/out -L native=/home/amir/work/qdrant/target/release/build/librocksdb-sys-88a70bdc670d21fe/out -L native=/home/amir/work/qdrant/target/release/build/ring-3f1e21ee488f9992/out -L native=/home/amir/work/qdrant/target/release/build/sys-info-4646848c5d0dabf2/out -L native=/home/amir/work/qdrant/target/release/build/tikv-jemalloc-sys-bd4540182a1fbc4f/out/build/lib` (signal: 11, SIGSEGV: invalid memory reference)

If I do the same, but use 1.72.1, all is well.

git clone https://github.com/qdrant/qdrant.git
cd qdrant/
cargo +1.72.1 build --release --no-default-features --features parking_lot --bin qdrant 

[saethlin]

Backtrace from a rustc build, tip of master with LLVM assertions and Rust debug assertions. No assertion of any kind is hit, we still segfault.

#0  0x00007f2a681df796 in llvm::DemandedBits::performAnalysis() ()
   from /home/ben/rust-master/build/x86_64-unknown-linux-gnu/stage1/lib/libLLVM-17-rust-1.75.0-nightly.so
#1  0x00007f2a681dd913 in llvm::BDCEPass::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) ()
   from /home/ben/rust-master/build/x86_64-unknown-linux-gnu/stage1/lib/libLLVM-17-rust-1.75.0-nightly.so
#2  0x00007f2a681dd38d in llvm::detail::PassModel<llvm::Function, llvm::BDCEPass, llvm::PreservedAnalyses, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) ()
   from /home/ben/rust-master/build/x86_64-unknown-linux-gnu/stage1/lib/libLLVM-17-rust-1.75.0-nightly.so
#3  0x00007f2a68513a0c in llvm::PassManager<llvm::Function, llvm::AnalysisManager<llvm::Function>>::run(llvm::Function&, llvm::AnalysisManager<llvm::Function>&) ()
   from /home/ben/rust-master/build/x86_64-unknown-linux-gnu/stage1/lib/libLLVM-17-rust-1.75.0-nightly.so
#4  0x00007f2a6826cb0d in llvm::ModuleToFunctionPassAdaptor::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) ()
   from /home/ben/rust-master/build/x86_64-unknown-linux-gnu/stage1/lib/libLLVM-17-rust-1.75.0-nightly.so
#5  0x00007f2a6826c88d in llvm::detail::PassModel<llvm::Module, llvm::ModuleToFunctionPassAdaptor, llvm::PreservedAnalyses, llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) ()
   from /home/ben/rust-master/build/x86_64-unknown-linux-gnu/stage1/lib/libLLVM-17-rust-1.75.0-nightly.so
#6  0x00007f2a6826b398 in llvm::PassManager<llvm::Module, llvm::AnalysisManager<llvm::Module>>::run(llvm::Module&, llvm::AnalysisManager<llvm::Module>&) ()
   from /home/ben/rust-master/build/x86_64-unknown-linux-gnu/stage1/lib/libLLVM-17-rust-1.75.0-nightly.so
#7  0x00007f2a6b3bad8c in LLVMRustOptimize ()
   from /home/ben/rust-master/build/x86_64-unknown-linux-gnu/stage1/lib/librustc_driver-ded147b729037202.so
#8  0x00007f2a6b2e7e82 in rustc_codegen_llvm::back::write::llvm_optimize ()
    at compiler/rustc_codegen_llvm/src/back/write.rs:553
#9  0x00007f2a6b3269fa in rustc_codegen_llvm::back::lto::run_pass_manager () at compiler/rustc_codegen_llvm/src/back/lto.rs:641
#10 0x00007f2a6b32a710 in rustc_codegen_llvm::{impl#3}::optimize_fat () at compiler/rustc_codegen_llvm/src/lib.rs:231

[dianqk]

Backtrace from a rustc build, tip of master with LLVM assertions and Rust debug assertions. No assertion of any kind is hit, we still segfault.

sanitizers=true may make sense.

[saethlin]

Here's the config I'm using:

profile = "compiler"
change-id = 115898

[llvm]
assertions = true
download-ci-llvm = true

[rust]
incremental = false
debug = true

[build]
sanitizers = true

Am I missing something to enable sanitizers in LLVM?

[dianqk]

Am I missing something to enable sanitizers in LLVM?

Sorry, I'm actually not sure about the exact behavior of sanitizers here.
We may need customized cmake settings, see: LLVM_USE_SANITIZER in https://www.llvm.org/docs/CMake.html.

[boulanlo]

It's probably obvious, but I managed to bisect it to nightly-2023-08-09, which contains the commit that upgraded to LLVM 17 (#114048). I didn't bisect individual commits because even on a beefy computing platform node it takes around 10 minutes to reach the segfault, and it's getting late here. I can also confirm the segfault only happens with --release, the build completes fine without it.

Edit: managed to finish the bisection.

searched nightlies: from nightly-2023-07-07 to nightly-2023-10-13
regressed nightly: nightly-2023-08-09
searched commit range: 03a119b...f88a8b7
regressed commit: 443c316

bisected with cargo-bisect-rustc v0.6.7

Host triple: x86_64-unknown-linux-gnu
Reproduce with:

cargo bisect-rustc --preserve -- build --release --no-default-features -F parking_lot --bin qdrant 

[boulanlo]

Minimizing this is proving to be quite difficult, but I managed to pinpoint a change. In lib/segment/src/common/mod.rs, changing any field assignment in the new() function of the structure OperationDurationsAggregator (line 174) from the value to todo!() makes the segmentation fault disappear:

impl OperationDurationsAggregator {
    pub fn new() -> Arc<Mutex<Self>> {
        Arc::new(Mutex::new(Self {
            ok_count: 0,
            fail_count: 0,
            timings: todo!(), //[0.; AVG_DATASET_LEN],
            timing_index: 0,
            timing_loops: 0,
            min_value: None,
            max_value: None,
            last_response_date: Some(Utc::now().round_subsecs(2)),
        }))
    }
}

Interestingly, while trying to go from main() and removing code to minimize the problem, the segmentation backtrace changed to this:

/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/librustc_driver-453cf35e1dd187fa.so(+0x3121c13)[0x7f88ee521c13]
/nix/store/yaz7pyf0ah88g2v505l38n0f3wg2vzdj-glibc-2.37-8/lib/libc.so.6(+0x38d60)[0x7f88eb068d60]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(_ZN4llvm11ConstantInt3getEPNS_4TypeEmb+0x6e)[0x7f88e7413a6e]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(_ZN4llvm31ConstantFoldCompareInstOperandsEjPNS_8ConstantES1_RKNS_10DataLayoutEPKNS_17TargetLibraryInfoEPKNS_11InstructionE+0x73)[0x7f88e7413383]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(+0x609135b)[0x7f88e749135b]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(+0x66068c5)[0x7f88e7a068c5]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(+0x5fd35b2)[0x7f88e73d35b2]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(+0x635b7a3)[0x7f88e775b7a3]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(+0x5e0fa65)[0x7f88e720fa65]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(_ZN4llvm11PassManagerINS_8FunctionENS_15AnalysisManagerIS1_JEEEJEE3runERS1_RS3_+0xeab)[0x7f88e774bbbb]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(_ZN4llvm27ModuleToFunctionPassAdaptor3runERNS_6ModuleERNS_15AnalysisManagerIS1_JEEE+0x68f)[0x7f88e774a665]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(+0x6349fd1)[0x7f88e7749fd1]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libLLVM-17-rust-1.73.0-stable.so(_ZN4llvm11PassManagerINS_6ModuleENS_15AnalysisManagerIS1_JEEEJEE3runERS1_RS3_+0xb9)[0x7f88e7436587]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/librustc_driver-453cf35e1dd187fa.so(+0x26d723c)[0x7f88edad723c]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/librustc_driver-453cf35e1dd187fa.so(+0x26cf1da)[0x7f88edacf1da]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/librustc_driver-453cf35e1dd187fa.so(+0x27c73f6)[0x7f88edbc73f6]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/librustc_driver-453cf35e1dd187fa.so(_RNvXs1_Cs1HXOGTbQ6wr_18rustc_codegen_llvmNtB5_18LlvmCodegenBackendNtNtNtCs1yZWZnyBLnz_17rustc_codegen_ssa6traits5write19WriteBackendMethods12optimize_fat+0x30)[0x7f88ee3c5fa0]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/librustc_driver-453cf35e1dd187fa.so(+0x27c5e7f)[0x7f88edbc5e7f]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/librustc_driver-453cf35e1dd187fa.so(+0x26ca808)[0x7f88edaca808]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/librustc_driver-453cf35e1dd187fa.so(+0x2659256)[0x7f88eda59256]
/nix/store/pqr2jmbbfgfd4gi2lz6zcndabr7igiy7-rust-default-1.73.0/lib/libstd-6498d8891e016dca.so(rust_metadata_std_3759e478f3a6c4f2+0x10b295)[0x7f88eb321295]
/nix/store/yaz7pyf0ah88g2v505l38n0f3wg2vzdj-glibc-2.37-8/lib/libc.so.6(+0x85e24)[0x7f88eb0b5e24]
/nix/store/yaz7pyf0ah88g2v505l38n0f3wg2vzdj-glibc-2.37-8/lib/libc.so.6(+0x1079b0)[0x7f88eb1379b0]

I'm not nearly enough of an LLVM expert to understand if this is relevant, but I thought I'd include it in the event it is.