rustc distributed for riscv64 linux segfaults on almost anything

[LaoLittle]

Meta

rustc --version --verbose:

rustc 1.73.0 (cc66ad468 2023-10-03)
binary: rustc
commit-hash: cc66ad468955717ab92600c770da8c1601a4ff33
commit-date: 2023-10-03
host: riscv64gc-unknown-linux-gnu
release: 1.73.0
LLVM version: 17.0.2

cat /proc/version:

Linux version 6.5.0-9-generic (buildd@riscv64-qemu-lgw01-068) (riscv64-linux-gnu-gcc-13 (Ubuntu 13.2.0-4ubuntu3) 13.2.0, GNU ld (GNU Binutils for Ubuntu) 2.41) #9.1-Ubuntu SMP Sat Oct  7 17:18:31 UTC 2023

Backtrace

/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0xbb93a6)[0x3f83fb93a6]
linux-vdso.so.1(__vdso_rt_sigreturn+0x0)[0x3f8b848800]
/lib/riscv64-linux-gnu/libc.so.6(read+0x44)[0x3f831e8ece]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/libstd-7c20a79be40626d5.so(_ZN51_$LT$$RF$std..fs..File$u20$as$u20$std..io..Read$GT$4read17h6080e84dd1fe0357E+0x28)[0x3f833295d4]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52eaf36)[0x3f886eaf36]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52eccc0)[0x3f886eccc0]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52ed3b6)[0x3f886ed3b6]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52ed416)[0x3f886ed416]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52edce8)[0x3f886edce8]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52eeac0)[0x3f886eeac0]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/libstd-7c20a79be40626d5.so(rust_metadata_std_bf3dc212422480e+0xaf7e2)[0x3f833457e2]
/lib/riscv64-linux-gnu/libc.so.6(+0x6a956)[0x3f8319e956]
/lib/riscv64-linux-gnu/libc.so.6(+0xbbbf0)[0x3f831efbf0]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0xbb93a6)[0x3faa1b93a6]
linux-vdso.so.1(__vdso_rt_sigreturn+0x0)[0x3fb1aca800]
/lib/riscv64-linux-gnu/libc.so.6(read+0x44)[0x3fa93e8ece]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/libstd-7c20a79be40626d5.so(_ZN51_$LT$$RF$std..fs..File$u20$as$u20$std..io..Read$GT$4read17h6080e84dd1fe0357E+0x28)[0x3fa95295d4]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52eaf36)[0x3fae8eaf36]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52eccc0)[0x3fae8eccc0]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52ed3b6)[0x3fae8ed3b6]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52ed416)[0x3fae8ed416]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52edce8)[0x3fae8edce8]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52eeac0)[0x3fae8eeac0]
/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/libstd-7c20a79be40626d5.so(rust_metadata_std_bf3dc212422480e+0xaf7e2)[0x3fa95457e2]
/lib/riscv64-linux-gnu/libc.so.6(+0x6a956)[0x3fa939e956]
/lib/riscv64-linux-gnu/libc.so.6(+0xbbbf0)[0x3fa93efbf0]
error: could not compile `libc` (lib)
                                                                                Caused by:                  
  process didn't exit successfully: `/home/laolittle/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/rustc --crate-name libc /home/laolittle/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libc-0.2.149/src/lib.rs --error-format=json --json=diagnostic-rendered-ansi,artifacts,future-incompat --diagnostic-width=80 --crate-type lib --emit=dep-info,metadata,link -C embed-bitcode=no -C debug-assertions=off --cfg 'feature="default"' --cfg 'feature="extra_traits"' --cfg 'feature="std"' -C metadata=7cb0156b76950299 -C extra-filename=-7cb0156b76950299 --out-dir /tmp/cargo-installoqXMnf/release/deps -C strip=symbols -L dependency=/tmp/cargo-installoqXMnf/release/deps --cap-lints allow --cfg freebsd11 --cfg libc_priv_mod_use --cfg libc_union --cfg libc_const_size_of --cfg libc_align --cfg libc_int128 --cfg libc_core_cvoid --cfg libc_packedN --cfg libc_cfg_target_vendor --cfg libc_non_exhaustive --cfg libc_long_array --cfg libc_ptr_addr_of --cfg libc_underscore_const_names --cfg libc_const_extern_fn` (signal: 11, SIGSEGV: invalid memory reference)
error: failed to compile `bat v0.24.0`, intermediate artifacts can be found at `/tmp/cargo-installoqXMnf`.
To reuse those artifacts with a future compilation, set the environment variable `CARGO_TARGET_DIR` to that path.

[saethlin]

Can you compile a hello world program? Anything at all?

The segfault here is from the Read impl on File, which makes me think something is deeply broken.

[LaoLittle]

Can you compile a hello world program? Anything at all?

The segfault here is from the Read impl on File, which makes me think something is deeply broken.

hello world is fine

[raw-bin]

Hi.

I think this may be related. Observed when running cargo and rustc on qemu's virt machine emulation with an ubuntu root filesystem:

robin@ubuntu:~/Work/workspace/rust-testing/misc$ cargo new hello
     Created binary (application) `hello` package

robin@ubuntu:~/Work/workspace/rust-testing/misc$ cd hello

robin@ubuntu:~/Work/workspace/rust-testing/misc/hello$ cargo run
   Compiling hello v0.1.0 (/home/robin/Work/workspace/rust-testing/misc/hello)
/lib/riscv64-linux-gnu/librustc_driver-192663414457c4de.so(+0x7d240c)[0x7fffa09d240c]
linux-vdso.so.1(__vdso_rt_sigreturn+0x0)[0x7fffa3d13800]
/lib/riscv64-linux-gnu/libc.so.6(syscall+0x16)[0x7fff9ffe1406]
/lib/riscv64-linux-gnu/libstd-e74a09c2d8dfb358.so(_ZN3std3sys4unix5locks11futex_mutex5Mutex14lock_contended17h82bb17465d9a013fE+0x96)[0x7fffa00f6c7e]
/lib/riscv64-linux-gnu/librustc_driver-192663414457c4de.so(+0x2a92d42)[0x7fffa2c92d42]
/lib/riscv64-linux-gnu/librustc_driver-192663414457c4de.so(+0x2a93234)[0x7fffa2c93234]
/lib/riscv64-linux-gnu/librustc_driver-192663414457c4de.so(+0x2a95160)[0x7fffa2c95160]
/lib/riscv64-linux-gnu/librustc_driver-192663414457c4de.so(+0x2a95208)[0x7fffa2c95208]
/lib/riscv64-linux-gnu/librustc_driver-192663414457c4de.so(+0x2a943ae)[0x7fffa2c943ae]
/lib/riscv64-linux-gnu/libstd-e74a09c2d8dfb358.so(rust_metadata_std_4809e9723d3e348c+0x7a798)[0x7fffa0103798]
/lib/riscv64-linux-gnu/libc.so.6(+0x6a956)[0x7fff9ff91956]
/lib/riscv64-linux-gnu/libc.so.6(+0xbbbf0)[0x7fff9ffe2bf0]
    Finished dev [unoptimized + debuginfo] target(s) in 7.10s
     Running `target/debug/hello`
Hello, world!

robin@ubuntu:~/Work/workspace/rust-testing/misc/hello$ uname -a
Linux ubuntu 6.5.0-10-generic #10.1-Ubuntu SMP Sat Oct 28 04:47:16 UTC 2023 riscv64 riscv64 riscv64 GNU/Linux

robin@ubuntu:~/Work/workspace/rust-testing/misc/hello$ rustc --version
rustc 1.71.1 (eb26296b5 2023-08-03) (built from a source tarball)

robin@ubuntu:~/Work/workspace/rust-testing/misc/hello$ cargo --version
cargo 1.71.1

Note that:

1. This is an ubuntu 23.10 preinstalled RISCV64 server image for qemu's virt machine platform. The image was obtained from here.

2. The qemu invocation was:

qemu-system-riscv64 -machine virt -nographic \
  -m 8G -smp 8 \
  -bios /usr/lib/riscv64-linux-gnu/opensbi/generic/fw_jump.bin \
  -kernel /usr/lib/u-boot/qemu-riscv64_smode/uboot.elf \
  -device virtio-rng-pci \
  -drive file=ubuntu-23.10-preinstalled-server-riscv64.img,format=raw,if=virtio \
  -netdev user,id=net0,hostfwd=tcp::2222-:22 -device virtio-net-device,netdev=net0

3. The host is an x86_64 box

The backtrace seems to be triggered by a read of some VDSO symbol by cargo~rustc et al which isn't nice. However, the hello binary is built and runs correctly.

Happy to provide any more info.

Thanks

[raw-bin]

Another data point. When I try to build the rust compiler natively I get a similar backtrace for rustc but this time it appears to trigger a fatal SIGSEGV:

robin@ubuntu:~/Work/workspace/rust-testing/rust$ ./x.py -h
INFO: Downloading and building bootstrap before processing --help command.
      See src/bootstrap/README.md for help with common commands.
Building bootstrap
   Compiling proc-macro2 v1.0.60
   Compiling memchr v2.5.0
   Compiling quote v1.0.26
error: rustc interrupted by SIGSEGV, printing backtrace

/home/robin/Work/workspace/rust-testing/rust/build/riscv64gc-unknown-linux-gnu/stage0/bin/../lib/librustc_driver-fdc49e368e40bdcf.so(+0xbcbbee)[0x7fff8d1cbbee]
linux-vdso.so.1(__vdso_rt_sigreturn+0x0)[0x7fff94ba3800]
/lib/riscv64-linux-gnu/libc.so.6(read+0x44)[0x7fff8c552ece]
/home/robin/Work/workspace/rust-testing/rust/build/riscv64gc-unknown-linux-gnu/stage0/bin/../lib/libstd-aa93b6b6286a632f.so(_ZN3std3sys4unix2fs4File4read17h39fb8e8834bf4c77E+0x26)[0x7fff94ae0068]
/home/robin/Work/workspace/rust-testing/rust/build/riscv64gc-unknown-linux-gnu/stage0/bin/../lib/librustc_driver-fdc49e368e40bdcf.so(+0x5330fe2)[0x7fff91930fe2]
/home/robin/Work/workspace/rust-testing/rust/build/riscv64gc-unknown-linux-gnu/stage0/bin/../lib/librustc_driver-fdc49e368e40bdcf.so(+0x532ffd0)[0x7fff9192ffd0]
/home/robin/Work/workspace/rust-testing/rust/build/riscv64gc-unknown-linux-gnu/stage0/bin/../lib/librustc_driver-fdc49e368e40bdcf.so(+0x5331dd6)[0x7fff91931dd6]
/home/robin/Work/workspace/rust-testing/rust/build/riscv64gc-unknown-linux-gnu/stage0/bin/../lib/librustc_driver-fdc49e368e40bdcf.so(+0x5331e2a)[0x7fff91931e2a]
/home/robin/Work/workspace/rust-testing/rust/build/riscv64gc-unknown-linux-gnu/stage0/bin/../lib/librustc_driver-fdc49e368e40bdcf.so(+0x5332662)[0x7fff91932662]
/home/robin/Work/workspace/rust-testing/rust/build/riscv64gc-unknown-linux-gnu/stage0/bin/../lib/librustc_driver-fdc49e368e40bdcf.so(+0x5333606)[0x7fff91933606]
/home/robin/Work/workspace/rust-testing/rust/build/riscv64gc-unknown-linux-gnu/stage0/bin/../lib/libstd-aa93b6b6286a632f.so(rust_metadata_std_9285b80fa7b34547+0xaec00)[0x7fff94ae5c00]
/lib/riscv64-linux-gnu/libc.so.6(+0x6a956)[0x7fff8c508956]
/lib/riscv64-linux-gnu/libc.so.6(+0xbbbf0)[0x7fff8c559bf0]

note: we would appreciate a report at https://github.com/rust-lang/rust
error: rustc interrupted by SIGSEGV, printing backtrace

Git hash for the rust source checkout:
commit d4c86cf (HEAD -> master, origin/master, origin/HEAD)

[saethlin]

Could these be related to #117101 (comment)?

This was caused by the update to LLVM 17: it adds additional attributes to the object file which indicate which ISA extensions are used. These are parsed by cc and turned into a -march parameter passed to ld. However it seems that older versions of ld don't understand the new target feature attributes added by LLVM 17.

[raw-bin]

@saethlin: It does appear likely. Will try to dig a bit.

[herrernst]

I probably have the same or similar issue: I am using current stable rustc directly on riscv64 (Ubuntu 23.10). Even a simple "hello world" spits out weird stuff:

ubuntu@rv:~/src$ cat > example.rs
fn main() {
    println!("Hello World!");
}
ubuntu@rv:~/src$ rustc example.rs 
/home/ubuntu/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0xbb93a6)[0x3f7b1b93a6]
linux-vdso.so.1(__vdso_rt_sigreturn+0x0)[0x3f82b23800]
/lib/riscv64-linux-gnu/libc.so.6(syscall+0x16)[0x3f7a558406]
/home/ubuntu/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/libstd-7c20a79be40626d5.so(_ZN3std3sys4unix5locks13futex_condvar7Condvar4wait17hb973eb74843e1e38E+0x64)[0x3f82a65c9c]
/home/ubuntu/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52ecc88)[0x3f7f8ecc88]
/home/ubuntu/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52ed3b6)[0x3f7f8ed3b6]
/home/ubuntu/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52ed416)[0x3f7f8ed416]
/home/ubuntu/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52edce8)[0x3f7f8edce8]
/home/ubuntu/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/librustc_driver-3ba4609cd58445ac.so(+0x52eeac0)[0x3f7f8eeac0]
/home/ubuntu/.rustup/toolchains/stable-riscv64gc-unknown-linux-gnu/bin/../lib/libstd-7c20a79be40626d5.so(rust_metadata_std_bf3dc212422480e+0xaf7e2)[0x3f82a647e2]
/lib/riscv64-linux-gnu/libc.so.6(+0x6a956)[0x3f7a508956]
/lib/riscv64-linux-gnu/libc.so.6(+0xbbbf0)[0x3f7a559bf0]

It run's though:

ubuntu@rv:~/src$ ./example 
Hello World!

System info:

ubuntu@rv:~/src$ rustc --version --verbose
rustc 1.73.0 (cc66ad468 2023-10-03)
binary: rustc
commit-hash: cc66ad468955717ab92600c770da8c1601a4ff33
commit-date: 2023-10-03
host: riscv64gc-unknown-linux-gnu
release: 1.73.0
LLVM version: 17.0.2
ubuntu@rv:~/src$ ld --version
GNU ld (GNU Binutils for Ubuntu) 2.41