Skip to content

CUDA 11.6 generates invalid device code for all physicalBoundaryFunctors #21

Closed

Description

nvcc 11.6.55 generates invalid device code for all test problems without fully-periodic boundary conditions:

Starting program: /avatar/bwibking/quokka/build/src/HydroShocktube/test_hydro_shocktube shocktube.in
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
warning: Cannot parse .gnu_debugdata section; LZMA support was disabled at compile time
warning: Cannot parse .gnu_debugdata section; LZMA support was disabled at compile time
[Detaching after fork from child process 231811]
[New Thread 0x7ffff4661000 (LWP 231815)]
[New Thread 0x7ffff3e60000 (LWP 231816)]
Initializing CUDA...
[Detaching after fork from child process 231818]
[New Thread 0x7ffff162f000 (LWP 231828)]
[New Thread 0x7ffff0e2e000 (LWP 231829)]
CUDA initialized with 1 GPU per MPI rank; 1 GPU(s) used in total
MPI initialized with 1 MPI processes
MPI initialized with thread support level 0
AMReX (22.01-1-ga73ce6759c91) initialized
warning: Cuda API error detected: cudaLaunchKernel returned (0x62)

warning: Cuda API error detected: cudaGetLastError returned (0x62)

terminate called after throwing an instance of 'std::runtime_error'
  what():  GPU last error detected in file /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX_GpuLaunchFunctsG.H line 862: invalid device function

Thread 1 "test_hydro_shoc" received signal SIGABRT, Aborted.
0x00007ffff63ae37f in raise () from /lib64/libc.so.6
(cuda-gdb) bt
#0  0x00007ffff63ae37f in raise () from /lib64/libc.so.6
#1  0x00007ffff6398db5 in abort () from /lib64/libc.so.6
#2  0x00007ffff6d6609b in ?? () from /lib64/libstdc++.so.6
#3  0x00007ffff6d6c53c in ?? () from /lib64/libstdc++.so.6
#4  0x00007ffff6d6b559 in ?? () from /lib64/libstdc++.so.6
#5  0x00007ffff6d6bed8 in __gxx_personality_v0 () from /lib64/libstdc++.so.6
#6  0x00007ffff674cb03 in ?? () from /lib64/libgcc_s.so.1
#7  0x00007ffff674d071 in _Unwind_RaiseException () from /lib64/libgcc_s.so.1
#8  0x00007ffff6d6c7eb in __cxa_throw () from /lib64/libstdc++.so.6
#9  0x0000000000484c22 in amrex::Abort_host (
    msg=0xec5c050 "GPU last error detected in file /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX_GpuLaunchFunctsG.H line 862: invalid device function") at /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX.cpp:237
#10 0x0000000000484a27 in amrex::Abort (
    msg=0xec5c050 "GPU last error detected in file /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX_GpuLaunchFunctsG.H line 862: invalid device function") at /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX.H:154
#11 amrex::Abort (msg=...) at /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX.cpp:197
#12 0x000000000041a394 in amrex::Gpu::ErrorCheck (
    file=0x81c988 "/avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX_GpuLaunchFunctsG.H", line=862)
    at /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX_GpuError.H:57
#13 0x0000000000453e94 in amrex::ParallelFor<__nv_dl_wrapper_t<__nv_dl_tag<void (amrex::GpuBndryFuncFab<setBoundaryFunctor<ShocktubeProblem> >::*)(amrex::Box const&, amrex::FArrayBox&, int, int, amrex::Geometry const&, double, amrex::Vector<amrex::BCRec, std::allocator<amrex::BCRec> > const&, int, int, amrex::FilccCell&&), &amrex::GpuBndryFuncFab<setBoundaryFunctor<ShocktubeProblem> >::ccfcdoit<amrex::FilccCell>, 1u>, amrex::FilccCell, amrex::Array4<double> const, int const, int const, amrex::Box const, amrex::BCRec*, setBoundaryFunctor<ShocktubeProblem> const, amrex::GeometryData const, double const, int const> > (box=..., f=...)
    at /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX_GpuLaunchFunctsG.H:862
#14 0x000000000044632b in amrex::ParallelFor<__nv_dl_wrapper_t<__nv_dl_tag<void (amrex::GpuBndryFuncFab<setBoundaryFunctor<ShocktubeProblem> >::*)(amrex::Box const&, amrex::FArrayBox&, int, int, amrex::Geometry const&, double, amrex::Vector<amrex::BCRec, std::allocator<amrex::BCRec> > const&, int, int, amrex::FilccCell&&), &amrex::GpuBndryFuncFab<setBoundaryFunctor<ShocktubeProblem> >::ccfcdoit<amrex::FilccCell>, 1u>, amrex::FilccCell, amrex::Array4<double> const, int const, int const, amrex::Box const, amrex::BCRec*, setBoundaryFunctor<ShocktubeProblem> const, amrex::GeometryData const, double const, int const> > (box=..., f=...)
    at /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX_GpuLaunchFunctsG.H:1275
#15 0x000000000042864d in amrex::GpuBndryFuncFab<setBoundaryFunctor<ShocktubeProblem> >::ccfcdoit<amrex::FilccCell> (
    this=0x7fffffff8130, bx=..., dest=..., dcomp=0, numcomp=9, geom=..., time=0, bcr=..., bcomp=0, orig_comp=0,
    fillfunc=...) at /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX_PhysBCFunct.H:383
#16 0x0000000000455a0d in amrex::GpuBndryFuncFab<setBoundaryFunctor<ShocktubeProblem> >::operator() (
    this=0x7fffffff8130, bx=..., dest=..., dcomp=0, numcomp=9, geom=..., time=0, bcr=..., bcomp=0, orig_comp=0)
    at /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX_PhysBCFunct.H:204
#17 0x0000000000449b91 in amrex::PhysBCFunct<amrex::GpuBndryFuncFab<setBoundaryFunctor<ShocktubeProblem> > >::operator() (this=0x7fffffff80c0, mf=..., icomp=0, ncomp=9, nghost=..., time=0, bccomp=0)
    at /avatar/bwibking/quokka/extern/amrex/Src/Base/AMReX_PhysBCFunct.H:177
#18 0x000000000046cca1 in AMRSimulation<ShocktubeProblem>::fillBoundaryConditions (this=0x7fffffff8620, S_filled=...,
    state=..., lev=0, time=0) at /avatar/bwibking/quokka/src/simulation.hpp:695
#19 0x000000000046a6e1 in AMRSimulation<ShocktubeProblem>::MakeNewLevelFromScratch (this=0x7fffffff8620, level=0,
    time=0, ba=..., dm=...) at /avatar/bwibking/quokka/src/simulation.hpp:628
#20 0x00000000007a0435 in amrex::AmrMesh::MakeNewGrids (this=0x7fffffff8620, time=0)
    at /avatar/bwibking/quokka/extern/amrex/Src/AmrCore/AMReX_AmrMesh.cpp:779
#21 0x00000000006fb236 in amrex::AmrCore::InitFromScratch (this=0x7fffffff8620, time=0)
    at /avatar/bwibking/quokka/extern/amrex/Src/AmrCore/AMReX_AmrCore.cpp:64
#22 0x000000000041e227 in AMRSimulation<ShocktubeProblem>::setInitialConditions (this=0x7fffffff8620)
    at /avatar/bwibking/quokka/src/simulation.hpp:280
#23 0x0000000000410ee8 in problem_main () at /avatar/bwibking/quokka/src/HydroShocktube/test_hydro_shocktube.cpp:332
#24 0x000000000040dde5 in main (argc=2, argv=0x7fffffff8d28) at /avatar/bwibking/quokka/src/main.cpp:41

Workaround: Use CUDA <= 11.5.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Assignees

No one assigned

    Labels

    compiler bugcaused by a bug in a compiler, not in the code itself

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions