Skip to content

Allow systems without NUMA support to start up #7524

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Ravi-Kappiyoor wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Allow systems without NUMA support to start up #7524

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b76bfd2
Allow systems without NUMA support to start up
Ravi-Kappiyoor Dec 2, 2025
90c8d8f
Check for ENOENT specifically on failure to open /sys/devices/system/…
Ravi-Kappiyoor Dec 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions src/util/shmem/fd_numa_linux.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ fd_numa_node_cnt( void ) {
char const * path = "/sys/devices/system/node";
DIR * dir = opendir( path );
if( FD_UNLIKELY( !dir ) ) {
// On some systems (such as a Linux container running in Docker on a Mac), parts of the file system
// are not set up the way we expect it to be on Linux. E.g., `/sys/devices/system/node` does not
// exist - if we are in such a system, assume that we only have a single NUMA node and continue
if ( errno == ENOENT ) return 1UL;
FD_LOG_WARNING(( "opendir( \"%s\" ) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
return 0UL;
}
Expand Down Expand Up @@ -118,8 +122,8 @@ fd_numa_node_idx( ulong cpu_idx ) {
FD_LOG_WARNING(( "closedir( \"%s\" ) failed (%i-%s); attempting to continue", path, errno, fd_io_strerror( errno ) ));

if( FD_UNLIKELY( node_idx<0 ) ) {
FD_LOG_WARNING(( "No numa node found in \"%s\"", path ));
return ULONG_MAX;
FD_LOG_WARNING(( "No numa node found in \"%s\" - assuming NUMA node 0", path ));
return 0UL;
Copy link
Contributor Author

@Ravi-Kappiyoor Ravi-Kappiyoor Dec 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without this change, fd_shmem_private_boot (run as part of fd_boot) would fail due to:

    ulong numa_idx = fd_numa_node_idx( cpu_idx );
    if( FD_UNLIKELY( numa_idx>=FD_SHMEM_NUMA_MAX) )
      FD_LOG_ERR(( "fd_shmem: unexpected numa idx (%lu) for cpu idx %lu", numa_idx, cpu_idx ));

}

return (ulong)node_idx;
Expand Down Expand Up @@ -178,6 +182,10 @@ fd_numa_get_mempolicy( int * mode,
void * addr,
uint flags ) {
long rc = syscall( SYS_get_mempolicy, mode, nodemask, maxnode, addr, flags );
if ( rc && errno == ENOSYS ) {
FD_LOG_WARNING(( "System appears to not support NUMA - unable to get mempolicy. Attempting to continue..." ));
return 0L;
Copy link
Contributor Author

@Ravi-Kappiyoor Ravi-Kappiyoor Dec 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without this change, we are unable to create a workspace due to the following in fd_shmem_create_multi_flags:

  if( FD_UNLIKELY( fd_numa_get_mempolicy( &orig_mempolicy, orig_nodemask, FD_SHMEM_NUMA_MAX, NULL, 0UL ) ) ) {
    FD_LOG_WARNING(( "fd_numa_get_mempolicy failed (%i-%s)", errno, fd_io_strerror( errno ) ));
    ERROR( done );
  }

}
if( rc==0 ) {
if( mode ) fd_msan_unpoison( mode, sizeof(int) );
if( nodemask ) fd_msan_unpoison( nodemask, 8UL*((maxnode+63UL)/64UL) );
Expand All @@ -189,7 +197,12 @@ long
fd_numa_set_mempolicy( int mode,
ulong const * nodemask,
ulong maxnode ) {
return syscall( SYS_set_mempolicy, mode, nodemask, maxnode );
long rc = syscall( SYS_set_mempolicy, mode, nodemask, maxnode );
if ( rc && errno == ENOSYS ) {
FD_LOG_WARNING(( "System appears to not support NUMA - unable to set mempolicy. Attempting to continue..." ));
return 0L;
Copy link
Contributor Author

@Ravi-Kappiyoor Ravi-Kappiyoor Dec 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without this change, we are unable to create a workspace due to the following in fd_shmem_create_multi_flags:

    if( FD_UNLIKELY( fd_numa_set_mempolicy( MPOL_BIND | MPOL_F_STATIC_NODES, nodemask, FD_SHMEM_NUMA_MAX ) ) ) {
      FD_LOG_WARNING(( "fd_numa_set_mempolicy failed (%i-%s)", errno, fd_io_strerror( errno ) ));
      ERROR( unmap );
    }

}
return rc;
}

long
Expand All @@ -199,7 +212,12 @@ fd_numa_mbind( void * addr,
ulong const * nodemask,
ulong maxnode,
uint flags ) {
return syscall( SYS_mbind, addr, len, mode, nodemask, maxnode, flags );
long rc = syscall( SYS_mbind, addr, len, mode, nodemask, maxnode, flags );
if ( rc && errno == ENOSYS ) {
FD_LOG_WARNING(( "System appears to not support NUMA - unable to bind memory. Attempting to continue..." ));
return 0L;
Copy link
Contributor Author

@Ravi-Kappiyoor Ravi-Kappiyoor Dec 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without this change, we are unable to create a workspace due to the following in fd_shmem_create_multi_flags:

    if( FD_UNLIKELY( fd_numa_mbind( sub_shmem, sub_sz, MPOL_BIND, nodemask, FD_SHMEM_NUMA_MAX, MPOL_MF_MOVE|MPOL_MF_STRICT ) ) ) {
      FD_LOG_WARNING(( "sub[%lu]: fd_numa_mbind(\"%s\",%lu KiB,MPOL_BIND,1UL<<%lu,MPOL_MF_MOVE|MPOL_MF_STRICT) failed (%i-%s)",
                       sub_idx, path, sub_sz>>10, sub_numa_idx, errno, fd_io_strerror( errno ) ));
      ERROR( unmap );
    }

}
return rc;
}

long
Expand Down