Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pkg/sentry/devices/nvproxy/nvconf/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ go_library(
name = "nvconf",
srcs = [
"caps.go",
"hostdriver_unsafe.go",
"hostsettings.go",
"nvconf.go",
"version.go",
Expand All @@ -17,7 +18,10 @@ go_library(
"//tools:__subpackages__",
],
deps = [
"//pkg/abi/linux",
"//pkg/abi/nvgpu",
"//pkg/log",
"@org_golang_x_sys//unix:go_default_library",
],
)

Expand Down
72 changes: 72 additions & 0 deletions pkg/sentry/devices/nvproxy/nvconf/hostdriver_unsafe.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// Copyright 2026 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nvconf

import (
"bytes"
"fmt"
"unsafe"

"golang.org/x/sys/unix"
"gvisor.dev/gvisor/pkg/abi/linux"
"gvisor.dev/gvisor/pkg/abi/nvgpu"
)

// HostDriverVersion returns the version of the host Nvidia driver.
func HostDriverVersion() (string, error) {
ctlFD, err := unix.Openat(-1, "/dev/nvidiactl", unix.O_RDONLY|unix.O_NOFOLLOW, 0)
if err != nil {
return "", fmt.Errorf("failed to open /dev/nvidiactl: %w", err)
}
defer unix.Close(ctlFD)

// From src/nvidia/arch/nvalloc/unix/include/nv-ioctl.h:
const NV_RM_API_VERSION_REPLY_RECOGNIZED = 1

// 530.30.02 and later versions of the host driver `#define
// NV_RM_API_VERSION_CMD_QUERY '2'`, which causes this ioctl to return the
// driver version without performing a check. Earlier versions of the
// driver `#define NV_RM_API_VERSION_CMD_OVERRIDE '2'`, which causes the
// ioctl to no-op. Try with Cmd '2' first, hoping that the driver
// interprets it as _QUERY; if the returned string is empty, then it was
// interpreted as _OVERRIDE and we need to perform an actual check (Cmd 0),
// which has the downside of logging an error message.
ioctlParams := nvgpu.RMAPIVersion{
Cmd: '2',
}
if _, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(ctlFD), uintptr(linux.IOWR(nvgpu.NV_IOCTL_MAGIC, nvgpu.NV_ESC_CHECK_VERSION_STR, uint32(unsafe.Sizeof(ioctlParams)))), uintptr(unsafe.Pointer(&ioctlParams))); errno != 0 {
return "", fmt.Errorf("NV_ESC_CHECK_VERSION_STR ioctl error: %w", errno)
}
if ioctlParams.Reply != NV_RM_API_VERSION_REPLY_RECOGNIZED {
return "", fmt.Errorf("unknown NV_ESC_CHECK_VERSION_STR reply: %d", ioctlParams.Reply)
}
if ioctlParams.VersionString[0] == '\x00' {
ioctlParams.Cmd = 0
ioctlParams.Reply = 0
// We expect the check to fail on our empty version string, so tolerate
// EINVAL.
if _, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(ctlFD), uintptr(linux.IOWR(nvgpu.NV_IOCTL_MAGIC, nvgpu.NV_ESC_CHECK_VERSION_STR, uint32(unsafe.Sizeof(ioctlParams)))), uintptr(unsafe.Pointer(&ioctlParams))); errno != 0 && errno != unix.EINVAL {
return "", fmt.Errorf("fallback NV_ESC_CHECK_VERSION_STR ioctl error: %w", errno)
}
if ioctlParams.Reply != NV_RM_API_VERSION_REPLY_RECOGNIZED {
return "", fmt.Errorf("unknown fallback NV_ESC_CHECK_VERSION_STR reply: %d", ioctlParams.Reply)
}
}

if i := bytes.IndexByte(ioctlParams.VersionString[:], '\x00'); i >= 0 {
return string(ioctlParams.VersionString[:i]), nil
}
return string(ioctlParams.VersionString[:]), nil
}
50 changes: 0 additions & 50 deletions pkg/sentry/devices/nvproxy/nvproxy_unsafe.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,61 +15,11 @@
package nvproxy

import (
"bytes"
"fmt"
"unsafe"

"golang.org/x/sys/unix"
"gvisor.dev/gvisor/pkg/abi/nvgpu"
)

// HostDriverVersion returns the version of the host Nvidia driver.
func HostDriverVersion() (string, error) {
ctlFD, err := unix.Openat(-1, "/dev/nvidiactl", unix.O_RDONLY|unix.O_NOFOLLOW, 0)
if err != nil {
return "", fmt.Errorf("failed to open /dev/nvidiactl: %w", err)
}
defer unix.Close(ctlFD)

// From src/nvidia/arch/nvalloc/unix/include/nv-ioctl.h:
const NV_RM_API_VERSION_REPLY_RECOGNIZED = 1

// 530.30.02 and later versions of the host driver `#define
// NV_RM_API_VERSION_CMD_QUERY '2'`, which causes this ioctl to return the
// driver version without performing a check. Earlier versions of the
// driver `#define NV_RM_API_VERSION_CMD_OVERRIDE '2'`, which causes the
// ioctl to no-op. Try with Cmd '2' first, hoping that the driver
// interprets it as _QUERY; if the returned string is empty, then it was
// interpreted as _OVERRIDE and we need to perform an actual check (Cmd 0),
// which has the downside of logging an error message.
ioctlParams := nvgpu.RMAPIVersion{
Cmd: '2',
}
if _, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(ctlFD), frontendIoctlCmd(nvgpu.NV_ESC_CHECK_VERSION_STR, uint32(unsafe.Sizeof(ioctlParams))), uintptr(unsafe.Pointer(&ioctlParams))); errno != 0 {
return "", fmt.Errorf("NV_ESC_CHECK_VERSION_STR ioctl error: %w", errno)
}
if ioctlParams.Reply != NV_RM_API_VERSION_REPLY_RECOGNIZED {
return "", fmt.Errorf("unknown NV_ESC_CHECK_VERSION_STR reply: %d", ioctlParams.Reply)
}
if ioctlParams.VersionString[0] == '\x00' {
ioctlParams.Cmd = 0
ioctlParams.Reply = 0
// We expect the check to fail on our empty version string, so tolerate
// EINVAL.
if _, _, errno := unix.RawSyscall(unix.SYS_IOCTL, uintptr(ctlFD), frontendIoctlCmd(nvgpu.NV_ESC_CHECK_VERSION_STR, uint32(unsafe.Sizeof(ioctlParams))), uintptr(unsafe.Pointer(&ioctlParams))); errno != 0 && errno != unix.EINVAL {
return "", fmt.Errorf("fallback NV_ESC_CHECK_VERSION_STR ioctl error: %w", errno)
}
if ioctlParams.Reply != NV_RM_API_VERSION_REPLY_RECOGNIZED {
return "", fmt.Errorf("unknown fallback NV_ESC_CHECK_VERSION_STR reply: %d", ioctlParams.Reply)
}
}

if i := bytes.IndexByte(ioctlParams.VersionString[:], '\x00'); i >= 0 {
return string(ioctlParams.VersionString[:i]), nil
}
return string(ioctlParams.VersionString[:]), nil
}

func p64FromPtr(ptr unsafe.Pointer) nvgpu.P64 {
return nvgpu.P64(uint64(uintptr(ptr)))
}
1 change: 1 addition & 0 deletions runsc/cmd/sentry/sentrycmd/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ go_library(
"//pkg/prometheus",
"//pkg/rdma",
"//pkg/ring0",
"//pkg/sentry/devices/nvproxy",
"//pkg/sentry/devices/nvproxy/nvconf",
"//pkg/sentry/hostmm",
"//pkg/sentry/platform",
Expand Down
11 changes: 8 additions & 3 deletions runsc/cmd/sentry/sentrycmd/boot.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
"gvisor.dev/gvisor/pkg/prometheus"
"gvisor.dev/gvisor/pkg/rdma"
"gvisor.dev/gvisor/pkg/ring0"
"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy"
"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf"
"gvisor.dev/gvisor/pkg/sentry/hostmm"
"gvisor.dev/gvisor/pkg/sentry/platform"
Expand Down Expand Up @@ -220,7 +221,8 @@ type Boot struct {
// used to synchronize rootless user namespace initialization.
syncUsernsFD int

// nvidiaDriverVersion is the Nvidia driver version on the host.
// nvidiaDriverVersion is the Nvidia driver ABI version to use, or "latest"
// for the newest supported one.
nvidiaDriverVersion string

// These correspond to fields in nvconf.HostSettings, which is not used
Expand Down Expand Up @@ -311,7 +313,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
f.BoolVar(&b.profilingMetricsLossy, "profiling-metrics-fd-lossy", false, "if true, treat the sentry profiling metrics FD as lossy and write a checksum to it.")

// Nvidia driver properties.
f.StringVar(&b.nvidiaDriverVersion, "nvidia-driver-version", "", "Nvidia driver version on the host")
f.StringVar(&b.nvidiaDriverVersion, "nvidia-driver-version", "", "Nvidia driver ABI version to use, or 'latest' for the newest supported one")
f.StringVar(&b.procDriverNvidiaParams, "nvidia-host-params", "", "value of /proc/driver/nvidia/params on the host")
f.Int64Var(&b.nvidiaFabricIMEXManagementDevMinor, "nvidia-fabric-imex-mgmt-minor", -1, "DeviceFileMinor in /proc/driver/nvidia/capabilities/fabric-imex-mgmt on the host")
}
Expand Down Expand Up @@ -666,7 +668,10 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
}

var nvidiaDriverVersion nvconf.DriverVersion
if b.nvidiaDriverVersion != "" {
if b.nvidiaDriverVersion == "latest" {
nvproxy.Init()
nvidiaDriverVersion = nvproxy.LatestDriver()
} else if b.nvidiaDriverVersion != "" {
nvidiaDriverVersion, err = nvconf.DriverVersionFrom(b.nvidiaDriverVersion)
if err != nil {
util.Fatalf("Failed to parse nvidia driver version: %v", err)
Expand Down
1 change: 0 additions & 1 deletion runsc/sandbox/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ go_library(
"//pkg/prometheus",
"//pkg/sentry/checkpoint",
"//pkg/sentry/control",
"//pkg/sentry/devices/nvproxy",
"//pkg/sentry/devices/nvproxy/nvconf",
"//pkg/sentry/fsimpl/erofs",
"//pkg/sentry/platform",
Expand Down
8 changes: 3 additions & 5 deletions runsc/sandbox/sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ import (
"gvisor.dev/gvisor/pkg/prometheus"
"gvisor.dev/gvisor/pkg/sentry/checkpoint"
"gvisor.dev/gvisor/pkg/sentry/control"
"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy"
"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf"
"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
"gvisor.dev/gvisor/pkg/sentry/platform"
Expand Down Expand Up @@ -2566,14 +2565,13 @@ func deviceFileForPlatform(name, devicePath string) (*fd.FD, error) {
}

// getNvproxyDriverVersion returns the NVIDIA driver ABI version to use by
// nvproxy.
// nvproxy, or "latest", which the sentry resolves to the newest supported ABI.
func getNvproxyDriverVersion(conf *config.Config) (string, error) {
switch conf.NVProxyDriverVersion {
case "":
return nvproxy.HostDriverVersion()
return nvconf.HostDriverVersion()
case "latest":
nvproxy.Init()
return nvproxy.LatestDriver().String(), nil
return conf.NVProxyDriverVersion, nil
default:
version, err := nvconf.DriverVersionFrom(conf.NVProxyDriverVersion)
return version.String(), err
Expand Down
2 changes: 1 addition & 1 deletion tools/ioctl_sniffer/sniffer/sniffer.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ func Init() error {
nvproxy.Init()

// Load the ABI for the host driver.
driverVerStr, err := nvproxy.HostDriverVersion()
driverVerStr, err := nvconf.HostDriverVersion()
if err != nil {
return fmt.Errorf("failed to get host driver version: %w", err)
}
Expand Down
Loading