program: perform CO-RE against all kmod

btf/core_reloc_test.go

@@ -36,7 +36,8 @@ func TestCORERelocationLoad(t *testing.T) {
 			}
 
 			prog, err := ebpf.NewProgramWithOptions(progSpec, ebpf.ProgramOptions{
-				KernelTypes: spec.Types,
+				KernelTypes:       spec.Types,
+				KernelModuleTypes: map[string]*btf.Spec{},
 			})
 			testutils.SkipIfNotSupported(t, err)
 
@@ -83,7 +84,8 @@ func TestCORERelocationRead(t *testing.T) {
 	for _, progSpec := range spec.Programs {
 		t.Run(progSpec.Name, func(t *testing.T) {
 			prog, err := ebpf.NewProgramWithOptions(progSpec, ebpf.ProgramOptions{
-				KernelTypes: targetSpec,
+				KernelTypes:       targetSpec,
+				KernelModuleTypes: map[string]*btf.Spec{},
 			})
 			testutils.SkipIfNotSupported(t, err)
 			if err != nil {
@@ -149,7 +151,8 @@ func TestCOREPoisonLineInfo(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			t.Log(progSpec.Instructions)
 			_, err := ebpf.NewProgramWithOptions(progSpec, ebpf.ProgramOptions{
-				KernelTypes: empty,
+				KernelTypes:       empty,
+				KernelModuleTypes: map[string]*btf.Spec{},
 			})
 			testutils.SkipIfNotSupported(t, err)
 

btf/kernel.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
+	"sort"
 	"sync"
 
 	"github.com/cilium/ebpf/internal"
@@ -189,7 +191,8 @@ func findVMLinux() (*os.File, error) {
 // It is not safe for concurrent use.
 type Cache struct {
 	KernelTypes   *Spec
-	KernelModules map[string]*Spec
+	ModuleTypes   map[string]*Spec
+	LoadedModules []string
 }
 
 // NewCache creates a new Cache.
@@ -214,7 +217,11 @@ func NewCache() *Cache {
 		modules[name] = &Spec{decoder: decoder}
 	}
 
-	return &Cache{kernel, modules}
+	if len(modules) == 0 {
+		return &Cache{kernel, nil, nil}
+	}
+
+	return &Cache{kernel, modules, nil}
 }
 
 // Kernel is equivalent to [LoadKernelSpec], except that repeated calls do
@@ -234,12 +241,12 @@ func (c *Cache) Kernel() (*Spec, error) {
 //
 // All modules also share the return value of [Kernel] as their base.
 func (c *Cache) Module(name string) (*Spec, error) {
-	if spec := c.KernelModules[name]; spec != nil {
+	if spec := c.ModuleTypes[name]; spec != nil {
 		return spec, nil
 	}
 
-	if c.KernelModules == nil {
-		c.KernelModules = make(map[string]*Spec)
+	if c.ModuleTypes == nil {
+		c.ModuleTypes = make(map[string]*Spec)
 	}
 
 	base, err := c.Kernel()
@@ -260,6 +267,32 @@ func (c *Cache) Module(name string) (*Spec, error) {
 	}
 
 	spec = &Spec{decoder: decoder}
-	c.KernelModules[name] = spec
+	c.ModuleTypes[name] = spec
 	return spec, err
 }
+
+// Modules returns a sorted list of all loaded modules.
+func (c *Cache) Modules() ([]string, error) {
+	if c.LoadedModules != nil {
+		return c.LoadedModules, nil
+	}
+
+	btfDir, err := os.Open("/sys/kernel/btf")
+	if err != nil {
+		return nil, err
+	}
+	defer btfDir.Close()
+
+	entries, err := btfDir.Readdirnames(-1)
+	if err != nil {
+		return nil, err
+	}
+
+	entries = slices.DeleteFunc(entries, func(s string) bool {
+		return s == "vmlinux"
+	})
+
+	sort.Strings(entries)
+	c.LoadedModules = entries
+	return entries, nil
+}

btf/kernel_test.go

@@ -34,7 +34,8 @@ func TestCache(t *testing.T) {
 	c := NewCache()
 
 	qt.Assert(t, qt.IsNil(c.KernelTypes))
-	qt.Assert(t, qt.HasLen(c.KernelModules, 0))
+	qt.Assert(t, qt.HasLen(c.ModuleTypes, 0))
+	qt.Assert(t, qt.IsNil(c.LoadedModules))
 
 	// Test that Kernel() creates only one copy
 	spec1, err := c.Kernel()
@@ -75,7 +76,12 @@ func TestCache(t *testing.T) {
 	qt.Assert(t, qt.IsNotNil(c.KernelTypes))
 	qt.Assert(t, qt.Not(qt.Equals(c.KernelTypes, vmlinux)))
 	if testmod != nil {
-		qt.Assert(t, qt.IsNotNil(c.KernelModules["bpf_testmod"]))
-		qt.Assert(t, qt.Not(qt.Equals(c.KernelModules["bpf_testmod"], testmod)))
+		qt.Assert(t, qt.IsNotNil(c.ModuleTypes["bpf_testmod"]))
+		qt.Assert(t, qt.Not(qt.Equals(c.ModuleTypes["bpf_testmod"], testmod)))
 	}
+
+	// Test that Modules only reads modules once.
+	_, err = c.Modules()
+	qt.Assert(t, qt.IsNil(err))
+	qt.Assert(t, qt.IsNotNil(c.LoadedModules))
 }

btf/unmarshal.go

@@ -170,7 +170,7 @@ func rebaseDecoder(d *decoder, base *decoder) (*decoder, error) {
 		return nil, fmt.Errorf("rebase split spec: not a split spec")
 	}
 
-	if &d.base.raw[0] != &base.raw[0] || len(d.base.raw) != len(base.raw) {
+	if len(d.base.raw) != len(base.raw) || (len(d.base.raw) > 0 && &d.base.raw[0] != &base.raw[0]) {
 		return nil, fmt.Errorf("rebase split spec: raw BTF differs")
 	}
 
@@ -269,7 +269,9 @@ func (d *decoder) TypesByName(name essentialName) ([]Type, error) {
 	}
 
 	if len(types) == 0 {
-		return nil, fmt.Errorf("type with name %s: %w", name, ErrNotFound)
+		// Return an unwrapped error because this is on the hot path
+		// for CO-RE.
+		return nil, ErrNotFound
 	}
 
 	return types, nil

elf_reader_test.go

@@ -1130,6 +1130,7 @@ func TestLibBPFCompat(t *testing.T) {
 
 				opts := opts // copy
 				opts.Programs.KernelTypes = btfSpec
+				opts.Programs.KernelModuleTypes = map[string]*btf.Spec{}
 				load(t, spec, opts, valid)
 			})
 		}

linker.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/fs"
 	"math"
 	"slices"
 	"strings"
@@ -124,7 +123,7 @@ func hasFunctionReferences(insns asm.Instructions) bool {
 //
 // Passing a nil target will relocate against the running kernel. insns are
 // modified in place.
-func applyRelocations(insns asm.Instructions, kmodName string, bo binary.ByteOrder, b *btf.Builder, c *btf.Cache) error {
+func applyRelocations(insns asm.Instructions, bo binary.ByteOrder, b *btf.Builder, c *btf.Cache) error {
 	var relos []*btf.CORERelocation
 	var reloInsns []*asm.Instruction
 	iter := insns.Iterate()
@@ -143,22 +142,26 @@ func applyRelocations(insns asm.Instructions, kmodName string, bo binary.ByteOrd
 		bo = internal.NativeEndian
 	}
 
-	var targets []*btf.Spec
 	kernelTarget, err := c.Kernel()
 	if err != nil {
 		return fmt.Errorf("load kernel spec: %w", err)
 	}
+
+	modules, err := c.Modules()
+	if err != nil {
+		return err
+	}
+
+	targets := make([]*btf.Spec, 0, 1+len(modules))
 	targets = append(targets, kernelTarget)
 
-	if kmodName != "" {
-		kmodTarget, err := c.Module(kmodName)
-		// Ignore ErrNotExists to cater to kernels which have CONFIG_DEBUG_INFO_BTF_MODULES disabled.
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("load kernel module spec: %w", err)
-		}
-		if err == nil {
-			targets = append(targets, kmodTarget)
+	for _, kmod := range modules {
+		spec, err := c.Module(kmod)
+		if err != nil {
+			return fmt.Errorf("load BTF for kmod %s: %w", kmod, err)
 		}
+
+		targets = append(targets, spec)
 	}
 
 	fixups, err := btf.CORERelocate(relos, targets, bo, b.Add)

prog.go

@@ -8,12 +8,12 @@ import (
 	"math"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"time"
 
 	"github.com/cilium/ebpf/asm"
 	"github.com/cilium/ebpf/btf"
 	"github.com/cilium/ebpf/internal"
-	"github.com/cilium/ebpf/internal/kallsyms"
 	"github.com/cilium/ebpf/internal/linux"
 	"github.com/cilium/ebpf/internal/platform"
 	"github.com/cilium/ebpf/internal/sys"
@@ -171,17 +171,6 @@ func (ps *ProgramSpec) Tag() (string, error) {
 	return ps.Instructions.Tag(internal.NativeEndian)
 }
 
-// kernelModule returns the kernel module providing the symbol in
-// ProgramSpec.AttachTo, if any. Returns an empty string if the symbol is not
-// present or not part of a kernel module.
-func (ps *ProgramSpec) kernelModule() (string, error) {
-	if ps.targetsKernelModule() {
-		return kallsyms.Module(ps.AttachTo)
-	}
-
-	return "", nil
-}
-
 // targetsKernelModule returns true if the program supports being attached to a
 // symbol provided by a kernel module.
 func (ps *ProgramSpec) targetsKernelModule() bool {
@@ -303,13 +292,8 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, c *btf.Cache)
 	insns := make(asm.Instructions, len(spec.Instructions))
 	copy(insns, spec.Instructions)
 
-	kmodName, err := spec.kernelModule()
-	if err != nil {
-		return nil, fmt.Errorf("kernel module search: %w", err)
-	}
-
 	var b btf.Builder
-	if err := applyRelocations(insns, kmodName, spec.ByteOrder, &b, c); err != nil {
+	if err := applyRelocations(insns, spec.ByteOrder, &b, c); err != nil {
 		return nil, fmt.Errorf("apply CO-RE relocations: %w", err)
 	}
 
@@ -1212,7 +1196,14 @@ func newBTFCache(opts *ProgramOptions) *btf.Cache {
 	c := btf.NewCache()
 	if opts.KernelTypes != nil {
 		c.KernelTypes = opts.KernelTypes
-		c.KernelModules = opts.KernelModuleTypes
+		c.ModuleTypes = opts.KernelModuleTypes
+		if opts.KernelModuleTypes != nil {
+			c.LoadedModules = []string{}
+			for name := range opts.KernelModuleTypes {
+				c.LoadedModules = append(c.LoadedModules, name)
+			}
+			sort.Strings(c.LoadedModules)
+		}
 	}
 	return c
 }

prog_test.go

@@ -881,7 +881,8 @@ func TestProgramLoadErrors(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			t.Log(progSpec.Instructions)
 			_, err := newProgram(t, progSpec, &ProgramOptions{
-				KernelTypes: empty,
+				KernelTypes:       empty,
+				KernelModuleTypes: map[string]*btf.Spec{},
 			})
 			testutils.SkipIfNotSupported(t, err)
 
@@ -902,15 +903,6 @@ func TestProgramTargetsKernelModule(t *testing.T) {
 	qt.Assert(t, qt.IsTrue(ps.targetsKernelModule()))
 }
 
-func TestProgramAttachToKernelModule(t *testing.T) {
-	requireTestmod(t)
-
-	ps := ProgramSpec{AttachTo: "bpf_testmod_test_read", Type: Tracing, AttachType: AttachTraceFEntry}
-	mod, err := ps.kernelModule()
-	qt.Assert(t, qt.IsNil(err))
-	qt.Assert(t, qt.Equals(mod, "bpf_testmod"))
-}
-
 func BenchmarkNewProgram(b *testing.B) {
 	testutils.SkipOnOldKernel(b, "5.18", "kfunc support")
 	spec, err := LoadCollectionSpec(testutils.NativeFile(b, "testdata/kfunc-%s.elf"))