btf: mmap vmlinux if possible

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
 
 env:
   TMPDIR: /tmp
-  CI_MAX_KERNEL_VERSION: '6.11'
+  CI_MAX_KERNEL_VERSION: '6.15'
   CI_MAX_EFW_VERSION: '0.21.0'
   CI_MIN_CLANG_VERSION: '13'
   go_version: '~1.24'

btf/btf.go

@@ -1,9 +1,7 @@
 package btf
 
 import (
-	"bufio"
 	"debug/elf"
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
@@ -69,11 +67,12 @@ func LoadSpec(file string) (*Spec, error) {
 func LoadSpecFromReader(rd io.ReaderAt) (*Spec, error) {
 	file, err := internal.NewSafeELFFile(rd)
 	if err != nil {
-		if bo := guessRawBTFByteOrder(rd); bo != nil {
-			return loadRawSpec(io.NewSectionReader(rd, 0, math.MaxInt64), bo, nil)
+		raw, err := io.ReadAll(io.NewSectionReader(rd, 0, math.MaxInt64))
+		if err != nil {
+			return nil, fmt.Errorf("read raw BTF: %w", err)
 		}
 
-		return nil, err
+		return loadRawSpec(raw, nil)
 	}
 
 	return loadSpecFromELF(file)
@@ -170,15 +169,20 @@ func loadSpecFromELF(file *internal.SafeELFFile) (*Spec, error) {
 		return nil, err
 	}
 
-	if btfSection.ReaderAt == nil {
-		return nil, fmt.Errorf("compressed BTF is not supported")
+	rawBTF, err := btfSection.Data()
+	if err != nil {
+		return nil, fmt.Errorf("reading .BTF section: %w", err)
 	}
 
-	spec, err := loadRawSpec(btfSection.ReaderAt, file.ByteOrder, nil)
+	spec, err := loadRawSpec(rawBTF, nil)
 	if err != nil {
 		return nil, err
 	}
 
+	if spec.decoder.byteOrder != file.ByteOrder {
+		return nil, fmt.Errorf("BTF byte order %s does not match ELF byte order %s", spec.decoder.byteOrder, file.ByteOrder)
+	}
+
 	spec.elf = &elfData{
 		sectionSizes,
 		offsets,
@@ -188,7 +192,7 @@ func loadSpecFromELF(file *internal.SafeELFFile) (*Spec, error) {
 	return spec, nil
 }
 
-func loadRawSpec(btf io.ReaderAt, bo binary.ByteOrder, base *Spec) (*Spec, error) {
+func loadRawSpec(btf []byte, base *Spec) (*Spec, error) {
 	var (
 		baseDecoder *decoder
 		baseStrings *stringTable
@@ -200,47 +204,39 @@ func loadRawSpec(btf io.ReaderAt, bo binary.ByteOrder, base *Spec) (*Spec, error
 		baseStrings = base.strings
 	}
 
-	buf := internal.NewBufferedSectionReader(btf, 0, math.MaxInt64)
-	header, err := parseBTFHeader(buf, bo)
+	header, bo, err := parseBTFHeader(btf)
 	if err != nil {
 		return nil, fmt.Errorf("parsing .BTF header: %v", err)
 	}
 
-	stringsSection := io.NewSectionReader(btf, header.stringStart(), int64(header.StringLen))
-	rawStrings, err := readStringTable(stringsSection, baseStrings)
+	if header.HdrLen > uint32(len(btf)) {
+		return nil, fmt.Errorf("BTF header length is out of bounds")
+	}
+	btf = btf[header.HdrLen:]
+
+	if int(header.StringOff+header.StringLen) > len(btf) {
+		return nil, fmt.Errorf("string table is out of bounds")
+	}
+	stringsSection := btf[header.StringOff : header.StringOff+header.StringLen]
+
+	rawStrings, err := newStringTable(stringsSection, baseStrings)
 	if err != nil {
 		return nil, fmt.Errorf("read string section: %w", err)
 	}
 
-	typesSection := io.NewSectionReader(btf, header.typeStart(), int64(header.TypeLen))
-	rawTypes := make([]byte, header.TypeLen)
-	if _, err := io.ReadFull(typesSection, rawTypes); err != nil {
-		return nil, fmt.Errorf("read type section: %w", err)
+	if int(header.TypeOff+header.TypeLen) > len(btf) {
+		return nil, fmt.Errorf("types section is out of bounds")
 	}
+	typesSection := btf[header.TypeOff : header.TypeOff+header.TypeLen]
 
-	decoder, err := newDecoder(rawTypes, bo, rawStrings, baseDecoder)
+	decoder, err := newDecoder(typesSection, bo, rawStrings, baseDecoder)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Spec{decoder, nil}, nil
 }
 
-func guessRawBTFByteOrder(r io.ReaderAt) binary.ByteOrder {
-	buf := new(bufio.Reader)
-	for _, bo := range []binary.ByteOrder{
-		binary.LittleEndian,
-		binary.BigEndian,
-	} {
-		buf.Reset(io.NewSectionReader(r, 0, math.MaxInt64))
-		if _, err := parseBTFHeader(buf, bo); err == nil {
-			return bo
-		}
-	}
-
-	return nil
-}
-
 // fixupDatasec attempts to patch up missing info in Datasecs and its members by
 // supplementing them with information from the ELF headers and symbol table.
 func (elf *elfData) fixupDatasec(typ Type) error {
@@ -505,7 +501,12 @@ func (s *Spec) TypeByName(name string, typ interface{}) error {
 // Types from base are used to resolve references in the split BTF.
 // The returned Spec only contains types from the split BTF, not from the base.
 func LoadSplitSpecFromReader(r io.ReaderAt, base *Spec) (*Spec, error) {
-	return loadRawSpec(r, internal.NativeEndian, base)
+	raw, err := io.ReadAll(io.NewSectionReader(r, 0, math.MaxInt64))
+	if err != nil {
+		return nil, fmt.Errorf("read raw BTF: %w", err)
+	}
+
+	return loadRawSpec(raw, base)
 }
 
 // All iterates over all types.

btf/btf_test.go

@@ -2,10 +2,8 @@ package btf
 
 import (
 	"bytes"
-	"encoding/binary"
 	"errors"
 	"fmt"
-	"io"
 	"io/fs"
 	"os"
 	"runtime"
@@ -46,34 +44,34 @@ var vmlinuxTestdata = sync.OnceValues(func() (specAndRawBTF, error) {
 		return specAndRawBTF{}, err
 	}
 
-	spec, err := loadRawSpec(bytes.NewReader(b), binary.LittleEndian, nil)
+	spec, err := loadRawSpec(b, nil)
 	if err != nil {
 		return specAndRawBTF{}, err
 	}
 
 	return specAndRawBTF{b, spec}, nil
 })
 
-func vmlinuxTestdataReader(tb testing.TB) *bytes.Reader {
+func vmlinuxTestdataSpec(tb testing.TB) *Spec {
 	tb.Helper()
 
 	td, err := vmlinuxTestdata()
 	if err != nil {
 		tb.Fatal(err)
 	}
 
-	return bytes.NewReader(td.raw)
+	return td.spec.Copy()
 }
 
-func vmlinuxTestdataSpec(tb testing.TB) *Spec {
+func vmlinuxTestdataBytes(tb testing.TB) []byte {
 	tb.Helper()
 
 	td, err := vmlinuxTestdata()
 	if err != nil {
 		tb.Fatal(err)
 	}
 
-	return td.spec.Copy()
+	return td.raw
 }
 
 func parseELFBTF(tb testing.TB, file string) *Spec {
@@ -211,32 +209,24 @@ func TestTypeByName(t *testing.T) {
 }
 
 func BenchmarkParseVmlinux(b *testing.B) {
-	rd := vmlinuxTestdataReader(b)
+	vmlinux := vmlinuxTestdataBytes(b)
 	b.ReportAllocs()
 	b.ResetTimer()
 
 	for n := 0; n < b.N; n++ {
-		if _, err := rd.Seek(0, io.SeekStart); err != nil {
-			b.Fatal(err)
-		}
-
-		if _, err := loadRawSpec(rd, binary.LittleEndian, nil); err != nil {
+		if _, err := loadRawSpec(vmlinux, nil); err != nil {
 			b.Fatal("Can't load BTF:", err)
 		}
 	}
 }
 
 func BenchmarkIterateVmlinux(b *testing.B) {
-	rd := vmlinuxTestdataReader(b)
+	vmlinux := vmlinuxTestdataBytes(b)
 	b.ReportAllocs()
 	b.ResetTimer()
 
 	for range b.N {
-		if _, err := rd.Seek(0, io.SeekStart); err != nil {
-			b.Fatal(err)
-		}
-
-		spec, err := loadRawSpec(rd, binary.LittleEndian, nil)
+		spec, err := loadRawSpec(vmlinux, nil)
 		if err != nil {
 			b.Fatal("Can't load BTF:", err)
 		}
@@ -325,13 +315,6 @@ func TestVerifierError(t *testing.T) {
 	}
 }
 
-func TestGuessBTFByteOrder(t *testing.T) {
-	bo := guessRawBTFByteOrder(vmlinuxTestdataReader(t))
-	if bo != binary.LittleEndian {
-		t.Fatalf("Guessed %s instead of %s", bo, binary.LittleEndian)
-	}
-}
-
 func TestSpecCopy(t *testing.T) {
 	qt.Check(t, qt.IsNil((*Spec)(nil).Copy()))
 

btf/btf_types.go

@@ -4,10 +4,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
-	"io"
 	"unsafe"
-
-	"github.com/cilium/ebpf/internal"
 )
 
 //go:generate go run golang.org/x/tools/cmd/stringer@latest -linecomment -output=btf_types_string.go -type=FuncLinkage,VarLinkage,btfKind
@@ -87,47 +84,49 @@ type btfHeader struct {
 	StringLen uint32
 }
 
-// typeStart returns the offset from the beginning of the .BTF section
-// to the start of its type entries.
-func (h *btfHeader) typeStart() int64 {
-	return int64(h.HdrLen + h.TypeOff)
-}
-
-// stringStart returns the offset from the beginning of the .BTF section
-// to the start of its string table.
-func (h *btfHeader) stringStart() int64 {
-	return int64(h.HdrLen + h.StringOff)
-}
-
 // parseBTFHeader parses the header of the .BTF section.
-func parseBTFHeader(r io.Reader, bo binary.ByteOrder) (*btfHeader, error) {
+func parseBTFHeader(buf []byte) (*btfHeader, binary.ByteOrder, error) {
 	var header btfHeader
-	if err := binary.Read(r, bo, &header); err != nil {
-		return nil, fmt.Errorf("can't read header: %v", err)
+	var bo binary.ByteOrder
+	for _, order := range []binary.ByteOrder{binary.LittleEndian, binary.BigEndian} {
+		n, err := binary.Decode(buf, order, &header)
+		if err != nil {
+			return nil, nil, fmt.Errorf("read header: %v", err)
+		}
+
+		if header.Magic != btfMagic {
+			continue
+		}
+
+		buf = buf[n:]
+		bo = order
+		break
 	}
 
-	if header.Magic != btfMagic {
-		return nil, fmt.Errorf("incorrect magic value %v", header.Magic)
+	if bo == nil {
+		return nil, nil, fmt.Errorf("no valid BTF header")
 	}
 
 	if header.Version != 1 {
-		return nil, fmt.Errorf("unexpected version %v", header.Version)
+		return nil, nil, fmt.Errorf("unexpected version %v", header.Version)
 	}
 
 	if header.Flags != 0 {
-		return nil, fmt.Errorf("unsupported flags %v", header.Flags)
+		return nil, nil, fmt.Errorf("unsupported flags %v", header.Flags)
 	}
 
 	remainder := int64(header.HdrLen) - int64(binary.Size(&header))
 	if remainder < 0 {
-		return nil, errors.New("header length shorter than btfHeader size")
+		return nil, nil, errors.New("header length shorter than btfHeader size")
 	}
 
-	if _, err := io.CopyN(internal.DiscardZeroes{}, r, remainder); err != nil {
-		return nil, fmt.Errorf("header padding: %v", err)
+	for _, b := range buf[:remainder] {
+		if b != 0 {
+			return nil, nil, errors.New("header contains non-zero trailer")
+		}
 	}
 
-	return &header, nil
+	return &header, bo, nil
 }
 
 // btfType is equivalent to struct btf_type in Documentation/bpf/btf.rst.

btf/fuzz_test.go

@@ -20,7 +20,7 @@ func FuzzSpec(f *testing.F) {
 			t.Skip("data is too short")
 		}
 
-		spec, err := loadRawSpec(bytes.NewReader(data), internal.NativeEndian, nil)
+		spec, err := loadRawSpec(data, nil)
 		if err != nil {
 			if spec != nil {
 				t.Fatal("spec is not nil")

btf/handle.go

@@ -1,7 +1,6 @@
 package btf
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"math"
@@ -153,7 +152,7 @@ func (h *Handle) Spec(base *Spec) (*Spec, error) {
 		return nil, fmt.Errorf("missing base types")
 	}
 
-	return loadRawSpec(bytes.NewReader(btfBuffer), internal.NativeEndian, base)
+	return loadRawSpec(btfBuffer, base)
 }
 
 // Close destroys the handle.