link: add TCX support

Add support for the new tcx link type. This supersedes netlink based
attachment to TC ingress and egress hooks. It is the first user of
the bpf_mprog API in the kernel, which allows attaching multiple
programs to the same interface. Semantically programs are put into
a list and then executed in order. Earlier programs may prevent the
execution of later programs.

The most interesting part is a way to express where a new program
should be inserted in the list. By default, a program attached via a
new link will be appended to the list, which is equivalent to
passing the BPF_F_AFTER flag. Passing the BPF_F_BEFORE flag prepends
instead.

BEFORE and AFTER can be combined with a reference to a program or a
link to specify any position in the list. The BPF_F_LINK and BPF_F_ID
flags indicate whether the reference is to a link or a program and
whether it is in the form of an FD or a ID. The default is to
refer to a program by FD. The other combinations are as follows:

- ..._ID: Refer to a program by ID.
- ..._LINK: Refer to a link by FD.
- ..._LINK | ..._ID: Refer to a link by ID.

There is a special case which allows replacing a program with another
program by specifying BPF_F_REPLACE. It's not possible to replace a
link.

The flag behaviour is pretty subtle, so the Go API doesn't expose
it directly. Instead a new concept called an Anchor is introduced,
which bundles a position with a reference to a program or link.
A couple of constructors are used to create only valid combinations.

bpf_mprog also introduces the concept of a revision which changes
with each modification of the list of attached programs. User space
can pass an expected revision when creating a new link. The kernel
then rejects the update if the revision has changed.

Breaking change: this commit removes RawAttachProgramOptions.Replace.
The fix is to set RawAttachProgramOptions.Anchor to ReplaceProgram(...).

Signed-off-by: Lorenz Bauer <lmb@isovalent.com>

internal/sys/syscall.go

@@ -139,6 +139,17 @@ const (
 	BPF_F_MMAPABLE
 	BPF_F_PRESERVE_ELEMS
 	BPF_F_INNER_MAP
+	BPF_F_LINK
+	BPF_F_PATH_FD
+)
+
+// Flags used by bpf_mprog.
+const (
+	BPF_F_REPLACE = 1 << (iota + 2)
+	BPF_F_BEFORE
+	BPF_F_AFTER
+	BPF_F_ID
+	BPF_F_LINK_MPROG = 1 << 13 // aka BPF_F_LINK
 )
 
 // wrappedErrno wraps syscall.Errno to prevent direct comparisons with

internal/unix/types_linux.go

@@ -25,6 +25,7 @@ const (
 	EACCES     = linux.EACCES
 	EILSEQ     = linux.EILSEQ
 	EOPNOTSUPP = linux.EOPNOTSUPP
+	ESTALE     = linux.ESTALE
 )
 
 const (

internal/unix/types_other.go

@@ -27,6 +27,7 @@ const (
 	EACCES
 	EILSEQ
 	EOPNOTSUPP
+	ESTALE
 )
 
 // Constants are distinct to avoid breaking switch statements.

link/anchor.go

@@ -0,0 +1,135 @@
+package link
+
+import (
+	"fmt"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/internal/sys"
+)
+
+const anchorFlags = sys.BPF_F_REPLACE |
+	sys.BPF_F_BEFORE |
+	sys.BPF_F_AFTER |
+	sys.BPF_F_ID |
+	sys.BPF_F_LINK_MPROG
+
+// Anchor is a reference to a link or program.
+//
+// It is used to describe where an attachment or detachment should take place
+// for link types which support multiple attachment.
+type Anchor interface {
+	// anchor returns an fd or ID and a set of flags.
+	//
+	// By default fdOrID is taken to reference a program, but BPF_F_LINK_MPROG
+	// changes this to refer to a link instead.
+	//
+	// BPF_F_BEFORE, BPF_F_AFTER, BPF_F_REPLACE modify where a link or program
+	// is attached. The default behaviour if none of these flags is specified
+	// matches BPF_F_AFTER.
+	anchor() (fdOrID, flags uint32, _ error)
+}
+
+type firstAnchor struct{}
+
+func (firstAnchor) anchor() (fdOrID, flags uint32, _ error) {
+	return 0, sys.BPF_F_BEFORE, nil
+}
+
+func First() Anchor {
+	return firstAnchor{}
+}
+
+type lastAnchor struct{}
+
+func (lastAnchor) anchor() (fdOrID, flags uint32, _ error) {
+	return 0, sys.BPF_F_AFTER, nil
+}
+
+func Last() Anchor {
+	return lastAnchor{}
+}
+
+// Before is the position just in front of target.
+func BeforeLink(target Link) Anchor {
+	return anchor{target, sys.BPF_F_BEFORE}
+}
+
+// After is the position just after target.
+func AfterLink(target Link) Anchor {
+	return anchor{target, sys.BPF_F_AFTER}
+}
+
+// Before is the position just in front of target.
+func BeforeLinkByID(target ID) Anchor {
+	return anchor{target, sys.BPF_F_BEFORE}
+}
+
+// After is the position just after target.
+func AfterLinkByID(target ID) Anchor {
+	return anchor{target, sys.BPF_F_AFTER}
+}
+
+// Before is the position just in front of target.
+func BeforeProgram(target *ebpf.Program) Anchor {
+	return anchor{target, sys.BPF_F_BEFORE}
+}
+
+// After is the position just after target.
+func AfterProgram(target *ebpf.Program) Anchor {
+	return anchor{target, sys.BPF_F_AFTER}
+}
+
+// Replace the target itself.
+func ReplaceProgram(target *ebpf.Program) Anchor {
+	return anchor{target, sys.BPF_F_REPLACE}
+}
+
+// Before is the position just in front of target.
+func BeforeProgramByID(target ebpf.ProgramID) Anchor {
+	return anchor{target, sys.BPF_F_BEFORE}
+}
+
+// After is the position just after target.
+func AfterProgramByID(target ebpf.ProgramID) Anchor {
+	return anchor{target, sys.BPF_F_AFTER}
+}
+
+// Replace the target itself.
+func ReplaceProgramByID(target ebpf.ProgramID) Anchor {
+	return anchor{target, sys.BPF_F_REPLACE}
+}
+
+type anchor struct {
+	target   any
+	position uint32
+}
+
+func (ap anchor) anchor() (fdOrID, flags uint32, _ error) {
+	var typeFlag uint32
+	switch target := ap.target.(type) {
+	case *ebpf.Program:
+		fd := target.FD()
+		if fd < 0 {
+			return 0, 0, sys.ErrClosedFd
+		}
+		fdOrID = uint32(fd)
+		typeFlag = 0
+	case ebpf.ProgramID:
+		fdOrID = uint32(target)
+		typeFlag = sys.BPF_F_ID
+	case interface{ FD() int }:
+		fd := target.FD()
+		if fd < 0 {
+			return 0, 0, sys.ErrClosedFd
+		}
+		fdOrID = uint32(fd)
+		typeFlag = sys.BPF_F_LINK_MPROG
+	case ID:
+		fdOrID = uint32(target)
+		typeFlag = sys.BPF_F_LINK_MPROG | sys.BPF_F_ID
+	default:
+		return 0, 0, fmt.Errorf("invalid target %T", ap.target)
+	}
+
+	return fdOrID, ap.position | typeFlag, nil
+}

link/cgroup.go

@@ -143,8 +143,7 @@ func (cg *progAttachCgroup) Update(prog *ebpf.Program) error {
 		// Atomically replacing multiple programs requires at least
 		// 5.5 (commit 7dd68b3279f17921 "bpf: Support replacing cgroup-bpf
 		// program in MULTI mode")
-		args.Flags |= uint32(flagReplace)
-		args.Replace = cg.current
+		args.Anchor = ReplaceProgram(cg.current)
 	}
 
 	if err := RawAttachProgram(args); err != nil {

link/link.go

@@ -98,6 +98,8 @@ func wrapRawLink(raw *RawLink) (_ Link, err error) {
 		return &kprobeMultiLink{*raw}, nil
 	case PerfEventType:
 		return nil, fmt.Errorf("recovering perf event fd: %w", ErrNotSupported)
+	case TCXType:
+		return &tcxLink{*raw}, nil
 	default:
 		return raw, nil
 	}
@@ -132,6 +134,7 @@ type TracingInfo sys.TracingLinkInfo
 type CgroupInfo sys.CgroupLinkInfo
 type NetNsInfo sys.NetNsLinkInfo
 type XDPInfo sys.XDPLinkInfo
+type TCXInfo sys.TcxLinkInfo
 
 // Tracing returns tracing type-specific link info.
 //
@@ -315,6 +318,8 @@ func (l *RawLink) Info() (*Info, error) {
 	case RawTracepointType, IterType,
 		PerfEventType, KprobeMultiType:
 		// Extra metadata not supported.
+	case TCXType:
+		extra = &TCXInfo{}
 	default:
 		return nil, fmt.Errorf("unknown link info type: %d", info.Type)
 	}

link/program.go

@@ -2,68 +2,100 @@ package link
 
 import (
 	"fmt"
+	"runtime"
 
 	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/internal/sys"
 )
 
 type RawAttachProgramOptions struct {
-	// File descriptor to attach to. This differs for each attach type.
+	// Target to query. This is usually a file descriptor but may refer to
+	// something else based on the attach type.
 	Target int
 	// Program to attach.
 	Program *ebpf.Program
-	// Program to replace (cgroups).
-	Replace *ebpf.Program
-	// Attach must match the attach type of Program (and Replace).
+	// Attach must match the attach type of Program.
 	Attach ebpf.AttachType
-	// Flags control the attach behaviour. This differs for each attach type.
+	// Attach relative to an anchor. Optional.
+	Anchor Anchor
+	// Flags control the attach behaviour. Specify an Anchor instead of
+	// F_LINK, F_ID, F_BEFORE, F_AFTER and F_REPLACE. Optional.
 	Flags uint32
+	// Only attach if the internal revision matches the given value.
+	ExpectedRevision uint64
 }
 
 // RawAttachProgram is a low level wrapper around BPF_PROG_ATTACH.
 //
 // You should use one of the higher level abstractions available in this
 // package if possible.
 func RawAttachProgram(opts RawAttachProgramOptions) error {
-	var replaceFd uint32
-	if opts.Replace != nil {
-		replaceFd = uint32(opts.Replace.FD())
+	if opts.Flags&anchorFlags != 0 {
+		return fmt.Errorf("disallowed flags: use Anchor to specify attach target")
 	}
 
 	attr := sys.ProgAttachAttr{
 		TargetFdOrIfindex: uint32(opts.Target),
 		AttachBpfFd:       uint32(opts.Program.FD()),
-		ReplaceBpfFd:      replaceFd,
 		AttachType:        uint32(opts.Attach),
 		AttachFlags:       uint32(opts.Flags),
+		ExpectedRevision:  opts.ExpectedRevision,
+	}
+
+	if opts.Anchor != nil {
+		fdOrID, flags, err := opts.Anchor.anchor()
+		if err != nil {
+			return fmt.Errorf("attach program: %w", err)
+		}
+
+		if flags == sys.BPF_F_REPLACE {
+			// Ensure that replacing a program works on old kernels.
+			attr.ReplaceBpfFd = fdOrID
+		} else {
+			attr.RelativeFdOrId = fdOrID
+			attr.AttachFlags |= flags
+		}
 	}
 
 	if err := sys.ProgAttach(&attr); err != nil {
 		if haveFeatErr := haveProgAttach(); haveFeatErr != nil {
 			return haveFeatErr
 		}
-		return fmt.Errorf("can't attach program: %w", err)
+		return fmt.Errorf("attach program: %w", err)
 	}
+	runtime.KeepAlive(opts.Program)
 
 	return nil
 }
 
-type RawDetachProgramOptions struct {
-	Target  int
-	Program *ebpf.Program
-	Attach  ebpf.AttachType
-}
+type RawDetachProgramOptions RawAttachProgramOptions
 
 // RawDetachProgram is a low level wrapper around BPF_PROG_DETACH.
 //
 // You should use one of the higher level abstractions available in this
 // package if possible.
 func RawDetachProgram(opts RawDetachProgramOptions) error {
+	if opts.Flags&anchorFlags != 0 {
+		return fmt.Errorf("disallowed flags: use Anchor to specify attach target")
+	}
+
 	attr := sys.ProgDetachAttr{
 		TargetFdOrIfindex: uint32(opts.Target),
 		AttachBpfFd:       uint32(opts.Program.FD()),
 		AttachType:        uint32(opts.Attach),
+		ExpectedRevision:  opts.ExpectedRevision,
 	}
+
+	if opts.Anchor != nil {
+		fdOrID, flags, err := opts.Anchor.anchor()
+		if err != nil {
+			return fmt.Errorf("detach program: %w", err)
+		}
+
+		attr.RelativeFdOrId = fdOrID
+		attr.AttachFlags |= flags
+	}
+
 	if err := sys.ProgDetach(&attr); err != nil {
 		if haveFeatErr := haveProgAttach(); haveFeatErr != nil {
 			return haveFeatErr