Skip to content

Commit

Permalink
Merge pull request #2227 from fasaxc/v3.13-gc-maps
Browse files Browse the repository at this point in the history
[v3.13] Clean up jump maps
  • Loading branch information
fasaxc authored Mar 4, 2020
2 parents ffd3291 + 060d045 commit 30c47af
Show file tree
Hide file tree
Showing 3 changed files with 305 additions and 22 deletions.
196 changes: 174 additions & 22 deletions bpf/tc/attach.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package tc

import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
Expand All @@ -26,11 +27,15 @@ import (
"os/exec"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"

"github.com/pkg/errors"
"github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"

"github.com/projectcalico/libcalico-go/lib/set"

"github.com/projectcalico/felix/bpf"
)
Expand All @@ -56,10 +61,12 @@ func (e ErrAttachFailed) Error() string {
// AttachProgram attaches a BPF program from a file to the TC attach point
func AttachProgram(attachPoint AttachPoint, hostIP net.IP) error {
// FIXME we use this lock so that two copies of tc running in parallel don't re-use the same jump map.
// This can happen if tc incorrectly decides the two programs are identical (when if fact they differ by attach
// This can happen if tc incorrectly decides the two programs are identical (when in fact they differ by attach
// point).
log.Debug("AttachProgram waiting for lock...")
tcLock.Lock()
defer tcLock.Unlock()
log.Debug("AttachProgram got lock.")

// Work around tc map name collision: when we load two identical BPF programs onto different interfaces, tc
// pins object-local maps to a namespace based on the hash of the BPF program, which is the same for both
Expand Down Expand Up @@ -88,7 +95,7 @@ func AttachProgram(attachPoint AttachPoint, hostIP net.IP) error {

hostIP = hostIP.To4()
if len(hostIP) == 4 {
logrus.WithField("ip", hostIP).Debug("Patching in host IP")
log.WithField("ip", hostIP).Debug("Patching in host IP")
replacement := make([]byte, 6)
copy(replacement[2:], hostIP)
exeData = bytes.ReplaceAll(exeData, []byte("\x00\x00HOST"), replacement)
Expand Down Expand Up @@ -118,11 +125,11 @@ func AttachProgram(attachPoint AttachPoint, hostIP net.IP) error {
if err != nil {
if strings.Contains(err.Error(), "Cannot find device") {
// Avoid a big, spammy log when the issue is that the interface isn't present.
logrus.WithField("iface", attachPoint.Iface).Info(
log.WithField("iface", attachPoint.Iface).Info(
"Failed to attach BPF program; interface not found. Will retry if it show up.")
return nil
}
logrus.WithError(err).WithFields(logrus.Fields{"out": string(out)}).
log.WithError(err).WithFields(log.Fields{"out": string(out)}).
WithField("command", tcCmd).Error("Failed to attach BPF program")
if err, ok := err.(*exec.ExitError); ok {
// ExitError is really unhelpful dumped to the log, swap it for a custom one.
Expand All @@ -144,49 +151,194 @@ func repinJumpMaps() {
return err
}
if info.Name() == "cali_jump" {
logrus.WithField("path", path).Debug("Queueing deletion of map")
log.WithField("path", path).Debug("Queueing deletion of map")

out, err := exec.Command("bpftool", "map", "dump", "pinned", path).Output()
if err != nil {
logrus.WithError(err).Panic("Failed to dump map")
if log.GetLevel() >= log.DebugLevel {
out, err := exec.Command("bpftool", "map", "dump", "pinned", path).Output()
if err != nil {
log.WithError(err).Panic("Failed to dump map")
}
log.WithField("dump", string(out)).Debug("Map dump before deletion")
}
logrus.WithField("dump", string(out)).Info("Map dump before deletion")

out, err = exec.Command("bpftool", "map", "show", "pinned", path).Output()
out, err := exec.Command("bpftool", "map", "show", "pinned", path).Output()
if err != nil {
logrus.WithError(err).Panic("Failed to show map")
log.WithError(err).Panic("Failed to show map")
}
logrus.WithField("dump", string(out)).Info("Map show before deletion")
log.WithField("dump", string(out)).Debug("Map show before deletion")
id := string(bytes.Split(out, []byte(":"))[0])

newPath := path + fmt.Sprint(rand.Uint32())
out, err = exec.Command("bpftool", "map", "pin", "id", id, newPath).Output()
if err != nil {
logrus.WithError(err).Panic("Failed to repin map")
log.WithError(err).Panic("Failed to repin map")
}
logrus.WithField("dump", string(out)).Debug("Repin output")
log.WithField("dump", string(out)).Debug("Repin output")

err = os.Remove(path)
if err != nil {
logrus.WithError(err).Panic("Failed to remove old map pin")
log.WithError(err).Panic("Failed to remove old map pin")
}

if log.GetLevel() >= log.DebugLevel {
out, err = exec.Command("bpftool", "map", "dump", "pinned", newPath).Output()
if err != nil {
log.WithError(err).Panic("Failed to show map")
}
log.WithField("dump", string(out)).Debug("Map show after repin")
}
}
return nil
})
if os.IsNotExist(err) {
log.WithError(err).Warn("tc directory missing from BPF file system?")
return
}
if err != nil {
log.WithError(err).Panic("Failed to walk BPF filesystem")
}
log.Debug("Finished moving map pins that we don't need.")
}

// tcDirRegex matches tc's auto-created directory names so we can clean them up when removing maps without accidentally
// removing other user-created dirs..
var tcDirRegex = regexp.MustCompile(`[0-9a-f]{40}`)

// CleanUpJumpMaps scans for cali_jump maps that are still pinned to the filesystem but no longer referenced by
// our BPF programs.
func CleanUpJumpMaps() {
// So that we serialise with AttachProgram()
log.Debug("CleanUpJumpMaps waiting for lock...")
tcLock.Lock()
defer tcLock.Unlock()
log.Debug("CleanUpJumpMaps got lock, cleaning up...")

// Find the maps we care about by walking the BPF filesystem.
mapIDToPath := make(map[int]string)
err := filepath.Walk("/sys/fs/bpf/tc", func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if strings.HasPrefix(info.Name(), "cali_jump") {
log.WithField("path", p).Debug("Examining map")

out, err = exec.Command("bpftool", "map", "dump", "pinned", newPath).Output()
out, err := exec.Command("bpftool", "map", "show", "pinned", p).Output()
if err != nil {
log.WithError(err).Panic("Failed to show map")
}
log.WithField("dump", string(out)).Debug("Map show before deletion")
idStr := string(bytes.Split(out, []byte(":"))[0])
id, err := strconv.Atoi(idStr)
if err != nil {
logrus.WithError(err).Panic("Failed to show map")
log.WithError(err).WithField("dump", string(out)).Error("Failed to parse bpftool output.")
return err
}
logrus.WithField("dump", string(out)).Info("Map show after repin")
mapIDToPath[id] = p
}
return nil
})
if os.IsNotExist(err) {
log.WithError(err).Warn("tc directory missing from BPF file system?")
return
}
if err != nil {
log.WithError(err).Error("Error while looking for maps.")
}

// Find all the programs that are attached to interfaces.
out, err := exec.Command("bpftool", "net", "-j").Output()
if err != nil {
log.WithError(err).Panic("Failed to list attached bpf programs")
}
log.WithField("dump", string(out)).Debug("Attached BPF programs")

var attached []struct {
TC []struct {
DevName string `json:"devname"`
ID int `json:"id"`
} `json:"tc"`
}
err = json.Unmarshal(out, &attached)
if err != nil {
log.WithError(err).WithField("dump", string(out)).Error("Failed to parse list of attached BPF programs")
}
attachedProgs := set.New()
for _, prog := range attached[0].TC {
log.WithField("prog", prog).Debug("Adding prog to attached set")
attachedProgs.Add(prog.ID)
}

// Find all the maps that the attached programs refer to and remove them from consideration.
progsJSON, err := exec.Command("bpftool", "prog", "list", "--json").Output()
if err != nil {
log.WithError(err).Info("Failed to list BPF programs, assuming there's nothing to clean up.")
return
}
var progs []struct {
ID int `json:"id"`
Name string `json:"name"`
Maps []int `json:"map_ids"`
}
err = json.Unmarshal(progsJSON, &progs)
if err != nil {
log.WithError(err).Info("Failed to parse bpftool output. Assuming nothing to clean up.")
return
}
for _, p := range progs {
if !attachedProgs.Contains(p.ID) {
log.WithField("prog", p).Debug("Prog is not in the attached set, skipping")
continue
}
for _, id := range p.Maps {
log.WithField("mapID", id).WithField("prog", p).Debug("Map is still in use")
delete(mapIDToPath, id)
}
}

// Remove the pins.
for id, p := range mapIDToPath {
log.WithFields(log.Fields{"id": id, "path": p}).Debug("Removing stale BPF map pin.")
err := os.Remove(p)
if err != nil {
log.WithError(err).Warn("Removed stale BPF map pin.")
}
log.WithFields(log.Fields{"id": id, "path": p}).Info("Removed stale BPF map pin.")
}

// Look for empty dirs.
emptyAutoDirs := set.New()
err = filepath.Walk("/sys/fs/bpf/tc", func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() && tcDirRegex.MatchString(info.Name()){
p := path.Clean(p)
log.WithField("path", p).Debug("Found tc auto-created dir.")
emptyAutoDirs.Add(p)
} else {
dirPath := path.Clean(path.Dir(p))
log.WithField("path", dirPath).Debug("tc dir is not empty.")
emptyAutoDirs.Discard(dirPath)
}
return nil
})
if os.IsNotExist(err) {
logrus.WithError(err).Warn("tc directory missing from BPF file system?")
log.WithError(err).Warn("tc directory missing from BPF file system?")
return
}
if err != nil {
logrus.WithError(err).Panic("Failed to walk BPF filesystem")
log.WithError(err).Error("Error while looking for maps.")
}
logrus.Debug("Finished moving map pins that we don't need.")

emptyAutoDirs.Iter(func(item interface{}) error {
p := item.(string)
log.WithField("path", p).Debug("Removing empty dir.")
err := os.Remove(p)
if err != nil {
log.WithError(err).Error("Error while removing empty dir.")
}
return nil
})
}

// EnsureQdisc makes sure that qdisc is attached to the given interface
Expand Down
125 changes: 125 additions & 0 deletions bpf/ut/attach_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
// Copyright (c) 2020 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ut

import (
"net"
"os"
"path/filepath"
"strings"
"testing"

. "github.com/onsi/gomega"
log "github.com/sirupsen/logrus"

"github.com/projectcalico/felix/bpf"
"github.com/projectcalico/felix/bpf/tc"
)

func TestJumpMapCleanup(t *testing.T) {
RegisterTestingT(t)

bpffs, err := bpf.MaybeMountBPFfs()
Expect(err).NotTo(HaveOccurred())
Expect(bpffs).To(Equal("/sys/fs/bpf"))

secName := tc.SectionName(tc.EpTypeWorkload, tc.ToEp)
prog := tc.ProgFilename(tc.EpTypeWorkload, tc.ToEp, false, true, true, "DEBUG")

t.Run(prog, func(t *testing.T) {
RegisterTestingT(t)
log.Debugf("Testing %v in %v", secName, prog)

vethName, veth := createVeth()
defer deleteLink(veth)

tc.EnsureQdisc(vethName)
ap := tc.AttachPoint{
Section: secName,
Hook: tc.HookIngress,
Iface: vethName,
Filename: prog,
}

// Start with a clean base state in case another test left something behind.
t.Log("Doing initial clean up")
tc.CleanUpJumpMaps()

t.Log("Adding program, should add one dir and one map.")
startingJumpMaps := countJumpMaps()
startingTCDirs := countTCDirs()
err := tc.AttachProgram(ap, net.ParseIP("10.0.0.1"))
Expect(err).NotTo(HaveOccurred())
Expect(countJumpMaps()).To(BeNumerically("==", startingJumpMaps+1), "unexpected number of jump maps")
Expect(countTCDirs()).To(BeNumerically("==", startingTCDirs+1), "unexpected number of TC dirs")

t.Log("Replacing program should add another map and dir.")
tc.EnsureQdisc(vethName)
err = tc.AttachProgram(ap, net.ParseIP("10.0.0.2"))
Expect(err).NotTo(HaveOccurred())
Expect(countJumpMaps()).To(BeNumerically("==", startingJumpMaps+2), "unexpected number of jump maps after replacing program")
Expect(countTCDirs()).To(BeNumerically("==", startingTCDirs+2), "unexpected number of TC dirs after replacing program")

t.Log("Cleaning up, should remove the first map.")
tc.CleanUpJumpMaps()
Expect(countJumpMaps()).To(BeNumerically("==", startingJumpMaps+1), "unexpected number of jump maps after clean up")
Expect(countTCDirs()).To(BeNumerically("==", startingTCDirs+1), "unexpected number of TC dirs after clean up")

// Remove the program.
t.Log("Removing all programs and cleaning up, should return to base state.")
tc.EnsureQdisc(vethName)
tc.CleanUpJumpMaps()
Expect(countJumpMaps()).To(BeNumerically("==", startingJumpMaps), "unexpected number of jump maps")
Expect(countTCDirs()).To(BeNumerically("==", startingTCDirs), "unexpected number of TC dirs")
})
}

func countJumpMaps() int {
var count int
err := filepath.Walk("/sys/fs/bpf/tc", func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if strings.HasPrefix(info.Name(), "cali_jump") {
log.Debugf("Jump map: %s", p)
count++
}
return nil
})

if err != nil {
panic(err)
}
return count
}

func countTCDirs() int {
var count int
err := filepath.Walk("/sys/fs/bpf/tc", func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() && len(info.Name()) == 40 {
log.Debugf("TC dir: %s", p)
count++
}
return nil
})

if err != nil {
panic(err)
}
return count
}
Loading

0 comments on commit 30c47af

Please sign in to comment.