Add fuse-manager

Signed-off-by: Zuti He <ilyeeelihe@gmail.com>

Makefile

@@ -23,7 +23,7 @@ VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
 REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
 GO_LD_FLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) $(GO_EXTRA_LDFLAGS)'
 
-CMD=containerd-stargz-grpc ctr-remote stargz-store
+CMD=containerd-stargz-grpc ctr-remote stargz-store stargz-fuse-manager
 
 CMD_BINARIES=$(addprefix $(PREFIX),$(CMD))
 
@@ -44,6 +44,9 @@ ctr-remote: FORCE
 stargz-store: FORCE
 	GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/stargz-store
 
+stargz-fuse-manager: FORCE
+	GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/stargz-fuse-manager
+
 check:
 	@echo "$@"
 	@GO111MODULE=$(GO111MODULE_VALUE) golangci-lint run

cmd/containerd-stargz-grpc/main.go

@@ -24,6 +24,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path/filepath"
 	"time"
@@ -34,11 +35,13 @@ import (
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/pkg/dialer"
 	"github.com/containerd/containerd/snapshots"
+	"github.com/containerd/stargz-snapshotter/fusemanager"
 	"github.com/containerd/stargz-snapshotter/service"
 	"github.com/containerd/stargz-snapshotter/service/keychain/cri"
 	"github.com/containerd/stargz-snapshotter/service/keychain/dockerconfig"
 	"github.com/containerd/stargz-snapshotter/service/keychain/kubeconfig"
 	"github.com/containerd/stargz-snapshotter/service/resolver"
+	snbase "github.com/containerd/stargz-snapshotter/snapshot"
 	"github.com/containerd/stargz-snapshotter/version"
 	sddaemon "github.com/coreos/go-systemd/v22/daemon"
 	metrics "github.com/docker/go-metrics"
@@ -57,21 +60,32 @@ const (
 	defaultLogLevel            = logrus.InfoLevel
 	defaultRootDir             = "/var/lib/containerd-stargz-grpc"
 	defaultImageServiceAddress = "/run/containerd/containerd.sock"
+	defaultFuseManagerAddress  = "/run/containerd-stargz-grpc/fuse-namanger.sock"
+
+	fuseManagerBin     = "stargz-fuse-manager"
+	fuseManagerAddress = "fuse-mananger.sock"
 )
 
 var (
-	address      = flag.String("address", defaultAddress, "address for the snapshotter's GRPC server")
-	configPath   = flag.String("config", defaultConfigPath, "path to the configuration file")
-	logLevel     = flag.String("log-level", defaultLogLevel.String(), "set the logging level [trace, debug, info, warn, error, fatal, panic]")
-	rootDir      = flag.String("root", defaultRootDir, "path to the root directory for this snapshotter")
-	printVersion = flag.Bool("version", false, "print the version")
+	address           = flag.String("address", defaultAddress, "address for the snapshotter's GRPC server")
+	configPath        = flag.String("config", defaultConfigPath, "path to the configuration file")
+	logLevel          = flag.String("log-level", defaultLogLevel.String(), "set the logging level [trace, debug, info, warn, error, fatal, panic]")
+	rootDir           = flag.String("root", defaultRootDir, "path to the root directory for this snapshotter")
+	detachFuseManager = flag.Bool("detach-fuse-manager", false, "whether detach fusemanager or not")
+	printVersion      = flag.Bool("version", false, "print the version")
 )
 
 type snapshotterConfig struct {
 	service.Config
 
 	// MetricsAddress is address for the metrics API
 	MetricsAddress string `toml:"metrics_address"`
+
+	// FuseManagerAddress is address for the fusemanager's GRPC server
+	FuseManagerAddress string `toml:"fusemanager_address"`
+
+	// FuseManagerPath is path to the fusemanager's executable
+	FuseManagerPath string `toml:"fusemanager_path"`
 }
 
 func main() {
@@ -153,9 +167,44 @@ func main() {
 		runtime.RegisterImageServiceServer(rpc, criServer)
 		credsFuncs = append(credsFuncs, f)
 	}
-	rs, err := service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config, service.WithCredsFuncs(credsFuncs...))
-	if err != nil {
-		log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
+
+	var rs snapshots.Snapshotter
+	if *detachFuseManager {
+		fmPath := config.FuseManagerPath
+		if fmPath == "" {
+			var err error
+			fmPath, err = exec.LookPath(fuseManagerBin)
+			if err != nil {
+				log.G(ctx).WithError(err).Fatalf("failed to find fusemanager bin")
+			}
+		}
+		fmAddr := config.FuseManagerAddress
+		if fmAddr == "" {
+			var err error
+			fmAddr, err = exec.LookPath(fuseManagerAddress)
+			if err != nil {
+				fmAddr = defaultFuseManagerAddress
+			}
+		}
+		err := service.StartFuseManager(ctx, fmPath, fmAddr, filepath.Join(*rootDir, "fusestore.db"), *logLevel, filepath.Join(*rootDir, "stargz-fuse-manager.log"))
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to start fusemanager")
+		}
+		fs, err := fusemanager.NewManagerClient(ctx, *rootDir, fmAddr, &config.Config)
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure fusemanager")
+		}
+		rs, err = snbase.NewSnapshotter(ctx, filepath.Join(*rootDir, "snapshotter"), fs, snbase.AsynchronousRemove)
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
+		}
+		log.G(ctx).Infof("Start snapshotter with fusemanager mode")
+	} else {
+		rs, err = service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config, service.WithCredsFuncs(credsFuncs...))
+		if err != nil {
+			log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
+		}
+		log.G(ctx).Infof("Start snapshotter with builtin mode")
 	}
 
 	cleanup, err := serve(ctx, rpc, *address, rs, config)

cmd/stargz-fuse-manager/main.go

@@ -0,0 +1,25 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	fusemanager "github.com/containerd/stargz-snapshotter/fusemanager"
+)
+
+func main() {
+	fusemanager.Run()
+}

fs/fs.go

@@ -68,6 +68,13 @@ const (
 	fusermountBin         = "fusermount"
 )
 
+var (
+	nsLock = sync.Mutex{}
+
+	ns         *metrics.Namespace
+	metricsCtr *fsmetrics.Controller
+)
+
 type Option func(*options)
 
 type options struct {
@@ -101,13 +108,19 @@ func NewFilesystem(root string, cfg config.Config, opts ...Option) (_ snapshot.F
 		return nil, errors.Wrapf(err, "failed to setup resolver")
 	}
 
-	var ns *metrics.Namespace
-	if !cfg.NoPrometheus {
-		ns = metrics.NewNamespace("stargz", "fs", nil)
-	}
-	c := fsmetrics.NewLayerMetrics(ns)
-	if ns != nil {
-		metrics.Register(ns)
+	nsLock.Lock()
+	defer nsLock.Unlock()
+
+	var c *fsmetrics.Controller
+	if cfg.NoPrometheus {
+		c = fsmetrics.NewLayerMetrics(nil)
+	} else {
+		if ns == nil {
+			ns = metrics.NewNamespace("stargz", "fs", nil)
+			metrics.Register(ns)
+			metricsCtr = fsmetrics.NewLayerMetrics(ns)
+		}
+		c = metricsCtr
 	}
 
 	return &filesystem{