Zdb deploying fix

pkg/flist/cleanup.go

@@ -10,6 +10,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zosbase/pkg/app"
+	"golang.org/x/sys/unix"
 )
 
 // Cleaner interface, implementer of this interface
@@ -134,12 +135,27 @@ func (f *flistModule) cleanUnusedMounts() error {
 
 	for _, entry := range entries {
 		path := filepath.Join(f.mountpoint, entry.Name())
-		if err := f.isMountpoint(path); err == nil {
+
+		// First check if it's a mountpoint - clean non-mountpoints as before
+		if err := f.isMountpoint(path); err != nil {
+			if err := os.Remove(path); err != nil {
+				log.Error().Err(err).Msgf("failed to clean mountpoint %s", path)
+			}
 			continue
 		}
 
-		if err := os.Remove(path); err != nil {
-			log.Error().Err(err).Msgf("failed to clean mountpoint %s", path)
+		// New check: If it's a mountpoint, verify it's not empty
+		contents, readErr := os.ReadDir(path)
+		if readErr == nil && len(contents) == 0 {
+			log.Debug().Str("path", path).Msg("cleaning empty mountpoint")
+			if unmountErr := f.system.Unmount(path, unix.MNT_DETACH|unix.MNT_FORCE); unmountErr != nil {
+				log.Error().Err(unmountErr).Str("path", path).Msg("failed to unmount empty mountpoint")
+				continue
+			}
+
+			if removeErr := os.RemoveAll(path); removeErr != nil {
+				log.Error().Err(removeErr).Str("path", path).Msg("failed to clean empty mountpoint")
+			}
 		}
 	}
 

pkg/flist/flist.go

@@ -27,6 +27,7 @@ import (
 	"github.com/threefoldtech/zosbase/pkg/kernel"
 	"github.com/threefoldtech/zosbase/pkg/network/namespace"
 	"github.com/threefoldtech/zosbase/pkg/stubs"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -430,11 +431,9 @@ func (f *flistModule) mountInNamespace(name, url string, opt pkg.MountOptions, n
 	sublog := log.With().Str("name", name).Str("url", url).Str("storage", opt.Storage).Logger()
 	sublog.Info().Msgf("request to mount flist: %+v", opt)
 
-	defer func() {
-		if err := f.cleanUnusedMounts(); err != nil {
-			log.Error().Err(err).Msg("failed to run clean up")
-		}
-	}()
+	if err := f.cleanUnusedMounts(); err != nil {
+		log.Error().Err(err).Msg("failed to run clean up")
+	}
 	// mount overlay
 	mountpoint, err := f.mountpath(name)
 	if err != nil {
@@ -599,7 +598,7 @@ func (f *flistModule) Unmount(name string) error {
 	}
 
 	if f.valid(mountpoint) == ErrAlreadyMounted {
-		if err := f.system.Unmount(mountpoint, 0); err != nil {
+		if err := f.system.Unmount(mountpoint, unix.MNT_DETACH|unix.MNT_FORCE); err != nil {
 			log.Error().Err(err).Str("path", mountpoint).Msg("fail to umount flist")
 		}
 	}

pkg/flist/flist_test.go

@@ -166,7 +166,7 @@ func TestMountUnmount(t *testing.T) {
 	os.Remove(cmder.m["pid"])
 	strg.On("VolumeDelete", mock.Anything, filepath.Base(mnt)).Return(nil)
 
-	sys.On("Unmount", mnt, 0).Return(nil)
+	sys.On("Unmount", mnt, 3).Return(nil)
 
 	err = flister.Unmount(name)
 	require.NoError(t, err)
@@ -193,7 +193,7 @@ func TestMountUnmountRO(t *testing.T) {
 	os.Remove(cmder.m["pid"])
 	strg.On("VolumeDelete", mock.Anything, filepath.Base(mnt)).Return(nil)
 
-	sys.On("Unmount", mnt, 0).Return(nil)
+	sys.On("Unmount", mnt, 3).Return(nil)
 
 	err = flister.Unmount(name)
 	require.NoError(t, err)
@@ -217,6 +217,8 @@ func TestIsolation(t *testing.T) {
 	strg.On("VolumeCreate", mock.Anything, mock.Anything, mock.Anything, uint64(256*mib)).
 		Return(backend, nil)
 
+	sys.On("Unmount", mock.Anything, 3).Return(nil)
+
 	name1 := "test1"
 	sys.On("Mount", "overlay", filepath.Join(root, "mountpoint", name1), "overlay", uintptr(syscall.MS_NOATIME), mock.Anything).Return(nil)
 

pkg/network/networker.go

@@ -355,12 +355,11 @@ func (n *networker) EnsureZDBPrepare(id string) (string, error) {
 // ZDBDestroy is the opposite of ZDPrepare, it makes sure network setup done
 // for zdb is rewind. ns param is the namespace return by the ZDBPrepare
 func (n *networker) ZDBDestroy(ns string) error {
-	panic("not implemented")
-	// if !strings.HasPrefix(ns, zdbNamespacePrefix) {
-	// 	return fmt.Errorf("invalid zdb namespace name '%s'", ns)
-	// }
+	if !strings.HasPrefix(ns, zdbNamespacePrefix) {
+		return fmt.Errorf("invalid zdb namespace name '%s'", ns)
+	}
 
-	// return n.destroy(ns)
+	return n.destroy(ns)
 }
 
 // func (n *networker) createMacVlan(iface string, master string, hw net.HardwareAddr, ips []*net.IPNet, routes []*netlink.Route, netNs ns.NetNS) error {

pkg/primitives/zdb/zdb.go

@@ -64,6 +64,29 @@ func (se *safeError) Error() string {
 func (z *tZDBContainer) DataMount() (string, error) {
 	for _, mnt := range z.Mounts {
 		if mnt.Target == zdbContainerDataMnt {
+			// Verify that this is a valid ZDB data directory by checking for expected subdirectories
+			source := mnt.Source
+			if stat, err := os.Stat(source); err != nil || !stat.IsDir() {
+				return "", fmt.Errorf("container '%s' data mount path doesn't exist or isn't a directory", z.Name)
+			}
+
+			// Check for data and index directories which should exist in a valid ZDB mount
+			requiredDirs := []string{"data", "index"}
+			valid := true
+
+			for _, dir := range requiredDirs {
+				path := filepath.Join(source, dir)
+				if stat, err := os.Stat(path); err != nil || !stat.IsDir() {
+					valid = false
+					log.Warn().Str("container", z.Name).Str("path", path).Msg("missing required ZDB directory")
+					break
+				}
+			}
+
+			if !valid {
+				return "", fmt.Errorf("container '%s' data mount doesn't appear to be a valid ZDB directory structure", z.Name)
+			}
+
 			return mnt.Source, nil
 		}
 	}
@@ -91,8 +114,31 @@ func (p *Manager) Provision(ctx context.Context, wl *gridtypes.WorkloadWithID) (
 	return res, newSafeError(err)
 }
 
+func (p *Manager) zdbCleanUp(ctx context.Context, z tZDBContainer, containerID pkg.ContainerID) {
+	var (
+		flist   = stubs.NewFlisterStub(p.zbus)
+		network = stubs.NewNetworkerStub(p.zbus)
+	)
+	rootFS, err := p.zdbRootFS(ctx)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to get zdb root fs")
+	}
+	for _, mnt := range z.Mounts {
+		if mnt.Target == zdbContainerDataMnt {
+			source := mnt.Source
+			if err := os.RemoveAll(source); err != nil {
+				log.Error().Err(err).Str("path", source).Msg("failed to delete invalid ZDB source directory")
+			}
+		}
+	}
+	if err := flist.Unmount(ctx, string(containerID)); err != nil {
+		log.Error().Err(err).Str("path", rootFS).Msgf("failed to unmount")
+	}
+	_ = network.ZDBDestroy(ctx, string(containerID))
+}
+
 func (p *Manager) zdbListContainers(ctx context.Context) (map[pkg.ContainerID]tZDBContainer, error) {
-	contmod := stubs.NewContainerModuleStub(p.zbus)
+	var contmod = stubs.NewContainerModuleStub(p.zbus)
 
 	containerIDs, err := contmod.List(ctx, zdbContainerNS)
 	if err != nil {
@@ -113,7 +159,7 @@ func (p *Manager) zdbListContainers(ctx context.Context) (map[pkg.ContainerID]tZ
 
 		if _, err = cont.DataMount(); err != nil {
 			log.Error().Err(err).Msg("failed to get data directory of zdb container")
-			continue
+			p.zdbCleanUp(ctx, cont, containerID)
 		}
 		m[containerID] = cont
 	}