DeviceManager: Added unit tests

Tests for both device managers: direct and LVM.

LVM tests used file backed loop device to mimic an logical volume group, This
requires /dev/loop-control file on host where the tests are run.

Intentionally disable running these device manager unit tests as it requires sudo/root user
permission.

Author: avalluri

go.mod

@@ -38,6 +38,7 @@ require (
 	google.golang.org/appengine v1.6.5 // indirect
 	google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03 // indirect
 	google.golang.org/grpc v1.24.0
+	gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
 	k8s.io/api v0.0.0

go.sum

@@ -622,6 +622,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124 h1:aPcd9iBdqpFyYkoGRQbQd+asp162GIRDvAVB0FhLxhc=
+gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124/go.mod h1:6LXpUYtVsrx91XiupFRJ8jVKOqLZf5PrbEVSGHta/84=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gcfg.v1 v1.2.0 h1:0HIbH907iBTAntm+88IJV2qmJALDAh8sPekI9Vc1fm0=

pkg/pmem-device-manager/pmd-lvm.go

@@ -60,6 +60,10 @@ func NewPmemDeviceManagerLVM() (PmemDeviceManager, error) {
 	}
 	ctx.Free()
 
+	return NewPmemDeviceManagerLVMForVGs(volumeGroups)
+}
+
+func NewPmemDeviceManagerLVMForVGs(volumeGroups []string) (PmemDeviceManager, error) {
 	devices, err := listDevices(volumeGroups...)
 	if err != nil {
 		return nil, err

pkg/pmem-device-manager/pmd-manager_test.go

@@ -0,0 +1,340 @@
+/*
+Copyright 2019  Intel Corporation.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+package pmdmanager
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"testing"
+
+	pmemexec "github.com/intel/pmem-csi/pkg/pmem-exec"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	losetup "gopkg.in/freddierice/go-losetup.v1"
+)
+
+const (
+	vgname = "test-group"
+	nsmode = "fsdax"
+	vgsize = uint64(1) * 1024 * 1024 * 1024 // 1Gb
+
+	ModeLVM    = "lvm"
+	ModeDirect = "direct"
+)
+
+func TestMain(m *testing.M) {
+	RegisterFailHandler(Fail)
+
+	os.Exit(m.Run())
+}
+
+func TestPmd(t *testing.T) {
+	RunSpecs(t, "PMEM Device manager Suite")
+}
+
+var _ = Describe("DeviceManager", func() {
+	Context(ModeLVM, func() { runTests(ModeLVM) })
+	Context(ModeDirect, func() { runTests(ModeDirect) })
+})
+
+func runTests(mode string) {
+	var dm PmemDeviceManager
+	var vg *testVGS
+	var cleanupList map[string]bool
+	var err error
+
+	BeforeEach(func() {
+		precheck()
+
+		cleanupList = map[string]bool{}
+
+		if mode == ModeLVM {
+			vg, err = createTestVGS(vgname, nsmode, vgsize)
+			Expect(err).Should(BeNil(), "Failed to create volume group")
+
+			dm, err = NewPmemDeviceManagerLVMForVGs([]string{vg.name})
+		} else {
+			dm, err = NewPmemDeviceManagerNdctl()
+		}
+		Expect(err).Should(BeNil(), "Failed to create LVM device manager")
+
+	})
+
+	AfterEach(func() {
+		for devName, ok := range cleanupList {
+			if !ok {
+				continue
+			}
+			By("Cleaning up device: " + devName)
+			dm.DeleteDevice(devName, false)
+		}
+		if mode == ModeLVM {
+			err := vg.Clean()
+			Expect(err).Should(BeNil(), "Failed to create LVM device manager")
+		}
+	})
+
+	It("Should create a new device", func() {
+		name := "test-dev-new"
+		size := uint64(2) * 1024 * 1024 // 2Mb
+		err := dm.CreateDevice(name, size, nsmode)
+		Expect(err).Should(BeNil(), "Failed to create new device")
+
+		cleanupList[name] = true
+
+		dev, err := dm.GetDevice(name)
+		Expect(err).Should(BeNil(), "Failed to retrieve device info")
+		Expect(dev.VolumeId).Should(Equal(name), "Name mismatch")
+		Expect(dev.Size >= size).Should(BeTrue(), "Size mismatch")
+		Expect(dev.Path).ShouldNot(BeNil(), "Null device path")
+	})
+
+	It("Should fail to retrieve non-existent device", func() {
+		dev, err := dm.GetDevice("unknown")
+		Expect(err).ShouldNot(BeNil(), "Error expected")
+		Expect(errors.Is(err, ErrDeviceNotFound)).Should(BeTrue(), "expected error is device not found error")
+		Expect(dev).Should(BeNil(), "returned device should be nil")
+	})
+
+	It("Should list devices", func() {
+		max_devices := 4
+		max_deletes := 2
+		sizes := map[string]uint64{}
+
+		list, err := dm.ListDevices()
+		Expect(err).Should(BeNil(), "Failed to list devices")
+		Expect(len(list)).Should(BeEquivalentTo(0), "count mismatch")
+
+		for i := 1; i <= max_devices; i++ {
+			name := fmt.Sprintf("list-dev-%d", i)
+			sizes[name] = rand.Uint64() % 16 * 1024 * 1024
+			err := dm.CreateDevice(name, sizes[name], nsmode)
+			Expect(err).Should(BeNil(), "Failed to create new device")
+			cleanupList[name] = true
+		}
+		list, err = dm.ListDevices()
+		Expect(err).Should(BeNil(), "Failed to list devices")
+		Expect(len(list)).Should(BeEquivalentTo(max_devices), "count mismatch")
+		for _, dev := range list {
+			size, ok := sizes[dev.VolumeId]
+			Expect(ok).Should(BeTrue(), "Unexpected device name:"+dev.VolumeId)
+			Expect(dev.Size >= size).Should(BeTrue(), "Device size mismatch")
+		}
+
+		for i := 1; i <= max_deletes; i++ {
+			name := fmt.Sprintf("list-dev-%d", i)
+			delete(sizes, name)
+			err = dm.DeleteDevice(name, false)
+			Expect(err).Should(BeNil(), "Error while deleting device '"+name+"'")
+			cleanupList[name] = false
+		}
+
+		// List device after deleting a device
+		list, err = dm.ListDevices()
+		Expect(err).Should(BeNil(), "Failed to list devices")
+		Expect(len(list)).Should(BeEquivalentTo(max_devices-max_deletes), "count mismatch")
+		for _, dev := range list {
+			size, ok := sizes[dev.VolumeId]
+			Expect(ok).Should(BeTrue(), "Unexpected device name:"+dev.VolumeId)
+			Expect(dev.Size >= size).Should(BeTrue(), "Device size mismatch")
+		}
+	})
+
+	It("Should delete devices", func() {
+		name := "delete-dev"
+		size := uint64(2) * 1024 * 1024 // 2Mb
+		err := dm.CreateDevice(name, size, nsmode)
+		Expect(err).Should(BeNil(), "Failed to create new device")
+		cleanupList[name] = true
+
+		dev, err := dm.GetDevice(name)
+		Expect(err).Should(BeNil(), "Failed to retrieve device info")
+		Expect(dev.VolumeId).Should(Equal(name), "Name mismatch")
+		Expect(dev.Size >= size).Should(BeTrue(), "Size mismatch")
+		Expect(dev.Path).ShouldNot(BeNil(), "Null device path")
+
+		mountPath, err := mountDevice(dev)
+		Expect(err).Should(BeNil(), "Failed to create mount path: %s", mountPath)
+
+		defer unmount(mountPath)
+
+		// Delete should fail as the device is in use
+		err = dm.DeleteDevice(name, true)
+		Expect(err).ShouldNot(BeNil(), "Error expected when deleting device in use: %s", dev.VolumeId)
+		Expect(errors.Is(err, ErrDeviceInUse)).Should(BeTrue(), "Expected device busy error: %s", dev.VolumeId)
+		cleanupList[name] = false
+
+		err = unmount(mountPath)
+		Expect(err).Should(BeNil(), "Failed to unmount the device: %s", dev.VolumeId)
+
+		// Delete should succeed
+		err = dm.DeleteDevice(name, true)
+		Expect(err).Should(BeNil(), "Failed to delete device")
+
+		dev, err = dm.GetDevice(name)
+		Expect(err).ShouldNot(BeNil(), "GetDevice() should fail on deleted device")
+		Expect(errors.Is(err, ErrDeviceNotFound)).Should(BeTrue(), "expected error is os.ErrNotExist")
+		Expect(dev).Should(BeNil(), "returned device should be nil")
+
+		// Delete call should not return any error on non-existing device
+		err = dm.DeleteDevice(name, true)
+		Expect(err).Should(BeNil(), "DeleteDevice() is not idempotent")
+	})
+}
+
+func precheck() {
+	if os.Geteuid() != 0 {
+		Skip("Root privileges are required to run these tests", 1)
+	}
+
+	info, err := os.Stat(losetup.LoopControlPath)
+	if err != nil {
+		Skip(fmt.Sprintf("Stat(%s) failure: %s", losetup.LoopControlPath, err.Error()), 1)
+	}
+	if isDev := info.Mode()&os.ModeDevice != 0; !isDev {
+		Skip(fmt.Sprintf("%s is not a loop device file", losetup.LoopControlPath), 1)
+	}
+}
+
+type testVGS struct {
+	name       string
+	loopDev    losetup.Device
+	backedFile string
+}
+
+func createTestVGS(vgname, nsmode string, size uint64) (*testVGS, error) {
+	var err error
+	var file *os.File
+	var dev losetup.Device
+	var out string
+
+	By("Creating temporary file")
+	if file, err = ioutil.TempFile("", "test-lvm-dev"); err != nil {
+		By("Cleaning temporary file")
+		return nil, fmt.Errorf("Fail to create temporary file : %s", err.Error())
+	}
+
+	defer func() {
+		if err != nil && file != nil {
+			By("Removing tmp file due to failure")
+			os.Remove(file.Name())
+		}
+	}()
+
+	By("Closing file")
+	if err = file.Close(); err != nil {
+		return nil, fmt.Errorf("Fail to close file: %s", err.Error())
+	}
+
+	By("File truncating")
+	if err = os.Truncate(file.Name(), int64(size)); err != nil {
+		return nil, fmt.Errorf("Fail to truncate file: %s", err.Error())
+	}
+
+	By("losetup.Attach")
+	dev, err = losetup.Attach(file.Name(), 0, false)
+	if err != nil {
+		return nil, fmt.Errorf("losetup failure: %s", err.Error())
+	}
+
+	defer func() {
+		if err != nil {
+			By("losetup.Detach due to failure")
+			dev.Detach() // nolint errcheck
+		}
+	}()
+
+	if err = waitDeviceAppears(&PmemDeviceInfo{Path: dev.Path()}); err != nil {
+		return nil, fmt.Errorf("created loop device not appeared: %s", err.Error())
+	}
+
+	By("Creating physical volume")
+	// TODO: reuse vgm code
+	cmdArgs := []string{"--force", dev.Path()}
+	if out, err = pmemexec.RunCommand("pvcreate", cmdArgs...); err != nil { // nolint gosec
+		return nil, fmt.Errorf("pvcreate failure(output:%s): %s", out, err.Error())
+	}
+
+	By("Creating volume group")
+	cmdArgs = []string{"--force", vgname, dev.Path()}
+	if out, err = pmemexec.RunCommand("vgcreate", cmdArgs...); err != nil { // nolint gosec
+		return nil, fmt.Errorf("vgcreate failure(output:%s): %s", out, err.Error())
+	}
+
+	defer func() {
+		if err != nil {
+			By("Removing volume group due to failure")
+			pmemexec.RunCommand("vgremove", "--force", vgname)
+		}
+	}()
+
+	By("Append tag(s) to volume group")
+	if out, err = pmemexec.RunCommand("vgchange", "--addtag", string(nsmode), vgname); err != nil {
+		return nil, fmt.Errorf("vgchange failure(output:%s): %s", out, err.Error())
+	}
+
+	return &testVGS{
+		name:       vgname,
+		loopDev:    dev,
+		backedFile: file.Name(),
+	}, nil
+}
+
+func (vg *testVGS) Clean() error {
+	By("Removing volume group")
+	if out, err := pmemexec.RunCommand("vgremove", "--force", vg.name); err != nil {
+		return fmt.Errorf("Fail to remove volume group(output:%s): %s", out, err.Error())
+	}
+
+	By("losetup.Detach()")
+	if err := vg.loopDev.Detach(); err != nil {
+		return fmt.Errorf("Fail detatch loop device: %s", err.Error())
+	}
+
+	By("Removing temp file")
+	if err := os.Remove(vg.backedFile); err != nil {
+		return fmt.Errorf("Fail remove temporary file: %s", err.Error())
+	}
+
+	return nil
+}
+
+func mountDevice(device *PmemDeviceInfo) (string, error) {
+	targetPath, err := ioutil.TempDir("/tmp", "lmv-mnt-path-")
+	if err != nil {
+		return "", err
+	}
+
+	cmd := "mkfs.ext4"
+	args := []string{"-b 4096", "-F", device.Path}
+
+	if _, err := pmemexec.RunCommand(cmd, args...); err != nil {
+		os.Remove(targetPath)
+		return "", err
+	}
+
+	cmd = "mount"
+	args = []string{"-c", device.Path, targetPath}
+
+	if _, err := pmemexec.RunCommand(cmd, args...); err != nil {
+		os.Remove(targetPath)
+		return "", err
+	}
+
+	return targetPath, nil
+}
+
+func unmount(path string) error {
+	args := []string{path}
+	if _, err := pmemexec.RunCommand("umount", args...); err != nil {
+		return err
+	}
+	return os.Remove(path)
+}

test/test.make

@@ -145,7 +145,7 @@ test_e2e: start
 .PHONY: run_tests
 test: run_tests
 RUN_TESTS = TEST_WORK=$(abspath _work) \
-	$(TEST_CMD) $(shell $(GO) list $(TEST_ARGS) | sed -e 's;$(IMPORT_PATH);.;')
+	$(TEST_CMD) $(shell $(GO) list $(TEST_ARGS) | grep -v pmem-device-manager | sed -e 's;$(IMPORT_PATH);.;')
 run_tests: _work/pmem-ca/.ca-stamp _work/evil-ca/.ca-stamp check-go-version-$(GO_BINARY)
 	$(RUN_TESTS)
 

vendor-bom.csv

@@ -55,6 +55,7 @@ golang.org/x/time;golang time;BSD 3-clause "New" or "Revised" License
 google.golang.org/appengine;Golang appengine;Apache License 2.0
 google.golang.org/genproto;Google go-genproto;Apache License 2.0
 google.golang.org/grpc;grpc grpc-go;Apache License 2.0
+gopkg.in/freddierice/go-losetup.v1; Go losetup; MIT License
 gopkg.in/fsnotify.v1;fsnotify;BSD 3-clause "New" or "Revised" License
 gopkg.in/inf.v0;go-inf;BSD 3-clause "New" or "Revised" License
 gopkg.in/square/go-jose.v2;go-jose;Apache License 2.0