Skip to content

[WIP] Fix remaining loop devices manipulations in NodeSetup E2Es #2987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

[WIP] Fix remaining loop devices manipulations in NodeSetup E2Es #2987

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 14 additions & 24 deletions test/e2e/set/nodeconfig/nodeconfig_disksetup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,19 @@ var _ = g.Describe("Node Setup", framework.Serial, func() {
f := framework.NewFramework("nodesetup")

ncTemplate := scyllafixture.NodeConfig.ReadOrFail()
var matchingNodes []*corev1.Node
var nodeUnderTest *corev1.Node

g.JustBeforeEach(func() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

g.By("Verifying there is at least one scylla node")
var err error
matchingNodes, err = utils.GetMatchingNodesForNodeConfig(ctx, f.KubeAdminClient().CoreV1(), ncTemplate)
matchingNodes, err := utils.GetMatchingNodesForNodeConfig(ctx, f.KubeAdminClient().CoreV1(), ncTemplate)
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(matchingNodes).NotTo(o.HaveLen(0))
framework.Infof("There are %d scylla nodes", len(matchingNodes))
nodeUnderTest = matchingNodes[0]
framework.Infof("There are %d scylla nodes. %s will be used in tests.", len(matchingNodes), nodeUnderTest.GetName())
})

g.DescribeTable("should make RAID0 array out of loop devices, format it to XFS, and mount at desired location", func(numberOfDevices int) {
Expand All @@ -61,7 +62,7 @@ var _ = g.Describe("Node Setup", framework.Serial, func() {
nc := ncTemplate.DeepCopy()

framework.By("Creating a client Pod")
clientPod := newClientPod(nc)
clientPod := newClientPod(nc, nodeUnderTest)

clientPod, err := f.KubeClient().CoreV1().Pods(f.Namespace()).Create(ctx, clientPod, metav1.CreateOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
Expand Down Expand Up @@ -258,7 +259,6 @@ var _ = g.Describe("Node Setup", framework.Serial, func() {
defer cancel()

nc := e.nodeConfigFunc()
nodeUnderTest := matchingNodes[0]

if e.preNodeConfigCreationFunc != nil {
cleanupFunc := e.preNodeConfigCreationFunc(ctx, nc, nodeUnderTest)
Expand Down Expand Up @@ -408,7 +408,7 @@ var _ = g.Describe("Node Setup", framework.Serial, func() {
hostMountPath := fmt.Sprintf("/host/var/lib/%s", f.Namespace())

framework.By("Creating a client Pod")
clientPod := newClientPod(nc)
clientPod := newClientPod(nc, nodeUnderTest)
clientPod.Spec.NodeName = nodeUnderTest.GetName()

clientPod, err := f.KubeClient().CoreV1().Pods(f.Namespace()).Create(ctx, clientPod, metav1.CreateOptions{})
Expand Down Expand Up @@ -468,7 +468,7 @@ var _ = g.Describe("Node Setup", framework.Serial, func() {
hostDevicePath := "/host/dev/loops/disk"

framework.By("Creating a client Pod")
clientPod := newClientPod(nc)
clientPod := newClientPod(nc, nodeUnderTest)
clientPod.Spec.NodeName = nodeUnderTest.GetName()

clientPod, err := f.KubeClient().CoreV1().Pods(f.Namespace()).Create(ctx, clientPod, metav1.CreateOptions{})
Expand Down Expand Up @@ -530,11 +530,7 @@ var _ = g.Describe("Node Setup", framework.Serial, func() {
}),
)

g.It("should successfully start a failed mount unit with a corrupted filesystem when it's overwritten with a clean one", func(ctx context.Context) {
o.Expect(matchingNodes).NotTo(o.BeEmpty())
o.Expect(matchingNodes[0]).NotTo(o.BeNil())
nodeUnderTest := matchingNodes[0]

g.FIt("should successfully start a failed mount unit with a corrupted filesystem when it's overwritten with a clean one", func(ctx context.Context) {
devicePath := fmt.Sprintf("/dev/loops/%s", f.Namespace())
hostDevicePath := path.Join("/host", devicePath)

Expand Down Expand Up @@ -573,7 +569,7 @@ var _ = g.Describe("Node Setup", framework.Serial, func() {
o.Expect(err).NotTo(o.HaveOccurred())

framework.By("Creating a client Pod")
clientPod := newClientPod(nc)
clientPod := newClientPod(nc, nodeUnderTest)

clientPod, err = f.KubeClient().CoreV1().Pods(f.Namespace()).Create(ctx, clientPod, metav1.CreateOptions{})
o.Expect(err).NotTo(o.HaveOccurred())
Expand Down Expand Up @@ -607,7 +603,6 @@ var _ = g.Describe("Node Setup", framework.Serial, func() {
framework.By("Getting the filesystem's block size")
stdout, stderr, err = executeInPod(ctx, f.ClientConfig(), f.KubeClient().CoreV1(), clientPod, "stat", "--file-system", "--format=%s", devicePathInContainer)
o.Expect(err).NotTo(o.HaveOccurred(), stdout, stderr)

blockSize := strings.TrimSpace(stdout)

framework.By("Corrupting the filesystem")
Expand Down Expand Up @@ -647,15 +642,7 @@ var _ = g.Describe("Node Setup", framework.Serial, func() {
o.Expect(err).NotTo(o.HaveOccurred())

framework.By("Overwriting the corrupted filesystem")
stdout, stderr, err = executeInPod(ctx, f.ClientConfig(), f.KubeClient().CoreV1(), clientPod, "mkfs", "-t", string(scyllav1alpha1.XFSFilesystem), "-b", fmt.Sprintf("size=%s", blockSize), "-K", "-f", hostDevicePath)
o.Expect(err).NotTo(o.HaveOccurred(), stdout, stderr)

framework.By("Zeroing XFS log")
stdout, stderr, err = executeInPod(ctx, f.ClientConfig(), f.KubeClient().CoreV1(), clientPod, "xfs_repair", "-L", "-f", hostDevicePath)
o.Expect(err).NotTo(o.HaveOccurred(), stdout, stderr)

framework.By("Verifying the filesystem's integrity")
stdout, stderr, err = executeInPod(ctx, f.ClientConfig(), f.KubeClient().CoreV1(), clientPod, "xfs_repair", "-o", "force_geometry", "-f", "-n", hostDevicePath)
stdout, stderr, err = executeInPod(ctx, f.ClientConfig(), f.KubeClient().CoreV1(), clientPod, "mkfs", "-t", string(scyllav1alpha1.XFSFilesystem), "-b", fmt.Sprintf("size=%s", blockSize), "-K", "-f", devicePathInContainer)
o.Expect(err).NotTo(o.HaveOccurred(), stdout, stderr)

framework.By("Waiting for NodeConfig to roll out")
Expand Down Expand Up @@ -702,12 +689,15 @@ var _ = g.Describe("Node Setup", framework.Serial, func() {
}, g.NodeTimeout(testTimeout))
})

func newClientPod(nc *scyllav1alpha1.NodeConfig) *corev1.Pod {
func newClientPod(nc *scyllav1alpha1.NodeConfig, nodeUnderTest *corev1.Node) *corev1.Pod {
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "client",
},
Spec: corev1.PodSpec{
// We want to schedule the Pod specifically on the node that is under test as we will perform operations
// on the host filesystem through that Pod.
NodeName: nodeUnderTest.Name,
Containers: []corev1.Container{
{
Name: "client",
Expand Down