OCPBUGS-52189, MCO-1626: [release-4.18] backport OCL to 4.18

Containerfile.helpers-ci

@@ -0,0 +1,11 @@
+FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.22-builder-multi-openshift-4.18 AS builder
+ENV GOCACHE="/go/.cache" \
+    GOMODCACHE="/go/pkg/mod"
+WORKDIR /go/src/github.com/openshift/machine-config-operator
+COPY . .
+RUN --mount=type=cache,target=/go/.cache,z \
+    --mount=type=cache,target=/go/pkg/mod,z \
+    make install-helpers DESTDIR=/helpers
+
+FROM registry.ci.openshift.org/ocp/builder:rhel-9-enterprise-base-multi-openshift-4.18 AS final
+COPY --from=builder /helpers/usr/bin /usr/bin

Makefile

@@ -1,6 +1,10 @@
+E2E_ROOT_DIR = ./test
+E2E_SUITES = $(notdir $(wildcard $(E2E_ROOT_DIR)/e2e*))
+
 MCO_COMPONENTS = daemon controller server operator
 EXTRA_COMPONENTS = apiserver-watcher machine-os-builder
 ALL_COMPONENTS = $(patsubst %,machine-config-%,$(MCO_COMPONENTS)) $(EXTRA_COMPONENTS)
+ALL_COMPONENTS_PATHS = $(patsubst %,cmd/%,$(ALL_COMPONENTS))
 PREFIX ?= /usr
 GO111MODULE?=on
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -36,9 +40,17 @@ clean:
 
 # Build machine configs. Intended to be called via another target.
 # Example:
-#    make _build-machine-config-operator
-_build-%:
-	WHAT=$* hack/build-go.sh
+#    make _build-component-machine-config-operator
+_build-component-%:
+	WHAT_PATH=cmd/$* WHAT=$(basename $*) hack/build-go.sh
+
+# Build the helpers under devex/cmd.
+_build-helper-%:
+	WHAT_PATH=devex/cmd/$* WHAT=$(basename $*) hack/build-go.sh
+
+# Verify that an e2e test is valid Golang by doing a trial compilation.
+_verify-e2e-%:
+	go test -c -tags=$(GOTAGS) -o _output/$* ./test/$*/...
 
 # Use podman to build the image.
 image:
@@ -98,28 +110,67 @@ endif
 
 install-tools: install-golangci-lint install-go-junit-report install-setup-envtest
 
-# Run verification steps
-# Example:
-#    make verify
-verify: install-tools
+# Runs golangci-lint
+lint: install-tools
 	./hack/golangci-lint.sh $(GOTAGS)
+
+# Verifies templates.
+verify-templates:
 	hack/verify-templates.sh
 
+# Verifies devex helpers
+verify-helpers:
+	# Conditionally tries to build the helper binaries in CI.
+	hack/verify-helpers.sh
+
+# Runs all verification steps
+# Example:
+#    make verify
+verify: install-tools verify-e2e lint verify-templates verify-helpers
+
+HELPERS_DIR := devex/cmd
+HELPER_BINARIES := $(notdir $(wildcard $(HELPERS_DIR)/*))
+
+.PHONY: helpers
+helpers: $(patsubst %,_build-helper-%,$(HELPER_BINARIES))
+
 # Template for defining build targets for binaries.
 define target_template =
  .PHONY: $(1)
- $(1): _build-$(1)
+ $(1): _build-component-$(1)
 endef
 # Create a target for each component
 $(foreach C, $(EXTRA_COMPONENTS), $(eval $(call target_template,$(C))))
 $(foreach C, $(MCO_COMPONENTS), $(eval $(call target_template,$(patsubst %,machine-config-%,$(C)))))
 
-.PHONY: binaries install
+# Template for defining build targets for helper binaries.
+define helper_target_template =
+ .PHONY: $(1)
+ $(1): _build-helper-$(1)
+endef
+# Create a target for each component
+$(foreach C, $(HELPER_BINARIES), $(eval $(call helper_target_template,$(C))))
+
+define verify_e2e_target_template =
+ .PHONY: $(1)
+ $(1): _verify-e2e-$(1)
+endef
+# Create a target for each e2e suite
+$(foreach C, $(E2E_SUITES), $(eval $(call verify_e2e_target_template,$(C))))
+
+
+.PHONY: binaries helpers install
 
 # Build all binaries:
 # Example:
 #    make binaries
-binaries: $(patsubst %,_build-%,$(ALL_COMPONENTS))
+binaries: $(patsubst %,_build-component-%,$(ALL_COMPONENTS))
+
+# Installs the helper binaries from devex/cmd.
+install-helpers: helpers
+	for helper in $(HELPER_BINARIES); do \
+		install -D -m 0755 _output/linux/$(GOARCH)/$${helper} $(DESTDIR)$(PREFIX)/bin/$${helper}; \
+	done
 
 install: binaries
 	for component in $(ALL_COMPONENTS); do \
@@ -130,6 +181,9 @@ Dockerfile.rhel7: Dockerfile Makefile
 	(echo '# THIS FILE IS GENERATED FROM '$<' DO NOT EDIT' && \
 	 sed -e s,org/openshift/release,org/ocp/builder, -e s,/openshift/origin-v4.0:base,/ocp/4.0:base, < $<) > $@.tmp && mv $@.tmp $@
 
+# Validates that all of the e2e test suites are valid Golang by performing a test compilation.
+verify-e2e: $(patsubst %,_verify-e2e-%,$(E2E_SUITES))
+
 # This was copied from https://github.com/openshift/cluster-image-registry-operator
 test-e2e: install-go-junit-report
 	set -o pipefail; go test -tags=$(GOTAGS) -failfast -timeout 170m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e/ ./test/e2e-techpreview-shared/ | ./hack/test-with-junit.sh $(@)
@@ -140,6 +194,9 @@ test-e2e-techpreview: install-go-junit-report
 test-e2e-single-node: install-go-junit-report
 	set -o pipefail; go test -tags=$(GOTAGS) -failfast -timeout 120m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e-single-node/ | ./hack/test-with-junit.sh $(@)
 
+test-e2e-ocl: install-go-junit-report
+	set -o pipefail; go test -tags=$(GOTAGS) -failfast -timeout 120m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e-ocl/ | ./hack/test-with-junit.sh $(@)
+
 bootstrap-e2e: install-go-junit-report install-setup-envtest
 	@echo "Setting up KUBEBUILDER_ASSETS"
 	@KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --index https://raw.githubusercontent.com/openshift/api/master/envtest-releases.yaml --bin-dir $(PROJECT_DIR)/bin -p path)" && \

cmd/common/helpers.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/openshift/machine-config-operator/internal/clients"
 	corev1 "k8s.io/api/core/v1"
@@ -98,3 +99,23 @@ func SignalHandler(runCancel context.CancelFunc) {
 	klog.Fatalf("Received shutdown signal twice, exiting: %s", sig)
 
 }
+
+func SignalHandlerWithDelay(runCancel context.CancelFunc, delay time.Duration) {
+	// make a signal handling channel for os signals
+	ch := make(chan os.Signal, 1)
+	// stop listening for signals when we leave this function
+	defer func() { signal.Stop(ch) }()
+	// catch SIGINT and SIGTERM
+	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
+	sig := <-ch
+	klog.Infof("Received shutdown signal: %s. Delaying shutdown...", sig)
+	// Wait for the delay
+	time.Sleep(delay)
+	// if we're shutting down, cancel the context so everything else will stop
+	klog.Infof("Shutting down after delay %s", delay)
+	runCancel()
+	klog.Infof("Context cancelled")
+	sig = <-ch
+	klog.Fatalf("Received shutdown signal twice, exiting: %s", sig)
+
+}

cmd/machine-config-controller/start.go

@@ -236,7 +236,7 @@ func createControllers(ctx *ctrlcommon.ControllerContext) []ctrlcommon.Controlle
 			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
 			ctx.KubeInformerFactory.Core().V1().Nodes(),
 			ctx.KubeInformerFactory.Core().V1().Pods(),
-			ctx.TechPreviewInformerFactory.Machineconfiguration().V1alpha1().MachineOSConfigs(),
+			ctx.InformerFactory.Machineconfiguration().V1().MachineOSConfigs(),
 			ctx.ConfigInformerFactory.Config().V1().Schedulers(),
 			ctx.ClientBuilder.KubeClientOrDie("node-update-controller"),
 			ctx.ClientBuilder.MachineConfigClientOrDie("node-update-controller"),

cmd/machine-os-builder/start.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"os"
+	"time"
 
 	"github.com/openshift/machine-config-operator/cmd/common"
 	"github.com/openshift/machine-config-operator/internal/clients"
@@ -53,7 +54,9 @@ func runStartCmd(_ *cobra.Command, _ []string) {
 	}
 
 	run := func(ctx context.Context) {
-		go common.SignalHandler(cancel)
+		// When shutting down, wait for 30 seconds before actually shutting down
+		// This will ensure that all the cleanup is done
+		go common.SignalHandlerWithDelay(cancel, 30*time.Second)
 
 		ctrlCtx := ctrlcommon.CreateControllerContext(ctx, cb)
 

devex/README.md

@@ -0,0 +1,18 @@
+# devex
+
+## Background
+
+This directory contains Golang programs which are most likely to be of use to
+fellow OpenShift developers, especially members of the
+[machine-config-operator](https://github.com/openshift/machine-config-operator)
+team. The helpers found here may be of use to you. They may not. They may
+completely break entirely.
+
+It is worth mentioning that these helpers may get your cluster into a
+difficult-to-recover-from state. So do not use these on a production OpenShift
+cluster.
+
+## Installation
+
+From the repo root, run: `make install-helpers`. Note: You'll periodically have
+to update the helpers based upon the current state of the MCO repository.

devex/cmd/mcdiff/diff.go

@@ -0,0 +1,170 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
+
+	ign3types "github.com/coreos/ignition/v2/config/v3_4/types"
+	ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
+	"github.com/openshift/machine-config-operator/test/framework"
+	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/yaml"
+)
+
+var (
+	diffCmd = &cobra.Command{
+		Use:   "diff",
+		Short: "Diffs MachineConfigs",
+		Long:  "",
+		RunE: func(_ *cobra.Command, args []string) error {
+			return diffMCs(args)
+		},
+	}
+
+	convertToYAML bool
+	keepFiles     bool
+)
+
+func init() {
+	rootCmd.AddCommand(diffCmd)
+	diffCmd.PersistentFlags().BoolVar(&convertToYAML, "convert-to-yaml", false, "Converts any JSON payloads that are found into YAML before diffing")
+	diffCmd.PersistentFlags().BoolVar(&keepFiles, "keep-files", false, "Keeps the files used for diffing")
+}
+
+func diffMCs(args []string) error {
+	if len(args) == 0 {
+		return fmt.Errorf("no MachineConfigs given")
+	}
+
+	if len(args) == 1 {
+		return fmt.Errorf("only one MachineConfig given")
+	}
+
+	cs := framework.NewClientSet("")
+
+	eg := errgroup.Group{}
+
+	mc1 := args[0]
+	mc2 := args[1]
+
+	dirname := ""
+	if keepFiles {
+		cwd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+
+		dirname = cwd
+	} else {
+
+		tempdir, err := os.MkdirTemp("", "")
+		if err != nil {
+			return err
+		}
+
+		defer os.RemoveAll(tempdir)
+
+		dirname = tempdir
+	}
+
+	eg.Go(func() error {
+		return getMCAndWriteToFile(cs, dirname, mc1)
+	})
+
+	eg.Go(func() error {
+		return getMCAndWriteToFile(cs, dirname, mc2)
+	})
+
+	if err := eg.Wait(); err != nil {
+		return err
+	}
+
+	klog.Infof("Running dyff command")
+	out := exec.Command("dyff", "between", getMCFilename(dirname, mc1), getMCFilename(dirname, mc2))
+	out.Stdout = os.Stdout
+	out.Stderr = os.Stderr
+
+	return out.Run()
+}
+
+func getMCAndWriteToFile(cs *framework.ClientSet, dirname, name string) error {
+	mc, err := cs.MachineConfigs().Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	outBytes, err := yaml.Marshal(mc)
+	if err != nil {
+		return err
+	}
+
+	genericized := map[string]interface{}{}
+
+	if err := yaml.Unmarshal(outBytes, &genericized); err != nil {
+		return err
+	}
+
+	parsedIgnConfig, err := getParsedIgnConfig(mc)
+	if err != nil {
+		return err
+	}
+
+	genericized["spec"].(map[string]interface{})["config"] = parsedIgnConfig
+
+	filename := getMCFilename(dirname, name)
+
+	outBytes, err = yaml.Marshal(genericized)
+	if err != nil {
+		return err
+	}
+
+	if err := os.WriteFile(filename, outBytes, 0o755); err != nil {
+		return err
+	}
+
+	klog.Infof("Wrote %s", filename)
+
+	return nil
+}
+
+func getMCFilename(dirname, mcName string) string {
+	return filepath.Join(dirname, fmt.Sprintf("%s.yaml", mcName))
+}
+
+func getParsedIgnConfig(mc *mcfgv1.MachineConfig) (*ign3types.Config, error) {
+	// Convert the raw Ignition bytes into an Ignition struct.
+	ignConfig, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw)
+	if err != nil {
+		return nil, err
+	}
+
+	for i, file := range ignConfig.Storage.Files {
+		// Decode each files contents
+		decoded, err := ctrlcommon.DecodeIgnitionFileContents(file.Contents.Source, file.Contents.Compression)
+		if err != nil {
+			return nil, err
+		}
+
+		if file.Contents.Source != nil {
+			if convertToYAML {
+				decoded, err = yaml.JSONToYAML(decoded)
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			out := string(decoded)
+			ignConfig.Storage.Files[i].Contents.Source = &out
+		}
+	}
+
+	return &ignConfig, nil
+}