Skip to content

Commit

Permalink
network: add a new operator config setting ROOK_ENFORCE_HOSTNETWORK
Browse files Browse the repository at this point in the history
    This new setting is of Boolean type and defaults to "false".

    When set to "true", it changes the behavior of the
     rook operator to
    nable host network on all pods created by the cephcluster controller

     new method to check the setting:  opcontroller.EnForceHostNetwork()

Signed-off-by: Michael Adam <obnox@samba.org>
  • Loading branch information
obnoxxx committed Sep 5, 2024
1 parent 04eecf7 commit e378588
Show file tree
Hide file tree
Showing 11 changed files with 97 additions and 7 deletions.
17 changes: 15 additions & 2 deletions pkg/apis/ceph.rook.io/v1/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@ import (
"github.com/pkg/errors"
)

// enforceHostNetwork is a private package variable that can be set via the rook-operator-config
// setting "ROOK_ENFORCE_HOST_NETWORK". when set to "true", it lets rook create all pods with host network enabled.
// This can be used, for example, to run Rook in k8s clusters with no CNI where host networking is required
var enforceHostNetwork bool = false

// IsMultus get whether to use multus network provider
func (n *NetworkSpec) IsMultus() bool {
return n.Provider == NetworkProviderMultus
Expand All @@ -40,7 +45,7 @@ func (n *NetworkSpec) IsMultus() bool {
// together with an empty or unset network provider has the same effect as
// network.Provider set to "host"
func (n *NetworkSpec) IsHost() bool {
return (n.HostNetwork && n.Provider == NetworkProviderDefault) || n.Provider == NetworkProviderHost
return enforceHostNetwork || (n.HostNetwork && n.Provider == NetworkProviderDefault) || n.Provider == NetworkProviderHost
}

func ValidateNetworkSpec(clusterNamespace string, spec NetworkSpec) error {
Expand All @@ -62,7 +67,7 @@ func ValidateNetworkSpec(clusterNamespace string, spec NetworkSpec) error {

if !spec.AddressRanges.IsEmpty() {
if !spec.IsMultus() && !spec.IsHost() {
// TODO: be sure to update docs that AddressRanges can be specified for host networking as
// TODO: be sure to update docs that AddressRanges can be specified for host networking as
// well as multus so that the override configmap doesn't need to be set
return errors.Errorf("network ranges can only be specified for %q and %q network providers", NetworkProviderHost, NetworkProviderMultus)
}
Expand Down Expand Up @@ -181,3 +186,11 @@ func (l *CIDRList) String() string {
}
return strings.Join(sl, ", ")
}

func SetEnforceHostNetwork(val bool) {
enforceHostNetwork = val
}

func EnforceHostNetwork() bool {
return enforceHostNetwork
}
24 changes: 23 additions & 1 deletion pkg/apis/ceph.rook.io/v1/network_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ func TestValidateNetworkSpec(t *testing.T) {
// test the NetworkSpec.IsHost method with different network providers
// Also test it in combination with the legacy
// "HostNetwork" setting.
// Also test the effect of the operator config setting
// ROOK_ENFORCE_HOST_NETWORK.
func TestNetworkCephIsHost(t *testing.T) {
net := NetworkSpec{HostNetwork: false}

Expand All @@ -85,6 +87,14 @@ func TestNetworkCephIsHost(t *testing.T) {
net.HostNetwork = true
assert.True(t, net.IsHost())

// enforcing does not change the result if host network is selected
// anyway in the cluster.
SetEnforceHostNetwork(true)
assert.True(t, net.IsHost())

SetEnforceHostNetwork(false)
assert.True(t, net.IsHost())

net = NetworkSpec{}
net.Provider = NetworkProviderDefault
net.HostNetwork = false
Expand All @@ -95,16 +105,28 @@ func TestNetworkCephIsHost(t *testing.T) {
net.HostNetwork = false
assert.False(t, net.IsHost())

// test that not enforcing does not change the result.
SetEnforceHostNetwork(false)
assert.False(t, net.IsHost())

// test enforcing of host network
SetEnforceHostNetwork(true)
assert.True(t, net.IsHost())

SetEnforceHostNetwork(false)
net = NetworkSpec{}
net.Provider = NetworkProviderMultus
net.HostNetwork = true
assert.False(t, net.IsHost())

// test with nonempty but invalid provider
// test with nonempty but invalid provider
net = NetworkSpec{}
net.HostNetwork = true
net.Provider = "foo"
SetEnforceHostNetwork(false)
assert.False(t, net.IsHost())
SetEnforceHostNetwork(true)
assert.True(t, net.IsHost())

}

Expand Down
2 changes: 2 additions & 0 deletions pkg/operator/ceph/cluster/cleanup.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
"github.com/rook/rook/pkg/operator/ceph/cluster/rbd"
"github.com/rook/rook/pkg/operator/ceph/controller"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/ceph/file/mds"
"github.com/rook/rook/pkg/operator/ceph/file/mirror"
"github.com/rook/rook/pkg/operator/ceph/object"
Expand Down Expand Up @@ -162,6 +163,7 @@ func (c *ClusterController) cleanUpJobTemplateSpec(cluster *cephv1.CephCluster,
RestartPolicy: v1.RestartPolicyOnFailure,
PriorityClassName: cephv1.GetCleanupPriorityClassName(cluster.Spec.PriorityClassNames),
ServiceAccountName: k8sutil.DefaultServiceAccount,
HostNetwork: opcontroller.EnforceHostNetwork(),
},
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/operator/ceph/cluster/osd/provision_spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/rook/rook/pkg/operator/ceph/cluster/mon"
"github.com/rook/rook/pkg/operator/ceph/cluster/osd/config"
"github.com/rook/rook/pkg/operator/ceph/controller"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
"github.com/rook/rook/pkg/operator/k8sutil"
batch "k8s.io/api/batch/v1"
v1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -154,7 +155,7 @@ func (c *Cluster) provisionPodTemplateSpec(osdProps osdProperties, restart v1.Re
},
RestartPolicy: restart,
Volumes: volumes,
HostNetwork: c.spec.Network.IsHost(),
HostNetwork: opcontroller.EnforceHostNetwork(),
PriorityClassName: cephv1.GetOSDPriorityClassName(c.spec.PriorityClassNames),
SchedulerName: osdProps.schedulerName,
}
Expand Down
1 change: 1 addition & 0 deletions pkg/operator/ceph/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ func (r *ReconcileConfig) reconcile(request reconcile.Request) (reconcile.Result
}

opcontroller.SetAllowLoopDevices(r.config.Parameters)
opcontroller.SetEnforceHostNetwork(r.config.Parameters)

logger.Infof("%s done reconciling", controllerName)
return reconcile.Result{}, nil
Expand Down
20 changes: 18 additions & 2 deletions pkg/operator/ceph/controller/controller_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ type ClusterHealth struct {

const (
// OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator
OperatorSettingConfigMapName string = "rook-ceph-operator-config"
OperatorSettingConfigMapName string = "rook-ceph-operator-config"
enforceHostNetworkSettingName string = "ROOK_ENFORCE_HOST_NETWORK"
enforceHostNetworkDefaultValue string = "false"

// UninitializedCephConfigError refers to the error message printed by the Ceph CLI when there is no ceph configuration file
// This typically is raised when the operator has not finished initializing
Expand Down Expand Up @@ -116,6 +118,21 @@ func LoopDevicesAllowed() bool {
return loopDevicesAllowed
}

func SetEnforceHostNetwork(data map[string]string) {
strval := k8sutil.GetValue(data, enforceHostNetworkSettingName, enforceHostNetworkDefaultValue)
val, err := strconv.ParseBool(strval)
if err != nil {
logger.Warningf("failed to parse value %q for %q. assuming false value", strval, enforceHostNetworkSettingName)
cephv1.SetEnforceHostNetwork(false)
return
}
cephv1.SetEnforceHostNetwork(val)
}

func EnforceHostNetwork() bool {
return cephv1.EnforceHostNetwork()
}

// canIgnoreHealthErrStatusInReconcile determines whether a status of HEALTH_ERR in the CephCluster can be ignored safely.
func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, controllerName string) bool {
// Get a list of all the keys causing the HEALTH_ERR status.
Expand Down Expand Up @@ -153,7 +170,6 @@ func IsReadyToReconcile(ctx context.Context, c client.Client, namespacedName typ
return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady
}
cephCluster = clusterList.Items[0]

// If the cluster has a cleanup policy to destroy the cluster and it has been marked for deletion, treat it as if it does not exist
if cephCluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() && !cephCluster.DeletionTimestamp.IsZero() {
logger.Infof("%q: CephCluster has a destructive cleanup policy, allowing %q to be deleted", controllerName, namespacedName)
Expand Down
27 changes: 27 additions & 0 deletions pkg/operator/ceph/controller/controller_utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,33 @@ func TestSetAllowLoopDevices(t *testing.T) {
assert.True(t, LoopDevicesAllowed())
}

func TestSetEnforceHostNetwork(t *testing.T) {
logger.Infof("testing default value for %v", enforceHostNetworkSettingName)
opConfig := map[string]string{}
SetEnforceHostNetwork(opConfig)
assert.False(t, EnforceHostNetwork())

// test invalid setting
var value string = "foo"
logger.Infof("testing invalid value'%v' for %v", value, enforceHostNetworkSettingName)
opConfig[enforceHostNetworkSettingName] = value
SetEnforceHostNetwork(opConfig)
assert.False(t, EnforceHostNetwork())

// test valid settings
value = "true"
logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName)
opConfig[enforceHostNetworkSettingName] = value
SetEnforceHostNetwork(opConfig)
assert.True(t, EnforceHostNetwork())

value = "false"
logger.Infof("testing valid value'%v' for %v", value, enforceHostNetworkSettingName)
opConfig[enforceHostNetworkSettingName] = value
SetEnforceHostNetwork(opConfig)
assert.False(t, EnforceHostNetwork())
}

func TestIsReadyToReconcile(t *testing.T) {
scheme := scheme.Scheme
scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{})
Expand Down
3 changes: 3 additions & 0 deletions pkg/operator/ceph/csi/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -382,6 +382,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
if tp.CSILogRotation {
applyLogrotateSidecar(&rbdProvisionerDeployment.Spec.Template, "csi-rbd-deployment-log-collector", LogrotateTemplatePath, tp)
}
rbdProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork()

// Create service if either liveness or GRPC metrics are enabled.
if CSIParam.EnableLiveness {
Expand Down Expand Up @@ -419,6 +420,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
if tp.CSILogRotation {
applyLogrotateSidecar(&cephfsProvisionerDeployment.Spec.Template, "csi-cephfs-deployment-log-collector", LogrotateTemplatePath, tp)
}
cephfsProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork()

// Create service if either liveness or GRPC metrics are enabled.
if CSIParam.EnableLiveness {
Expand Down Expand Up @@ -457,6 +459,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI
if tp.CSILogRotation {
applyLogrotateSidecar(&nfsProvisionerDeployment.Spec.Template, "csi-nfs-deployment-log-collector", LogrotateTemplatePath, tp)
}
nfsProvisionerDeployment.Spec.Template.Spec.HostNetwork = opcontroller.EnforceHostNetwork()

enabledDrivers = append(enabledDrivers, driverDetails{
name: NFSDriverShortName,
Expand Down
2 changes: 2 additions & 0 deletions pkg/operator/ceph/object/cosi/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/operator/ceph/controller"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -83,6 +84,7 @@ func createCOSIPodSpec(cephCOSIDriver *cephv1.CephCOSIDriver) (corev1.PodTemplat
cosiSideCarContainer := createCOSISideCarContainer(cephCOSIDriver)

podSpec := corev1.PodSpec{
HostNetwork: opcontroller.EnforceHostNetwork(),
Containers: []corev1.Container{
cosiDriverContainer,
cosiSideCarContainer,
Expand Down
3 changes: 2 additions & 1 deletion pkg/operator/discover/discover.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/rook/rook/pkg/clusterd"
discoverDaemon "github.com/rook/rook/pkg/daemon/discover"
"github.com/rook/rook/pkg/operator/ceph/controller"
opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
k8sutil "github.com/rook/rook/pkg/operator/k8sutil"
"github.com/rook/rook/pkg/util/sys"

Expand Down Expand Up @@ -173,7 +174,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco
},
},
},
HostNetwork: false,
HostNetwork: opcontroller.EnforceHostNetwork(),
PriorityClassName: k8sutil.GetValue(data, discoverDaemonsetPriorityClassNameEnv, ""),
},
},
Expand Down
2 changes: 2 additions & 0 deletions pkg/operator/k8sutil/cmdreporter/cmdreporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

"github.com/coreos/pkg/capnslog"
"github.com/pkg/errors"
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/daemon/util"
"github.com/rook/rook/pkg/operator/k8sutil"
batch "k8s.io/api/batch/v1"
Expand Down Expand Up @@ -302,6 +303,7 @@ func (cr *cmdReporterCfg) initJobSpec() (*batch.Job, error) {
},
RestartPolicy: v1.RestartPolicyOnFailure,
ServiceAccountName: k8sutil.DefaultServiceAccount,
HostNetwork: cephv1.EnforceHostNetwork(),
}
copyBinsVol, _ := copyBinariesVolAndMount()
podSpec.Volumes = []v1.Volume{copyBinsVol}
Expand Down

0 comments on commit e378588

Please sign in to comment.