Skip to content

Commit

Permalink
fix: ConfigMap xxx-rsm-env not found (#8222)
Browse files Browse the repository at this point in the history
  • Loading branch information
free6om authored Sep 30, 2024
1 parent edf8598 commit 2b09ac5
Show file tree
Hide file tree
Showing 13 changed files with 19 additions and 199 deletions.
1 change: 0 additions & 1 deletion cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,6 @@ func init() {
viper.SetDefault(constant.CfgKBReconcileWorkers, 8)
viper.SetDefault(constant.FeatureGateIgnoreConfigTemplateDefaultMode, false)
viper.SetDefault(constant.FeatureGateInPlacePodVerticalScaling, false)
viper.SetDefault(constant.FeatureGateNoRSMEnv, false)
}

type flagName string
Expand Down
9 changes: 7 additions & 2 deletions controllers/apps/transformer_component_workload_upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package apps

import (
"context"
"fmt"
"reflect"

appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -65,7 +66,7 @@ func (t *componentWorkloadUpgradeTransformer) Transform(ctx graph.TransformConte
ml := constant.GetCompLabels(synthesizeComp.ClusterName, synthesizeComp.Name)
inNS := client.InNamespace(comp.Namespace)
defaultHeadlessSvc := constant.GenerateDefaultComponentHeadlessServiceName(synthesizeComp.ClusterName, synthesizeComp.Name)
envCM := instanceset.GetEnvConfigMapName(constant.GenerateClusterComponentName(synthesizeComp.ClusterName, synthesizeComp.Name))
envCM := getEnvConfigMapName(constant.GenerateClusterComponentName(synthesizeComp.ClusterName, synthesizeComp.Name))
var revision string
for _, list := range objectList {
if err := graphCli.List(transCtx.Context, list, client.MatchingLabels(ml), inNS); err == nil {
Expand Down Expand Up @@ -169,6 +170,10 @@ func buildRevision(synthesizeComp *component.SynthesizedComponent, componentDef
if err != nil {
return "", err
}
template := instanceset.BuildPodTemplate(its, instanceset.GetEnvConfigMapName(its.Name))
template := instanceset.BuildPodTemplate(its)
return instanceset.BuildInstanceTemplateRevision(template, its)
}

func getEnvConfigMapName(rsmName string) string {
return fmt.Sprintf("%s-rsm-env", rsmName)
}
2 changes: 0 additions & 2 deletions deploy/helm/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -178,8 +178,6 @@ spec:
value: {{ .Values.featureGates.componentReplicasAnnotation.enabled | quote }}
- name: IN_PLACE_POD_VERTICAL_SCALING
value: {{ .Values.featureGates.inPlacePodVerticalScaling.enabled | quote }}
- name: NO_RSM_ENV
value: {{ .Values.featureGates.noRSMEnv.enabled | quote }}
{{- with .Values.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
Expand Down
2 changes: 0 additions & 2 deletions deploy/helm/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1925,8 +1925,6 @@ featureGates:
enabled: true
inPlacePodVerticalScaling:
enabled: false
noRSMEnv:
enabled: false

vmagent:

Expand Down
3 changes: 0 additions & 3 deletions pkg/constant/flag.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,4 @@ const (
// FeatureGateInPlacePodVerticalScaling specifies to enable in-place pod vertical scaling
// NOTE: This feature depends on the InPlacePodVerticalScaling feature of the K8s cluster in which the KubeBlocks runs.
FeatureGateInPlacePodVerticalScaling = "IN_PLACE_POD_VERTICAL_SCALING"

// FeatureGateNoRSMEnv specifies not to create the '$(instanceset.name)-rsm-env' ConfigMap object.
FeatureGateNoRSMEnv = "NO_RSM_ENV"
)
3 changes: 1 addition & 2 deletions pkg/controller/instanceset/instance_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -682,8 +682,7 @@ func BuildInstanceTemplateRevision(template *corev1.PodTemplateSpec, parent *wor
}

func buildInstanceTemplateExts(itsExt *instanceSetExt) []*instanceTemplateExt {
envConfigName := GetEnvConfigMapName(itsExt.its.Name)
defaultTemplate := BuildPodTemplate(itsExt.its, envConfigName)
defaultTemplate := BuildPodTemplate(itsExt.its)
makeInstanceTemplateExt := func(templateName string) *instanceTemplateExt {
var claims []corev1.PersistentVolumeClaim
for _, template := range itsExt.its.Spec.VolumeClaimTemplates {
Expand Down
9 changes: 3 additions & 6 deletions pkg/controller/instanceset/instance_util_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,7 @@ var _ = Describe("instance util test", func() {
Expect(nameTemplate).Should(HaveKey(its.Name + "-1"))
Expect(nameTemplate).Should(HaveKey(its.Name + "-2"))
nameTemplate[name0].PodTemplateSpec.Spec.Volumes = nil
envConfigName := GetEnvConfigMapName(its.Name)
defaultTemplate := BuildPodTemplate(its, envConfigName)
defaultTemplate := BuildPodTemplate(its)
Expect(nameTemplate[name0].PodTemplateSpec.Spec).Should(Equal(defaultTemplate.Spec))
})

Expand Down Expand Up @@ -169,8 +168,7 @@ var _ = Describe("instance util test", func() {
Expect(nameTemplate).Should(HaveKey(name0))
Expect(nameTemplate).Should(HaveKey(name1))
Expect(nameTemplate).Should(HaveKey(nameOverride0))
envConfigName := GetEnvConfigMapName(its.Name)
expectedTemplate := BuildPodTemplate(its, envConfigName)
expectedTemplate := BuildPodTemplate(its)
Expect(nameTemplate[name0].PodTemplateSpec.Spec).Should(Equal(expectedTemplate.Spec))
Expect(nameTemplate[name1].PodTemplateSpec.Spec).Should(Equal(expectedTemplate.Spec))
Expect(nameTemplate[nameOverride0].PodTemplateSpec.Spec).ShouldNot(Equal(expectedTemplate.Spec))
Expand Down Expand Up @@ -201,8 +199,7 @@ var _ = Describe("instance util test", func() {
Expect(instance.pod.Namespace).Should(Equal(its.Namespace))
Expect(instance.pod.Spec.Volumes).Should(HaveLen(1))
Expect(instance.pod.Spec.Volumes[0].Name).Should(Equal(volumeClaimTemplates[0].Name))
envConfigName := GetEnvConfigMapName(its.Name)
expectedTemplate := BuildPodTemplate(its, envConfigName)
expectedTemplate := BuildPodTemplate(its)
Expect(instance.pod.Spec).ShouldNot(Equal(expectedTemplate.Spec))
// reset pod.volumes, pod.hostname and pod.subdomain
instance.pod.Spec.Volumes = nil
Expand Down
89 changes: 1 addition & 88 deletions pkg/controller/instanceset/object_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,35 +82,8 @@ func getHeadlessSvcName(itsName string) string {
return strings.Join([]string{itsName, "headless"}, "-")
}

func buildEnvConfigMap(its workloads.InstanceSet, labels map[string]string) (*corev1.ConfigMap, error) {
if viper.GetBool(constant.FeatureGateNoRSMEnv) {
return nil, nil
}
envData, err := buildEnvConfigData(its)
if err != nil {
return nil, err
}
annotations := ParseAnnotationsOfScope(ConfigMapScope, its.Annotations)
return builder.NewConfigMapBuilder(its.Namespace, GetEnvConfigMapName(its.Name)).
AddAnnotationsInMap(annotations).
AddLabelsInMap(labels).
SetData(envData).GetObject(), nil
}

func BuildPodTemplate(its *workloads.InstanceSet, envConfigName string) *corev1.PodTemplateSpec {
func BuildPodTemplate(its *workloads.InstanceSet) *corev1.PodTemplateSpec {
template := its.Spec.Template.DeepCopy()
// inject env ConfigMap into workload pods only
for i := range template.Spec.Containers {
template.Spec.Containers[i].EnvFrom = append(template.Spec.Containers[i].EnvFrom,
corev1.EnvFromSource{
ConfigMapRef: &corev1.ConfigMapEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: envConfigName,
},
Optional: func() *bool { optional := false; return &optional }(),
}})
}

injectRoleProbeContainer(its, template)

return template
Expand Down Expand Up @@ -357,63 +330,3 @@ func injectCustomRoleProbeContainer(its *workloads.InstanceSet, template *corev1
template.Spec.Containers = append(template.Spec.Containers, container)
}
}

func buildEnvConfigData(its workloads.InstanceSet) (map[string]string, error) {
envData := map[string]string{}
svcName := getHeadlessSvcName(its.Name)
uid := string(its.UID)
strReplicas := strconv.Itoa(int(*its.Spec.Replicas))
generateReplicaEnv := func(prefix string, podNames []string) {
for _, podName := range podNames {
_, ordinal := ParseParentNameAndOrdinal(podName)
hostNameTplKey := prefix + strconv.Itoa(ordinal) + "_HOSTNAME"
hostNameTplValue := its.Name + "-" + strconv.Itoa(ordinal)
envData[hostNameTplKey] = fmt.Sprintf("%s.%s", hostNameTplValue, svcName)
}
}
// build member related envs from set.Status.MembersStatus
generateMemberEnv := func(prefix string) {
followers := ""
for _, memberStatus := range its.Status.MembersStatus {
if memberStatus.PodName == "" || memberStatus.PodName == defaultPodName || memberStatus.ReplicaRole == nil {
continue
}
switch {
case memberStatus.ReplicaRole.IsLeader:
envData[prefix+"LEADER"] = memberStatus.PodName
case memberStatus.ReplicaRole.CanVote:
if len(followers) > 0 {
followers += ","
}
followers += memberStatus.PodName
}
}
if followers != "" {
envData[prefix+"FOLLOWERS"] = followers
}
}
// generate all pod names
generatePodNames := func() ([]string, error) {
var instances []InstanceTemplate
for i := range its.Spec.Instances {
instances = append(instances, &its.Spec.Instances[i])
}
return GenerateAllInstanceNames(its.Name, *its.Spec.Replicas, instances, its.Spec.OfflineInstances, its.Spec.DefaultTemplateOrdinals)
}

// all pod names
podNames, err := generatePodNames()
if err != nil {
return nil, err
}

prefix := constant.KBPrefix + "_ITS_"
envData[prefix+"N"] = strReplicas
generateReplicaEnv(prefix, podNames)
generateMemberEnv(prefix)
// set owner uid to let pod know if the owner is recreated
envData[prefix+"OWNER_UID"] = uid
envData[prefix+"OWNER_UID_SUFFIX8"] = uid[len(uid)-4:]

return envData, nil
}
69 changes: 0 additions & 69 deletions pkg/controller/instanceset/object_builder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package instanceset

import (
"fmt"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

Expand All @@ -48,73 +46,6 @@ var _ = Describe("object generation transformer test.", func() {
GetObject()
})

Context("buildEnvConfigData function", func() {
It("should work well", func() {
By("build env config data")
its.Status.MembersStatus = []workloads.MemberStatus{
{
PodName: getPodName(its.Name, 1),
ReplicaRole: &workloads.ReplicaRole{Name: "leader", IsLeader: true},
},
{
PodName: getPodName(its.Name, 0),
ReplicaRole: &workloads.ReplicaRole{Name: "follower", CanVote: true},
},
{
PodName: getPodName(its.Name, 2),
ReplicaRole: &workloads.ReplicaRole{Name: "follower", CanVote: true},
},
}
requiredKeys := []string{
"KB_ITS_N",
"KB_ITS_0_HOSTNAME",
}
cfg, err := buildEnvConfigData(*its)
Expect(err).Should(BeNil())
By("builds Env Config correctly")
Expect(cfg).ShouldNot(BeNil())
for _, k := range requiredKeys {
_, ok := cfg[k]
Expect(ok).Should(BeTrue())
}

By("builds Env Config with ConsensusSet status correctly")
toCheckKeys := append(requiredKeys, []string{
"KB_ITS_LEADER",
"KB_ITS_FOLLOWERS",
}...)
for _, k := range toCheckKeys {
_, ok := cfg[k]
Expect(ok).Should(BeTrue())
}
})

It("non-sequential ordinal", func() {
By("build env config data")
its.Spec.OfflineInstances = []string{
getPodName(its.Name, 1),
}
hostname := func(i int) string {
return fmt.Sprintf("%s.%s", getPodName(its.Name, i), getHeadlessSvcName(its.Name))
}
requiredKeys := map[string]string{
"KB_ITS_N": "3",
"KB_ITS_0_HOSTNAME": hostname(0),
"KB_ITS_2_HOSTNAME": hostname(2),
"KB_ITS_3_HOSTNAME": hostname(3),
}
cfg, err := buildEnvConfigData(*its)
Expect(err).Should(BeNil())

By("builds Env Config correctly")
Expect(cfg).ShouldNot(BeNil())
for k, v := range requiredKeys {
Expect(cfg).Should(HaveKeyWithValue(k, v))
}
Expect(cfg).ShouldNot(HaveKey("KB_ITS_1_HOSTNAME"))
})
})

Context("well-known service labels", func() {
It("should work well", func() {
svc := buildSvc(*its, getMatchLabels(its.Name), getSvcSelector(its, false))
Expand Down
7 changes: 0 additions & 7 deletions pkg/controller/instanceset/reconciler_assistant_object.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,18 +57,11 @@ func (a *assistantObjectReconciler) Reconcile(tree *kubebuilderx.ObjectTree) (ku

svc := buildSvc(*its, labels, selectors)
headLessSvc := buildHeadlessSvc(*its, labels, headlessSelectors)
envConfig, err := buildEnvConfigMap(*its, labels)
if err != nil {
return kubebuilderx.Continue, err
}
var objects []client.Object
if svc != nil {
objects = append(objects, svc)
}
objects = append(objects, headLessSvc)
if envConfig != nil {
objects = append(objects, envConfig)
}
for _, object := range objects {
if err := intctrlutil.SetOwnership(its, object, model.GetScheme(), finalizer); err != nil {
return kubebuilderx.Continue, err
Expand Down
18 changes: 7 additions & 11 deletions pkg/controller/instanceset/reconciler_assistant_object_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/apecloud/kubeblocks/pkg/controller/builder"
"github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx"
"github.com/apecloud/kubeblocks/pkg/controller/model"
Expand Down Expand Up @@ -55,17 +53,15 @@ var _ = Describe("assistant object reconciler test", func() {
res, err := reconciler.Reconcile(tree)
Expect(err).Should(BeNil())
Expect(res).Should(Equal(kubebuilderx.Continue))
// desired: svc: "bar-headless", cm: "bar"
// desired: svc: "bar-headless"
objects := tree.GetSecondaryObjects()
Expect(objects).Should(HaveLen(2))
Expect(objects).Should(HaveLen(1))
svc := builder.NewHeadlessServiceBuilder(namespace, name+"-headless").GetObject()
cm := builder.NewConfigMapBuilder(namespace, GetEnvConfigMapName(name)).GetObject()
for _, object := range []client.Object{svc, cm} {
name, err := model.GetGVKName(object)
Expect(err).Should(BeNil())
_, ok := objects[*name]
Expect(ok).Should(BeTrue())
}
name, err := model.GetGVKName(svc)
Expect(err).Should(BeNil())
_, ok := objects[*name]
Expect(ok).Should(BeTrue())

})
})
})
2 changes: 0 additions & 2 deletions pkg/controller/instanceset/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ const (
RoleLabelKey = "kubeblocks.io/role"
AccessModeLabelKey = "workloads.kubeblocks.io/access-mode"

defaultPodName = "Unknown"

LegacyRSMFinalizerName = "rsm.workloads.kubeblocks.io/finalizer"

roleProbeContainerName = "kb-role-probe"
Expand Down
4 changes: 0 additions & 4 deletions pkg/controller/instanceset/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,10 +189,6 @@ func ParseAnnotationsOfScope(scope AnnotationScope, scopedAnnotations map[string
return annotations
}

func GetEnvConfigMapName(itsName string) string {
return fmt.Sprintf("%s-rsm-env", itsName)
}

func composeRoleMap(its workloads.InstanceSet) map[string]workloads.ReplicaRole {
roleMap := make(map[string]workloads.ReplicaRole)
for _, role := range its.Spec.Roles {
Expand Down

0 comments on commit 2b09ac5

Please sign in to comment.