Skip to content

Commit 2ee4cd1

Browse files
sbueringersbueringer
committed
repro upgrade issue
1 parent d62c78f commit 2ee4cd1

File tree

13 files changed

+184
-27
lines changed

13 files changed

+184
-27
lines changed

‎bootstrap/kubeadm/config/manager/manager.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ spec:
3333
path: /readyz
3434
port: healthz
3535
livenessProbe:
36+
failureThreshold: 12
37+
periodSeconds: 10
3638
httpGet:
3739
path: /healthz
3840
port: healthz

‎config/manager/manager.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ spec:
3434
path: /readyz
3535
port: healthz
3636
livenessProbe:
37+
failureThreshold: 12
38+
periodSeconds: 10
3739
httpGet:
3840
path: /healthz
3941
port: healthz

‎controllers/remote/cluster_cache_tracker_test.go

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"testing"
2222

23+
"github.com/davecgh/go-spew/spew"
2324
. "github.com/onsi/gomega"
2425
corev1 "k8s.io/api/core/v1"
2526
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -79,9 +80,12 @@ func TestClusterCacheTracker(t *testing.T) {
7980
mgrContext, mgrCancel = context.WithCancel(ctx)
8081
t.Log("Starting the manager")
8182
go func() {
82-
g.Expect(mgr.Start(mgrContext)).To(Succeed())
83+
t.Helper()
84+
err := mgr.Start(mgrContext)
85+
spew.Dump(err)
86+
g.Expect(err).ToNot(HaveOccurred())
8387
}()
84-
<-env.Manager.Elected()
88+
<-mgr.Elected()
8589

8690
k8sClient = mgr.GetClient()
8791

@@ -115,11 +119,16 @@ func TestClusterCacheTracker(t *testing.T) {
115119

116120
teardown := func(t *testing.T, g *WithT, ns *corev1.Namespace) {
117121
t.Helper()
122+
defer close(c.ch)
118123

119124
t.Log("Deleting any Secrets")
120125
g.Expect(cleanupTestSecrets(ctx, k8sClient)).To(Succeed())
121126
t.Log("Deleting any Clusters")
122127
g.Expect(cleanupTestClusters(ctx, k8sClient)).To(Succeed())
128+
g.Expect(<-c.ch).To(Equal("mapped-" + clusterA.Name))
129+
g.Consistently(func() int {
130+
return len(c.ch)
131+
}).Should(Equal(0))
123132
t.Log("Deleting Namespace")
124133
g.Expect(env.Delete(ctx, ns)).To(Succeed())
125134
t.Log("Stopping the manager")
@@ -196,7 +205,11 @@ type testController struct {
196205
}
197206

198207
func (c *testController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
199-
c.ch <- req.Name
208+
spew.Dump(req)
209+
select {
210+
case <-ctx.Done():
211+
case c.ch <- req.Name:
212+
}
200213
return ctrl.Result{}, nil
201214
}
202215

‎controlplane/kubeadm/config/manager/manager.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ spec:
3333
path: /readyz
3434
port: healthz
3535
livenessProbe:
36+
failureThreshold: 12
37+
periodSeconds: 10
3638
httpGet:
3739
path: /healthz
3840
port: healthz

‎go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,5 @@ require (
4141
sigs.k8s.io/controller-runtime v0.10.2
4242
sigs.k8s.io/yaml v1.3.0
4343
)
44+
45+
replace sigs.k8s.io/controller-runtime => github.com/vincepri/controller-runtime v0.10.1-0.20211103153228-423ec7419d00

‎go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
561561
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
562562
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
563563
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
564+
github.com/vincepri/controller-runtime v0.10.1-0.20211013215004-81098136c987 h1:mSW5ZlfBIPa8VlJUhbfy7BkGP12Fs2uhYBNvhJtTH/I=
565+
github.com/vincepri/controller-runtime v0.10.1-0.20211013215004-81098136c987/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY=
564566
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
565567
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
566568
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@@ -1114,8 +1116,6 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
11141116
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
11151117
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
11161118
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
1117-
sigs.k8s.io/controller-runtime v0.10.2 h1:jW8qiY+yMnnPx6O9hu63tgcwaKzd1yLYui+mpvClOOc=
1118-
sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY=
11191119
sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g=
11201120
sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs=
11211121
sigs.k8s.io/kustomize/kustomize/v4 v4.2.0/go.mod h1:MOkR6fmhwG7hEDRXBYELTi5GSFcLwfqwzTRHW3kv5go=

‎internal/envtest/environment.go

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -160,17 +160,17 @@ func new(uncachedObjs ...client.Object) *Environment {
160160
filepath.Join(root, "controlplane", "kubeadm", "config", "crd", "bases"),
161161
filepath.Join(root, "bootstrap", "kubeadm", "config", "crd", "bases"),
162162
},
163-
CRDs: []apiextensionsv1.CustomResourceDefinition{
164-
*builder.GenericBootstrapConfigCRD.DeepCopy(),
165-
*builder.GenericBootstrapConfigTemplateCRD.DeepCopy(),
166-
*builder.GenericControlPlaneCRD.DeepCopy(),
167-
*builder.GenericControlPlaneTemplateCRD.DeepCopy(),
168-
*builder.GenericInfrastructureMachineCRD.DeepCopy(),
169-
*builder.GenericInfrastructureMachineTemplateCRD.DeepCopy(),
170-
*builder.GenericInfrastructureClusterCRD.DeepCopy(),
171-
*builder.GenericInfrastructureClusterTemplateCRD.DeepCopy(),
172-
*builder.GenericRemediationCRD.DeepCopy(),
173-
*builder.GenericRemediationTemplateCRD.DeepCopy(),
163+
CRDs: []*apiextensionsv1.CustomResourceDefinition{
164+
builder.GenericBootstrapConfigCRD.DeepCopy(),
165+
builder.GenericBootstrapConfigTemplateCRD.DeepCopy(),
166+
builder.GenericControlPlaneCRD.DeepCopy(),
167+
builder.GenericControlPlaneTemplateCRD.DeepCopy(),
168+
builder.GenericInfrastructureMachineCRD.DeepCopy(),
169+
builder.GenericInfrastructureMachineTemplateCRD.DeepCopy(),
170+
builder.GenericInfrastructureClusterCRD.DeepCopy(),
171+
builder.GenericInfrastructureClusterTemplateCRD.DeepCopy(),
172+
builder.GenericRemediationCRD.DeepCopy(),
173+
builder.GenericRemediationTemplateCRD.DeepCopy(),
174174
},
175175
// initialize webhook here to be able to test the envtest install via webhookOptions
176176
// This should set LocalServingCertDir and LocalServingPort that are used below.
@@ -274,7 +274,7 @@ func (e *Environment) start(ctx context.Context) {
274274
panic(fmt.Sprintf("Failed to start the test environment manager: %v", err))
275275
}
276276
}()
277-
e.Manager.Elected()
277+
<-e.Manager.Elected()
278278
e.WaitForWebhooks()
279279
}
280280

‎internal/envtest/webhooks.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ const (
3939
)
4040

4141
func initWebhookInstallOptions() envtest.WebhookInstallOptions {
42-
validatingWebhooks := []admissionv1.ValidatingWebhookConfiguration{}
43-
mutatingWebhooks := []admissionv1.MutatingWebhookConfiguration{}
42+
validatingWebhooks := []*admissionv1.ValidatingWebhookConfiguration{}
43+
mutatingWebhooks := []*admissionv1.MutatingWebhookConfiguration{}
4444

4545
// Get the root of the current file to use in CRD paths.
4646
_, filename, _, _ := goruntime.Caller(0) //nolint
@@ -86,7 +86,7 @@ func initWebhookInstallOptions() envtest.WebhookInstallOptions {
8686

8787
// Mutate the name of each webhook, because kubebuilder generates the same name for all controllers.
8888
// In normal usage, kustomize will prefix the controller name, which we have to do manually here.
89-
func appendWebhookConfiguration(mutatingWebhooks []admissionv1.MutatingWebhookConfiguration, validatingWebhooks []admissionv1.ValidatingWebhookConfiguration, configyamlFile []byte, tag string) ([]admissionv1.MutatingWebhookConfiguration, []admissionv1.ValidatingWebhookConfiguration, error) {
89+
func appendWebhookConfiguration(mutatingWebhooks []*admissionv1.MutatingWebhookConfiguration, validatingWebhooks []*admissionv1.ValidatingWebhookConfiguration, configyamlFile []byte, tag string) ([]*admissionv1.MutatingWebhookConfiguration, []*admissionv1.ValidatingWebhookConfiguration, error) {
9090
objs, err := utilyaml.ToUnstructured(configyamlFile)
9191
if err != nil {
9292
klog.Fatalf("failed to parse yaml")
@@ -104,7 +104,7 @@ func appendWebhookConfiguration(mutatingWebhooks []admissionv1.MutatingWebhookCo
104104
klog.Fatalf("failed to convert MutatingWebhookConfiguration %s", o.GetName())
105105
}
106106

107-
mutatingWebhooks = append(mutatingWebhooks, *webhook)
107+
mutatingWebhooks = append(mutatingWebhooks, webhook)
108108
}
109109
}
110110
if o.GetKind() == validatingWebhookKind {
@@ -117,7 +117,7 @@ func appendWebhookConfiguration(mutatingWebhooks []admissionv1.MutatingWebhookCo
117117
klog.Fatalf("failed to convert ValidatingWebhookConfiguration %s", o.GetName())
118118
}
119119

120-
validatingWebhooks = append(validatingWebhooks, *webhook)
120+
validatingWebhooks = append(validatingWebhooks, webhook)
121121
}
122122
}
123123
}

‎test/e2e/config/docker.yaml

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ providers:
3434
type: "url"
3535
contract: v1alpha3
3636
replacements:
37+
- old: "--leader-elect"
38+
new: "--leader-elect=false"
3739
- old: --metrics-addr=127.0.0.1:8080
3840
new: --metrics-addr=:8080
3941
files:
@@ -43,13 +45,17 @@ providers:
4345
type: "url"
4446
contract: v1alpha4
4547
replacements:
48+
- old: "--leader-elect"
49+
new: "--leader-elect=false"
4650
- old: --metrics-addr=127.0.0.1:8080
4751
new: --metrics-addr=:8080
4852
files:
4953
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
5054
- name: v1.0.99 # next; use manifest from source files
5155
value: ../../../config/default
5256
replacements:
57+
- old: "--leader-elect"
58+
new: "--leader-elect=false"
5359
- old: --metrics-bind-addr=localhost:8080
5460
new: --metrics-bind-addr=:8080
5561
files:
@@ -63,6 +69,8 @@ providers:
6369
type: "url"
6470
contract: v1alpha3
6571
replacements:
72+
- old: "--leader-elect"
73+
new: "--leader-elect=false"
6674
- old: --metrics-addr=127.0.0.1:8080
6775
new: --metrics-addr=:8080
6876
files:
@@ -72,13 +80,17 @@ providers:
7280
type: "url"
7381
contract: v1alpha4
7482
replacements:
83+
- old: "--leader-elect"
84+
new: "--leader-elect=false"
7585
- old: --metrics-addr=127.0.0.1:8080
7686
new: --metrics-addr=:8080
7787
files:
7888
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
7989
- name: v1.0.99 # next; use manifest from source files
8090
value: ../../../bootstrap/kubeadm/config/default
8191
replacements:
92+
- old: "--leader-elect"
93+
new: "--leader-elect=false"
8294
- old: --metrics-bind-addr=localhost:8080
8395
new: --metrics-bind-addr=:8080
8496
files:
@@ -92,6 +104,8 @@ providers:
92104
type: "url"
93105
contract: v1alpha3
94106
replacements:
107+
- old: "--leader-elect"
108+
new: "--leader-elect=false"
95109
- old: --metrics-addr=127.0.0.1:8080
96110
new: --metrics-addr=:8080
97111
files:
@@ -101,13 +115,17 @@ providers:
101115
type: "url"
102116
contract: v1alpha4
103117
replacements:
118+
- old: "--leader-elect"
119+
new: "--leader-elect=false"
104120
- old: --metrics-addr=127.0.0.1:8080
105121
new: --metrics-addr=:8080
106122
files:
107123
- sourcePath: "../data/shared/v1alpha4/metadata.yaml"
108124
- name: v1.0.99 # next; use manifest from source files
109125
value: ../../../controlplane/kubeadm/config/default
110126
replacements:
127+
- old: "--leader-elect"
128+
new: "--leader-elect=false"
111129
- old: --metrics-bind-addr=localhost:8080
112130
new: --metrics-bind-addr=:8080
113131
files:
@@ -121,6 +139,8 @@ providers:
121139
type: "url"
122140
contract: v1alpha3
123141
replacements:
142+
- old: "--leader-elect"
143+
new: "--leader-elect=false"
124144
- old: --metrics-addr=127.0.0.1:8080
125145
new: --metrics-addr=:8080
126146
files:
@@ -131,6 +151,8 @@ providers:
131151
type: "url"
132152
contract: v1alpha4
133153
replacements:
154+
- old: "--leader-elect"
155+
new: "--leader-elect=false"
134156
- old: --metrics-addr=127.0.0.1:8080
135157
new: --metrics-addr=:8080
136158
files:
@@ -139,6 +161,8 @@ providers:
139161
- name: v1.0.99 # next; use manifest from source files
140162
value: ../../../test/infrastructure/docker/config/default
141163
replacements:
164+
- old: "--leader-elect"
165+
new: "--leader-elect=false"
142166
- old: --metrics-bind-addr=localhost:8080
143167
new: --metrics-bind-addr=:8080
144168
files:
@@ -176,9 +200,9 @@ variables:
176200
NODE_DRAIN_TIMEOUT: "60s"
177201
# NOTE: INIT_WITH_BINARY and INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize
178202
# the management cluster to be upgraded.
179-
INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.4/clusterctl-{OS}-{ARCH}"
180-
INIT_WITH_PROVIDERS_CONTRACT: "v1alpha4"
181-
INIT_WITH_KUBERNETES_VERSION: "v1.22.0"
203+
INIT_WITH_BINARY: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.23/clusterctl-{OS}-{ARCH}"
204+
INIT_WITH_PROVIDERS_CONTRACT: "v1alpha3"
205+
INIT_WITH_KUBERNETES_VERSION: "v1.21.2"
182206

183207
intervals:
184208
default/wait-controllers: ["3m", "10s"]

‎test/framework/alltypes_helpers.go

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import (
2626

2727
. "github.com/onsi/ginkgo"
2828
. "github.com/onsi/gomega"
29-
3029
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
3130
apierrors "k8s.io/apimachinery/pkg/api/errors"
3231
apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -72,6 +71,36 @@ func GetCAPIResources(ctx context.Context, input GetCAPIResourcesInput) []*unstr
7271
}
7372
}
7473

74+
// cluster-wide
75+
types = []metav1.TypeMeta{
76+
{
77+
Kind: "Deployment",
78+
APIVersion: "apps/v1",
79+
},
80+
{
81+
Kind: "Pod",
82+
APIVersion: "v1",
83+
},
84+
}
85+
86+
for i := range types {
87+
typeMeta := types[i]
88+
typeList := new(unstructured.UnstructuredList)
89+
typeList.SetAPIVersion(typeMeta.APIVersion)
90+
typeList.SetKind(typeMeta.Kind)
91+
92+
if err := input.Lister.List(ctx, typeList); err != nil {
93+
if apierrors.IsNotFound(err) {
94+
continue
95+
}
96+
Fail(fmt.Sprintf("failed to list %q resources: %v", typeList.GroupVersionKind(), err))
97+
}
98+
for i := range typeList.Items {
99+
obj := typeList.Items[i]
100+
objList = append(objList, &obj)
101+
}
102+
}
103+
75104
return objList
76105
}
77106

0 commit comments

Comments
 (0)