From ee214586d17e9b1c2a64c09ab1afde25183f69dd Mon Sep 17 00:00:00 2001 From: Tao Yi Date: Tue, 24 Sep 2024 16:09:52 +0800 Subject: [PATCH] feat(konnect): KongTarget reconciler (#627) * Add KongTarget reconciler * fix apiauth ref and add unit tests * add assertions for updated entity * add KongTarget sample and do not delete when upstream being deleted --- .mockery.yaml | 1 + CHANGELOG.md | 2 + config/rbac/role/role.yaml | 2 + config/samples/konnect_kongtarget.yaml | 46 ++ controller/konnect/conditions/conditions.go | 14 + controller/konnect/constraints/constraints.go | 4 +- controller/konnect/errors.go | 25 + controller/konnect/ops/kongtarget.go | 14 + controller/konnect/ops/kongtarget_mock.go | 259 +++++++++++ controller/konnect/ops/ops.go | 8 + controller/konnect/ops/ops_kongtarget.go | 139 ++++++ controller/konnect/ops/sdkfactory.go | 6 + controller/konnect/ops/sdkfactory_mock.go | 6 + controller/konnect/reconciler_generic.go | 65 ++- controller/konnect/reconciler_generic_rbac.go | 3 + controller/konnect/reconciler_upstreamref.go | 190 ++++++++ .../konnect/reconciler_upstreamref_test.go | 429 ++++++++++++++++++ controller/konnect/watch.go | 2 + controller/konnect/watch_kongtarget.go | 99 ++++ modules/manager/controller_setup.go | 11 + test/integration/test_konnect_entities.go | 74 +++ 21 files changed, 1396 insertions(+), 3 deletions(-) create mode 100644 config/samples/konnect_kongtarget.yaml create mode 100644 controller/konnect/ops/kongtarget.go create mode 100644 controller/konnect/ops/kongtarget_mock.go create mode 100644 controller/konnect/ops/ops_kongtarget.go create mode 100644 controller/konnect/reconciler_upstreamref.go create mode 100644 controller/konnect/reconciler_upstreamref_test.go create mode 100644 controller/konnect/watch_kongtarget.go diff --git a/.mockery.yaml b/.mockery.yaml index 546f34022..5c99a5df7 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -18,6 +18,7 @@ packages: ConsumerGroupSDK: PluginSDK: UpstreamsSDK: + TargetsSDK: MeSDK: KongCredentialBasicAuthSDK: CACertificatesSDK: diff --git a/CHANGELOG.md b/CHANGELOG.md index 143297738..987cc0d04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,8 @@ [#516](https://github.com/Kong/gateway-operator/pull/516) - Add `KongPluginBinding` reconciler for Konnect Plugins. [#513](https://github.com/Kong/gateway-operator/pull/513), [#535](https://github.com/Kong/gateway-operator/pull/535) +- Add `KongTarget` reconciler for Konnect Targets. + [#627](https://github.com/Kong/gateway-operator/pull/627) - The `DataPlaneKonnectExtension` CRD has been introduced. Such a CRD can be attached to a `DataPlane` via the extensions field to have a konnect-flavored `DataPlane`. [#453](https://github.com/Kong/gateway-operator/pull/453), [#578](https://github.com/Kong/gateway-operator/pull/578) diff --git a/config/rbac/role/role.yaml b/config/rbac/role/role.yaml index 29942ec13..40ceb13d1 100644 --- a/config/rbac/role/role.yaml +++ b/config/rbac/role/role.yaml @@ -155,6 +155,7 @@ rules: - kongplugins/status - kongroutes/status - kongservices/status + - kongtargets/status - kongupstreampolicies/status - kongupstreams/status - kongvaults/status @@ -182,6 +183,7 @@ rules: resources: - kongroutes - kongservices + - kongtargets - kongupstreams verbs: - get diff --git a/config/samples/konnect_kongtarget.yaml b/config/samples/konnect_kongtarget.yaml new file mode 100644 index 000000000..7f56144c0 --- /dev/null +++ b/config/samples/konnect_kongtarget.yaml @@ -0,0 +1,46 @@ +kind: KonnectAPIAuthConfiguration +apiVersion: konnect.konghq.com/v1alpha1 +metadata: + name: konnect-api-auth-dev-1 + namespace: default +spec: + type: token + token: kpat_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + serverURL: us.api.konghq.com +--- +kind: KonnectGatewayControlPlane +apiVersion: konnect.konghq.com/v1alpha1 +metadata: + name: test1 + namespace: default +spec: + name: test1 + labels: + app: test1 + key1: test1 + konnect: + authRef: + name: konnect-api-auth-dev-1 +--- +kind: KongUpstream +apiVersion: configuration.konghq.com/v1alpha1 +metadata: + name: upstream-1 + namespace: default +spec: + name: upstream-1 + controlPlaneRef: + type: konnectNamespacedRef + konnectNamespacedRef: + name: test1 +--- +kind: KongTarget +apiVersion: configuration.konghq.com/v1alpha1 +metadata: + name: target-1 + namespace: default +spec: + upstreamRef: + name: upstream-1 + target: "10.0.0.1" + weight: 100 diff --git a/controller/konnect/conditions/conditions.go b/controller/konnect/conditions/conditions.go index c8bc5c9a8..555328207 100644 --- a/controller/konnect/conditions/conditions.go +++ b/controller/konnect/conditions/conditions.go @@ -114,3 +114,17 @@ const ( // condition type indicating that one or more KongConsumerGroup references are invalid. KongConsumerGroupRefsReasonInvalid = "Invalid" ) + +const ( + // KongUpstreamRefValidConditionType is the type of the condition that indicates + // whether the KongUpstream reference is valid and points to an existing + // KongUpstreamRefValid. + KongUpstreamRefValidConditionType = "KongUpstreamRefValid" + + // KongUpstreamRefReasonValid is the reason used with the KongUpstreamRefValid + // condition type indicating that the KongUpstream reference is valid. + KongUpstreamRefReasonValid = "Valid" + // KongUpstreamRefReasonInvalid is the reason used with the KongUpstreamRefValid + // condition type indicating that the KongUpstream reference is invalid. + KongUpstreamRefReasonInvalid = "Invalid" +) diff --git a/controller/konnect/constraints/constraints.go b/controller/konnect/constraints/constraints.go index 7716065db..77adced99 100644 --- a/controller/konnect/constraints/constraints.go +++ b/controller/konnect/constraints/constraints.go @@ -21,7 +21,9 @@ type SupportedKonnectEntityType interface { configurationv1alpha1.KongPluginBinding | configurationv1alpha1.KongCredentialBasicAuth | configurationv1alpha1.KongUpstream | - configurationv1alpha1.KongCACertificate + configurationv1alpha1.KongCACertificate | + configurationv1alpha1.KongTarget + // TODO: add other types GetTypeName() string diff --git a/controller/konnect/errors.go b/controller/konnect/errors.go index e38d1dba6..44d5d875e 100644 --- a/controller/konnect/errors.go +++ b/controller/konnect/errors.go @@ -61,3 +61,28 @@ type ReferencedKongConsumerDoesNotExist struct { func (e ReferencedKongConsumerDoesNotExist) Error() string { return fmt.Sprintf("referenced Kong Consumer %s does not exist: %v", e.Reference, e.Err) } + +// ReferencedKongUpstreamIsBeingDeleted is an error type that is returned when +// a Konnect entity references a Kong Upstream which is being deleted. +type ReferencedKongUpstreamIsBeingDeleted struct { + Reference types.NamespacedName + DeletionTimestamp time.Time +} + +// Error implements the error interface. +func (e ReferencedKongUpstreamIsBeingDeleted) Error() string { + return fmt.Sprintf("referenced Kong Upstream %s is being deleted (deletion timestamp: %s)", + e.Reference, e.DeletionTimestamp) +} + +// ReferencedKongUpstreamDoesNotExist is an error type that is returned when +// a Konnect entity references a Kong Upstream which does not exist. +type ReferencedKongUpstreamDoesNotExist struct { + Reference types.NamespacedName + Err error +} + +// Error implements the error interface. +func (e ReferencedKongUpstreamDoesNotExist) Error() string { + return fmt.Sprintf("referenced Kong Upstream %s does not exist: %v", e.Reference, e.Err) +} diff --git a/controller/konnect/ops/kongtarget.go b/controller/konnect/ops/kongtarget.go new file mode 100644 index 000000000..ed1326170 --- /dev/null +++ b/controller/konnect/ops/kongtarget.go @@ -0,0 +1,14 @@ +package ops + +import ( + "context" + + sdkkonnectops "github.com/Kong/sdk-konnect-go/models/operations" +) + +// TargetsSDK is the interface for the Konnect Taret SDK. +type TargetsSDK interface { + CreateTargetWithUpstream(ctx context.Context, req sdkkonnectops.CreateTargetWithUpstreamRequest, opts ...sdkkonnectops.Option) (*sdkkonnectops.CreateTargetWithUpstreamResponse, error) + UpsertTargetWithUpstream(ctx context.Context, req sdkkonnectops.UpsertTargetWithUpstreamRequest, opts ...sdkkonnectops.Option) (*sdkkonnectops.UpsertTargetWithUpstreamResponse, error) + DeleteTargetWithUpstream(ctx context.Context, req sdkkonnectops.DeleteTargetWithUpstreamRequest, opts ...sdkkonnectops.Option) (*sdkkonnectops.DeleteTargetWithUpstreamResponse, error) +} diff --git a/controller/konnect/ops/kongtarget_mock.go b/controller/konnect/ops/kongtarget_mock.go new file mode 100644 index 000000000..dbc1f71f3 --- /dev/null +++ b/controller/konnect/ops/kongtarget_mock.go @@ -0,0 +1,259 @@ +// Code generated by mockery. DO NOT EDIT. + +package ops + +import ( + context "context" + + operations "github.com/Kong/sdk-konnect-go/models/operations" + mock "github.com/stretchr/testify/mock" +) + +// MockTargetsSDK is an autogenerated mock type for the TargetsSDK type +type MockTargetsSDK struct { + mock.Mock +} + +type MockTargetsSDK_Expecter struct { + mock *mock.Mock +} + +func (_m *MockTargetsSDK) EXPECT() *MockTargetsSDK_Expecter { + return &MockTargetsSDK_Expecter{mock: &_m.Mock} +} + +// CreateTargetWithUpstream provides a mock function with given fields: ctx, req, opts +func (_m *MockTargetsSDK) CreateTargetWithUpstream(ctx context.Context, req operations.CreateTargetWithUpstreamRequest, opts ...operations.Option) (*operations.CreateTargetWithUpstreamResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, req) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for CreateTargetWithUpstream") + } + + var r0 *operations.CreateTargetWithUpstreamResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, operations.CreateTargetWithUpstreamRequest, ...operations.Option) (*operations.CreateTargetWithUpstreamResponse, error)); ok { + return rf(ctx, req, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, operations.CreateTargetWithUpstreamRequest, ...operations.Option) *operations.CreateTargetWithUpstreamResponse); ok { + r0 = rf(ctx, req, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*operations.CreateTargetWithUpstreamResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, operations.CreateTargetWithUpstreamRequest, ...operations.Option) error); ok { + r1 = rf(ctx, req, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockTargetsSDK_CreateTargetWithUpstream_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateTargetWithUpstream' +type MockTargetsSDK_CreateTargetWithUpstream_Call struct { + *mock.Call +} + +// CreateTargetWithUpstream is a helper method to define mock.On call +// - ctx context.Context +// - req operations.CreateTargetWithUpstreamRequest +// - opts ...operations.Option +func (_e *MockTargetsSDK_Expecter) CreateTargetWithUpstream(ctx interface{}, req interface{}, opts ...interface{}) *MockTargetsSDK_CreateTargetWithUpstream_Call { + return &MockTargetsSDK_CreateTargetWithUpstream_Call{Call: _e.mock.On("CreateTargetWithUpstream", + append([]interface{}{ctx, req}, opts...)...)} +} + +func (_c *MockTargetsSDK_CreateTargetWithUpstream_Call) Run(run func(ctx context.Context, req operations.CreateTargetWithUpstreamRequest, opts ...operations.Option)) *MockTargetsSDK_CreateTargetWithUpstream_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]operations.Option, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(operations.Option) + } + } + run(args[0].(context.Context), args[1].(operations.CreateTargetWithUpstreamRequest), variadicArgs...) + }) + return _c +} + +func (_c *MockTargetsSDK_CreateTargetWithUpstream_Call) Return(_a0 *operations.CreateTargetWithUpstreamResponse, _a1 error) *MockTargetsSDK_CreateTargetWithUpstream_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockTargetsSDK_CreateTargetWithUpstream_Call) RunAndReturn(run func(context.Context, operations.CreateTargetWithUpstreamRequest, ...operations.Option) (*operations.CreateTargetWithUpstreamResponse, error)) *MockTargetsSDK_CreateTargetWithUpstream_Call { + _c.Call.Return(run) + return _c +} + +// DeleteTargetWithUpstream provides a mock function with given fields: ctx, req, opts +func (_m *MockTargetsSDK) DeleteTargetWithUpstream(ctx context.Context, req operations.DeleteTargetWithUpstreamRequest, opts ...operations.Option) (*operations.DeleteTargetWithUpstreamResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, req) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for DeleteTargetWithUpstream") + } + + var r0 *operations.DeleteTargetWithUpstreamResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, operations.DeleteTargetWithUpstreamRequest, ...operations.Option) (*operations.DeleteTargetWithUpstreamResponse, error)); ok { + return rf(ctx, req, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, operations.DeleteTargetWithUpstreamRequest, ...operations.Option) *operations.DeleteTargetWithUpstreamResponse); ok { + r0 = rf(ctx, req, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*operations.DeleteTargetWithUpstreamResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, operations.DeleteTargetWithUpstreamRequest, ...operations.Option) error); ok { + r1 = rf(ctx, req, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockTargetsSDK_DeleteTargetWithUpstream_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteTargetWithUpstream' +type MockTargetsSDK_DeleteTargetWithUpstream_Call struct { + *mock.Call +} + +// DeleteTargetWithUpstream is a helper method to define mock.On call +// - ctx context.Context +// - req operations.DeleteTargetWithUpstreamRequest +// - opts ...operations.Option +func (_e *MockTargetsSDK_Expecter) DeleteTargetWithUpstream(ctx interface{}, req interface{}, opts ...interface{}) *MockTargetsSDK_DeleteTargetWithUpstream_Call { + return &MockTargetsSDK_DeleteTargetWithUpstream_Call{Call: _e.mock.On("DeleteTargetWithUpstream", + append([]interface{}{ctx, req}, opts...)...)} +} + +func (_c *MockTargetsSDK_DeleteTargetWithUpstream_Call) Run(run func(ctx context.Context, req operations.DeleteTargetWithUpstreamRequest, opts ...operations.Option)) *MockTargetsSDK_DeleteTargetWithUpstream_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]operations.Option, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(operations.Option) + } + } + run(args[0].(context.Context), args[1].(operations.DeleteTargetWithUpstreamRequest), variadicArgs...) + }) + return _c +} + +func (_c *MockTargetsSDK_DeleteTargetWithUpstream_Call) Return(_a0 *operations.DeleteTargetWithUpstreamResponse, _a1 error) *MockTargetsSDK_DeleteTargetWithUpstream_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockTargetsSDK_DeleteTargetWithUpstream_Call) RunAndReturn(run func(context.Context, operations.DeleteTargetWithUpstreamRequest, ...operations.Option) (*operations.DeleteTargetWithUpstreamResponse, error)) *MockTargetsSDK_DeleteTargetWithUpstream_Call { + _c.Call.Return(run) + return _c +} + +// UpsertTargetWithUpstream provides a mock function with given fields: ctx, req, opts +func (_m *MockTargetsSDK) UpsertTargetWithUpstream(ctx context.Context, req operations.UpsertTargetWithUpstreamRequest, opts ...operations.Option) (*operations.UpsertTargetWithUpstreamResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, req) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for UpsertTargetWithUpstream") + } + + var r0 *operations.UpsertTargetWithUpstreamResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, operations.UpsertTargetWithUpstreamRequest, ...operations.Option) (*operations.UpsertTargetWithUpstreamResponse, error)); ok { + return rf(ctx, req, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, operations.UpsertTargetWithUpstreamRequest, ...operations.Option) *operations.UpsertTargetWithUpstreamResponse); ok { + r0 = rf(ctx, req, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*operations.UpsertTargetWithUpstreamResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, operations.UpsertTargetWithUpstreamRequest, ...operations.Option) error); ok { + r1 = rf(ctx, req, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockTargetsSDK_UpsertTargetWithUpstream_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpsertTargetWithUpstream' +type MockTargetsSDK_UpsertTargetWithUpstream_Call struct { + *mock.Call +} + +// UpsertTargetWithUpstream is a helper method to define mock.On call +// - ctx context.Context +// - req operations.UpsertTargetWithUpstreamRequest +// - opts ...operations.Option +func (_e *MockTargetsSDK_Expecter) UpsertTargetWithUpstream(ctx interface{}, req interface{}, opts ...interface{}) *MockTargetsSDK_UpsertTargetWithUpstream_Call { + return &MockTargetsSDK_UpsertTargetWithUpstream_Call{Call: _e.mock.On("UpsertTargetWithUpstream", + append([]interface{}{ctx, req}, opts...)...)} +} + +func (_c *MockTargetsSDK_UpsertTargetWithUpstream_Call) Run(run func(ctx context.Context, req operations.UpsertTargetWithUpstreamRequest, opts ...operations.Option)) *MockTargetsSDK_UpsertTargetWithUpstream_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]operations.Option, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(operations.Option) + } + } + run(args[0].(context.Context), args[1].(operations.UpsertTargetWithUpstreamRequest), variadicArgs...) + }) + return _c +} + +func (_c *MockTargetsSDK_UpsertTargetWithUpstream_Call) Return(_a0 *operations.UpsertTargetWithUpstreamResponse, _a1 error) *MockTargetsSDK_UpsertTargetWithUpstream_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockTargetsSDK_UpsertTargetWithUpstream_Call) RunAndReturn(run func(context.Context, operations.UpsertTargetWithUpstreamRequest, ...operations.Option) (*operations.UpsertTargetWithUpstreamResponse, error)) *MockTargetsSDK_UpsertTargetWithUpstream_Call { + _c.Call.Return(run) + return _c +} + +// NewMockTargetsSDK creates a new instance of MockTargetsSDK. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockTargetsSDK(t interface { + mock.TestingT + Cleanup(func()) +}) *MockTargetsSDK { + mock := &MockTargetsSDK{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/controller/konnect/ops/ops.go b/controller/konnect/ops/ops.go index ba9f06a86..c38121d94 100644 --- a/controller/konnect/ops/ops.go +++ b/controller/konnect/ops/ops.go @@ -70,6 +70,8 @@ func Create[ return e, createKongCredentialBasicAuth(ctx, sdk.GetBasicAuthCredentials(), ent) case *configurationv1alpha1.KongCACertificate: return e, createCACertificate(ctx, sdk.GetCACertificatesSDK(), ent) + case *configurationv1alpha1.KongTarget: + return e, createTarget(ctx, sdk.GetTargetsSDK(), ent) // --------------------------------------------------------------------- // TODO: add other Konnect types @@ -114,6 +116,9 @@ func Delete[ return deleteKongCredentialBasicAuth(ctx, sdk.GetBasicAuthCredentials(), ent) case *configurationv1alpha1.KongCACertificate: return deleteCACertificate(ctx, sdk.GetCACertificatesSDK(), ent) + case *configurationv1alpha1.KongTarget: + return deleteTarget(ctx, sdk.GetTargetsSDK(), ent) + // --------------------------------------------------------------------- // TODO: add other Konnect types @@ -182,6 +187,9 @@ func Update[ return ctrl.Result{}, updateKongCredentialBasicAuth(ctx, sdk.GetBasicAuthCredentials(), ent) case *configurationv1alpha1.KongCACertificate: return ctrl.Result{}, updateCACertificate(ctx, sdk.GetCACertificatesSDK(), ent) + case *configurationv1alpha1.KongTarget: + return ctrl.Result{}, updateTarget(ctx, sdk.GetTargetsSDK(), ent) + // --------------------------------------------------------------------- // TODO: add other Konnect types diff --git a/controller/konnect/ops/ops_kongtarget.go b/controller/konnect/ops/ops_kongtarget.go new file mode 100644 index 000000000..bb01ea60b --- /dev/null +++ b/controller/konnect/ops/ops_kongtarget.go @@ -0,0 +1,139 @@ +package ops + +import ( + "context" + "errors" + "fmt" + "net/http" + "slices" + + sdkkonnectcomp "github.com/Kong/sdk-konnect-go/models/components" + sdkkonnectops "github.com/Kong/sdk-konnect-go/models/operations" + sdkkonnecterrs "github.com/Kong/sdk-konnect-go/models/sdkerrors" + "github.com/samber/lo" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" + "github.com/kong/kubernetes-configuration/pkg/metadata" +) + +func createTarget( + ctx context.Context, + sdk TargetsSDK, + target *configurationv1alpha1.KongTarget, +) error { + cpID := target.GetControlPlaneID() + if cpID == "" { + return fmt.Errorf("can't create %T %s without a Konnect ControlPlane ID", target, client.ObjectKeyFromObject(target)) + } + if target.Status.Konnect == nil || target.Status.Konnect.UpstreamID == "" { + return fmt.Errorf("can't create %T %s without a Konnect Upstream ID", target, client.ObjectKeyFromObject(target)) + } + + resp, err := sdk.CreateTargetWithUpstream(ctx, sdkkonnectops.CreateTargetWithUpstreamRequest{ + ControlPlaneID: cpID, + UpstreamIDForTarget: target.Status.Konnect.UpstreamID, + TargetWithoutParents: kongTargetToTargetWithoutParents(target), + }) + + if errWrapped := wrapErrIfKonnectOpFailed(err, CreateOp, target); errWrapped != nil { + SetKonnectEntityProgrammedConditionFalse(target, "FailedToCreate", errWrapped.Error()) + return errWrapped + } + + target.Status.Konnect.SetKonnectID(*resp.Target.ID) + SetKonnectEntityProgrammedCondition(target) + + return nil +} + +func updateTarget( + ctx context.Context, + sdk TargetsSDK, + target *configurationv1alpha1.KongTarget, +) error { + cpID := target.GetControlPlaneID() + if cpID == "" { + return fmt.Errorf("can't update %T %s without a Konnect ControlPlane ID", target, client.ObjectKeyFromObject(target)) + } + if target.Status.Konnect == nil || target.Status.Konnect.UpstreamID == "" { + return fmt.Errorf("can't update %T %s without a Konnect Upstream ID", target, client.ObjectKeyFromObject(target)) + } + + _, err := sdk.UpsertTargetWithUpstream(ctx, sdkkonnectops.UpsertTargetWithUpstreamRequest{ + ControlPlaneID: cpID, + UpstreamIDForTarget: target.Status.Konnect.UpstreamID, + TargetID: target.GetKonnectID(), + TargetWithoutParents: kongTargetToTargetWithoutParents(target), + }) + + if errWrapped := wrapErrIfKonnectOpFailed(err, UpdateOp, target); errWrapped != nil { + SetKonnectEntityProgrammedConditionFalse(target, "FailedToUpdate", errWrapped.Error()) + return errWrapped + } + + SetKonnectEntityProgrammedCondition(target) + return nil +} + +func deleteTarget( + ctx context.Context, + sdk TargetsSDK, + target *configurationv1alpha1.KongTarget, +) error { + cpID := target.GetControlPlaneID() + if cpID == "" { + return fmt.Errorf("can't update %T %s without a Konnect ControlPlane ID", target, client.ObjectKeyFromObject(target)) + } + if target.Status.Konnect == nil || target.Status.Konnect.UpstreamID == "" { + return fmt.Errorf("can't update %T %s without a Konnect Upstream ID", target, client.ObjectKeyFromObject(target)) + } + id := target.GetKonnectID() + + _, err := sdk.DeleteTargetWithUpstream(ctx, sdkkonnectops.DeleteTargetWithUpstreamRequest{ + ControlPlaneID: cpID, + UpstreamIDForTarget: target.Status.Konnect.UpstreamID, + TargetID: id, + }) + + if errWrapped := wrapErrIfKonnectOpFailed(err, DeleteOp, target); errWrapped != nil { + // Service delete operation returns an SDKError instead of a NotFoundError. + var sdkError *sdkkonnecterrs.SDKError + if errors.As(errWrapped, &sdkError) { + if sdkError.StatusCode == http.StatusNotFound { + ctrllog.FromContext(ctx). + Info("entity not found in Konnect, skipping delete", + "op", DeleteOp, "type", target.GetTypeName(), "id", id, + ) + return nil + } + return FailedKonnectOpError[configurationv1alpha1.KongTarget]{ + Op: DeleteOp, + Err: sdkError, + } + } + return FailedKonnectOpError[configurationv1alpha1.KongTarget]{ + Op: DeleteOp, + Err: errWrapped, + } + } + + return nil +} + +func kongTargetToTargetWithoutParents(target *configurationv1alpha1.KongTarget) sdkkonnectcomp.TargetWithoutParents { + var ( + specTags = target.Spec.KongTargetAPISpec.Tags + annotationTags = metadata.ExtractTags(target) + k8sTags = GenerateKubernetesMetadataTags(target) + ) + // Deduplicate tags to avoid rejection by Konnect. + tags := lo.Uniq(slices.Concat(specTags, annotationTags, k8sTags)) + + return sdkkonnectcomp.TargetWithoutParents{ + Target: lo.ToPtr(target.Spec.Target), + Weight: lo.ToPtr(int64(target.Spec.Weight)), + Tags: tags, + } +} diff --git a/controller/konnect/ops/sdkfactory.go b/controller/konnect/ops/sdkfactory.go index fe365716e..94232ba70 100644 --- a/controller/konnect/ops/sdkfactory.go +++ b/controller/konnect/ops/sdkfactory.go @@ -14,6 +14,7 @@ type SDKWrapper interface { GetConsumerGroupsSDK() ConsumerGroupSDK GetPluginSDK() PluginSDK GetUpstreamsSDK() UpstreamsSDK + GetTargetsSDK() TargetsSDK GetMeSDK() MeSDK GetBasicAuthCredentials() KongCredentialBasicAuthSDK GetCACertificatesSDK() CACertificatesSDK @@ -60,6 +61,11 @@ func (w sdkWrapper) GetUpstreamsSDK() UpstreamsSDK { return w.sdk.Upstreams } +// GetTargetsSDK returns the SDK to operate Targets. +func (w sdkWrapper) GetTargetsSDK() TargetsSDK { + return w.sdk.Targets +} + // GetMeSDK returns the "me" SDK to get current organization. func (w sdkWrapper) GetMeSDK() MeSDK { return w.sdk.Me diff --git a/controller/konnect/ops/sdkfactory_mock.go b/controller/konnect/ops/sdkfactory_mock.go index d701f5aca..bddf68c60 100644 --- a/controller/konnect/ops/sdkfactory_mock.go +++ b/controller/konnect/ops/sdkfactory_mock.go @@ -14,6 +14,7 @@ type MockSDKWrapper struct { ConsumerGroupSDK *MockConsumerGroupSDK PluginSDK *MockPluginSDK UpstreamsSDK *MockUpstreamsSDK + TargetsSDK *MockTargetsSDK MeSDK *MockMeSDK BasicAuthCredentials *MockKongCredentialBasicAuthSDK CACertificatesSDK *MockCACertificatesSDK @@ -30,6 +31,7 @@ func NewMockSDKWrapperWithT(t *testing.T) *MockSDKWrapper { ConsumerGroupSDK: NewMockConsumerGroupSDK(t), PluginSDK: NewMockPluginSDK(t), UpstreamsSDK: NewMockUpstreamsSDK(t), + TargetsSDK: NewMockTargetsSDK(t), MeSDK: NewMockMeSDK(t), BasicAuthCredentials: NewMockKongCredentialBasicAuthSDK(t), CACertificatesSDK: NewMockCACertificatesSDK(t), @@ -68,6 +70,10 @@ func (m MockSDKWrapper) GetBasicAuthCredentials() KongCredentialBasicAuthSDK { return m.BasicAuthCredentials } +func (m MockSDKWrapper) GetTargetsSDK() TargetsSDK { + return m.TargetsSDK +} + func (m MockSDKWrapper) GetMeSDK() MeSDK { return m.MeSDK } diff --git a/controller/konnect/reconciler_generic.go b/controller/konnect/reconciler_generic.go index b9ef41023..788582ea2 100644 --- a/controller/konnect/reconciler_generic.go +++ b/controller/konnect/reconciler_generic.go @@ -211,6 +211,45 @@ func (r *KonnectEntityReconciler[T, TEnt]) Reconcile( return res, nil } + // If a type has a KongUpstream ref (KongTarget), handle it. + res, err = handleKongUpstreamRef(ctx, r.Client, ent) + if err != nil { + // If the referenced KongUpstream is being deleted and the object + // is not being deleted yet then requeue until it will + // get the deletion timestamp set due to having the owner set to KongUpstream. + if errDel := (&ReferencedKongUpstreamIsBeingDeleted{}); errors.As(err, errDel) && + ent.GetDeletionTimestamp().IsZero() { + return ctrl.Result{ + RequeueAfter: time.Until(errDel.DeletionTimestamp), + }, nil + } + + // If the referenced KongUpstream is not found or is being deleted + // and the object is being deleted, remove the finalizer and let the + // deletion proceed without trying to delete the entity from Konnect + // as the KongUpstream deletion will take care of it on the Konnect side. + if errors.As(err, &ReferencedKongUpstreamIsBeingDeleted{}) || + errors.As(err, &ReferencedKongUpstreamDoesNotExist{}) { + if !ent.GetDeletionTimestamp().IsZero() { + if controllerutil.RemoveFinalizer(ent, KonnectCleanupFinalizer) { + if err := r.Client.Update(ctx, ent); err != nil { + if k8serrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to remove finalizer %s: %w", KonnectCleanupFinalizer, err) + } + log.Debug(logger, "finalizer removed as the owning KongUpstream is being deleted or is already gone", ent, + "finalizer", KonnectCleanupFinalizer, + ) + } + } + } + + return ctrl.Result{}, err + } else if res.Requeue { + return res, nil + } + apiAuthRef, err := getAPIAuthRefNN(ctx, r.Client, ent) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to get APIAuth ref for %s: %w", client.ObjectKeyFromObject(ent), err) @@ -492,7 +531,7 @@ func getCPForRef( var cp konnectv1alpha1.KonnectGatewayControlPlane if err := cl.Get(ctx, nn, &cp); err != nil { - return nil, fmt.Errorf("failed to get ControlPlane %s", nn) + return nil, fmt.Errorf("failed to get ControlPlane %s: %w", nn, err) } return &cp, nil } @@ -574,10 +613,31 @@ func getAPIAuthRefNN[T constraints.SupportedKonnectEntityType, TEnt constraints. return getCPAuthRefForRef(ctx, cl, cpRef, ent.GetNamespace()) } + // If the entity has a KongUpstreamRef, get the KonnectAPIAuthConfiguration + // ref from the referenced KongUpstream. + upsteramRef, ok := getKongUpstreamRef(ent).Get() + if ok { + nn := types.NamespacedName{ + Name: upsteramRef.Name, + Namespace: ent.GetNamespace(), + } + + var upstream configurationv1alpha1.KongUpstream + if err := cl.Get(ctx, nn, &upstream); err != nil { + return types.NamespacedName{}, fmt.Errorf("failed to get KongUpstream %s", nn) + } + + cpRef, ok := getControlPlaneRef(&upstream).Get() + if !ok { + return types.NamespacedName{}, fmt.Errorf("KongUpstream %s does not have a ControlPlaneRef", nn) + } + return getCPAuthRefForRef(ctx, cl, cpRef, ent.GetNamespace()) + } + if ref, ok := any(ent).(constraints.EntityWithKonnectAPIAuthConfigurationRef); ok { return types.NamespacedName{ Name: ref.GetKonnectAPIAuthConfigurationRef().Name, - // TODO(pmalek): enable if cross namespace refs are allowed + // TODO: enable if cross namespace refs are allowed Namespace: ent.GetNamespace(), }, nil } @@ -923,6 +983,7 @@ func getControlPlaneRef[T constraints.SupportedKonnectEntityType, TEnt constrain switch e := any(e).(type) { case *konnectv1alpha1.KonnectGatewayControlPlane, *configurationv1alpha1.KongRoute, + *configurationv1alpha1.KongTarget, *configurationv1alpha1.KongCredentialBasicAuth: return mo.None[configurationv1alpha1.ControlPlaneRef]() case *configurationv1.KongConsumer: diff --git a/controller/konnect/reconciler_generic_rbac.go b/controller/konnect/reconciler_generic_rbac.go index e091efb4a..004a7c094 100644 --- a/controller/konnect/reconciler_generic_rbac.go +++ b/controller/konnect/reconciler_generic_rbac.go @@ -18,6 +18,9 @@ package konnect //+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongupstreams,verbs=get;list;watch;update;patch //+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongupstreams/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongtargets,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongtargets/status,verbs=get;update;patch + //+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongconsumers,verbs=get;list;watch //+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongconsumers/status,verbs=get;update;patch diff --git a/controller/konnect/reconciler_upstreamref.go b/controller/konnect/reconciler_upstreamref.go new file mode 100644 index 000000000..774c37049 --- /dev/null +++ b/controller/konnect/reconciler_upstreamref.go @@ -0,0 +1,190 @@ +package konnect + +import ( + "context" + "fmt" + + "github.com/samber/mo" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/kong/gateway-operator/controller/konnect/conditions" + "github.com/kong/gateway-operator/controller/konnect/constraints" + k8sutils "github.com/kong/gateway-operator/pkg/utils/kubernetes" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" + konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" +) + +// getKongUpstreamRef gets the reference of KongUpstream. +func getKongUpstreamRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]]( + e TEnt, +) mo.Option[configurationv1alpha1.TargetRef] { + switch e := any(e).(type) { + case *configurationv1alpha1.KongTarget: + // Since upstreamRef is required for KongTarget, we directly return spec.UpstreamRef here. + return mo.Some(e.Spec.UpstreamRef) + default: + return mo.None[configurationv1alpha1.TargetRef]() + } +} + +// handleKongUpstreamRef handles KongUpstream reference if the entity references a KongUpstream. +// Now applies to KongTarget. +func handleKongUpstreamRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]]( + ctx context.Context, + cl client.Client, + ent TEnt, +) (ctrl.Result, error) { + upstreamRef, ok := getKongUpstreamRef(ent).Get() + if !ok { + return ctrl.Result{}, nil + } + + kongUpstream := &configurationv1alpha1.KongUpstream{} + nn := types.NamespacedName{ + Name: upstreamRef.Name, + // TODO: handle cross namespace refs + Namespace: ent.GetNamespace(), + } + err := cl.Get(ctx, nn, kongUpstream) + if err != nil { + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.KongUpstreamRefValidConditionType, + metav1.ConditionFalse, + conditions.KongUpstreamRefReasonInvalid, + err.Error(), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + return ctrl.Result{}, ReferencedKongUpstreamDoesNotExist{ + Reference: nn, + Err: err, + } + } + + // If referenced KongUpstream is being deleted, return an error so that we + // can remove the entity from Konnect first. + if delTimestamp := kongUpstream.GetDeletionTimestamp(); !delTimestamp.IsZero() { + return ctrl.Result{}, ReferencedKongUpstreamIsBeingDeleted{ + Reference: nn, + DeletionTimestamp: delTimestamp.Time, + } + } + + // requeue it if referenced KongUpstream is not programmed yet so we cannot do the following work. + cond, ok := k8sutils.GetCondition(conditions.KonnectEntityProgrammedConditionType, kongUpstream) + if !ok || cond.Status != metav1.ConditionTrue { + ent.SetKonnectID("") + if res, err := updateStatusWithCondition( + ctx, cl, ent, + conditions.KongUpstreamRefValidConditionType, + metav1.ConditionFalse, + conditions.KongUpstreamRefReasonInvalid, + fmt.Sprintf("Referenced KongUpstream %s is not programmed yet", nn), + ); err != nil || res.Requeue { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + + // Set owner reference of referenced KongUpstream and the reconciled entity. + old := ent.DeepCopyObject().(TEnt) + if err := controllerutil.SetOwnerReference(kongUpstream, ent, cl.Scheme(), controllerutil.WithBlockOwnerDeletion(true)); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to set owner reference: %w", err) + } + if err := cl.Patch(ctx, ent, client.MergeFrom(old)); err != nil { + if k8serrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to update status: %w", err) + } + + // TODO: make this more generic. + if target, ok := any(ent).(*configurationv1alpha1.KongTarget); ok { + if target.Status.Konnect == nil { + target.Status.Konnect = &konnectv1alpha1.KonnectEntityStatusWithControlPlaneAndUpstreamRefs{} + } + target.Status.Konnect.UpstreamID = kongUpstream.GetKonnectID() + } + + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.KongUpstreamRefValidConditionType, + metav1.ConditionTrue, + conditions.KongUpstreamRefReasonValid, + fmt.Sprintf("Referenced KongUpstream %s programmed", nn), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + cpRef, ok := getControlPlaneRef(kongUpstream).Get() + // TODO: ignore the entity if referenced KongUpstream does not have a Konnect control plane reference + // because this situation is likely to mean that they are not controlled by us: + // https://github.com/Kong/gateway-operator/issues/629 + if !ok { + return ctrl.Result{}, fmt.Errorf( + "%T references a KongUpstream %s which does not have a ControlPlane ref", + ent, client.ObjectKeyFromObject(kongUpstream), + ) + } + cp, err := getCPForRef(ctx, cl, cpRef, ent.GetNamespace()) + if err != nil { + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.ControlPlaneRefValidConditionType, + metav1.ConditionFalse, + conditions.ControlPlaneRefReasonInvalid, + err.Error(), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + if k8serrors.IsNotFound(err) { + return ctrl.Result{}, ReferencedControlPlaneDoesNotExistError{ + Reference: types.NamespacedName{ + Namespace: ent.GetNamespace(), + Name: cpRef.KonnectNamespacedRef.Name, + }, + Err: err, + } + } + return ctrl.Result{}, err + } + + cond, ok = k8sutils.GetCondition(conditions.KonnectEntityProgrammedConditionType, cp) + if !ok || cond.Status != metav1.ConditionTrue || cond.ObservedGeneration != cp.GetGeneration() { + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.ControlPlaneRefValidConditionType, + metav1.ConditionFalse, + conditions.ControlPlaneRefReasonInvalid, + fmt.Sprintf("Referenced ControlPlane %s is not programmed yet", nn), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + return ctrl.Result{Requeue: true}, nil + } + + if resource, ok := any(ent).(EntityWithControlPlaneRef); ok { + resource.SetControlPlaneID(cp.Status.ID) + } + + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.ControlPlaneRefValidConditionType, + metav1.ConditionTrue, + conditions.ControlPlaneRefReasonValid, + fmt.Sprintf("Referenced ControlPlane %s is programmed", nn), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + return ctrl.Result{}, nil +} diff --git a/controller/konnect/reconciler_upstreamref_test.go b/controller/konnect/reconciler_upstreamref_test.go new file mode 100644 index 000000000..20629ba2d --- /dev/null +++ b/controller/konnect/reconciler_upstreamref_test.go @@ -0,0 +1,429 @@ +package konnect + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/samber/lo" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kong/gateway-operator/controller/konnect/conditions" + "github.com/kong/gateway-operator/controller/konnect/constraints" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" + konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" +) + +type handleUpstreamRefTestCase[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]] struct { + name string + ent TEnt + objects []client.Object + expectResult ctrl.Result + expectError bool + expectErrorContains string + // Returns true if the updated entity satisfy the assertion. + // Returns false and error message if entity fails to satisfy it. + updatedEntAssertions []func(TEnt) (ok bool, message string) +} + +var testKongUpstreamOK = &configurationv1alpha1.KongUpstream{ + ObjectMeta: metav1.ObjectMeta{ + Name: "upstream-ok", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongUpstreamSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: "cp-ok", + }, + }, + KongUpstreamAPISpec: configurationv1alpha1.KongUpstreamAPISpec{ + Slots: lo.ToPtr(int64(12345)), + }, + }, + Status: configurationv1alpha1.KongUpstreamStatus{ + Konnect: &konnectv1alpha1.KonnectEntityStatusWithControlPlaneRef{ + KonnectEntityStatus: konnectv1alpha1.KonnectEntityStatus{ + ID: "12345", + }, + ControlPlaneID: "123456789", + }, + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionTrue, + }, + }, + }, +} + +var testKongUpstreamNotProgrammed = &configurationv1alpha1.KongUpstream{ + ObjectMeta: metav1.ObjectMeta{ + Name: "upstream-not-programmed", + Namespace: "default", + }, + Status: configurationv1alpha1.KongUpstreamStatus{ + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionFalse, + }, + }, + }, +} + +var testKongUpstreamNoControlPlaneRef = &configurationv1alpha1.KongUpstream{ + ObjectMeta: metav1.ObjectMeta{ + Name: "upstream-no-cp-ref", + Namespace: "default", + }, + Status: configurationv1alpha1.KongUpstreamStatus{ + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionTrue, + }, + }, + }, +} + +var testKongUpstreamBeingDeleted = &configurationv1alpha1.KongUpstream{ + ObjectMeta: metav1.ObjectMeta{ + Name: "upstream-being-deleted", + Namespace: "default", + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Finalizers: []string{"target-0"}, + }, +} + +var testKongUpstreamControlPlaneRefNotFound = &configurationv1alpha1.KongUpstream{ + ObjectMeta: metav1.ObjectMeta{ + Name: "upstream-cpref-not-found", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongUpstreamSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: "cp-not-found", + }, + }, + KongUpstreamAPISpec: configurationv1alpha1.KongUpstreamAPISpec{ + Slots: lo.ToPtr(int64(12345)), + }, + }, + Status: configurationv1alpha1.KongUpstreamStatus{ + Konnect: &konnectv1alpha1.KonnectEntityStatusWithControlPlaneRef{ + KonnectEntityStatus: konnectv1alpha1.KonnectEntityStatus{ + ID: "12345", + }, + ControlPlaneID: "123456789", + }, + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionTrue, + }, + }, + }, +} + +var testKongUpstreamControlPlaneRefNotProgrammed = &configurationv1alpha1.KongUpstream{ + ObjectMeta: metav1.ObjectMeta{ + Name: "upstream-cpref-not-programmed", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongUpstreamSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: "cp-not-programmed", + }, + }, + KongUpstreamAPISpec: configurationv1alpha1.KongUpstreamAPISpec{ + Slots: lo.ToPtr(int64(12345)), + }, + }, + Status: configurationv1alpha1.KongUpstreamStatus{ + Konnect: &konnectv1alpha1.KonnectEntityStatusWithControlPlaneRef{ + KonnectEntityStatus: konnectv1alpha1.KonnectEntityStatus{ + ID: "12345", + }, + ControlPlaneID: "123456789", + }, + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionTrue, + }, + }, + }, +} + +var testControlPlaneOK = &konnectv1alpha1.KonnectGatewayControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cp-ok", + Namespace: "default", + }, + Spec: konnectv1alpha1.KonnectGatewayControlPlaneSpec{}, + Status: konnectv1alpha1.KonnectGatewayControlPlaneStatus{ + KonnectEntityStatus: konnectv1alpha1.KonnectEntityStatus{ + ID: "123456789", + }, + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionTrue, + }, + }, + }, +} + +var testControlPlaneNotProgrammed = &konnectv1alpha1.KonnectGatewayControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cp-not-programmed", + Namespace: "default", + }, + Spec: konnectv1alpha1.KonnectGatewayControlPlaneSpec{}, + Status: konnectv1alpha1.KonnectGatewayControlPlaneStatus{ + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionFalse, + }, + }, + }, +} + +func TestHandleUpstreamRef(t *testing.T) { + // The test cases here includes test cases for handling upstream ref for KongTarget, which are expected to have KongUpstream reference. + // We can define test cases for other types and call `testHandleUpstreamRef` to test handling entities with other types. + testCases := []handleUpstreamRefTestCase[configurationv1alpha1.KongTarget, *configurationv1alpha1.KongTarget]{ + { + name: "has upstream ref and control plane ref", + ent: &configurationv1alpha1.KongTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "target-ok", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongTargetSpec{ + UpstreamRef: configurationv1alpha1.TargetRef{ + Name: "upstream-ok", + }, + }, + }, + objects: []client.Object{ + testKongUpstreamOK, + testControlPlaneOK, + }, + expectResult: ctrl.Result{}, + expectError: false, + updatedEntAssertions: []func(*configurationv1alpha1.KongTarget) (bool, string){ + func(kt *configurationv1alpha1.KongTarget) (bool, string) { + return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.KongUpstreamRefValidConditionType && c.Status == metav1.ConditionTrue + }), "KongTarget does not have KongUpsteamRefValid condition set to True" + }, + func(kt *configurationv1alpha1.KongTarget) (bool, string) { + return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.ControlPlaneRefValidConditionType && c.Status == metav1.ConditionTrue + }), "KongTarget does not have ControlPlaneRefValid condition set to True" + }, + func(kt *configurationv1alpha1.KongTarget) (bool, string) { + return lo.ContainsBy(kt.OwnerReferences, func(o metav1.OwnerReference) bool { + return o.Kind == "KongUpstream" && o.Name == "upstream-ok" + }), "OwnerReference of KongTarget is not set" + }, + }, + }, + { + name: "upstream ref not found", + ent: &configurationv1alpha1.KongTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "target-upstream-notfound", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongTargetSpec{ + UpstreamRef: configurationv1alpha1.TargetRef{ + Name: "upstream-nonexist", + }, + }, + }, + expectError: true, + expectErrorContains: "referenced Kong Upstream default/upstream-nonexist does not exist", + updatedEntAssertions: []func(*configurationv1alpha1.KongTarget) (bool, string){ + func(kt *configurationv1alpha1.KongTarget) (bool, string) { + return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.KongUpstreamRefValidConditionType && c.Status == metav1.ConditionFalse + }), "KongTarget does not have KongUpsteamRefValid condition set to False" + }, + }, + }, + { + name: "referenced KongUpstream not programmed", + ent: &configurationv1alpha1.KongTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "target-upstream-not-programmed", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongTargetSpec{ + UpstreamRef: configurationv1alpha1.TargetRef{ + Name: "upstream-not-programmed", + }, + }, + }, + objects: []client.Object{testKongUpstreamNotProgrammed}, + expectError: false, + expectResult: ctrl.Result{Requeue: true}, + updatedEntAssertions: []func(*configurationv1alpha1.KongTarget) (bool, string){ + func(kt *configurationv1alpha1.KongTarget) (bool, string) { + return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.KongUpstreamRefValidConditionType && c.Status == metav1.ConditionFalse && + c.Message == fmt.Sprintf("Referenced KongUpstream %s/%s is not programmed yet", + testKongUpstreamNotProgrammed.Namespace, testKongUpstreamNotProgrammed.Name) + }), "KongTarget does not have KongUpsteamRefValid condition set to False" + }, + }, + }, + { + name: "referenced KongUpstream has no ControlPlaneRef", + ent: &configurationv1alpha1.KongTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "target-upstream-no-cpref", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongTargetSpec{ + UpstreamRef: configurationv1alpha1.TargetRef{ + Name: "upstream-no-cp-ref", + }, + }, + }, + objects: []client.Object{testKongUpstreamNoControlPlaneRef}, + expectError: true, + expectErrorContains: fmt.Sprintf("references a KongUpstream %s/%s which does not have a ControlPlane ref", + testKongUpstreamNoControlPlaneRef.Namespace, testKongUpstreamNoControlPlaneRef.Name), + updatedEntAssertions: []func(*configurationv1alpha1.KongTarget) (bool, string){ + func(kt *configurationv1alpha1.KongTarget) (bool, string) { + return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.KongUpstreamRefValidConditionType && c.Status == metav1.ConditionTrue + }), "KongTarget does not have KongUpsteamRefValid condition set to True" + }, + }, + }, + { + name: "referenced KongUpstream is being deleted", + ent: &configurationv1alpha1.KongTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "target-upstream-being-deleted", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongTargetSpec{ + UpstreamRef: configurationv1alpha1.TargetRef{ + Name: "upstream-being-deleted", + }, + }, + }, + objects: []client.Object{testKongUpstreamBeingDeleted}, + expectError: true, + expectErrorContains: fmt.Sprintf("referenced Kong Upstream %s/%s is being deleted", testKongUpstreamBeingDeleted.Namespace, testKongUpstreamBeingDeleted.Name), + }, + { + name: "ControlPlaneRef not found", + ent: &configurationv1alpha1.KongTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "target-upstream-cpref-not-found", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongTargetSpec{ + UpstreamRef: configurationv1alpha1.TargetRef{ + Name: "upstream-cpref-not-found", + }, + }, + }, + objects: []client.Object{testKongUpstreamControlPlaneRefNotFound}, + expectError: true, + expectErrorContains: fmt.Sprintf("referenced Control Plane %s/%s does not exist", + testKongUpstreamControlPlaneRefNotFound.Namespace, + testKongUpstreamControlPlaneRefNotFound.Spec.ControlPlaneRef.KonnectNamespacedRef.Name, + ), + }, + { + name: "ControlPlaneRef not programmed", + ent: &configurationv1alpha1.KongTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: "target-upstream-cpref-not-programmed", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongTargetSpec{ + UpstreamRef: configurationv1alpha1.TargetRef{ + Name: "upstream-cpref-not-programmed", + }, + }, + }, + objects: []client.Object{ + testKongUpstreamControlPlaneRefNotProgrammed, + testControlPlaneNotProgrammed, + }, + expectError: false, + expectResult: ctrl.Result{Requeue: true}, + updatedEntAssertions: []func(*configurationv1alpha1.KongTarget) (bool, string){ + func(kt *configurationv1alpha1.KongTarget) (bool, string) { + return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.KongUpstreamRefValidConditionType && c.Status == metav1.ConditionTrue + }), "KongTarget does not have KongUpsteamRefValid condition set to True" + }, + func(kt *configurationv1alpha1.KongTarget) (bool, string) { + return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.ControlPlaneRefValidConditionType && c.Status == metav1.ConditionFalse + }), "KongTarget does not have ControlPlaneRefValid condition set to False" + }, + }, + }, + } + testHandleUpstreamRef(t, testCases) +} + +func testHandleUpstreamRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]]( + t *testing.T, testCases []handleUpstreamRefTestCase[T, TEnt]) { + t.Helper() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, configurationv1alpha1.AddToScheme(scheme)) + require.NoError(t, konnectv1alpha1.AddToScheme(scheme)) + fakeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(tc.ent).WithObjects(tc.objects...). + // WithStatusSubresource is required for updating status of handled entity. + WithStatusSubresource(tc.ent).Build() + require.NoError(t, fakeClient.SubResource("status").Update(context.Background(), tc.ent)) + + res, err := handleKongUpstreamRef(context.Background(), fakeClient, tc.ent) + + var updatedEnt TEnt = tc.ent.DeepCopyObject().(TEnt) + require.NoError(t, fakeClient.Get(context.Background(), client.ObjectKeyFromObject(tc.ent), updatedEnt)) + for _, assertion := range tc.updatedEntAssertions { + ok, msg := assertion(updatedEnt) + require.True(t, ok, msg) + } + + if tc.expectError { + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectErrorContains) + t.Logf("%#v", err) + return + } + + require.NoError(t, err) + require.Equal(t, tc.expectResult, res) + }) + } +} diff --git a/controller/konnect/watch.go b/controller/konnect/watch.go index 617368981..5177c4b07 100644 --- a/controller/konnect/watch.go +++ b/controller/konnect/watch.go @@ -42,6 +42,8 @@ func ReconciliationWatchOptionsForEntity[ return kongCredentialBasicAuthReconciliationWatchOptions(cl) case *configurationv1alpha1.KongCACertificate: return KongCACertificateReconciliationWatchOptions(cl) + case *configurationv1alpha1.KongTarget: + return KongTargetReconciliationWatchOptions(cl) default: panic(fmt.Sprintf("unsupported entity type %T", ent)) } diff --git a/controller/konnect/watch_kongtarget.go b/controller/konnect/watch_kongtarget.go new file mode 100644 index 000000000..e92c5c892 --- /dev/null +++ b/controller/konnect/watch_kongtarget.go @@ -0,0 +1,99 @@ +package konnect + +import ( + "context" + "reflect" + + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + operatorerrors "github.com/kong/gateway-operator/internal/errors" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" +) + +// KongTargetReconciliationWatchOptions returns the watch options for +// the KongTarget. +func KongTargetReconciliationWatchOptions(cl client.Client, +) []func(*ctrl.Builder) *ctrl.Builder { + return []func(*ctrl.Builder) *ctrl.Builder{ + func(b *ctrl.Builder) *ctrl.Builder { + return b.For( + &configurationv1alpha1.KongTarget{}, + builder.WithPredicates( + predicate.NewPredicateFuncs(kongTargetRefersToKonnectGatewayControlPlane(cl)), + ), + ) + }, + func(b *ctrl.Builder) *ctrl.Builder { + return b.Watches( + &configurationv1alpha1.KongUpstream{}, + handler.EnqueueRequestsFromMapFunc(enqueueKongTargetForKongUpstream(cl)), + ) + }, + } +} + +// kongTargetRefersToKonnectGatewayControlPlane returns the predict +// that checks whether a KongTarget is referring a Konnect Control Plane via upstream. +func kongTargetRefersToKonnectGatewayControlPlane(cl client.Client) func(obj client.Object) bool { + return func(obj client.Object) bool { + kongTarget, ok := obj.(*configurationv1alpha1.KongTarget) + if !ok { + ctrllog.FromContext(context.Background()).Error( + operatorerrors.ErrUnexpectedObject, + "failed to run predicate function", + "expected", "KongTarget", "found", reflect.TypeOf(obj), + ) + return false + } + + upstream := configurationv1alpha1.KongUpstream{} + nn := types.NamespacedName{ + Namespace: kongTarget.Namespace, + Name: kongTarget.Spec.UpstreamRef.Name, + } + if err := cl.Get(context.Background(), nn, &upstream); client.IgnoreNotFound(err) != nil { + return true + } + cpRef := upstream.Spec.ControlPlaneRef + return cpRef != nil && cpRef.Type == configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef + } +} + +func enqueueKongTargetForKongUpstream(cl client.Client, +) func(ctx context.Context, obj client.Object) []reconcile.Request { + return func(ctx context.Context, obj client.Object) []reconcile.Request { + kongUpstream, ok := obj.(*configurationv1alpha1.KongUpstream) + if !ok { + return nil + } + cpRef := kongUpstream.Spec.ControlPlaneRef + if cpRef == nil || cpRef.Type != configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef { + return nil + } + var targetList configurationv1alpha1.KongTargetList + if err := cl.List(ctx, &targetList, &client.ListOptions{ + // TODO: change this when cross namespace refs are allowed. + Namespace: kongUpstream.GetNamespace(), + }); err != nil { + return nil + } + + var ret []reconcile.Request + for _, target := range targetList.Items { + if target.Spec.UpstreamRef.Name == kongUpstream.Name { + ret = append(ret, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&target), + }) + } + } + return ret + } +} diff --git a/modules/manager/controller_setup.go b/modules/manager/controller_setup.go index 48498c0a8..01b1470d2 100644 --- a/modules/manager/controller_setup.go +++ b/modules/manager/controller_setup.go @@ -78,6 +78,8 @@ const ( KongPluginControllerName = "KongPlugin" // KongUpstreamControllerName is the name of the KongUpstream controller. KongUpstreamControllerName = "KongUpstream" + // KongTargetControllerName is the name of the KongTarget controller. + KongTargetControllerName = "KongTarget" // KongServicePluginBindingFinalizerControllerName is the name of the KongService PluginBinding finalizer controller. KongServicePluginBindingFinalizerControllerName = "KongServicePluginBindingFinalizer" // KongCredentialsSecretControllerName is the name of the Credentials Secret controller. @@ -401,6 +403,15 @@ func SetupControllers(mgr manager.Manager, c *Config) (map[string]ControllerDef, konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongCACertificate](c.KonnectSyncPeriod), ), }, + KongTargetControllerName: { + Enabled: c.KonnectControllersEnabled, + Controller: konnect.NewKonnectEntityReconciler( + sdkFactory, + c.DevelopmentMode, + mgr.GetClient(), + konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongTarget](c.KonnectSyncPeriod), + ), + }, KongPluginBindingControllerName: { Enabled: c.KonnectControllersEnabled, Controller: konnect.NewKonnectEntityReconciler( diff --git a/test/integration/test_konnect_entities.go b/test/integration/test_konnect_entities.go index 7db26e040..7d8af100d 100644 --- a/test/integration/test_konnect_entities.go +++ b/test/integration/test_konnect_entities.go @@ -102,6 +102,7 @@ func TestKonnectEntities(t *testing.T) { KongServiceAPISpec: configurationv1alpha1.KongServiceAPISpec{ Name: lo.ToPtr(ksName), URL: lo.ToPtr("http://example.com"), + Host: "example.com", }, }, } @@ -247,6 +248,79 @@ func TestKonnectEntities(t *testing.T) { require.NoError(t, err) assertKonnectEntityProgrammed(t, kpb.GetConditions(), kpb.GetKonnectStatus()) }, testutils.ObjectUpdateTimeout, time.Second) + + t.Log("Creating KongUpstream") + kupName := "kup-" + testID + kup := &configurationv1alpha1.KongUpstream{ + ObjectMeta: metav1.ObjectMeta{ + Name: kupName, + Namespace: ns.Name, + }, + Spec: configurationv1alpha1.KongUpstreamSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: cp.Name, + }, + }, + KongUpstreamAPISpec: configurationv1alpha1.KongUpstreamAPISpec{ + Name: ks.Spec.Host, + Slots: lo.ToPtr(int64(16384)), + Algorithm: sdkkonnectcomp.UpstreamAlgorithmConsistentHashing.ToPointer(), + }, + }, + } + err = GetClients().MgrClient.Create(GetCtx(), kup) + require.NoError(t, err) + + t.Log("Waiting for KongUpstream to be updated with Konnect ID") + require.EventuallyWithT(t, func(t *assert.CollectT) { + err := GetClients().MgrClient.Get(GetCtx(), types.NamespacedName{Name: kup.Name, Namespace: ns.Name}, kup) + require.NoError(t, err) + + if !assert.NotNil(t, kup.Status.Konnect) { + return + } + assert.NotEmpty(t, kup.Status.Konnect.KonnectEntityStatus.GetKonnectID()) + assert.NotEmpty(t, kup.Status.Konnect.KonnectEntityStatus.GetOrgID()) + assert.NotEmpty(t, kup.Status.Konnect.KonnectEntityStatus.GetServerURL()) + }, testutils.ObjectUpdateTimeout, time.Second) + + t.Log("Creating KongTarget") + ktName := "kt-" + testID + kt := &configurationv1alpha1.KongTarget{ + ObjectMeta: metav1.ObjectMeta{ + Name: ktName, + Namespace: ns.Name, + }, + Spec: configurationv1alpha1.KongTargetSpec{ + UpstreamRef: configurationv1alpha1.TargetRef{ + Name: kupName, + }, + KongTargetAPISpec: configurationv1alpha1.KongTargetAPISpec{ + Target: "example.com", + Weight: 100, + }, + }, + } + err = GetClients().MgrClient.Create(GetCtx(), kt) + require.NoError(t, err) + + t.Log("Waiting for KongTarget to be updated with Konnect ID") + require.EventuallyWithT(t, func(t *assert.CollectT) { + err := GetClients().MgrClient.Get(GetCtx(), types.NamespacedName{Name: kt.Name, Namespace: ns.Name}, kt) + require.NoError(t, err) + if !assert.NotNil(t, kt.Status.Konnect) { + return + } + assert.NotEmpty(t, kt.Status.Konnect.KonnectEntityStatus.GetKonnectID()) + assert.NotEmpty(t, kt.Status.Konnect.KonnectEntityStatus.GetOrgID()) + assert.NotEmpty(t, kt.Status.Konnect.KonnectEntityStatus.GetServerURL()) + }, testutils.ObjectUpdateTimeout, time.Second) + + // Should delete KongTarget because it will block deletion of KongUpstream owning it. + t.Cleanup(deleteObjectAndWaitForDeletionFn(t, kt)) + } // deleteObjectAndWaitForDeletionFn returns a function that deletes the given object and waits for it to be gone.