Skip to content

Commit b052a8e

Browse files
pbenasGouthamML
pbenas
authored and
GouthamML
committed
OKE-33554 Rule Sets support
Allow user to specify Load Balancer Rule Sets (as specifified in documentation) through a Kubernetes Load Balancer service annotation. The value is a JSON object corresponding to a map of RuleSetDetails as specified by OCI API docs.
1 parent 828d00d commit b052a8e

File tree

18 files changed

+1130
-37
lines changed

18 files changed

+1130
-37
lines changed

‎docs/load-balancer-annotations.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ spec:
5757
| `oci.oraclecloud.com/oci-load-balancer-listener-ssl-config` | Specifies the cipher suite on the listener of the LB managed by CCM. | `N/A` | `'{"CipherSuiteName":"oci-default-http2-ssl-cipher-suite-v1", "Protocols":["TLSv1.2"]}'` |
5858
| `oci.oraclecloud.com/oci-load-balancer-backendset-ssl-config"` | Specifies the cipher suite on the backendsets of the LB managed by CCM. | `N/A` | `'{"CipherSuiteName":"oci-default-http2-ssl-cipher-suite-v1", "Protocols":["TLSv1.2"]}'` |
5959
| `oci.oraclecloud.com/ingress-ip-mode` | Specifies ".status.loadBalancer.ingress.ipMode" for a Service with type set to LoadBalancer. Refer: [Specifying IPMode to adjust traffic routing][11] | `VIP` | `"proxy"` |
60+
| `oci.oraclecloud.com/oci-load-balancer-rule-sets` | [Rule Sets][11] configuration. A JSON object mapping strings to RuleSetDetails objects as specified in [OCI API documentation][12]. All rule sets will be attached to all configured listeners. | `N/A` |
6061

6162

6263
Note:
@@ -148,3 +149,5 @@ Note:
148149
[11]: https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringloadbalancersnetworkloadbalancers-subtopic.htm#contengcreatingloadbalancer_topic_Specifying_IPMode
149150
[12]: https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringloadbalancersnetworkloadbalancers-subtopic.htm#contengcreatingloadbalancer_topic_Skip_private_IP_addresses
150151
[13]: https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengcreatingloadbalancers-subtopic.htm#listenerprotocol
152+
[12]: https://docs.oracle.com/en-us/iaas/Content/Balance/Tasks/managingrulesets.htm
153+
[13]: https://docs.oracle.com/en-us/iaas/api/#/en/loadbalancer/20170115/datatypes/RuleSetDetails

‎pkg/cloudprovider/providers/oci/instances_test.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"github.com/oracle/oci-go-sdk/v65/filestorage"
2929
"github.com/oracle/oci-go-sdk/v65/identity"
3030

31+
"github.com/oracle/oci-go-sdk/v65/loadbalancer"
3132
"go.uber.org/zap"
3233
authv1 "k8s.io/api/authentication/v1"
3334
v1 "k8s.io/api/core/v1"
@@ -935,6 +936,17 @@ func (MockComputeClient) GetPrimaryVNICForInstance(ctx context.Context, compartm
935936
func (MockComputeClient) GetSecondaryVNICsForInstance(ctx context.Context, compartmentID, instanceID string) ([]*core.Vnic, error) {
936937
return []*core.Vnic{instanceVnics[instanceID]}, nil
937938
}
939+
func (c *MockComputeClient) ListVnicAttachments(ctx context.Context, compartmentID, instanceID string) ([]core.VnicAttachment, error) {
940+
return nil, nil
941+
}
942+
943+
func (c *MockComputeClient) GetVnicAttachment(ctx context.Context, vnicAttachmentId *string) (response *core.VnicAttachment, err error) {
944+
return nil, nil
945+
}
946+
947+
func (c *MockComputeClient) AttachVnic(ctx context.Context, instanceID, subnetID *string, nsgIds []*string, skipSourceDestCheck *bool) (response core.VnicAttachment, err error) {
948+
return core.VnicAttachment{}, nil
949+
}
938950

939951
func (MockComputeClient) FindVolumeAttachment(ctx context.Context, compartmentID, volumeID string) (core.VolumeAttachment, error) {
940952
return nil, nil
@@ -1121,6 +1133,18 @@ var updateLoadBalancerErrors = map[string]error{
11211133
"work request fail": errors.New("internal server error"),
11221134
}
11231135

1136+
func (c *MockLoadBalancerClient) CreateRuleSet(ctx context.Context, lbID string, name string, details *loadbalancer.RuleSetDetails) (string, error) {
1137+
return "", nil
1138+
}
1139+
1140+
func (c *MockLoadBalancerClient) UpdateRuleSet(ctx context.Context, lbID string, name string, details *loadbalancer.RuleSetDetails) (string, error) {
1141+
return "", nil
1142+
}
1143+
1144+
func (c *MockLoadBalancerClient) DeleteRuleSet(ctx context.Context, lbID string, name string) (string, error) {
1145+
return "", nil
1146+
}
1147+
11241148
func (c *MockLoadBalancerClient) UpdateLoadBalancer(ctx context.Context, lbID string, details *client.GenericUpdateLoadBalancerDetails) (string, error) {
11251149
if err, ok := updateLoadBalancerErrors[lbID]; ok {
11261150
return "", err
@@ -1223,6 +1247,18 @@ func (c *MockNetworkLoadBalancerClient) DeleteListener(ctx context.Context, lbID
12231247
return "", nil
12241248
}
12251249

1250+
func (c *MockNetworkLoadBalancerClient) CreateRuleSet(ctx context.Context, lbID string, name string, details *loadbalancer.RuleSetDetails) (string, error) {
1251+
return "", nil
1252+
}
1253+
1254+
func (c *MockNetworkLoadBalancerClient) UpdateRuleSet(ctx context.Context, lbID string, name string, details *loadbalancer.RuleSetDetails) (string, error) {
1255+
return "", nil
1256+
}
1257+
1258+
func (c *MockNetworkLoadBalancerClient) DeleteRuleSet(ctx context.Context, lbID string, name string) (string, error) {
1259+
return "", nil
1260+
}
1261+
12261262
func (c *MockNetworkLoadBalancerClient) AwaitWorkRequest(ctx context.Context, id string) (*client.GenericWorkRequest, error) {
12271263
if err, ok := awaitLoadbalancerWorkrequestMap[id]; ok {
12281264
return nil, err

‎pkg/cloudprovider/providers/oci/load_balancer.go‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,7 @@ func (clb *CloudLoadBalancerProvider) createLoadBalancer(ctx context.Context, sp
413413
FreeformTags: spec.FreeformTags,
414414
DefinedTags: spec.DefinedTags,
415415
IpVersion: spec.IpVersions.LbEndpointIpVersion,
416+
RuleSets: spec.RuleSets,
416417
}
417418
// do not block creation if the defined tag limit is reached. defer LB to tracked by backfilling
418419
if len(details.DefinedTags) > MaxDefinedTagPerResource {
@@ -1046,13 +1047,18 @@ func (clb *CloudLoadBalancerProvider) updateLoadBalancer(ctx context.Context, lb
10461047
}
10471048
}
10481049

1050+
var ruleSetActions []Action
1051+
if spec.RuleSets != nil {
1052+
ruleSetActions = getRuleSetChanges(lb.RuleSets, spec.RuleSets)
1053+
}
1054+
10491055
actualBackendSets := lb.BackendSets
10501056
desiredBackendSets := spec.BackendSets
10511057
backendSetActions := getBackendSetChanges(logger, actualBackendSets, desiredBackendSets)
10521058

10531059
actualListeners := lb.Listeners
10541060
desiredListeners := spec.Listeners
1055-
listenerActions := getListenerChanges(logger, actualListeners, desiredListeners)
1061+
listenerActions := getListenerChanges(logger, actualListeners, desiredListeners, spec.RuleSets)
10561062

10571063
if len(backendSetActions) == 0 && len(listenerActions) == 0 {
10581064
// If there are no backendSetActions or Listener actions
@@ -1065,7 +1071,8 @@ func (clb *CloudLoadBalancerProvider) updateLoadBalancer(ctx context.Context, lb
10651071
return err
10661072
}
10671073
}
1068-
actions := sortAndCombineActions(logger, backendSetActions, listenerActions)
1074+
actions := sortAndCombineActions(logger, backendSetActions, listenerActions, ruleSetActions)
1075+
10691076
for _, action := range actions {
10701077
switch a := action.(type) {
10711078
case *BackendSetAction:
@@ -1090,6 +1097,11 @@ func (clb *CloudLoadBalancerProvider) updateLoadBalancer(ctx context.Context, lb
10901097
if err != nil {
10911098
return errors.Wrap(err, "updating listener")
10921099
}
1100+
case *RuleSetAction:
1101+
err := clb.updateRuleSet(ctx, lbID, a, spec)
1102+
if err != nil {
1103+
return errors.Wrap(err, "updating RuleSet")
1104+
}
10931105
}
10941106
}
10951107

@@ -1333,6 +1345,39 @@ func (clb *CloudLoadBalancerProvider) updateListener(ctx context.Context, lbID s
13331345
return nil
13341346
}
13351347

1348+
func (clb *CloudLoadBalancerProvider) updateRuleSet(ctx context.Context, lbID string, action *RuleSetAction, spec *LBSpec) (err error) {
1349+
var workRequestID string
1350+
1351+
logger := clb.logger.With(
1352+
"actionType", action.Type(),
1353+
"ruleSetName", action.Name(),
1354+
"loadBalancerID", lbID,
1355+
"loadBalancerType", getLoadBalancerType(spec.service))
1356+
logger.Info("Applying action on rule set")
1357+
1358+
switch action.Type() {
1359+
case Create:
1360+
workRequestID, err = clb.lbClient.CreateRuleSet(ctx, lbID, action.Name(), &action.RuleSetDetails)
1361+
case Update:
1362+
workRequestID, err = clb.lbClient.UpdateRuleSet(ctx, lbID, action.Name(), &action.RuleSetDetails)
1363+
case Delete:
1364+
workRequestID, err = clb.lbClient.DeleteRuleSet(ctx, lbID, action.Name())
1365+
}
1366+
1367+
if err != nil {
1368+
return err
1369+
}
1370+
logger = logger.With("workRequestID", workRequestID)
1371+
logger.Info("Await work request for loadbalancer rule set")
1372+
_, err = clb.lbClient.AwaitWorkRequest(ctx, workRequestID)
1373+
if err != nil {
1374+
return err
1375+
}
1376+
logger.Info("Work request for loadbalancer rule set completed successfully")
1377+
1378+
return nil
1379+
}
1380+
13361381
// UpdateLoadBalancer updates an existing loadbalancer
13371382
func (cp *CloudProvider) UpdateLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) error {
13381383
startTime := time.Now()

‎pkg/cloudprovider/providers/oci/load_balancer_spec.go‎

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,13 @@ import (
1919
"fmt"
2020
"net"
2121
"net/http"
22+
"slices"
2223
"strconv"
2324
"strings"
2425

26+
"github.com/oracle/oci-go-sdk/v65/loadbalancer"
2527
"go.uber.org/zap"
28+
"golang.org/x/exp/maps"
2629
v1 "k8s.io/api/core/v1"
2730
"k8s.io/apimachinery/pkg/util/sets"
2831
apiservice "k8s.io/kubernetes/pkg/api/v1/service"
@@ -180,6 +183,12 @@ const (
180183
// with type set to LoadBalancer.
181184
// https://kubernetes.io/docs/concepts/services-networking/service/#load-balancer-ip-mode:~:text=Specifying%20IPMode%20of%20load%20balancer%20status
182185
ServiceAnnotationIngressIpMode = "oci.oraclecloud.com/ingress-ip-mode"
186+
187+
// ServiceAnnotationRuleSets allows the user to specify rule sets of actions applied to traffic at a load balancer listener
188+
// https://docs.oracle.com/en-us/iaas/Content/Balance/Tasks/managingrulesets.htm
189+
// Expected format is a JSON blob containing a JSON object literal with keys being rule names and values being a JSON
190+
// representation of a valid Rule object. https://docs.oracle.com/en-us/iaas/api/#/en/loadbalancer/20170115/datatypes/Rule
191+
ServiceAnnotationRuleSets = "oci.oraclecloud.com/oci-load-balancer-rule-sets"
183192
)
184193

185194
// NLB specific annotations
@@ -294,7 +303,7 @@ type ManagedNetworkSecurityGroup struct {
294303
}
295304

296305
func requiresCertificate(svc *v1.Service) bool {
297-
if svc.Annotations[ServiceAnnotationLoadBalancerType] == NLB {
306+
if getLoadBalancerType(svc) == NLB {
298307
return false
299308
}
300309
_, ok := svc.Annotations[ServiceAnnotationLoadBalancerSSLPorts]
@@ -354,6 +363,7 @@ type LBSpec struct {
354363
SystemTags map[string]map[string]interface{}
355364
ingressIpMode *v1.LoadBalancerIPMode
356365
Compartment string
366+
RuleSets map[string]loadbalancer.RuleSetDetails
357367

358368
service *v1.Service
359369
nodes []*v1.Node
@@ -390,6 +400,11 @@ func NewLBSpec(logger *zap.SugaredLogger, svc *v1.Service, provisionedNodes []*v
390400
return nil, err
391401
}
392402

403+
ruleSets, err := getRuleSets(svc)
404+
if err != nil {
405+
return nil, err
406+
}
407+
393408
listeners, err := getListeners(svc, sslConfig, convertOciIpVersionsToOciIpFamilies(versions.ListenerBackendIpVersion))
394409
if err != nil {
395410
return nil, err
@@ -476,14 +491,14 @@ func NewLBSpec(logger *zap.SugaredLogger, svc *v1.Service, provisionedNodes []*v
476491
SystemTags: getResourceTrackingSystemTagsFromConfig(logger, initialLBTags),
477492
ingressIpMode: ingressIpMode,
478493
Compartment: compartment,
494+
RuleSets: ruleSets,
479495
}, nil
480496
}
481497

482498
func getLoadBalancerCompartment(svc *v1.Service, clusterCompartment string) (compartment string) {
499+
compartment = clusterCompartment
483500
if value, exist := svc.Annotations[util.CompartmentIDAnnotation]; exist {
484501
compartment = value
485-
} else {
486-
compartment = clusterCompartment
487502
}
488503
return
489504
}
@@ -1044,6 +1059,13 @@ func getListenersOciLoadBalancer(svc *v1.Service, sslCfg *SSLConfig) (map[string
10441059
proxyProtocolVersion = common.Int(version)
10451060
}
10461061

1062+
ruleSets, _ := getRuleSets(svc)
1063+
var rs []string
1064+
if ruleSets != nil {
1065+
rs = maps.Keys(ruleSets)
1066+
slices.Sort(rs)
1067+
}
1068+
10471069
listeners := make(map[string]client.GenericListener)
10481070
for _, servicePort := range svc.Spec.Ports {
10491071
protocol := string(servicePort.Protocol)
@@ -1088,6 +1110,7 @@ func getListenersOciLoadBalancer(svc *v1.Service, sslCfg *SSLConfig) (map[string
10881110
DefaultBackendSetName: common.String(getBackendSetName(string(servicePort.Protocol), int(servicePort.Port))),
10891111
Protocol: &protocol,
10901112
Port: &port,
1113+
RuleSetNames: rs,
10911114
SslConfiguration: sslConfiguration,
10921115
}
10931116

@@ -1648,3 +1671,20 @@ func isSkipPrivateIP(svc *v1.Service) (bool, error) {
16481671
}
16491672
return skipPrivateIp, nil
16501673
}
1674+
1675+
func getRuleSets(svc *v1.Service) (rs map[string]loadbalancer.RuleSetDetails, err error) {
1676+
annotation, exists := svc.Annotations[ServiceAnnotationRuleSets]
1677+
if !exists {
1678+
return nil, nil
1679+
}
1680+
1681+
if getLoadBalancerType(svc) == NLB {
1682+
return rs, fmt.Errorf("invalid annotation %s. Rule Sets are not supported by Network Load Balancer", ServiceAnnotationRuleSets)
1683+
}
1684+
1685+
if annotation == "" {
1686+
annotation = "{}"
1687+
}
1688+
err = json.NewDecoder(strings.NewReader(annotation)).Decode(&rs)
1689+
return rs, err
1690+
}

0 commit comments

Comments
 (0)