[SFS-1439] Implement custom sort for Unstructured

.github/workflows/publish.yaml

@@ -27,13 +27,13 @@ jobs:
 
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: '1.22'
 
       - name: Build, tag, and push docker image to Amazon ECR Public
         uses: int128/kaniko-action@v1
         with:
           push: true
-          tags: ${{ steps.login-ecr-public.outputs.registry }}/f8y0w2c4/cleaner-controller:${{ github.ref_name }}
+          tags: ${{ steps.login-ecr-public.outputs.registry }}/f8y0w2c4/cleaner-controller:manager-${{ github.ref_name }}
 
       - name: Publish helm chart
         run: |

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.19 as builder
+FROM golang:1.22 as builder
 ARG TARGETOS
 ARG TARGETARCH
 
@@ -15,6 +15,7 @@ RUN go mod download
 COPY main.go main.go
 COPY api/ api/
 COPY controllers/ controllers/
+COPY custom_cel/ custom_cel/
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command

Makefile

@@ -196,7 +196,7 @@ CRD_REF_DOCS ?= $(LOCALBIN)/crd-ref-docs
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v3.8.7
-CONTROLLER_TOOLS_VERSION ?= v0.10.0
+CONTROLLER_TOOLS_VERSION ?= v0.16.3
 
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
 .PHONY: kustomize

controllers/conditionalttl_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/vtex/cleaner-controller/custom_cel"
 	"time"
 
 	cloudevents "github.com/cloudevents/sdk-go/v2"
@@ -138,13 +139,13 @@ func (r *ConditionalTTLReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		return ctrl.Result{}, err
 	}
 
-	celCtx := buildCELContext(ts, t)
-	celOpts := buildCELOptions(cTTL)
+	celCtx := custom_cel.BuildCELContext(ts, t)
+	celOpts := custom_cel.BuildCELOptions(cTTL)
 
 	readyCondition := metav1.Condition{
 		ObservedGeneration: cTTL.GetGeneration(),
 	}
-	condsMet, retryable := evaluateCELConditions(celOpts, celCtx, cTTL.Spec.Conditions, &readyCondition)
+	condsMet, retryable := custom_cel.EvaluateCELConditions(celOpts, celCtx, cTTL.Spec.Conditions, &readyCondition)
 	apimeta.SetStatusCondition(&cTTL.Status.Conditions, readyCondition)
 
 	if !condsMet {

controllers/cel.go → custom_cel/cel.go

@@ -1,4 +1,4 @@
-package controllers
+package custom_cel
 
 import (
 	"fmt"
@@ -10,12 +10,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// buildCELOptions builds the list of env options to be used when
+// BuildCELOptions builds the list of env options to be used when
 // building the CEL environment used to evaluated the conditions
 // of a given cTTL.
-func buildCELOptions(cTTL *cleanerv1alpha1.ConditionalTTL) []cel.EnvOption {
+func BuildCELOptions(cTTL *cleanerv1alpha1.ConditionalTTL) []cel.EnvOption {
 	r := []cel.EnvOption{
-		ext.Strings(), // helper string functions
+		ext.Strings(),  // helper string functions
+		ext.Bindings(), // helper binding functions
+		Lists(),        // custom VTEX helper for list functions
 		cel.Variable("time", cel.TimestampType),
 	}
 	for _, t := range cTTL.Spec.Targets {
@@ -26,9 +28,9 @@ func buildCELOptions(cTTL *cleanerv1alpha1.ConditionalTTL) []cel.EnvOption {
 	return r
 }
 
-// buildCELContext builds the map of parameters to be passed to the CEL
+// BuildCELContext builds the map of parameters to be passed to the CEL
 // evaluation given a list of TargetStatus and an evaluation time.
-func buildCELContext(targets []cleanerv1alpha1.TargetStatus, time time.Time) map[string]interface{} {
+func BuildCELContext(targets []cleanerv1alpha1.TargetStatus, time time.Time) map[string]interface{} {
 	ctx := make(map[string]interface{})
 	for _, ts := range targets {
 		if !ts.IncludeWhenEvaluating {
@@ -40,12 +42,12 @@ func buildCELContext(targets []cleanerv1alpha1.TargetStatus, time time.Time) map
 	return ctx
 }
 
-// evaluateCELConditions compiles and evaluates all the conditions on the passed CEL context,
+// EvaluateCELConditions compiles and evaluates all the conditions on the passed CEL context,
 // returning true only when all conditions evaluate to true. It stops evaluating on the first
 // encountered error but otherwise all conditions are evaluated in order to find and report
 // compilation and/or evaluation errors early. It also updates the passed
 // readyCondition Status, Type, Reason and Message fields.
-func evaluateCELConditions(opts []cel.EnvOption, celCtx map[string]interface{}, conditions []string, readyCondition *metav1.Condition) (conditionsMet bool, retryable bool) {
+func EvaluateCELConditions(opts []cel.EnvOption, celCtx map[string]interface{}, conditions []string, readyCondition *metav1.Condition) (conditionsMet bool, retryable bool) {
 	readyCondition.Status = metav1.ConditionFalse
 	readyCondition.Type = cleanerv1alpha1.ConditionTypeReady
 	env, err := cel.NewEnv(opts...)

custom_cel/lists.go

@@ -0,0 +1,204 @@
+package custom_cel
+
+import (
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common"
+	"github.com/google/cel-go/common/ast"
+	"github.com/google/cel-go/common/operators"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+	"github.com/google/cel-go/common/types/traits"
+	"github.com/google/cel-go/parser"
+	"k8s.io/apiserver/pkg/cel/library"
+	"slices"
+	"sort"
+)
+
+// Lists returns a cel.EnvOption to configure extended functions Lists manipulation.
+//
+// # SortBy
+//
+// Returns a new sorted list by the field defined.
+// It supports all types that implements the base traits.Comparer interface.
+//
+// <list>.sort_by(obj, obj.field) ==> <list>
+//
+// Examples:
+//
+// [2,3,1].sort_by(i,i) ==> [1,2,3]
+//
+// [{Name: "c", Age: 10}, {Name: "a", Age: 30}, {Name: "b", Age: 1}].sort_by(obj, obj.age) ==> [{Name: "b", Age: 1}, {Name: "c", Age: 10}, {Name: "a", Age: 30}]
+//
+// # ReverseList
+//
+// Returns a new list in reverse order.
+// It supports all types that implements the base traits.Comparer interface
+//
+// <list>.reverse_list() ==> <list>
+//
+// # Examples
+//
+// [1,2,3].reverse_list() ==> [3,2,1]
+//
+// ["x", "y", "z"].reverse_list() ==> ["z", "y", "x"]
+func Lists() cel.EnvOption {
+	return cel.Lib(listsLib{})
+}
+
+type listsLib struct{}
+
+// CompileOptions implements the Library interface method defining the basic compile configuration
+func (u listsLib) CompileOptions() []cel.EnvOption {
+	dynListType := cel.ListType(cel.DynType)
+	sortByMacro := parser.NewReceiverMacro("sort_by", 2, makeSortBy)
+	return []cel.EnvOption{
+		library.Lists(),
+		cel.Macros(sortByMacro),
+		cel.Function(
+			"pair",
+			cel.Overload(
+				"make_pair",
+				[]*cel.Type{cel.DynType, cel.DynType},
+				cel.DynType,
+				cel.BinaryBinding(makePair),
+			),
+		),
+		cel.Function(
+			"sort",
+			cel.Overload(
+				"sort_list",
+				[]*cel.Type{dynListType},
+				dynListType,
+				cel.UnaryBinding(makeSort),
+			),
+		),
+		cel.Function(
+			"reverse_list",
+			cel.MemberOverload(
+				"reverse_list_id",
+				[]*cel.Type{cel.ListType(cel.DynType)},
+				cel.ListType(cel.DynType),
+				cel.UnaryBinding(makeReverse),
+			),
+		),
+	}
+}
+
+// ProgramOptions implements the Library interface method defining the basic program options
+func (u listsLib) ProgramOptions() []cel.ProgramOption {
+	return []cel.ProgramOption{}
+}
+
+type pair struct {
+	order ref.Val
+	value ref.Val
+}
+
+var (
+	orderKey = types.DefaultTypeAdapter.NativeToValue("order")
+	valueKey = types.DefaultTypeAdapter.NativeToValue("value")
+)
+
+func makePair(order ref.Val, value ref.Val) ref.Val {
+	if _, ok := order.(traits.Comparer); !ok {
+		return types.ValOrErr(order, "unable to build ordered pair with value %v", order.Value())
+	}
+	return types.NewStringInterfaceMap(types.DefaultTypeAdapter, map[string]any{
+		"order": order.Value(),
+		"value": value.Value(),
+	})
+}
+
+func makeSort(itemsVal ref.Val) ref.Val {
+	items, ok := itemsVal.(traits.Lister)
+	if !ok {
+		return types.ValOrErr(itemsVal, "unable to convert to traits.Lister")
+	}
+
+	pairs := make([]pair, 0, items.Size().Value().(int64))
+	index := 0
+	for it := items.Iterator(); it.HasNext().(types.Bool); {
+		curr, ok := it.Next().(traits.Mapper)
+		if !ok {
+			return types.NewErr("unable to convert elem %d to traits.Mapper", index)
+		}
+
+		pairs = append(pairs, pair{
+			order: curr.Get(orderKey),
+			value: curr.Get(valueKey),
+		})
+		index++
+	}
+
+	sort.Slice(pairs, func(i, j int) bool {
+		return pairs[i].order.(traits.Comparer).Compare(pairs[j].order) == types.IntNegOne
+	})
+
+	var ordered []interface{}
+	for _, v := range pairs {
+		ordered = append(ordered, v.value.Value())
+	}
+
+	return types.NewDynamicList(types.DefaultTypeAdapter, ordered)
+}
+
+func extractIdent(e ast.Expr) (string, bool) {
+	if e.Kind() == ast.IdentKind {
+		return e.AsIdent(), true
+	}
+	return "", false
+}
+
+func makeSortBy(eh parser.ExprHelper, target ast.Expr, args []ast.Expr) (ast.Expr, *common.Error) {
+	v, found := extractIdent(args[0])
+	if !found {
+		return nil, eh.NewError(args[0].ID(), "argument is not an identifier")
+	}
+
+	var fn = args[1]
+
+	init := eh.NewList()
+	condition := eh.NewLiteral(types.True)
+
+	step := eh.NewCall(operators.Add, eh.NewAccuIdent(), eh.NewList(
+		eh.NewCall("pair", fn, args[0]),
+	))
+
+	/*
+	   This comprehension is expanded to:
+	   __result__ = [] # init expr
+	   for $v in $target:
+	       __result__ += [pair(fn(v), v)] # step expr
+	   return sort(__result__) # result expr
+	*/
+	mapped := eh.NewComprehension(
+		target,
+		v,
+		parser.AccumulatorName,
+		init,
+		condition,
+		step,
+		eh.NewCall(
+			"sort",
+			eh.NewAccuIdent(),
+		),
+	)
+
+	return mapped, nil
+}
+
+func makeReverse(itemsVal ref.Val) ref.Val {
+	items, ok := itemsVal.(traits.Lister)
+	if !ok {
+		return types.ValOrErr(itemsVal, "unable to convert to traits.Lister")
+	}
+
+	orderedItems := make([]ref.Val, 0, items.Size().Value().(int64))
+	for it := items.Iterator(); it.HasNext().(types.Bool); {
+		orderedItems = append(orderedItems, it.Next())
+	}
+
+	slices.Reverse(orderedItems)
+
+	return types.NewDynamicList(types.DefaultTypeAdapter, orderedItems)
+}