Skip to content

Commit

Permalink
add identifier annotation (#600)
Browse files Browse the repository at this point in the history
  • Loading branch information
yuwenma authored Jul 6, 2022
1 parent 78748f0 commit 7181f45
Show file tree
Hide file tree
Showing 9 changed files with 280 additions and 3 deletions.
15 changes: 15 additions & 0 deletions go/fn/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,18 @@ const (
// KptLocalConfig marks a KRM resource to be skipped from deploying to the cluster via `kpt live apply`.
KptLocalConfig = ConfigPrefix + "local-config"
)

// For Kpt use only constants
const (
// KptUseOnlyPrefix is the prefix of kpt-only annotations. Users are not expected to touch these annotations.
KptUseOnlyPrefix = "internal.kpt.dev/"

// UpstreamIdentifier is the annotation to record a resource's upstream origin.
// It is in the form of <GROUP>|<KIND>|<NAMESPACE>|<NAME>
UpstreamIdentifier = KptUseOnlyPrefix + "upstream-identifier"

// UnknownNamespace is the special char for cluster-scoped or unknown-scoped resources. This is only used in upstream-identifier
UnknownNamespace = "~C"
// DefaultNamespace is the actual namespace value if a namespace-scoped resource has its namespace field unspecified.
DefaultNamespace = "default"
)
14 changes: 14 additions & 0 deletions go/fn/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,17 @@ func (e *errResultEnd) Error() string {
}
return fmt.Sprintf("function is terminated: %v", e.message)
}

type ErrAttemptToTouchUpstreamIdentifier struct{}

func (ErrAttemptToTouchUpstreamIdentifier) Error() string {
return fmt.Sprintf("annotation %v is managed by kpt and should not be modified", UpstreamIdentifier)
}

type ErrInternalAnnotation struct {
Message string
}

func (e *ErrInternalAnnotation) Error() string {
return e.Message
}
2 changes: 1 addition & 1 deletion go/fn/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/GoogleContainerTools/kpt-functions-sdk/go/fn
go 1.17

require (
github.com/go-errors/errors v1.0.1
github.com/google/go-cmp v0.5.7
github.com/stretchr/testify v1.7.1
// We must not include any core k8s modules (e.g. k8s.io/apimachinery) in
Expand All @@ -16,7 +17,6 @@ require (
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-logr/logr v1.2.0 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.6 // indirect
Expand Down
3 changes: 1 addition & 2 deletions go/fn/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
Expand Down
15 changes: 15 additions & 0 deletions go/fn/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -322,10 +322,21 @@ func (o *SubObject) SetOrDie(val interface{}, fields ...string) {
}
}

// onLockedFields locks the SubObject fields which are expected for kpt internal use only.
func (o *SubObject) onLockedFields(val interface{}, fields ...string) error {
if o.hasUpstreamIdentifier(val, fields...) {
return ErrAttemptToTouchUpstreamIdentifier{}
}
return nil
}

// SetNestedField sets a nested field located by fields to the value provided as val. val
// should not be a yaml.RNode. If you want to deal with yaml.RNode, you should
// use Get method and modify the underlying yaml.Node.
func (o *SubObject) SetNestedField(val interface{}, fields ...string) error {
if err := o.onLockedFields(val, fields...); err != nil {
return err
}
err := func() error {
if val == nil {
return fmt.Errorf("the passed-in object must not be nil")
Expand Down Expand Up @@ -682,6 +693,10 @@ func (o *KubeObject) SetNamespace(name string) {
}

func (o *KubeObject) SetAnnotation(k, v string) {
// Keep upstream-identifier untouched from users
if k == UpstreamIdentifier {
panic(ErrAttemptToTouchUpstreamIdentifier{})
}
if err := o.SetNestedField(v, "metadata", "annotations", k); err != nil {
panic(fmt.Errorf("cannot set metadata annotations '%v': %v", k, err))
}
Expand Down
33 changes: 33 additions & 0 deletions go/fn/object_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"sort"
"testing"

"github.com/go-errors/errors"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
)
Expand Down Expand Up @@ -237,6 +238,38 @@ func TestSetNestedFields(t *testing.T) {
}
}

func TestInternalAnnotationsUntouchable(t *testing.T) {
o := NewEmptyKubeObject()
// Verify the "upstream-identifier" annotation cannot be changed via SetNestedStringMap
o.SetNestedStringMap(map[string]string{"owner": "kpt"}, "metadata", "annotations")
if stringMapVal := o.NestedStringMapOrDie("metadata", "annotations"); !reflect.DeepEqual(stringMapVal, map[string]string{"owner": "kpt"}) {
t.Errorf("annotations cannot be set via SetNestedStringMap, got %v", stringMapVal)
}
err := o.SetNestedStringMap(map[string]string{UpstreamIdentifier: "apps|Deployment|default|dp"}, "metadata", "annotations")
if !errors.Is(ErrAttemptToTouchUpstreamIdentifier{}, err) {
t.Errorf("set internal annotation via SetNestedStringMap() failed, expect %e, got %e", ErrAttemptToTouchUpstreamIdentifier{}, err)
}

// Verify the "upstream-identifier" annotation cannot be changed via SetAnnotation
o.SetAnnotation("owner", "kpt")
if o.GetAnnotation("owner") != "kpt" {
t.Errorf("annotations cannot be set via SetAnnotation(), got %v", o.GetAnnotation("owner"))
}
defer func() {
if r := recover(); r == nil {
t.Errorf("set internal annotation via SetAnnotation() expect panic (%v), got pass",
ErrAttemptToTouchUpstreamIdentifier{})
}
}()
o.SetAnnotation(UpstreamIdentifier, "apps|Deployment|default|dp")

// Verify the "upstream-identifier" annotation cannot be changed via SetNestedField
err = o.SetNestedField(map[string]string{UpstreamIdentifier: "apps|Deployment|default|dp"}, "metadata", "annotations")
if !errors.Is(ErrAttemptToTouchUpstreamIdentifier{}, err) {
t.Errorf("set internal annotation via SetNestedField() failed, expect %e, got %e", ErrAttemptToTouchUpstreamIdentifier{}, err)
}
}

func generate(t *testing.T) *KubeObject {
doc := `
apiVersion: v1
Expand Down
138 changes: 138 additions & 0 deletions go/fn/origin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fn

import (
"fmt"
"reflect"
"regexp"
"strings"
)

const (
// upstreamIdentifierRegexPattern provides the rough regex to parse a upstream-identiifier annotation.
// "group" should be a domain name. We accept empty string for kubernetes core v1 resources.
// "kind" should be the resource type with initial in capitals.
// "namespace" should follow RFC 1123 Label Names. We accept "~C~ for cluster-scoped resource or unknown scope resources.
// "name" should follow RFC 1123 Label Names https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
upstreamIdentifierRegexPattern = `(?P<group>[a-z0-9-.]*)\|(?P<kind>[A-Z][a-zA-Z0-9]*)\|(?P<namespace>[a-z0-9-]{1,63}|~C)\|(?P<name>[a-z0-9-]{1,63})`
upstreamIdentifierFormat = "<GROUP>|<KIND>|<NAMESPACE>|<NAME>"
regexPatternGroup = "group"
regexPatternKind = "kind"
regexPatterNamespace = "namespace"
regexPatternName = "name"
)

type ResourceIdentifier struct {
Group string
Version string
Kind string
Name string
Namespace string
}

func (r *ResourceIdentifier) String() string {
return fmt.Sprintf("%v|%v|%v|%v", r.Group, r.Kind, r.Namespace, r.Name)
}

// hasUpstreamIdentifier determines whether the args are touching the kpt only annotation "internal.kpt.dev/upstream-identifier"
func (o *SubObject) hasUpstreamIdentifier(val interface{}, fields ...string) bool {
kind := reflect.ValueOf(val).Kind()
if kind == reflect.Ptr {
kind = reflect.TypeOf(val).Elem().Kind()
}
switch kind {
case reflect.String:
if fields[len(fields)-1] == UpstreamIdentifier {
return true
}
case reflect.Map:
if fields[len(fields)-1] == "annotations" {
for _, key := range reflect.ValueOf(val).MapKeys() {
if key.String() == UpstreamIdentifier {
return true
}
}
}
}
return false
}

func (o *KubeObject) effectiveNamespace() string {
if o.HasNamespace() {
return o.GetNamespace()
}
if o.IsNamespaceScoped() {
return DefaultNamespace
}
return UnknownNamespace
}

// GetId gets the Group, Kind, Namespace and Name as the ResourceIdentifier.
func (o *KubeObject) GetId() *ResourceIdentifier {
group, _ := ParseGroupVersion(o.GetAPIVersion())
return &ResourceIdentifier{
Group: group,
Kind: o.GetKind(),
Namespace: o.effectiveNamespace(),
Name: o.GetName(),
}
}

func parseUpstreamIdentifier(upstreamId string) *ResourceIdentifier {
upstreamId = strings.TrimSpace(upstreamId)
r := regexp.MustCompile(upstreamIdentifierRegexPattern)
match := r.FindStringSubmatch(upstreamId)
if match == nil {
panic(ErrInternalAnnotation{Message: fmt.Sprintf("annotation %v: %v is in bad format. expect %q",
UpstreamIdentifier, upstreamId, upstreamIdentifierFormat)})
}
matchGroups := make(map[string]string)
for i, name := range r.SubexpNames() {
if i > 0 && i <= len(match) {
matchGroups[name] = match[i]
}
}
return &ResourceIdentifier{
Group: matchGroups[regexPatternGroup],
Kind: matchGroups[regexPatternKind],
Namespace: matchGroups[regexPatterNamespace],
Name: matchGroups[regexPatternName],
}
}

// GetOriginId provides the `ResourceIdentifier` to identify the upstream origin of a KRM resource.
// This origin is generated and maintained by kpt pkg management and is stored in the `internal.kpt.dev/upstream-identiifer` annotation.
// If a resource does not have an upstream origin, we use its current meta resource ID instead.
func (o *KubeObject) GetOriginId() *ResourceIdentifier {
upstreamId := o.GetAnnotation(UpstreamIdentifier)
if upstreamId != "" {
return parseUpstreamIdentifier(upstreamId)
}
return o.GetId()
}

// HasUpstreamOrigin tells whether a resource is sourced from an upstream package resource.
func (o *KubeObject) HasUpstreamOrigin() bool {
upstreamId := o.GetAnnotation(UpstreamIdentifier)
return upstreamId != ""
}

// ParseGroupVersion parses a "apiVersion" to get the "group" and "version" values.
func ParseGroupVersion(apiVersion string) (group, version string) {
if i := strings.Index(apiVersion, "/"); i > -1 {
return apiVersion[:i], apiVersion[i+1:]
}
return "", apiVersion
}
55 changes: 55 additions & 0 deletions go/fn/origin_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fn

import (
"testing"
)

var resource = []byte(`
apiVersion: v1
kind: ConfigMap
metadata:
name: cm
annotations:
internal.kpt.dev/upstream-identifier: '|ConfigMap|example|example'
`)

var resourceCustom = []byte(`
apiVersion: test.kpt.dev/v1
kind: Custom
metadata:
name: cm
`)

func TestOrigin(t *testing.T) {
o_noGroup, _ := ParseKubeObject(resource)
if o_noGroup.GetOriginId().String() != "|ConfigMap|example|example" {
t.Fatalf("GetOriginId() expect %v, got %v", "|ConfigMap|example|example", o_noGroup.GetOriginId())
}
o_defaultNamespace, _ := ParseKubeObject(resource)
if o_defaultNamespace.GetId().String() != "|ConfigMap|default|cm" {
t.Fatalf("GetId() expect %v, got %v", "|ConfigMap|default|cm", o_defaultNamespace.GetId())
}
o_sameIdAndOrigin, _ := ParseKubeObject(resourceCustom)
if o_sameIdAndOrigin.GetOriginId().String() != o_sameIdAndOrigin.GetId().String() {
t.Fatalf("expect the origin and id the same if upstream-identifier is not given, got OriginID %v, got ID %v",
o_sameIdAndOrigin.GetOriginId(), o_sameIdAndOrigin.GetId())
}
o_unknownNamespace, _ := ParseKubeObject(resourceCustom)
if o_unknownNamespace.GetId().Namespace != UnknownNamespace {
t.Fatalf("expect unknown custom resource use namespace %v, got %v",
UnknownNamespace, o_unknownNamespace.GetId().Namespace)
}
}
8 changes: 8 additions & 0 deletions go/fn/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,14 @@ func Run(p ResourceListProcessor, input []byte) (out []byte, err error) {
err = &t
case *errResultEnd:
err = t
case ErrAttemptToTouchUpstreamIdentifier:
err = &t
case *ErrAttemptToTouchUpstreamIdentifier:
err = t
case ErrInternalAnnotation:
err = &t
case *ErrInternalAnnotation:
err = t
default:
panic(v)
}
Expand Down

0 comments on commit 7181f45

Please sign in to comment.