Inject cross-resource references from provider metadata

pkg/config/provider.go

@@ -96,11 +96,21 @@ type Provider struct {
 	// resource name.
 	Resources map[string]*Resource
 
+	// refInjectors is an ordered list of `ReferenceInjector`s for
+	// injecting references across this Provider's resources.
+	refInjectors []ReferenceInjector
+
 	// resourceConfigurators is a map holding resource configurators where key
 	// is Terraform resource name.
 	resourceConfigurators map[string]ResourceConfiguratorChain
 }
 
+// ReferenceInjector injects cross-resource references across the resources
+// of this Provider.
+type ReferenceInjector interface {
+	InjectReferences(map[string]*Resource) error
+}
+
 // A ProviderOption configures a Provider.
 type ProviderOption func(*Provider)
 
@@ -147,10 +157,19 @@ func WithDefaultResourceOptions(opts ...ResourceOption) ProviderOption {
 	}
 }
 
+// WithReferenceInjectors configures an ordered list of `ReferenceInjector`s
+// for this Provider. The configured reference resolvers are executed in order
+// to inject cross-resource references across this Provider's resources.
+func WithReferenceInjectors(refInjectors []ReferenceInjector) ProviderOption {
+	return func(p *Provider) {
+		p.refInjectors = refInjectors
+	}
+}
+
 // NewProvider builds and returns a new Provider from provider
 // tfjson schema, that is generated using Terraform CLI with:
 // `terraform providers schema --json`
-func NewProvider(schema []byte, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider {
+func NewProvider(schema []byte, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider { // nolint:gocyclo
 	ps := tfjson.ProviderSchemas{}
 	if err := ps.UnmarshalJSON(schema); err != nil {
 		panic(err)
@@ -202,6 +221,11 @@ func NewProvider(schema []byte, prefix string, modulePath string, metadata []byt
 	if err := p.loadMetadata(metadata); err != nil {
 		panic(errors.Wrap(err, "cannot load provider metadata"))
 	}
+	for i, refInjector := range p.refInjectors {
+		if err := refInjector.InjectReferences(p.Resources); err != nil {
+			panic(errors.Wrapf(err, "cannot inject references using the configured ReferenceInjector at index %d", i))
+		}
+	}
 	return p
 }
 

pkg/config/resource.go

@@ -116,6 +116,11 @@ type Reference struct {
 	// Type is the type name of the CRD if it is in the same package or
 	// <package-path>.<type-name> if it is in a different package.
 	Type string
+	// TerraformName is the name of the Terraform resource
+	// which will be referenced. The supplied resource name is
+	// converted to a type name of the corresponding CRD using
+	// the configured TerraformTypeMapper.
+	TerraformName string
 	// Extractor is the function to be used to extract value from the
 	// referenced type. Defaults to getting external name.
 	// Optional
@@ -287,6 +292,11 @@ type Resource struct {
 	// References keeps the configuration to build cross resource references
 	References References
 
+	// SkipReferencesTo configures attributes for which
+	// references will not be injected. Expected format
+	// for each element is <Terraform resource name>.<attribute name>.
+	SkipReferencesTo []string
+
 	// Sensitive keeps the configuration to handle sensitive information
 	Sensitive Sensitive
 

pkg/pipeline/crd.go

@@ -21,13 +21,17 @@ import (
 	tjtypes "github.com/upbound/upjet/pkg/types"
 )
 
-// GenStatement is printed on every generated file.
-const GenStatement = "// Code generated by upjet. DO NOT EDIT."
+const (
+	// GenStatement is printed on every generated file.
+	GenStatement = "// Code generated by upjet. DO NOT EDIT."
+
+	apiRoot = "apis"
+)
 
 // NewCRDGenerator returns a new CRDGenerator.
 func NewCRDGenerator(pkg *types.Package, rootDir, providerShortName, group, version string) *CRDGenerator {
 	return &CRDGenerator{
-		LocalDirectoryPath: filepath.Join(rootDir, "apis", strings.ToLower(strings.Split(group, ".")[0]), version),
+		LocalDirectoryPath: filepath.Join(rootDir, apiRoot, strings.ToLower(strings.Split(group, ".")[0]), version),
 		LicenseHeaderPath:  filepath.Join(rootDir, "hack", "boilerplate.go.txt"),
 		Group:              group,
 		ProviderShortName:  providerShortName,

pkg/pipeline/example.go

@@ -18,123 +18,64 @@ import (
 	"sigs.k8s.io/yaml"
 
 	"github.com/upbound/upjet/pkg/config"
+	"github.com/upbound/upjet/pkg/registry/reference"
 	tjtypes "github.com/upbound/upjet/pkg/types"
 )
 
 var (
-	reRef  = regexp.MustCompile(`\${(.+)}`)
 	reFile = regexp.MustCompile(`file\("(.+)"\)`)
 )
 
-type pavedWithManifest struct {
-	manifestPath string
-	paved        *fieldpath.Paved
-	refsResolved bool
-}
-
 // ExampleGenerator represents a pipeline for generating example manifests.
 // Generates example manifests for Terraform resources under examples-generated.
 type ExampleGenerator struct {
-	rootDir        string
-	configResource map[string]*config.Resource
-	resources      map[string]*pavedWithManifest
+	reference.Injector
+	rootDir         string
+	configResources map[string]*config.Resource
+	resources       map[string]*reference.PavedWithManifest
 }
 
 // NewExampleGenerator returns a configured ExampleGenerator
-func NewExampleGenerator(rootDir string, configResource map[string]*config.Resource) *ExampleGenerator {
+func NewExampleGenerator(rootDir, modulePath, shortName string, configResources map[string]*config.Resource) *ExampleGenerator {
 	return &ExampleGenerator{
-		rootDir:        rootDir,
-		configResource: configResource,
-		resources:      make(map[string]*pavedWithManifest),
+		Injector: reference.Injector{
+			ModulePath:        modulePath,
+			ProviderShortName: shortName,
+		},
+		rootDir:         rootDir,
+		configResources: configResources,
+		resources:       make(map[string]*reference.PavedWithManifest),
 	}
 }
 
 // StoreExamples stores the generated example manifests under examples-generated in
 // their respective API groups.
 func (eg *ExampleGenerator) StoreExamples() error {
 	for n, pm := range eg.resources {
-		if err := eg.resolveReferencesOfPaved(pm); err != nil {
+		if err := eg.ResolveReferencesOfPaved(pm, eg.resources); err != nil {
 			return errors.Wrapf(err, "cannot resolve references for resource: %s", n)
 		}
-		u := pm.paved.UnstructuredContent()
+		u := pm.Paved.UnstructuredContent()
 		delete(u["spec"].(map[string]interface{})["forProvider"].(map[string]interface{}), "depends_on")
 		buff, err := yaml.Marshal(u)
 		if err != nil {
 			return errors.Wrapf(err, "cannot marshal example manifest for resource: %s", n)
 		}
-		manifestDir := filepath.Dir(pm.manifestPath)
+		manifestDir := filepath.Dir(pm.ManifestPath)
 		if err := os.MkdirAll(manifestDir, 0750); err != nil {
 			return errors.Wrapf(err, "cannot mkdir %s", manifestDir)
 		}
 		// no sensitive info in the example manifest
-		if err := ioutil.WriteFile(pm.manifestPath, buff, 0644); err != nil { // nolint:gosec
-			return errors.Wrapf(err, "cannot write example manifest file %s for resource %s", pm.manifestPath, n)
-		}
-	}
-	return nil
-}
-
-func (eg *ExampleGenerator) resolveReferencesOfPaved(pm *pavedWithManifest) error {
-	if pm.refsResolved {
-		return nil
-	}
-	pm.refsResolved = true
-	return errors.Wrap(eg.resolveReferences(pm.paved.UnstructuredContent()), "failed to resolve references of paved")
-}
-
-func (eg *ExampleGenerator) resolveReferences(params map[string]interface{}) error { // nolint:gocyclo
-	for k, v := range params {
-		switch t := v.(type) {
-		case map[string]interface{}:
-			if err := eg.resolveReferences(t); err != nil {
-				return err
-			}
-
-		case []interface{}:
-			for _, e := range t {
-				eM, ok := e.(map[string]interface{})
-				if !ok {
-					continue
-				}
-				if err := eg.resolveReferences(eM); err != nil {
-					return err
-				}
-			}
-
-		case string:
-			g := reRef.FindStringSubmatch(t)
-			if len(g) != 2 {
-				continue
-			}
-			path := strings.Split(g[1], ".")
-			// expected reference format is <resource type>.<resource name>.<field name>
-			if len(path) < 3 {
-				continue
-			}
-			pm := eg.resources[path[0]]
-			if pm == nil || pm.paved == nil {
-				continue
-			}
-			if err := eg.resolveReferencesOfPaved(pm); err != nil {
-				return errors.Wrapf(err, "cannot recursively resolve references for %q", path[0])
-			}
-			pathStr := strings.Join(append([]string{"spec", "forProvider"}, path[2:]...), ".")
-			s, err := pm.paved.GetString(pathStr)
-			if fieldpath.IsNotFound(err) {
-				continue
-			}
-			if err != nil {
-				return errors.Wrapf(err, "cannot get string value from paved: %s", pathStr)
-			}
-			params[k] = s
+		if err := ioutil.WriteFile(pm.ManifestPath, buff, 0600); err != nil {
+			return errors.Wrapf(err, "cannot write example manifest file %s for resource %s", pm.ManifestPath, n)
 		}
 	}
 	return nil
 }
 
 // Generate generates an example manifest for the specified Terraform resource.
 func (eg *ExampleGenerator) Generate(group, version string, r *config.Resource, fieldTransformations map[string]tjtypes.Transformation) error {
-	rm := eg.configResource[r.Name].MetaResource
+	rm := eg.configResources[r.Name].MetaResource
 	if rm == nil || len(rm.Examples) == 0 {
 		return nil
 	}
@@ -158,9 +99,10 @@ func (eg *ExampleGenerator) Generate(group, version string, r *config.Resource,
 		},
 	}
 	manifestDir := filepath.Join(eg.rootDir, "examples-generated", strings.ToLower(strings.Split(group, ".")[0]))
-	eg.resources[r.Name] = &pavedWithManifest{
-		manifestPath: filepath.Join(manifestDir, fmt.Sprintf("%s.yaml", strings.ToLower(r.Kind))),
-		paved:        fieldpath.Pave(example),
+	eg.resources[r.Name] = &reference.PavedWithManifest{
+		ManifestPath: filepath.Join(manifestDir, fmt.Sprintf("%s.yaml", strings.ToLower(r.Kind))),
+		Paved:        fieldpath.Pave(example),
+		ParamsPrefix: []string{"spec", "forProvider"},
 	}
 	return nil
 }
@@ -198,30 +140,31 @@ func transformFields(params map[string]interface{}, omittedFields []string, t ma
 		}
 	}
 
-	for hn, transform := range t {
-		for n, v := range params {
-			if hn == getHierarchicalName(namePrefix, n) {
-				delete(params, n)
-				if transform.IsRef {
-					if !transform.IsSensitive {
-						params[transform.TransformedName] = getRefField(v,
-							map[string]interface{}{
-								"name": "example",
-							})
-					} else {
-						secretName, secretKey := getSecretRef(v)
-						params[transform.TransformedName] = getRefField(v,
-							map[string]interface{}{
-								"name":      secretName,
-								"namespace": "crossplane-system",
-								"key":       secretKey,
-							})
-					}
-				} else {
-					params[transform.TransformedName] = v
-				}
-				break
+	for n, v := range params {
+		hName := getHierarchicalName(namePrefix, n)
+		for hn, transform := range t {
+			if hn != hName {
+				continue
+			}
+			delete(params, n)
+			switch {
+			case !transform.IsRef:
+				params[transform.TransformedName] = v
+			case transform.IsSensitive:
+				secretName, secretKey := getSecretRef(v)
+				params[transform.TransformedName] = getRefField(v,
+					map[string]interface{}{
+						"name":      secretName,
+						"namespace": "crossplane-system",
+						"key":       secretKey,
+					})
+			default:
+				params[transform.TransformedName] = getRefField(v,
+					map[string]interface{}{
+						"name": "example",
+					})
 			}
+			break
 		}
 	}
 }
@@ -245,7 +188,7 @@ func getSecretRef(v interface{}) (string, string) {
 	if !ok {
 		return secretName, secretKey
 	}
-	g := reRef.FindStringSubmatch(s)
+	g := reference.ReRef.FindStringSubmatch(s)
 	if len(g) != 2 {
 		return secretName, secretKey
 	}

pkg/pipeline/run.go

@@ -1,17 +1,5 @@
 /*
- Copyright 2021 Upbound Inc.
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+Copyright 2022 Upbound Inc.
 */
 
 package pipeline
@@ -57,6 +45,10 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 		resourcesGroups[group][resource.Version][name] = resource
 	}
 
+	exampleGen := NewExampleGenerator(rootDir, pc.ModulePath, pc.ShortName, pc.Resources)
+	if err := exampleGen.SetReferenceTypes(pc.Resources); err != nil {
+		panic(errors.Wrap(err, "cannot set reference types for resources"))
+	}
 	// Add ProviderConfig API package to the list of API version packages.
 	apiVersionPkgList := make([]string, 0)
 	for _, p := range pc.BasePackages.APIVersion {
@@ -68,7 +60,6 @@ func Run(pc *config.Provider, rootDir string) { // nolint:gocyclo
 		controllerPkgList = append(controllerPkgList, filepath.Join(pc.ModulePath, p))
 	}
 	count := 0
-	exampleGen := NewExampleGenerator(rootDir, pc.Resources)
 	for group, versions := range resourcesGroups {
 		for version, resources := range versions {
 			var tfResources []*terraformedInput