Skip to content

Commit 326f3f8

Browse files
Yuval ShavitDavidSeptimus
Yuval Shavit
authored andcommitted
use a provided graph instead of our own (#397)
Also, use more normal graph traversal (not our own janky one). This will let us use the graph that the provider plugin gives us. Also, simplify some tree-sitter query stuff. Also also, fix the templates_ts (they were trying to check the .gitignore "directory" because I mis-typed `find -type d` as `find -d`)
1 parent b4ec7fa commit 326f3f8

File tree

9 files changed

+230
-134
lines changed

9 files changed

+230
-134
lines changed

.github/workflows/templates_ts.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@ on:
33
branches: [main, ifc2-foundations] # TODO ifc2-foundations only while that feature branch is active
44
pull_request:
55
branches: [main, ifc2-foundations]
6-
workflow_dispatch: {}
6+
workflow_dispatch: {}
77
concurrency:
88
cancel-in-progress: true
99
group: templates-compilation-${{ github.ref }}
1010
name: iac-templates
1111
jobs:
12-
ls:
12+
list-templates:
1313
outputs:
1414
to_test: ${{ steps.find_dirs.outputs.to_test }}
1515
runs-on: ubuntu-latest
@@ -18,16 +18,16 @@ jobs:
1818
- name: find templates
1919
id: find_dirs
2020
run: |
21-
json="$(find pkg/infra/iac2/templates -d -maxdepth 1 -mindepth 1 -exec basename {} \; \
21+
json="$(find pkg/infra/iac2/templates -type d -maxdepth 1 -mindepth 1 -exec basename {} \; \
2222
| jq -csR 'split("\n") | map(select(. != ""))')"
2323
echo "to_test=$json" > $GITHUB_OUTPUT
2424
checks:
25-
needs: [ls]
25+
needs: [list-templates]
2626
runs-on: ubuntu-latest
2727
strategy:
2828
fail-fast: false
2929
matrix:
30-
template_dir: ${{ fromJson(needs.ls.outputs.to_test) }}
30+
template_dir: ${{ fromJson(needs.list-templates.outputs.to_test) }}
3131
defaults:
3232
run:
3333
working-directory: pkg/infra/iac2/templates/${{ matrix.template_dir }}

go.mod

-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ require (
3939
github.com/kr/pretty v0.3.0 // indirect
4040
github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
4141
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
42-
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
4342
github.com/rivo/uniseg v0.4.4 // indirect
4443
golang.org/x/oauth2 v0.4.0 // indirect
4544
)

go.sum

-2
Original file line numberDiff line numberDiff line change
@@ -515,8 +515,6 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI
515515
github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
516516
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
517517
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
518-
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
519-
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
520518
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
521519
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
522520
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=

pkg/graph/graph.go

+4
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ func (d *Directed[V]) Roots() []V {
5555
return roots
5656
}
5757

58+
func (d *Directed[V]) VertexIdsInTopologicalOrder() ([]string, error) {
59+
return graph.TopologicalSort(d.underlying)
60+
}
61+
5862
func (d *Directed[V]) OutgoingEdges(from V) []Edge[V] {
5963
return handleOutgoingEdges(d, from, func(destination V) Edge[V] {
6064
return Edge[V]{

pkg/infra/iac2/iac_templates.go

+13-16
Original file line numberDiff line numberDiff line change
@@ -61,28 +61,25 @@ func ParseResourceCreationTemplate(contents []byte) ResourceCreationTemplate {
6161
result.inputTypes[inputName] = inputType
6262
}
6363

64-
// return type
65-
returnTypeFunc := query.Select(doQuery(node, findCreateFuncQuery), query.ContentOf(query.ParamNamed("return_type")))
66-
returnType, err := returnTypeFunc()
67-
if !err {
64+
// return type and expression
65+
createFunc := doQuery(node, findCreateFuncQuery)
66+
create, found := createFunc()
67+
if !found {
6868
// unexpected, since all inputs are from resources in the klotho binary
6969
panic("couldn't find valid create() function")
7070
}
71-
result.outputType = returnType
72-
73-
// return expression
74-
returnBodyFunc := query.Select(doQuery(node, findCreateFuncQuery), query.ContentOf(query.ParamNamed("return_body")))
75-
returnBody, err := returnBodyFunc()
76-
if !err {
77-
// unexpected, since all inputs are from resources in the klotho binary
78-
panic("couldn't find valid create() function")
79-
}
80-
result.expressionTemplate = parameterizeArgs(returnBody)
71+
result.outputType = create["return_type"].Content()
72+
result.expressionTemplate = parameterizeArgs(create["return_body"].Content())
8173

8274
// imports
8375
result.imports = make(map[string]struct{})
84-
importsQuery := query.Select(doQuery(node, findImportsQuery), query.ContentOf(query.ParamNamed("import")))
85-
for _, importLine := range query.Collect(importsQuery) {
76+
importsQuery := doQuery(node, findImportsQuery)
77+
for {
78+
match, found := importsQuery()
79+
if !found {
80+
break
81+
}
82+
importLine := match["import"].Content()
8683
// Trim any trailing semicolons. This helps normalize imports, so that we don't include them twice if one file
8784
// includes the semicolon and the other doesn't.
8885
importLine = strings.TrimRight(importLine, ";")

pkg/infra/iac2/templates_compiler.go

+81-85
Original file line numberDiff line numberDiff line change
@@ -6,31 +6,30 @@ import (
66
"fmt"
77
"github.com/klothoplatform/klotho/pkg/graph"
88
"github.com/klothoplatform/klotho/pkg/multierr"
9-
"github.com/mitchellh/hashstructure/v2"
109
"github.com/pkg/errors"
1110
"io"
1211
"io/fs"
1312
"reflect"
1413
"regexp"
1514
"sort"
16-
"strings"
1715
)
1816

1917
type (
2018
varNamer interface {
2119
VariableName() string
2220
}
2321

24-
resource struct {
25-
hash string
26-
element any
27-
}
28-
2922
templatesCompiler struct {
30-
templates fs.FS
31-
resourceGraph *graph.Directed[resource]
32-
resources map[any]resource
23+
// templates is the fs.FS where we read all of our `<struct>/factory.ts` files
24+
templates fs.FS
25+
// resourceGraph is the graph of resources to render
26+
resourceGraph *graph.Directed[graph.Identifiable]
27+
// templatesByStructName is a cache from struct name (e.g. "CloudwatchLogs") to the template for that struct.
3328
templatesByStructName map[string]ResourceCreationTemplate
29+
// resourceVarNames is a set of all variable names
30+
resourceVarNames map[string]struct{}
31+
// resourceVarNamesById is a map from resource id to the variable name for that resource
32+
resourceVarNamesById map[string]string
3433
}
3534
)
3635

@@ -41,69 +40,52 @@ var (
4140
nonIdentifierChars = regexp.MustCompile(`\W`)
4241
)
4342

44-
func CreateTemplatesCompiler() *templatesCompiler {
43+
func CreateTemplatesCompiler(resources *graph.Directed[graph.Identifiable]) *templatesCompiler {
4544
subTemplates, err := fs.Sub(standardTemplates, "templates")
4645
if err != nil {
4746
panic(err) // unexpected, since standardTemplates is statically built into klotho
4847
}
4948
return &templatesCompiler{
5049
templates: subTemplates,
51-
resourceGraph: graph.NewDirected[resource](),
52-
resources: make(map[any]resource),
50+
resourceGraph: resources,
5351
templatesByStructName: make(map[string]ResourceCreationTemplate),
52+
resourceVarNames: make(map[string]struct{}),
53+
resourceVarNamesById: make(map[string]string),
5454
}
5555
}
5656

57-
func (tc templatesCompiler) AddResource(v any) {
58-
if _, exists := tc.resources[v]; exists {
59-
return
60-
61-
}
62-
res := resource{
63-
hash: fmt.Sprintf(`%x`, len(tc.resources)),
64-
element: v,
65-
}
66-
tc.resources[v] = res
67-
tc.resourceGraph.AddVertex(res)
68-
for _, child := range getStructValues(v) {
69-
if reflect.TypeOf(child).Kind() == reflect.Struct {
70-
tc.AddResource(child)
71-
childRes := tc.getResource(child)
72-
tc.resourceGraph.AddEdge(res.Id(), childRes.Id())
73-
}
74-
}
75-
}
76-
77-
func (tc templatesCompiler) getResource(v any) resource {
78-
childRes, childExists := tc.resources[v]
79-
if !childExists {
80-
panic(fmt.Sprintf(`compiler has inconsistent state: no resource for %v`, v))
81-
}
82-
return childRes
83-
}
84-
8557
func (tc templatesCompiler) RenderBody(out io.Writer) error {
86-
// TODO: for now, assume a nice little tree
87-
// TODO: need a stable sorting of outputs!
88-
8958
errs := multierr.Error{}
90-
for _, res := range tc.resourceGraph.Roots() {
91-
err := tc.renderResource(out, res.element)
59+
vertexIds, err := tc.resourceGraph.VertexIdsInTopologicalOrder()
60+
if err != nil {
61+
return err
62+
}
63+
reverseInPlace(vertexIds)
64+
for _, id := range vertexIds {
65+
resource := tc.resourceGraph.GetVertex(id)
66+
err := tc.renderResource(out, resource)
9267
errs.Append(err)
9368
}
9469
return errs.ErrOrNil()
9570
}
9671

9772
func (tc templatesCompiler) RenderImports(out io.Writer) error {
98-
// TODO: for now, assume a nice little tree
73+
errs := multierr.Error{}
9974

10075
allImports := make(map[string]struct{})
101-
for _, res := range tc.resources {
102-
tmpl := tc.GetTemplate(res.element)
76+
for _, res := range tc.resourceGraph.GetAllVertices() {
77+
tmpl, err := tc.GetTemplate(res)
78+
if err != nil {
79+
errs.Append(err)
80+
continue
81+
}
10382
for statement, _ := range tmpl.imports {
10483
allImports[statement] = struct{}{}
10584
}
10685
}
86+
if err := errs.ErrOrNil(); err != nil {
87+
return err
88+
}
10789

10890
sortedImports := make([]string, 0, len(allImports))
10991
for statement, _ := range allImports {
@@ -123,67 +105,81 @@ func (tc templatesCompiler) RenderImports(out io.Writer) error {
123105
return nil
124106
}
125107

126-
func (tc templatesCompiler) renderResource(out io.Writer, res any) error {
108+
func (tc templatesCompiler) renderResource(out io.Writer, resource graph.Identifiable) error {
127109
// TODO: for now, assume a nice little tree
128110
errs := multierr.Error{}
129111

130112
inputArgs := make(map[string]string)
131-
for fieldName, child := range getStructValues(res) {
132-
childType := reflect.TypeOf(child) // todo cache in the resource?
113+
for fieldName, child := range getStructValues(resource) {
114+
childType := reflect.TypeOf(child)
133115
switch childType.Kind() {
134116
case reflect.String:
135117
inputArgs[fieldName] = quoteTsString(child.(string))
136-
case reflect.Struct:
137-
errs.Append(tc.renderResource(out, child))
138-
inputArgs[fieldName] = tc.getResource(child).VariableName()
118+
case reflect.Struct, reflect.Pointer:
119+
if child, ok := child.(graph.Identifiable); ok {
120+
inputArgs[fieldName] = tc.getVarName(child)
121+
} else {
122+
errs.Append(errors.Errorf(`child struct of %v is not of a known type: %v`, resource, child))
123+
}
139124
default:
140-
errs.Append(errors.Errorf(`unrecognized input type for %v [%s]: %v`, res, fieldName, child))
125+
errs.Append(errors.Errorf(`unrecognized input type for %v [%s]: %v`, resource, fieldName, child))
141126
}
142127
}
143128

144-
varName := tc.getResource(res).VariableName()
129+
varName := tc.getVarName(resource)
130+
tmpl, err := tc.GetTemplate(resource)
131+
if err != nil {
132+
return err
133+
}
134+
145135
fmt.Fprintf(out, `const %s = `, varName)
146-
errs.Append(tc.GetTemplate(res).RenderCreate(out, inputArgs))
136+
errs.Append(tmpl.RenderCreate(out, inputArgs))
147137
out.Write([]byte(";\n"))
148138

149139
return errs.ErrOrNil()
140+
}
150141

142+
// getVarName gets a unique but nice-looking variable for the given item.
143+
//
144+
// It does this by first calculating an ideal variable name, which is a camel-cased ${structName}${Id}. For example, if
145+
// you had an object CoolResource{id: "foo-bar"}, the ideal variable name is coolResourceFooBar.
146+
//
147+
// If that ideal variable name hasn't been used yet, this function returns it. If it has been used, we append `_${i}` to
148+
// it, where ${i} is the lowest positive integer that would give us a new, unique variable name. This isn't expected
149+
// to happen often, if at all, since ids are globally unique.
150+
func (tc templatesCompiler) getVarName(v graph.Identifiable) string {
151+
if name, alreadyResolved := tc.resourceVarNamesById[v.Id()]; alreadyResolved {
152+
return name
153+
}
154+
155+
// Generate something like "lambdaFoo", where Lambda is the name of the struct and "foo" is the id
156+
desiredName := lowercaseFirst(structName(v)) + toUpperCamel(v.Id())
157+
resolvedName := desiredName
158+
for i := 1; ; i++ {
159+
_, varNameTaken := tc.resourceVarNames[resolvedName]
160+
if varNameTaken {
161+
resolvedName = fmt.Sprintf("%s_%d", desiredName, i)
162+
} else {
163+
break
164+
}
165+
}
166+
tc.resourceVarNames[resolvedName] = struct{}{}
167+
tc.resourceVarNamesById[v.Id()] = resolvedName
168+
return resolvedName
151169
}
152170

153-
func (tc templatesCompiler) GetTemplate(v any) ResourceCreationTemplate {
154-
// TODO cache into the resource
155-
vType := reflect.TypeOf(v)
156-
typeName := vType.Name()
171+
func (tc templatesCompiler) GetTemplate(v graph.Identifiable) (ResourceCreationTemplate, error) {
172+
typeName := structName(v)
157173
existing, ok := tc.templatesByStructName[typeName]
158174
if ok {
159-
return existing
175+
return existing, nil
160176
}
161177
templateName := camelToSnake(typeName)
162178
contents, err := fs.ReadFile(tc.templates, templateName+`/factory.ts`)
163179
if err != nil {
164-
// Shouldn't ever happen; would mean an error in how we set up our structs
165-
panic(err)
180+
return ResourceCreationTemplate{}, err
166181
}
167182
template := ParseResourceCreationTemplate(contents)
168183
tc.templatesByStructName[typeName] = template
169-
return template
170-
}
171-
172-
func (r resource) Id() string {
173-
if r.hash == "" {
174-
h, err := hashstructure.Hash(r.element, hashstructure.FormatV2, nil)
175-
if err != nil {
176-
// Shouldn't ever happen; would mean an error in how we set up our structs
177-
panic(err)
178-
}
179-
r.hash = fmt.Sprintf("%x", h)
180-
}
181-
return r.hash
182-
}
183-
184-
func (r resource) VariableName() string {
185-
name := fmt.Sprintf(`%s_%s`, reflect.TypeOf(r.element).Name(), r.Id())
186-
firstChar := name[:1]
187-
rest := name[1:]
188-
return strings.ToLower(firstChar) + rest
184+
return template, nil
189185
}

0 commit comments

Comments
 (0)