Skip to content

Commit 577156e

Browse files
dimsdims
authored
Support for EKS internal environments (#8294)
* Support for EKS internal beta/gamma environments Allow setting `AWS_ENDPOINT_URL_EKS` to internal URLs for testing/development cycles. We setup a AWS Lambda based CloudFormation Resource that acts as a proxy to pass through calls from CFN templates to the internal URLs. Added a couple of beta.go to encapsulate creating custom resources from the same info used to create the EKS related CFN resources. The hard part is the python based lambda code that turns the CFN handler payload into calls to various eks/iam and other APIs. Signed-off-by: Davanum Srinivas <davanum@gmail.com> * Incorporate review feedback Signed-off-by: Davanum Srinivas <davanum@gmail.com> --------- Signed-off-by: Davanum Srinivas <davanum@gmail.com>
1 parent 3cad04f commit 577156e

File tree

12 files changed

+828
-12
lines changed

12 files changed

+828
-12
lines changed

‎pkg/apis/eksctl.io/v1alpha5/beta.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package v1alpha5
2+
3+
import (
4+
"os"
5+
"strings"
6+
)
7+
8+
func (c *ClusterConfig) IsCustomEksEndpoint() bool {
9+
eksEndpoint := os.Getenv("AWS_EKS_ENDPOINT")
10+
if eksEndpoint == "" {
11+
eksEndpoint = os.Getenv("AWS_ENDPOINT_URL_EKS")
12+
}
13+
return strings.Contains(eksEndpoint, "beta") || strings.Contains(eksEndpoint, "gamma")
14+
}

‎pkg/cfn/builder/beta.go

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
package builder
2+
3+
import (
4+
"context"
5+
_ "embed"
6+
"fmt"
7+
"os"
8+
"strings"
9+
10+
"github.com/aws/aws-sdk-go-v2/service/sts"
11+
12+
"github.com/weaveworks/eksctl/pkg/awsapi"
13+
cft "github.com/weaveworks/eksctl/pkg/cfn/template"
14+
"github.com/weaveworks/eksctl/pkg/goformation"
15+
gfn "github.com/weaveworks/eksctl/pkg/goformation/cloudformation"
16+
"github.com/weaveworks/eksctl/pkg/goformation/cloudformation/cloudformation"
17+
gfneks "github.com/weaveworks/eksctl/pkg/goformation/cloudformation/eks"
18+
"github.com/weaveworks/eksctl/pkg/goformation/cloudformation/lambda"
19+
gfnt "github.com/weaveworks/eksctl/pkg/goformation/cloudformation/types"
20+
)
21+
22+
//go:embed templates/beta-resources.yaml
23+
var betaResourcesTemplate []byte
24+
25+
//go:embed templates/beta.py
26+
var lambdaBetaPy []byte
27+
28+
func addBetaResources(stsAPI awsapi.STS, stackName string, clusterTemplate *gfn.Template, g *gfneks.Cluster) error {
29+
30+
identity, err := stsAPI.GetCallerIdentity(context.TODO(), &sts.GetCallerIdentityInput{})
31+
if err != nil {
32+
return fmt.Errorf("unable to get identity: %w", err)
33+
}
34+
userArn := *identity.Arn
35+
baseArn := userArn[:strings.LastIndex(userArn, "/")]
36+
roleArn := fmt.Sprintf("%s%s", baseArn, "/{{SessionName}}")
37+
iamARN := strings.Replace(
38+
strings.Replace(baseArn, "assumed-role", "role", 1),
39+
"sts", "iam", 1)
40+
41+
clusterName := "eksctl-" + stackName + "-cluster"
42+
43+
template, err := goformation.ParseYAML(betaResourcesTemplate)
44+
if err != nil {
45+
return err
46+
}
47+
for resourceName, resource := range template.Resources {
48+
clusterTemplate.Resources[resourceName] = resource
49+
}
50+
for key, output := range template.Outputs {
51+
clusterTemplate.Outputs[key] = output
52+
}
53+
customResource := clusterTemplate.Resources["ControlPlane"].(*gfn.CustomResource)
54+
if g.AccessConfig != nil {
55+
customResource.Properties["AccessConfig"] = g.AccessConfig
56+
}
57+
if g.BootstrapSelfManagedAddons != nil {
58+
customResource.Properties["BootstrapSelfManagedAddons"] = g.BootstrapSelfManagedAddons
59+
}
60+
if g.ComputeConfig != nil {
61+
customResource.Properties["ComputeConfig"] = g.ComputeConfig
62+
}
63+
if g.EncryptionConfig != nil {
64+
customResource.Properties["EncryptionConfig"] = g.EncryptionConfig
65+
}
66+
if g.KubernetesNetworkConfig != nil {
67+
customResource.Properties["KubernetesNetworkConfig"] = g.KubernetesNetworkConfig
68+
}
69+
if g.Logging != nil {
70+
customResource.Properties["Logging"] = g.Logging
71+
}
72+
if g.Name != nil {
73+
customResource.Properties["Name"] = g.Name
74+
}
75+
if g.OutpostConfig != nil {
76+
customResource.Properties["OutpostConfig"] = g.OutpostConfig
77+
}
78+
if g.RemoteNetworkConfig != nil {
79+
customResource.Properties["RemoteNetworkConfig"] = g.RemoteNetworkConfig
80+
}
81+
if g.ResourcesVpcConfig != nil {
82+
customResource.Properties["ResourcesVpcConfig"] = g.ResourcesVpcConfig
83+
}
84+
if g.RoleArn != nil {
85+
customResource.Properties["RoleArn"] = g.RoleArn
86+
}
87+
if g.StorageConfig != nil {
88+
customResource.Properties["StorageConfig"] = g.StorageConfig
89+
}
90+
if g.Tags != nil {
91+
g.Tags = append(g.Tags, cloudformation.Tag{
92+
Key: gfnt.NewString("Name"),
93+
Value: gfnt.NewString(clusterName + "/ControlPlane"),
94+
})
95+
customResource.Properties["Tags"] = g.Tags
96+
} else {
97+
customResource.Properties["Tags"] = []cloudformation.Tag{
98+
{
99+
Key: gfnt.NewString("Name"),
100+
Value: gfnt.NewString(clusterName + "/ControlPlane"),
101+
},
102+
}
103+
}
104+
if g.UpgradePolicy != nil {
105+
customResource.Properties["UpgradePolicy"] = g.UpgradePolicy
106+
}
107+
if g.Version != nil {
108+
customResource.Properties["Version"] = g.Version
109+
}
110+
if g.ZonalShiftConfig != nil {
111+
customResource.Properties["ZonalShiftConfig"] = g.ZonalShiftConfig
112+
}
113+
114+
customResource.Properties["IAMPrincipalArn"] = gfnt.NewString(iamARN)
115+
customResource.Properties["STSRoleArn"] = gfnt.NewString(roleArn)
116+
117+
customFunction := clusterTemplate.Resources["CustomEKSFunction"].(*lambda.Function)
118+
customFunction.Code = &lambda.Function_Code{
119+
ZipFile: gfnt.NewString(string(lambdaBetaPy)),
120+
}
121+
122+
clusterTemplate.Outputs["EKSFunctionArn"] = gfn.Output{
123+
Value: gfnt.MakeFnGetAttString("CustomEKSFunction", "Arn"),
124+
Export: &gfn.Export{
125+
Name: gfnt.MakeFnSubString(fmt.Sprintf("${%s}::EKSFunctionArn", gfnt.StackName)),
126+
},
127+
}
128+
129+
clusterTemplate.Parameters["EksEndpointUrl"] = gfn.Parameter{
130+
Type: "String",
131+
Description: "The endpoint URL for the EKS service",
132+
Default: gfnt.NewString(os.Getenv("AWS_ENDPOINT_URL_EKS")),
133+
}
134+
return nil
135+
}
136+
137+
func addBetaManagedNodeGroupResources(managedResource *gfneks.Nodegroup, stackName string) *gfn.CustomResource {
138+
customResource := &gfn.CustomResource{
139+
Type: "Custom::EksManagedNodeGroup",
140+
}
141+
customResource.Properties = make(map[string]interface{})
142+
functionArn := gfnt.MakeFnImportValueString(fmt.Sprintf("eksctl-%s-cluster::EKSFunctionArn", stackName))
143+
customResource.Properties["ServiceToken"] = functionArn
144+
145+
if managedResource.AmiType != nil {
146+
customResource.Properties["AmiType"] = managedResource.AmiType
147+
}
148+
if managedResource.CapacityType != nil {
149+
customResource.Properties["CapacityType"] = managedResource.CapacityType
150+
}
151+
if managedResource.ClusterName != nil {
152+
customResource.Properties["ClusterName"] = managedResource.ClusterName
153+
}
154+
if managedResource.DiskSize != nil {
155+
customResource.Properties["DiskSize"] = managedResource.DiskSize
156+
}
157+
if managedResource.ForceUpdateEnabled != nil {
158+
customResource.Properties["ForceUpdateEnabled"] = managedResource.ForceUpdateEnabled
159+
}
160+
if managedResource.InstanceTypes != nil {
161+
customResource.Properties["InstanceTypes"] = managedResource.InstanceTypes
162+
}
163+
if managedResource.Labels != nil {
164+
customResource.Properties["Labels"] = managedResource.Labels
165+
}
166+
if managedResource.LaunchTemplate != nil {
167+
customResource.Properties["LaunchTemplate"] = managedResource.LaunchTemplate
168+
}
169+
if managedResource.NodeRepairConfig != nil {
170+
customResource.Properties["NodeRepairConfig"] = managedResource.NodeRepairConfig
171+
}
172+
if managedResource.NodeRole != nil {
173+
customResource.Properties["NodeRole"] = managedResource.NodeRole
174+
}
175+
if managedResource.NodegroupName != nil {
176+
customResource.Properties["NodegroupName"] = managedResource.NodegroupName
177+
}
178+
if managedResource.ReleaseVersion != nil {
179+
customResource.Properties["ReleaseVersion"] = managedResource.ReleaseVersion
180+
}
181+
if managedResource.RemoteAccess != nil {
182+
customResource.Properties["RemoteAccess"] = managedResource.RemoteAccess
183+
}
184+
if managedResource.ScalingConfig != nil {
185+
customResource.Properties["ScalingConfig"] = managedResource.ScalingConfig
186+
}
187+
if managedResource.Subnets != nil {
188+
customResource.Properties["Subnets"] = managedResource.Subnets
189+
}
190+
if managedResource.Tags != nil {
191+
customResource.Properties["Tags"] = managedResource.Tags
192+
}
193+
if managedResource.Taints != nil {
194+
customResource.Properties["Taints"] = managedResource.Taints
195+
}
196+
if managedResource.UpdateConfig != nil {
197+
customResource.Properties["UpdateConfig"] = managedResource.UpdateConfig
198+
}
199+
if managedResource.Version != nil {
200+
customResource.Properties["Version"] = managedResource.Version
201+
}
202+
203+
return customResource
204+
}
205+
206+
func createBetaAssumeRolePolicy() interface{} {
207+
statements := []cft.MapOfInterfaces{
208+
{
209+
"Effect": "Allow",
210+
"Principal": cft.MapOfInterfaces{
211+
"Service": "eks.amazonaws.com",
212+
},
213+
"Action": []string{
214+
"sts:AssumeRole",
215+
"sts:TagSession",
216+
},
217+
},
218+
{
219+
"Effect": "Allow",
220+
"Principal": cft.MapOfInterfaces{
221+
"Service": "eks-beta.aws.internal",
222+
},
223+
"Action": []string{
224+
"sts:AssumeRole",
225+
"sts:TagSession",
226+
},
227+
},
228+
{
229+
"Effect": "Allow",
230+
"Principal": cft.MapOfInterfaces{
231+
"Service": "eks-gamma.aws.internal",
232+
},
233+
"Action": []string{
234+
"sts:AssumeRole",
235+
"sts:TagSession",
236+
},
237+
},
238+
}
239+
return cft.MakePolicyDocument(statements...)
240+
}
241+
242+
func addBetaAccessEntry(stackName string, accessEntryType string) *gfn.CustomResource {
243+
customResource := &gfn.CustomResource{
244+
Type: "Custom::EksAccessEntry",
245+
}
246+
customResource.Properties = make(map[string]interface{})
247+
functionArn := gfnt.MakeFnImportValueString(fmt.Sprintf("eksctl-%s-cluster::EKSFunctionArn", stackName))
248+
customResource.Properties["ServiceToken"] = functionArn
249+
customResource.Properties["PrincipalArn"] = gfnt.MakeFnGetAttString(cfnIAMInstanceRoleName, "Arn")
250+
customResource.Properties["ClusterName"] = gfnt.NewString(stackName)
251+
customResource.Properties["Type"] = gfnt.NewString(accessEntryType)
252+
return customResource
253+
}

‎pkg/cfn/builder/cluster.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,11 @@ type ClusterResourceSet struct {
2929
region string
3030
vpcResourceSet VPCResourceSet
3131
securityGroups *gfnt.Value
32+
stsAPI awsapi.STS
3233
}
3334

3435
// NewClusterResourceSet returns a resource set for the new cluster.
35-
func NewClusterResourceSet(ec2API awsapi.EC2, region string, spec *api.ClusterConfig, existingStack *gjson.Result, extendForOutposts bool) *ClusterResourceSet {
36+
func NewClusterResourceSet(ec2API awsapi.EC2, stsAPI awsapi.STS, region string, spec *api.ClusterConfig, existingStack *gjson.Result, extendForOutposts bool) *ClusterResourceSet {
3637
var usesExistingVPC bool
3738
if existingStack != nil {
3839
unsetExistingResources(existingStack, spec)
@@ -59,6 +60,7 @@ func NewClusterResourceSet(ec2API awsapi.EC2, region string, spec *api.ClusterCo
5960
rs: rs,
6061
spec: spec,
6162
ec2API: ec2API,
63+
stsAPI: stsAPI,
6264
region: region,
6365
vpcResourceSet: vpcResourceSet,
6466
}
@@ -409,7 +411,14 @@ func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDe
409411
}
410412
}
411413

412-
c.newResource("ControlPlane", &cluster)
414+
if c.spec.IsCustomEksEndpoint() {
415+
err := addBetaResources(c.stsAPI, c.spec.Metadata.Name, c.rs.template, &cluster)
416+
if err != nil {
417+
return fmt.Errorf("unable to add beta resources: %w", err)
418+
}
419+
} else {
420+
c.newResource("ControlPlane", &cluster)
421+
}
413422

414423
if c.spec.Status == nil {
415424
c.spec.Status = &api.ClusterStatus{}

‎pkg/cfn/builder/cluster_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ var _ = Describe("Cluster Template Builder", func() {
4545
})
4646

4747
JustBeforeEach(func() {
48-
crs = builder.NewClusterResourceSet(provider.EC2(), provider.Region(), cfg, existingStack, false)
48+
crs = builder.NewClusterResourceSet(provider.EC2(), provider.STS(), provider.Region(), cfg, existingStack, false)
4949
})
5050

5151
Describe("AddAllResources", func() {

‎pkg/cfn/builder/iam.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,9 @@ func (c *ClusterResourceSet) addResourcesForServiceRole() {
126126
),
127127
ManagedPolicyArns: gfnt.NewSlice(makePolicyARNs(managedPolicyARNs...)...),
128128
}
129+
if c.spec.IsCustomEksEndpoint() {
130+
role.AssumeRolePolicyDocument = createBetaAssumeRolePolicy()
131+
}
129132
}
130133

131134
if api.IsSetAndNonEmptyString(c.spec.IAM.ServiceRolePermissionsBoundary) {

‎pkg/cfn/builder/managed_nodegroup.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,11 @@ func (m *ManagedNodeGroupResourceSet) AddAllResources(ctx context.Context) error
220220
}
221221

222222
managedResource.LaunchTemplate = launchTemplate
223-
m.newResource(ManagedNodeGroupResourceName, managedResource)
223+
if m.clusterConfig.IsCustomEksEndpoint() {
224+
m.newResource(ManagedNodeGroupResourceName, addBetaManagedNodeGroupResources(managedResource, m.clusterConfig.Metadata.Name))
225+
} else {
226+
m.newResource(ManagedNodeGroupResourceName, managedResource)
227+
}
224228
return nil
225229
}
226230

‎pkg/cfn/builder/nodegroup.go

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -176,11 +176,17 @@ func (n *NodeGroupResourceSet) addAccessEntry() {
176176
return
177177
}
178178

179-
n.newResource("AccessEntry", &gfneks.AccessEntry{
180-
PrincipalArn: gfnt.MakeFnGetAttString(cfnIAMInstanceRoleName, "Arn"),
181-
ClusterName: gfnt.NewString(n.options.ClusterConfig.Metadata.Name),
182-
Type: gfnt.NewString(string(api.GetAccessEntryType(n.options.NodeGroup))),
183-
})
179+
if n.options.ClusterConfig.IsCustomEksEndpoint() {
180+
n.newResource("AccessEntry",
181+
addBetaAccessEntry(n.options.ClusterConfig.Metadata.Name,
182+
string(api.GetAccessEntryType(n.options.NodeGroup))))
183+
} else {
184+
n.newResource("AccessEntry", &gfneks.AccessEntry{
185+
PrincipalArn: gfnt.MakeFnGetAttString(cfnIAMInstanceRoleName, "Arn"),
186+
ClusterName: gfnt.NewString(n.options.ClusterConfig.Metadata.Name),
187+
Type: gfnt.NewString(string(api.GetAccessEntryType(n.options.NodeGroup))),
188+
})
189+
}
184190
}
185191

186192
func (n *NodeGroupResourceSet) addResourcesForSecurityGroups() {

0 commit comments

Comments
 (0)