6100: Add default security group for ecs tasks and services

docs/usage/properties/instance/cdk/common.md

@@ -9,6 +9,7 @@ The following instance properties are commonly used throughout Sleeper.
 | sleeper.config.bucket                                            | The S3 bucket name used to store configuration files.                                                                                                                      |
 | sleeper.account                                                  | The AWS account number. This is the AWS account that the instance is deployed in.                                                                                          |
 | sleeper.region                                                   | The AWS region the instance is deployed in.                                                                                                                                |
+| sleeper.ecs.security.group.id                                    | The security group ID to be used for ECS tasks and services.                                                                                                               |
 | sleeper.data.bucket                                              | The S3 bucket name used to store table data.                                                                                                                               |
 | sleeper.tables.name.index.dynamo.table                           | The name of the DynamoDB table indexing tables by their name.                                                                                                              |
 | sleeper.tables.id.index.dynamo.table                             | The name of the DynamoDB table indexing tables by their ID.                                                                                                                |

docs/usage/properties/instance/user/common.md

@@ -30,6 +30,5 @@ The following instance properties are commonly used throughout Sleeper.
 | sleeper.task.runner.memory.mb               | The amount of memory in MB for the lambda that creates ECS tasks to execute compaction and ingest jobs.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | 1024                                                                                                                                                                                 | true                        |
 | sleeper.task.runner.timeout.seconds         | The timeout in seconds for the lambda that creates ECS tasks to execute compaction jobs and ingest jobs.<br>This must be >0 and <= 900.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | 900                                                                                                                                                                                  | true                        |
 | sleeper.properties.force.reload             | If true, properties will be reloaded every time a long running job is started or a lambda is run. This will mainly be used in test scenarios to ensure properties are up to date.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           | false                                                                                                                                                                                | false                       |
-| sleeper.ecs.security.groups                 | A comma-separated list of up to 5 security group IDs to be used when running ECS tasks.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |                                                                                                                                                                                      | true                        |
 | sleeper.default.lambda.concurrency.reserved | Default value for the reserved concurrency for each lambda in the Sleeper instance that scales according to the number of Sleeper tables.<br>The state store committer lambda is an exception to this, as it has reserved concurrency by default. This is set in the property sleeper.statestore.committer.concurrency.reserved. Other lambdas are present that do not scale by the number of Sleeper tables, and are not set from this property.<br>By default no concurrency is reserved for the lambdas. Each lambda also has its own property that overrides the value found here.<br>See reserved concurrency overview at: https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html |                                                                                                                                                                                      | false                       |
 | sleeper.default.lambda.concurrency.max      | Default value for the maximum concurrency for each lambda in the Sleeper instance that scales according to the number of Sleeper tables.<br>Other lambdas are present that do not scale by the number of Sleeper tables, and are not set from this property.<br>By default the maximum concurrency is set to 10, which is enough for 10 online tables. If there are more online tables, this number may need to be increased. Each lambda also has its own property that overrides the value found here.<br>See maximum concurrency overview at: https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/                     | 10                                                                                                                                                                                   | false                       |

example/full/instance.properties

@@ -194,10 +194,6 @@ sleeper.logging.root.level=INFO
 # (default value shown below, uncomment to set a value)
 # sleeper.properties.force.reload=false
 
-# A comma-separated list of up to 5 security group IDs to be used when running ECS tasks.
-# (uncomment to set a value)
-# sleeper.ecs.security.groups=
-
 # Default value for the reserved concurrency for each lambda in the Sleeper instance that scales
 # according to the number of Sleeper tables.
 # The state store committer lambda is an exception to this, as it has reserved concurrency by default.

java/common/common-task/src/main/java/sleeper/common/task/RunDataProcessingTasks.java

@@ -45,8 +45,8 @@
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_TASK_EC2_DEFINITION_FAMILY;
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_TASK_FARGATE_DEFINITION_FAMILY;
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET;
+import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ECS_SECURITY_GROUP;
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.LEAF_PARTITION_BULK_EXPORT_QUEUE_URL;
-import static sleeper.core.properties.instance.CommonProperty.ECS_SECURITY_GROUPS;
 import static sleeper.core.properties.instance.CommonProperty.FARGATE_VERSION;
 import static sleeper.core.properties.instance.CommonProperty.SUBNETS;
 import static sleeper.core.properties.instance.CompactionProperty.COMPACTION_ECS_LAUNCHTYPE;
@@ -337,7 +337,7 @@ private static TaskOverride createOverride(InstanceProperties instanceProperties
     private static NetworkConfiguration networkConfig(InstanceProperties instanceProperties) {
         AwsVpcConfiguration vpcConfiguration = AwsVpcConfiguration.builder()
                 .subnets(instanceProperties.getList(SUBNETS))
-                .securityGroups(instanceProperties.getList(ECS_SECURITY_GROUPS))
+                .securityGroups(instanceProperties.get(ECS_SECURITY_GROUP))
                 .build();
         return NetworkConfiguration.builder()
                 .awsvpcConfiguration(vpcConfiguration)

java/common/common-task/src/main/java/sleeper/common/task/RunIngestTasks.java

@@ -30,10 +30,10 @@
 import java.util.List;
 
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET;
+import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ECS_SECURITY_GROUP;
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_CLUSTER;
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_JOB_QUEUE_URL;
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_TASK_DEFINITION_FAMILY;
-import static sleeper.core.properties.instance.CommonProperty.ECS_SECURITY_GROUPS;
 import static sleeper.core.properties.instance.CommonProperty.FARGATE_VERSION;
 import static sleeper.core.properties.instance.CommonProperty.SUBNETS;
 import static sleeper.core.properties.instance.IngestProperty.MAXIMUM_CONCURRENT_INGEST_TASKS;
@@ -103,7 +103,7 @@ public void run() {
                 .networkConfiguration(networkConfiguration -> networkConfiguration
                         .awsvpcConfiguration(vpcConfiguration -> vpcConfiguration
                                 .subnets(properties.getList(SUBNETS))
-                                .securityGroups(properties.getList(ECS_SECURITY_GROUPS))))
+                                .securityGroups(properties.get(ECS_SECURITY_GROUP))))
                 .overrides(overrides -> overrides
                         .containerOverrides(ContainerOverride.builder()
                                 .name(containerName)

java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java

@@ -73,6 +73,12 @@ static List<CdkDefinedInstanceProperty> getAllInGroup(PropertyGroup group) {
             .propertyGroup(InstancePropertyGroup.COMMON)
             .build();
 
+    // Security Groups
+    CdkDefinedInstanceProperty ECS_SECURITY_GROUP = Index.propertyBuilder("sleeper.ecs.security.group.id")
+            .description("The security group ID to be used for ECS tasks and services.")
+            .propertyGroup(InstancePropertyGroup.COMMON)
+            .build();
+
     // Data
     CdkDefinedInstanceProperty DATA_BUCKET = Index.propertyBuilder("sleeper.data.bucket")
             .description("The S3 bucket name used to store table data.")

java/core/src/main/java/sleeper/core/properties/instance/CommonProperty.java

@@ -219,11 +219,6 @@ public interface CommonProperty {
             .validationPredicate(SleeperPropertyValueUtils::isTrueOrFalse)
             .propertyGroup(InstancePropertyGroup.COMMON)
             .build();
-    UserDefinedInstanceProperty ECS_SECURITY_GROUPS = Index.propertyBuilder("sleeper.ecs.security.groups")
-            .description("A comma-separated list of up to 5 security group IDs to be used when running ECS tasks.")
-            .validationPredicate(value -> SleeperPropertyValueUtils.isListWithMaxSize(value, 5))
-            .runCdkDeployWhenChanged(true)
-            .propertyGroup(InstancePropertyGroup.COMMON).build();
     UserDefinedInstanceProperty DEFAULT_LAMBDA_CONCURRENCY_RESERVED = Index.propertyBuilder("sleeper.default.lambda.concurrency.reserved")
             .description("Default value for the reserved concurrency for each lambda in the Sleeper instance " +
                     "that scales according to the number of Sleeper tables.\n" +

java/deployment/cdk/src/main/java/sleeper/cdk/stack/SleeperCoreStacks.java

@@ -40,8 +40,8 @@
 import sleeper.cdk.networking.SleeperNetworking;
 import sleeper.cdk.stack.compaction.CompactionTrackerResources;
 import sleeper.cdk.stack.core.AutoDeleteS3ObjectsStack;
-import sleeper.cdk.stack.core.AutoStopEcsClusterTasksStack;
 import sleeper.cdk.stack.core.ConfigBucketStack;
+import sleeper.cdk.stack.core.EcsClusterTasksStack;
 import sleeper.cdk.stack.core.LoggingStack;
 import sleeper.cdk.stack.core.LoggingStack.LogGroupRef;
 import sleeper.cdk.stack.core.ManagedPoliciesStack;
@@ -81,7 +81,7 @@ public class SleeperCoreStacks {
     private final IngestTrackerResources ingestTracker;
     private final CompactionTrackerResources compactionTracker;
     private final AutoDeleteS3ObjectsStack autoDeleteS3Stack;
-    private final AutoStopEcsClusterTasksStack autoStopEcsStack;
+    private final EcsClusterTasksStack ecsClusterTasksStack;
     private SleeperInstanceRoles roles;
 
     @SuppressWarnings("checkstyle:ParameterNumberCheck")
@@ -93,7 +93,7 @@ private SleeperCoreStacks(
             IngestTrackerResources ingestTracker,
             CompactionTrackerResources compactionTracker,
             AutoDeleteS3ObjectsStack autoDeleteS3Stack,
-            AutoStopEcsClusterTasksStack autoStopEcsStack) {
+            EcsClusterTasksStack ecsClusterTasksStack) {
         this.networking = networking;
         this.loggingStack = loggingStack;
         this.deadLetters = deadLetters;
@@ -106,7 +106,7 @@ private SleeperCoreStacks(
         this.ingestTracker = ingestTracker;
         this.compactionTracker = compactionTracker;
         this.autoDeleteS3Stack = autoDeleteS3Stack;
-        this.autoStopEcsStack = autoStopEcsStack;
+        this.ecsClusterTasksStack = ecsClusterTasksStack;
     }
 
     public static SleeperCoreStacks create(Stack stack, SleeperInstanceProps props) {
@@ -132,7 +132,7 @@ public static SleeperCoreStacks create(
                 new TopicStack(scope, "Topic", instanceProperties));
 
         // Custom resource providers
-        AutoStopEcsClusterTasksStack autoStopEcsStack = new AutoStopEcsClusterTasksStack(scope, "AutoStopEcsClusterTasks", instanceProperties, jars, loggingStack);
+        EcsClusterTasksStack ecsClusterTasksStack = new EcsClusterTasksStack(scope, "AutoStopEcsClusterTasks", instanceProperties, jars, loggingStack);
         ManagedPoliciesStack policiesStack = new ManagedPoliciesStack(scope, "Policies", instanceProperties);
 
         // Stacks for tables
@@ -150,11 +150,11 @@ public static SleeperCoreStacks create(
                 instanceProperties, jars,
                 loggingStack, configBucketStack, tableIndexStack,
                 stateStoreStacks, ingestTracker, compactionTracker,
-                policiesStack, deadLetters);
+                policiesStack, ecsClusterTasksStack, deadLetters);
 
         SleeperCoreStacks stacks = new SleeperCoreStacks(networking, loggingStack, deadLetters,
                 configBucketStack, tableIndexStack, policiesStack, stateStoreStacks, dataStack,
-                stateStoreCommitterStack, ingestTracker, compactionTracker, autoDeleteS3Stack, autoStopEcsStack);
+                stateStoreCommitterStack, ingestTracker, compactionTracker, autoDeleteS3Stack, ecsClusterTasksStack);
 
         // Table state store maintenance
         new TransactionLogSnapshotStack(scope, "TransactionLogSnapshot",
@@ -236,11 +236,11 @@ public void addAutoDeleteS3Objects(Construct scope, IBucket bucket) {
     }
 
     public void addAutoStopEcsClusterTasksAfterTaskCreatorIsDeleted(Construct scope, ICluster cluster, IFunction taskCreator) {
-        autoStopEcsStack.addAutoStopEcsClusterTasksAfterTaskCreatorIsDeleted(scope, cluster, taskCreator);
+        ecsClusterTasksStack.addAutoStopEcsClusterTasksAfterTaskCreatorIsDeleted(scope, cluster, taskCreator);
     }
 
-    public AutoStopEcsClusterTasksStack getAutoStopEcsStack() {
-        return autoStopEcsStack;
+    public EcsClusterTasksStack getEcsClusterTasksStack() {
+        return ecsClusterTasksStack;
     }
 
     // The Lambda IFunction.getRole method is annotated as nullable, even though it will never return null in practice.

java/deployment/cdk/src/main/java/sleeper/cdk/stack/compaction/CompactionOnEc2Resources.java

@@ -25,7 +25,6 @@
 import software.amazon.awscdk.services.ec2.EbsDeviceOptions;
 import software.amazon.awscdk.services.ec2.EbsDeviceVolumeType;
 import software.amazon.awscdk.services.ec2.IMachineImage;
-import software.amazon.awscdk.services.ec2.ISecurityGroup;
 import software.amazon.awscdk.services.ec2.IVpc;
 import software.amazon.awscdk.services.ec2.InstanceArchitecture;
 import software.amazon.awscdk.services.ec2.InstanceClass;
@@ -51,7 +50,6 @@
 import software.amazon.awscdk.services.iam.PolicyStatement;
 import software.amazon.awscdk.services.lambda.CfnPermission;
 import software.amazon.awscdk.services.lambda.IFunction;
-import software.constructs.Construct;
 import software.constructs.IDependable;
 
 import sleeper.cdk.jars.SleeperLambdaCode;
@@ -70,14 +68,10 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT;
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_AUTO_SCALING_GROUP;
 import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_TASK_EC2_DEFINITION_FAMILY;
-import static sleeper.core.properties.instance.CommonProperty.ECS_SECURITY_GROUPS;
 import static sleeper.core.properties.instance.CompactionProperty.COMPACTION_EC2_POOL_MAXIMUM;
 import static sleeper.core.properties.instance.CompactionProperty.COMPACTION_EC2_POOL_MINIMUM;
 import static sleeper.core.properties.instance.CompactionProperty.COMPACTION_EC2_ROOT_SIZE;
@@ -166,8 +160,7 @@ private void addEC2CapacityProvider(Cluster cluster, IVpc vpc, SleeperLambdaCode
                         .build())
                 .instanceProfile(InstanceProfile.Builder.create(stack, "CompactionScalingInstanceProfile").build())
                 .build();
-        addSecurityGroupReferences(stack, instanceProperties)
-                .forEach(scalingLaunchTemplate::addSecurityGroup);
+
         AutoScalingGroup ec2scalingGroup = AutoScalingGroup.Builder.create(stack, "CompactionScalingGroup")
                 .vpc(vpc)
                 .launchTemplate(scalingLaunchTemplate)
@@ -266,13 +259,4 @@ private static AmiHardwareType lookupAmiHardwareType(InstanceArchitecture archit
                 throw new IllegalArgumentException("Unrecognised architecture: " + architecture);
         }
     }
-
-    private static List<ISecurityGroup> addSecurityGroupReferences(Construct scope, InstanceProperties instanceProperties) {
-        AtomicInteger index = new AtomicInteger(1);
-        return instanceProperties.getList(ECS_SECURITY_GROUPS).stream()
-                .filter(Predicate.not(String::isBlank))
-                .map(groupId -> SecurityGroup.fromLookupById(scope, "CompactionScalingSG" + index.getAndIncrement(), groupId))
-                .collect(Collectors.toList());
-    }
-
 }