- NodeJS/npm is now managed via NVM (#64: Contributor @cfsnate)

- Fixed IAM policies required to install SOCA and added support for cdk boostrap (#64: Contributor @cfsnate)
- More consistent way to install EPEL repository across distros
- Better way to install SSM on the Scheduler host (similar to what we are already doing with ComputeNodes)
- Updated remote job submission to fix error with group ownership when using a remote input file

Author: mcrozes

.gitignore

@@ -10,6 +10,7 @@ installer/resources/src/installer_history.txt
 installer/resources/src/cdk.context.json
 installer/resources/src/cdk.out/
 installer/resources/src/envs/
+installer/resources/src/.requirements_installed
 
 # C extensions
 *.so

README.md

@@ -13,12 +13,12 @@ Scale-Out Computing on AWS project consists in a collection of CloudFormation te
 ```bash
 .
 ├── soca
-│   ├── cluster_analytics                               [ Scripts to ingest cluster/job data into ELK ]
-│   ├── cluster_hooks                                   [ Scheduler Hooks ]
-│   ├── cluster_logs_management                         [ Scripts to manage cluster log rotation ]
-│   ├── cluster_manager                                 [ Scripts to control Soca cluster ]
-│   ├── cluster_web_ui                                  [ Web Interface ]
-│   └── cluster_node_bootstrap                          [ Script to configure compute nodes]
+│   ├── cluster_analytics                               [ Scripts to ingest cluster/job data into ELK ]
+│   ├── cluster_hooks                                   [ Scheduler Hooks ]
+│   ├── cluster_logs_management                         [ Scripts to manage cluster log rotation ]
+│   ├── cluster_manager                                 [ Scripts to control Soca cluster ]
+│   ├── cluster_web_ui                                  [ Web Interface ]
+│   └── cluster_node_bootstrap                          [ Script to configure compute nodes]
 └── scripts
    ├── config.cfg                                      [ List of all packages to install ]
    ├── Scheduler.sh                                    [ Configure Schedule Node ]
@@ -30,7 +30,7 @@ This solution collects anonymous operational metrics to help AWS improve the qua
 
 ***
 
-Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

installer/Makefile

@@ -0,0 +1,10 @@
+
+ifeq (${QUIET_MODE},true)
+    pip_flags := --quiet
+endif
+
+resources/src/.requirements_installed: resources/src/requirements.txt
+	@echo "Installing/upgrading required dependencies (this can take a couple of minutes)"
+	pip3 install --upgrade pip ${pip_flags}
+	pip3 install -r resources/src/requirements.txt ${pip_flags}
+	touch $@

installer/SOCAInstallerIamPolicy.json

@@ -42,34 +42,33 @@
                 "ec2:CreateNatGateway",
                 "ec2:CreateNetworkInterface",
                 "ec2:CreateNetworkInterfacePermission",
-                "ec2:CreateNetworkInterfacePermission",
                 "ec2:CreateRoute",
                 "ec2:CreateRouteTable",
-                "ec2:CreateRouteTable",
                 "ec2:CreateSecurityGroup",
                 "ec2:CreateSubnet",
                 "ec2:CreateTags",
                 "ec2:CreateVpc",
                 "ec2:CreateVpcEndpoint",
+                "ec2:DescribeAddresses",
                 "ec2:DescribeAvailabilityZones",
                 "ec2:DescribeFlowLogs",
                 "ec2:DescribeInstances",
                 "ec2:DescribeInternetGateways",
                 "ec2:DescribeKeyPairs",
                 "ec2:DescribeNatGateways",
                 "ec2:DescribeNetworkInterfaces",
+                "ec2:DescribeNetwork*",
                 "ec2:DescribeRegions",
                 "ec2:DescribeRouteTables",
-                "ec2:DescribeRouteTables",
                 "ec2:DescribeSecurityGroups",
                 "ec2:DescribeSubnets",
+                "ec2:DescribeVpcAttribute",
                 "ec2:DescribeVpcEndpoints",
                 "ec2:DescribeVpcs",
                 "ec2:ModifySubnetAttribute",
                 "ec2:ModifyVpcAttribute",
                 "ec2:RevokeSecurityGroupEgress",
                 "ec2:RunInstances",
-                "ec2:describeAddresses",
                 "elasticfilesystem:CreateFileSystem",
                 "elasticfilesystem:CreateMountTarget",
                 "elasticfilesystem:DescribeFileSystems",
@@ -90,7 +89,6 @@
                 "elasticloadbalancing:ModifyRule",
                 "elasticloadbalancing:RegisterTargets",
                 "es:AddTags",
-                "es:AddTags",
                 "es:CreateElasticsearchDomain",
                 "es:DescribeElasticsearchDomain",
                 "fsx:CreateFileSystem",
@@ -123,7 +121,6 @@
                 "route53resolver:CreateResolverEndpoint",
                 "route53resolver:CreateResolverRule",
                 "route53resolver:GetResolverEndpoint",
-                "route53resolver:GetResolverEndpoint",
                 "route53resolver:GetResolverRule",
                 "route53resolver:GetResolverRuleAssociation",
                 "route53resolver:PutResolverRulePolicy",
@@ -151,12 +148,10 @@
                 "ds:DeleteDirectory",
                 "ec2:DeleteFlowLogs",
                 "ec2:DeleteInternetGateway",
-                "ec2:DeleteInternetGateway",
                 "ec2:DeleteNatGateway",
                 "ec2:DeleteNetworkInterface",
                 "ec2:DeleteRoute",
                 "ec2:DeleteRouteTable",
-                "ec2:DeleteRouteTable",
                 "ec2:DeleteSecurityGroup",
                 "ec2:DeleteSubnet",
                 "ec2:DeleteVpc",
@@ -168,6 +163,7 @@
                 "ec2:ReleaseAddress",
                 "ec2:RevokeSecurityGroupIngress",
                 "ec2:TerminateInstances",
+                "ecr:DeleteRepository",
                 "elasticfilesystem:DeleteFileSystem",
                 "elasticfilesystem:DeleteMountTarget",
                 "elasticloadbalancing:DeRegisterTargets",
@@ -179,18 +175,50 @@
                 "iam:DeleteInstanceProfile",
                 "iam:DeleteRole",
                 "iam:DeleteRolePolicy",
+                "iam:DeleteServiceLinkedRole",
                 "iam:DetachRolePolicy",
                 "iam:RemoveRoleFromInstanceProfile",
                 "lambda:DeleteFunction",
                 "lambda:RemovePermission",
                 "logs:DeleteLogGroup",
                 "route53resolver:DeleteResolverEndpoint",
-                "route53resolver:DeleteResolverEndpoint",
                 "route53resolver:DeleteResolverRule",
                 "route53resolver:DisassociateResolverRule",
-                "secretsmanager:DeleteSecret"
+                "secretsmanager:DeleteSecret",
+                "s3:DeleteBucketPolicy"
+            ],
+            "Resource": "*"
+        },
+          {
+            "Sid": "CreateCDKToolkitIfNeeded",
+            "Effect": "Allow",
+            "Action": [
+                "s3:PutEncryptionConfiguration",
+                "s3:PutBucketVersioning",
+                "s3:PutBucketPublicAccessBlock",
+                "s3:PutBucketPolicy",
+                "s3:CreateBucket"
+            ],
+            "Resource": "arn:aws:s3:::cdk*"
+        },
+        {
+            "Sid": "CreateCDKToolkitIfNeeded2",
+            "Effect": "Allow",
+            "Action": [
+                "ecr:CreateRepository",
+                "ecr:DescribeRepositories",
+                "ec2:DescribeAccountAttributes",
+                "es:CreateServiceLinkedRole",
+                "ssm:GetParameter",
+                "ssm:GetParameters",
+                "ssm:PutParameter",
+                "ssm:DeleteParameter",
+                "iam:CreateRole",
+                "iam:CreateServiceLinkedRole",
+                "iam:GetRole",
+                "iam:PutRolePolicy"
             ],
             "Resource": "*"
         }
     ]
-}
+}

installer/default_config.yml

@@ -1,5 +1,5 @@
 Config:
-  version: "v2.7.0" # version automatically populated as part of RELEASE-PIPELINE.sh (see github_release.py)
+  version: "2.7.0" # version automatically populated as part of RELEASE-PIPELINE.sh (see github_release.py)
   termination_protection: True # Enable (recommended) or Disable Cloudformation Stack termination protection
   entry_points_subnets: "Public" # Public (recommended) or Private. In public mode the scheduler and ELB are deployed on PublicSubnets and assigned Public IPS. In Private mode scheduler and ELB are deployed in private subnets. In both case compute nodes and ElasticSearch/EFS/FSxL are deployed in private subnets. Public does not means your cluster will be accessible to everyone by default, access to your cluster is still protected by security groups
 

installer/resources/src/cdk_construct.py

@@ -390,6 +390,7 @@ def iam_roles(self):
             self.soca_resources["scheduler_role"] = iam.Role(self, "SchedulerRole", description="IAM role assigned to the scheduler host", assumed_by=iam.CompositePrincipal(iam.ServicePrincipal(principals_suffix["ssm"]), iam.ServicePrincipal(principals_suffix["ec2"])))
             self.soca_resources["compute_node_role"] = iam.Role(self, "ComputeNodeRole", description="IAM role assigned to the compute nodes", assumed_by=iam.CompositePrincipal(iam.ServicePrincipal(principals_suffix["ssm"]), iam.ServicePrincipal(principals_suffix["ec2"])))
             self.soca_resources["spot_fleet_role"] = iam.Role(self, "SpotFleetRole", description="IAM role to manage SpotFleet requests", assumed_by=iam.ServicePrincipal(principals_suffix["spotfleet"]))
+            self.soca_resources["spot_fleet_role"].add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("service-role/AmazonEC2SpotFleetTaggingRole"))
             self.soca_resources["compute_node_instance_profile"] = iam.CfnInstanceProfile(self, "ComputeNodeInstanceProfile", roles=[self.soca_resources["compute_node_role"].role_name])
         else:
             # Reference existing Scheduler/ComputeNode/SpotFleet roles
@@ -1077,4 +1078,4 @@ def viewer(self):
     install = SOCAInstall(app, user_specified_variables.cluster_id, env=cdk_env,
                           description=f"SOCA cluster version {install_props.Config.version}",
                           termination_protection=install_props.Config.termination_protection)
-    app.synth()
+    app.synth()

installer/resources/src/install_soca.py

@@ -495,7 +495,7 @@ def validate_soca_config(user_specified_inputs, install_properties):
     os.chdir(install_directory)
 
     # Append Solution ID to Boto3 Construct
-    aws_solution_user_agent = {"user_agent_extra": "AwsSolution/SO0072/v2.7.0"}
+    aws_solution_user_agent = {"user_agent_extra": "AwsSolution/SO0072/2.7.0"}
     boto_extra_config = config.Config(**aws_solution_user_agent)
 
     print(f"""

installer/soca_installer.sh

@@ -77,25 +77,19 @@ if [[ -n $VIRTUAL_ENV ]]; then
 else
   # Check if Python Virtual environment exist
   # If not, create the venv and install required python libraries
-  if [[ ! -d $PYTHON_VENV ]]; then
+  if [[ ! -e $PYTHON_VENV/bin/activate ]]; then
       echo "No Python virtual environment found. Creating one ..."
+      rm -rf $PYTHON_VENV
       $PYTHON -m venv $PYTHON_VENV
       # shellcheck disable=SC1090
       . "$PYTHON_VENV/bin/activate"
-      echo "Installing/upgrading required dependencies (this can take a couple of minutes)"
-      if [[ $QUIET_MODE = "true" ]]; then
-        pip3 install --upgrade pip --quiet
-        pip3 install -r resources/src/requirements.txt --quiet
-      else
-        pip3 install --upgrade pip
-        pip3 install -r resources/src/requirements.txt
-      fi
   else
     # Load Python environment
     echo "Loading Python Virtual Environment"
     source "$PYTHON_VENV/bin/activate"
   fi
 fi
+make resources/src/.requirements_installed
 
 # Install local NodeJS environment and CDK
 if [[ ! -d $NVM_DIR ]]; then

source/scripts/Scheduler.sh

@@ -23,9 +23,16 @@ if [[ $# -lt 2 ]]; then
 fi
 
 # Install SSM
-yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm
-systemctl enable amazon-ssm-agent
-systemctl restart amazon-ssm-agent
+machine=$(uname -m)
+if ! systemctl status amazon-ssm-agent; then
+    if [[ $machine == "x86_64" ]]; then
+        yum install -y $SSM_X86_64_URL
+    elif [[ $machine == "aarch64" ]]; then
+        yum install -y $SSM_AARCH64_URL
+    fi
+    systemctl enable amazon-ssm-agent || true
+    systemctl restart amazon-ssm-agent
+fi
 
 mkdir -p /apps/soca/$SOCA_CONFIGURATION
 FS_DATA_PROVIDER=$1
@@ -37,22 +44,26 @@ SERVER_HOSTNAME=$(hostname)
 SERVER_HOSTNAME_ALT=$(echo $SERVER_HOSTNAME | cut -d. -f1)
 echo $SERVER_IP $SERVER_HOSTNAME $SERVER_HOSTNAME_ALT >> /etc/hosts
 
-# Install Epel repo
-if [[ $SOCA_BASE_OS == "amazonlinux2" ]]; then
-  sudo amazon-linux-extras install -y epel
+# Install System required libraries / EPEL
+if [[ $SOCA_BASE_OS == "rhel7" ]]; then
+  # RHEL7
+  curl "$EPEL_URL" -o $EPEL_RPM
+  if [[ $(md5sum "$EPEL_RPM" | awk '{print $1}') != "$EPEL_HASH" ]];  then
+      echo -e "FATAL ERROR: Checksum for EPEL failed. File may be compromised." > /etc/motd
+      exit 1
+  fi
+  yum -y install $EPEL_RPM
+  yum install -y $(echo ${SYSTEM_PKGS[*]} ${SCHEDULER_PKGS[*]}) --enablerepo rhui-REGION-rhel-server-optional
 elif [[ $SOCA_BASE_OS == "centos7" ]]; then
+  # CentOS
   yum -y install epel-release
+  yum install -y $(echo ${SYSTEM_PKGS[*]} ${SCHEDULER_PKGS[*]})
 else
-  # rhel7
-  yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-fi
-
-if [[ $SOCA_BASE_OS == "rhel7" ]]; then
-    yum install -y $(echo ${SYSTEM_PKGS[*]} ${SCHEDULER_PKGS[*]}) --enablerepo rhui-REGION-rhel-server-optional
-else
-    yum install -y $(echo ${SYSTEM_PKGS[*]} ${SCHEDULER_PKGS[*]})
+  # AL2
+  amazon-linux-extras install -y epel
+  yum update --security
+  yum install -y $(echo ${SYSTEM_PKGS[*]} ${SCHEDULER_PKGS[*]})
 fi
-
 yum install -y $(echo ${OPENLDAP_SERVER_PKGS[*]} ${SSSD_PKGS[*]})
 
 # Mount File system
@@ -63,7 +74,7 @@ if [[ "$FS_DATA_PROVIDER" == "fsx_lustre" ]] || [[ "$FS_APPS_PROVIDER" == "fsx_l
     if [[ -z "$(rpm -qa lustre-client)" ]]; then
         # Install FSx for Lustre Client
         if [[ "$SOCA_BASE_OS" == "amazonlinux2" ]]; then
-            sudo amazon-linux-extras install -y lustre2.10
+            amazon-linux-extras install -y lustre2.10
         else
             kernel=$(uname -r)
             machine=$(uname -m)

source/scripts/SchedulerPostReboot.sh

@@ -302,11 +302,30 @@ fi
 IFS="-" read name sanitized_cluster_name <<< "echo $SOCA_CONFIGURATION"
 sed -i "s/__SOCA_CLUSTER__NAME__/$sanitized_cluster_name/g" /apps/soca/$SOCA_CONFIGURATION/cluster_web_ui/templates/common/horizontal_menu_bar.html
 
-# Start Web UI
+# Install NodeJS/NPM if needed
+if [[ ! $(command -v npm) ]];
+then
+  echo "npm not detected, installing it ... "
+  export NVM_DIR="/root/nvm/$(date +%s)/.nvm"
+  mkdir -p $NVM_DIR
+  echo "Downloading $NVM_URL"
+  wget "$NVM_URL"
+  if [[ $(md5sum $NVM_INSTALL_SCRIPT | awk '{print $1}') != $NVM_HASH ]];  then
+        echo -e "FATAL ERROR: Checksum for NVM failed. File may be compromised." > /etc/motd
+        exit 1
+  fi
+  chmod +x $NVM_INSTALL_SCRIPT
+  /bin/bash $NVM_INSTALL_SCRIPT
+  source "$NVM_DIR/nvm.sh"  # This loads nvm
+  # shellcheck disable=SC1090
+  source "$NVM_DIR/bash_completion"
+  nvm install node
+fi
 
-# Install required node module
+# Install required Node module
 npm install --prefix /apps/soca/"$SOCA_CONFIGURATION"/cluster_web_ui/static monaco-editor@0.24.0
 
+# Start Web UI
 chmod +x /apps/soca/"$SOCA_CONFIGURATION"/cluster_web_ui/socawebui.sh
 /apps/soca/"$SOCA_CONFIGURATION"/cluster_web_ui/socawebui.sh start