TECH-3227: Modernizing cloud-init.yml (#22)

Author: fjoeaz

README.md

@@ -50,6 +50,18 @@ Note: The VPC must have `enableDnsHostnames` = `true` and `enableDnsSupport` = `
 
 If you can SSH onto one of the nodes you can run:
 `docker exec rabbitmq rabbitmqctl cluster_status` to see the cluster status of that node.
+
+## Upgrading
+
+Sometimes we will need to do a hot restart of a node in the cluster in order to preform some maintenance, upgrade, or infrastructure improvements. To do this graceful we need to remove the current node from the cluster. This helps keep the node in sync and organized correctly. To do this we need to stop the app and reset the node as follows:
+
+```SH
+docker exec rabbitmq rabbitmqctl cluster_status
+docker exec rabbitmq rabbitmqctl stop_app
+docker exec rabbitmq rabbitmqctl reset
+docker exec rabbitmq rabbitmqctl cluster_status
+```
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 README.md updated successfully
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

cloud-init.yaml

@@ -2,12 +2,19 @@
 write_files:
   - path: /root/conf/enabled_plugins
     content: |
-        [prometheus_rabbitmq_exporter,rabbitmq_management].
-  - path: /root/conf/rabbitmq.config
+      [
+        rabbitmq_management
+      ].
+  - path: /root/conf/rabbitmq.conf
     content: |
-        [ { rabbit, [
-          { loopback_users, [ ] } ] }
-         ].
+      loopback_users = none
+  - path: /root/conf/advanced.config
+    content: |
+      [
+        {rabbit, [
+          ${feature_flags}
+        ]}
+      ].
   - path: /etc/sysconfig/docker
     content: |
       # The max number of open files for the daemon itself, and all
@@ -24,6 +31,13 @@ write_files:
       # when starting the daemon.
       DAEMON_PIDFILE_TIMEOUT=10
 
+  - path: /root/erlang_cookie.sh
+    content: |
+        #!/usr/bin/env bash
+        mkdir /root/data/ #/root/data:/var/lib/rabbitmq
+        aws ssm get-parameter --name ${secret_cookie} --with-decryption --region ${region} | jq -r '.Parameter.Value' > /root/data/.erlang.cookie
+        chmod 400 /root/data/.erlang.cookie #400 is required by rabbitmq
+
   - path: /root/forget_hosts.sh
     content: |
         #!/usr/bin/env bash
@@ -54,12 +68,12 @@ write_files:
 
         for run in {1..3}; do
           sleep $[ ( $RANDOM % 10 )  + 1 ]s
-          echo "stopping rabbit to try and join other nodes"
+          echo "    stopping rabbit to try and join other nodes"
           rabbitmqctl stop_app
 
           NEW_HOSTNAMES=()
           for peerhostname in $HOSTNAMES; do
-            echo "trying to join $${peerhostname}"
+            echo "    trying to join $${peerhostname}"
             rabbitmqctl join_cluster rabbit@$peerhostname
             st=$?
             if [ $st -ne 0 ] && [ $st -ne 130 ]; then  # 130 is "already joined"
@@ -69,7 +83,7 @@ write_files:
 
           HOSTNAMES=( $${NEW_HOSTNAMES[@]} )
           rabbitmqctl start_app
-          echo "starting rabbit after trying to join other nodes"
+          echo "    starting rabbit after trying to join other nodes"
 
           if [ $${#HOSTNAMES[@]} -eq 0 ]; then
             exit 0
@@ -131,8 +145,10 @@ write_files:
         ####### Defaults
         ## Provides auto detected defaults, for vanilla Docker environments,
         ## please see datadog.yaml.example for all supported options
+        ## https://github.com/DataDog/datadog-agent/blob/main/pkg/config/config_template.yaml
 
         api_key: "ENC[DATADOG_API_KEY]"
+        ec2_prefer_imdsv2: true
         secret_backend_command: /etc/datadog-agent/secrets.py
 
         # Auto discovery settings for vanilla Docker
@@ -184,24 +200,29 @@ write_files:
         docker exec rabbitmq rabbitmqctl set_permissions -p / datadog "^aliveness-test$" "^amq\.default$" ".*"
 
 runcmd:
+  - set -x
   - yum update -y
   - yum install -y docker jq
   - pip3 install boto3
-  - DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_API_KEY=$(aws ssm get-parameter --name ${dd_api_key} --with-decryption --region ${region} | jq -r '.Parameter.Value') DD_SITE="${dd_site}" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script.sh)"
+  - (set +x; DD_AGENT_MAJOR_VERSION=7 DD_INSTALL_ONLY=true DD_API_KEY=$(aws ssm get-parameter --name ${dd_api_key} --with-decryption --region ${region} | jq -r '.Parameter.Value') DD_SITE="${dd_site}" bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)")
   - systemctl start docker
   - chkconfig docker on
   - usermod -a -G docker dd-agent
-  - sed -i "s/replace_ip_address_here/$(curl http://169.254.169.254/latest/meta-data/local-ipv4)/g" /root/datadog-agent/conf.d/rabbitmq.d/conf.yaml
+  - |
+    TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"`
+    sed -i "s/replace_ip_address_here/$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)/g" /root/datadog-agent/conf.d/rabbitmq.d/conf.yaml
   - cp /root/datadog-agent/datadog.yaml /etc/datadog-agent/datadog.yaml && chown dd-agent /etc/datadog-agent/datadog.yaml
   - cp /root/datadog-agent/conf.d/rabbitmq.d/conf.yaml /etc/datadog-agent/conf.d/rabbitmq.d/conf.yaml && chown dd-agent /etc/datadog-agent/conf.d/rabbitmq.d/conf.yaml
   - cp /root/datadog-agent/secrets.py /etc/datadog-agent/secrets.py && chown dd-agent /etc/datadog-agent/secrets.py && chmod 0700 /etc/datadog-agent/secrets.py
   - systemctl start datadog-agent
-  - $(aws ecr get-login --no-include-email --region ${region} --registry-ids ${ecr_registry_id})
-  - docker run -d --name rabbitmq --hostname $HOSTNAME --log-driver=local --log-opt max-size=10m -p 4369:4369 -p 5672:5672 -p 15672:15672 -p 25672:25672 -e RABBITMQ_ERLANG_COOKIE=$(aws ssm get-parameter --name ${secret_cookie} --with-decryption --region ${region} | jq -r '.Parameter.Value') -v /root/data:/var/lib/rabbitmq -v /root/conf/:/etc/rabbitmq -v /root/bin:/tmp/bin ${rabbitmq_image}
+  - $(aws ecr get-login-password --region ${region} | docker login --username AWS --password-stdin ${ecr_registry_id}.dkr.ecr.${region}.amazonaws.com)
+  - bash /root/erlang_cookie.sh
+  - docker run -d --name rabbitmq --hostname $HOSTNAME --log-driver=local --log-opt max-size=10m -p 4369:4369 -p 5672:5672 -p 15672:15672 -p 25672:25672 -v /root/data:/var/lib/rabbitmq -v /root/conf/:/etc/rabbitmq -v /root/bin:/tmp/bin ${rabbitmq_image}
   - sleep 1
   - bash /root/find_hosts.sh
   - docker exec rabbitmq bash /tmp/bin/join_cluster.sh $(bash /root/find_hosts.sh)
   - sleep 1
   - bash /root/configure.sh
   - sleep 1
   - bash /root/configure_datadog_user.sh
+  - set +x

data.tf

@@ -0,0 +1,48 @@
+
+data "aws_region" "current" {}
+
+# For use when setting up a new instance or updating to the latest AMI
+data "aws_ami" "amazon_linux_2_latest" {
+  owners      = ["amazon"] # force 1st party AMI's
+  most_recent = true       # latest
+
+  filter {
+    # HVM seems better for many reasons if not only tighter hardware integration
+    # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    # We are switching to ARM64 (Graviton) to leverage the best price performance
+    # https://aws.amazon.com/pm/ec2-graviton/
+    name   = "architecture"
+    values = ["arm64"]
+  }
+
+  filter {
+    # gp3 is preferred but not available at time of writing: 8-14-23
+    # https://aws.amazon.com/ebs/general-purpose/
+    name   = "block-device-mapping.volume-type"
+    values = ["gp2"]
+  }
+
+  filter {
+    # Kernel 5.* selected for is tighter integration with Graviton processors
+    # https://aws.amazon.com/about-aws/whats-new/2021/11/amazon-linux-2-ami-kernel-5-10/
+    name   = "name"
+    values = ["amzn2-ami-kernel-5.*"]
+  }
+}
+
+# Note: bootstraps with latest if ami is not provided
+data "aws_ami" "amazon_linux_2" {
+  owners             = ["amazon"]
+  most_recent        = true
+  include_deprecated = true # statically passed AMI's may be deprecated
+
+  filter {
+    name   = "image-id"
+    values = [var.ami_id != "" ? var.ami_id : data.aws_ami.amazon_linux_2_latest.image_id]
+  }
+}

iam.tf

@@ -0,0 +1,82 @@
+data "aws_iam_policy_document" "policy_doc" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["ec2.amazonaws.com"]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "policy_permissions_doc" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "autoscaling:DescribeAutoScalingInstances",
+      "ec2:DescribeInstances"
+    ]
+    resources = [
+      "*"
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "ecr:GetAuthorizationToken",
+      "ecr:ListImages",
+      "ecr:BatchCheckLayerAvailability",
+      "ecr:BatchGetImage",
+      "ecr:DescribeImages",
+      "ecr:DescribeRepositories",
+      "ecr:GetDownloadUrlForLayer",
+      "ecr:GetRepositoryPolicy"
+    ]
+    resources = [
+      "*"
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "ssm:GetParameter"
+    ]
+    resources = [
+      aws_ssm_parameter.datadog_api_key.arn,
+      aws_ssm_parameter.datadog_user_password.arn,
+      aws_ssm_parameter.rabbit_admin_password.arn,
+      aws_ssm_parameter.rabbit_password.arn,
+      aws_ssm_parameter.secret_cookie.arn,
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "kms:Decrypt"
+    ]
+    resources = [
+      var.kms_key_arn
+    ]
+  }
+}
+
+resource "aws_iam_role_policy" "iam_policy" {
+  name = "${var.name}-${data.aws_region.current.name}"
+  role = aws_iam_role.iam_role.id
+
+  policy = data.aws_iam_policy_document.policy_permissions_doc.json
+}
+
+resource "aws_iam_instance_profile" "iam_profile" {
+  name_prefix = "${var.name}-${data.aws_region.current.name}-"
+  role        = aws_iam_role.iam_role.name
+}
+
+resource "aws_iam_role" "iam_role" {
+  name               = "${var.name}-${data.aws_region.current.name}"
+  assume_role_policy = data.aws_iam_policy_document.policy_doc.json
+  tags               = var.tags
+}