Feat/add oss multi arch comp metal amd arm intel redis latest (#140)

* Add deployment for comparing Redis versions (7.2, 8.0) on multiple archs (amd, intel, arm)

* Add deployment for comparing Redis on multiple archs (amd, intel, arm) using metal instances

Author: paulorsousa

terraform/oss-multi-arch-comp-amd-arm-intel-4xlarge-redis-7.2-redis-8.0.0/client-cloud-init-memtier.yaml

@@ -0,0 +1,45 @@
+#cloud-config
+
+package_update: true
+package_upgrade: true
+packages:
+  - git
+  - build-essential
+  - autoconf
+  - automake
+  - libpcre3-dev
+  - libevent-dev
+  - pkg-config
+  - zlib1g-dev
+  - libssl-dev
+  - libtool
+  - ca-certificates
+  - wget
+
+# Create the memtier installation script
+write_files:
+  - path: /tmp/install_memtier.sh
+    permissions: "0755"
+    content: |
+      #!/bin/bash
+
+      set -e # exit immediately on error
+
+      # Clone memtier benchmark
+      cd /tmp
+      rm -rf memtier_benchmark
+      git clone https://github.com/RedisLabs/memtier_benchmark.git
+      cd memtier_benchmark
+
+      # Build and install
+      autoreconf -ivf
+      ./configure
+      make -j
+      sudo make install
+
+      echo "Memtier benchmark installed successfully"
+
+# Run the installation script
+runcmd:
+  - [bash, /tmp/install_memtier.sh]
+  - [echo, "Cloud-init installation completed"]

terraform/oss-multi-arch-comp-amd-arm-intel-4xlarge-redis-7.2-redis-8.0.0/client-resources.tf

@@ -0,0 +1,32 @@
+resource "aws_instance" "client" {
+  ami                    = var.client_instance_ami
+  instance_type          = var.client_instance_type
+  subnet_id              = data.terraform_remote_state.shared_resources.outputs.subnet_public_us_east_2b_id
+  vpc_security_group_ids = ["${data.terraform_remote_state.shared_resources.outputs.performance_cto_sg_id}"]
+  key_name               = var.key_name
+  placement_group        = data.terraform_remote_state.shared_resources.outputs.placement_group_name_us_east_2b
+  availability_zone      = "us-east-2b"
+
+  user_data = file("./client-cloud-init-memtier.yaml") # Use cloud-init to install memtier benchmark
+
+  root_block_device {
+    volume_size           = var.instance_volume_size
+    volume_type           = var.instance_volume_type
+    encrypted             = var.instance_volume_encrypted
+    delete_on_termination = true
+    tags = merge(
+      local.base_tags,
+      {
+        Name = "ebs_block_device-${var.setup_name}-CLIENT"
+      }
+    )
+  }
+
+  tags = merge(
+    local.base_tags,
+    {
+      Name         = "${var.setup_name}-CLIENT"
+      InstanceType = var.client_instance_type
+    }
+  )
+}

terraform/oss-multi-arch-comp-amd-arm-intel-4xlarge-redis-7.2-redis-8.0.0/common.tf

@@ -0,0 +1,11 @@
+
+################################################################################
+# This is the bucket holding this specific setup tfstate
+################################################################################
+terraform {
+  backend "s3" {
+    bucket = "performance-cto-group"
+    key    = "benchmarks/infrastructure/oss-multi-arch-comp-amd-arm-intel-4xlarge-redis-7.2-redis-8.0.0.tfstate"
+    region = "us-east-1"
+  }
+}

terraform/oss-multi-arch-comp-amd-arm-intel-4xlarge-redis-7.2-redis-8.0.0/db-cloud-init-dbs.yaml

@@ -0,0 +1,51 @@
+#cloud-config
+
+package_update: true
+package_upgrade: true
+packages:
+  - git
+  - dpkg-dev
+  - gcc
+  - g++
+  - libc6-dev
+  - libssl-dev
+  - make
+  - cmake
+  - clang
+  - automake
+  - autoconf
+  - libtool
+  - ca-certificates
+  - wget
+
+# Create the Redis installation script
+write_files:
+  - path: /tmp/install_redis.sh
+    permissions: "0755"
+    content: |
+      #!/bin/bash
+
+      set -e # exit immediately on error
+
+      GH_ORG=${1:-"redis"}
+      GH_REPO=${2:-"redis"}
+      COMMIT=${3:-"HEAD"}
+      FOLDER=${4:-${GH_REPO}}
+
+      echo "Installing ${GH_ORG}/${GH_REPO} at commit ${COMMIT} in folder ${FOLDER}"
+      rm -rf ${FOLDER}
+      git clone https://github.com/${GH_ORG}/${GH_REPO} ${FOLDER}
+
+      cd ${FOLDER}
+      git checkout ${COMMIT}
+
+      make distclean
+      make BUILD_TLS=yes -j
+
+      echo "Installed successfully"
+
+# Run the installation script
+runcmd:
+  - [su, ubuntu, -c, "/tmp/install_redis.sh redis redis 7.2 ~/redis-7.2"]
+  - [su, ubuntu, -c, "/tmp/install_redis.sh redis redis 8.0.0 ~/redis-8.0.0"]
+  - [echo, "Cloud-init installation completed"]

terraform/oss-multi-arch-comp-amd-arm-intel-4xlarge-redis-7.2-redis-8.0.0/db-resources.tf

@@ -0,0 +1,34 @@
+resource "aws_instance" "server" {
+  for_each               = var.server_instances_configs # Create instances for each architecture
+  ami                    = each.value.ami
+  instance_type          = each.value.instance_type
+  subnet_id              = data.terraform_remote_state.shared_resources.outputs.subnet_public_us_east_2b_id
+  vpc_security_group_ids = ["${data.terraform_remote_state.shared_resources.outputs.performance_cto_sg_id}"]
+  key_name               = var.key_name
+  placement_group        = data.terraform_remote_state.shared_resources.outputs.placement_group_name_us_east_2b
+  availability_zone      = "us-east-2b"
+
+  user_data = file("./db-cloud-init-dbs.yaml") # Use cloud-init to install Redis versions
+
+  root_block_device {
+    volume_size           = var.instance_volume_size
+    volume_type           = var.instance_volume_type
+    encrypted             = var.instance_volume_encrypted
+    delete_on_termination = true
+    tags = merge(
+      local.base_tags,
+      {
+        Name = "ebs_block_device-${var.setup_name}-${each.value.arch_label}"
+      }
+    )
+  }
+
+  tags = merge(
+    local.base_tags,
+    {
+      Name         = "${var.setup_name}-${each.value.arch_label}"
+      Architecture = each.value.arch_label
+      InstanceType = each.value.instance_type
+    }
+  )
+}

terraform/oss-multi-arch-comp-amd-arm-intel-4xlarge-redis-7.2-redis-8.0.0/output.tf

@@ -0,0 +1,19 @@
+output "all_server_public_ips" {
+  value = {
+    for k, v in var.server_instances_configs : v.name => aws_instance.server[k].public_ip
+  }
+}
+
+output "all_server_private_ips" {
+  value = {
+    for k, v in var.server_instances_configs : v.name => aws_instance.server[k].private_ip
+  }
+}
+
+output "client_public_ip" {
+  value = ["${aws_instance.client[*].public_ip}"]
+}
+
+output "client_private_ip" {
+  value = ["${aws_instance.client[*].private_ip}"]
+}

terraform/oss-multi-arch-comp-amd-arm-intel-4xlarge-redis-7.2-redis-8.0.0/shared_resources.tf

@@ -0,0 +1,17 @@
+# provider
+provider "aws" {
+  region = var.region
+}
+
+################################################################################
+# This is the shared resources bucket key -- you will need it across environments like security rules,etc...
+# !! do not change this !!
+################################################################################
+data "terraform_remote_state" "shared_resources" {
+  backend = "s3"
+  config = {
+    bucket = "performance-cto-group"
+    key    = "benchmarks/infrastructure/shared_resources.tfstate"
+    region = "us-east-1"
+  }
+}

terraform/oss-multi-arch-comp-amd-arm-intel-4xlarge-redis-7.2-redis-8.0.0/variables.tf

@@ -0,0 +1,145 @@
+################################################################################
+# Variables used for deployment tag
+################################################################################
+
+variable "setup_name" {
+  description = "setup name"
+  default     = "oss-multi-arch-comp-amd-arm-intel-4xlarge-redis-7.2-redis-8.0.0"
+}
+
+variable "github_actor" {
+  description = "The name of the person or app that initiated the deployment."
+  default     = "N/A"
+}
+
+variable "github_org" {
+  description = "The owner name. For example, RedisModules."
+  default     = "N/A"
+}
+
+variable "github_repo" {
+  description = "The owner and repository name. For example, octocat/Hello-World."
+  default     = "N/A"
+}
+
+variable "github_sha" {
+  description = "The commit SHA that triggered the deployment."
+  default     = "N/A"
+}
+
+variable "timeout_secs" {
+  description = "The maximum time to wait prior destroying the VM via the watchdog."
+  default     = "3600"
+}
+
+variable "triggering_env" {
+  description = "The triggering environment. For example circleci."
+  default     = "N/A"
+}
+
+variable "environment" {
+  description = "The cost tag."
+  default     = "N/A"
+}
+
+variable "redis_module" {
+  description = "redis_module"
+  default     = "N/A"
+}
+
+################################################################################
+# Instance(s) options
+################################################################################
+
+variable "region" {
+  default = "us-east-2"
+}
+
+variable "server_instances_configs" {
+  description = "Configuration for each instance type"
+  type = map(object({
+    name          = string
+    instance_type = string
+    ami           = string
+    arch_label    = string
+  }))
+  default = {
+    "amd" = {
+      name          = "AMD m7a.4xlarge"
+      instance_type = "m7a.4xlarge"
+      ami           = "ami-04f167a56786e4b09" # Ubuntu 24.04 LTS x86_64
+      arch_label    = "AMD"
+    },
+    "intel" = {
+      name          = "Intel m7i.4xlarge"
+      instance_type = "m7i.4xlarge"
+      ami           = "ami-04f167a56786e4b09" # Ubuntu 24.04 LTS x86_64
+      arch_label    = "INTEL"
+    },
+    "arm" = {
+      name          = "ARM m8g.4xlarge"
+      instance_type = "m8g.4xlarge"
+      ami           = "ami-0ae6f07ad3a8ef182" # Ubuntu 24.04 LTS ARM64
+      arch_label    = "ARM"
+    }
+  }
+}
+
+variable "client_instance_ami" {
+  description = "AMI for aws EC2 instance - Ubuntu 24.04 LTS x86_64"
+  default     = "ami-04f167a56786e4b09"
+}
+
+variable "client_instance_type" {
+  description = "type for aws EC2 instance"
+  default     = "c6in.8xlarge"
+}
+
+variable "instance_volume_size" {
+  description = "EC2 instance volume_size"
+  default     = "128"
+}
+
+variable "instance_volume_type" {
+  description = "EC2 instance volume_type"
+  default     = "gp3"
+}
+
+
+variable "instance_volume_encrypted" {
+  description = "EC2 instance instance_volume_encrypted"
+  default     = "false"
+}
+
+################################################################################
+# SSH Access
+################################################################################
+variable "private_key" {
+  description = "private key"
+  default     = "/tmp/benchmarks.redislabs.pem"
+}
+
+variable "key_name" {
+  description = "key name"
+  default     = "perf-cto-us-east-2"
+}
+
+variable "ssh_user" {
+  description = "ssh_user"
+  default     = "ubuntu"
+}
+
+# Define base tags that are common to multiple resources
+locals {
+  base_tags = {
+    Environment    = var.environment
+    setup          = var.setup_name
+    redis_module   = var.redis_module
+    triggering_env = var.triggering_env
+    github_actor   = var.github_actor
+    github_org     = var.github_org
+    github_repo    = var.github_repo
+    github_sha     = var.github_sha
+    timeout_secs   = var.timeout_secs
+  }
+}

terraform/oss-multi-arch-comp-metal-amd-arm-intel-redis-latest/client-cloud-init-benchmark-client-runner.yaml

@@ -0,0 +1,20 @@
+#cloud-config
+
+package_update: true
+package_upgrade: true
+packages:
+  - python3-pip
+  - docker.io
+  - python3.12-venv
+
+# Run the installation script
+runcmd:
+  - [mkdir, /home/ubuntu/redis-benchmarks-specification-venv]
+  - [python3, -m, venv, /home/ubuntu/redis-benchmarks-specification-venv/]
+  - [
+      /home/ubuntu/redis-benchmarks-specification-venv/bin/pip,
+      install,
+      redis-benchmarks-specification,
+    ]
+  - [usermod, -aG, docker, ubuntu]
+  - [echo, "Cloud-init installation completed"]