|
| 1 | +# Terraform configuration for nightly tests running on Azure. |
| 2 | +# |
| 3 | +# To perform the required one-time Azure setup: |
| 4 | +# 1. Create a resource group for the tests and set ${var.azure_location} to |
| 5 | +# its name. |
| 6 | +# 2. Create a storage account and set ${var.azure_vhd_storage_account} to its |
| 7 | +# name. |
| 8 | +# 3. Create a storage container for the previously created storage account and |
| 9 | +# set ${var.vhd_storage_container} to its name. |
| 10 | + |
| 11 | +provider "azurerm" { |
| 12 | + # There are no Azure credentials here. |
| 13 | + # |
| 14 | + # So, set the ARM_SUBSCRIPTION_ID, ARM_CLIENT_ID, ARM_CLIENT_SECRET, |
| 15 | + # ARM_TENANT_ID environment variables to provide credentials for Azure |
| 16 | + # Resource Manager. |
| 17 | + # |
| 18 | + # See https://www.terraform.io/docs/providers/azurerm to understand the Azure |
| 19 | + # permissions needed to run Terraform against it. |
| 20 | +} |
| 21 | + |
| 22 | +# |
| 23 | +# Networking. |
| 24 | +# |
| 25 | + |
| 26 | +resource "azurerm_virtual_network" "cockroach" { |
| 27 | + name = "${var.prefix}-vn" |
| 28 | + address_space = ["192.168.0.0/16"] |
| 29 | + location = "${var.azure_location}" |
| 30 | + resource_group_name = "${var.azure_resource_group}" |
| 31 | +} |
| 32 | + |
| 33 | +# Firewall rules. |
| 34 | +resource "azurerm_network_security_group" "cockroach" { |
| 35 | + name = "${var.prefix}-nsg" |
| 36 | + location = "${var.azure_location}" |
| 37 | + resource_group_name = "${var.azure_resource_group}" |
| 38 | + |
| 39 | + security_rule { |
| 40 | + name = "${var.prefix}-cockroach-ssh" |
| 41 | + priority = 100 |
| 42 | + direction = "Inbound" |
| 43 | + access = "Allow" |
| 44 | + protocol = "Tcp" |
| 45 | + source_port_range = "*" |
| 46 | + destination_port_range = "22" |
| 47 | + source_address_prefix = "*" |
| 48 | + destination_address_prefix = "*" |
| 49 | + } |
| 50 | + |
| 51 | + security_rule { |
| 52 | + name = "${var.prefix}-cockroach-http" |
| 53 | + priority = 101 |
| 54 | + direction = "Inbound" |
| 55 | + access = "Allow" |
| 56 | + protocol = "Tcp" |
| 57 | + source_port_range = "*" |
| 58 | + destination_port_range = "8080" |
| 59 | + source_address_prefix = "*" |
| 60 | + destination_address_prefix = "*" |
| 61 | + } |
| 62 | + |
| 63 | + security_rule { |
| 64 | + name = "${var.prefix}-cockroach-sql" |
| 65 | + priority = 102 |
| 66 | + direction = "Inbound" |
| 67 | + access = "Allow" |
| 68 | + protocol = "Tcp" |
| 69 | + source_port_range = "*" |
| 70 | + destination_port_range = "26257" |
| 71 | + source_address_prefix = "*" |
| 72 | + destination_address_prefix = "*" |
| 73 | + } |
| 74 | + |
| 75 | + # Azure Network Security Groups have a low-priority default deny all rule. |
| 76 | + # See: |
| 77 | + # https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-nsg#default-rules |
| 78 | + |
| 79 | + tags { |
| 80 | + environment = "test" |
| 81 | + } |
| 82 | +} |
| 83 | + |
| 84 | +resource "azurerm_subnet" "cockroach" { |
| 85 | + name = "${var.prefix}-subnet" |
| 86 | + resource_group_name = "${var.azure_resource_group}" |
| 87 | + virtual_network_name = "${azurerm_virtual_network.cockroach.name}" |
| 88 | + address_prefix = "192.168.1.0/24" |
| 89 | +} |
| 90 | + |
| 91 | +# |
| 92 | +# CockroachDB nodes. |
| 93 | +# |
| 94 | + |
| 95 | +resource "azurerm_public_ip" "cockroach" { |
| 96 | + count = "${var.num_instances}" |
| 97 | + name = "${var.prefix}-ip-${count.index + 1}" |
| 98 | + location = "${var.azure_location}" |
| 99 | + resource_group_name = "${var.azure_resource_group}" |
| 100 | + public_ip_address_allocation = "dynamic" |
| 101 | + domain_name_label="${var.prefix}-cockroach-${count.index + 1}" |
| 102 | + |
| 103 | + tags { |
| 104 | + environment = "test" |
| 105 | + } |
| 106 | +} |
| 107 | + |
| 108 | +resource "azurerm_network_interface" "cockroach" { |
| 109 | + count = "${var.num_instances}" |
| 110 | + |
| 111 | + name = "${var.prefix}-cockroach-nic-${count.index + 1}" |
| 112 | + location = "${var.azure_location}" |
| 113 | + resource_group_name = "${var.azure_resource_group}" |
| 114 | + network_security_group_id = "${azurerm_network_security_group.cockroach.id}" |
| 115 | + |
| 116 | + ip_configuration { |
| 117 | + name = "testconfiguration1" |
| 118 | + subnet_id = "${azurerm_subnet.cockroach.id}" |
| 119 | + private_ip_address_allocation = "dynamic" |
| 120 | + public_ip_address_id = "${element(azurerm_public_ip.cockroach.*.id, count.index)}" |
| 121 | + } |
| 122 | +} |
| 123 | + |
| 124 | +resource "azurerm_virtual_machine" "cockroach" { |
| 125 | + count = "${var.num_instances}" |
| 126 | + name = "${var.prefix}-cockroach-${count.index + 1}" |
| 127 | + location = "${var.azure_location}" |
| 128 | + resource_group_name = "${var.azure_resource_group}" |
| 129 | + network_interface_ids = ["${element(azurerm_network_interface.cockroach.*.id, count.index)}"] |
| 130 | + vm_size = "${var.azure_vm_size}" |
| 131 | + delete_os_disk_on_termination = "true" |
| 132 | + |
| 133 | + storage_image_reference { |
| 134 | + publisher = "Canonical" |
| 135 | + offer = "UbuntuServer" |
| 136 | + sku = "16.04.0-LTS" |
| 137 | + version = "latest" |
| 138 | + } |
| 139 | + |
| 140 | + # Don't recreate this VM when the VHD URI changes, because that may have |
| 141 | + # unique identifiers that change every time this config is applied. |
| 142 | + lifecycle { |
| 143 | + ignore_changes = [ "storage_os_disk" ] |
| 144 | + } |
| 145 | + |
| 146 | + storage_os_disk { |
| 147 | + name = "disk1" |
| 148 | + vhd_uri = "https://${var.azure_vhd_storage_account}.blob.core.windows.net/${var.vhd_storage_container}/${var.prefix}-cockroach-${count.index + 1}.vhd" |
| 149 | + create_option = "FromImage" |
| 150 | + } |
| 151 | + |
| 152 | + os_profile { |
| 153 | + computer_name = "${var.prefix}-cockroach-${count.index + 1}" |
| 154 | + admin_username = "ubuntu" |
| 155 | + # This password doesn't matter, because password auth is disabled below. |
| 156 | + admin_password = "password_auth_disabled" |
| 157 | + } |
| 158 | + |
| 159 | + os_profile_linux_config { |
| 160 | + disable_password_authentication = true |
| 161 | + ssh_keys { |
| 162 | + path = "/home/ubuntu/.ssh/authorized_keys" |
| 163 | + key_data = "${file("~/.ssh/${var.key_name}.pub")}" |
| 164 | + } |
| 165 | + } |
| 166 | + |
| 167 | + tags { |
| 168 | + environment = "test" |
| 169 | + } |
| 170 | +} |
| 171 | + |
| 172 | +# Supervisor config for CockroachDB nodes. |
| 173 | +data "template_file" "supervisor" { |
| 174 | + count = "${var.num_instances}" |
| 175 | + template = "${file("../common/supervisor.conf.tpl")}" |
| 176 | + depends_on = [ "azurerm_virtual_machine.cockroach" ] |
| 177 | + |
| 178 | + vars { |
| 179 | + stores = "${var.stores}" |
| 180 | + cockroach_port = "${var.sql_port}" |
| 181 | + # The value of the --join flag must be empty for the first node, |
| 182 | + # and a running node for all others. We build a list of addresses |
| 183 | + # shifted by one (first element is empty), then take the value at index "instance.index". |
| 184 | + join_address = "${element(concat(split(",", ""), azurerm_public_ip.cockroach.*.fqdn), count.index)}" |
| 185 | + cockroach_flags = "${var.cockroach_flags}" |
| 186 | + # If the following changes, (*terrafarm.Farmer).Add() must change too. |
| 187 | + cockroach_env = "${var.cockroach_env}" |
| 188 | + benchmark_name = "${var.benchmark_name}" |
| 189 | + } |
| 190 | +} |
| 191 | + |
| 192 | +# Set up CockroachDB nodes. |
| 193 | +resource "null_resource" "cockroach-runner" { |
| 194 | + count = "${var.num_instances}" |
| 195 | + depends_on = [ "azurerm_virtual_machine.cockroach" ] |
| 196 | + |
| 197 | + connection { |
| 198 | + user = "ubuntu" |
| 199 | + private_key = "${file(format("~/.ssh/%s", var.key_name))}" |
| 200 | + host = "${element(azurerm_public_ip.cockroach.*.fqdn, count.index)}" |
| 201 | + } |
| 202 | + |
| 203 | + provisioner "file" { |
| 204 | + source = "../common/download_binary.sh" |
| 205 | + destination = "/home/ubuntu/download_binary.sh" |
| 206 | + } |
| 207 | + |
| 208 | + provisioner "file" { |
| 209 | + source = "../common/nodectl" |
| 210 | + destination = "/home/ubuntu/nodectl" |
| 211 | + } |
| 212 | + |
| 213 | + # This writes the filled-in supervisor template. It would be nice if we could |
| 214 | + # use rendered templates in the file provisioner. |
| 215 | + provisioner "remote-exec" { |
| 216 | + inline = <<FILE |
| 217 | +echo '${element(data.template_file.supervisor.*.rendered, count.index)}' > supervisor.conf |
| 218 | +FILE |
| 219 | + } |
| 220 | + |
| 221 | + provisioner "file" { |
| 222 | + # If no binary is specified, we'll copy /dev/null (always 0 bytes) to the |
| 223 | + # instance. The "remote-exec" block will then overwrite that. There's no |
| 224 | + # such thing as conditional file copying in Terraform, so we fake it. |
| 225 | + source = "${coalesce(var.cockroach_binary, "/dev/null")}" |
| 226 | + destination = "/home/ubuntu/cockroach" |
| 227 | + } |
| 228 | + |
| 229 | + # Launch CockroachDB. |
| 230 | + provisioner "remote-exec" { |
| 231 | + inline = [ |
| 232 | + # For consistency with other Terraform configs, we create the store in |
| 233 | + # /mnt/data0. |
| 234 | + "sudo mkdir /mnt/data0", |
| 235 | + "sudo chown ubuntu:ubuntu /mnt/data0", |
| 236 | + # This sleep is needed to avoid apt-get errors below. It appears that when |
| 237 | + # the VM first launches, something is interfering with launches of apt-get. |
| 238 | + "sleep 30", |
| 239 | + # Install test dependencies. NTP synchronization is especially needed for |
| 240 | + # Azure VMs. |
| 241 | + "sudo apt-get -qqy update >/dev/null", |
| 242 | + "sudo apt-get -qqy install supervisor ntpdate >/dev/null", |
| 243 | + "sudo ntpdate -b pool.ntp.org", |
| 244 | + "sudo apt-get -qqy install ntp >/dev/null", |
| 245 | + "sudo sed -i 's/^#statsdir/statsdir/' /etc/ntp.conf", |
| 246 | + "sudo service supervisor stop", |
| 247 | + # TODO(cuongdo): Remove this dependency on Google Cloud SDK after we move |
| 248 | + # the test data to Azure Storage. |
| 249 | + "export CLOUD_SDK_REPO=\"cloud-sdk-$(lsb_release -c -s)\"", |
| 250 | + "echo \"deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main\" | sudo tee /etc/apt/sources.list.d/google-cloud-sdk.list", |
| 251 | + "curl -sS https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -", |
| 252 | + "sudo apt-get -qqy update >/dev/null", |
| 253 | + "sudo apt-get -qqy install google-cloud-sdk >/dev/null", |
| 254 | + # Install CockroachDB. |
| 255 | + "mkdir /mnt/data0/logs", |
| 256 | + "ln -sf /mnt/data0/logs logs", |
| 257 | + "chmod 755 cockroach nodectl", |
| 258 | + "[ $(stat --format=%s cockroach) -ne 0 ] || bash download_binary.sh cockroach/cockroach ${var.cockroach_sha}", |
| 259 | + "if [ ! -e supervisor.pid ]; then supervisord -c supervisor.conf; fi", |
| 260 | + "supervisorctl -c supervisor.conf start cockroach", |
| 261 | + # Install load generators. |
| 262 | + "bash download_binary.sh examples-go/block_writer ${var.block_writer_sha}", |
| 263 | + "bash download_binary.sh examples-go/photos ${var.photos_sha}", |
| 264 | + ] |
| 265 | + } |
| 266 | +} |
0 commit comments