Exoscale deploy

clojurians_log/README.md

@@ -14,7 +14,7 @@ This is a work in progress and is subject to significant changes over time.
   Contact either of the [maintainers](https://github.com/clojureverse/nebula#maintainers)
   for this.
 - Create your own SSH keypair on the Console: [guide](https://community.exoscale.com/documentation/compute/ssh-keypairs/)
-- Install Terraform v0.11+.
+- Install Terraform v0.12.6+.
 - Download the latest Exoscale terraform provider for your OS from [here](https://github.com/terraform-providers/terraform-provider-exoscale).
 - Decompress the archive and follow [the plugin installation](https://www.terraform.io/docs/configuration/providers.html#third-party-plugins).
 - Install [GPG](https://gnupg.org/download/)
@@ -24,9 +24,19 @@ This is a work in progress and is subject to significant changes over time.
   - Run `export TF_VAR_exoscale_api_key="The Exoscale API key here"`
   - Run `export TF_VAR_exoscale_secret_key="The Exoscale secret key here"`
   - Run `export TF_VAR_exoscale_ssh_keypair_name="The Exoscale key pair name you created"`
+  - Run `export TF_VAR_cloudflare_email="Your cloudflare account"`
+  - Run `export TF_VAR_cloudlfare_api_key="Cloudflare 'global API key' (not to be confused with cloudflare tokens)"`
+  - Run `bin/terraform_apply`
+  - To destroy: `terraform destroy`
+  - To ssh into the instance
+	- `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $(terraform output username)@$(terraform output ip_address)`
+  - You wil want to reboot the instance after provisioning the first time. (why?)
+  - You will need to update the IP address on the host name in the DNS as well.
+
+The `terraform_apply script does the following`
   - In the `playbooks/vars` dir:
     - Run `gpg --decrypt clojurians_log_secrets.yml.gpg > clojurians_log_secrets.yml`
-    - (Optional, for maintainers) Run:
+    - If you need to update the secret file later on you'll have to do this:
       ```bash
       gpg --encrypt --recipient your_email \
                     --recipient others_emails \
@@ -41,13 +51,7 @@ This is a work in progress and is subject to significant changes over time.
       -backend-config="secret_key=${TF_VAR_exoscale_secret_key}"
     ```
   - Run `terraform plan -out plan`
-  - Make sure all looks good.
   - Run `terraform apply plan`
-  - To destroy: `terraform destroy`
-  - To ssh into the instance
-	- `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $(terraform output username)@$(terraform output ip_address)`
-  - You wil want to reboot the instance after provisioning the first time.
-  - You will need to update the IP address on the host name in the DNS as well.
 
 ## How to use Vagrant
 

clojurians_log/Vagrantfile

@@ -76,7 +76,8 @@ Vagrant.configure("2") do |config|
     ansible.raw_arguments = Shellwords.shellsplit(ENV['ANSIBLE_ARGS']) if ENV['ANSIBLE_ARGS']
     ansible.extra_vars = { ansible_python_interpreter: "/usr/bin/python3",
                            clojurians_app_fqdn: "clojurians-log-staging.clojureverse.org",
-                           use_demo_logs: true, # for only loading the demo logs on a vagrant host
+                           use_demo_logs: true, # Load the demo logs on a vagrant host
+                           acme_sh_default_staging: true # Use ACME default servers on a vagrant host
                            acme_sh_default_force_issue: true, # Force ACME cert issue on a vagrant host
                          }
   end

clojurians_log/bin/terraform_apply

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+if [[ -z "${TF_VAR_exoscale_ssh_keypair_name}" ]]; then
+    echo "TF_VAR_exoscale_ssh_keypair_name not set, aborting."
+    exit 1
+fi
+
+if [[ -z "${TF_VAR_exoscale_api_key}" ]]; then
+    echo "TF_VAR_exoscale_api_key not set, aborting."
+    exit 1
+fi
+
+if [[ -z "${TF_VAR_exoscale_secret_key}" ]]; then
+    echo "TF_VAR_exoscale_secret_key not set, aborting."
+    exit 1
+fi
+
+# File does not exist, or encrypted version is newer: decrypt
+if [[ "playbooks/vars/clojurians_log_secrets.yml.gpg" -nt "playbooks/vars/clojurians_log_secrets.yml" ]]; then
+    gpg --decrypt playbooks/vars/clojurians_log_secrets.yml.gpg > playbooks/vars/clojurians_log_secrets.yml
+fi
+
+if [[ ! -d ".terraform" ]]; then
+    terraform init \
+              -backend-config="access_key=${TF_VAR_exoscale_api_key}" \
+              -backend-config="secret_key=${TF_VAR_exoscale_secret_key}"
+fi
+
+terraform plan -out plan
+terraform apply plan

clojurians_log/main.tf

@@ -1,28 +1,42 @@
+locals {
+  instance_name = "${terraform.workspace == "default" ? "clojurians-log" : terraform.workspace}"
+}
+
 provider "exoscale" {
   version = "~> 0.10"
   key = "${var.exoscale_api_key}"
   secret = "${var.exoscale_secret_key}"
 }
 
+provider "cloudflare" {
+  version = "~> 1.17"
+
+  email = "${var.cloudflare_email}"
+  token = "${var.cloudflare_api_key}"
+}
+
+provider "template" {
+   version = "~> 2.1"
+}
+
 data "template_file" "userdata" {
   template = "${file("userdata.sh.tmpl")}"
   vars = {
-    # For development / testing, use (uncomment) the line below
-    ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3 clojurians_app_fqdn=clojurians-log-staging.clojureverse.org use_demo_logs=true acme_sh_default_force_issue=true\""
-    git_clone_params = "--single-branch --branch exoscale-staging"
-    # For normal production use, the values set in ansible are appropriate.
-    # So, can leave the following empty.
-    # ansible_playbook_params = ""
-    # git_clone_params = ""
+    # For development / testing, use (uncomment) the lines below
+    # ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3 clojurians_app_fqdn=clojurians-log-staging.clojureverse.org use_demo_logs=true acme_sh_default_staging=true acme_sh_default_force_issue=true\""
+    # git_clone_params = "--single-branch --branch exoscale-deploy"
+    # The following can be used for production
+    ansible_playbook_params = "--extra-vars \"ansible_python_interpreter=/usr/bin/python3\" --extra-vars \"clojurians_app_fqdn=${local.instance_name}.clojureverse.org\""
+    git_clone_params = ""
   }
 }
 
 resource "exoscale_compute" "clojurians_log" {
-  display_name = "clojurians-log"
+  display_name = "${local.instance_name}"
   template = "Linux Ubuntu 18.04 LTS 64-bit"
-  zone = "de-fra-1"
-  size = "Medium"
-  disk_size = 10
+  zone = "ch-gva-2"
+  size = "Large"
+  disk_size = 50
   key_pair = "${var.exoscale_ssh_keypair_name}"
   user_data = "${data.template_file.userdata.rendered}"
 
@@ -34,6 +48,22 @@ resource "exoscale_compute" "clojurians_log" {
 output "ip_address" {
   value = exoscale_compute.clojurians_log.ip_address
 }
+
 output "username" {
   value = exoscale_compute.clojurians_log.username
 }
+
+resource "cloudflare_record" "clojurians_log_internal" {
+  domain = "clojureverse.org"
+  name   = "${local.instance_name}-internal"
+  value  = "${exoscale_compute.clojurians_log.ip_address}"
+  type   = "A"
+}
+
+resource "cloudflare_record" "clojurians_log" {
+  domain  = "clojureverse.org"
+  name    = "${local.instance_name}"
+  value   = "${exoscale_compute.clojurians_log.ip_address}"
+  type    = "A"
+  proxied = true
+}

clojurians_log/playbooks/clojurians-log.yml

@@ -22,10 +22,19 @@
     clojurians_app_http_port: 4242
     datomic_pro_version: 0.9.5561.56
     datomic_object_cache_max: 2g
+    datomic_transactor_max_heap: 3584m
+    clojure_socket_repl_port: 50505
     clojure_app_env_vars: |
-      JVM_OPTS="-Dclojure.server.myrepl={:port,50505,:accept,clojure.core.server/repl} -Xmx8g -Xms2g -Ddatomic.ObjectCacheMax=7g -Ddatomic.memcachedServers=127.0.0.1:11211"
+      JVM_OPTS="-Dclojure.server.myrepl={:port,{{ clojure_socket_repl_port }},:accept,clojure.core.server/repl} -Xmx3584m -Xms2g -Ddatomic.ObjectCacheMax=2g -Ddatomic.memcachedServers=127.0.0.1:11211"
+    # -Dcom.sun.management.jmxremote.port=17264 -Dcom.sun.management.jmxremote.rmi.port=17264
 
   tasks:
+    - name: Set up librato first so we have eyes
+      import_role:
+        name: librato.librato
+      vars:
+        librato_enabled_plugins: ['nginx', 'memcached'] #jvm, postgresql
+        librato_logging_use_syslog: true
 
     # Keys first, then repos, then the dist-upgrade.
     - name: Add repository for ansible
@@ -177,7 +186,7 @@
         clojure_app_health_check_url: "http://localhost:{{clojurians_app_http_port}}/healthcheck"
         clojure_app_service_start_after: datomic.service
 
-    - name: Install the Clojurians Log app
+    - name: Install and configure the clojurians-log-app
       tags: clojurians-log
       import_role:
         name: plexus.clojurians-log
@@ -237,6 +246,9 @@
             state: "present"
             nginx_access_log: off
             extra_parameters: |
+              location = /basic_status {
+                    stub_status;
+              }
               location / {
                   root {{ clojure_app_app_dir }}/resources/public;
                   try_files $uri @proxy_app;

clojurians_log/playbooks/roles/librato.librato/.gitignore

@@ -0,0 +1,5 @@
+tmp/
+.kitchen/
+.librarian/
+Ansiblefile.lock
+hosts

clojurians_log/playbooks/roles/librato.librato/.kitchen.yml

@@ -0,0 +1,102 @@
+driver:
+  name: vagrant
+
+provisioner:
+  name: ansible_playbook
+  require_ansible_repo: true
+  ansible_verbose: true
+  require_chef_for_busser: false
+  require_ruby_for_busser: true
+  hosts: hosts
+  roles_path: ../
+
+verifier:
+  ruby_bindir: '/usr/bin'
+
+platforms:
+  - name: centos-6
+    driver:
+      box: centos/6
+  - name: centos-7
+    driver:
+      box: centos/7
+  - name: fedora-23
+    driver:
+      box: fedora/23-cloud-base
+# Due to a bug in kitchen-ansible, statically set the ansible_platform to Amazon
+# See: https://github.com/neillturner/kitchen-ansible/issues/216
+  - name: amazonlinux-2016.03
+    driver:
+      name: ec2
+      image_id: ami-7172b611
+      aws_ssh_key_id: test-kitchen
+      region: us-west-2
+      availability_zone: a
+      instance_type: t2.large
+      associate_public_ip: true
+      interface: dns
+    transport:
+      username: ec2-user
+      ssh_key: ~/.ssh/id_rsa
+    provisioner:
+      ansible_platform: 'amazon'
+      require_chef_for_busser: true
+      require_ruby_for_busser: false
+    verifier:
+      ruby_bindir: '/opt/chef/embedded/bin'
+  - name: amazonlinux-2016.09
+    driver:
+      name: ec2
+      image_id: ami-5ec1673e
+      aws_ssh_key_id: test-kitchen
+      region: us-west-2
+      availability_zone: a
+      instance_type: t2.large
+      associate_public_ip: true
+      interface: dns
+    transport:
+      username: ec2-user
+      ssh_key: ~/.ssh/id_rsa
+    provisioner:
+      ansible_platform: 'amazon'
+      require_chef_for_busser: true
+      require_ruby_for_busser: false
+    verifier:
+      ruby_bindir: '/opt/chef/embedded/bin'
+# Ansible doesn't provide Debian repos so we use the omnibus installer to provide Ansible
+  - name: debian-7
+    driver:
+      box: debian/wheezy64
+    provisioner:
+      require_ansible_omnibus: true
+      require_ansible_repo: false
+  - name: debian-8
+    driver:
+      box: debian/jessie64
+    provisioner:
+      require_ansible_omnibus: true
+      require_ansible_repo: false
+# Ubuntu 12.04 uses Ruby 1.8.7 while busser requires Ruby 1.9+, so we need to use Chef to run busser
+  - name: ubuntu-12.04
+    driver:
+      box: ubuntu/precise64
+    provisioner:
+      require_chef_for_busser: true
+      require_ruby_for_busser: false
+    verifier:
+      ruby_bindir: '/opt/chef/embedded/bin'
+  - name: ubuntu-14.04
+    driver:
+      box: ubuntu/trusty64
+  - name: ubuntu-15.04
+    driver:
+      box: larryli/vivid64
+  - name: ubuntu-15.10
+    driver:
+      box: ubuntu/wily64
+  - name: ubuntu-16.04
+    driver:
+      box: ubuntu/xenial64
+
+suites:
+  - name: default

clojurians_log/playbooks/roles/librato.librato/Ansiblefile

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+#^syntax detection
+
+site "https://galaxy.ansible.com/api/v1";

clojurians_log/playbooks/roles/librato.librato/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'kitchen-ansible'
+gem 'kitchen-vagrant'
+gem 'kitchen-verifier-serverspec'