Database replication

extra/install_config.bat

@@ -0,0 +1,40 @@
+@echo off
+SETLOCAL ENABLEDELAYEDEXPANSION
+set /p numberOfServers=How many extra databases do you need? 
+set /p replicatorPassword=What password do you want to use for the replicator account? 
+echo # -*- mode: ruby -*- > ../Vagrantfile
+echo #vi: set ft=ruby : >> ../Vagrantfile
+echo. >> ../Vagrantfile
+echo VAGRANTFILE_API_VERSION = "2" >> ../Vagrantfile
+echo. >> ../Vagrantfile
+echo Vagrant.configure(VAGRANTFILE_API_VERSION) do ^|config^| >> ../Vagrantfile
+echo   config.vm.box = "ubuntu/trusty64" >> ../Vagrantfile
+echo   config.ssh.shell = "bash -c 'BASH_ENV=/etc/profile exec bash'" >> ../Vagrantfile
+echo   #This runs the provision script on both servers >> ../Vagrantfile
+echo. >> ../Vagrantfile
+echo   #Main server that runs the FBCTF >> ../Vagrantfile
+echo   config.vm.define "main" do ^|main^| >> ../Vagrantfile
+echo     main.vm.network "private_network", ip: "10.10.10.5" >> ../Vagrantfile
+echo     main.vm.hostname = "facebookCTF-Dev" >> ../Vagrantfile
+echo     main.vm.provision "shell", path: "extra/provision.sh", args: "'-r %numberOfServers%' '-N 1' '-P%replicatorPassword%' ENV[\'FBCTF_PROVISION_ARGS\']", privileged: true >> ../Vagrantfile
+echo     config.vm.provider "virtualbox" do ^|v^| >> ../Vagrantfile
+echo       v.memory = 4096 >> ../Vagrantfile
+echo       v.cpus = 4 >> ../Vagrantfile
+echo     end >> ../Vagrantfile
+echo   end >> ../Vagrantfile
+FOR /L %%i in (1,1,%numberOfServers%) DO (
+SET /A addr=%%i+5
+SET /A serverNumber=%%i+1
+echo. >> ../Vagrantfile
+echo   #Replication server >> ../Vagrantfile
+echo   config.vm.define "replication%%i" do ^|replication%%i^| >> ../Vagrantfile
+echo     replication%%i.vm.network "private_network", ip: "10.10.10.!addr!" >> ../Vagrantfile
+echo     replication%%i.vm.hostname = "fbctf-dbreplication" >> ../Vagrantfile
+echo     replication%%i.vm.provision "shell", path: "extra/replication.sh", args: "'-r %numberOfServers%' '-N !serverNumber!' '-P%replicatorPassword%' ENV[\'FBCTF_PROVISION_ARGS\']", privileged: true >> ../Vagrantfile
+echo     config.vm.provider "virtualbox" do ^|v^| >> ../Vagrantfile
+echo       v.memory = 2048 >> ../Vagrantfile
+echo       v.cpus = 2 >> ../Vagrantfile
+echo     end >> ../Vagrantfile
+echo   end >> ../Vagrantfile
+)
+echo end >> ../Vagrantfile

extra/lib.sh

@@ -307,6 +307,83 @@ function import_empty_db() {
   log "The password for admin is: $PASSWORD"
 }
 
+function create_replication_user() {
+  local __user=$1
+  local __pwd=$2
+  local __db=$3
+  local __replicator_password=$4
+
+  log "Creating replication user..."
+  mysql -u "$__user" --password="$__pwd" -e "CREATE USER 'replicator'@'%' IDENTIFIED BY '$__replicator_password';" || true # don't fail if the user exists
+  #Unfortunately, I need to grant SUPER to replicator, because vagrant does not let us run a second script after the other server is finished setting up
+  #There is a "trigger" vagrant plugin that could resolve this issue, and help ensure we do not grant more permissions then we should be
+  mysql -u "$__user" --password="$__pwd" -e "GRANT SUPER ON *.* TO 'replicator'@'%'"
+  mysql -u "$__user" --password="$__pwd" -e "GRANT ALL PRIVILEGES ON \`$__db\`.* TO 'replicator'@'%';"
+  mysql -u "$__user" --password="$__pwd" -e "GRANT REPLICATION SLAVE ON *.* TO 'replicator'@'%';"
+  mysql -u "$__user" --password="$__pwd" -e "FLUSH PRIVILEGES;"
+}
+
+function setup_db_replication() {
+  NUMBER_OF_SERVERS=$1
+  CURRENT_SERVER_NUMBER=$2
+  PREVIOUS_SERVER_IP=$(($CURRENT_SERVER_NUMBER+3))
+  CURRENT_SERVER_IP=$((CURRENT_SERVER_NUMBER+4))
+  replicator_password=$3
+
+  log "Adding replication settings to my.cnf..."
+  sed -i '/bind-address\s\+= 127.0.0.1/c\#bind-address		= 127.0.0.1' "/etc/mysql/my.cnf"
+  sed -i "/#server-id\s\+= 1/c\server-id		= $CURRENT_SERVER_NUMBER" "/etc/mysql/my.cnf"
+  sed -i '/#binlog_do_db\s\+= include_database_name/c\binlog_do_db		= fbctf' "/etc/mysql/my.cnf"
+  sed -i '/#log_bin\s\+= \/var\/log\/mysql\/mysql-bin.log/c\log_bin		= /var/log/mysql/mysql-bin.log' "/etc/mysql/my.cnf"
+
+  #This value should always be however many databsase servers are going to be setup, value N
+  echo "auto_increment_increment = $NUMBER_OF_SERVERS" >> "/etc/mysql/my.cnf"
+
+  #This value should be different for every database server. The first server should be 1, incrementing until the final server has the same value as N.
+  #Example: 5 servers. Server 1 has an offset of 1, Server 2 an offset of 2, up to Server 5 with an offset of 5.
+  echo "auto_increment_offset = $CURRENT_SERVER_NUMBER" >> "/etc/mysql/my.cnf"
+
+  #Unfortunately Vagrant does not allow for a provision script to be run AFTER all machines have been created
+  #So I have to configure the setting in this script
+
+  if [ "$CURRENT_SERVER_NUMBER" -eq 1 ]; then
+    log "This is the first server, the next server will connect to mysql and set it up..."
+    log "Restarting mysql..."
+    sudo service mysql restart
+  elif [ "$NUMBER_OF_SERVERS" -ne "$CURRENT_SERVER_NUMBER"  -a  "$CURRENT_SERVER_NUMBER" -ne 1 ]; then 
+    log "Restarting mysql so it generates bin_log info..."
+    sudo service mysql restart
+    log "Retrieve bin_log information to give to the other server..."
+    SMS=/tmp/show_master_status.txt
+    mysql -ureplicator --password="${replicator_password}" -ANe "SHOW MASTER STATUS" > ${SMS}
+    CURRENT_LOG=`cat ${SMS} | awk '{print $1}'`
+    CURRENT_POS=`cat ${SMS} | awk '{print $2}'`
+
+    log "Configuring Master info onto remote mysql database..."
+    mysql -h 10.10.10.${PREVIOUS_SERVER_IP} -u replicator --password="${replicator_password}" -e "CHANGE MASTER TO MASTER_HOST = '10.10.10.${CURRENT_SERVER_IP}', MASTER_USER = 'replicator', MASTER_PASSWORD = '${replicator_password}', MASTER_LOG_FILE = '${CURRENT_LOG}', MASTER_LOG_POS = ${CURRENT_POS}"
+  else
+    log "Restarting mysql so it generates bin_log info..."
+    sudo service mysql restart
+    log "Retrieve bin_log information to give to the other server..."
+    SMS2=/tmp/show_master_status2.txt
+    mysql -ureplicator --password="${replicator_password}" -ANe "SHOW MASTER STATUS" > ${SMS2}
+    CURRENT_LOG2=`cat ${SMS2} | awk '{print $1}'`
+    CURRENT_POS2=`cat ${SMS2} | awk '{print $2}'`
+
+    log "Configuring Master info onto remote mysql database..."
+    mysql -h 10.10.10.${PREVIOUS_SERVER_IP} -u "replicator" --password="${replicator_password}" -e "CHANGE MASTER TO MASTER_HOST = '10.10.10.${CURRENT_SERVER_IP}', MASTER_USER = 'replicator', MASTER_PASSWORD = '${replicator_password}', MASTER_LOG_FILE = '${CURRENT_LOG2}', MASTER_LOG_POS = ${CURRENT_POS2}"
+
+   log "Connecting to other master server to retrieve bin_log information..."
+   SMS=/tmp/show_master_status.txt
+   mysql -h 10.10.10.5 -ureplicator --password="${replicator_password}" -ANe "SHOW MASTER STATUS" > ${SMS}
+   CURRENT_LOG=`cat ${SMS} | awk '{print $1}'`
+   CURRENT_POS=`cat ${SMS} | awk '{print $2}'`
+
+   log "Configuring Master info onto local mysql database..."
+   mysql -u "root" --password="root" -e "CHANGE MASTER TO MASTER_HOST = '10.10.10.5', MASTER_USER = 'replicator', MASTER_PASSWORD = '${replicator_password}', MASTER_LOG_FILE = '${CURRENT_LOG}', MASTER_LOG_POS = ${CURRENT_POS}"
+  fi
+}
+
 function set_password() {
   local __admin_pwd=$1
   local __user=$2

extra/provision.sh

@@ -85,6 +85,9 @@ function usage() {
   printf "  -e EMAIL, --email EMAIL \tDomain for the SSL certificate to be generated using letsencrypt.\n"
   printf "  -s PATH, --code PATH \t\tPath to fbctf code. Default is /vagrant\n"
   printf "  -d PATH, --destination PATH \tDestination path to place the fbctf folder. Default is /var/www/fbctf\n"
+  printf "  -r NUMBER, --replication NUMBER \tExtra number of databases"
+  printf "  -N NUMBER, --server-number NUMBER \tCurrent server number"
+  printf "  -P PASSWORD, --replicator-password PASSWORD \tReplicator user password"
   printf "\nExamples:\n"
   printf "  Provision fbctf in development mode:\n"
   printf "\t%s -m dev -s /home/foobar/fbctf -d /var/fbctf\n" "${0}"
@@ -94,7 +97,7 @@ function usage() {
   printf "\t%s -m dev -U -s /home/foobar/fbctf -d /var/fbctf\n" "${0}"
 }
 
-ARGS=$(getopt -n "$0" -o hm:c:URk:C:D:e:s:d: -l "help,mode:,cert:,update,repo-mode,keyfile:,certfile:,domain:,email:,code:,destination:,docker" -- "$@")
+ARGS=$(getopt -n "$0" -o hm:c:URk:C:D:e:s:d:r:N:P: -l "help,mode:,cert:,update,repo-mode,keyfile:,certfile:,domain:,email:,code:,destination:,replication,server-number,replicator-password,docker" -- "$@")
 
 eval set -- "$ARGS"
 
@@ -156,6 +159,18 @@ while true; do
       CTF_PATH=$2
       shift 2
       ;;
+    -r|--replication)
+      NUMBER_OF_SERVERS=$(($2+1))
+      shift 2
+      ;;
+    -N|--server-number)
+      CURRENT_SERVER_NUMBER=$2
+      shift 2
+      ;;
+    -P|--replicator-password)
+      REPLICATOR_PASSWORD=$2
+      shift 2
+      ;;
     --docker)
       DOCKER=true
       shift
@@ -285,6 +300,11 @@ log "Remember install the same version of unison (2.48.3) in your host machine"
 # Database creation
 import_empty_db "root" "$P_ROOT" "$DB" "$CTF_PATH" "$MODE"
 
+if [[ "$NUMBER_OF_SERVERS"  ||  "$CURRENT_SERVER_NUMBER" ]]; then
+  create_replication_user "root" "$P_ROOT" "$DB" "$REPLICATOR_PASSWORD"
+  setup_db_replication "$NUMBER_OF_SERVERS" "$CURRENT_SERVER_NUMBER" "$REPLICATOR_PASSWORD"
+fi
+
 # Make attachments folder world writable
 sudo chmod 777 "$CTF_PATH/src/data/attachments"
 sudo chmod 777 "$CTF_PATH/src/data/attachments/deleted"

extra/replication.sh

@@ -0,0 +1,203 @@
+#!/bin/bash
+ # We want the provision script to fail as soon as there are any errors
+set -e
+
+DB="fbctf"
+U="ctf"
+P="ctf"
+P_ROOT="root"
+
+# Default values
+MODE="dev"
+NOREPOMODE=false
+TYPE="self"
+KEYFILE="none"
+CERTFILE="none"
+DOMAIN="none"
+EMAIL="none"
+CODE_PATH="/vagrant"
+CTF_PATH="/var/www/fbctf"
+HHVM_CONFIG_PATH="/etc/hhvm/server.ini"
+
+ARGS=$(getopt -n "$0" -o hm:c:URk:C:D:e:s:d:r:N:P: -l "help,mode:,cert:,update,repo-mode,keyfile:,certfile:,domain:,email:,code:,destination:,replication,server-number,replicator-password,docker" -- "$@")
+
+eval set -- "$ARGS"
+
+while true; do
+  case "$1" in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    -m|-mode)
+      GIVEN_ARG=$2
+      if [[ "${VALID_MODE[@]}" =~ "${GIVEN_ARG}" ]]; then
+        MODE=$2
+        shift 2
+      else
+        usage
+        exit 1
+      fi
+      ;;
+    -c|--cert)
+      GIVEN_ARG=$2
+      if [[ "${VALID_TYPE[@]}" =~ "${GIVEN_ARG}" ]]; then
+        TYPE=$2
+        shift 2
+      else
+        usage
+        exit 1
+      fi
+      ;;
+    -U|--update)
+      UPDATE=true
+      shift
+      ;;
+    -R|--no-repo-mode)
+      NOREPOMODE=true
+      shift
+      ;;
+    -k|--keyfile)
+      KEYFILE=$2
+      shift 2
+      ;;
+    -C|--certfile)
+      CERTFILE=$2
+      shift 2
+      ;;
+    -D|--domain)
+      DOMAIN=$2
+      shift 2
+      ;;
+    -e|--email)
+      EMAIL=$2
+      shift 2
+      ;;
+    -s|--code)
+      CODE_PATH=$2
+      shift 2
+      ;;
+    -d|--destination)
+      CTF_PATH=$2
+      shift 2
+      ;;
+    -r|--replication)
+      NUMBER_OF_SERVERS=$(($2+1))
+      shift 2
+      ;;
+    -N|--server-number)
+      CURRENT_SERVER_NUMBER=$2
+      shift 2
+      ;;
+    -P|--replicator-password)
+      REPLICATOR_PASSWORD=$2
+      shift 2
+      ;;
+    --docker)
+      DOCKER=true
+      shift
+      ;;
+    --)
+      shift
+      break
+      ;;
+    *)
+      usage
+      exit 1
+      ;;
+  esac
+done
+
+source "$CODE_PATH/extra/lib.sh"
+
+if [[ "$CURRENT_SERVER_NUMBER" -eq 1 ]] ; then
+
+#Create the replication user
+create_replication_user "root" "$P_ROOT"
+
+setup_db_replication "$NUMBER_OF_SERVERS" "$CURRENT_SERVER_NUMBER"
+
+  log "Restarting mysql to enable bin_log and the replication..."
+  sudo service mysql restart
+
+
+else
+
+ # Install git first
+ package git
+
+ # Are we just updating a running fbctf?
+ if [[ "$UPDATE" == true ]] ; then
+    update_repo "$MODE" "$CODE_PATH" "$CTF_PATH"
+    exit 0
+ fi
+
+ log "Provisioning in $MODE mode"
+ log "Using $TYPE certificate"
+ log "Source code folder $CODE_PATH"
+ log "Destination folder $CTF_PATH"
+
+ # We only create a new directory and rsync files over if it's different from the
+ # original code path
+ if [[ "$CODE_PATH" != "$CTF_PATH" ]]; then
+    log "Creating code folder $CTF_PATH"
+    [[ -d "$CTF_PATH" ]] || sudo mkdir -p "$CTF_PATH"
+
+    log "Copying all CTF code to destination folder"
+    sudo rsync -a --exclude node_modules --exclude vendor "$CODE_PATH/" "$CTF_PATH/"
+
+    # This is because sync'ing files is done with unison
+    if [[ "$MODE" == "dev" ]]; then
+        log "Configuring git to ignore permission changes"
+        git -C "$CTF_PATH/" config core.filemode false
+        log "Setting permissions"
+        sudo chmod -R 777 "$CTF_PATH/"
+    fi
+ fi
+
+ # Some Ubuntu distros don't come with curl installed
+ package curl
+
+ # We only run this once so provisioning is faster
+ sudo apt-get update
+
+ # Some people need this language pack installed or HHVM will report errors
+ package language-pack-en
+
+ # Packages to be installed in dev mode
+ if [[ "$MODE" == "dev" ]]; then
+    sudo apt-get install -y build-essential python-all-dev python-setuptools
+    package python-pip
+    sudo -H pip install --upgrade pip
+    sudo -H pip install mycli
+    package emacs
+    package htop
+ fi
+
+ # Install memcached
+ package memcached
+
+ # Install MySQL
+ install_mysql "$P_ROOT"
+
+ # Install HHVM
+ install_hhvm "$CTF_PATH" "$HHVM_CONFIG_PATH"
+
+ # Install Composer
+ install_composer "$CTF_PATH"
+ # This step has done `cd "$CTF_PATH"`
+ composer.phar install
+
+ # Database creation
+ import_empty_db "root" "$P_ROOT" "$DB" "$CTF_PATH" "$MODE"
+
+ #Create the replication user
+ create_replication_user "root" "$P_ROOT" "$DB" "$REPLICATOR_PASSWORD"
+
+ setup_db_replication "$NUMBER_OF_SERVERS" "$CURRENT_SERVER_NUMBER" "$REPLICATOR_PASSWORD"
+
+  log "Restarting mysql to enable bin_log and the replication..."
+  sudo service mysql restart
+
+fi
+  exit 0

src/Db.php

@@ -54,7 +54,16 @@ public function isConnected(): bool {
   }
 
   private async function genConnect(): Awaitable<void> {
-    $host = must_have_idx($this->config, 'DB_HOST');
+    if (array_key_exists('DB_SLAVE', $this->config)) {
+      if (random_int(1,2) == 1){
+        $host = must_have_idx($this->config, 'DB_MASTER');
+      } else {
+        $host = must_have_idx($this->config, 'DB_SLAVE');
+      }
+    } else {
+      $host = must_have_idx($this->config, 'DB_HOST');
+    }
+
     $port = must_have_idx($this->config, 'DB_PORT');
     $db_name = must_have_idx($this->config, 'DB_NAME');
     $username = must_have_idx($this->config, 'DB_USERNAME');