Upgrade from Ubuntu 14.04 (Trusty) to Ubuntu 16.04 (Xenial)

Dockerfile

@@ -1,5 +1,4 @@
-FROM ubuntu:trusty
-LABEL maintainer="Boik Su <boik@tdohacker.org>"
+FROM ubuntu:xenial
 
 ENV HOME /root
 
@@ -10,9 +9,11 @@ ARG TYPE=self
 ARG KEY
 ARG CRT
 
+ENV HHVM_DISABLE_NUMA true
+
 WORKDIR $HOME
 COPY . $HOME
-RUN chown www-data:www-data $HOME
 
+RUN apt-get update && apt-get -y install sudo apt-utils
 RUN ./extra/provision.sh -m $MODE -c $TYPE -k $KEY -C $CRT -D $DOMAIN -e $EMAIL -s `pwd` --docker
 CMD ["./extra/service_startup.sh"]

Vagrantfile

@@ -4,7 +4,7 @@
 VAGRANTFILE_API_VERSION = "2"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "ubuntu/xenial64"
   config.vm.network "private_network", ip: "10.10.10.5"
   config.vm.hostname = "FacebookCTF-Dev"
   config.ssh.shell = "bash -c 'BASH_ENV=/etc/profile exec bash'"

Vagrantfile-multi

@@ -4,7 +4,7 @@
 VAGRANTFILE_API_VERSION = "2"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "ubuntu/xenial64"
   config.ssh.shell = "bash -c 'BASH_ENV=/etc/profile exec bash'"
 
   # MySQL Server

Vagrantfile-single

@@ -4,7 +4,7 @@
 VAGRANTFILE_API_VERSION = "2"
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "ubuntu/xenial64"
   config.vm.network "private_network", ip: "10.10.10.5"
   config.vm.hostname = "facebookCTF-Dev"
   config.ssh.shell = "bash -c 'BASH_ENV=/etc/profile exec bash'"

extra/cache/Dockerfile

@@ -1,5 +1,4 @@
-FROM ubuntu:trusty
-LABEL maintainer="Boik Su <boik@tdohacker.org>"
+FROM ubuntu:xenial
 
 ENV HOME /root
 
@@ -13,5 +12,6 @@ ARG CRT
 WORKDIR $HOME
 COPY . $HOME
 
+RUN apt-get update && apt-get -y install sudo apt-utils
 RUN ./extra/provision.sh -m $MODE -c $TYPE -k $KEY -C $CRT -D $DOMAIN -e $EMAIL -s `pwd` --docker --multiple-servers --server-type cache
 CMD ["./extra/cache/cache_startup.sh"]

extra/hhvm/Dockerfile

@@ -1,5 +1,4 @@
-FROM ubuntu:trusty
-LABEL maintainer="Boik Su <boik@tdohacker.org>"
+FROM ubuntu:xenial
 
 ENV HOME /root
 
@@ -10,8 +9,11 @@ ARG TYPE=self
 ARG KEY
 ARG CRT
 
+ENV HHVM_DISABLE_NUMA true
+
 WORKDIR $HOME
 COPY . $HOME
 
+RUN apt-get update && apt-get -y install sudo apt-utils
 RUN ./extra/provision.sh -m $MODE -c $TYPE -k $KEY -C $CRT -D $DOMAIN -e $EMAIL -s `pwd` --docker --multiple-servers --server-type hhvm --mysql-server mysql --cache-server cache
 CMD ["./extra/hhvm/hhvm_startup.sh"]

extra/hhvm/hhvm_startup.sh

@@ -2,7 +2,8 @@
 
 set -e
 
-service hhvm restart
+chown -R www-data:www-data /var/www/fbctf
+sudo -u www-data service hhvm restart
 
 while true; do
     if [[ -e /var/run/hhvm/sock ]]; then

extra/lib.sh

@@ -59,7 +59,7 @@ function install_unison() {
 function repo_osquery() {
   log "Adding osquery repository keys"
   sudo DEBIAN_FRONTEND=noninteractive apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1484120AC4E9F8A1A577AEEE97A80C63C9D8B80B
-  sudo DEBIAN_FRONTEND=noninteractive add-apt-repository "deb [arch=amd64] https://osquery-packages.s3.amazonaws.com/trusty trusty main"
+  sudo DEBIAN_FRONTEND=noninteractive add-apt-repository "deb [arch=amd64] https://pkg.osquery.io/deb deb main"
 }
 
 function install_mysql() {
@@ -229,16 +229,14 @@ function install_hhvm() {
 
   package software-properties-common
 
-  log "Adding HHVM key"
+  log "Adding HHVM keys"
   sudo DEBIAN_FRONTEND=noninteractive apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0x5a16e7281be7a449
+  sudo DEBIAN_FRONTEND=noninteractive apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xB4112585D386EB94
 
   log "Adding HHVM repo"
-  sudo DEBIAN_FRONTEND=noninteractive add-apt-repository "deb http://dl.hhvm.com/ubuntu $(lsb_release -sc) main"
+  sudo DEBIAN_FRONTEND=noninteractive add-apt-repository "deb http://dl.hhvm.com/ubuntu xenial-lts-3.21 main"
 
   package_repo_update
-
-  log "Installing HHVM"
-  # Installing the package so the dependencies are installed too
   package hhvm
 
   log "Enabling HHVM to start by default"
@@ -274,7 +272,7 @@ function hhvm_performance() {
   cat "$__config" | sed "s|$__oldrepo|$__repofile|g" | sudo tee "$__config"
   sudo hhvm-repo-mode enable "$__path"
   sudo chown www-data:www-data "$__repofile"
-  sudo service hhvm start
+  sudo service hhvm restart
 }
 
 function install_composer() {
@@ -288,9 +286,6 @@ function install_composer() {
 }
 
 function install_nodejs() {
-  log "Removing node.js legacy version"
-  sudo DEBIAN_FRONTEND=noninteractive apt-get remove --purge nodejs -y
-
   log "Downloading and setting node.js version 6.x repo information"
   dl_pipe "https://deb.nodesource.com/setup_6.x" | sudo -E bash -
 

extra/mysql/Dockerfile

@@ -1,5 +1,4 @@
-FROM ubuntu:trusty
-LABEL maintainer="Boik Su <boik@tdohacker.org>"
+FROM ubuntu:xenial
 
 ENV HOME /root
 
@@ -13,5 +12,6 @@ ARG CRT
 WORKDIR $HOME
 COPY . $HOME
 
+RUN apt-get update && apt-get -y install sudo apt-utils
 RUN ./extra/provision.sh -m $MODE -c $TYPE -k $KEY -C $CRT -D $DOMAIN -e $EMAIL -s `pwd` --docker --multiple-servers --server-type mysql
 CMD ["./extra/mysql/mysql_startup.sh"]

extra/mysql/mysql_startup.sh

@@ -2,6 +2,10 @@
 
 set -e
 
+chown -R mysql:mysql /var/lib/mysql
+chown -R mysql:mysql /var/run/mysqld
+chown -R mysql:mysql /var/log/mysql
+
 service mysql restart
 
 while true; do

extra/nginx/Dockerfile

@@ -1,5 +1,4 @@
-FROM ubuntu:trusty
-LABEL maintainer="Boik Su <boik@tdohacker.org>"
+FROM ubuntu:xenial
 
 ENV HOME /root
 
@@ -12,7 +11,7 @@ ARG CRT
 
 WORKDIR $HOME
 COPY . $HOME
-RUN chown www-data:www-data $HOME
 
+RUN apt-get update && apt-get -y install sudo apt-utils
 RUN ./extra/provision.sh -m $MODE -c $TYPE -k $KEY -C $CRT -D $DOMAIN -e $EMAIL -s `pwd` --docker --multiple-servers --server-type nginx --hhvm-server hhvm
 CMD ["./extra/nginx/nginx_startup.sh"]

extra/nginx/nginx_startup.sh

@@ -6,6 +6,7 @@ if [[ -e /root/tmp/certbot.sh ]]; then
     /bin/bash /root/tmp/certbot.sh
 fi
 
+chown -R www-data:www-data /var/www/fbctf
 service nginx restart
 
 while true; do

extra/provision.sh

@@ -244,7 +244,6 @@ fi
 
 # If multiple servers are being utilized, ensure provision was called from the "nginx" or "hhvm" servers
     if [[ "$MULTIPLE_SERVERS" == false || "$SERVER_TYPE" = "nginx" || $SERVER_TYPE = "hhvm" ]]; then
-    package language-pack-en
 
     if [[ "$UPDATE" == true ]] ; then
         log "Updating repo"
@@ -265,7 +264,6 @@ fi
         log "Installing HHVM"
         install_hhvm "$CTF_PATH" "$HHVM_CONFIG_PATH" "$MULTIPLE_SERVERS"
 
-        # Install Composer
         log "Installing Composer"
         install_composer "$CTF_PATH"
         log "Installing Composer in /usr/bin"
@@ -359,10 +357,10 @@ if [[ "$MULTIPLE_SERVERS" == false || "$SERVER_TYPE" = "mysql" ]]; then
     # Configuration for MySQL
     if [[ "$MULTIPLE_SERVERS" == true ]] && [[ "$SERVER_TYPE" = "mysql" ]]; then
         # This is required in order to generate password hash (since HHVM is not being installed)
-        package php5-cli
+        package php7.0-cli
 
-        sudo sed -e '/^bind-address/ s/^#*/#/' -i /etc/mysql/my.cnf
-        sudo sed -e '/^skip-external-locking/ s/^#*/#/' -i /etc/mysql/my.cnf
+        sudo sed -e '/^bind-address/ s/^#*/#/' -i /etc/mysql/mysql.conf.d/mysqld.cnf
+        sudo sed -e '/^skip-external-locking/ s/^#*/#/' -i /etc/mysql/mysql.conf.d/mysqld.cnf
 	fi
 
     # Database creation

extra/service_startup.sh

@@ -10,7 +10,12 @@ if [[ -e /var/run/hhvm/sock ]]; then
     rm -f /var/run/hhvm/sock
 fi
 
-service hhvm restart
+chown -R mysql:mysql /var/lib/mysql
+chown -R mysql:mysql /var/run/mysqld
+chown -R mysql:mysql /var/log/mysql
+chown -R www-data:www-data /var/www/fbctf
+
+sudo -u www-data service hhvm restart
 service nginx restart
 service mysql restart
 service memcached restart

src/models/Progressive.php

@@ -71,7 +71,7 @@ private static function progressiveFromRow(
       $progressive = array();
       $result =
         await $db->queryf(
-          'SELECT * FROM progressive_log GROUP BY team_name, iteration ORDER BY points ASC',
+          'SELECT MAX(id) as id, MAX(ts) as ts, team_name, MAX(points) as points, iteration FROM progressive_log GROUP BY team_name, iteration, id ORDER BY points ASC',
         );
       foreach ($result->mapRows() as $row) {
         $progressive[$row->get('team_name')][] =

src/models/Session.php

@@ -439,7 +439,13 @@ private static function sessionFromRow(Map<string, string> $row): Session {
       $sessions = array();
       $cached_sessions = array();
       /* HH_IGNORE_ERROR[4053]: HHVM doesn't beleive there is a getAllKeys() method, there is... */
-      $mc_keys = $mc->getAllKeys();
+      //$mc_keys = $mc->getAllKeys();
+      /* Memcached::getAllKeys() is not working in HHVM 3.21 - For now we will flush Memcache in place of the call.
+       *  Flushing the cache will be slower and will negatively impact performance.
+       *  As soon as getAllKeys() regains support in HHVM, or the functionality it replaced, this should be updated.
+       */
+      $mc_keys = array();
+      self::flushMCCluster();
       $all_sessions = preg_grep(
         '/'.self::$MC_KEY.self::$MC_KEYS->get('SESSIONS').'/',
         $mc_keys,
@@ -492,7 +498,13 @@ public static function invalidateMCSessions(?string $key = null): void {
     $key = str_replace(' ', '', $key);
     if ($key === null) {
       /* HH_IGNORE_ERROR[4053]: HHVM doesn't beleive there is a getAllKeys() method, there is... */
-      $mc_keys = $mc->getAllKeys();
+      //$mc_keys = $mc->getAllKeys();
+      /* Memcached::getAllKeys() is not working in HHVM 3.21 - For now we will flush Memcache in place of the call.
+       *  Flushing the cache will be slower and will negatively impact performance.
+       *  As soon as getAllKeys() regains support in HHVM, or the functionality it replaced, this should be updated.
+       */
+      $mc_keys = array();
+      self::flushMCCluster();
       $all_sessions = preg_grep(
         '/'.self::$MC_KEY.self::$MC_KEYS->get('SESSIONS').'/',
         $mc_keys,

tests/_files/seed.xml

@@ -278,13 +278,15 @@
     <column>team_id</column>
     <column>created_ts</column>
     <column>last_access_ts</column>
+    <column>last_page_access</column>
     <row>
       <value>1</value>
       <value>cookie</value>
       <value>data</value>
       <value>1</value>
       <value>2015-04-24 17:15:23</value>
       <value>2016-04-24 17:15:23</value>
+      <value>index</value>
     </row>
   </table>
   <table name="registration_tokens">