Run tests for different connectors in different builds

This allows us to avoid spinning up all the backends (hadoop, mysql,
cassandra, and postgres) to reduce memory usage.

.travis.yml

@@ -73,7 +73,7 @@ script:
   - |
     if [[ -v PRODUCT_TESTS_BASIC_ENVIRONMENT ]]; then
       presto-product-tests/bin/run_on_docker.sh \
-        multinode -x quarantine,big_query,storage_formats,profile_specific_tests,tpcds
+        multinode -x quarantine,big_query,storage_formats,profile_specific_tests,tpcds,cassandra,mysql_connector,postgresql_connector,mysql
     fi
   - |
     if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then
@@ -98,6 +98,21 @@ script:
       presto-product-tests/bin/run_on_docker.sh \
         multinode-tls -g smoke,cli,group-by,join,tls
     fi
+  - |
+    if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then
+      presto-product-tests/bin/run_on_docker.sh \
+        singlenode-mysql -g mysql_connector,mysql
+    fi
+  - |
+    if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then
+      presto-product-tests/bin/run_on_docker.sh \
+        singlenode-postgresql -g postgresql_connector
+    fi
+  - |
+    if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then
+      presto-product-tests/bin/run_on_docker.sh \
+        singlenode-cassandra -g cassandra
+    fi
   - |
     if [[ -v HIVE_TESTS ]]; then
       presto-hive-hadoop2/bin/run_hive_tests.sh

presto-product-tests/README.md

@@ -119,7 +119,7 @@ and deficiencies of `extends:` syntax (see the note
 To ease the pain of passing multiple `-f` arguments to `docker-compose`,
 each environment has a `compose.sh` wrapper script. Thanks to it, instead of e.g.
 
-`docker-compose -f ./docker-compose.yml -f ../common/standard.yml -f ../common/jdbc_db.yml [compose commands]`
+`docker-compose -f ./docker-compose.yml -f ../common/standard.yml [compose commands]`
 
 one can simply write
 

presto-product-tests/bin/run_on_docker.sh

@@ -142,12 +142,16 @@ fi
 # catch terminate signals
 trap terminate INT TERM EXIT
 
-if [[ "$ENVIRONMENT" == "singlenode" || "$ENVIRONMENT" == "multinode" ]]; then
-  EXTERNAL_SERVICES="hadoop-master mysql postgres cassandra"
-elif [[ "$ENVIRONMENT" == "singlenode-sqlserver" ]]; then
+if [[ "$ENVIRONMENT" == "singlenode-sqlserver" ]]; then
   EXTERNAL_SERVICES="hadoop-master sqlserver"
 elif [[ "$ENVIRONMENT" == "singlenode-ldap" ]]; then
   EXTERNAL_SERVICES="hadoop-master ldapserver"
+elif [[ "$ENVIRONMENT" == "singlenode-mysql" ]]; then
+  EXTERNAL_SERVICES="hadoop-master mysql"
+elif [[ "$ENVIRONMENT" == "singlenode-postgresql" ]]; then
+  EXTERNAL_SERVICES="hadoop-master postgres"
+elif [[ "$ENVIRONMENT" == "singlenode-cassandra" ]]; then
+  EXTERNAL_SERVICES="hadoop-master cassandra"
 else
   EXTERNAL_SERVICES="hadoop-master"
 fi

presto-product-tests/conf/docker/multinode-tls/compose.sh

@@ -8,6 +8,5 @@ source "${SCRIPT_DIRECTORY}/../common/compose-commons.sh"
 
 docker-compose \
     -f ${SCRIPT_DIRECTORY}/../common/standard.yml \
-    -f ${SCRIPT_DIRECTORY}/../common/jdbc_db.yml \
     -f ${SCRIPT_DIRECTORY}/docker-compose.yml \
     "$@"

presto-product-tests/conf/docker/multinode/compose.sh

@@ -6,7 +6,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh"
 
 docker-compose \
     -f ${BASH_SOURCE%/*}/../common/standard.yml \
-    -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \
-    -f ${BASH_SOURCE%/*}/../common/cassandra.yml \
     -f ${BASH_SOURCE%/*}/docker-compose.yml \
     "$@"

presto-product-tests/conf/docker/singlenode-cassandra/compose.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "${BASH_SOURCE%/*}/../common/compose-commons.sh"
+
+docker-compose \
+    -f ${BASH_SOURCE%/*}/../common/standard.yml \
+    -f ${BASH_SOURCE%/*}/docker-compose.yml \
+    "$@"

presto-product-tests/conf/docker/singlenode-cassandra/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '2'
+services:
+
+  cassandra:
+      hostname: cassandra
+      image: 'cassandra:3.9'
+      ports:
+        - '9042:9042'
+      environment:
+          - HEAP_NEWSIZE=128M
+          - MAX_HEAP_SIZE=512M

presto-product-tests/conf/docker/singlenode-hdfs-impersonation/compose.sh

@@ -6,6 +6,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh"
 
 docker-compose \
     -f ${BASH_SOURCE%/*}/../common/standard.yml \
-    -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \
     -f ${BASH_SOURCE%/*}/docker-compose.yml \
     "$@"

presto-product-tests/conf/docker/singlenode-kerberos-hdfs-impersonation/compose.sh

@@ -7,6 +7,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh"
 docker-compose \
     -f ${BASH_SOURCE%/*}/../common/standard.yml \
     -f ${BASH_SOURCE%/*}/../common/kerberos.yml \
-    -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \
     -f ${BASH_SOURCE%/*}/docker-compose.yml \
     "$@"

presto-product-tests/conf/docker/singlenode-kerberos-hdfs-no-impersonation/compose.sh

@@ -7,6 +7,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh"
 docker-compose \
     -f ${BASH_SOURCE%/*}/../common/standard.yml \
     -f ${BASH_SOURCE%/*}/../common/kerberos.yml \
-    -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \
     -f ${BASH_SOURCE%/*}/docker-compose.yml \
     "$@"

presto-product-tests/conf/docker/singlenode-ldap/compose.sh

@@ -6,6 +6,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh"
 
 docker-compose \
     -f ${BASH_SOURCE%/*}/../common/standard.yml \
-    -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \
     -f ${BASH_SOURCE%/*}/docker-compose.yml \
     "$@"

presto-product-tests/conf/docker/singlenode-mysql/compose.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "${BASH_SOURCE%/*}/../common/compose-commons.sh"
+
+docker-compose \
+    -f ${BASH_SOURCE%/*}/../common/standard.yml \
+    -f ${BASH_SOURCE%/*}/docker-compose.yml \
+    "$@"

presto-product-tests/conf/docker/common/jdbc_db.yml → presto-product-tests/conf/docker/singlenode-mysql/docker-compose.yml

@@ -1,17 +1,6 @@
 version: '2'
 services:
 
-  postgres:
-    hostname: postgres
-    image: 'postgres'
-    ports:
-      - '15432:15432'
-    environment:
-      POSTGRES_PASSWORD: swarm
-      POSTGRES_USER: swarm
-      POSTGRES_DB: test
-      PGPORT: 15432
-
   mysql:
     hostname: mysql
     image: 'mysql'

presto-product-tests/conf/docker/singlenode-postgresql/compose.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "${BASH_SOURCE%/*}/../common/compose-commons.sh"
+
+docker-compose \
+    -f ${BASH_SOURCE%/*}/../common/standard.yml \
+    -f ${BASH_SOURCE%/*}/docker-compose.yml \
+    "$@"

presto-product-tests/conf/docker/singlenode-postgresql/docker-compose.yml

@@ -0,0 +1,13 @@
+version: '2'
+services:
+
+  postgres:
+    hostname: postgres
+    image: 'postgres'
+    ports:
+      - '15432:15432'
+    environment:
+      POSTGRES_PASSWORD: swarm
+      POSTGRES_USER: swarm
+      POSTGRES_DB: test
+      PGPORT: 15432

presto-product-tests/conf/docker/singlenode-sqlserver/compose.sh

@@ -6,6 +6,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh"
 
 docker-compose \
     -f ${BASH_SOURCE%/*}/../common/standard.yml \
-    -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \
     -f ${BASH_SOURCE%/*}/docker-compose.yml \
     "$@"

presto-product-tests/conf/docker/singlenode/compose.sh

@@ -6,7 +6,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh"
 
 docker-compose \
     -f ${BASH_SOURCE%/*}/../common/standard.yml \
-    -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \
-    -f ${BASH_SOURCE%/*}/../common/cassandra.yml \
     -f ${BASH_SOURCE%/*}/docker-compose.yml \
     "$@"