From 35a17a015936eada51205e715d60c8955ce0a884 Mon Sep 17 00:00:00 2001 From: Andrii Rosa Date: Wed, 18 Apr 2018 13:37:44 -0400 Subject: [PATCH] Run tests for different connectors in different builds This allows us to avoid spinning up all the backends (hadoop, mysql, cassandra, and postgres) to reduce memory usage. --- .travis.yml | 17 ++++++++++++++++- presto-product-tests/README.md | 2 +- presto-product-tests/bin/run_on_docker.sh | 10 +++++++--- .../conf/docker/common/cassandra.yml | 11 ----------- .../conf/docker/multinode-tls/compose.sh | 1 - .../conf/docker/multinode/compose.sh | 2 -- .../conf/docker/singlenode-cassandra/compose.sh | 10 ++++++++++ .../singlenode-cassandra/docker-compose.yml | 11 +++++++++++ .../singlenode-hdfs-impersonation/compose.sh | 1 - .../compose.sh | 1 - .../compose.sh | 1 - .../conf/docker/singlenode-ldap/compose.sh | 1 - .../conf/docker/singlenode-mysql/compose.sh | 10 ++++++++++ .../docker-compose.yml} | 11 ----------- .../docker/singlenode-postgresql/compose.sh | 10 ++++++++++ .../singlenode-postgresql/docker-compose.yml | 13 +++++++++++++ .../conf/docker/singlenode-sqlserver/compose.sh | 1 - .../conf/docker/singlenode/compose.sh | 2 -- 18 files changed, 78 insertions(+), 37 deletions(-) delete mode 100644 presto-product-tests/conf/docker/common/cassandra.yml create mode 100755 presto-product-tests/conf/docker/singlenode-cassandra/compose.sh create mode 100644 presto-product-tests/conf/docker/singlenode-cassandra/docker-compose.yml create mode 100755 presto-product-tests/conf/docker/singlenode-mysql/compose.sh rename presto-product-tests/conf/docker/{common/jdbc_db.yml => singlenode-mysql/docker-compose.yml} (56%) create mode 100755 presto-product-tests/conf/docker/singlenode-postgresql/compose.sh create mode 100644 presto-product-tests/conf/docker/singlenode-postgresql/docker-compose.yml diff --git a/.travis.yml b/.travis.yml index 4ee0e32f9abb0..d2012ea7bf88b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,7 +73,7 @@ script: - | if [[ -v PRODUCT_TESTS_BASIC_ENVIRONMENT ]]; then presto-product-tests/bin/run_on_docker.sh \ - multinode -x quarantine,big_query,storage_formats,profile_specific_tests,tpcds + multinode -x quarantine,big_query,storage_formats,profile_specific_tests,tpcds,cassandra,mysql_connector,postgresql_connector,mysql fi - | if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then @@ -98,6 +98,21 @@ script: presto-product-tests/bin/run_on_docker.sh \ multinode-tls -g smoke,cli,group-by,join,tls fi + - | + if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then + presto-product-tests/bin/run_on_docker.sh \ + singlenode-mysql -g mysql_connector,mysql + fi + - | + if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then + presto-product-tests/bin/run_on_docker.sh \ + singlenode-postgresql -g postgresql_connector + fi + - | + if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT ]]; then + presto-product-tests/bin/run_on_docker.sh \ + singlenode-cassandra -g cassandra + fi - | if [[ -v HIVE_TESTS ]]; then presto-hive-hadoop2/bin/run_hive_tests.sh diff --git a/presto-product-tests/README.md b/presto-product-tests/README.md index 2b54aa67e4112..f432434bfb47b 100644 --- a/presto-product-tests/README.md +++ b/presto-product-tests/README.md @@ -119,7 +119,7 @@ and deficiencies of `extends:` syntax (see the note To ease the pain of passing multiple `-f` arguments to `docker-compose`, each environment has a `compose.sh` wrapper script. Thanks to it, instead of e.g. -`docker-compose -f ./docker-compose.yml -f ../common/standard.yml -f ../common/jdbc_db.yml [compose commands]` +`docker-compose -f ./docker-compose.yml -f ../common/standard.yml [compose commands]` one can simply write diff --git a/presto-product-tests/bin/run_on_docker.sh b/presto-product-tests/bin/run_on_docker.sh index f50ee771ae9e4..27cce3c7b9b23 100755 --- a/presto-product-tests/bin/run_on_docker.sh +++ b/presto-product-tests/bin/run_on_docker.sh @@ -142,12 +142,16 @@ fi # catch terminate signals trap terminate INT TERM EXIT -if [[ "$ENVIRONMENT" == "singlenode" || "$ENVIRONMENT" == "multinode" ]]; then - EXTERNAL_SERVICES="hadoop-master mysql postgres cassandra" -elif [[ "$ENVIRONMENT" == "singlenode-sqlserver" ]]; then +if [[ "$ENVIRONMENT" == "singlenode-sqlserver" ]]; then EXTERNAL_SERVICES="hadoop-master sqlserver" elif [[ "$ENVIRONMENT" == "singlenode-ldap" ]]; then EXTERNAL_SERVICES="hadoop-master ldapserver" +elif [[ "$ENVIRONMENT" == "singlenode-mysql" ]]; then + EXTERNAL_SERVICES="hadoop-master mysql" +elif [[ "$ENVIRONMENT" == "singlenode-postgresql" ]]; then + EXTERNAL_SERVICES="hadoop-master postgres" +elif [[ "$ENVIRONMENT" == "singlenode-cassandra" ]]; then + EXTERNAL_SERVICES="hadoop-master cassandra" else EXTERNAL_SERVICES="hadoop-master" fi diff --git a/presto-product-tests/conf/docker/common/cassandra.yml b/presto-product-tests/conf/docker/common/cassandra.yml deleted file mode 100644 index a946793e2f20d..0000000000000 --- a/presto-product-tests/conf/docker/common/cassandra.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: '2' -services: - - cassandra: - hostname: cassandra - image: 'cassandra:3.9' - ports: - - '9042:9042' - environment: - - HEAP_NEWSIZE=128M - - MAX_HEAP_SIZE=512M diff --git a/presto-product-tests/conf/docker/multinode-tls/compose.sh b/presto-product-tests/conf/docker/multinode-tls/compose.sh index 267bfce53d83f..af7fbac71667c 100755 --- a/presto-product-tests/conf/docker/multinode-tls/compose.sh +++ b/presto-product-tests/conf/docker/multinode-tls/compose.sh @@ -8,6 +8,5 @@ source "${SCRIPT_DIRECTORY}/../common/compose-commons.sh" docker-compose \ -f ${SCRIPT_DIRECTORY}/../common/standard.yml \ - -f ${SCRIPT_DIRECTORY}/../common/jdbc_db.yml \ -f ${SCRIPT_DIRECTORY}/docker-compose.yml \ "$@" diff --git a/presto-product-tests/conf/docker/multinode/compose.sh b/presto-product-tests/conf/docker/multinode/compose.sh index 40ef1ab5c1b0b..9d94d229f79eb 100755 --- a/presto-product-tests/conf/docker/multinode/compose.sh +++ b/presto-product-tests/conf/docker/multinode/compose.sh @@ -6,7 +6,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh" docker-compose \ -f ${BASH_SOURCE%/*}/../common/standard.yml \ - -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \ - -f ${BASH_SOURCE%/*}/../common/cassandra.yml \ -f ${BASH_SOURCE%/*}/docker-compose.yml \ "$@" diff --git a/presto-product-tests/conf/docker/singlenode-cassandra/compose.sh b/presto-product-tests/conf/docker/singlenode-cassandra/compose.sh new file mode 100755 index 0000000000000..9d94d229f79eb --- /dev/null +++ b/presto-product-tests/conf/docker/singlenode-cassandra/compose.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "${BASH_SOURCE%/*}/../common/compose-commons.sh" + +docker-compose \ + -f ${BASH_SOURCE%/*}/../common/standard.yml \ + -f ${BASH_SOURCE%/*}/docker-compose.yml \ + "$@" diff --git a/presto-product-tests/conf/docker/singlenode-cassandra/docker-compose.yml b/presto-product-tests/conf/docker/singlenode-cassandra/docker-compose.yml new file mode 100644 index 0000000000000..6a75d3c0a3c95 --- /dev/null +++ b/presto-product-tests/conf/docker/singlenode-cassandra/docker-compose.yml @@ -0,0 +1,11 @@ +version: '2' +services: + + cassandra: + hostname: cassandra + image: 'cassandra:3.9' + ports: + - '9042:9042' + environment: + - HEAP_NEWSIZE=128M + - MAX_HEAP_SIZE=512M diff --git a/presto-product-tests/conf/docker/singlenode-hdfs-impersonation/compose.sh b/presto-product-tests/conf/docker/singlenode-hdfs-impersonation/compose.sh index f9053af4ed538..9d94d229f79eb 100755 --- a/presto-product-tests/conf/docker/singlenode-hdfs-impersonation/compose.sh +++ b/presto-product-tests/conf/docker/singlenode-hdfs-impersonation/compose.sh @@ -6,6 +6,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh" docker-compose \ -f ${BASH_SOURCE%/*}/../common/standard.yml \ - -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \ -f ${BASH_SOURCE%/*}/docker-compose.yml \ "$@" diff --git a/presto-product-tests/conf/docker/singlenode-kerberos-hdfs-impersonation/compose.sh b/presto-product-tests/conf/docker/singlenode-kerberos-hdfs-impersonation/compose.sh index c52bf88313285..ded20a6688252 100755 --- a/presto-product-tests/conf/docker/singlenode-kerberos-hdfs-impersonation/compose.sh +++ b/presto-product-tests/conf/docker/singlenode-kerberos-hdfs-impersonation/compose.sh @@ -7,6 +7,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh" docker-compose \ -f ${BASH_SOURCE%/*}/../common/standard.yml \ -f ${BASH_SOURCE%/*}/../common/kerberos.yml \ - -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \ -f ${BASH_SOURCE%/*}/docker-compose.yml \ "$@" diff --git a/presto-product-tests/conf/docker/singlenode-kerberos-hdfs-no-impersonation/compose.sh b/presto-product-tests/conf/docker/singlenode-kerberos-hdfs-no-impersonation/compose.sh index c52bf88313285..ded20a6688252 100755 --- a/presto-product-tests/conf/docker/singlenode-kerberos-hdfs-no-impersonation/compose.sh +++ b/presto-product-tests/conf/docker/singlenode-kerberos-hdfs-no-impersonation/compose.sh @@ -7,6 +7,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh" docker-compose \ -f ${BASH_SOURCE%/*}/../common/standard.yml \ -f ${BASH_SOURCE%/*}/../common/kerberos.yml \ - -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \ -f ${BASH_SOURCE%/*}/docker-compose.yml \ "$@" diff --git a/presto-product-tests/conf/docker/singlenode-ldap/compose.sh b/presto-product-tests/conf/docker/singlenode-ldap/compose.sh index f9053af4ed538..9d94d229f79eb 100755 --- a/presto-product-tests/conf/docker/singlenode-ldap/compose.sh +++ b/presto-product-tests/conf/docker/singlenode-ldap/compose.sh @@ -6,6 +6,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh" docker-compose \ -f ${BASH_SOURCE%/*}/../common/standard.yml \ - -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \ -f ${BASH_SOURCE%/*}/docker-compose.yml \ "$@" diff --git a/presto-product-tests/conf/docker/singlenode-mysql/compose.sh b/presto-product-tests/conf/docker/singlenode-mysql/compose.sh new file mode 100755 index 0000000000000..9d94d229f79eb --- /dev/null +++ b/presto-product-tests/conf/docker/singlenode-mysql/compose.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "${BASH_SOURCE%/*}/../common/compose-commons.sh" + +docker-compose \ + -f ${BASH_SOURCE%/*}/../common/standard.yml \ + -f ${BASH_SOURCE%/*}/docker-compose.yml \ + "$@" diff --git a/presto-product-tests/conf/docker/common/jdbc_db.yml b/presto-product-tests/conf/docker/singlenode-mysql/docker-compose.yml similarity index 56% rename from presto-product-tests/conf/docker/common/jdbc_db.yml rename to presto-product-tests/conf/docker/singlenode-mysql/docker-compose.yml index 2e63da0070526..0a412fc534ea5 100644 --- a/presto-product-tests/conf/docker/common/jdbc_db.yml +++ b/presto-product-tests/conf/docker/singlenode-mysql/docker-compose.yml @@ -1,17 +1,6 @@ version: '2' services: - postgres: - hostname: postgres - image: 'postgres' - ports: - - '15432:15432' - environment: - POSTGRES_PASSWORD: swarm - POSTGRES_USER: swarm - POSTGRES_DB: test - PGPORT: 15432 - mysql: hostname: mysql image: 'mysql' diff --git a/presto-product-tests/conf/docker/singlenode-postgresql/compose.sh b/presto-product-tests/conf/docker/singlenode-postgresql/compose.sh new file mode 100755 index 0000000000000..9d94d229f79eb --- /dev/null +++ b/presto-product-tests/conf/docker/singlenode-postgresql/compose.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source "${BASH_SOURCE%/*}/../common/compose-commons.sh" + +docker-compose \ + -f ${BASH_SOURCE%/*}/../common/standard.yml \ + -f ${BASH_SOURCE%/*}/docker-compose.yml \ + "$@" diff --git a/presto-product-tests/conf/docker/singlenode-postgresql/docker-compose.yml b/presto-product-tests/conf/docker/singlenode-postgresql/docker-compose.yml new file mode 100644 index 0000000000000..a41e3797351c4 --- /dev/null +++ b/presto-product-tests/conf/docker/singlenode-postgresql/docker-compose.yml @@ -0,0 +1,13 @@ +version: '2' +services: + + postgres: + hostname: postgres + image: 'postgres' + ports: + - '15432:15432' + environment: + POSTGRES_PASSWORD: swarm + POSTGRES_USER: swarm + POSTGRES_DB: test + PGPORT: 15432 diff --git a/presto-product-tests/conf/docker/singlenode-sqlserver/compose.sh b/presto-product-tests/conf/docker/singlenode-sqlserver/compose.sh index f9053af4ed538..9d94d229f79eb 100755 --- a/presto-product-tests/conf/docker/singlenode-sqlserver/compose.sh +++ b/presto-product-tests/conf/docker/singlenode-sqlserver/compose.sh @@ -6,6 +6,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh" docker-compose \ -f ${BASH_SOURCE%/*}/../common/standard.yml \ - -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \ -f ${BASH_SOURCE%/*}/docker-compose.yml \ "$@" diff --git a/presto-product-tests/conf/docker/singlenode/compose.sh b/presto-product-tests/conf/docker/singlenode/compose.sh index 40ef1ab5c1b0b..9d94d229f79eb 100755 --- a/presto-product-tests/conf/docker/singlenode/compose.sh +++ b/presto-product-tests/conf/docker/singlenode/compose.sh @@ -6,7 +6,5 @@ source "${BASH_SOURCE%/*}/../common/compose-commons.sh" docker-compose \ -f ${BASH_SOURCE%/*}/../common/standard.yml \ - -f ${BASH_SOURCE%/*}/../common/jdbc_db.yml \ - -f ${BASH_SOURCE%/*}/../common/cassandra.yml \ -f ${BASH_SOURCE%/*}/docker-compose.yml \ "$@"