Test: Add a container to run the tests (#479)

.gitignore

@@ -12,6 +12,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 env/
+venv/
 bin/
 build/
 develop-eggs/

TEST.rst

@@ -2,12 +2,43 @@
 Tests
 =====
 
+Test Container
+==============
+
+Instead of installing the necessary requirements onto your host machine, the `test_container/` directory contains
+instructions for building a docker image with all the necessary requirements for running the tests across all
+supported python versions. When you are finished the container and associated data can be removed.
+
+Install and run the container::
+
+    $ ./test_container/build.sh
+    $ ./test_container/run.sh
+
+Run the tests inside the container::
+
+    # tox --workdir /output -vv
+
+Remove the container and associated data::
+
+    $ sudo rm -rf test_container/output
+    $ docker image rm geoalchemy2
+    $ docker system prune
+
+
+Host System
+===========
+
+If you have a Linux system that you want to run the tests on instead of the test container, follow these steps:
+
+
 Install system dependencies
-===========================
+---------------------------
+
+(instructions for Ubuntu 22.04)
 
 Install PostgreSQL and PostGIS::
 
-    $ sudo apt-get install postgresql postgis
+    $ sudo apt-get install postgresql postgresql-14-postgis-3 postgresql-14-postgis-3-scripts
 
 Install the Python and PostgreSQL development packages::
 
@@ -29,7 +60,7 @@ Install the Python dependencies::
 Or you can use the Conda environment provided in the `GeoAlchemy2_dev.yml` file.
 
 Set up the PostGIS database
-===========================
+---------------------------
 
 Create the ``gis`` role::
 
@@ -51,7 +82,7 @@ With PostGIS 3 enable PostGIS Raster as well::
     $ sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis_raster;"
 
 Set the path to the SpatiaLite module
-=====================================
+-------------------------------------
 
 By default the SpatiaLite functional tests are not run. To run them the ``SPATIALITE_LIBRARY_PATH``
 environment variable must be set.
@@ -62,7 +93,7 @@ the SpatiaLite library is ``/usr/lib/x86_64-linux-gnu/mod_spatialite.so``, so yo
     $ export SPATIALITE_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/mod_spatialite.so"
 
 Set up the MySQL database
-=========================
+-------------------------
 
 Create the ``gis`` role::
 
@@ -71,10 +102,10 @@ Create the ``gis`` role::
 
 Create the ``gis`` database::
 
-    $ mysql -u gis -p -e "CREATE DATABASE gis;"
+    $ mysql -u gis --password=gis -e "CREATE DATABASE gis;"
 
 Run Tests
-=========
+---------
 
 To run the tests::
 

test_container/.dockerignore

@@ -0,0 +1 @@
+/output/

test_container/.gitignore

@@ -0,0 +1 @@
+/output/

test_container/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:22.04
+
+COPY ./helpers/install_requirements.sh /
+RUN /install_requirements.sh
+
+COPY ./helpers/init_postgres.sh /
+env PGDATA="/var/lib/postgresql/data"
+env POSTGRES_PATH="/usr/lib/postgresql/14"
+RUN su postgres -c /init_postgres.sh
+
+ENV SPATIALITE_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/mod_spatialite.so"
+
+COPY ./helpers/init_mysql.sh /
+RUN /init_mysql.sh
+
+COPY ./helpers/entrypoint.sh /
+ENTRYPOINT ["/entrypoint.sh"]

test_container/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+cd "${SCRIPT_DIR}"
+docker build -t geoalchemy2 .

test_container/helpers/entrypoint.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+echo "starting postgres"
+(su postgres -c '${POSTGRES_PATH}/bin/pg_ctl start' > /dev/null 2>&1) &
+
+while ! ${POSTGRES_PATH}/bin/pg_isready --quiet; do
+    sleep 0.2
+done
+
+echo "starting mysql"
+/etc/init.d/mysql start
+
+echo "waiting for mysql to start"
+while ! mysqladmin ping -h 127.0.0.1 --silent; do
+    sleep 0.2
+done
+
+echo "###############################"
+echo "GeoAlchemy2 Test Container"
+echo ""
+echo 'run tests with `tox --workdir /output -vv`'
+echo "###############################"
+
+###############################
+# workarounds to get the tests working while mounting the code in as read-only
+mkdir /geoalchemy2
+find /geoalchemy2_read_only -mindepth 1 -maxdepth 1 | while read -r item; do
+    ln -s "${item}" "/geoalchemy2/$(basename "${item}")"
+done
+
+cd /geoalchemy2
+
+# remove links that would cause issues if they are present and read-only
+rm -f .mypy_cache .eggs *.egg-info .git .gitignore doc reports
+
+# copy these items instead of symlinking
+cp -r /geoalchemy2_read_only/doc /geoalchemy2/doc
+cp /geoalchemy2_read_only/.gitignore ./
+
+# store reports in the output directory
+mkdir -p /output/reports
+ln -s /output/reports /geoalchemy2/reports
+
+# to allow pre-commit to run
+git config --global init.defaultBranch master
+git config --global user.email "user@example.com"
+git config --global user.name "user"
+git init > /dev/null
+git add --all
+git commit -m "dummy commit" > /dev/null
+
+export MYPY_CACHE_DIR=/output/.mypy_cache
+
+###############################
+
+exec bash

test_container/helpers/init_mysql.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+
+if [ $(whoami) != "root" ]; then
+    echo "must run as the root user"
+    exit 1
+fi
+
+/etc/init.d/mysql start
+
+echo "waiting for mysql to start"
+while ! mysqladmin ping -h 127.0.0.1 --silent; do
+    sleep 0.2
+done
+
+echo "Create the 'gis' role"
+mysql -e "CREATE USER 'gis'@'%' IDENTIFIED BY 'gis';"
+mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'gis'@'%' WITH GRANT OPTION;"
+
+echo "Create the 'gis' database"
+mysql -u gis --password=gis -e "CREATE DATABASE gis;"

test_container/helpers/init_postgres.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -e
+
+if [ $(whoami) != "postgres" ]; then
+    echo "must run as the postgres user"
+    exit 1
+fi
+
+${POSTGRES_PATH}/bin/initdb --auth=trust --username=postgres -E 'UTF-8'
+
+# by default only listens on localhost so host cannot connect
+echo "listen_addresses = '0.0.0.0'" >> "${PGDATA}/postgresql.conf"
+
+(${POSTGRES_PATH}/bin/pg_ctl start > /dev/null 2>&1) &
+
+while ! ${POSTGRES_PATH}/bin/pg_isready --quiet; do
+    sleep 0.2
+done
+
+echo "Create the 'gis' role"
+psql -c "CREATE ROLE gis PASSWORD 'gis' SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN;"
+
+echo "Create the 'gis' database"
+createdb -E UTF-8 gis
+psql -d gis -c 'CREATE SCHEMA gis;'
+psql -c 'GRANT CREATE ON DATABASE gis TO "gis";'
+psql -d gis -c 'GRANT USAGE,CREATE ON SCHEMA gis TO "gis";'
+
+
+echo "Enable PostGIS for the 'gis' database"
+psql -d gis -c "CREATE EXTENSION postgis;"
+
+echo "With PostGIS 3 enable PostGIS Raster as well"
+psql -d gis -c "CREATE EXTENSION postgis_raster;"
+
+
+${POSTGRES_PATH}/bin/pg_ctl stop
+sleep 1

test_container/helpers/install_requirements.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -e
+
+# based on geoalchemy2/TEST.rst
+packages=(
+    # for managing virtual environments
+    tox
+    git
+    pypy3
+    pypy3-dev
+    pypy3-venv
+    python3.7
+    python3.7-dev
+    python3.7-venv
+    python3.8
+    python3.8-dev
+    python3.8-venv
+    python3.9
+    python3.9-dev
+    python3.9-venv
+    python3.10
+    python3.10-dev
+    python3.10-venv
+    python3.11
+    python3.11-dev
+    python3.11-venv
+
+    # PostgreSQL and PostGIS
+    postgresql
+    postgresql-14-postgis-3
+    postgresql-14-postgis-3-scripts
+    libpq-dev
+    libgeos-dev
+
+    # SpatiaLite
+    libsqlite3-mod-spatialite
+
+    # MySQL
+    mysql-client
+    mysql-server
+    default-libmysqlclient-dev
+
+    # mysqlclient requirements
+    # https://github.com/PyMySQL/mysqlclient#linux
+    default-libmysqlclient-dev
+    build-essential
+    pkg-config
+
+    # rasterio requirements with pypy
+    libgdal-dev
+)
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update -y
+apt-get install --no-install-recommends -y software-properties-common gnupg2
+add-apt-repository ppa:deadsnakes/ppa
+apt-get update -y
+
+apt-get install --no-install-recommends -y "${packages[@]}"
+
+# clear the package list cache (populated with apt-get update)
+rm -rf /var/lib/apt/lists/*

test_container/run.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+mkdir -p "${SCRIPT_DIR}/output"
+
+docker run --rm -it \
+    --mount type=bind,source="${ROOT}",target=/geoalchemy2_read_only,ro \
+    --mount type=bind,source="${SCRIPT_DIR}/output",target=/output \
+    geoalchemy2