Merge pull request #66 from lsst-sqre/u/jsickcodes/v2-devenv

Support mysql and postgres in development and testing

.github/workflows/ci.yaml

@@ -3,14 +3,32 @@ name: CI
 "on": [push]
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Run pre-commit
+        uses: pre-commit/action@v2.0.3
+
   test:
     runs-on: ubuntu-latest
+    needs: [lint]
 
     strategy:
       matrix:
         python:
-          - 3.8
           - 3.9
+        db:
+          - sqlite
+          - postgres
+          - mysql
 
     steps:
       - uses: actions/checkout@v2
@@ -20,11 +38,10 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
-      - name: Run pre-commit
-        uses: pre-commit/action@v2.0.3
-
       - name: Install tox
-        run: pip install tox
+        run: |
+          pip install tox
+          pip install --pre tox-docker
 
       - name: Cache tox environments
         id: cache-tox
@@ -33,27 +50,33 @@ jobs:
           path: .tox
           # requirements/*.txt and pyproject.toml have versioning info
           # that would impact the tox environment.
-          key: tox-${{ matrix.python }}-${{ hashFiles('requirements/*.txt') }}-${{ hashFiles('pyproject.toml') }}
+          key: tox-${{ matrix.python }}--${{ matrix.db }}-${{ hashFiles('requirements/*.txt') }}-${{ hashFiles('pyproject.toml') }}
           restore-keys: |
-            tox-${{ matrix.python }}-${{ hashFiles('requirements/*.txt') }}-
+            tox-${{ matrix.python }}-${{ matrix.db }}-${{ hashFiles('requirements/*.txt') }}-
 
-      - name: Run tox
+      - name: Run tox with external services
+        if: ${{ matrix.python == 3.9 && matrix.db == 'postgres' }}
         env:
           LTD_KEEPER_TEST_AWS_ID: ${{ secrets.LTD_KEEPER_TEST_AWS_ID }}
           LTD_KEEPER_TEST_AWS_SECRET: ${{ secrets.LTD_KEEPER_TEST_AWS_SECRET }}
           LTD_KEEPER_TEST_BUCKET: ${{ secrets.LTD_KEEPER_TEST_BUCKET }}
-        run: tox -e py,coverage-report,typing
+        run: tox -e typing,${{matrix.db}},coverage-report  # run tox using Python in path
+
+      - name: Run tox without external services
+        if: ${{ !(matrix.python != 3.9 && matrix.db != 'postgres') }}
+        run: tox -e typing,${{matrix.db}},coverage-report  # run tox using Python in path
 
   docs:
     runs-on: ubuntu-latest
+    needs: [lint]
 
     steps:
       - uses: actions/checkout@v2
 
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Install Python dependencies
         run: pip install tox ltd-conveyor
@@ -82,8 +105,9 @@ jobs:
     runs-on: ubuntu-latest
     needs: [test]
 
-    # Only do Docker builds of ticket branches and tagged releases.
-    if: startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/tickets/')
+    # Only do Docker builds of ticket branches and tagged releases, as well
+    # as JSick Codes branches.
+    if: startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/tickets/') || startsWith(github.ref, 'refs/heads/u/jsickcodes/')
 
     steps:
       - uses: actions/checkout@v2

.gitignore

@@ -62,7 +62,10 @@ target/
 #Ipython Notebook
 .ipynb_checkpoints
 
+# Development/test Databases
 *.sqlite
+mysqldb/
+pgdb/
 
 integration_tests/ltd_keeper_doc_examples.txt
 

CHANGELOG.rst

@@ -2,6 +2,11 @@
 Change log
 ##########
 
+2.0.0 (Unreleased)
+==================
+
+- Add support the testing mysql and postgres databases locally and in GitHub Actions.
+
 1.20.3 (2020-11-17)
 ===================
 

Dockerfile

@@ -14,7 +14,7 @@
 #   - Runs a non-root user.
 #   - Sets up the entrypoint and port.
 
-FROM python:3.9.4-slim-buster AS base-image
+FROM python:3.9.5-slim-buster AS base-image
 
 # Update system packages
 COPY bin/install-base-packages.sh .

Makefile

@@ -14,7 +14,6 @@ help:
 	@echo "  make docs ...... (make Sphinx docs)"
 	@echo "  make docs-clean  (clean Sphinx docs)"
 	@echo "  make image ..... (make tagged Docker image)"
-	@echo "  make travis-docker-deploy (push image to Docker Hub from Travis CI)"
 	@echo "  make version ... (print the app version)"
 
 .PHONY: update-deps
@@ -29,6 +28,7 @@ init:
 	pip install --upgrade -r requirements/main.txt -r requirements/dev.txt
 	rm -rf .tox
 	pip install --upgrade pre-commit tox
+	pip install --pre --upgrade tox-docker
 	pre-commit install
 
 .PHONY: update
@@ -40,21 +40,36 @@ test:
 
 .PHONY: run
 run:
-	FLASK_APP=keeper LTD_KEEPER_PROFILE=development flask run
+	FLASK_APP=keeper LTD_KEEPER_PROFILE=development LTD_KEEPER_DEV_DB_URL="mysql+pymysql://user:password@localhost:3306/db" flask run
+
+.PHONY: run-pg
+runpg:
+	FLASK_APP=keeper LTD_KEEPER_PROFILE=development LTD_KEEPER_DEV_DB_URL=postgresql+psycopg2://user:password@localhost:3308/db flask run
+
+.PHONY: db-init-pg
+db-init-pg:
+	FLASK_APP=keeper LTD_KEEPER_PROFILE=development LTD_KEEPER_DEV_DB_URL="postgresql+psycopg2://user:password@localhost:3308/db" flask createdb
+	FLASK_APP=keeper LTD_KEEPER_PROFILE=development LTD_KEEPER_DEV_DB_URL="postgresql+psycopg2://user:password@localhost:3308/db" flask init
 
 .PHONY: db-init
 db-init:
-	FLASK_APP=keeper LTD_KEEPER_PROFILE=development flask createdb
-	FLASK_APP=keeper LTD_KEEPER_PROFILE=development flask init
+	FLASK_APP=keeper LTD_KEEPER_PROFILE=development LTD_KEEPER_DEV_DB_URL="mysql+pymysql://user:password@localhost:3306/db" flask createdb
+	FLASK_APP=keeper LTD_KEEPER_PROFILE=development LTD_KEEPER_DEV_DB_URL="mysql+pymysql://user:password@localhost:3306/db" flask init
 
 .PHONY: db-upgrade
 db-upgrade:
-	FLASK_APP=keeper LTD_KEEPER_PROFILE=development flask db upgrade
+	FLASK_APP=keeper LTD_KEEPER_PROFILE=development LTD_KEEPER_DEV_DB_URL="mysql+pymysql://user:password@localhost:3306/db" flask db upgrade
+
+.PHONY: db-upgrade-pg
+db-upgrade-pg:
+	FLASK_APP=keeper LTD_KEEPER_PROFILE=development LTD_KEEPER_DEV_DB_URL="postgresql+psycopg2://user:password@localhost:3308/db" flask db upgrade
 
 .PHONY: db-clean
 db-clean:
-	rm ltd-keeper-dev.sqlite
-	rm ltd-keeper-test.sqlite
+	rm -f ltd-keeper-dev.sqlite
+	rm -f ltd-keeper-test.sqlite
+	rm -rf mysqldb
+	rm -rf pgdb
 
 .PHONY: redis
 redis:

docker-compose.pg.yaml

@@ -0,0 +1,16 @@
+version: "3"
+services:
+  postgres:
+    image: postgres:11
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: 'password'
+      POSTGRES_USER: 'user'
+      POSTGRES_DB: 'db'
+      PGPORT: '3308'
+    ports:
+      - '3308:3308'
+    expose:
+      - '3308'
+    volumes:
+      - ./pgdb:/var/lib/postgresql/data

docker-compose.yaml

@@ -0,0 +1,22 @@
+version: "3"
+services:
+  mysql:
+    image: mysql:5.7
+    restart: always
+    environment:
+      MYSQL_DATABASE: 'db'
+      # So you don't have to use root, but you can if you like
+      MYSQL_USER: 'user'
+      # You can use whatever password you like
+      MYSQL_PASSWORD: 'password'
+      # Password for root access
+      MYSQL_ROOT_PASSWORD: 'password'
+    ports:
+      # <Port exposed> : < MySQL Port running inside container>
+      - '3306:3306'
+    expose:
+      # Opens port 3306 on the container
+      - '3306'
+      # Where our data will be persisted
+    volumes:
+      - ./mysqldb:/var/lib/mysql

keeper/appfactory.py

@@ -34,7 +34,7 @@ def create_flask_app(profile: Optional[str] = None) -> Flask:
 
     # Add the middleware to respect headers forwarded from the proxy server
     if app.config["PROXY_FIX"]:
-        app.wsgi_app = ProxyFix(  # type: ignore
+        app.wsgi_app = ProxyFix(
             app.wsgi_app,
             x_for=app.config["TRUST_X_FOR"],
             x_proto=app.config["TRUST_X_PROTO"],

keeper/config.py

@@ -101,6 +101,8 @@ def init_app(cls, app: Flask) -> None:
         stream_handler = logging.StreamHandler(stream=sys.stdout)
         stream_handler.setFormatter(logging.Formatter("%(message)s"))
         logger = logging.getLogger("keeper")
+        if logger.hasHandlers():
+            logger.handlers.clear()
         logger.addHandler(stream_handler)
         logger.setLevel(logging.DEBUG)
 
@@ -128,9 +130,9 @@ class TestConfig(Config):
     """Test configuration (for py.test harness)."""
 
     SERVER_NAME = "example.test"
-    SQLALCHEMY_DATABASE_URI = "sqlite:///" + os.path.join(
-        BASEDIR, "ltd-keeper-test.sqlite"
-    )
+    SQLALCHEMY_DATABASE_URI = os.environ.get(
+        "LTD_KEEPER_TEST_DB_URL"
+    ) or "sqlite:///" + os.path.join(BASEDIR, "ltd-keeper-test.sqlite")
 
     @classmethod
     def init_app(cls, app: Flask) -> None:
@@ -140,6 +142,8 @@ def init_app(cls, app: Flask) -> None:
         stream_handler = logging.StreamHandler(stream=sys.stdout)
         stream_handler.setFormatter(logging.Formatter("%(message)s"))
         logger = logging.getLogger("keeper")
+        if logger.hasHandlers():
+            logger.handlers.clear()
         logger.addHandler(stream_handler)
         logger.setLevel(logging.DEBUG)
 
@@ -181,6 +185,8 @@ def init_app(cls, app: Flask) -> None:
         stream_handler.setFormatter(logging.Formatter("%(message)s"))
         logger = logging.getLogger("keeper")
         logger.addHandler(stream_handler)
+        if logger.hasHandlers():
+            logger.handlers.clear()
         logger.setLevel(logging.INFO)
 
         structlog.configure(

keeper/logutils.py

@@ -40,6 +40,7 @@ def decorated_function(*args, **kwargs):  # type: ignore
             # Close out the logger
             end_time = timer()
             log.info(
+                "Closing endpoint handler",
                 status=response.status_code,
                 response_time=end_time - start_time,
             )

keeper/utils.py

@@ -8,7 +8,16 @@
 
 import json
 import re
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    List,
+    Optional,
+    SupportsIndex,
+    Tuple,
+    Union,
+)
 
 from dateutil import parser as datetime_parser
 from dateutil.tz import tzutc
@@ -194,7 +203,7 @@ def __setitem__(self, index: Any, value: Any) -> None:
         list.__setitem__(self, index, value)
         self.changed()
 
-    def __delitem__(self, index: Union[int, slice]) -> None:
+    def __delitem__(self, index: Union[SupportsIndex, slice]) -> None:
         """Detect list del events and emit change events."""
         list.__delitem__(self, index)
         self.changed()

requirements/dev.in

@@ -5,15 +5,20 @@
 # After editing, update requirements/dev.txt by running:
 #     make update-deps
 
+-c main.txt
+
 pytest
 responses
 pytest-mock
 sqlalchemy-stubs
 mock
 coverage[toml]
 mypy
-Sphinx
-docutils==0.15.2  # due to boto3
+Sphinx<4
 sphinx-rtd-theme
 numpydoc
 sphinxcontrib-httpdomain
+types-requests
+types-setuptools
+types-python-dateutil
+types-mock