From 096c8b60d359661e27c9d2b955b715311fb67795 Mon Sep 17 00:00:00 2001 From: George Hickman Date: Fri, 15 Oct 2021 15:39:49 +0100 Subject: [PATCH] SQLite -> PostgreSQL --- .github/workflows/main.yml | 16 ++++++++++++++++ .gitignore | 3 --- Dockerfile | 5 ----- INSTALL.md | 12 +----------- README.md | 2 +- docker-compose.yml | 16 +++++++++++++++- jobserver/settings.py | 4 +++- sqlite.load | 5 +++++ 8 files changed, 41 insertions(+), 22 deletions(-) create mode 100644 sqlite.load diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4d51e7ea..6044902a7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -104,6 +104,21 @@ jobs: needs: [assets, format, lint, sort, upgrade] runs-on: ubuntu-latest + services: + postgres: + image: postgres:13 + env: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: jobserver + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - uses: actions/checkout@v2 - uses: "actions/setup-python@v2" @@ -119,6 +134,7 @@ jobs: - name: Run tests env: + DATABASE_URL: postgres://user:password@localhost/jobserver SECRET_KEY: 12345 SOCIAL_AUTH_GITHUB_KEY: test SOCIAL_AUTH_GITHUB_SECRET: test diff --git a/.gitignore b/.gitignore index d6ca85003..6ea39ecf2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,6 @@ __pycache__/ /assets/dist /assets/stats.html /coverage -db.sqlite3 -db.sqlite3-shm -db.sqlite3-wal htmlcov node_modules/ releases diff --git a/Dockerfile b/Dockerfile index d21879c7e..1f017aed6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,11 +19,6 @@ ENV PIP_NO_CACHE_DIR=1 \ WORKDIR /app -RUN apt-get update && \ - apt-get install --no-install-recommends -y sqlite3=3.27.2-3+deb10u1 && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - # Only requirements to cache them in docker layer so we can skip package # installation if they haven't changed COPY requirements.prod.txt . diff --git a/INSTALL.md b/INSTALL.md index 7fb53e010..7d20ac73f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -7,23 +7,13 @@ dokku$ dokku domains:add job-server jobs.opensafely.org dokku$ dokku git:set job-server deploy-branch main ``` -## Create storage & load sqlite db into it - -```bash -dokku$ mkdir /var/lib/dokku/data/storage/job-server -dokku$ chown dokku:dokku /var/lib/dokku/data/storage/job-server -dokku$ cp ./job-server-db.sqlite3 /var/lib/dokku/data/storage/job-server/db.sqlite3 -dokku$ chown dokku:dokku /var/lib/dokku/data/storage/job-server/* -dokku$ dokku storage:mount job-server /var/lib/dokku/data/storage/job-server/:/storage -``` - ## Configure app ```bash dokku config:set job-server ADMIN_USERS='xxx' dokku config:set job-server BACKENDS='tpp,emis' dokku config:set job-server BASE_URL='https://jobs.opensafely.org' -dokku config:set job-server DATABASE_URL='sqlite:////storage/db.sqlite3' +dokku config:set job-server DATABASE_URL='postgres://localhost/jobserver' dokku config:set job-server EMAIL_BACKEND='anymail.backends.mailgun.EmailBackend' dokku config:set job-server GITHUB_TOKEN='xxx' dokku config:set job-server MAILGUN_API_KEY='xxx' diff --git a/README.md b/README.md index b2d63bc4a..1b903b912 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ It provides an API to [job-runner](https://github.com/opensafely-core/job-runner ## Stack This is a [Django](https://www.djangoproject.com) project. -It uses [Django Rest Framework](https://www.django-rest-framework.org) for the API and [SQLite](https://www.sqlite.org/index.html) for the database. +It uses [Django Rest Framework](https://www.django-rest-framework.org) for the API and [PostgreSQL](https://www.postgresql.org/) for the database. It is deployed via [dokku](https://dokku.com), serves static files using the [whitenoise](http://whitenoise.evans.io) package, and is itself served by [gunicorn](https://gunicorn.org). We authenticate Users with the [Python Social Auth](https://python-social-auth.readthedocs.io) Django-specific package, using [GitHub](https://github.com/) as the OAuth Provider backend. diff --git a/docker-compose.yml b/docker-compose.yml index 50fdfb875..c766fef60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,13 +2,24 @@ version: "3.6" services: + db: + image: "postgres:13" + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: pass + POSTGRES_DB: jobserver + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data/ + job-server: build: . command: /app/manage.py runserver 0.0.0.0:8000 environment: - ADMIN_USERS - AUTHORIZATION_ORGS - - DATABASE_URL + - DATABASE_URL=postgres://user:pass@db:5432/jobserver - DEBUG - GITHUB_TOKEN - SECRET_KEY=${SECRET_KEY:-12345} @@ -18,3 +29,6 @@ services: - "8000:8000" volumes: - .:/app + +volumes: + postgres_data: diff --git a/jobserver/settings.py b/jobserver/settings.py index 2920e9091..8c89baa69 100644 --- a/jobserver/settings.py +++ b/jobserver/settings.py @@ -101,7 +101,9 @@ # Database # https://docs.djangoproject.com/en/3.0/ref/settings/#databases -DATABASES = {"default": env.dj_db_url("DATABASE_URL", default="sqlite:///db.sqlite3")} +DATABASES = { + "default": env.dj_db_url("DATABASE_URL", default="postgres://localhost/jobserver") +} # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field diff --git a/sqlite.load b/sqlite.load new file mode 100644 index 000000000..32237b53a --- /dev/null +++ b/sqlite.load @@ -0,0 +1,5 @@ +load database + from sqlite://db.sqlite3 + into postgresql://localhost/jobserver + + with include drop, reset sequences;