Skip to content

Commit dd9c11f

Browse files
lucernaelucernae
authored
fix kartoza#321: POSTGRES_INITDB_WALDIR handling (kartoza#324)
This is mainly to fix kartoza#321 Fix POSTGRES_INITDB_WALDIR behaviour as expected for the default setup. With added several minor improvements: * Add scenario tests for testing POSTGRES_INITDB_WALDIR possible setup * Improve README.md * Fix symlink resolution check
1 parent aee92e7 commit dd9c11f

File tree

9 files changed

+331
-136
lines changed

9 files changed

+331
-136
lines changed

‎.github/workflows/build-latest.yaml

Lines changed: 27 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -14,72 +14,13 @@ on:
1414
# description: Base image variant
1515
# required: true
1616
# default: slim
17-
push:
17+
workflow_dispatch:
1818
pull_request:
1919
branches:
2020
- develop
2121
jobs:
22-
build-image:
23-
runs-on: ubuntu-latest
24-
env:
25-
latest-ref: refs/heads/develop
26-
defaultRepo: kartoza
27-
strategy:
28-
matrix:
29-
postgresMajorVersion:
30-
- 13
31-
postgisMajorVersion:
32-
- 3
33-
postgisMinorRelease:
34-
- 1
35-
include:
36-
- distro: debian
37-
imageVersion: bullseye
38-
imageVariant: slim
39-
steps:
40-
- uses: actions/checkout@v2
41-
- name: Set up QEMU
42-
uses: docker/setup-qemu-action@v1
43-
- name: Set up Docker Buildx
44-
uses: docker/setup-buildx-action@v1
45-
- name: Get build cache
46-
uses: actions/cache@v2
47-
with:
48-
path: /tmp/.buildx-cache
49-
# Build inputs are:
50-
# - Dockerfile recipe
51-
# - docker-compose.build.yml
52-
# - build args (.example.env)
53-
# - base_build directory
54-
# - scripts directory
55-
key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts', 'scenario_tests/utils/requirements.txt') }}-${{ github.sha }}
56-
restore-keys: |
57-
buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts', 'scenario_tests/utils/requirements.txt') }}-
58-
59-
- name: Build all stages
60-
id: docker_build_base
61-
uses: docker/build-push-action@v2
62-
with:
63-
context: .
64-
file: Dockerfile
65-
push: false
66-
load: true
67-
tags: ${{ secrets.DOCKERHUB_REPO || env.defaultRepo }}/postgis:base-${{ matrix.distro }}-${{ matrix.imageVersion }}-${{ matrix.imageVariant }}
68-
build-args: |
69-
DISTRO=${{ matrix.distro }}
70-
IMAGE_VERSION=${{ matrix.imageVersion }}
71-
IMAGE_VARIANT=${{ matrix.imageVariant }}
72-
LANGS=en_US.UTF-8,id_ID.UTF-8
73-
GENERATE_ALL_LOCALE=0
74-
POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }}
75-
POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }}
76-
POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }}
77-
cache-from: type=local,src=/tmp/.buildx-cache
78-
cache-to: type=local,mode=max,dest=/tmp/.buildx-cache
79-
8022
run-scenario-tests:
8123
runs-on: ubuntu-latest
82-
needs: [ build-image ]
8324
strategy:
8425
matrix:
8526
postgresMajorVersion:
@@ -104,20 +45,6 @@ jobs:
10445
uses: docker/setup-qemu-action@v1
10546
- name: Set up Docker Buildx
10647
uses: docker/setup-buildx-action@v1
107-
- name: Get build cache
108-
uses: actions/cache@v2
109-
with:
110-
path: /tmp/.buildx-cache
111-
# Build inputs are:
112-
# - Dockerfile recipe
113-
# - docker-compose.build.yml
114-
# - build args (.example.env)
115-
# - base_build directory
116-
# - scripts directory
117-
key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts', 'scenario_tests/utils/requirements.txt') }}-${{ github.sha }}
118-
restore-keys: |
119-
buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts', 'scenario_tests/utils/requirements.txt') }}-
120-
12148
- name: Build image for testing
12249
id: docker_build_testing_image
12350
uses: docker/build-push-action@v2
@@ -136,22 +63,25 @@ jobs:
13663
POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }}
13764
POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }}
13865
POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }}
139-
cache-from: type=local,src=/tmp/.buildx-cache
66+
cache-from: |
67+
type=gha,scope=test
68+
type=gha,scope=prod
69+
type=gha,scope=base
70+
cache-to: type=gha,scope=test
14071
target: postgis-test
14172

142-
- name: Run scenario test
73+
- name: Run scenario test ${{ matrix.scenario }}
74+
working-directory: scenario_tests/${{ matrix.scenario }}
14375
env:
14476
COMPOSE_INTERACTIVE_NO_CLI: 1
14577
PRINT_TEST_LOGS: 1
14678
run: |
147-
pushd scenario_tests/${{ matrix.scenario }}
148-
./test.sh
149-
popd
79+
bash ./test.sh
15080
15181
push-internal-pr-images:
15282
if: github.event_name == 'pull_request' && github.event.pull_request.base.repo.url == github.event.pull_request.head.repo.url
15383
runs-on: ubuntu-latest
154-
needs: [ build-image ]
84+
needs: [ run-scenario-tests ]
15585
strategy:
15686
matrix:
15787
postgresMajorVersion:
@@ -165,37 +95,25 @@ jobs:
16595
imageVersion: bullseye
16696
imageVariant: slim
16797
steps:
98+
- uses: actions/checkout@v2
99+
- name: Set up QEMU
100+
uses: docker/setup-qemu-action@v1
101+
- name: Set up Docker Buildx
102+
uses: docker/setup-buildx-action@v1
168103
- name: Login to DockerHub
169104
uses: docker/login-action@v1
170105
with:
171106
username: ${{ secrets.DOCKERHUB_USERNAME }}
172107
password: ${{ secrets.DOCKERHUB_PASSWORD }}
173-
174108
- name: Docker meta
175109
id: docker_meta
176-
uses: crazy-max/ghaction-docker-meta@v1
110+
uses: docker/metadata-action@v3
177111
with:
178112
images: ${{ secrets.DOCKERHUB_REPO}}/postgis
179-
tag-latest: false
180-
181-
- uses: actions/checkout@v2
182-
- name: Set up QEMU
183-
uses: docker/setup-qemu-action@v1
184-
- name: Set up Docker Buildx
185-
uses: docker/setup-buildx-action@v1
186-
- name: Get build cache
187-
uses: actions/cache@v2
188-
with:
189-
path: /tmp/.buildx-cache
190-
# Build inputs are:
191-
# - Dockerfile recipe
192-
# - docker-compose.build.yml
193-
# - build args (.example.env)
194-
# - base_build directory
195-
# - scripts directory
196-
key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts') }}-${{ github.sha }}
197-
restore-keys: |
198-
buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts') }}-
113+
tags: |
114+
type=semver,pattern=\d-\d.\d
115+
type=ref,event=branch
116+
type=ref,event=pr
199117
200118
- name: Build image for testing
201119
id: docker_build_testing_image
@@ -204,7 +122,8 @@ jobs:
204122
context: .
205123
file: Dockerfile
206124
push: true
207-
tags: ${{ steps.docker_meta.outputs.tags }}-${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }}
125+
tags: |
126+
${{ steps.docker_meta.outputs.tags }}-${{ matrix.postgresMajorVersion }}-${{ matrix.postgisMajorVersion }}.${{ matrix.postgisMinorRelease }}
208127
build-args: |
209128
DISTRO=${{ matrix.distro }}
210129
IMAGE_VERSION=${{ matrix.imageVersion }}
@@ -214,5 +133,9 @@ jobs:
214133
POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }}
215134
POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }}
216135
POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }}
217-
cache-from: type=local,src=/tmp/.buildx-cache
136+
cache-from: |
137+
type=gha,scope=test
138+
type=gha,scope=prod
139+
type=gha,scope=base
140+
cache-to: type=gha,scope=test
218141
target: postgis-test

‎.github/workflows/deploy-image.yaml

Lines changed: 10 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -44,19 +44,6 @@ jobs:
4444
uses: docker/setup-qemu-action@v1
4545
- name: Set up Docker Buildx
4646
uses: docker/setup-buildx-action@v1
47-
- name: Get build cache
48-
uses: actions/cache@v2
49-
with:
50-
path: /tmp/.buildx-cache
51-
# Build inputs are:
52-
# - Dockerfile recipe
53-
# - docker-compose.build.yml
54-
# - build args (.example.env)
55-
# - base_build directory
56-
# - scripts directory
57-
key: buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts', 'scenario_tests/utils/requirements.txt') }}-${{ github.sha }}
58-
restore-keys: |
59-
buildx-${{ hashFiles('Dockerfile', 'docker-compose.build.yml', '.example.env', 'base_build', 'scripts', 'scenario_tests/utils/requirements.txt') }}-
6047

6148
- name: Login to DockerHub
6249
uses: docker/login-action@v1
@@ -82,8 +69,11 @@ jobs:
8269
POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }}
8370
POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }}
8471
POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }}
85-
cache-from: type=local,src=/tmp/.buildx-cache
86-
cache-to: type=local,dest=/tmp/.buildx-cache
72+
cache-from: |
73+
type=gha,scope=test
74+
type=gha,scope=prod
75+
type=gha,scope=base
76+
cache-to: type=gha,scope=base
8777
target: postgis-base
8878

8979
- name: Build prod image
@@ -105,6 +95,9 @@ jobs:
10595
POSTGRES_MAJOR_VERSION=${{ matrix.postgresMajorVersion }}
10696
POSTGIS_MAJOR_VERSION=${{ matrix.postgisMajorVersion }}
10797
POSTGIS_MINOR_VERSION=${{ matrix.postgresMinorVersion }}
108-
cache-from: type=local,src=/tmp/.buildx-cache
109-
cache-to: type=local,dest=/tmp/.buildx-cache
98+
cache-from: |
99+
type=gha,scope=test
100+
type=gha,scope=prod
101+
type=gha,scope=base
102+
cache-to: type=gha,scope=prod
110103
target: postgis-prod

‎README.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
[![Build Status](https://travis-ci.org/kartoza/docker-postgis.svg?branch=develop)](https://travis-ci.org/kartoza/docker-postgis)
1+
[![Scenario Tests](https://github.com/kartoza/docker-postgis/actions/workflows/build-latest.yaml/badge.svg?branch=develop&event=push)](https://github.com/kartoza/docker-postgis/actions/workflows/build-latest.yaml)
2+
[![deploy-image](https://github.com/kartoza/docker-postgis/actions/workflows/deploy-image.yaml/badge.svg)](https://github.com/kartoza/docker-postgis/actions/workflows/deploy-image.yaml)
23

34
# docker-postgis
45

@@ -143,7 +144,7 @@ You need to specify different empty directory, like this
143144
The containers will use above parameters to initialize a new db cluster in the
144145
specified directory. If the directory is not empty, then initialization parameter will be ignored.
145146

146-
These are some initialization parameters that will only get used to initialize a new cluster.
147+
These are some initialization parameters that will only be used to initialize a new cluster.
147148
If the container uses an existing cluster, it is ignored (for example, when the container restarts).
148149

149150
* `DEFAULT_ENCODING`: cluster encoding
@@ -152,6 +153,7 @@ If the container uses an existing cluster, it is ignored (for example, when the
152153
* `WAL_SEGSIZE`: WAL segsize option
153154
* `PASSWORD_AUTHENTICATION` : PASSWORD AUTHENTICATION
154155
* `INITDB_EXTRA_ARGS`: extra parameter that will be passed down to `initdb` command
156+
* `POSTGRES_INITDB_WALDIR`: parameter to tell postgres about the initial waldir location. Note that you must always mount persistent volume to this location. Postgres will expect that the directory will always be available, even though it doesn't need the environment variable anymore. If you didn't persist this location, postgres will not be able to find the `pg_wal` directory and consider the instance to be broken.
155157

156158
In addition to that, we have another parameter: `RECREATE_DATADIR` that can be used to force database reinitializations.
157159
If this parameter is specified as `TRUE` it will act as explicit consent to delete `DATADIR` and create

‎scenario_tests/datadir_init/docker-compose.yml

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ version: '2.1'
22
volumes:
33
default-pg-data-dir:
44
new-pg-data-dir:
5+
new-pg-data-dir-2:
6+
new-pg-data-dir-3:
57
recreate-pg-data-dir:
8+
init-waldir:
69
services:
710
pg-default:
811
image: 'kartoza/postgis:${TAG:-manual-build}'
@@ -58,3 +61,83 @@ services:
5861
timeout: 30s
5962
retries: 3
6063
test: "pg_isready"
64+
65+
pg-custom-waldir-wrong:
66+
image: 'kartoza/postgis:${TAG:-manual-build}'
67+
volumes:
68+
# Mount to a locations where there are initial data
69+
- new-pg-data-dir-2:/opt/mypostgis/data
70+
# Specify different waldir location
71+
- init-waldir:/opt/mypostgis/data/waldir
72+
- ./tests:/tests
73+
- ../utils:/lib/utils
74+
environment:
75+
TEST_CLASS: TestCustomWALdir
76+
DATADIR: /opt/mypostgis/data
77+
POSTGRES_INITDB_WALDIR: /opt/mypostgis/data/waldir
78+
POSTGRES_PASS: 'docker'
79+
healthcheck:
80+
interval: 60s
81+
timeout: 30s
82+
retries: 3
83+
test: "pg_isready"
84+
85+
pg-custom-waldir-correct:
86+
image: 'kartoza/postgis:${TAG:-manual-build}'
87+
volumes:
88+
# Mount to new locations where there are no initial data
89+
- new-pg-data-dir-3:/opt/mypostgis/data
90+
# Specify different waldir location
91+
- init-waldir:/opt/mypostgis/waldir
92+
- ./tests:/tests
93+
- ../utils:/lib/utils
94+
environment:
95+
TEST_CLASS: TestCustomWALdir
96+
DATADIR: /opt/mypostgis/data
97+
POSTGRES_INITDB_WALDIR: /opt/mypostgis/waldir
98+
POSTGRES_PASS: 'docker'
99+
healthcheck:
100+
interval: 60s
101+
timeout: 30s
102+
retries: 3
103+
test: "pg_isready"
104+
105+
pg-custom-waldir-not-match-1:
106+
image: 'kartoza/postgis:${TAG:-manual-build}'
107+
volumes:
108+
# Mount to a locations where there are initial data
109+
- new-pg-data-dir-3:/opt/mypostgis/data
110+
# Specify different waldir location
111+
- init-waldir:/opt/mypostgis/waldir
112+
- ./tests:/tests
113+
- ../utils:/lib/utils
114+
environment:
115+
TEST_CLASS: TestCustomWALdirNotMatch
116+
DATADIR: /opt/mypostgis/data
117+
POSTGRES_INITDB_WALDIR: /opt/waldir
118+
POSTGRES_PASS: 'docker'
119+
healthcheck:
120+
interval: 60s
121+
timeout: 30s
122+
retries: 3
123+
test: "pg_isready"
124+
125+
pg-custom-waldir-not-match-2:
126+
image: 'kartoza/postgis:${TAG:-manual-build}'
127+
volumes:
128+
# Mount to a locations where there are initial data
129+
- new-pg-data-dir-2:/opt/mypostgis/data
130+
# Specify different waldir location
131+
- init-waldir:/opt/waldir
132+
- ./tests:/tests
133+
- ../utils:/lib/utils
134+
environment:
135+
TEST_CLASS: TestCustomWALdirNotMatch
136+
DATADIR: /opt/mypostgis/data
137+
POSTGRES_INITDB_WALDIR: /opt/waldir
138+
POSTGRES_PASS: 'docker'
139+
healthcheck:
140+
interval: 60s
141+
timeout: 30s
142+
retries: 3
143+
test: "pg_isready"

‎scenario_tests/datadir_init/test.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ set -e
66
source ../test-env.sh
77

88
# Run service
9-
docker-compose up -d
9+
docker-compose up -d pg-default pg-new pg-recreate
1010

1111
if [[ -n "${PRINT_TEST_LOGS}" ]]; then
1212
docker-compose logs -f &
@@ -28,4 +28,7 @@ for service in "${services[@]}"; do
2828

2929
done
3030

31+
# special meta test to check the setup
32+
bash ./test_custom_waldir.sh
33+
3134
docker-compose down -v

0 commit comments

Comments
 (0)