Skip to content

Commit 592972f

Browse files
pascallapointepascallapointe
authored
Merge pull request #43 from PaskLab/development
Added db-sync create snapshot support.
2 parents cba0a30 + 7a27237 commit 592972f

File tree

7 files changed

+430
-55
lines changed

7 files changed

+430
-55
lines changed

‎Dockerfiles/db-sync/Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@ LABEL maintainer="https://github.com/pascallapointe"
77
WORKDIR /root
88

99
# Install base utilities and dependencies
10+
# (libicu70 & tree required by db-sync-tool)
1011
RUN apt-get update && \
11-
apt-get install -y libpq-dev postgresql-client
12+
apt-get install -y libpq-dev postgresql-client libicu70 tree
1213

1314
# Download Db-Sync repository
1415
ARG RELEASE
@@ -37,6 +38,7 @@ RUN mv cardano-db-sync/schema /cardano/.
3738
# Add startup scripts
3839
COPY files/start.sh /cardano/scripts/start.sh
3940
COPY files/sleep.sh /cardano/scripts/sleep.sh
41+
COPY files/postgresql-setup.sh /cardano/scripts/postgresql-setup.sh
4042

4143
# Add permissions
4244
RUN chmod g+s /cardano && \

‎Dockerfiles/db-sync/files/postgresql-setup.sh‎

Lines changed: 353 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,353 @@
1+
#!/usr/bin/env bash
2+
3+
# Unoffiical bash strict mode.
4+
# See: http://redsymbol.net/articles/unofficial-bash-strict-mode/
5+
set -u
6+
set -o pipefail
7+
IFS=$'\n\t'
8+
9+
progname="$0"
10+
11+
case "$(uname)" in
12+
Linux)
13+
# Linux tools have long names for these options.
14+
recursive="--recursive"
15+
directory="--directory"
16+
force="--force"
17+
test="--test"
18+
;;
19+
*)
20+
# These shoud work on any POSIX system
21+
recursive="-r"
22+
directory="-d"
23+
force="-f"
24+
test="-t"
25+
esac
26+
27+
# This should work on all Linux variants as well as FreeBSD.
28+
numcores=$(getconf _NPROCESSORS_ONLN)
29+
if test "${numcores}" -le 2 ; then
30+
numcores=1
31+
else
32+
numcores=$(( numcores - 1 ))
33+
fi
34+
35+
function die {
36+
echo "$1"
37+
exit 1
38+
}
39+
40+
function check_pgpass_file {
41+
if test -z ${PGPASSFILE+x} ; then
42+
echo "Error: The PGPASSFILE env var should be set to the location of the pgpass file."
43+
echo
44+
echo "Eg for mainnet:"
45+
echo "export PGPASSFILE=$(pwd)/config/pgpass"
46+
echo
47+
exit 1
48+
fi
49+
50+
if test ! -f "${PGPASSFILE}" ; then
51+
echo "Error: PostgreSQL password file ${PGPASSFILE} does not exist."
52+
exit 1
53+
fi
54+
55+
PGHOST=$(cut -d ":" -f 1 "${PGPASSFILE}")
56+
PGPORT=$(cut -d ":" -f 2 "${PGPASSFILE}")
57+
PGDATABASE=$(cut -d ":" -f 3 "${PGPASSFILE}")
58+
user=$(cut -d ":" -f 4 "${PGPASSFILE}")
59+
if [ "$user" != "*" ]; then
60+
PGUSER=$user
61+
export PGUSER
62+
fi;
63+
64+
export PGHOST
65+
export PGPORT
66+
export PGDATABASE
67+
}
68+
69+
function check_for_psql {
70+
# Make sure we have the psql executable.
71+
psql -V > /dev/null 2>&1 || die "Error : Missing 'psql' executable!"
72+
}
73+
74+
function check_psql_superuser {
75+
set +e
76+
psql -l "${PGDATABASE}" > /dev/null 2>&1 || psql -l > /dev/null 2>&1
77+
if test $? -ne 0 ; then
78+
echo
79+
echo "Error : User '${PGUSER:-$(whoami)}' can't access postgres."
80+
echo
81+
echo "To fix this, log into the postgres account and run:"
82+
echo " createuser --createdb --superuser ${PGUSER:-$(whoami)}"
83+
echo
84+
exit 1
85+
fi
86+
set -e
87+
}
88+
89+
function check_connect_as_user {
90+
psql "${PGDATABASE}" --no-password --command='\dt' > /dev/null
91+
if test $? -ne 0 ; then
92+
echo
93+
echo "Error : Not able to connect as '${PGUSER:-$(whoami)}' user."
94+
echo
95+
exit 1
96+
fi
97+
}
98+
99+
function check_db_exists {
100+
set +e
101+
count=$(psql -l "${PGDATABASE}" | grep -c "${PGDATABASE} ")
102+
if test "${count}" -lt 1 ; then
103+
echo
104+
echo "Error : No '${PGDATABASE}' database."
105+
echo
106+
echo "To create one run:"
107+
echo " $progname --createdb"
108+
echo
109+
exit 1
110+
fi
111+
count=$(psql -l "${PGDATABASE}" | grep "${PGDATABASE} " | cut -d \| -f 3 | grep -c UTF8)
112+
if test "${count}" -ne 1 ; then
113+
echo
114+
echo "Error : '${PGDATABASE}' database exists, but is not UTF8."
115+
echo
116+
echo "To fix this you should drop the current one and create a new one using:"
117+
echo " $progname --dropdb"
118+
echo " $progname --createdb"
119+
echo
120+
exit 1
121+
fi
122+
set -e
123+
}
124+
125+
function create_db {
126+
if test "$( psql "${PGDATABASE}" -tAc "SELECT 1 FROM pg_database WHERE datname='${PGDATABASE}'" )" != '1' ; then
127+
createdb -T template0 --owner="${PGUSER:-$(whoami)}" --encoding=UTF8 "${PGDATABASE}"
128+
fi
129+
}
130+
131+
132+
133+
function drop_db {
134+
if test "$( psql "${PGDATABASE}" -tAc "SELECT 1 FROM pg_database WHERE datname='${PGDATABASE}'" )" = '1' ; then
135+
psql "${PGDATABASE}" --command="DROP OWNED BY CURRENT_USER cascade;"
136+
fi
137+
}
138+
139+
function list_views {
140+
psql "${PGDATABASE}" \
141+
--command="select table_name from information_schema.views where table_catalog = '${PGDATABASE}' and table_schema = 'public' ;"
142+
}
143+
144+
function create_migration {
145+
echo "To create a migration:"
146+
echo "cabal run cardano-db-tool -- create-migration --mdir schema/"
147+
exit 0
148+
}
149+
150+
function run_migrations {
151+
echo "To run migrations:"
152+
echo "cabal run cardano-db-tool -- run-migrations --mdir schema/ --ldir ."
153+
echo "You probably do not need to do this."
154+
exit 0
155+
}
156+
157+
function dump_schema {
158+
pg_dump -s --schema=public "${PGDATABASE}"
159+
}
160+
161+
function create_snapshot {
162+
tgz_file=$1.tgz
163+
ledger_file=$2
164+
tmp_dir=$(mktemp "${directory}" -t db-sync-snapshot-XXXXXXXXXX)
165+
echo $"Working directory: ${tmp_dir}"
166+
pg_dump --no-owner --schema=public --jobs="${numcores}" "${PGDATABASE}" --format=directory --file="${tmp_dir}/db/"
167+
lstate_gz_file=$(basename "${ledger_file}").gz
168+
gzip --to-stdout "${ledger_file}" > "${tmp_dir}/$(basename "${ledger_file}").gz"
169+
tree "${tmp_dir}"
170+
# Use plain tar here because the database dump files and the ledger state file are already gzipped.
171+
tar cvf - --directory "${tmp_dir}" "db" "${lstate_gz_file}" | tee "${tgz_file}.tmp" \
172+
| sha256sum | head -c 64 | sed -e "s/$/ ${tgz_file}\n/" > "${tgz_file}.sha256sum"
173+
mv "${tgz_file}.tmp" "${tgz_file}"
174+
rm "${recursive}" "${force}" "${tmp_dir}"
175+
if test "$(tar "${test}" --file "${tgz_file}")" ; then
176+
echo "Tar reports the snapshot file as being corrupt."
177+
echo "It is not safe to drop the database and restore using this file."
178+
exit 1
179+
fi
180+
echo "Created ${tgz_file} + .sha256sum"
181+
}
182+
183+
function restore_snapshot {
184+
file_count=$(find "$2" -type f -name '*.lstate' | wc -l)
185+
if test "${file_count}" -gt 0 ; then
186+
echo "Ledger state directory ($2) is not empty. Please empty it and then retry."
187+
exit 1
188+
fi
189+
tmp_dir=$(mktemp "${directory}" -t db-sync-snapshot-XXXXXXXXXX)
190+
tar xvf "$1" --directory "$tmp_dir"
191+
if test -d "${tmp_dir}/db/" ; then
192+
# New pg_dump format
193+
lstate_gz_file=$(find "${tmp_dir}/" -iname "*.lstate.gz")
194+
lstate_file=$(basename "${lstate_gz_file}" | sed 's/.gz$//')
195+
gunzip --to-stdout "${lstate_gz_file}" > "$2/${lstate_file}"
196+
197+
# Important: specify --schema=public below to skip over `create schema public`
198+
# statement generated by pg_dump
199+
pg_restore \
200+
--schema=public \
201+
--jobs="${numcores}" \
202+
--format=directory \
203+
--dbname="${PGDATABASE}" \
204+
--no-owner \
205+
--exit-on-error \
206+
"${tmp_dir}/db/"
207+
else
208+
# Old snapshot format produced by this script
209+
db_file=$(find "${tmp_dir}/" -iname "*.sql")
210+
lstate_file=$(find "${tmp_dir}/" -iname "*.lstate")
211+
mv "${lstate_file}" "$2"
212+
psql --dbname="${PGDATABASE}" --file="${db_file}"
213+
fi
214+
rm "${recursive}" "${tmp_dir}"
215+
}
216+
217+
function usage_exit {
218+
echo
219+
echo "Usage:"
220+
echo " $progname --check - Check database exists and is set up correctly."
221+
echo " $progname --createdb - Create database."
222+
echo " $progname --dropdb - Drop database."
223+
echo " $progname --list-views - List the currently definied views."
224+
echo " $progname --recreatedb - Drop and recreate database."
225+
echo " $progname --create-user - Create database user (from config/pgass file)."
226+
echo " $progname --create-migration - Create a migration (if one is needed)."
227+
echo " $progname --run-migrations - Run all migrations applying as needed."
228+
echo " $progname --dump-schema - Dump the schema of the database."
229+
echo
230+
echo " - Create a db-sync state snapshot"
231+
echo " $progname --create-snapshot <snapshot-file> <ledger-state-file>"
232+
echo
233+
echo " - Restore a db-sync state snapshot."
234+
echo " $progname --restore-snapshot <snapshot-file> <ledger-state-dir>"
235+
echo
236+
exit 0
237+
}
238+
239+
# postgresql_version=$(psql -V | head -1 | sed -e "s/.* //;s/\.[0-9]*$//")
240+
241+
set -e
242+
243+
case "${1:-""}" in
244+
--check)
245+
check_pgpass_file
246+
check_for_psql
247+
check_psql_superuser
248+
check_db_exists
249+
check_connect_as_user
250+
;;
251+
--createdb)
252+
check_pgpass_file
253+
check_for_psql
254+
check_psql_superuser
255+
create_db
256+
;;
257+
--dropdb)
258+
check_pgpass_file
259+
check_for_psql
260+
check_psql_superuser
261+
drop_db
262+
;;
263+
--list-views)
264+
check_pgpass_file
265+
check_for_psql
266+
check_psql_superuser
267+
check_db_exists
268+
check_connect_as_user
269+
list_views
270+
;;
271+
--recreatedb)
272+
check_pgpass_file
273+
check_for_psql
274+
check_psql_superuser
275+
check_db_exists
276+
check_connect_as_user
277+
drop_db
278+
create_db
279+
echo "The database ${PGDATABASE} has been dropped and recreated."
280+
echo "The tables will be recreated when the application is run."
281+
exit 0
282+
;;
283+
--create-user)
284+
check_pgpass_file
285+
check_for_psql
286+
check_psql_superuser
287+
create_user
288+
;;
289+
--create-migration)
290+
create_migration
291+
;;
292+
--run-migrations)
293+
run_migrations
294+
;;
295+
--dump-schema)
296+
check_pgpass_file
297+
check_db_exists
298+
dump_schema
299+
;;
300+
--create-snapshot)
301+
check_pgpass_file
302+
check_db_exists
303+
if test $# -ne 3 ; then
304+
echo "Expecting exactly 2 more arguments, the snapshot file name template and the ledger state file."
305+
exit 1
306+
fi
307+
if test -z "$2" ; then
308+
echo "Second argument should be the snapshot file name template."
309+
exit 1
310+
fi
311+
if test -z "$3" ; then
312+
echo "Third argument should be the ledger state file."
313+
exit 1
314+
fi
315+
if test -d "$3" ; then
316+
echo "Third argument is a directory and expecting a file."
317+
exit 1
318+
fi
319+
create_snapshot "$2" "$3"
320+
;;
321+
--restore-snapshot)
322+
check_pgpass_file
323+
check_for_psql
324+
check_psql_superuser
325+
if [ "${RESTORE_RECREATE_DB:-Y}" == "Y" ]; then
326+
drop_db
327+
create_db
328+
fi
329+
if test $# -ne 3 ; then
330+
echo "Expecting exactly 2 more arguments, the snapshot file and the ledger state directory."
331+
exit 1
332+
fi
333+
if test -z "$2" ; then
334+
echo "Second argument should be the snapshot file."
335+
exit 1
336+
fi
337+
if test -z "$3" ; then
338+
echo "Third argument should be the ledger state directory."
339+
exit 1
340+
fi
341+
if test -f "$3" ; then
342+
echo "Third argument is a file and expecting a directory."
343+
exit 1
344+
fi
345+
restore_snapshot "$2" "$3"
346+
;;
347+
*)
348+
usage_exit
349+
;;
350+
esac
351+
352+
echo "All good!"
353+
exit 0

‎Dockerfiles/mithril-signer/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,15 @@ WORKDIR /root
1010
RUN mkdir -p /cardano/config /cardano/bin /cardano/db /cardano/socket /cardano/scripts
1111

1212
# Download cardano-cli repository
13-
ARG ARCHITECTURE
1413
ARG CLI_TAG
1514
RUN git clone https://github.com/IntersectMBO/cardano-cli.git
1615
WORKDIR /root/cardano-cli
17-
ARG GHC_VERSION=ghc-9.6.4
1816
RUN git checkout cardano-cli-${CLI_TAG} && git submodule update --init --recursive
1917

2018
# Build cardano-cli binary
2119
RUN cabal update && cabal user-config update && cabal build cardano-cli --disable-tests
20+
ARG ARCHITECTURE
21+
ARG GHC_VERSION=ghc-9.6.4
2222
RUN cp -p dist-newstyle/build/${ARCHITECTURE}-linux/${GHC_VERSION}/cardano-cli-${CLI_TAG}/x/cardano-cli/build/cardano-cli/cardano-cli /cardano/bin
2323

2424
# Download mithril repository

0 commit comments

Comments
 (0)