Repository navigation
Expand file tree
/
Copy pathdeploy.sh
More file actions
executable file
·354 lines (315 loc) · 12.3 KB
/
Copy pathdeploy.sh
File metadata and controls
executable file
·354 lines (315 loc) · 12.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
#!/bin/bash
set -euo pipefail
# Bikontrol Deployment Script
# Usage: ./scripts/deploy.sh [deploy|status|logs|verify|rollback]
DEPLOY_DIR="${DEPLOY_DIR:-/opt/bikontrol}"
LOG_FILE="${LOG_FILE:-/tmp/bikontrol-deploy.log}"
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
BACKUP_DIR="${DEPLOY_DIR}/backups/backup-${TIMESTAMP}"
BACKUP_RETENTION="${BACKUP_RETENTION:-7}"
# Public base URL the post-deploy smoke test hits (frontend + API under one
# origin). Override for staging: SMOKE_URL=https://staging.example ./deploy.sh deploy
SMOKE_URL="${SMOKE_URL:-https://bikontrol.santidev21.tech}"
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}
error_exit() {
log "ERROR: $1"
exit 1
}
validate_docker() {
log "Checking Docker..."
docker info >/dev/null 2>&1 || error_exit "Docker is not running or not accessible"
log "Docker available."
}
validate_env() {
log "Validating .env file..."
if [ ! -f "$DEPLOY_DIR/.env" ]; then
error_exit ".env file not found at $DEPLOY_DIR/.env (copy .env.example and fill it in)"
fi
if grep -q "CHANGE_ME" "$DEPLOY_DIR/.env"; then
error_exit ".env still contains CHANGE_ME placeholders — set real values before deploying"
fi
log ".env validated."
}
validate_config() {
log "Validating docker compose configuration..."
(cd "$DEPLOY_DIR" && docker compose config --quiet) || error_exit "docker compose config validation failed"
log "Configuration valid."
}
backup() {
mkdir -p "$(dirname "$BACKUP_DIR")"
if [ -d "$DEPLOY_DIR" ]; then
log "Creating config backup at ${BACKUP_DIR}.tgz..."
tar czf "${BACKUP_DIR}.tgz" -C / \
--exclude=opt/bikontrol/.git \
--exclude=opt/bikontrol/.env \
--exclude=opt/bikontrol/backups \
opt/bikontrol 2>/dev/null || true
chmod 700 "${BACKUP_DIR}.tgz"
log "Backup created."
fi
}
backup_database() {
if [ ! -f "$DEPLOY_DIR/.env" ]; then
return 0
fi
set -a
# shellcheck disable=SC1091
source "$DEPLOY_DIR/.env"
set +a
mkdir -p "$(dirname "$BACKUP_DIR")"
mkdir -p "$BACKUP_DIR"
if docker compose -f "$DEPLOY_DIR/docker-compose.yml" ps -q db >/dev/null 2>&1; then
log "Backing up database to ${BACKUP_DIR}/db.sql.gz ..."
if ! docker compose -f "$DEPLOY_DIR/docker-compose.yml" exec -T db \
pg_dump -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" 2>/dev/null | gzip > "${BACKUP_DIR}/db.sql.gz"; then
error_exit "database backup failed — aborting"
fi
if ! gzip -t "${BACKUP_DIR}/db.sql.gz" 2>/dev/null; then
error_exit "database backup looks corrupt (${BACKUP_DIR}/db.sql.gz) — aborting"
fi
# A gzip-valid dump can still be unrestorable. Restore it into a
# throwaway database and check the core tables before trusting it.
if ! "${DEPLOY_DIR}/scripts/db-verify-backup.sh" "${BACKUP_DIR}/db.sql.gz"; then
error_exit "database backup failed verification — aborting"
fi
# prune old backups, keep last N
log "Pruning old backups (keeping last $BACKUP_RETENTION) ..."
ls -dt "${DEPLOY_DIR}/backups"/backup-* 2>/dev/null | tail -n +$((BACKUP_RETENTION + 1)) | xargs -r rm -rf
ls -t "${DEPLOY_DIR}/backups"/backup-*/db.sql.gz 2>/dev/null | tail -n +$((BACKUP_RETENTION + 1)) | xargs -r rm -f || true
fi
}
backup_only() {
BACKUP_DIR="${DEPLOY_DIR}/backups/backup-${TIMESTAMP}"
log "=== Manual backup: ${BACKUP_DIR} ==="
mkdir -p "$BACKUP_DIR"
backup_database
log "Backup complete at ${BACKUP_DIR}"
ls -lh "${BACKUP_DIR}/" 2>/dev/null || true
}
# Read-only data integrity audit (scripts/db-integrity-audit.sql).
# Every block must return 0 rows; any output aborts the deploy.
audit_database() {
if [ ! -f "$DEPLOY_DIR/.env" ]; then
return 0
fi
set -a
# shellcheck disable=SC1091
source "$DEPLOY_DIR/.env"
set +a
local audit_sql="${DEPLOY_DIR}/scripts/db-integrity-audit.sql"
if [ ! -f "$audit_sql" ]; then
log "WARNING: ${audit_sql} not found — skipping integrity audit"
return 0
fi
if ! docker compose -f "$DEPLOY_DIR/docker-compose.yml" ps -q db >/dev/null 2>&1; then
log "WARNING: db service not running — skipping integrity audit"
return 0
fi
log "Running read-only integrity audit..."
local findings
findings="$(docker compose -f "$DEPLOY_DIR/docker-compose.yml" exec -T db \
psql -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -t -A -q -v ON_ERROR_STOP=1 -f - \
< "$audit_sql" 2>/dev/null || true)"
if [ -n "$findings" ]; then
log "ERROR: integrity audit returned findings:"
echo "$findings" | tee -a "$LOG_FILE"
error_exit "integrity audit failed — fix the data before deploying"
fi
log "Integrity audit clean (0 rows)."
}
install_cron() {
local schedule="0 2 * * 0"
local cmd="${DEPLOY_DIR}/scripts/deploy.sh backup-db >> /var/log/bikontrol-backup.log 2>&1"
local cron_line="${schedule} ${cmd}"
# Allow custom schedule: ./scripts/deploy.sh install-cron "0 3 * * 0"
if [ -n "${2:-}" ]; then
# second argument overrides schedule
cron_line="${2} ${cmd}"
fi
log "Installing weekly backup cron: ${cron_line}"
# Ensure backup dir + log file exist
mkdir -p "${DEPLOY_DIR}/backups"
touch /var/log/bikontrol-backup.log 2>/dev/null || true
# Remove any previous bikontrol backup crons to avoid duplicates
(crontab -l 2>/dev/null | grep -v "bikontrol.*backup-db" || true; echo "$cron_line") | crontab -
log "Cron installed. Current crontab:"
crontab -l 2>/dev/null | grep bikontrol || true
log "Backups run weekly on Sunday 02:00 UTC. Change with: ./scripts/deploy.sh install-cron \"<cron>\""
}
remove_cron() {
log "Removing bikontrol backup cron..."
(crontab -l 2>/dev/null | grep -v "bikontrol.*backup-db" || true) | crontab -
log "Cron removed."
}
pull() {
log "Pulling latest code..."
(cd "$DEPLOY_DIR" && git fetch origin main && git reset --hard origin/main)
log "Code updated."
}
build() {
log "Building images..."
(cd "$DEPLOY_DIR" && docker compose build --no-cache)
log "Build complete."
}
# Apply EF Core migrations as a dedicated, one-shot step — before the API
# starts and without racing replicas. Uses the freshly built API image in
# migration-only mode (`--migrate`), on the internal network so it can reach
# the database. Idempotent: already-applied migrations are skipped.
migrate() {
log "Applying database migrations (dedicated step)..."
# The DB must be reachable; start it (idempotent) and give it a moment. The
# api service depends on a healthy db, but `run` starts on demand.
(cd "$DEPLOY_DIR" && docker compose up -d db)
sleep 5
(cd "$DEPLOY_DIR" && \
docker compose run --rm --no-deps api --migrate) \
|| error_exit "database migration failed — aborting (nothing was started)"
log "Migrations applied."
}
up() {
log "Starting containers..."
(cd "$DEPLOY_DIR" && docker compose up -d --remove-orphans)
log "Containers started."
}
wait_healthy() {
log "Waiting for services to become healthy..."
local TIMEOUT=180
local INTERVAL=5
local ELAPSED=0
while [ $ELAPSED -lt $TIMEOUT ]; do
local API_STATUS WEB_STATUS
API_STATUS=$(docker inspect --format='{{.State.Health.Status}}' bikontrol-api 2>/dev/null || echo "not_found")
WEB_STATUS=$(docker inspect --format='{{.State.Health.Status}}' bikontrol 2>/dev/null || echo "not_found")
if [ "$API_STATUS" = "healthy" ] && [ "$WEB_STATUS" = "healthy" ]; then
log "Services healthy."
return 0
fi
log "Waiting... ($ELAPSED/$TIMEOUT) api: $API_STATUS web: $WEB_STATUS"
sleep $INTERVAL
ELAPSED=$((ELAPSED + INTERVAL))
done
(cd "$DEPLOY_DIR" && docker compose ps && docker compose logs --tail=50)
error_exit "Health check timeout"
}
verify() {
log "Verifying deployment..."
local API_STATUS WEB_STATUS
API_STATUS=$(docker inspect --format='{{.State.Health.Status}}' bikontrol-api 2>/dev/null || echo "not_found")
WEB_STATUS=$(docker inspect --format='{{.State.Health.Status}}' bikontrol 2>/dev/null || echo "not_found")
[ "$API_STATUS" = "healthy" ] || error_exit "bikontrol-api is not healthy (status: $API_STATUS)"
[ "$WEB_STATUS" = "healthy" ] || error_exit "bikontrol is not healthy (status: $WEB_STATUS)"
log "Deployment verified (containers healthy)."
}
# End-to-end smoke test against the public origin. Catches regressions that the
# container healthcheck cannot (e.g. the gateway not routing, a broken build
# being served, the SPA shell not loading).
smoke() {
local script="${DEPLOY_DIR}/scripts/smoke-test.sh"
if [ ! -f "$script" ]; then
log "WARNING: ${script} not found — skipping smoke test"
return 0
fi
log "Running post-deploy smoke test against ${SMOKE_URL} ..."
if ! SMOKE_TIMEOUT="${SMOKE_TIMEOUT:-10}" "$script" "$SMOKE_URL"; then
error_exit "smoke test failed against ${SMOKE_URL}"
fi
log "Smoke test passed."
}
rollback() {
log "Rolling back..."
(cd "$DEPLOY_DIR" && docker compose down) || true
# Support both .sql and .sql.gz backups
local _dump=""
if [ -f "${BACKUP_DIR}/db.sql.gz" ]; then _dump="${BACKUP_DIR}/db.sql.gz"
elif [ -f "${BACKUP_DIR}/db.sql" ]; then _dump="${BACKUP_DIR}/db.sql"
fi
if [ -n "$_dump" ] && [ -f "$DEPLOY_DIR/.env" ]; then
set -a
# shellcheck disable=SC1091
source "$DEPLOY_DIR/.env"
set +a
log "Starting db to restore dump..."
(cd "$DEPLOY_DIR" && docker compose up -d db)
sleep 10
if [[ "$_dump" == *.gz ]]; then
gunzip -c "$_dump" | docker compose -f "$DEPLOY_DIR/docker-compose.yml" exec -T db \
psql -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" \
|| log "WARNING: database restore failed"
else
docker compose -f "$DEPLOY_DIR/docker-compose.yml" exec -T db \
psql -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" < "$_dump" \
|| log "WARNING: database restore failed"
fi
fi
if [ -f "${BACKUP_DIR}.tgz" ]; then
log "Restoring config backup..."
tar xzf "${BACKUP_DIR}.tgz" -C /
fi
(cd "$DEPLOY_DIR" && docker compose up -d --remove-orphans)
log "Rollback complete."
# Best-effort: confirm the restored stack actually serves (do not fail the
# rollback itself if the smoke test has a transient hiccup).
if [ -f "${DEPLOY_DIR}/scripts/smoke-test.sh" ]; then
log "Verifying rollback (best effort) ..."
SMOKE_TIMEOUT="${SMOKE_TIMEOUT:-10}" "${DEPLOY_DIR}/scripts/smoke-test.sh" "$SMOKE_URL" \
|| log "WARNING: smoke test after rollback did not pass — investigate"
fi
}
deploy() {
log "=== Starting Bikontrol deployment ==="
validate_docker
validate_env
validate_config
backup
backup_database
audit_database
pull
build
migrate
up
if ! wait_healthy; then
log "Deployment failed health check — rolling back."
rollback
error_exit "Deployment failed and was rolled back"
fi
verify
# A green healthcheck is not enough: prove the public surface actually works
# end-to-end, and roll back if it does not.
if ! smoke; then
log "Deployment failed smoke test — rolling back."
rollback
error_exit "Deployment failed the smoke test and was rolled back"
fi
log "=== Deployment successful ==="
status
}
status() {
(cd "$DEPLOY_DIR" && docker compose ps)
echo ""
(cd "$DEPLOY_DIR" && docker compose logs --tail=20)
}
logs() {
(cd "$DEPLOY_DIR" && docker compose logs --tail=100)
}
case "${1:-deploy}" in
pull) pull ;;
build) build ;;
migrate) migrate ;;
up) up ;;
deploy) deploy ;;
status) status ;;
logs) logs ;;
verify) verify ;;
smoke) smoke ;;
rollback) rollback ;;
backup|backup-db|backup-only) backup_only ;;
audit-db) audit_database ;;
install-cron) install_cron "$@" ;;
remove-cron) remove_cron ;;
*)
echo "Usage: $0 [pull|build|migrate|up|deploy|status|logs|verify|smoke|rollback|backup-db|audit-db|install-cron|remove-cron]"
exit 1
;;
esac