mirror of
https://github.com/TheRealOwenRees/chess-endgame-trainer.git
synced 2026-11-18 03:32:06 +00:00
The VPS now takes a daily logical dump, verifies it, encrypts it, and hands it to a workstation; a monthly drill proves the archive restores. - deploy/backup: pg_dump -Fc plus pg_dumpall --globals-only, pg_restore -l and globals-role checks before anything is promoted, row counts for the drill, sha256, age encryption into offsite/, retention (3d plaintext, 14d encrypted, 5 pre-deploy sets), JSON logging and a healthchecks.io heartbeat - systemd service and timer (03:15 UTC, Persistent, jittered) plus an OnFailure unit that logs a distinct event - deploy/activate takes a pre-deploy dump after extracting and before flipping current, because migrations run in ExecStartPre; a failed dump aborts the activation, with SKIP_PRE_DEPLOY_BACKUP=1 as the escape hatch - deploy/workstation/backup-pull: rsync the encrypted archives over SSH, verify by decrypting and checking the inner sha256, retain 7d/4w/6m, and fail when the newest archive is older than 48h - deploy/workstation/backup-drill: restore the newest archive into a throwaway postgres:18.6, replay the globals dump as the same superuser, assert both roles, compare users/endgames/attempts, and append the result to drills.log - user units for the daily pull and monthly drill, with a shared OnFailure alert template - docs: install, key handling, retention, restore, disaster recovery, a run by hand section, monitoring and troubleshooting Verified against production: the first backup completed and pinged the heartbeat, the pull verified the archive, and the drill restored users=1 endgames=39 attempts=0, matching the live database.
114 lines
3.5 KiB
Bash
Executable File
114 lines
3.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Activate a timestamped release on the VPS.
|
|
#
|
|
# Usage:
|
|
# sudo deploy/activate /tmp/chesstrainer-20261109.tar.gz
|
|
#
|
|
# Environment overrides:
|
|
# APP_USER, APP_ROOT, SERVICE, HEALTH_URL, KEEP_RELEASES, RELEASE_ID,
|
|
# BACKUP_BIN, SKIP_PRE_DEPLOY_BACKUP=1
|
|
|
|
if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then
|
|
echo "activate: run with sudo" >&2
|
|
exit 1
|
|
fi
|
|
|
|
TARBALL="${1:-}"
|
|
if [[ -z "$TARBALL" || ! -f "$TARBALL" ]]; then
|
|
echo "usage: sudo $0 /path/to/chesstrainer-release.tar.gz" >&2
|
|
exit 1
|
|
fi
|
|
|
|
APP_USER="${APP_USER:-chesstrainer}"
|
|
APP_ROOT="${APP_ROOT:-/opt/chesstrainer}"
|
|
SERVICE="${SERVICE:-chesstrainer}"
|
|
HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:4100/health}"
|
|
KEEP_RELEASES="${KEEP_RELEASES:-5}"
|
|
RELEASE_ID="${RELEASE_ID:-$(date -u +%Y%m%d%H%M%S)}"
|
|
BACKUP_BIN="${BACKUP_BIN:-/usr/local/sbin/chesstrainer-backup}"
|
|
SKIP_PRE_DEPLOY_BACKUP="${SKIP_PRE_DEPLOY_BACKUP:-0}"
|
|
|
|
if [[ ! "$RELEASE_ID" =~ ^[A-Za-z0-9._-]+$ ]]; then
|
|
echo "activate: RELEASE_ID may contain only letters, digits, dot, underscore and dash" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! "$KEEP_RELEASES" =~ ^[0-9]+$ ]] || [[ "$KEEP_RELEASES" -lt 1 ]]; then
|
|
echo "activate: KEEP_RELEASES must be a positive integer" >&2
|
|
exit 1
|
|
fi
|
|
|
|
health_check() {
|
|
curl --fail --silent --show-error \
|
|
--retry 10 --retry-delay 2 --retry-connrefused \
|
|
"$HEALTH_URL" >/dev/null
|
|
}
|
|
|
|
install -d -o "$APP_USER" -g "$APP_USER" "$APP_ROOT/releases"
|
|
|
|
PREVIOUS="$(readlink -f "$APP_ROOT/current" 2>/dev/null || true)"
|
|
RELEASE_DIR="$APP_ROOT/releases/$RELEASE_ID"
|
|
|
|
echo "activate: extracting $TARBALL -> $RELEASE_DIR"
|
|
install -d -o "$APP_USER" -g "$APP_USER" "$RELEASE_DIR"
|
|
tar -xzf "$TARBALL" -C "$RELEASE_DIR"
|
|
chown -R "$APP_USER:$APP_USER" "$RELEASE_DIR"
|
|
|
|
if [[ "$SKIP_PRE_DEPLOY_BACKUP" == "1" ]]; then
|
|
echo "activate: SKIP_PRE_DEPLOY_BACKUP=1, not taking a pre-deploy dump"
|
|
else
|
|
backup_script="$BACKUP_BIN"
|
|
if [[ ! -x "$backup_script" ]]; then
|
|
backup_script="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/backup"
|
|
fi
|
|
|
|
if [[ ! -x "$backup_script" ]]; then
|
|
echo "activate: no backup script at $BACKUP_BIN or next to activate" >&2
|
|
echo "activate: install deploy/backup or set SKIP_PRE_DEPLOY_BACKUP=1" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Migrations run in ExecStartPre after this, so the dump must exist first.
|
|
echo "activate: taking a pre-deploy dump"
|
|
if ! BACKUP_TAG=predeploy LOCK_WAIT=120 "$backup_script"; then
|
|
echo "activate: pre-deploy dump failed, aborting" >&2
|
|
echo "activate: fix the backup or re-run with SKIP_PRE_DEPLOY_BACKUP=1" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
echo "activate: pointing current at $RELEASE_DIR"
|
|
ln -sfn "$RELEASE_DIR" "$APP_ROOT/current"
|
|
|
|
if systemctl restart "$SERVICE" && health_check; then
|
|
echo "activate: $RELEASE_ID is live"
|
|
|
|
if [[ "$KEEP_RELEASES" -gt 0 ]]; then
|
|
find "$APP_ROOT/releases" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' \
|
|
| sort -nr \
|
|
| tail -n +"$((KEEP_RELEASES + 1))" \
|
|
| cut -d' ' -f2- \
|
|
| while read -r old_release; do
|
|
[[ -z "$old_release" ]] && continue
|
|
[[ "$old_release" == "$(readlink -f "$APP_ROOT/current")" ]] && continue
|
|
echo "activate: pruning $old_release"
|
|
rm -rf -- "$old_release"
|
|
done
|
|
fi
|
|
|
|
exit 0
|
|
fi
|
|
|
|
echo "activate: health check failed" >&2
|
|
|
|
if [[ -n "$PREVIOUS" && -d "$PREVIOUS" ]]; then
|
|
echo "activate: rolling back to $PREVIOUS" >&2
|
|
ln -sfn "$PREVIOUS" "$APP_ROOT/current"
|
|
systemctl restart "$SERVICE" || true
|
|
health_check || echo "activate: rollback health check also failed" >&2
|
|
fi
|
|
|
|
exit 1
|