mirror of
https://github.com/TheRealOwenRees/chess-endgame-trainer.git
synced 2026-11-18 11:42:07 +00:00
The VPS now takes a daily logical dump, verifies it, encrypts it, and hands it to a workstation; a monthly drill proves the archive restores. - deploy/backup: pg_dump -Fc plus pg_dumpall --globals-only, pg_restore -l and globals-role checks before anything is promoted, row counts for the drill, sha256, age encryption into offsite/, retention (3d plaintext, 14d encrypted, 5 pre-deploy sets), JSON logging and a healthchecks.io heartbeat - systemd service and timer (03:15 UTC, Persistent, jittered) plus an OnFailure unit that logs a distinct event - deploy/activate takes a pre-deploy dump after extracting and before flipping current, because migrations run in ExecStartPre; a failed dump aborts the activation, with SKIP_PRE_DEPLOY_BACKUP=1 as the escape hatch - deploy/workstation/backup-pull: rsync the encrypted archives over SSH, verify by decrypting and checking the inner sha256, retain 7d/4w/6m, and fail when the newest archive is older than 48h - deploy/workstation/backup-drill: restore the newest archive into a throwaway postgres:18.6, replay the globals dump as the same superuser, assert both roles, compare users/endgames/attempts, and append the result to drills.log - user units for the daily pull and monthly drill, with a shared OnFailure alert template - docs: install, key handling, retention, restore, disaster recovery, a run by hand section, monitoring and troubleshooting Verified against production: the first backup completed and pinged the heartbeat, the pull verified the archive, and the drill restored users=1 endgames=39 attempts=0, matching the live database.
This commit is contained in:
Executable
+190
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Restore the newest (or a named) archive into a throwaway Postgres container
|
||||
# and prove that the dump, the roles and the row counts all came back.
|
||||
#
|
||||
# backup-drill # newest archive in DEST
|
||||
# backup-drill 20261111T031500Z.tar.age
|
||||
#
|
||||
# Runs on the workstation. The production database is never contacted: the
|
||||
# container is created and destroyed by this script. Results are appended to
|
||||
# $DRILL_LOG (default $DEST/drills.log).
|
||||
#
|
||||
# Configuration: same file as backup-pull,
|
||||
# ~/.config/chesstrainer/backup-pull.env.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
CONFIG="${PULL_ENV_FILE:-${XDG_CONFIG_HOME:-$HOME/.config}/chesstrainer/backup-pull.env}"
|
||||
if [[ -r "$CONFIG" ]]; then
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG"
|
||||
fi
|
||||
|
||||
DEST="${DEST:-$HOME/chesstrainer-backups}"
|
||||
AGE_KEY="${AGE_KEY:-${XDG_CONFIG_HOME:-$HOME/.config}/chesstrainer/age.key}"
|
||||
EXPECTED_ROLES="${EXPECTED_ROLES:-chesstrainer chesstrainer_ro}"
|
||||
DRILL_IMAGE="${DRILL_IMAGE:-postgres:18.6}"
|
||||
DRILL_DB="${DRILL_DB:-chesstrainer_restore}"
|
||||
DRILL_LOG="${DRILL_LOG:-$DEST/drills.log}"
|
||||
DRILL_CONTAINER="${DRILL_CONTAINER:-chesstrainer-drill-$$}"
|
||||
|
||||
START_SECONDS=$SECONDS
|
||||
FAILURE=""
|
||||
TMPDIR_DRILL=""
|
||||
|
||||
log_event() {
|
||||
local event="$1"
|
||||
local extra="${2:-}"
|
||||
|
||||
printf '{"timestamp":"%s","event":"%s"%s}\n' \
|
||||
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$event" "$extra"
|
||||
}
|
||||
|
||||
record() {
|
||||
local line="$1"
|
||||
|
||||
printf '%s\n' "$line"
|
||||
printf '%s\n' "$line" >>"$DRILL_LOG"
|
||||
}
|
||||
|
||||
fail() {
|
||||
FAILURE="$*"
|
||||
exit 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
|
||||
docker rm -f "$DRILL_CONTAINER" >/dev/null 2>&1 || true
|
||||
|
||||
if [[ -n "$TMPDIR_DRILL" ]]; then
|
||||
rm -rf "$TMPDIR_DRILL"
|
||||
fi
|
||||
|
||||
if ((status != 0)); then
|
||||
if [[ -d "$DEST" ]]; then
|
||||
record "$(log_event drill.failed ",\"reason\":\"${FAILURE:-exit $status}\"")"
|
||||
else
|
||||
log_event drill.failed ",\"reason\":\"${FAILURE:-exit $status}\""
|
||||
fi
|
||||
fi
|
||||
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
trap 'exit 143' TERM INT
|
||||
|
||||
[[ -d "$DEST" ]] || fail "archive directory does not exist: $DEST"
|
||||
[[ -r "$AGE_KEY" ]] || fail "age identity not readable: $AGE_KEY"
|
||||
|
||||
for cmd in docker age tar sha256sum; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || fail "missing command: $cmd"
|
||||
done
|
||||
|
||||
if [[ $# -gt 0 ]]; then
|
||||
case "$1" in
|
||||
/*) archive="$1" ;;
|
||||
*) archive="$DEST/$1" ;;
|
||||
esac
|
||||
else
|
||||
archive="$(find "$DEST" -maxdepth 1 -type f -name '*.tar.age' -printf '%p\n' | sort | tail -n 1)"
|
||||
[[ -n "$archive" ]] || fail "no archives in $DEST"
|
||||
fi
|
||||
|
||||
[[ -f "$archive" ]] || fail "archive not found: $archive"
|
||||
|
||||
name="${archive##*/}"
|
||||
[[ "$name" =~ ^[0-9]{8}T[0-9]{6}Z(-[a-z0-9]+)?\.tar\.age$ ]] ||
|
||||
fail "unexpected archive name: $name"
|
||||
|
||||
prefix="${name%%.*}"
|
||||
TMPDIR_DRILL="$(mktemp -d "${TMPDIR:-/tmp}/chesstrainer-drill.XXXXXX")"
|
||||
|
||||
echo "backup-drill: decrypting $name" >&2
|
||||
age --decrypt --identity "$AGE_KEY" --output "$TMPDIR_DRILL/$prefix.tar" "$archive" ||
|
||||
fail "could not decrypt $name"
|
||||
|
||||
tar -xf "$TMPDIR_DRILL/$prefix.tar" -C "$TMPDIR_DRILL" ||
|
||||
fail "archive does not contain a readable tar file"
|
||||
|
||||
(
|
||||
cd "$TMPDIR_DRILL"
|
||||
sha256sum --check --strict --quiet "$prefix.sha256"
|
||||
) || fail "checksum verification failed for $name"
|
||||
|
||||
echo "backup-drill: starting $DRILL_IMAGE" >&2
|
||||
docker image inspect "$DRILL_IMAGE" >/dev/null 2>&1 ||
|
||||
docker pull "$DRILL_IMAGE" >/dev/null ||
|
||||
fail "could not pull $DRILL_IMAGE"
|
||||
|
||||
docker run --rm --detach \
|
||||
--name "$DRILL_CONTAINER" \
|
||||
--env POSTGRES_USER=chesstrainer \
|
||||
--env POSTGRES_PASSWORD=drill \
|
||||
--env POSTGRES_HOST_AUTH_METHOD=trust \
|
||||
"$DRILL_IMAGE" >/dev/null || fail "could not start the drill container"
|
||||
|
||||
ready=0
|
||||
for _ in $(seq 1 60); do
|
||||
if docker exec "$DRILL_CONTAINER" pg_isready -U chesstrainer -d postgres >/dev/null 2>&1; then
|
||||
ready=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
((ready == 1)) || fail "postgres did not become ready"
|
||||
|
||||
# Recreating the cluster roles is half of disaster recovery. The scratch
|
||||
# cluster uses the same superuser name as production (`chesstrainer`), because
|
||||
# the globals dump ends with grants issued "GRANTED BY chesstrainer". The dump
|
||||
# always repeats the cluster's own superuser role, so "already exists" is
|
||||
# expected; anything else is a real failure.
|
||||
echo "backup-drill: applying globals" >&2
|
||||
docker exec -i "$DRILL_CONTAINER" psql -U chesstrainer -d postgres \
|
||||
-q -v ON_ERROR_STOP=0 -f - \
|
||||
<"$TMPDIR_DRILL/$prefix.globals.sql" >/dev/null 2>"$TMPDIR_DRILL/globals.err" ||
|
||||
fail "applying the globals dump failed"
|
||||
|
||||
globals_errors="$(grep -v 'already exists' "$TMPDIR_DRILL/globals.err" | sed '/^$/d' || true)"
|
||||
if [[ -n "$globals_errors" ]]; then
|
||||
fail "globals restore reported errors: $(printf '%s' "$globals_errors" | head -n 1)"
|
||||
fi
|
||||
|
||||
for role in $EXPECTED_ROLES; do
|
||||
found="$(docker exec "$DRILL_CONTAINER" psql -U chesstrainer -d postgres -Atc \
|
||||
"select 1 from pg_roles where rolname = '$role'")"
|
||||
[[ "$found" == "1" ]] || fail "globals did not recreate role $role"
|
||||
done
|
||||
|
||||
owner="${EXPECTED_ROLES%% *}"
|
||||
|
||||
docker exec "$DRILL_CONTAINER" psql -U chesstrainer -d postgres \
|
||||
-q -v ON_ERROR_STOP=1 -c "create database $DRILL_DB owner $owner" ||
|
||||
fail "could not create $DRILL_DB"
|
||||
|
||||
echo "backup-drill: restoring into $DRILL_DB" >&2
|
||||
docker exec -i "$DRILL_CONTAINER" pg_restore -U chesstrainer -d "$DRILL_DB" \
|
||||
--single-transaction --exit-on-error <"$TMPDIR_DRILL/$prefix.dump" ||
|
||||
fail "pg_restore failed"
|
||||
|
||||
restored_counts=""
|
||||
for table in users endgames attempts; do
|
||||
expected="$(awk -F= -v key="$table" '$1 == key { print $2 }' "$TMPDIR_DRILL/$prefix.counts")"
|
||||
[[ "$expected" =~ ^[0-9]+$ ]] || fail "counts file has no value for $table"
|
||||
|
||||
actual="$(docker exec "$DRILL_CONTAINER" psql -U chesstrainer -d "$DRILL_DB" -Atc \
|
||||
"select count(*) from $table")"
|
||||
[[ "$actual" =~ ^[0-9]+$ ]] || fail "could not count $table"
|
||||
|
||||
if ((actual < expected)); then
|
||||
fail "restored $table=$actual is below the recorded $expected"
|
||||
fi
|
||||
|
||||
echo "backup-drill: $table restored=$actual recorded=$expected drift=$((actual - expected))" >&2
|
||||
restored_counts+="${restored_counts:+,}$table=$actual"
|
||||
done
|
||||
|
||||
record "$(log_event drill.completed ",\"archive\":\"$name\",\"result\":\"pass\",\"counts\":\"$restored_counts\",\"duration_s\":$((SECONDS - START_SECONDS))")"
|
||||
Executable
+248
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Pull the encrypted database archives from the VPS, verify them end to end,
|
||||
# and apply the 7 daily / 4 weekly / 6 monthly retention policy.
|
||||
#
|
||||
# Runs on the workstation, never on the server. Install as
|
||||
# ~/.local/bin/chesstrainer-backup-pull and enable
|
||||
# chesstrainer-backup-pull.timer as a user unit.
|
||||
#
|
||||
# Configuration: ~/.config/chesstrainer/backup-pull.env (0600). See
|
||||
# deploy/README.md §Backups.
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
CONFIG="${PULL_ENV_FILE:-${XDG_CONFIG_HOME:-$HOME/.config}/chesstrainer/backup-pull.env}"
|
||||
if [[ -r "$CONFIG" ]]; then
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG"
|
||||
fi
|
||||
|
||||
VPS_HOST="${VPS_HOST:-}"
|
||||
REMOTE_DIR="${REMOTE_DIR:-/var/backups/chesstrainer/offsite}"
|
||||
DEST="${DEST:-$HOME/chesstrainer-backups}"
|
||||
AGE_KEY="${AGE_KEY:-${XDG_CONFIG_HOME:-$HOME/.config}/chesstrainer/age.key}"
|
||||
MAX_STALENESS_HOURS="${MAX_STALENESS_HOURS:-48}"
|
||||
KEEP_DAILY_DAYS="${KEEP_DAILY_DAYS:-7}"
|
||||
KEEP_WEEKLY_WEEKS="${KEEP_WEEKLY_WEEKS:-4}"
|
||||
KEEP_MONTHLY_MONTHS="${KEEP_MONTHLY_MONTHS:-6}"
|
||||
PULL_HEALTHCHECK_URL="${PULL_HEALTHCHECK_URL:-}"
|
||||
SSH_OPTIONS="${SSH_OPTIONS:--o BatchMode=yes -o ConnectTimeout=10 -o ServerAliveInterval=15}"
|
||||
RSYNC_ATTEMPTS="${RSYNC_ATTEMPTS:-3}"
|
||||
|
||||
START_SECONDS=$SECONDS
|
||||
FAILURE=""
|
||||
VERIFIED=0
|
||||
|
||||
log_event() {
|
||||
local event="$1"
|
||||
local extra="${2:-}"
|
||||
|
||||
printf '{"timestamp":"%s","event":"%s"%s}\n' \
|
||||
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$event" "$extra"
|
||||
}
|
||||
|
||||
heartbeat() {
|
||||
local outcome="$1"
|
||||
|
||||
[[ -n "$PULL_HEALTHCHECK_URL" ]] || return 0
|
||||
|
||||
local url="${PULL_HEALTHCHECK_URL%/}"
|
||||
[[ "$outcome" == "fail" ]] && url="$url/fail"
|
||||
|
||||
if ! curl --fail --silent --show-error \
|
||||
--max-time 10 --retry 2 --retry-delay 2 \
|
||||
--output /dev/null "$url"; then
|
||||
echo "backup-pull: heartbeat ping failed (run result is unaffected)" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
fail() {
|
||||
FAILURE="$*"
|
||||
exit 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
|
||||
if [[ -d "$DEST" ]]; then
|
||||
find "$DEST" -maxdepth 1 -name '.verify.*' -type d -exec rm -rf {} + 2>/dev/null || true
|
||||
fi
|
||||
|
||||
if ((status != 0)); then
|
||||
log_event pull.failed ",\"reason\":\"${FAILURE:-exit $status}\""
|
||||
heartbeat fail
|
||||
fi
|
||||
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
trap 'exit 143' TERM INT
|
||||
|
||||
[[ -n "$VPS_HOST" ]] || fail "VPS_HOST is not set"
|
||||
[[ -r "$AGE_KEY" ]] || fail "age identity not readable: $AGE_KEY"
|
||||
[[ "$MAX_STALENESS_HOURS" =~ ^[0-9]+$ ]] || fail "MAX_STALENESS_HOURS must be an integer"
|
||||
[[ "$KEEP_DAILY_DAYS" =~ ^[0-9]+$ ]] || fail "KEEP_DAILY_DAYS must be an integer"
|
||||
[[ "$KEEP_WEEKLY_WEEKS" =~ ^[0-9]+$ ]] || fail "KEEP_WEEKLY_WEEKS must be an integer"
|
||||
[[ "$KEEP_MONTHLY_MONTHS" =~ ^[0-9]+$ ]] || fail "KEEP_MONTHLY_MONTHS must be an integer"
|
||||
[[ "$RSYNC_ATTEMPTS" =~ ^[0-9]+$ ]] || fail "RSYNC_ATTEMPTS must be an integer"
|
||||
|
||||
for cmd in rsync ssh age tar sha256sum curl flock; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || fail "missing command: $cmd"
|
||||
done
|
||||
|
||||
# Never create DEST on the root filesystem when the backup disk is unmounted.
|
||||
[[ -d "$(dirname "$DEST")" ]] ||
|
||||
fail "parent directory of $DEST does not exist (is the backup disk mounted?)"
|
||||
|
||||
install -d -m 0700 "$DEST"
|
||||
install -d -m 0700 "$DEST/.verified"
|
||||
|
||||
exec 9>"$DEST/.pull.lock"
|
||||
if ! flock -n 9; then
|
||||
# A manual run overlapping the timer is not an error; the other run reports.
|
||||
log_event pull.skipped ',"reason":"another pull is running"'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
archive_epoch() {
|
||||
local name="$1"
|
||||
local ts="${name:0:16}"
|
||||
|
||||
[[ "$ts" =~ ^[0-9]{8}T[0-9]{6}Z$ ]] || return 1
|
||||
|
||||
date -u -d "${ts:0:4}-${ts:4:2}-${ts:6:2} ${ts:9:2}:${ts:11:2}:${ts:13:2}Z" +%s
|
||||
}
|
||||
|
||||
verify_archive() {
|
||||
local archive="$1"
|
||||
local name="${archive##*/}"
|
||||
local marker="$DEST/.verified/$name"
|
||||
local sum tmp
|
||||
|
||||
sum="$(sha256sum "$archive" | awk '{ print $1 }')"
|
||||
|
||||
if [[ -f "$marker" && "$(cat "$marker")" == "$sum" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
tmp="$(mktemp -d "$DEST/.verify.XXXXXX")"
|
||||
|
||||
if ! age --decrypt --identity "$AGE_KEY" --output "$tmp/$name.tar" "$archive"; then
|
||||
rm -rf "$tmp"
|
||||
echo "backup-pull: $name failed to decrypt" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! tar -xf "$tmp/$name.tar" -C "$tmp"; then
|
||||
rm -rf "$tmp"
|
||||
echo "backup-pull: $name did not contain a readable tar archive" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! (cd "$tmp" && sha256sum --check --strict --quiet ./*.sha256); then
|
||||
rm -rf "$tmp"
|
||||
echo "backup-pull: $name failed its checksum verification" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm -rf "$tmp"
|
||||
|
||||
printf '%s\n' "$sum" >"$marker.tmp"
|
||||
mv "$marker.tmp" "$marker"
|
||||
|
||||
VERIFIED=$((VERIFIED + 1))
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Copy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
attempt=1
|
||||
while true; do
|
||||
if rsync --archive --partial --compress --timeout=120 \
|
||||
--include='*.tar.age' --exclude='*' \
|
||||
--rsh "ssh $SSH_OPTIONS" \
|
||||
"$VPS_HOST:$REMOTE_DIR/" "$DEST/"; then
|
||||
break
|
||||
fi
|
||||
|
||||
if ((attempt >= RSYNC_ATTEMPTS)); then
|
||||
fail "rsync from $VPS_HOST failed after $attempt attempts"
|
||||
fi
|
||||
|
||||
echo "backup-pull: rsync attempt $attempt failed, retrying" >&2
|
||||
attempt=$((attempt + 1))
|
||||
sleep 20
|
||||
done
|
||||
|
||||
mapfile -t archives < <(
|
||||
find "$DEST" -maxdepth 1 -type f -name '*.tar.age' -printf '%f\n' | sort
|
||||
)
|
||||
|
||||
((${#archives[@]} > 0)) || fail "no archives in $DEST after the pull"
|
||||
|
||||
for name in "${archives[@]}"; do
|
||||
[[ "$name" =~ ^[0-9]{8}T[0-9]{6}Z(-[a-z0-9]+)?\.tar\.age$ ]] ||
|
||||
fail "unexpected file in $DEST: $name"
|
||||
done
|
||||
|
||||
newest="${archives[${#archives[@]} - 1]}"
|
||||
newest_epoch="$(archive_epoch "$newest")" || fail "cannot parse a timestamp from $newest"
|
||||
age_seconds=$(($(date -u +%s) - newest_epoch))
|
||||
|
||||
if ((age_seconds > MAX_STALENESS_HOURS * 3600)); then
|
||||
fail "newest archive $newest is $((age_seconds / 3600))h old (limit ${MAX_STALENESS_HOURS}h): VPS backups may have stopped"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Verify every archive once; unchanged ciphertext is only re-hashed.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
for name in "${archives[@]}"; do
|
||||
verify_archive "$DEST/$name" || fail "$name failed verification"
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Retention: 7 daily, then one per ISO week for 4 weeks, then one per calendar
|
||||
# month for 6 months. Archives outside all three windows are deleted.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
now="$(date -u +%s)"
|
||||
pruned=0
|
||||
declare -A kept_week=()
|
||||
declare -A kept_month=()
|
||||
|
||||
mapfile -t newest_first < <(
|
||||
find "$DEST" -maxdepth 1 -type f -name '*.tar.age' -printf '%f\n' | sort -r
|
||||
)
|
||||
|
||||
for name in "${newest_first[@]}"; do
|
||||
epoch="$(archive_epoch "$name")" || continue
|
||||
age_days=$(((now - epoch) / 86400))
|
||||
|
||||
if ((age_days < KEEP_DAILY_DAYS)); then
|
||||
continue
|
||||
fi
|
||||
|
||||
week="$(date -u -d "@$epoch" +%G-W%V)"
|
||||
if ((age_days < KEEP_WEEKLY_WEEKS * 7)) && [[ -z "${kept_week[$week]:-}" ]]; then
|
||||
kept_week[$week]=1
|
||||
continue
|
||||
fi
|
||||
|
||||
month="$(date -u -d "@$epoch" +%Y-%m)"
|
||||
if ((age_days < KEEP_MONTHLY_MONTHS * 31)) && [[ -z "${kept_month[$month]:-}" ]]; then
|
||||
kept_month[$month]=1
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "backup-pull: pruning $name" >&2
|
||||
rm -f -- "$DEST/$name" "$DEST/.verified/$name"
|
||||
pruned=$((pruned + 1))
|
||||
done
|
||||
|
||||
log_event pull.completed ",\"newest\":\"$newest\",\"archives\":$(( ${#archives[@]} - pruned )),\"verified\":$VERIFIED,\"pruned\":$pruned,\"duration_s\":$((SECONDS - START_SECONDS))"
|
||||
heartbeat ok
|
||||
@@ -0,0 +1,37 @@
|
||||
# ~/.config/chesstrainer/backup-pull.env -- mode 0600.
|
||||
#
|
||||
# Sourced by ~/.local/bin/chesstrainer-backup-pull. Values below are the
|
||||
# defaults; VPS_HOST and AGE_KEY must be right. Quote anything containing
|
||||
# spaces.
|
||||
|
||||
VPS_HOST=owen@159.69.21.225
|
||||
REMOTE_DIR=/var/backups/chesstrainer/offsite
|
||||
DEST="/mnt/Mint Data/chesstrainer-backups"
|
||||
|
||||
# Private half of the age key. Back this file up somewhere safe: without it
|
||||
# every archive is unreadable.
|
||||
AGE_KEY="$HOME/.config/chesstrainer/age.key"
|
||||
|
||||
# Fail the pull when the newest archive from the VPS is older than this, which
|
||||
# means the VPS backup timer has stopped.
|
||||
MAX_STALENESS_HOURS=48
|
||||
|
||||
# Keep every archive from the last 7 days, then one per ISO week for 4 weeks,
|
||||
# then one per calendar month for 6 months.
|
||||
KEEP_DAILY_DAYS=7
|
||||
KEEP_WEEKLY_WEEKS=4
|
||||
KEEP_MONTHLY_MONTHS=6
|
||||
|
||||
# Restore drill. It runs in a throwaway container matching the production
|
||||
# Postgres major and appends results to DRILL_LOG.
|
||||
# DRILL_IMAGE=postgres:18.6
|
||||
# DRILL_DB=chesstrainer_restore
|
||||
# DRILL_LOG="$DEST/drills.log"
|
||||
EXPECTED_ROLES="chesstrainer chesstrainer_ro"
|
||||
|
||||
# Optional second heartbeat check with a long period (for example 72h), so you
|
||||
# hear about it when the workstation stops pulling. Leave empty to disable.
|
||||
# PULL_HEALTHCHECK_URL=https://hc-ping.com/00000000-0000-0000-0000-000000000000
|
||||
|
||||
# Extra ssh options, space separated.
|
||||
# SSH_OPTIONS="-o BatchMode=yes -o ConnectTimeout=10 -o ServerAliveInterval=15"
|
||||
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Report a failed Inner Square Chess backup unit
|
||||
Documentation=https://github.com/TheRealOwenRees/chess-endgame-trainer/blob/main/deploy/README.md
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/logger --tag chesstrainer-backup --priority user.err --stderr backup.unit-failed: %i failed, see journalctl --user -u %i
|
||||
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=Monthly restore drill for Inner Square Chess backups
|
||||
Documentation=https://github.com/TheRealOwenRees/chess-endgame-trainer/blob/main/deploy/README.md
|
||||
OnFailure=chesstrainer-backup-alert@%N.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=%h/.local/bin/chesstrainer-backup-drill
|
||||
WorkingDirectory=%h
|
||||
Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
|
||||
Nice=10
|
||||
TimeoutStartSec=20min
|
||||
SyslogIdentifier=chesstrainer-backup-drill
|
||||
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=Monthly restore drill for Inner Square Chess backups
|
||||
Documentation=https://github.com/TheRealOwenRees/chess-endgame-trainer/blob/main/deploy/README.md
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-01 09:00:00
|
||||
RandomizedDelaySec=30m
|
||||
AccuracySec=5m
|
||||
Persistent=true
|
||||
Unit=chesstrainer-backup-drill.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,15 @@
|
||||
[Unit]
|
||||
Description=Pull encrypted Inner Square Chess backups from the VPS
|
||||
Documentation=https://github.com/TheRealOwenRees/chess-endgame-trainer/blob/main/deploy/README.md
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
OnFailure=chesstrainer-backup-alert@%N.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=%h/.local/bin/chesstrainer-backup-pull
|
||||
WorkingDirectory=%h
|
||||
Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
|
||||
Nice=10
|
||||
TimeoutStartSec=15min
|
||||
SyslogIdentifier=chesstrainer-backup-pull
|
||||
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=Daily pull of encrypted Inner Square Chess backups
|
||||
Documentation=https://github.com/TheRealOwenRees/chess-endgame-trainer/blob/main/deploy/README.md
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 12:00:00
|
||||
OnBootSec=10m
|
||||
RandomizedDelaySec=10m
|
||||
AccuracySec=1m
|
||||
Persistent=true
|
||||
Unit=chesstrainer-backup-pull.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user