feat: backup and restore

services/vpn/backup-docker-compose.yml

@@ -0,0 +1,48 @@
+services:
+  # Nightly backup of every named volume on this host to S3. Containers
+  # labeled docker-volume-backup.stop-during-backup=true (see
+  # vpn-docker-compose.yml) are stopped for the duration of the backup so
+  # their data is captured in a consistent state, then restarted.
+  backup:
+    image: offen/docker-volume-backup:v2
+    container_name: backup
+    environment:
+      BACKUP_FILENAME: "vpn-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
+      BACKUP_CRON_EXPRESSION: "0 3 * * *"
+      BACKUP_RETENTION_DAYS: "14"
+      BACKUP_PRUNING_LEEWAY: "5m"
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - openvpn_data:/backup/openvpn_data:ro
+      - letsencrypt_data:/backup/letsencrypt_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+
+  # One-shot: restores empty volumes from the latest S3 backup, then exits.
+  # Run explicitly by spinup.sh before the services that own these volumes
+  # start - not started by `docker compose up`.
+  restore:
+    image: amazon/aws-cli:2
+    container_name: backup-restore
+    entrypoint: ["/bin/bash", "/restore.sh"]
+    environment:
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - ./restore.sh:/restore.sh:ro
+      - openvpn_data:/restore/openvpn_data
+      - letsencrypt_data:/restore/letsencrypt_data
+    restart: "no"
+
+volumes:
+  openvpn_data:
+    name: openvpn_data
+  letsencrypt_data:
+    name: letsencrypt_data

services/vpn/restore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Restores this host's named volumes from the latest S3 backup, but only into
+# volumes that are currently empty - never overwrites data that's already
+# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml
+# run --rm restore` before any service that owns one of the mounted volumes
+# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh
+# estate): it just leaves the empty volumes alone.
+set -euo pipefail
+
+S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}"
+ENDPOINT_ARGS=()
+if [ -n "${AWS_ENDPOINT:-}" ]; then
+  ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}")
+fi
+
+empty_volumes=()
+for dir in /restore/*/; do
+  name="$(basename "$dir")"
+  if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+    empty_volumes+=("$name")
+  fi
+done
+
+if [ "${#empty_volumes[@]}" -eq 0 ]; then
+  echo "All volumes already contain data - skipping restore."
+  exit 0
+fi
+
+echo "Empty volumes: ${empty_volumes[*]}"
+
+latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)"
+if [ -z "$latest" ]; then
+  echo "No existing backup found at ${S3_URI} - starting with empty volumes."
+  exit 0
+fi
+
+echo "Restoring ${S3_URI}/${latest} into empty volumes only..."
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz"
+mkdir "$tmp/extracted"
+tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted"
+
+for name in "${empty_volumes[@]}"; do
+  if [ -d "$tmp/extracted/$name" ]; then
+    echo "Restoring $name..."
+    cp -a "$tmp/extracted/$name/." "/restore/$name/"
+  else
+    echo "No $name/ in backup - leaving volume empty."
+  fi
+done
+
+echo "Restore complete."

services/vpn/spindown.sh

@@ -5,7 +5,9 @@ cd "$(dirname "$0")"
 docker compose -f vpn-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
+docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-docker volume prune -f -a
+# Deliberately no `docker volume prune` - openvpn_data/letsencrypt_data are
+# named volumes holding real data, backed up to S3 but not disposable.

services/vpn/spinup.sh

@@ -3,6 +3,18 @@
 set -e
 cd "$(dirname "$0")"
 
+if [ ! -f .env ]; then
+  echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
+  exit 1
+fi
+
+echo "Restoring named volumes from S3 backup if any are empty..."
+docker compose -f backup-docker-compose.yml pull restore
+docker compose -f backup-docker-compose.yml run --rm restore
+
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 docker compose -f vpn-docker-compose.yml pull && docker compose -f vpn-docker-compose.yml up -d
 docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d
+
+docker compose -f backup-docker-compose.yml pull backup
+docker compose -f backup-docker-compose.yml up -d backup

services/vpn/traefik-docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - "80:80"
       - "443:443"
     volumes:
-      - "./letsencrypt:/letsencrypt"
+      - "letsencrypt_data:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
     networks:
       - proxy
@@ -36,3 +36,7 @@ services:
 networks:
   proxy:
     name: proxy
+
+volumes:
+  letsencrypt_data:
+    name: letsencrypt_data

services/vpn/vpn-docker-compose.yml

@@ -8,11 +8,14 @@ services:
     environment:
         HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address 
     volumes:
-        - ./openvpn_conf:/opt/Dockovpn_data
+        - openvpn_data:/opt/Dockovpn_data
     labels:
       ## Expose vpn Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
 
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.vpn-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.vpn-insecure.rule=Host(`vpn.luke-else.co.uk`)"
@@ -21,4 +24,8 @@ services:
       - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)"
       - "traefik.http.routers.vpn.entrypoints=websecure"
       - "traefik.http.routers.vpn.tls.certresolver=myresolver"
-    restart: always
+    restart: always
+
+volumes:
+  openvpn_data:
+    name: openvpn_data