feat: backup and restore

services/dev/backup-docker-compose.yml

@@ -0,0 +1,48 @@
+services:
+  # Nightly backup of every named volume on this host to S3. Containers
+  # labeled docker-volume-backup.stop-during-backup=true (see
+  # gitea-docker-compose.yml) are stopped for the duration of the backup so
+  # their data is captured in a consistent state, then restarted.
+  backup:
+    image: offen/docker-volume-backup:v2
+    container_name: backup
+    environment:
+      BACKUP_FILENAME: "dev-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
+      BACKUP_CRON_EXPRESSION: "0 3 * * *"
+      BACKUP_RETENTION_DAYS: "14"
+      BACKUP_PRUNING_LEEWAY: "5m"
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - gitea_data:/backup/gitea_data:ro
+      - letsencrypt_data:/backup/letsencrypt_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+
+  # One-shot: restores empty volumes from the latest S3 backup, then exits.
+  # Run explicitly by spinup.sh before the services that own these volumes
+  # start - not started by `docker compose up`.
+  restore:
+    image: amazon/aws-cli:2
+    container_name: backup-restore
+    entrypoint: ["/bin/bash", "/restore.sh"]
+    environment:
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - ./restore.sh:/restore.sh:ro
+      - gitea_data:/restore/gitea_data
+      - letsencrypt_data:/restore/letsencrypt_data
+    restart: "no"
+
+volumes:
+  gitea_data:
+    name: gitea_data
+  letsencrypt_data:
+    name: letsencrypt_data

services/dev/gitea-docker-compose.yml

@@ -4,7 +4,7 @@ services:
     image: gitea/gitea:latest
     container_name: gitea
     volumes:
-      - ${DATA_DIR}/gitea:/data
+      - gitea_data:/data
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
     networks:
@@ -29,7 +29,11 @@ services:
     labels:
       ## Expose Gitea Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
+      # Stopped for the duration of each backup run so the sqlite db/repos are
+      # captured in a consistent state - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowmethods=*"
       - "traefik.http.middlewares.cors-gitea.headers.accesscontrolalloworiginlist=*"
       - "traefik.http.middlewares.cors-gitea.headers.addvaryheader=true"
@@ -51,4 +55,8 @@ services:
 
 networks:
   proxy:
-    external: true
+    external: true
+
+volumes:
+  gitea_data:
+    name: gitea_data

services/dev/restore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Restores this host's named volumes from the latest S3 backup, but only into
+# volumes that are currently empty - never overwrites data that's already
+# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml
+# run --rm restore` before any service that owns one of the mounted volumes
+# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh
+# estate): it just leaves the empty volumes alone.
+set -euo pipefail
+
+S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}"
+ENDPOINT_ARGS=()
+if [ -n "${AWS_ENDPOINT:-}" ]; then
+  ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}")
+fi
+
+empty_volumes=()
+for dir in /restore/*/; do
+  name="$(basename "$dir")"
+  if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+    empty_volumes+=("$name")
+  fi
+done
+
+if [ "${#empty_volumes[@]}" -eq 0 ]; then
+  echo "All volumes already contain data - skipping restore."
+  exit 0
+fi
+
+echo "Empty volumes: ${empty_volumes[*]}"
+
+latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)"
+if [ -z "$latest" ]; then
+  echo "No existing backup found at ${S3_URI} - starting with empty volumes."
+  exit 0
+fi
+
+echo "Restoring ${S3_URI}/${latest} into empty volumes only..."
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz"
+mkdir "$tmp/extracted"
+tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted"
+
+for name in "${empty_volumes[@]}"; do
+  if [ -d "$tmp/extracted/$name" ]; then
+    echo "Restoring $name..."
+    cp -a "$tmp/extracted/$name/." "/restore/$name/"
+  else
+    echo "No $name/ in backup - leaving volume empty."
+  fi
+done
+
+echo "Restore complete."

services/dev/spindown.sh

@@ -9,7 +9,9 @@ cd ..
 docker compose -f gitea-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
+docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-docker volume prune -f -a
+# Deliberately no `docker volume prune` - gitea_data/letsencrypt_data/etc. are
+# named volumes holding real data, backed up to S3 but not disposable.

services/dev/spinup.sh

@@ -4,10 +4,14 @@ set -e
 cd "$(dirname "$0")"
 
 if [ ! -f .env ]; then
-  echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
+  echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
   exit 1
 fi
 
+echo "Restoring named volumes from S3 backup if any are empty..."
+docker compose -f backup-docker-compose.yml pull restore
+docker compose -f backup-docker-compose.yml run --rm restore
+
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 
 docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d
@@ -31,3 +35,6 @@ docker compose pull && docker compose up -d
 cd ..
 
 docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d
+
+docker compose -f backup-docker-compose.yml pull backup
+docker compose -f backup-docker-compose.yml up -d backup

services/dev/traefik-docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - "80:80"
       - "443:443"
     volumes:
-      - "${DATA_DIR}/letsencrypt:/letsencrypt"
+      - "letsencrypt_data:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
     networks:
       - proxy
@@ -36,3 +36,7 @@ services:
 networks:
   proxy:
     name: proxy
+
+volumes:
+  letsencrypt_data:
+    name: letsencrypt_data

services/prod/backup-docker-compose.yml

@@ -0,0 +1,59 @@
+services:
+  # Nightly backup of every named volume on this host to S3. Containers
+  # labeled docker-volume-backup.stop-during-backup=true (see
+  # bitwarden-docker-compose.yml, rd-docker-compose.yml,
+  # status-docker-compose.yml, web-docker-compose.yml) are stopped for the
+  # duration of the backup so their data is captured in a consistent state,
+  # then restarted.
+  backup:
+    image: offen/docker-volume-backup:v2
+    container_name: backup
+    environment:
+      BACKUP_FILENAME: "prod-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
+      BACKUP_CRON_EXPRESSION: "0 3 * * *"
+      BACKUP_RETENTION_DAYS: "14"
+      BACKUP_PRUNING_LEEWAY: "5m"
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - bitwarden_data:/backup/bitwarden_data:ro
+      - rustdesk_data:/backup/rustdesk_data:ro
+      - uptime_kuma_data:/backup/uptime_kuma_data:ro
+      - snexo_data:/backup/snexo_data:ro
+      - letsencrypt_data:/backup/letsencrypt_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+
+  # One-shot: restores empty volumes from the latest S3 backup, then exits.
+  # Run explicitly by spinup.sh before the services that own these volumes
+  # start - not started by `docker compose up`.
+  restore:
+    image: amazon/aws-cli:2
+    container_name: backup-restore
+    entrypoint: [ "/bin/bash", "/restore.sh" ]
+    environment:
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - ./restore.sh:/restore.sh:ro
+      - bitwarden_data:/restore/bitwarden_data
+      - rustdesk_data:/restore/rustdesk_data
+      - uptime_kuma_data:/restore/uptime_kuma_data
+      - letsencrypt_data:/restore/letsencrypt_data
+    restart: "no"
+
+volumes:
+  bitwarden_data:
+    name: bitwarden_data
+  rustdesk_data:
+    name: rustdesk_data
+  uptime_kuma_data:
+    name: uptime_kuma_data
+  letsencrypt_data:
+    name: letsencrypt_data

services/prod/bitwarden-docker-compose.yml

@@ -5,13 +5,16 @@ services:
     image: "vaultwarden/server:latest"
     container_name: vaultwarden
     volumes:
-      - ${DATA_DIR}/bitwarden/:/data/
+      - bitwarden_data:/data/
     networks:
       - proxy
     labels:
       ## Expose Bitwarden Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.bitwarden-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.bitwarden-insecure.rule=Host(`bitwarden.luke-else.co.uk`)"
@@ -24,4 +27,8 @@ services:
 
 networks:
   proxy:
-    external: true
+    external: true
+
+volumes:
+  bitwarden_data:
+    name: bitwarden_data

services/prod/rd-docker-compose.yml

@@ -6,7 +6,7 @@ services:
     restart: unless-stopped
 
     volumes:
-      - ${DATA_DIR}/rustdesk:/root
+      - rustdesk_data:/root
 
     networks:
       - proxy
@@ -20,6 +20,9 @@ services:
     labels:
       - "traefik.enable=true"
 
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.rustdesk-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.rustdesk-insecure.rule=Host(`rd.luke-else.co.uk`)"
@@ -39,7 +42,7 @@ services:
     restart: unless-stopped
 
     volumes:
-      - ${DATA_DIR}/rustdesk:/root
+      - rustdesk_data:/root
 
     networks:
       - proxy
@@ -50,4 +53,8 @@ services:
 
 networks:
   proxy:
-    external: true
+    external: true
+
+volumes:
+  rustdesk_data:
+    name: rustdesk_data

services/prod/restore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Restores this host's named volumes from the latest S3 backup, but only into
+# volumes that are currently empty - never overwrites data that's already
+# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml
+# run --rm restore` before any service that owns one of the mounted volumes
+# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh
+# estate): it just leaves the empty volumes alone.
+set -euo pipefail
+
+S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}"
+ENDPOINT_ARGS=()
+if [ -n "${AWS_ENDPOINT:-}" ]; then
+  ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}")
+fi
+
+empty_volumes=()
+for dir in /restore/*/; do
+  name="$(basename "$dir")"
+  if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+    empty_volumes+=("$name")
+  fi
+done
+
+if [ "${#empty_volumes[@]}" -eq 0 ]; then
+  echo "All volumes already contain data - skipping restore."
+  exit 0
+fi
+
+echo "Empty volumes: ${empty_volumes[*]}"
+
+latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)"
+if [ -z "$latest" ]; then
+  echo "No existing backup found at ${S3_URI} - starting with empty volumes."
+  exit 0
+fi
+
+echo "Restoring ${S3_URI}/${latest} into empty volumes only..."
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz"
+mkdir "$tmp/extracted"
+tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted"
+
+for name in "${empty_volumes[@]}"; do
+  if [ -d "$tmp/extracted/$name" ]; then
+    echo "Restoring $name..."
+    cp -a "$tmp/extracted/$name/." "/restore/$name/"
+  else
+    echo "No $name/ in backup - leaving volume empty."
+  fi
+done
+
+echo "Restore complete."

services/prod/spindown.sh

@@ -8,7 +8,9 @@ docker compose -f bitwarden-docker-compose.yml down
 docker compose -f rd-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
+docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-docker volume prune -f -a
+# Deliberately no `docker volume prune` - bitwarden_data/rustdesk_data/etc. are
+# named volumes holding real data, backed up to S3 but not disposable.

services/prod/spinup.sh

@@ -4,10 +4,14 @@ set -e
 cd "$(dirname "$0")"
 
 if [ ! -f .env ]; then
-  echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
+  echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
   exit 1
 fi
 
+echo "Restoring named volumes from S3 backup if any are empty..."
+docker compose -f backup-docker-compose.yml pull restore
+docker compose -f backup-docker-compose.yml run --rm restore
+
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 
 sleep 20 # Allow Traefik + registry auth to settle before starting the rest of the services
@@ -17,3 +21,6 @@ docker compose -f status-docker-compose.yml pull && docker compose -f status-doc
 docker compose -f web-docker-compose.yml pull && docker compose -f web-docker-compose.yml up -d
 docker compose -f bitwarden-docker-compose.yml pull && docker compose -f bitwarden-docker-compose.yml up -d
 docker compose -f rd-docker-compose.yml pull && docker compose -f rd-docker-compose.yml up -d
+
+docker compose -f backup-docker-compose.yml pull backup
+docker compose -f backup-docker-compose.yml up -d backup

services/prod/status-docker-compose.yml

@@ -3,13 +3,16 @@ services:
     image: louislam/uptime-kuma:latest
     container_name: status
     volumes:
-      - ${DATA_DIR}/uptime-kuma/data:/app/data
+      - uptime_kuma_data:/app/data
     networks:
       - proxy
     labels:
       ## Expose uptime-kuma Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
 
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)"
@@ -23,3 +26,7 @@ services:
 networks:
   proxy:
     external: true
+
+volumes:
+  uptime_kuma_data:
+    name: uptime_kuma_data

services/prod/traefik-docker-compose.yml

@@ -20,7 +20,7 @@ services:
       - "443:443"
       - "27017:27017"
     volumes:
-      - "${DATA_DIR}/letsencrypt:/letsencrypt"
+      - "letsencrypt_data:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
     networks:
       - proxy
@@ -41,3 +41,7 @@ services:
 networks:
   proxy:
     name: proxy
+
+volumes:
+  letsencrypt_data:
+    name: letsencrypt_data

services/prod/web-docker-compose.yml

@@ -1,6 +1,6 @@
 services:
 
-#Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080)
+  #Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080)
   luke-else:
     image: git.luke-else.co.uk/luke-else/luke-else.co.uk
     container_name: luke-else
@@ -9,7 +9,7 @@ services:
     labels:
       ## Expose luke-else Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.personal-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.personal-insecure.rule=Host(`luke-else.co.uk`)"
@@ -28,7 +28,7 @@ services:
     labels:
       ## Expose luke-else Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.personal-dev-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.personal-dev-insecure.rule=Host(`dev.luke-else.co.uk`)"
@@ -47,7 +47,7 @@ services:
     labels:
       ## Expose metarius Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.metarius-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.metarius-insecure.rule=Host(`metarius.luke-else.co.uk`)"
@@ -66,7 +66,7 @@ services:
     labels:
       ## Expose divine-couture Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.divine-couture-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.divine-couture-insecure.rule=Host(`www.divine-couture.co.uk`)"
@@ -77,27 +77,6 @@ services:
       - "traefik.http.routers.divine-couture.tls.certresolver=myresolver"
     restart: unless-stopped
 
-  snexo:
-    image: "php:apache"
-    container_name: snexo
-    volumes:
-      - ${DATA_DIR}/snexo.co.uk/:/var/www/html
-    networks:
-      - proxy
-    labels:
-      ## Expose Snexo Through Trefik ##
-      - "traefik.enable=true" # <== Enable traefik to proxy this container
-
-      - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
-      - "traefik.http.routers.snexo-insecure.middlewares=redirect-web-secure"
-      - "traefik.http.routers.snexo-insecure.rule=Host(`snexo.co.uk`)"
-      - "traefik.http.routers.snexo-insecure.entrypoints=web"
-
-      - "traefik.http.routers.snexo.rule=Host(`snexo.co.uk`)"
-      - "traefik.http.routers.snexo.entrypoints=websecure"
-      - "traefik.http.routers.snexo.tls.certresolver=myresolver"
-    restart: unless-stopped
-
 networks:
   proxy:
     external: true

services/vpn/backup-docker-compose.yml

@@ -0,0 +1,48 @@
+services:
+  # Nightly backup of every named volume on this host to S3. Containers
+  # labeled docker-volume-backup.stop-during-backup=true (see
+  # vpn-docker-compose.yml) are stopped for the duration of the backup so
+  # their data is captured in a consistent state, then restarted.
+  backup:
+    image: offen/docker-volume-backup:v2
+    container_name: backup
+    environment:
+      BACKUP_FILENAME: "vpn-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
+      BACKUP_CRON_EXPRESSION: "0 3 * * *"
+      BACKUP_RETENTION_DAYS: "14"
+      BACKUP_PRUNING_LEEWAY: "5m"
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - openvpn_data:/backup/openvpn_data:ro
+      - letsencrypt_data:/backup/letsencrypt_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+
+  # One-shot: restores empty volumes from the latest S3 backup, then exits.
+  # Run explicitly by spinup.sh before the services that own these volumes
+  # start - not started by `docker compose up`.
+  restore:
+    image: amazon/aws-cli:2
+    container_name: backup-restore
+    entrypoint: ["/bin/bash", "/restore.sh"]
+    environment:
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - ./restore.sh:/restore.sh:ro
+      - openvpn_data:/restore/openvpn_data
+      - letsencrypt_data:/restore/letsencrypt_data
+    restart: "no"
+
+volumes:
+  openvpn_data:
+    name: openvpn_data
+  letsencrypt_data:
+    name: letsencrypt_data

services/vpn/restore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Restores this host's named volumes from the latest S3 backup, but only into
+# volumes that are currently empty - never overwrites data that's already
+# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml
+# run --rm restore` before any service that owns one of the mounted volumes
+# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh
+# estate): it just leaves the empty volumes alone.
+set -euo pipefail
+
+S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}"
+ENDPOINT_ARGS=()
+if [ -n "${AWS_ENDPOINT:-}" ]; then
+  ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}")
+fi
+
+empty_volumes=()
+for dir in /restore/*/; do
+  name="$(basename "$dir")"
+  if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+    empty_volumes+=("$name")
+  fi
+done
+
+if [ "${#empty_volumes[@]}" -eq 0 ]; then
+  echo "All volumes already contain data - skipping restore."
+  exit 0
+fi
+
+echo "Empty volumes: ${empty_volumes[*]}"
+
+latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)"
+if [ -z "$latest" ]; then
+  echo "No existing backup found at ${S3_URI} - starting with empty volumes."
+  exit 0
+fi
+
+echo "Restoring ${S3_URI}/${latest} into empty volumes only..."
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz"
+mkdir "$tmp/extracted"
+tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted"
+
+for name in "${empty_volumes[@]}"; do
+  if [ -d "$tmp/extracted/$name" ]; then
+    echo "Restoring $name..."
+    cp -a "$tmp/extracted/$name/." "/restore/$name/"
+  else
+    echo "No $name/ in backup - leaving volume empty."
+  fi
+done
+
+echo "Restore complete."

services/vpn/spindown.sh

@@ -5,7 +5,9 @@ cd "$(dirname "$0")"
 docker compose -f vpn-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
+docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-docker volume prune -f -a
+# Deliberately no `docker volume prune` - openvpn_data/letsencrypt_data are
+# named volumes holding real data, backed up to S3 but not disposable.

services/vpn/spinup.sh

@@ -3,6 +3,18 @@
 set -e
 cd "$(dirname "$0")"
 
+if [ ! -f .env ]; then
+  echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
+  exit 1
+fi
+
+echo "Restoring named volumes from S3 backup if any are empty..."
+docker compose -f backup-docker-compose.yml pull restore
+docker compose -f backup-docker-compose.yml run --rm restore
+
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 docker compose -f vpn-docker-compose.yml pull && docker compose -f vpn-docker-compose.yml up -d
 docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d
+
+docker compose -f backup-docker-compose.yml pull backup
+docker compose -f backup-docker-compose.yml up -d backup

services/vpn/traefik-docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - "80:80"
       - "443:443"
     volumes:
-      - "./letsencrypt:/letsencrypt"
+      - "letsencrypt_data:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
     networks:
       - proxy
@@ -36,3 +36,7 @@ services:
 networks:
   proxy:
     name: proxy
+
+volumes:
+  letsencrypt_data:
+    name: letsencrypt_data

services/vpn/vpn-docker-compose.yml

@@ -8,11 +8,14 @@ services:
     environment:
         HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address 
     volumes:
-        - ./openvpn_conf:/opt/Dockovpn_data
+        - openvpn_data:/opt/Dockovpn_data
     labels:
       ## Expose vpn Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
 
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.vpn-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.vpn-insecure.rule=Host(`vpn.luke-else.co.uk`)"
@@ -21,4 +24,8 @@ services:
       - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)"
       - "traefik.http.routers.vpn.entrypoints=websecure"
       - "traefik.http.routers.vpn.tls.certresolver=myresolver"
-    restart: always
+    restart: always
+
+volumes:
+  openvpn_data:
+    name: openvpn_data