feat: backup and restore

ansible/README.md

@@ -2,25 +2,26 @@
 
 Bootstraps, syncs, and starts/stops the Docker Compose stacks in `../services/`
 on the three servers OpenTofu provisions (see `../infra/`). OpenTofu still owns
-the actual cloud resources (servers, network, firewalls, volumes, DNS);
+the actual cloud resources (servers, network, firewalls, DNS);
 everything downstream of "the server exists" - installing Docker, creating the
 `deploy` user, hardening SSH, copying `services/<host>/`, rendering `.env` /
 the Gitea runner compose file, and running `spinup.sh`/`spindown.sh` - lives
-here instead.
+here instead. Persistent service data lives in named Docker volumes backed up
+to S3, not on a Hetzner volume - see `../readme.md`.
 
 ## Contents
 
 - `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API
   directly (not Terraform state) and groups servers by their `role` label
   (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules/<host>/main.tf`).
-- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, `HCLOUD_TOKEN`
-  lookup), `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` (per-role vars,
-  e.g. `dev_runner_count`).
+- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path,
+  `backup_s3_*` S3 backup credentials), `role_dev.yml` / `role_prod.yml` /
+  `role_vpn.yml` (per-role vars, e.g. `dev_runner_count`).
 - `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening,
   unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`.
-- `roles/deploy/` - copies `services/<host>/` to the server, looks up the
-  host's data volume directly from the Hetzner API and renders `.env`
-  (`DATA_DIR=...`) and, for `dev`, `Runners/docker-compose.yml`.
+- `roles/deploy/` - copies `services/<host>/` to the server and renders `.env`
+  (S3 backup credentials, see `../readme.md`) and, for `dev`,
+  `Runners/docker-compose.yml`.
 - `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`,
   and `site.yml` (all three in order).
 
@@ -30,6 +31,10 @@ here instead.
 cd ansible
 ansible-galaxy collection install -r requirements.yml
 export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
+export BACKUP_S3_BUCKET=your-bucket-name
+export BACKUP_S3_ACCESS_KEY_ID=your-access-key-id
+export BACKUP_S3_SECRET_ACCESS_KEY=your-secret-access-key
+export BACKUP_S3_ENDPOINT=your-s3-endpoint   # omit for real AWS S3
 ```
 
 `ansible_ssh_private_key_file` (see `group_vars/all.yml`) defaults to

ansible/group_vars/all.yml

@@ -6,9 +6,16 @@ deploy_user: deploy
 ansible_user: "{{ deploy_user }}"
 ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}"
 
-# Used by the deploy role to look up each host's data volume directly from the
-# Hetzner API (hcloud_volume_info) rather than from Terraform state/outputs.
-hcloud_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
-
 # Repo root's services/ directory, relative to wherever a playbook lives under ansible/playbooks/.
 services_root: "{{ playbook_dir }}/../../services"
+
+# S3-compatible bucket that services/<host>/backup-docker-compose.yml backs up
+# to and restores from - see readme.md. Not provisioned by OpenTofu (the
+# hcloud provider has no Object Storage resource), so these must point at a
+# bucket you've created yourself. AWS_ENDPOINT is only needed for a
+# non-AWS S3-compatible provider (e.g. Hetzner Object Storage); leave it unset
+# to use real AWS S3.
+backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}"
+backup_s3_access_key_id: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY_ID') }}"
+backup_s3_secret_access_key: "{{ lookup('env', 'BACKUP_S3_SECRET_ACCESS_KEY') }}"
+backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}"

ansible/roles/deploy/tasks/main.yml

@@ -3,6 +3,10 @@
 # used to be generated by OpenTofu (services/<host>/.env and, for dev,
 # Runners/docker-compose.yml) - replaces the null_resource.deploy_services +
 # local_file provisioners previously in infra/modules/<host>/main.tf.
+#
+# Persistent data lives in named Docker volumes now, not on a Hetzner volume -
+# see services/<host>/backup-docker-compose.yml. .env only carries the S3
+# credentials that backup/restore need, rendered for every host.
 
 - name: Ensure services directory exists on the host
   ansible.builtin.file:
@@ -16,28 +20,11 @@
     dest: "/home/{{ deploy_user }}/services/{{ service_group }}/"
     mode: preserve
 
-- name: Look up this host's data volume
-  when: service_group in ['dev', 'prod']
-  delegate_to: localhost
-  become: false
-  hetzner.hcloud.hcloud_volume_info:
-    api_token: "{{ hcloud_token }}"
-    name: "{{ service_group }}-storage"
-  register: _volume_info
-
-# Hetzner's automount tooling always mounts an auto-mounted volume at
-# /mnt/HC_Volume_<volume-id> - deterministic once the volume exists.
-- name: Compute DATA_DIR from the volume's id
-  when: service_group in ['dev', 'prod']
-  ansible.builtin.set_fact:
-    data_dir: "/mnt/HC_Volume_{{ _volume_info.hcloud_volume_info[0].id }}"
-
-- name: Render DATA_DIR into .env (auto-loaded by docker compose)
-  when: service_group in ['dev', 'prod']
+- name: Render S3 backup credentials into .env (auto-loaded by docker compose)
   ansible.builtin.template:
     src: env.j2
     dest: "/home/{{ deploy_user }}/services/{{ service_group }}/.env"
-    mode: "0644"
+    mode: "0600"
 
 - name: Render Gitea Actions runner compose file
   when: service_group == 'dev'
@@ -46,10 +33,11 @@
     dest: "/home/{{ deploy_user }}/services/dev/Runners/docker-compose.yml"
     mode: "0644"
 
-- name: Make spinup/spindown scripts executable
+- name: Make spinup/spindown/restore scripts executable
   ansible.builtin.file:
     path: "/home/{{ deploy_user }}/services/{{ service_group }}/{{ item }}"
     mode: "0755"
   loop:
     - spinup.sh
     - spindown.sh
+    - restore.sh

ansible/roles/deploy/templates/env.j2

@@ -1 +1,5 @@
-DATA_DIR={{ data_dir }}
+AWS_ACCESS_KEY_ID={{ backup_s3_access_key_id }}
+AWS_SECRET_ACCESS_KEY={{ backup_s3_secret_access_key }}
+AWS_S3_BUCKET_NAME={{ backup_s3_bucket }}
+AWS_S3_PATH={{ service_group }}
+AWS_ENDPOINT={{ backup_s3_endpoint }}

ansible/roles/deploy/templates/runners-docker-compose.yml.j2

@@ -7,10 +7,9 @@
 # (not baked in here) - services/dev/spinup.sh generates a fresh token and
 # writes it to Runners/.env immediately before starting these containers.
 #
-# /data lives on dev's volume ({{ data_dir }}) so runner identity survives a
-# server rebuild - baked in directly rather than via .env, since Runners/.env
-# is already reserved for the registration token above and gets overwritten
-# on every deploy.
+# /data is a disposable local cache (runner identity re-registers on every
+# spinup.sh run anyway), so it's a plain named volume - not part of the S3
+# backup/restore stack in services/dev/backup-docker-compose.yml.
 services:
 {% for i in range(1, dev_runner_count + 1) %}
   runner-{{ i }}:
@@ -18,7 +17,7 @@ services:
     container_name: gitea_runner_{{ i }}
     volumes:
       - ./config.yaml:/config.yaml
-      - {{ data_dir }}/gitea_runner_{{ i }}:/data
+      - gitea_runner_{{ i }}_data:/data
       - /var/run/docker.sock:/var/run/docker.sock
     networks:
       - proxy
@@ -33,3 +32,9 @@ services:
 networks:
   proxy:
     external: true
+
+volumes:
+{% for i in range(1, dev_runner_count + 1) %}
+  gitea_runner_{{ i }}_data:
+    name: gitea_runner_{{ i }}_data
+{% endfor %}

infra/main.tf

@@ -1,6 +1,8 @@
 # Root module: wires together the shared network and the three host modules
 # (one per server in docs/architecture.md). Each host module owns its own
-# firewall (and volume, where applicable) - see modules/<host>/main.tf.
+# firewall - see modules/<host>/main.tf. Persistent data lives in named Docker
+# volumes backed up to S3 (see services/<host>/backup-docker-compose.yml), not
+# on Hetzner volumes.
 
 locals {
   # SSH to dev/prod is only permitted from the vpn server's public IP: admins must
@@ -28,8 +30,6 @@ locals {
     { zone = "luke-else.co.uk", name = "traefik.cicd", value = module.dev.ipv4 },
     { zone = "luke-else.co.uk", name = "vpn", value = module.vpn.ipv4 },
     { zone = "luke-else.co.uk", name = "traefik.vpn", value = module.vpn.ipv4 },
-    { zone = "divine-couture.co.uk", name = "www", value = module.prod.ipv4 },
-    { zone = "snexo.co.uk", name = "@", value = module.prod.ipv4 },
   ]
 }
 
@@ -50,7 +50,6 @@ module "dev" {
   ssh_key_ids            = local.ssh_key_ids
   network_id             = module.network.id
   private_ip             = var.dev_private_ip
-  volume_size            = var.dev_volume_size
   allowed_ssh_source_ips = local.vpn_ssh_source_ips
   network_ip_range       = var.network_ip_range
 
@@ -68,7 +67,6 @@ module "prod" {
   ssh_key_ids            = local.ssh_key_ids
   network_id             = module.network.id
   private_ip             = var.prod_private_ip
-  volume_size            = var.prod_volume_size
   allowed_ssh_source_ips = local.vpn_ssh_source_ips
   network_ip_range       = var.network_ip_range
 

infra/modules/dev/main.tf

@@ -64,24 +64,3 @@ resource "hcloud_server" "this" {
   }
 }
 
-resource "hcloud_volume" "storage" {
-  name      = "dev-storage"
-  size      = var.volume_size
-  server_id = hcloud_server.this.id
-  automount = true
-  format    = "ext4"
-
-  lifecycle {
-    prevent_destroy = true
-  }
-}
-
-locals {
-  # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is
-  # always /mnt/HC_Volume_<id> - deterministic once the volume exists, since the
-  # id is stable for the volume's lifetime regardless of which server it's
-  # attached to. Kept here only as a human-readable output (see outputs.tf) -
-  # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't
-  # read this value.
-  data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}"
-}

infra/modules/dev/outputs.tf

@@ -5,8 +5,3 @@ output "ipv4" {
 output "private_ipv4" {
   value = var.private_ip
 }
-
-output "data_dir" {
-  description = "Deterministic host path of the auto-mounted volume - see local.data_dir."
-  value       = local.data_dir
-}

infra/modules/dev/variables.tf

@@ -28,11 +28,6 @@ variable "private_ip" {
   type        = string
 }
 
-variable "volume_size" {
-  description = "Size in GB of the volume attached to dev (disk2 in docs/architecture.md)."
-  type        = number
-}
-
 variable "allowed_ssh_source_ips" {
   description = "CIDRs allowed to reach port 22 on dev. Set by the root module to the vpn server's public IP."
   type        = list(string)

infra/modules/prod/main.tf

@@ -91,24 +91,3 @@ resource "hcloud_server" "this" {
   }
 }
 
-resource "hcloud_volume" "storage" {
-  name      = "prod-storage"
-  size      = var.volume_size
-  server_id = hcloud_server.this.id
-  automount = true
-  format    = "ext4"
-
-  lifecycle {
-    prevent_destroy = true
-  }
-}
-
-locals {
-  # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is
-  # always /mnt/HC_Volume_<id> - deterministic once the volume exists, since the
-  # id is stable for the volume's lifetime regardless of which server it's
-  # attached to. Kept here only as a human-readable output (see outputs.tf) -
-  # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't
-  # read this value.
-  data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}"
-}

infra/modules/prod/outputs.tf

@@ -5,8 +5,3 @@ output "ipv4" {
 output "private_ipv4" {
   value = var.private_ip
 }
-
-output "data_dir" {
-  description = "Deterministic host path of the auto-mounted volume - see local.data_dir."
-  value       = local.data_dir
-}

infra/modules/prod/variables.tf

@@ -28,11 +28,6 @@ variable "private_ip" {
   type        = string
 }
 
-variable "volume_size" {
-  description = "Size in GB of the volume attached to prod (disk1 in docs/architecture.md)."
-  type        = number
-}
-
 variable "allowed_ssh_source_ips" {
   description = "CIDRs allowed to reach port 22 on prod. Set by the root module to the vpn server's public IP."
   type        = list(string)

infra/outputs.tf

@@ -14,16 +14,6 @@ output "prod_private_ipv4" {
   value = module.prod.private_ipv4
 }
 
-output "dev_data_dir" {
-  description = "Deterministic host path of dev's auto-mounted volume - also written to services/dev/.env as DATA_DIR."
-  value       = module.dev.data_dir
-}
-
-output "prod_data_dir" {
-  description = "Deterministic host path of prod's auto-mounted volume - also written to services/prod/.env as DATA_DIR."
-  value       = module.prod.data_dir
-}
-
 output "vpn_ipv4" {
   value = module.vpn.ipv4
 }

infra/terraform.tfvars.example

@@ -14,7 +14,5 @@ ssh_key_names = ["your-key-name", "laptop", "phone"]
 # dev_server_type        = "cx22"
 # prod_server_type       = "cx22"
 # vpn_server_type        = "cx22"
-# dev_volume_size        = 20
-# prod_volume_size       = 20
 # allowed_ssh_source_ips = ["203.0.113.4/32"]
 # dns_zones              = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"]

infra/variables.tf

@@ -1,5 +1,5 @@
 variable "location" {
-  description = "Hetzner Cloud datacenter location for all servers and volumes."
+  description = "Hetzner Cloud datacenter location for all servers."
   type        = string
   default     = "nbg1"
 }
@@ -34,18 +34,6 @@ variable "vpn_server_type" {
   default     = "cx22"
 }
 
-variable "dev_volume_size" {
-  description = "Size in GB of the volume attached to dev"
-  type        = number
-  default     = 20
-}
-
-variable "prod_volume_size" {
-  description = "Size in GB of the volume attached to prod (disk1 in docs/architecture.md)."
-  type        = number
-  default     = 20
-}
-
 variable "network_ip_range" {
   description = "IP range of the private network shared by dev and prod."
   type        = string
@@ -84,5 +72,5 @@ variable "allowed_ssh_source_ips" {
 variable "dns_zones" {
   description = "Domains to manage as Hetzner DNS zones. Point each domain's registrar NS records at tofu output dns_nameservers for Hetzner to actually become authoritative."
   type        = list(string)
-  default     = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"]
+  default     = ["luke-else.co.uk"]
 }

readme.md

@@ -28,9 +28,6 @@ architecture-beta
 
     group network(cloud)[network] in cloud
 
-    service disk1(mdi:disk)[Storage] in cloud
-    service disk2(mdi:disk)[Storage] in cloud
-
     service dev(mdi:server)[dev] in network
 
     service prod(mdi:server)[prod] in network
@@ -41,24 +38,30 @@ architecture-beta
 
     service gateway(mdi:web)[gateway] in cloud
 
+    service backups(mdi:bucket)[S3 Backups]
+
     dev:L -- R:prod
-    disk1:B -- T:prod
-    disk2:B -- T:dev
 
     prod:B -- T:prodfirewall
     vpn:B -- T:vpnfirewall
 
     prodfirewall: L -- R: gateway
     vpnfirewall: B -- T: gateway
+
+    dev:T -- B:backups
+    prod:T -- B:backups
+    vpn:R -- L:backups
 ```
 
-`gateway` represents the public internet, not a provisioned resource.
+`gateway` represents the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from - not a Hetzner resource, see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3).
 
-| Server | Purpose | Network | Volume |
-|---|---|---|---|
-| `dev`  | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) | `dev-storage` |
-| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | `prod-storage` |
-| `vpn`  | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | none |
+| Server | Purpose | Network |
+|---|---|---|
+| `dev`  | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) |
+| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall |
+| `vpn`  | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` |
+
+Each host's stateful data lives in local named Docker volumes, backed up nightly to a shared S3 bucket and restored automatically on first boot - see below. No host has a Hetzner Cloud Volume attached any more.
 
 `dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach deployment targets on `prod` without exposing that traffic publicly. `vpn` is deliberately kept off this network. Each server has its own Hetzner Cloud Firewall (see `infra/modules/<host>/main.tf`) that only opens the ports actually used by the compose stacks running on it, plus SSH restricted to `var.allowed_ssh_source_ips`.
 
@@ -66,7 +69,7 @@ architecture-beta
 
 ```
 .
-├── infra/                    # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes, DNS
+├── infra/                    # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, DNS
 │   ├── main.tf                # root module: wires network + dev/prod/vpn/dns modules together
 │   ├── variables.tf           # shared inputs (sizes, locations, IP ranges, SSH key names)
 │   ├── outputs.tf             # pass-through outputs from each host module
@@ -75,21 +78,21 @@ architecture-beta
 │   ├── terraform.tfvars.example
 │   └── modules/
 │       ├── network/           # shared private network + subnet (used by dev and prod)
-│       ├── dev/                # dev server + firewall + volume, labeled role=dev
-│       ├── prod/               # prod server + firewall + volume, labeled role=prod
-│       ├── vpn/                # vpn server + firewall (no private network, no volume), labeled role=vpn
+│       ├── dev/                # dev server + firewall, labeled role=dev
+│       ├── prod/               # prod server + firewall, labeled role=prod
+│       ├── vpn/                # vpn server + firewall (no private network), labeled role=vpn
 │       └── dns/                # Hetzner DNS zones + A records for every domain in var.dns_zones
 ├── ansible/                   # Ansible: bootstraps each server and deploys/starts services/<host>/ onto it
 │   ├── inventory/hcloud.yml   # dynamic inventory — queries the Hetzner API, groups by the role label above
-│   ├── group_vars/            # deploy_user, dev_runner_count, etc.
+│   ├── group_vars/            # deploy_user, dev_runner_count, backup_s3_*, etc.
 │   ├── roles/
 │   │   ├── bootstrap/          # Docker, deploy user, sshd hardening, unattended-upgrades
-│   │   └── deploy/             # copies services/<host>/, renders .env + Runners/docker-compose.yml
+│   │   └── deploy/             # copies services/<host>/, renders .env (S3 backup credentials) + Runners/docker-compose.yml
 │   └── playbooks/              # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml
 ├── services/                 # Docker Compose stacks, grouped by which server they run on
-│   ├── dev/                   # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk)
-│   ├── prod/                  # Websites, Bitwarden, RustDesk, status page + Traefik
-│   ├── vpn/                    # OpenVPN + Traefik
+│   ├── dev/                   # Gitea + CI runner + Traefik + backup/restore (git.luke-else.co.uk, cicd.luke-else.co.uk)
+│   ├── prod/                  # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore
+│   ├── vpn/                    # OpenVPN + Traefik + backup/restore
 │   └── todo.md                # Outstanding manual setup/hardening tasks
 ├── docs/
 │   └── architecture.md       # Source of the architecture diagram above
@@ -97,9 +100,9 @@ architecture-beta
 └── assets/
 ```
 
-Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side.
+Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, a `backup-docker-compose.yml` + `restore.sh` pair for that host's S3 backup/restore (see below), plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side.
 
-OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, volumes, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services/<host>/`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown.
+OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services/<host>/`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown.
 
 ## Prerequisites
 
@@ -108,6 +111,7 @@ OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud res
 - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0`
 - [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`)
 - Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner (see [Managing DNS](#managing-dns) — the zones and records themselves are created for you)
+- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/), AWS S3, or any other S3-compatible provider) and an access key/secret pair with read/write access to it - not provisioned by OpenTofu (the `hcloud` provider has no Object Storage resource), so create this yourself and pass it to Ansible as `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` (and `BACKUP_S3_ENDPOINT` for anything other than real AWS S3) - see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3)
 
 Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below).
 
@@ -127,10 +131,11 @@ tofu apply
 This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` / `module.dns` in `infra/main.tf`:
 - `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`)
 - one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses, each server labeled `role = "dev"/"prod"/"vpn"` for Ansible's dynamic inventory (`modules/dev`, `modules/prod`, `modules/vpn`)
-- one `hcloud_volume` each for `dev` and `prod` (`modules/dev`, `modules/prod`; `vpn` has none)
 - one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns))
 
-Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`, `tofu output dev_data_dir`, `tofu output prod_data_dir` (the last two are informational only now — Ansible looks the data directory up itself, see below).
+No servers have an attached volume — persistent data lives in named Docker volumes backed up to S3 instead, see below.
+
+Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`.
 
 `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`.
 
@@ -148,7 +153,7 @@ Only A records for IPv4 are managed here — none of the modules currently track
 
 ## Bootstrapping and deploying with Ansible (`ansible/`)
 
-Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services/<host>/` to each server, rendering the files OpenTofu used to generate (`.env`'s `DATA_DIR`, `dev`'s `Runners/docker-compose.yml`), and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version:
+Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services/<host>/` to each server, rendering `.env` (S3 backup credentials) and, for `dev`, `Runners/docker-compose.yml`, and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version:
 
 ```sh
 cd ansible
@@ -164,13 +169,25 @@ ansible-playbook playbooks/site.yml -l role_dev,role_prod
 
 Hosts are discovered dynamically from the Hetzner API (`ansible/inventory/hcloud.yml`), grouped into `role_dev`/`role_prod`/`role_vpn` by the `role` label OpenTofu sets on each server — there's no static inventory file to keep in sync, and nothing here reads Terraform state.
 
-### Persistent data lives on the volumes, not the server disk
+### Persistent data lives in named Docker volumes, backed up to S3
 
-`dev` and `prod`'s compose files bind-mount container data to paths on the `dev-storage` / `prod-storage` volumes rather than the server's local disk, so it survives the server being destroyed and recreated: `${DATA_DIR}/gitea:/data`, `${DATA_DIR}/bitwarden/:/data/`, and so on across `services/dev/*.yml` and `services/prod/*.yml`. `vpn` has no volume, so it's untouched - see the architecture table above.
+Every stateful service's compose file mounts a plain named Docker volume rather than a bind mount to a Hetzner volume - `gitea_data:/data`, `bitwarden_data:/data/`, `letsencrypt_data:/letsencrypt`, and so on across `services/dev/*.yml`, `services/prod/*.yml`, and `services/vpn/*.yml`. Each volume is declared with an explicit `name:` in every compose file that mounts it (the same pattern already used for the shared `proxy` network), so Compose creates it once and every stack that references it reuses the same underlying volume regardless of which compose file happens to run first.
 
-`DATA_DIR` resolves deterministically: Hetzner always automounts a volume at `/mnt/HC_Volume_<volume-id>`. Ansible's `deploy` role looks the volume up by name (`hcloud_volume_info` for `dev-storage`/`prod-storage`) directly against the Hetzner API and renders that path into `services/<host>/.env` as `DATA_DIR=...` on the remote host — `docker compose` auto-loads `.env` from its working directory for variable substitution. Nothing is written locally; re-run `ansible-playbook playbooks/deploy.yml` if a volume is ever destroyed and recreated (new id). `spinup.sh` on both hosts refuses to start anything if `.env` is missing, rather than silently falling back to a relative path.
+Each host's `backup-docker-compose.yml` defines two services against those same named volumes:
+- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), running on a nightly cron (`0 3 * * *`), tars up every mounted volume and uploads it to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/<host>-<timestamp>.tar.gz`, keeping 14 days of history (`BACKUP_RETENTION_DAYS`). Containers labeled `docker-volume-backup.stop-during-backup=true` (Gitea, Bitwarden, RustDesk, Uptime Kuma, snexo, OpenVPN) are stopped for the duration of the backup so their data is captured in a consistent state, then restarted automatically. Started by `spinup.sh` alongside everything else.
+- **`restore`** — a one-shot container (`amazon/aws-cli` running `restore.sh`) that `spinup.sh` runs *before* anything else starts. It checks each of the host's named volumes; any that are already empty are populated from the most recent object under that host's S3 prefix, any that already contain data are left completely untouched. On a fresh server with no prior backup (or a bucket that doesn't exist yet), it's a no-op - services just start with empty volumes as before.
 
-The Gitea Actions runners are the one exception to the `.env` mechanism: their `/data` path is baked directly into the rendered `Runners/docker-compose.yml`, because `Runners/.env` is already reserved for the registration token and gets overwritten on every deploy.
+`AWS_S3_BUCKET_NAME`, `AWS_S3_PATH` (set to the host name, e.g. `dev`), `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_ENDPOINT` are rendered into `services/<host>/.env` by Ansible's `deploy` role from `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`/`BACKUP_S3_ENDPOINT` in your shell environment (see [Prerequisites](#prerequisites)) — `docker compose` auto-loads `.env` from its working directory. `spinup.sh` on every host refuses to start anything if `.env` is missing.
+
+The Gitea Actions runners are the one exception: their `/data` is a disposable local cache (runner identity re-registers with Gitea on every `spinup.sh` run anyway), so it's a plain named volume per runner (`gitea_runner_N_data`, baked directly into the rendered `Runners/docker-compose.yml`) that's deliberately left out of the backup/restore stack.
+
+To trigger a backup or restore manually rather than waiting for cron or a restart:
+
+```sh
+cd services/<host>
+docker compose -f backup-docker-compose.yml exec backup backup   # force an immediate backup
+docker compose -f backup-docker-compose.yml run --rm restore     # restore any currently-empty volumes
+```
 
 ### Scaling Gitea Actions runners
 
@@ -232,6 +249,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi
 | Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` |
 | Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | N/A |
 | Watchtower | `watchtower-docker-compose.yml` | — |
+| Backup/restore | `backup-docker-compose.yml` | — |
 
 ### `prod`
 
@@ -243,6 +261,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi
 | Bitwarden (Vaultwarden) | `bitwarden-docker-compose.yml` | `bitwarden.luke-else.co.uk` |
 | RustDesk relay (hbbs/hbbr) | `rd-docker-compose.yml` | `rd.luke-else.co.uk`, ports `21115-21119` |
 | Watchtower | `watchtower-docker-compose.yml` | — |
+| Backup/restore | `backup-docker-compose.yml` | — |
 
 ### `vpn`
 
@@ -251,6 +270,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi
 | Traefik | `traefik-docker-compose.yml` | `traefik.vpn.luke-else.co.uk` |
 | OpenVPN (Dockovpn) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `1194` |
 | Watchtower | `watchtower-docker-compose.yml` | — |
+| Backup/restore | `backup-docker-compose.yml` | — |
 
 ## First-time setup
 
@@ -270,7 +290,7 @@ Then reopen the repo in VS Code with the Dev Containers extension.
 
 ## Security notes
 
-- Real secrets (`HCLOUD_TOKEN`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`.
+- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. The S3 credentials end up in each host's `services/<host>/.env` (rendered by Ansible, mode `0600`, gitignored) - anyone with root on a host can read them, same trust boundary as everything else `deploy` can already do there.
 - SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s).
 - `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP.
 - Firewalls are allowlists scoped per host in `infra/modules/<host>/main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open.

services/dev/backup-docker-compose.yml

@@ -0,0 +1,48 @@
+services:
+  # Nightly backup of every named volume on this host to S3. Containers
+  # labeled docker-volume-backup.stop-during-backup=true (see
+  # gitea-docker-compose.yml) are stopped for the duration of the backup so
+  # their data is captured in a consistent state, then restarted.
+  backup:
+    image: offen/docker-volume-backup:v2
+    container_name: backup
+    environment:
+      BACKUP_FILENAME: "dev-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
+      BACKUP_CRON_EXPRESSION: "0 3 * * *"
+      BACKUP_RETENTION_DAYS: "14"
+      BACKUP_PRUNING_LEEWAY: "5m"
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - gitea_data:/backup/gitea_data:ro
+      - letsencrypt_data:/backup/letsencrypt_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+
+  # One-shot: restores empty volumes from the latest S3 backup, then exits.
+  # Run explicitly by spinup.sh before the services that own these volumes
+  # start - not started by `docker compose up`.
+  restore:
+    image: amazon/aws-cli:2
+    container_name: backup-restore
+    entrypoint: ["/bin/bash", "/restore.sh"]
+    environment:
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - ./restore.sh:/restore.sh:ro
+      - gitea_data:/restore/gitea_data
+      - letsencrypt_data:/restore/letsencrypt_data
+    restart: "no"
+
+volumes:
+  gitea_data:
+    name: gitea_data
+  letsencrypt_data:
+    name: letsencrypt_data

services/dev/gitea-docker-compose.yml

@@ -4,7 +4,7 @@ services:
     image: gitea/gitea:latest
     container_name: gitea
     volumes:
-      - ${DATA_DIR}/gitea:/data
+      - gitea_data:/data
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
     networks:
@@ -29,7 +29,11 @@ services:
     labels:
       ## Expose Gitea Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
+      # Stopped for the duration of each backup run so the sqlite db/repos are
+      # captured in a consistent state - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowmethods=*"
       - "traefik.http.middlewares.cors-gitea.headers.accesscontrolalloworiginlist=*"
       - "traefik.http.middlewares.cors-gitea.headers.addvaryheader=true"
@@ -51,4 +55,8 @@ services:
 
 networks:
   proxy:
-    external: true
+    external: true
+
+volumes:
+  gitea_data:
+    name: gitea_data

services/dev/restore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Restores this host's named volumes from the latest S3 backup, but only into
+# volumes that are currently empty - never overwrites data that's already
+# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml
+# run --rm restore` before any service that owns one of the mounted volumes
+# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh
+# estate): it just leaves the empty volumes alone.
+set -euo pipefail
+
+S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}"
+ENDPOINT_ARGS=()
+if [ -n "${AWS_ENDPOINT:-}" ]; then
+  ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}")
+fi
+
+empty_volumes=()
+for dir in /restore/*/; do
+  name="$(basename "$dir")"
+  if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+    empty_volumes+=("$name")
+  fi
+done
+
+if [ "${#empty_volumes[@]}" -eq 0 ]; then
+  echo "All volumes already contain data - skipping restore."
+  exit 0
+fi
+
+echo "Empty volumes: ${empty_volumes[*]}"
+
+latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)"
+if [ -z "$latest" ]; then
+  echo "No existing backup found at ${S3_URI} - starting with empty volumes."
+  exit 0
+fi
+
+echo "Restoring ${S3_URI}/${latest} into empty volumes only..."
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz"
+mkdir "$tmp/extracted"
+tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted"
+
+for name in "${empty_volumes[@]}"; do
+  if [ -d "$tmp/extracted/$name" ]; then
+    echo "Restoring $name..."
+    cp -a "$tmp/extracted/$name/." "/restore/$name/"
+  else
+    echo "No $name/ in backup - leaving volume empty."
+  fi
+done
+
+echo "Restore complete."

services/dev/spindown.sh

@@ -9,7 +9,9 @@ cd ..
 docker compose -f gitea-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
+docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-docker volume prune -f -a
+# Deliberately no `docker volume prune` - gitea_data/letsencrypt_data/etc. are
+# named volumes holding real data, backed up to S3 but not disposable.

services/dev/spinup.sh

@@ -4,10 +4,14 @@ set -e
 cd "$(dirname "$0")"
 
 if [ ! -f .env ]; then
-  echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
+  echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
   exit 1
 fi
 
+echo "Restoring named volumes from S3 backup if any are empty..."
+docker compose -f backup-docker-compose.yml pull restore
+docker compose -f backup-docker-compose.yml run --rm restore
+
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 
 docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d
@@ -31,3 +35,6 @@ docker compose pull && docker compose up -d
 cd ..
 
 docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d
+
+docker compose -f backup-docker-compose.yml pull backup
+docker compose -f backup-docker-compose.yml up -d backup

services/dev/traefik-docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - "80:80"
       - "443:443"
     volumes:
-      - "${DATA_DIR}/letsencrypt:/letsencrypt"
+      - "letsencrypt_data:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
     networks:
       - proxy
@@ -36,3 +36,7 @@ services:
 networks:
   proxy:
     name: proxy
+
+volumes:
+  letsencrypt_data:
+    name: letsencrypt_data

services/prod/backup-docker-compose.yml

@@ -0,0 +1,59 @@
+services:
+  # Nightly backup of every named volume on this host to S3. Containers
+  # labeled docker-volume-backup.stop-during-backup=true (see
+  # bitwarden-docker-compose.yml, rd-docker-compose.yml,
+  # status-docker-compose.yml, web-docker-compose.yml) are stopped for the
+  # duration of the backup so their data is captured in a consistent state,
+  # then restarted.
+  backup:
+    image: offen/docker-volume-backup:v2
+    container_name: backup
+    environment:
+      BACKUP_FILENAME: "prod-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
+      BACKUP_CRON_EXPRESSION: "0 3 * * *"
+      BACKUP_RETENTION_DAYS: "14"
+      BACKUP_PRUNING_LEEWAY: "5m"
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - bitwarden_data:/backup/bitwarden_data:ro
+      - rustdesk_data:/backup/rustdesk_data:ro
+      - uptime_kuma_data:/backup/uptime_kuma_data:ro
+      - snexo_data:/backup/snexo_data:ro
+      - letsencrypt_data:/backup/letsencrypt_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+
+  # One-shot: restores empty volumes from the latest S3 backup, then exits.
+  # Run explicitly by spinup.sh before the services that own these volumes
+  # start - not started by `docker compose up`.
+  restore:
+    image: amazon/aws-cli:2
+    container_name: backup-restore
+    entrypoint: [ "/bin/bash", "/restore.sh" ]
+    environment:
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - ./restore.sh:/restore.sh:ro
+      - bitwarden_data:/restore/bitwarden_data
+      - rustdesk_data:/restore/rustdesk_data
+      - uptime_kuma_data:/restore/uptime_kuma_data
+      - letsencrypt_data:/restore/letsencrypt_data
+    restart: "no"
+
+volumes:
+  bitwarden_data:
+    name: bitwarden_data
+  rustdesk_data:
+    name: rustdesk_data
+  uptime_kuma_data:
+    name: uptime_kuma_data
+  letsencrypt_data:
+    name: letsencrypt_data

services/prod/bitwarden-docker-compose.yml

@@ -5,13 +5,16 @@ services:
     image: "vaultwarden/server:latest"
     container_name: vaultwarden
     volumes:
-      - ${DATA_DIR}/bitwarden/:/data/
+      - bitwarden_data:/data/
     networks:
       - proxy
     labels:
       ## Expose Bitwarden Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.bitwarden-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.bitwarden-insecure.rule=Host(`bitwarden.luke-else.co.uk`)"
@@ -24,4 +27,8 @@ services:
 
 networks:
   proxy:
-    external: true
+    external: true
+
+volumes:
+  bitwarden_data:
+    name: bitwarden_data

services/prod/rd-docker-compose.yml

@@ -6,7 +6,7 @@ services:
     restart: unless-stopped
 
     volumes:
-      - ${DATA_DIR}/rustdesk:/root
+      - rustdesk_data:/root
 
     networks:
       - proxy
@@ -20,6 +20,9 @@ services:
     labels:
       - "traefik.enable=true"
 
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.rustdesk-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.rustdesk-insecure.rule=Host(`rd.luke-else.co.uk`)"
@@ -39,7 +42,7 @@ services:
     restart: unless-stopped
 
     volumes:
-      - ${DATA_DIR}/rustdesk:/root
+      - rustdesk_data:/root
 
     networks:
       - proxy
@@ -50,4 +53,8 @@ services:
 
 networks:
   proxy:
-    external: true
+    external: true
+
+volumes:
+  rustdesk_data:
+    name: rustdesk_data

services/prod/restore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Restores this host's named volumes from the latest S3 backup, but only into
+# volumes that are currently empty - never overwrites data that's already
+# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml
+# run --rm restore` before any service that owns one of the mounted volumes
+# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh
+# estate): it just leaves the empty volumes alone.
+set -euo pipefail
+
+S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}"
+ENDPOINT_ARGS=()
+if [ -n "${AWS_ENDPOINT:-}" ]; then
+  ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}")
+fi
+
+empty_volumes=()
+for dir in /restore/*/; do
+  name="$(basename "$dir")"
+  if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+    empty_volumes+=("$name")
+  fi
+done
+
+if [ "${#empty_volumes[@]}" -eq 0 ]; then
+  echo "All volumes already contain data - skipping restore."
+  exit 0
+fi
+
+echo "Empty volumes: ${empty_volumes[*]}"
+
+latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)"
+if [ -z "$latest" ]; then
+  echo "No existing backup found at ${S3_URI} - starting with empty volumes."
+  exit 0
+fi
+
+echo "Restoring ${S3_URI}/${latest} into empty volumes only..."
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz"
+mkdir "$tmp/extracted"
+tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted"
+
+for name in "${empty_volumes[@]}"; do
+  if [ -d "$tmp/extracted/$name" ]; then
+    echo "Restoring $name..."
+    cp -a "$tmp/extracted/$name/." "/restore/$name/"
+  else
+    echo "No $name/ in backup - leaving volume empty."
+  fi
+done
+
+echo "Restore complete."

services/prod/spindown.sh

@@ -8,7 +8,9 @@ docker compose -f bitwarden-docker-compose.yml down
 docker compose -f rd-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
+docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-docker volume prune -f -a
+# Deliberately no `docker volume prune` - bitwarden_data/rustdesk_data/etc. are
+# named volumes holding real data, backed up to S3 but not disposable.

services/prod/spinup.sh

@@ -4,10 +4,14 @@ set -e
 cd "$(dirname "$0")"
 
 if [ ! -f .env ]; then
-  echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
+  echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
   exit 1
 fi
 
+echo "Restoring named volumes from S3 backup if any are empty..."
+docker compose -f backup-docker-compose.yml pull restore
+docker compose -f backup-docker-compose.yml run --rm restore
+
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 
 sleep 20 # Allow Traefik + registry auth to settle before starting the rest of the services
@@ -17,3 +21,6 @@ docker compose -f status-docker-compose.yml pull && docker compose -f status-doc
 docker compose -f web-docker-compose.yml pull && docker compose -f web-docker-compose.yml up -d
 docker compose -f bitwarden-docker-compose.yml pull && docker compose -f bitwarden-docker-compose.yml up -d
 docker compose -f rd-docker-compose.yml pull && docker compose -f rd-docker-compose.yml up -d
+
+docker compose -f backup-docker-compose.yml pull backup
+docker compose -f backup-docker-compose.yml up -d backup

services/prod/status-docker-compose.yml

@@ -3,13 +3,16 @@ services:
     image: louislam/uptime-kuma:latest
     container_name: status
     volumes:
-      - ${DATA_DIR}/uptime-kuma/data:/app/data
+      - uptime_kuma_data:/app/data
     networks:
       - proxy
     labels:
       ## Expose uptime-kuma Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
 
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)"
@@ -23,3 +26,7 @@ services:
 networks:
   proxy:
     external: true
+
+volumes:
+  uptime_kuma_data:
+    name: uptime_kuma_data

services/prod/traefik-docker-compose.yml

@@ -20,7 +20,7 @@ services:
       - "443:443"
       - "27017:27017"
     volumes:
-      - "${DATA_DIR}/letsencrypt:/letsencrypt"
+      - "letsencrypt_data:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
     networks:
       - proxy
@@ -41,3 +41,7 @@ services:
 networks:
   proxy:
     name: proxy
+
+volumes:
+  letsencrypt_data:
+    name: letsencrypt_data

services/prod/web-docker-compose.yml

@@ -1,6 +1,6 @@
 services:
 
-#Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080)
+  #Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080)
   luke-else:
     image: git.luke-else.co.uk/luke-else/luke-else.co.uk
     container_name: luke-else
@@ -9,7 +9,7 @@ services:
     labels:
       ## Expose luke-else Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.personal-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.personal-insecure.rule=Host(`luke-else.co.uk`)"
@@ -28,7 +28,7 @@ services:
     labels:
       ## Expose luke-else Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.personal-dev-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.personal-dev-insecure.rule=Host(`dev.luke-else.co.uk`)"
@@ -47,7 +47,7 @@ services:
     labels:
       ## Expose metarius Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.metarius-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.metarius-insecure.rule=Host(`metarius.luke-else.co.uk`)"
@@ -66,7 +66,7 @@ services:
     labels:
       ## Expose divine-couture Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.divine-couture-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.divine-couture-insecure.rule=Host(`www.divine-couture.co.uk`)"
@@ -77,27 +77,6 @@ services:
       - "traefik.http.routers.divine-couture.tls.certresolver=myresolver"
     restart: unless-stopped
 
-  snexo:
-    image: "php:apache"
-    container_name: snexo
-    volumes:
-      - ${DATA_DIR}/snexo.co.uk/:/var/www/html
-    networks:
-      - proxy
-    labels:
-      ## Expose Snexo Through Trefik ##
-      - "traefik.enable=true" # <== Enable traefik to proxy this container
-
-      - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
-      - "traefik.http.routers.snexo-insecure.middlewares=redirect-web-secure"
-      - "traefik.http.routers.snexo-insecure.rule=Host(`snexo.co.uk`)"
-      - "traefik.http.routers.snexo-insecure.entrypoints=web"
-
-      - "traefik.http.routers.snexo.rule=Host(`snexo.co.uk`)"
-      - "traefik.http.routers.snexo.entrypoints=websecure"
-      - "traefik.http.routers.snexo.tls.certresolver=myresolver"
-    restart: unless-stopped
-
 networks:
   proxy:
     external: true

services/vpn/backup-docker-compose.yml

@@ -0,0 +1,48 @@
+services:
+  # Nightly backup of every named volume on this host to S3. Containers
+  # labeled docker-volume-backup.stop-during-backup=true (see
+  # vpn-docker-compose.yml) are stopped for the duration of the backup so
+  # their data is captured in a consistent state, then restarted.
+  backup:
+    image: offen/docker-volume-backup:v2
+    container_name: backup
+    environment:
+      BACKUP_FILENAME: "vpn-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
+      BACKUP_CRON_EXPRESSION: "0 3 * * *"
+      BACKUP_RETENTION_DAYS: "14"
+      BACKUP_PRUNING_LEEWAY: "5m"
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - openvpn_data:/backup/openvpn_data:ro
+      - letsencrypt_data:/backup/letsencrypt_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+
+  # One-shot: restores empty volumes from the latest S3 backup, then exits.
+  # Run explicitly by spinup.sh before the services that own these volumes
+  # start - not started by `docker compose up`.
+  restore:
+    image: amazon/aws-cli:2
+    container_name: backup-restore
+    entrypoint: ["/bin/bash", "/restore.sh"]
+    environment:
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - ./restore.sh:/restore.sh:ro
+      - openvpn_data:/restore/openvpn_data
+      - letsencrypt_data:/restore/letsencrypt_data
+    restart: "no"
+
+volumes:
+  openvpn_data:
+    name: openvpn_data
+  letsencrypt_data:
+    name: letsencrypt_data

services/vpn/restore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Restores this host's named volumes from the latest S3 backup, but only into
+# volumes that are currently empty - never overwrites data that's already
+# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml
+# run --rm restore` before any service that owns one of the mounted volumes
+# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh
+# estate): it just leaves the empty volumes alone.
+set -euo pipefail
+
+S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}"
+ENDPOINT_ARGS=()
+if [ -n "${AWS_ENDPOINT:-}" ]; then
+  ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}")
+fi
+
+empty_volumes=()
+for dir in /restore/*/; do
+  name="$(basename "$dir")"
+  if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+    empty_volumes+=("$name")
+  fi
+done
+
+if [ "${#empty_volumes[@]}" -eq 0 ]; then
+  echo "All volumes already contain data - skipping restore."
+  exit 0
+fi
+
+echo "Empty volumes: ${empty_volumes[*]}"
+
+latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)"
+if [ -z "$latest" ]; then
+  echo "No existing backup found at ${S3_URI} - starting with empty volumes."
+  exit 0
+fi
+
+echo "Restoring ${S3_URI}/${latest} into empty volumes only..."
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz"
+mkdir "$tmp/extracted"
+tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted"
+
+for name in "${empty_volumes[@]}"; do
+  if [ -d "$tmp/extracted/$name" ]; then
+    echo "Restoring $name..."
+    cp -a "$tmp/extracted/$name/." "/restore/$name/"
+  else
+    echo "No $name/ in backup - leaving volume empty."
+  fi
+done
+
+echo "Restore complete."

services/vpn/spindown.sh

@@ -5,7 +5,9 @@ cd "$(dirname "$0")"
 docker compose -f vpn-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
+docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-docker volume prune -f -a
+# Deliberately no `docker volume prune` - openvpn_data/letsencrypt_data are
+# named volumes holding real data, backed up to S3 but not disposable.

services/vpn/spinup.sh

@@ -3,6 +3,18 @@
 set -e
 cd "$(dirname "$0")"
 
+if [ ! -f .env ]; then
+  echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
+  exit 1
+fi
+
+echo "Restoring named volumes from S3 backup if any are empty..."
+docker compose -f backup-docker-compose.yml pull restore
+docker compose -f backup-docker-compose.yml run --rm restore
+
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 docker compose -f vpn-docker-compose.yml pull && docker compose -f vpn-docker-compose.yml up -d
 docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d
+
+docker compose -f backup-docker-compose.yml pull backup
+docker compose -f backup-docker-compose.yml up -d backup

services/vpn/traefik-docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - "80:80"
       - "443:443"
     volumes:
-      - "./letsencrypt:/letsencrypt"
+      - "letsencrypt_data:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
     networks:
       - proxy
@@ -36,3 +36,7 @@ services:
 networks:
   proxy:
     name: proxy
+
+volumes:
+  letsencrypt_data:
+    name: letsencrypt_data

services/vpn/vpn-docker-compose.yml

@@ -8,11 +8,14 @@ services:
     environment:
         HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address 
     volumes:
-        - ./openvpn_conf:/opt/Dockovpn_data
+        - openvpn_data:/opt/Dockovpn_data
     labels:
       ## Expose vpn Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
 
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.vpn-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.vpn-insecure.rule=Host(`vpn.luke-else.co.uk`)"
@@ -21,4 +24,8 @@ services:
       - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)"
       - "traefik.http.routers.vpn.entrypoints=websecure"
       - "traefik.http.routers.vpn.tls.certresolver=myresolver"
-    restart: always
+    restart: always
+
+volumes:
+  openvpn_data:
+    name: openvpn_data