Merge pull request 'Feat/open tofu' (#8) from feat/open-tofu into main

Reviewed-on: #8

.gitignore

@@ -13,6 +13,13 @@
 # services/docker-compose.yml carve-out above.
 **/.env
 
+# Variables set through control.sh's "Set variables" menu (HCLOUD_TOKEN,
+# BACKUP_S3_*, etc.) - never committed.
+.control.env
+
+### Ansible ###
+*.retry
+
 ### Terraform ###
 # Local .terraform directories
 **/.terraform/*

ansible/README.md

@@ -0,0 +1,93 @@
+# Ansible
+
+Bootstraps, syncs, and starts/stops the Docker Compose stacks in `../services/`
+on the three servers OpenTofu provisions (see `../infra/`). OpenTofu still owns
+the actual cloud resources (servers, network, firewalls, DNS);
+everything downstream of "the server exists" - installing Docker, creating the
+`deploy` user, hardening SSH, copying `services/<host>/`, rendering `.env` /
+the Gitea runner compose file, and running `spinup.sh`/`spindown.sh` - lives
+here instead. Persistent service data lives in named Docker volumes backed up
+to S3, not on a Hetzner volume - see `../readme.md`.
+
+## Contents
+
+- `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API
+  directly (not Terraform state) and groups servers by their `role` label
+  (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules/<host>/main.tf`).
+- `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening,
+  unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`.
+- `roles/deploy/` - copies `services/<host>/` to the server and renders `.env`
+  (S3 backup credentials, see `../readme.md`) and, for `dev`,
+  `Runners/docker-compose.yml`.
+- `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`,
+  `site.yml` (all three in order), and `group_vars/` (`all.yml` - shared:
+  `deploy_user`, SSH key path, `backup_s3_*` S3 backup credentials;
+  `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` - per-role vars, e.g.
+  `service_group`). `group_vars/` lives here rather than at the `ansible/`
+  root because Ansible only auto-loads `group_vars/`/`host_vars/` from
+  directories adjacent to the playbook file or the inventory source - not
+  from an arbitrary sibling directory.
+
+## Prerequisites
+
+```sh
+cd ansible
+ansible-galaxy collection install -r requirements.yml
+export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
+export BACKUP_S3_BUCKET=your-bucket-name
+export BACKUP_S3_ACCESS_KEY_ID=your-access-key-id
+export BACKUP_S3_SECRET_ACCESS_KEY=your-secret-access-key
+export BACKUP_S3_ENDPOINT=your-s3-endpoint   # omit for real AWS S3
+```
+
+`../control.sh`'s **Set variables** menu does the same thing interactively and
+persists your answers to `../.control.env` (gitignored) so you don't have to
+re-export them every session.
+
+`deploy_user` (see `playbooks/group_vars/all.yml`) defaults to `deploy` - override with
+the `DEPLOY_USER` env var if you want a different non-root user created on each
+host. `ansible_ssh_private_key_file` (same file) defaults to
+`~/.ssh/id_ed25519` - override with `ANSIBLE_SSH_PRIVATE_KEY_FILE` if you use a
+different key. It must match one of the SSH keys OpenTofu installed on the
+servers (`var.ssh_key_names`).
+
+## Usage
+
+Same chicken-and-egg as OpenTofu's own first-apply ordering (see
+`../readme.md`): `dev`/`prod`'s firewalls only accept SSH from `vpn`'s own
+public IP, so you must bring `vpn` up and connect to it before `dev`/`prod`
+are reachable at all.
+
+```sh
+# 1. vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly
+ansible-playbook playbooks/bootstrap.yml -l role_vpn
+ansible-playbook playbooks/deploy.yml    -l role_vpn
+ansible-playbook playbooks/spinup.yml    -l role_vpn
+
+# 2. visit https://vpn.luke-else.co.uk, complete wg-easy's first-run setup,
+#    add a client peer, and connect with a WireGuard client - now dev/prod
+#    are reachable, since your egress IP matches their firewall rule
+ansible-playbook playbooks/bootstrap.yml -l role_dev,role_prod
+ansible-playbook playbooks/deploy.yml    -l role_dev,role_prod
+ansible-playbook playbooks/spinup.yml    -l role_dev,role_prod
+
+# ...or once already bootstrapped and connected, do everything in one go:
+ansible-playbook playbooks/site.yml -l role_dev,role_prod
+```
+
+Re-run `deploy.yml` + `spinup.yml` any time a compose file changes - `spinup.sh`
+only starts/updates what's changed, same as before. `dev` always runs three
+Gitea Actions runners (fixed in
+`roles/deploy/templates/runners-docker-compose.yml.j2`).
+
+`spindown.yml` runs `spindown.sh`, which does a full `docker system/volume
+prune -a` on the host - it prompts for a `yes` confirmation before running,
+since the dynamic inventory makes it trivial to target multiple hosts at once
+(`-l role_dev,role_prod`).
+
+Inspect what the dynamic inventory currently sees:
+
+```sh
+ansible-inventory --graph
+ansible-inventory --list
+```

ansible/ansible.cfg

@@ -0,0 +1,9 @@
+[defaults]
+inventory = inventory/hcloud.yml
+roles_path = roles
+host_key_checking = False
+retry_files_enabled = False
+interpreter_python = auto_silent
+
+[inventory]
+enable_plugins = hetzner.hcloud.hcloud, auto

ansible/inventory/hcloud.yml

@@ -0,0 +1,11 @@
+---
+# Dynamic inventory: queries the Hetzner Cloud API directly (HCLOUD_TOKEN),
+# grouping hosts by the `role` label set on each hcloud_server resource in
+# infra/modules/<host>/main.tf (role = dev/prod/vpn).
+plugin: hetzner.hcloud.hcloud
+token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+connect_with: public_ipv4
+keyed_groups:
+  - key: hcloud_labels.role
+    prefix: role
+    separator: "_"

ansible/playbooks/bootstrap.yml

@@ -0,0 +1,9 @@
+---
+- name: Bootstrap newly created servers (idempotent, safe to re-run)
+  hosts: all
+  gather_facts: true
+  become: false
+  vars:
+    ansible_user: root
+  roles:
+    - bootstrap

ansible/playbooks/deploy.yml

@@ -0,0 +1,6 @@
+---
+- name: Render templates and sync service definitions to each host
+  hosts: all
+  gather_facts: false
+  roles:
+    - deploy

ansible/playbooks/group_vars/all.yml

@@ -0,0 +1,14 @@
+---
+deploy_user: "{{ lookup('env', 'DEPLOY_USER') | default('deploy', true) }}"
+ansible_user: "{{ deploy_user }}"
+ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}"
+
+# Repo root's services/ directory, relative to wherever a playbook lives under ansible/playbooks/.
+services_root: "{{ playbook_dir }}/../../services"
+
+# S3-compatible bucket that services/<host>/backup-docker-compose.yml backs up
+# to and restores from
+backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}"
+backup_s3_access_key_id: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY_ID') }}"
+backup_s3_secret_access_key: "{{ lookup('env', 'BACKUP_S3_SECRET_ACCESS_KEY') }}"
+backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}"

ansible/playbooks/group_vars/role_dev.yml

@@ -0,0 +1,2 @@
+---
+service_group: dev

ansible/playbooks/group_vars/role_prod.yml

@@ -0,0 +1,2 @@
+---
+service_group: prod

ansible/playbooks/group_vars/role_vpn.yml

@@ -0,0 +1,2 @@
+---
+service_group: vpn

ansible/playbooks/site.yml

@@ -0,0 +1,4 @@
+---
+- ansible.builtin.import_playbook: bootstrap.yml
+- ansible.builtin.import_playbook: deploy.yml
+- ansible.builtin.import_playbook: spinup.yml

ansible/playbooks/spindown.yml

@@ -0,0 +1,27 @@
+---
+- name: Stop every compose stack on each host and prune Docker resources
+  hosts: all
+  gather_facts: false
+  vars_prompt:
+    - name: confirm
+      prompt: >-
+        This runs spindown.sh (docker compose down, then a full
+        docker image/volume prune -a) on the targeted host(s).
+        Type 'yes' to continue
+      private: false
+  tasks:
+    - name: Abort unless confirmed
+      ansible.builtin.fail:
+        msg: Aborted.
+      when: confirm != 'yes'
+
+    - name: Run spindown.sh
+      ansible.builtin.command: ./spindown.sh
+      args:
+        chdir: "/home/{{ deploy_user }}/services/{{ service_group }}"
+      register: _spindown
+      changed_when: true
+
+    - name: Show spindown output
+      ansible.builtin.debug:
+        var: _spindown.stdout_lines

ansible/playbooks/spinup.yml

@@ -0,0 +1,15 @@
+---
+- name: Start every compose stack on each host, in the order spinup.sh defines
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Run spinup.sh
+      ansible.builtin.command: ./spinup.sh
+      args:
+        chdir: "/home/{{ deploy_user }}/services/{{ service_group }}"
+      register: _spinup
+      changed_when: true
+
+    - name: Show spinup output
+      ansible.builtin.debug:
+        var: _spinup.stdout_lines

ansible/requirements.yml

@@ -0,0 +1,4 @@
+---
+collections:
+  - name: hetzner.hcloud
+    version: ">=4.0.0"

ansible/roles/bootstrap/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Reload sshd
+  ansible.builtin.systemd:
+    name: ssh
+    state: reloaded

ansible/roles/bootstrap/tasks/main.yml

@@ -0,0 +1,89 @@
+---
+# Post-install bootstrap - Idempotent: safe to
+# re-run against an already-bootstrapped host.
+
+- name: Update apt cache and upgrade packages
+  ansible.builtin.apt:
+    update_cache: true
+    upgrade: dist
+    cache_valid_time: 3600
+
+- name: Install Docker Engine and the Compose plugin
+  ansible.builtin.apt:
+    name:
+      - docker.io
+      - docker-compose-v2
+    state: present
+
+- name: Ensure Docker is enabled and running
+  ansible.builtin.systemd:
+    name: docker
+    enabled: true
+    state: started
+
+- name: Create non-root sudo user, given the same key root was provisioned with
+  ansible.builtin.user:
+    name: "{{ deploy_user }}"
+    shell: /bin/bash
+    groups: sudo,docker
+    append: true
+    create_home: true
+
+- name: Ensure deploy user's .ssh directory exists
+  ansible.builtin.file:
+    path: "/home/{{ deploy_user }}/.ssh"
+    state: directory
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+    mode: "0700"
+
+- name: Give deploy user the same authorized_keys as root
+  ansible.builtin.copy:
+    src: /root/.ssh/authorized_keys
+    dest: "/home/{{ deploy_user }}/.ssh/authorized_keys"
+    remote_src: true
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+    mode: "0600"
+
+- name: Grant deploy user passwordless sudo
+  ansible.builtin.copy:
+    content: "{{ deploy_user }} ALL=(ALL) NOPASSWD:ALL\n"
+    dest: "/etc/sudoers.d/90-{{ deploy_user }}"
+    validate: "visudo -cf %s"
+    mode: "0440"
+
+# Root keeps key-only login as a fallback rather than being fully locked out,
+# in case deploy user setup above ever fails silently on a future image.
+- name: Restrict root login to key-only
+  ansible.builtin.lineinfile:
+    path: /etc/ssh/sshd_config
+    regexp: "^#?PermitRootLogin"
+    line: "PermitRootLogin prohibit-password"
+  notify: Reload sshd
+
+- name: Disable SSH password authentication
+  ansible.builtin.lineinfile:
+    path: /etc/ssh/sshd_config
+    regexp: "^#?PasswordAuthentication"
+    line: "PasswordAuthentication no"
+  notify: Reload sshd
+
+- name: Install unattended-upgrades
+  ansible.builtin.apt:
+    name: unattended-upgrades
+    state: present
+
+- name: Enable automatic security upgrades
+  ansible.builtin.copy:
+    dest: /etc/apt/apt.conf.d/20auto-upgrades
+    content: |
+      APT::Periodic::Update-Package-Lists "1";
+      APT::Periodic::Unattended-Upgrade "1";
+    mode: "0644"
+
+- name: Enable and start unattended-upgrades
+  ansible.builtin.systemd:
+    name: unattended-upgrades
+    enabled: true
+    state: started

ansible/roles/deploy/tasks/main.yml

@@ -0,0 +1,41 @@
+---
+# Copies services/<service_group>/ to the server and renders the files that
+# need templating (e.g. .env, docker-compose.yml).
+#
+# Persistent data lives in named Docker volumes
+# see services/<host>/backup-docker-compose.yml. .env only carries the S3
+# credentials that backup/restore need, rendered for every host.
+
+- name: Ensure services directory exists on the host
+  ansible.builtin.file:
+    path: "/home/{{ deploy_user }}/services/{{ service_group }}"
+    state: directory
+    mode: "0755"
+
+- name: Copy static service files (compose files, scripts, configs)
+  ansible.builtin.copy:
+    src: "{{ services_root }}/{{ service_group }}/"
+    dest: "/home/{{ deploy_user }}/services/{{ service_group }}/"
+    mode: preserve
+
+- name: Render S3 backup credentials into .env (auto-loaded by docker compose)
+  ansible.builtin.template:
+    src: env.j2
+    dest: "/home/{{ deploy_user }}/services/{{ service_group }}/.env"
+    mode: "0600"
+
+- name: Render Gitea Actions runner compose file
+  when: service_group == 'dev'
+  ansible.builtin.template:
+    src: runners-docker-compose.yml.j2
+    dest: "/home/{{ deploy_user }}/services/dev/Runners/docker-compose.yml"
+    mode: "0644"
+
+- name: Make spinup/spindown/restore scripts executable
+  ansible.builtin.file:
+    path: "/home/{{ deploy_user }}/services/{{ service_group }}/{{ item }}"
+    mode: "0755"
+  loop:
+    - spinup.sh
+    - spindown.sh
+    - restore.sh

ansible/roles/deploy/templates/env.j2

@@ -0,0 +1,5 @@
+AWS_ACCESS_KEY_ID={{ backup_s3_access_key_id }}
+AWS_SECRET_ACCESS_KEY={{ backup_s3_secret_access_key }}
+AWS_S3_BUCKET_NAME={{ backup_s3_bucket }}
+AWS_S3_PATH={{ service_group }}
+AWS_ENDPOINT={{ backup_s3_endpoint }}

ansible/roles/deploy/templates/runners-docker-compose.yml.j2

@@ -0,0 +1,40 @@
+# Generated by Ansible - do not hand-edit. Three Gitea Actions runners, always.
+# To change their shape (image, volumes, env), edit this template and re-run
+# playbooks/deploy.yml.
+#
+# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable
+# (not baked in here) - services/dev/spinup.sh generates a fresh token and
+# writes it to Runners/.env immediately before starting these containers.
+#
+# /data is a disposable local cache (runner identity re-registers on every
+# spinup.sh run anyway), so it's a plain named volume - not part of the S3
+# backup/restore stack in services/dev/backup-docker-compose.yml.
+{% set runner_count = 3 %}
+services:
+{% for i in range(1, runner_count + 1) %}
+  runner-{{ i }}:
+    image: gitea/act_runner:latest
+    container_name: gitea_runner_{{ i }}
+    volumes:
+      - ./config.yaml:/config.yaml
+      - gitea_runner_{{ i }}_data:/data
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - proxy
+    environment:
+      CONFIG_FILE: /config.yaml
+      GITEA_INSTANCE_URL: "https://git.luke-else.co.uk"
+      GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}"
+      GITEA_RUNNER_NAME: "CICD#{{ i }}"
+    restart: unless-stopped
+{% endfor %}
+
+networks:
+  proxy:
+    external: true
+
+volumes:
+{% for i in range(1, runner_count + 1) %}
+  gitea_runner_{{ i }}_data:
+    name: gitea_runner_{{ i }}_data
+{% endfor %}

control.sh

@@ -0,0 +1,321 @@
+#!/usr/bin/env bash
+# control.sh - interactive control panel for this repo.
+#
+# Steps you through the pre-setup and day-to-day operations for both the
+# OpenTofu infra (infra/) and the Ansible config (ansible/), in the order the
+# firewall rules require (vpn first, then tunnel in, then dev/prod). It only
+# ever runs `tofu`/`ansible-playbook` for you - it's a menu wrapper, nothing
+# more, so anything it does you can also do by hand from infra/ or ansible/.
+
+# Re-exec under bash if invoked as `sh control.sh` (or another POSIX shell) -
+# this script relies on bash-only features (indirect expansion, arrays, [[ ]])
+# that fail with cryptic errors like "bad substitution" under dash/sh.
+if [ -z "${BASH_VERSION:-}" ]; then
+  exec bash "$0" "$@"
+fi
+
+set -uo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INFRA_DIR="$REPO_ROOT/infra"
+ANSIBLE_DIR="$REPO_ROOT/ansible"
+CONTROL_ENV="$REPO_ROOT/.control.env"
+
+# --- pretty output --------------------------------------------------------
+if [ -t 1 ]; then
+  BOLD=$(tput bold); DIM=$(tput dim); RESET=$(tput sgr0)
+  RED=$(tput setaf 1); GREEN=$(tput setaf 2); YELLOW=$(tput setaf 3); CYAN=$(tput setaf 6)
+else
+  BOLD=""; DIM=""; RESET=""; RED=""; GREEN=""; YELLOW=""; CYAN=""
+fi
+
+ok()    { printf "%s  ok %s %s\n"   "$GREEN" "$RESET" "$1"; }
+warn()  { printf "%s warn%s %s\n"   "$YELLOW" "$RESET" "$1"; }
+bad()   { printf "%s miss%s %s\n"   "$RED" "$RESET" "$1"; }
+title() { printf "\n%s%s%s\n" "$BOLD$CYAN" "$1" "$RESET"; }
+
+# Pick the OpenTofu binary: prefer `tofu`, fall back to `terraform`.
+TOFU=""
+detect_tofu() {
+  if command -v tofu >/dev/null 2>&1; then TOFU="tofu"
+  elif command -v terraform >/dev/null 2>&1; then TOFU="terraform"
+  fi
+}
+detect_tofu
+
+# --- variable persistence --------------------------------------------------
+# Variables set via the "Set variables" menu (HCLOUD_TOKEN, BACKUP_S3_*,
+# DEPLOY_USER, ANSIBLE_SSH_PRIVATE_KEY_FILE) live in .control.env, gitignored,
+# so they survive between runs of this script. They are NOT exported into your
+# normal shell - only into control.sh's own process and whatever it runs.
+load_control_env() {
+  if [ -f "$CONTROL_ENV" ]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$CONTROL_ENV"
+    set +a
+  fi
+}
+load_control_env
+
+# Upsert KEY=VALUE into .control.env and export it into this process.
+save_var() {
+  local key="$1" val="$2"
+  touch "$CONTROL_ENV"; chmod 600 "$CONTROL_ENV"
+  local tmp; tmp="$(mktemp)"
+  grep -v "^${key}=" "$CONTROL_ENV" > "$tmp" 2>/dev/null || true
+  mv "$tmp" "$CONTROL_ENV"
+  printf '%s=%q\n' "$key" "$val" >> "$CONTROL_ENV"
+  export "${key?}=${val}"
+}
+
+# Run a command inside a directory, echoing it first. Returns the command's exit code.
+run_in() {
+  local dir="$1"; shift
+  printf "%s\$ (cd %s && %s)%s\n" "$DIM" "${dir#$REPO_ROOT/}" "$*" "$RESET"
+  ( cd "$dir" && "$@" )
+}
+
+pause() { read -rp $'\nPress Enter to return to the menu... ' _; }
+
+confirm() {
+  local reply
+  read -rp "$1 [y/N] " reply
+  [[ "$reply" == "y" || "$reply" == "Y" ]]
+}
+
+# --- prerequisite checks --------------------------------------------------
+check_prereqs() {
+  title "Tools"
+  if [ -n "$TOFU" ]; then ok "OpenTofu ($TOFU: $("$TOFU" version 2>/dev/null | head -n1))"
+  else bad "Neither 'tofu' nor 'terraform' found on PATH"; fi
+  if command -v ansible-playbook >/dev/null 2>&1; then ok "Ansible ($(ansible --version 2>/dev/null | head -n1))"
+  else bad "'ansible-playbook' not found on PATH"; fi
+
+  title "Environment variables"
+  [ -n "${HCLOUD_TOKEN:-}" ]                 && ok "HCLOUD_TOKEN set"                 || bad "HCLOUD_TOKEN not set (needed by both tofu and the ansible inventory)"
+  [ -n "${BACKUP_S3_BUCKET:-}" ]             && ok "BACKUP_S3_BUCKET set"             || warn "BACKUP_S3_BUCKET not set (needed for deploy/spinup - S3 backup .env)"
+  [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ]      && ok "BACKUP_S3_ACCESS_KEY_ID set"      || warn "BACKUP_S3_ACCESS_KEY_ID not set"
+  [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ]  && ok "BACKUP_S3_SECRET_ACCESS_KEY set"  || warn "BACKUP_S3_SECRET_ACCESS_KEY not set"
+  [ -n "${BACKUP_S3_ENDPOINT:-}" ]           && ok "BACKUP_S3_ENDPOINT set"           || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)"
+  ok "DEPLOY_USER: ${DEPLOY_USER:-deploy} (default 'deploy' if unset)"
+
+  local key="${ANSIBLE_SSH_PRIVATE_KEY_FILE:-$HOME/.ssh/id_ed25519}"
+  [ -f "${key/#\~/$HOME}" ] && ok "SSH key present ($key)" || warn "SSH key not found at $key (set ANSIBLE_SSH_PRIVATE_KEY_FILE)"
+
+  [ -f "$CONTROL_ENV" ] && ok ".control.env present (loaded above)" || warn "No .control.env yet - use Configuration > Set variables to persist the above across runs"
+
+  title "Files"
+  [ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \
+    || bad "infra/terraform.tfvars missing (Configuration > Copy terraform.tfvars.example)"
+  [ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \
+    || warn "infra/.terraform missing - run OpenTofu > init"
+  if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed"
+  else warn "hetzner.hcloud collection not installed - run Ansible > Install collections"; fi
+}
+
+# --- configuration actions -------------------------------------------------
+# Prompts for one variable, keeping the current value if the user hits Enter.
+# $3 = true masks input and the shown current value, for tokens/secrets.
+prompt_var() {
+  local key="$1" desc="$2" secret="${3:-false}" current input shown
+  current="${!key:-}"
+  shown="$current"
+  [ "$secret" = "true" ] && [ -n "$current" ] && shown="(set, hidden)"
+  if [ "$secret" = "true" ]; then read -rsp "$desc [$shown]: " input; echo
+  else read -rp "$desc [$shown]: " input; fi
+  [ -n "$input" ] && save_var "$key" "$input"
+}
+
+set_variables() {
+  title "Set variables"
+  echo "Leave a prompt blank to keep its current value. Saved to .control.env" \
+       "(gitignored) and reloaded automatically next time you run control.sh -" \
+       "not exported to your normal shell outside of it."
+  echo
+  prompt_var HCLOUD_TOKEN                 "Hetzner Cloud API token" true
+  prompt_var DEPLOY_USER                  "Non-root deploy user Ansible creates (default: deploy)" false
+  prompt_var ANSIBLE_SSH_PRIVATE_KEY_FILE "SSH private key path (default: ~/.ssh/id_ed25519)" false
+  prompt_var BACKUP_S3_BUCKET             "S3 backup bucket name" false
+  prompt_var BACKUP_S3_ACCESS_KEY_ID      "S3 backup access key ID" true
+  prompt_var BACKUP_S3_SECRET_ACCESS_KEY  "S3 backup secret access key" true
+  prompt_var BACKUP_S3_ENDPOINT           "S3 backup endpoint (blank for real AWS S3)" false
+  ok "Variables saved."
+}
+
+copy_tfvars() {
+  local example="$INFRA_DIR/terraform.tfvars.example"
+  local target="$INFRA_DIR/terraform.tfvars"
+  if [ -f "$target" ]; then
+    warn "infra/terraform.tfvars already exists."
+    confirm "Overwrite it with a fresh copy of the example?" || return 0
+  fi
+  cp "$example" "$target"
+  ok "Copied terraform.tfvars.example -> terraform.tfvars"
+
+  local keys quoted k
+  read -rp "SSH key name(s) from Hetzner Console > Security > SSH Keys, comma-separated (blank = edit manually later): " keys
+  if [ -n "$keys" ]; then
+    quoted=""
+    IFS=',' read -ra _key_arr <<< "$keys"
+    for k in "${_key_arr[@]}"; do
+      k="$(echo "$k" | xargs)"
+      quoted+="\"$k\", "
+    done
+    quoted="${quoted%, }"
+    sed -i "s|^ssh_key_names = .*|ssh_key_names = [${quoted}]|" "$target"
+    ok "Set ssh_key_names = [${quoted}]"
+  else
+    warn "ssh_key_names still has the example's placeholder values - edit infra/terraform.tfvars before running tofu."
+  fi
+}
+
+# --- OpenTofu actions -----------------------------------------------------
+require_tofu() {
+  if [ -z "$TOFU" ]; then bad "No tofu/terraform binary found."; return 1; fi
+  if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then
+    bad "infra/terraform.tfvars is missing - copy terraform.tfvars.example first."; return 1
+  fi
+}
+tofu_init()    { require_tofu && run_in "$INFRA_DIR" "$TOFU" init; }
+tofu_plan()    { require_tofu && run_in "$INFRA_DIR" "$TOFU" plan; }
+tofu_apply()   { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply; }
+tofu_output()  { require_tofu && run_in "$INFRA_DIR" "$TOFU" output; }
+tofu_destroy() {
+  require_tofu || return 1
+  confirm "${RED}Destroy ALL infra managed by OpenTofu?${RESET}" && run_in "$INFRA_DIR" "$TOFU" destroy
+}
+
+# --- Ansible actions ------------------------------------------------------
+ansible_collections() { run_in "$ANSIBLE_DIR" ansible-galaxy collection install -r requirements.yml; }
+ansible_inventory()   { run_in "$ANSIBLE_DIR" ansible-inventory --graph; }
+
+# $1 = playbook, $2 = -l limit expression
+play() { run_in "$ANSIBLE_DIR" ansible-playbook "playbooks/$1" -l "$2"; }
+
+ansible_bootstrap_vpn()   { play bootstrap.yml role_vpn; }
+ansible_deploy_vpn()      { play deploy.yml role_vpn; }
+ansible_spinup_vpn()      { play spinup.yml role_vpn; }
+ansible_bootstrap_dev()   { play bootstrap.yml role_dev; }
+ansible_deploy_dev()      { play deploy.yml role_dev; }
+ansible_spinup_dev()      { play spinup.yml role_dev; }
+ansible_bootstrap_prod()  { play bootstrap.yml role_prod; }
+ansible_deploy_prod()     { play deploy.yml role_prod; }
+ansible_spinup_prod()     { play spinup.yml role_prod; }
+
+ansible_spindown() {
+  local target
+  read -rp "Spindown which hosts? (e.g. role_vpn, role_dev,role_prod, all): " target
+  [ -n "$target" ] && play spindown.yml "$target"
+}
+
+# --- guided walkthrough ---------------------------------------------------
+guided() {
+  title "Guided full setup"
+  cat <<EOF
+This walks the whole bring-up in the order the firewalls require. dev/prod only
+accept SSH from the vpn server's public IP, so vpn must exist and you must be
+connected to it (WireGuard, via wg-easy) before dev/prod are reachable.
+
+Steps, in order:
+  0. Configuration: variables + terraform.tfvars (skip if already done)
+  1. OpenTofu: init -> plan -> apply   (creates servers, network, firewalls, DNS)
+  2. Ansible:  bootstrap + deploy + spinup for role_vpn
+  3. YOU:      visit https://vpn.luke-else.co.uk, complete wg-easy's setup,
+               add a client peer, and connect with a WireGuard client
+  4. Ansible:  bootstrap + deploy + spinup for role_dev, then role_prod
+EOF
+  confirm "Start the guided walkthrough?" || return 0
+
+  echo; title "0/4  Configuration"
+  if confirm "Set/update variables now (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)?"; then
+    set_variables
+  fi
+  if [ ! -f "$INFRA_DIR/terraform.tfvars" ] && confirm "infra/terraform.tfvars is missing - create it from the example now?"; then
+    copy_tfvars
+  fi
+
+  echo; title "1/4  OpenTofu"
+  if confirm "Run $TOFU init/plan/apply now?"; then
+    tofu_init && tofu_plan && tofu_apply || { bad "OpenTofu step failed - stopping."; return 1; }
+  fi
+
+  echo; title "2/4  Ansible - vpn"
+  if confirm "Bootstrap + deploy + spinup role_vpn now?"; then
+    ansible_bootstrap_vpn && ansible_deploy_vpn && ansible_spinup_vpn || { bad "vpn step failed - stopping."; return 1; }
+  fi
+
+  echo; title "3/4  Connect the VPN"
+  warn "Visit https://vpn.luke-else.co.uk, complete wg-easy's setup wizard, add a client peer, and connect with a WireGuard client now."
+  confirm "Are you connected to the VPN?" || { warn "Stopping - reconnect and re-run this step."; return 0; }
+
+  echo; title "4/4  Ansible - dev + prod"
+  if confirm "Bootstrap + deploy + spinup role_dev now?"; then
+    ansible_bootstrap_dev && ansible_deploy_dev && ansible_spinup_dev || { bad "dev step failed."; return 1; }
+  fi
+  if confirm "Bootstrap + deploy + spinup role_prod now?"; then
+    ansible_bootstrap_prod && ansible_deploy_prod && ansible_spinup_prod || { bad "prod step failed."; return 1; }
+  fi
+  ok "Guided setup complete."
+}
+
+# --- menu -----------------------------------------------------------------
+menu() {
+  clear
+  printf "%s==== server control panel ====%s\n" "$BOLD$CYAN" "$RESET"
+  printf "%srepo: %s   tofu: %s%s\n" "$DIM" "$REPO_ROOT" "${TOFU:-none}" "$RESET"
+
+  title "Setup"
+  echo "  1) Check prerequisites"
+  echo "  2) Guided full setup (vpn -> connect -> dev/prod)"
+
+  title "Configuration"
+  echo "  3) Set variables (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)"
+  echo "  4) Copy terraform.tfvars.example -> terraform.tfvars"
+
+  title "OpenTofu (infra/)"
+  echo "  5) init        6) plan        7) apply"
+  echo "  8) output      9) destroy"
+
+  title "Ansible (ansible/)"
+  echo " 10) Install collections        11) Show inventory"
+  echo " 12) Bootstrap vpn              13) Deploy vpn          14) Spinup vpn"
+  echo " 15) Bootstrap dev               16) Deploy dev          17) Spinup dev"
+  echo " 18) Bootstrap prod             19) Deploy prod         20) Spinup prod"
+  echo " 21) Spindown (choose target)"
+
+  echo
+  echo "  0) Quit"
+  echo
+  read -rp "Select an option: " choice
+}
+
+while true; do
+  menu
+  case "$choice" in
+    1)  check_prereqs; pause ;;
+    2)  guided; pause ;;
+    3)  set_variables; pause ;;
+    4)  copy_tfvars; pause ;;
+    5)  tofu_init; pause ;;
+    6)  tofu_plan; pause ;;
+    7)  tofu_apply; pause ;;
+    8)  tofu_output; pause ;;
+    9)  tofu_destroy; pause ;;
+    10) ansible_collections; pause ;;
+    11) ansible_inventory; pause ;;
+    12) ansible_bootstrap_vpn; pause ;;
+    13) ansible_deploy_vpn; pause ;;
+    14) ansible_spinup_vpn; pause ;;
+    15) ansible_bootstrap_dev; pause ;;
+    16) ansible_deploy_dev; pause ;;
+    17) ansible_spinup_dev; pause ;;
+    18) ansible_bootstrap_prod; pause ;;
+    19) ansible_deploy_prod; pause ;;
+    20) ansible_spinup_prod; pause ;;
+    21) ansible_spindown; pause ;;
+    0)  echo "Bye."; exit 0 ;;
+    "") ;;
+    *)  warn "Unknown option: $choice"; pause ;;
+  esac
+done

infra/.terraform.lock.hcl

@@ -33,46 +33,9 @@ provider "registry.opentofu.org/hashicorp/hcloud" {
   ]
 }
 
-provider "registry.opentofu.org/hashicorp/local" {
-  version     = "2.9.0"
-  constraints = "~> 2.5"
-  hashes = [
-    "h1:1dtKYW/5a1qob3yneL6WzOlnSGfYtJ6a2XeejCk9yb4=",
-    "h1:5NseXq5wU8O20ersTtV4ocrLYFFtgFr7n0pRLO1W2Rw=",
-    "h1:5d22ZPPK4iiygPbwRz/PJF5Es/0axVpMlPRpCR0Padw=",
-    "h1:AnwyolirmIlBMjH6+tV8bKkvT+5axJNYxi2y2IguiX4=",
-    "h1:PBp+HeseY021Fw3sLznCG27idgwPoff4cBuNmKgPL2w=",
-    "h1:VDxIhe4GbzdOCdmt7mQaqdwERQW6GSI7Roonts42Gr0=",
-    "h1:ZO6eWWnf8LjjV1q/JNeL9WLtZ6fwIttOnyN5LjCNSEo=",
-    "h1:dPIAf8oUAz+vW2E0iZunMvpuPddRZIztRsPSY1u+VnY=",
-    "h1:fwTDVG9AhFVKQZIb1EXkHv4FqzsZNlLWgkyPGDmZZEE=",
-    "h1:kDc465XPC7/6XFCjrMC4mTqhA9ef0FHKuJ3ZgfGNfeg=",
-    "h1:kGbjxrI2P8MHeyVtE1U3Q1TbyF71ExnHxtkrE+Aj6UU=",
-    "h1:kcoK6Afbsj54u9zaEqpecWAFKytqjBijtguCNwV3d4M=",
-    "h1:rxomJjDwOo+YZ+WIPc25FqEgsz9orh/2MCyUcZmFjvw=",
-    "h1:t0CMn/Rkwquw8l2yQ+O4ApzbMZfY2UazbsDnZygzACA=",
-    "h1:tJwgm2BS4xCGlElCDQEFXQoefY9Y4t0JdSKTtsPBbBo=",
-    "zh:13ef7ecd1e397ec5b20ea588508dd3e3b8d6c50d809ae76b079abf9dd8d02e4b",
-    "zh:2190c9325980076489ce02b0f5dd2c0b91fc8711cefa99e714d8619a32827ad1",
-    "zh:2a0cfc5600730093705071707e4a4e4e953e7d9091859e0f66b46daa1060dd5d",
-    "zh:2ff53eac1af43ab9a2248a0e53c963d46e19cf04bc4c3f323591cfcebb218252",
-    "zh:4ebc3dee700f60af9da29970052fd02fa947813162b224716862dc9d7f1f7542",
-    "zh:5fe6dab84ceeaa8eb3f1567c5f05578333370c472240ca5c5bfc25e92d4d5586",
-    "zh:66bbec16367bbf440045502c9779b11f4ac5b022c8d8d17afe12d431950838b5",
-    "zh:7641e5c2e4b529e869cde29ab5b1de2fd1091489eb745b19ac2709bd7f4dfd84",
-    "zh:855bfba0756d17ce07595ff57d7cf664443d1495127cb88fb063362734b8b22a",
-    "zh:aaec10f237921d60c581d1b7a66f0a8a8019d9802dc04af11b5b981f6682e01d",
-    "zh:e460835a38ffa1e74f6929904bfd14ef473d217fd537b7ce834abe5ce5e2ce07",
-    "zh:ecc4295215db0e4aea3c9329611c31e09a853e1ae207d56742403bd4f5516703",
-    "zh:ee6d9fae63a612072e00402894e14826af7a3351c235b9c5b423b7629a77ca29",
-    "zh:f2b5c8db74aa7ebcf7cd423672358437d42401675069ef67b01ff910054e49d5",
-    "zh:f5aff74d3eb96d4592c7bca5cd3ea89b469e84efbf382944bd0f844a57059c09",
-  ]
-}
-
 provider "registry.opentofu.org/hetznercloud/hcloud" {
   version     = "1.66.0"
-  constraints = "~> 1.45"
+  constraints = "~> 1.54"
   hashes = [
     "h1:+R3t/5wqCb2ics5bNZ4JgtfaAa08Z6uRzXzNtwovVEY=",
     "h1:9c1ECk3N9xhdQKz4NVCfYEFxebJVVeQZS3E6wrR/Hx8=",

infra/main.tf

@@ -1,6 +1,8 @@
 # Root module: wires together the shared network and the three host modules
 # (one per server in docs/architecture.md). Each host module owns its own
-# firewall (and volume, where applicable) - see modules/<host>/main.tf.
+# firewall - see modules/<host>/main.tf. Persistent data lives in named Docker
+# volumes backed up to S3 (see services/<host>/backup-docker-compose.yml), not
+# on Hetzner volumes.
 
 locals {
   # SSH to dev/prod is only permitted from the vpn server's public IP: admins must
@@ -9,11 +11,34 @@ locals {
   # var.allowed_ssh_source_ips for direct SSH access.
   vpn_ssh_source_ips = ["${module.vpn.ipv4}/32"]
 
-  # Rendered once and uploaded+run by every host module immediately after its
+  # IDs for every named key in var.ssh_key_names - installed on every server.
-  # server is created - see scripts/bootstrap.sh.tftpl.
+  ssh_key_ids = [for k in data.hcloud_ssh_key.main : k.id]
-  bootstrap_script = templatefile("${path.module}/scripts/bootstrap.sh.tftpl", {
+
-    deploy_user = var.deploy_user
+  # Every A record currently referenced by a Traefik Host() rule anywhere under
-  })
+  # services/ - kept in sync with the actual compose files, not aspirational.
+  # "dev.luke-else.co.uk" is a *website* served from prod (a dev-tagged build of
+  # luke-else.co.uk), not the dev server - don't confuse the two.
+  dns_records = [
+    { zone = "luke-else.co.uk", name = "@", value = module.prod.ipv4 },
+    { zone = "luke-else.co.uk", name = "dev", value = module.prod.ipv4 },
+    { zone = "luke-else.co.uk", name = "metarius", value = module.prod.ipv4 },
+    { zone = "luke-else.co.uk", name = "traefik", value = module.prod.ipv4 },
+    { zone = "luke-else.co.uk", name = "status", value = module.prod.ipv4 },
+    { zone = "luke-else.co.uk", name = "bitwarden", value = module.prod.ipv4 },
+    { zone = "luke-else.co.uk", name = "rd", value = module.prod.ipv4 },
+    { zone = "luke-else.co.uk", name = "git", value = module.dev.ipv4 },
+    { zone = "luke-else.co.uk", name = "traefik.cicd", value = module.dev.ipv4 },
+    { zone = "luke-else.co.uk", name = "vpn", value = module.vpn.ipv4 },
+    { zone = "luke-else.co.uk", name = "traefik.vpn", value = module.vpn.ipv4 },
+  ]
+
+  # Microsoft 365 mail records for luke-else.co.uk
+  mail_records = [
+    { zone = "luke-else.co.uk", name = "@", type = "MX", value = "0 lukeelse-co-uk01b.mail.protection.outlook.com." },
+    { zone = "luke-else.co.uk", name = "@", type = "TXT", value = "\"v=spf1 include:spf.protection.outlook.com -all\"" },
+    { zone = "luke-else.co.uk", name = "@", type = "TXT", value = "\"google-site-verification=aKLk5BATowKUwNbFcU5zyfu-AIyCW9W9PBStePRjH3U\"" },
+    { zone = "luke-else.co.uk", name = "autodiscover", type = "CNAME", value = "autodiscover.outlook.com." },
+  ]
 }
 
 module "network" {
@@ -30,15 +55,11 @@ module "dev" {
   server_type            = var.dev_server_type
   image                  = var.server_image
   location               = var.location
-  ssh_key_id             = data.hcloud_ssh_key.main.id
+  ssh_key_ids            = local.ssh_key_ids
   network_id             = module.network.id
   private_ip             = var.dev_private_ip
-  volume_size            = var.dev_volume_size
   allowed_ssh_source_ips = local.vpn_ssh_source_ips
   network_ip_range       = var.network_ip_range
-  bootstrap_script       = local.bootstrap_script
-  ssh_private_key_path   = var.ssh_private_key_path
-  runner_count           = var.dev_runner_count
 
   # module.network.id alone doesn't guarantee the subnet exists yet, and a server
   # can't join a network before it has a subnet.
@@ -51,14 +72,11 @@ module "prod" {
   server_type            = var.prod_server_type
   image                  = var.server_image
   location               = var.location
-  ssh_key_id             = data.hcloud_ssh_key.main.id
+  ssh_key_ids            = local.ssh_key_ids
   network_id             = module.network.id
   private_ip             = var.prod_private_ip
-  volume_size            = var.prod_volume_size
   allowed_ssh_source_ips = local.vpn_ssh_source_ips
   network_ip_range       = var.network_ip_range
-  bootstrap_script       = local.bootstrap_script
-  ssh_private_key_path   = var.ssh_private_key_path
 
   depends_on = [module.network]
 }
@@ -69,8 +87,13 @@ module "vpn" {
   server_type            = var.vpn_server_type
   image                  = var.server_image
   location               = var.location
-  ssh_key_id             = data.hcloud_ssh_key.main.id
+  ssh_key_ids            = local.ssh_key_ids
   allowed_ssh_source_ips = var.allowed_ssh_source_ips
-  bootstrap_script       = local.bootstrap_script
+}
-  ssh_private_key_path   = var.ssh_private_key_path
+
+module "dns" {
+  source = "./modules/dns"
+
+  zone_names = var.dns_zones
+  records    = concat(local.dns_records, local.mail_records)
 }

infra/modules/dev/main.tf

@@ -1,26 +1,24 @@
 # dev: Gitea + Runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk)
 
-# Port sources: services/dev/*docker-compose.yml (published ports) and services/todo.md
+# Port sources: services/dev/*docker-compose.yml (published ports)
-# (the documented UFW allow-list). dev has no firewall in docs/architecture.md, but we
-# add one anyway for baseline safety - see conversation history.
 resource "hcloud_firewall" "this" {
   name = "dev-firewall"
 
-  rule { # server ssh - wired to the vpn server's public IP only, see infra/main.tf
+  rule { # server ssh - wired to the vpn server's public IP only
     direction  = "in"
     protocol   = "tcp"
     port       = "22"
     source_ips = var.allowed_ssh_source_ips
   }
 
-  rule { # gitea ssh (git.luke-else.co.uk, published as 222:22)
+  rule { # gitea ssh
     direction  = "in"
     protocol   = "tcp"
     port       = "222"
     source_ips = ["0.0.0.0/0", "::/0"]
   }
 
-  rule { # Traefik http/https (git.luke-else.co.uk, cicd.luke-else.co.uk)
+  rule { # Traefik http/https
     direction  = "in"
     protocol   = "tcp"
     port       = "80"
@@ -54,53 +52,13 @@ resource "hcloud_server" "this" {
   server_type  = var.server_type
   image        = var.image
   location     = var.location
-  ssh_keys     = [var.ssh_key_id]
+  ssh_keys     = var.ssh_key_ids
   firewall_ids = [hcloud_firewall.this.id]
+  labels       = { role = "dev" }
 
   network {
     network_id = var.network_id
     ip         = var.private_ip
   }
-
-  connection {
-    type        = "ssh"
-    host        = self.ipv4_address
-    user        = "root"
-    private_key = file(var.ssh_private_key_path)
-  }
-
-  provisioner "file" {
-    content     = var.bootstrap_script
-    destination = "/root/bootstrap.sh"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "chmod +x /root/bootstrap.sh",
-      "/root/bootstrap.sh",
-    ]
-  }
 }
 
-resource "hcloud_volume" "storage" {
-  name      = "dev-storage"
-  size      = var.volume_size
-  server_id = hcloud_server.this.id
-  automount = true
-  format    = "ext4"
-
-  lifecycle {
-    prevent_destroy = true
-  }
-}
-
-# Renders services/dev/Runners/docker-compose.yml directly into the repo, with
-# one runner service per var.runner_count. This only touches the local working
-# tree - deploying the change still goes through the normal scp + spinup.sh flow.
-resource "local_file" "runners_compose" {
-  filename = "${path.root}/../services/dev/Runners/docker-compose.yml"
-  content = templatefile("${path.module}/templates/runners-docker-compose.yml.tftpl", {
-    runner_count = var.runner_count
-  })
-  file_permission = "0644"
-}

infra/modules/dev/templates/runners-docker-compose.yml.tftpl

@@ -1,30 +0,0 @@
-# Generated by OpenTofu from var.dev_runner_count - do not hand-edit.
-# To change the number of runners, edit dev_runner_count in terraform.tfvars and
-# run tofu apply. To change their shape, edit
-# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead.
-#
-# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable
-# (not baked in here) - services/dev/spinup.sh generates a fresh token and
-# writes it to Runners/.env immediately before starting these containers.
-services:
-%{ for i in range(runner_count) ~}
-  runner-${i + 1}:
-    image: gitea/act_runner:latest
-    container_name: gitea_runner_${i + 1}
-    volumes:
-      - ./config.yaml:/config.yaml
-      - ./gitea_runner_${i + 1}:/data
-      - /var/run/docker.sock:/var/run/docker.sock
-    networks:
-      - proxy
-    environment:
-      CONFIG_FILE: /config.yaml
-      GITEA_INSTANCE_URL: "https://git.luke-else.co.uk"
-      GITEA_RUNNER_REGISTRATION_TOKEN: "$${GITEA_RUNNER_REGISTRATION_TOKEN}"
-      GITEA_RUNNER_NAME: "CICD#${i + 1}"
-    restart: unless-stopped
-%{ endfor ~}
-
-networks:
-  proxy:
-    external: true

infra/modules/dev/variables.tf

@@ -13,9 +13,9 @@ variable "location" {
   type        = string
 }
 
-variable "ssh_key_id" {
+variable "ssh_key_ids" {
-  description = "ID of the Hetzner Cloud SSH key to install on the server."
+  description = "IDs of the Hetzner Cloud SSH keys to install on the server."
-  type        = string
+  type        = list(string)
 }
 
 variable "network_id" {
@@ -28,11 +28,6 @@ variable "private_ip" {
   type        = string
 }
 
-variable "volume_size" {
-  description = "Size in GB of the volume attached to dev (disk2 in docs/architecture.md)."
-  type        = number
-}
-
 variable "allowed_ssh_source_ips" {
   description = "CIDRs allowed to reach port 22 on dev. Set by the root module to the vpn server's public IP."
   type        = list(string)
@@ -42,18 +37,3 @@ variable "network_ip_range" {
   description = "CIDR of the private network, allowed through the firewall for traffic from prod."
   type        = string
 }
-
-variable "bootstrap_script" {
-  description = "Rendered post-install script, uploaded and executed on the server immediately after creation."
-  type        = string
-}
-
-variable "ssh_private_key_path" {
-  description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH."
-  type        = string
-}
-
-variable "runner_count" {
-  description = "Number of Gitea Actions runner containers to render into services/dev/Runners/docker-compose.yml."
-  type        = number
-}

infra/modules/dns/main.tf

@@ -0,0 +1,37 @@
+# One Hetzner DNS zone per domain in var.zone_names. Hetzner becomes
+# authoritative for these once the domain's registrar NS records are pointed at
+# the nameservers in the `nameservers` output - that step happens at whichever
+# registrar the domain was bought through, and can't be done from here.
+resource "hcloud_zone" "this" {
+  for_each = toset(var.zone_names)
+
+  name = each.value
+  mode = "primary"
+}
+
+locals {
+  # Group flat var.records entries by zone/name/type into one set of values
+  # each, since a Hetzner RRSet is keyed on (zone, name, type) and can hold
+  # several values (e.g. an SPF and a google-site-verification TXT both live
+  # under the same "@" TXT RRSet).
+  rrsets = {
+    for key, items in { for r in var.records : "${r.zone}/${r.name}/${r.type}" => r... } : key => {
+      zone   = items[0].zone
+      name   = items[0].name
+      type   = items[0].type
+      ttl    = items[0].ttl
+      values = [for i in items : i.value]
+    }
+  }
+}
+
+resource "hcloud_zone_rrset" "this" {
+  for_each = local.rrsets
+
+  zone = hcloud_zone.this[each.value.zone].id
+  name = each.value.name
+  type = each.value.type
+  ttl  = each.value.ttl
+
+  records = [for v in each.value.values : { value = v }]
+}

infra/modules/dns/moved.tf

@@ -0,0 +1,60 @@
+# Generalizing hcloud_zone_rrset.a (for_each keyed "zone/name", A records
+# only) into hcloud_zone_rrset.this (keyed "zone/name/type", any record type)
+# changed every existing instance's resource address. Without these, `tofu
+# apply` would destroy and recreate every A record already in state instead of
+# just adding the new mail records - these moved blocks make it a no-op rename
+# for anything that already existed. Safe to delete once applied and merged.
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/@"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/@/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/dev"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/dev/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/metarius"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/metarius/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/traefik"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/traefik/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/status"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/status/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/bitwarden"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/bitwarden/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/rd"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/rd/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/git"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/git/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/traefik.cicd"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/traefik.cicd/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/vpn"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/vpn/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/traefik.vpn"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/traefik.vpn/A"]
+}

infra/modules/dns/outputs.tf

@@ -0,0 +1,9 @@
+output "zone_ids" {
+  description = "Map of domain name to Hetzner zone ID."
+  value       = { for name, z in hcloud_zone.this : name => z.id }
+}
+
+output "nameservers" {
+  description = "Map of domain name to the nameservers Hetzner assigned it - point your registrar's NS records at these to delegate DNS."
+  value       = { for name, z in hcloud_zone.this : name => z.authoritative_nameservers.assigned }
+}

infra/modules/dns/variables.tf

@@ -0,0 +1,24 @@
+variable "zone_names" {
+  description = "Domains to create as Hetzner DNS zones."
+  type        = list(string)
+}
+
+variable "records" {
+  description = <<-EOT
+    Records to create across the zones in var.zone_names. Use name = "@" for
+    the zone apex. Entries sharing the same zone/name/type become a single
+    Hetzner RRSet with multiple values (e.g. two TXT strings under the same
+    name) - Hetzner has no concept of separate same-type records otherwise.
+    For MX, embed the priority in value using standard zonefile syntax (e.g.
+    "10 mail.example.com."); Hetzner's API has no separate priority field.
+    For TXT, wrap value in escaped double quotes (e.g. "\"v=spf1 ...\"").
+  EOT
+  type = list(object({
+    zone  = string
+    name  = string
+    value = string
+    type  = optional(string, "A")
+    ttl   = optional(number, 300)
+  }))
+  default = []
+}

infra/modules/network/main.tf

@@ -1,5 +1,5 @@
-# Private network shared by dev and prod (see docs/architecture.md "network" group).
+# Private network shared by dev and prod (see readme.md "network" group).
-# vpn is intentionally not attached to this network - it sits outside it in the diagram.
+# vpn is intentionally not attached to this network
 resource "hcloud_network" "main" {
   name     = "server-network"
   ip_range = var.ip_range

infra/modules/prod/main.tf

@@ -1,18 +1,15 @@
-# prod: Traefik, Websites, Bitwarden, RustDesk, status page
+# Port sources: services/prod/*docker-compose.yml (published ports)
-
-# Port sources: services/prod/*docker-compose.yml (published ports) and services/todo.md
-# (the documented UFW allow-list).
 resource "hcloud_firewall" "this" {
   name = "prod-firewall"
 
-  rule { # server ssh - wired to the vpn server's public IP only, see infra/main.tf
+  rule { # server ssh - wired to the vpn server's public IP only
     direction  = "in"
     protocol   = "tcp"
     port       = "22"
     source_ips = var.allowed_ssh_source_ips
   }
 
-  rule { # Traefik http/https (Websites, Bitwarden, status page)
+  rule { # Traefik http/https
     direction  = "in"
     protocol   = "tcp"
     port       = "80"
@@ -61,7 +58,7 @@ resource "hcloud_firewall" "this" {
     source_ips = ["0.0.0.0/0", "::/0"]
   }
 
-  rule { # traffic from dev over the private network
+  rule { # traffic from over the private network
     direction  = "in"
     protocol   = "tcp"
     port       = "1-65535"
@@ -81,42 +78,13 @@ resource "hcloud_server" "this" {
   server_type  = var.server_type
   image        = var.image
   location     = var.location
-  ssh_keys     = [var.ssh_key_id]
+  ssh_keys     = var.ssh_key_ids
   firewall_ids = [hcloud_firewall.this.id]
+  labels       = { role = "prod" }
 
   network {
     network_id = var.network_id
     ip         = var.private_ip
   }
-
-  connection {
-    type        = "ssh"
-    host        = self.ipv4_address
-    user        = "root"
-    private_key = file(var.ssh_private_key_path)
-  }
-
-  provisioner "file" {
-    content     = var.bootstrap_script
-    destination = "/root/bootstrap.sh"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "chmod +x /root/bootstrap.sh",
-      "/root/bootstrap.sh",
-    ]
-  }
 }
 
-resource "hcloud_volume" "storage" {
-  name      = "prod-storage"
-  size      = var.volume_size
-  server_id = hcloud_server.this.id
-  automount = true
-  format    = "ext4"
-
-  lifecycle {
-    prevent_destroy = true
-  }
-}

infra/modules/prod/variables.tf

@@ -13,9 +13,9 @@ variable "location" {
   type        = string
 }
 
-variable "ssh_key_id" {
+variable "ssh_key_ids" {
-  description = "ID of the Hetzner Cloud SSH key to install on the server."
+  description = "IDs of the Hetzner Cloud SSH keys to install on the server."
-  type        = string
+  type        = list(string)
 }
 
 variable "network_id" {
@@ -28,11 +28,6 @@ variable "private_ip" {
   type        = string
 }
 
-variable "volume_size" {
-  description = "Size in GB of the volume attached to prod (disk1 in docs/architecture.md)."
-  type        = number
-}
-
 variable "allowed_ssh_source_ips" {
   description = "CIDRs allowed to reach port 22 on prod. Set by the root module to the vpn server's public IP."
   type        = list(string)
@@ -42,13 +37,3 @@ variable "network_ip_range" {
   description = "CIDR of the private network, allowed through the firewall for traffic from dev."
   type        = string
 }
-
-variable "bootstrap_script" {
-  description = "Rendered post-install script, uploaded and executed on the server immediately after creation."
-  type        = string
-}
-
-variable "ssh_private_key_path" {
-  description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH."
-  type        = string
-}

infra/modules/vpn/main.tf

@@ -1,4 +1,4 @@
-# vpn: OpenVPN + Traefik. Not attached to the private network (see docs/architecture.md).
+# vpn: WireGuard (wg-easy) + Traefik. Not attached to the private network.
 resource "hcloud_firewall" "this" {
   name = "vpn-firewall"
 
@@ -9,7 +9,7 @@ resource "hcloud_firewall" "this" {
     source_ips = var.allowed_ssh_source_ips
   }
 
-  rule { # Traefik http/https (traefik.vpn.luke-else.co.uk)
+  rule { # Traefik http/https
     direction  = "in"
     protocol   = "tcp"
     port       = "80"
@@ -23,10 +23,10 @@ resource "hcloud_firewall" "this" {
     source_ips = ["0.0.0.0/0", "::/0"]
   }
 
-  rule { # OpenVPN tunnel - always direct to this server's public IP, never via a load balancer
+  rule { # WireGuard tunnel - always direct to this server's public IP, never via a load balancer
     direction  = "in"
     protocol   = "udp"
-    port       = "1194"
+    port       = "51820"
     source_ips = ["0.0.0.0/0", "::/0"]
   }
 }
@@ -36,25 +36,7 @@ resource "hcloud_server" "this" {
   server_type  = var.server_type
   image        = var.image
   location     = var.location
-  ssh_keys     = [var.ssh_key_id]
+  ssh_keys     = var.ssh_key_ids
   firewall_ids = [hcloud_firewall.this.id]
-
+  labels       = { role = "vpn" }
-  connection {
-    type        = "ssh"
-    host        = self.ipv4_address
-    user        = "root"
-    private_key = file(var.ssh_private_key_path)
-  }
-
-  provisioner "file" {
-    content     = var.bootstrap_script
-    destination = "/root/bootstrap.sh"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "chmod +x /root/bootstrap.sh",
-      "/root/bootstrap.sh",
-    ]
-  }
 }

infra/modules/vpn/variables.tf

@@ -1,5 +1,5 @@
 variable "server_type" {
-  description = "Server type for vpn (OpenVPN + Traefik)."
+  description = "Server type for vpn (WireGuard/wg-easy + Traefik)."
   type        = string
 }
 
@@ -13,22 +13,12 @@ variable "location" {
   type        = string
 }
 
-variable "ssh_key_id" {
+variable "ssh_key_ids" {
-  description = "ID of the Hetzner Cloud SSH key to install on the server."
+  description = "IDs of the Hetzner Cloud SSH keys to install on the server."
-  type        = string
+  type        = list(string)
 }
 
 variable "allowed_ssh_source_ips" {
   description = "CIDRs allowed to reach port 22 on vpn."
   type        = list(string)
 }
-
-variable "bootstrap_script" {
-  description = "Rendered post-install script, uploaded and executed on the server immediately after creation."
-  type        = string
-}
-
-variable "ssh_private_key_path" {
-  description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH."
-  type        = string
-}

infra/outputs.tf

@@ -17,3 +17,8 @@ output "prod_private_ipv4" {
 output "vpn_ipv4" {
   value = module.vpn.ipv4
 }
+
+output "dns_nameservers" {
+  description = "Nameservers Hetzner assigned each zone - point each domain's registrar NS records at these to delegate DNS to Hetzner."
+  value       = module.dns.nameservers
+}

infra/scripts/bootstrap.sh.tftpl

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-# Post-install bootstrap - uploaded and executed once by OpenTofu immediately
-# after server creation (see modules/<host>/main.tf). Idempotent-ish: safe to
-# re-run by hand, but only ever runs automatically on first create.
-set -euo pipefail
-
-export DEBIAN_FRONTEND=noninteractive
-DEPLOY_USER="${deploy_user}"
-
-apt-get update -y
-apt-get upgrade -y
-
-# --- Docker Engine + Compose plugin ---
-if ! command -v docker >/dev/null 2>&1; then
-  apt-get install -y ca-certificates curl gnupg
-  install -m 0755 -d /etc/apt/keyrings
-  curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
-  chmod a+r /etc/apt/keyrings/docker.asc
-  . /etc/os-release
-  echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $VERSION_CODENAME stable" \
-    > /etc/apt/sources.list.d/docker.list
-  apt-get update -y
-  apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-fi
-
-# --- Non-root sudo user, given the same key root was provisioned with ---
-if ! id -u "$DEPLOY_USER" >/dev/null 2>&1; then
-  useradd --create-home --shell /bin/bash "$DEPLOY_USER"
-  usermod -aG sudo,docker "$DEPLOY_USER"
-  mkdir -p "/home/$DEPLOY_USER/.ssh"
-  cp /root/.ssh/authorized_keys "/home/$DEPLOY_USER/.ssh/authorized_keys"
-  chown -R "$DEPLOY_USER:$DEPLOY_USER" "/home/$DEPLOY_USER/.ssh"
-  chmod 700 "/home/$DEPLOY_USER/.ssh"
-  chmod 600 "/home/$DEPLOY_USER/.ssh/authorized_keys"
-  echo "$DEPLOY_USER ALL=(ALL) NOPASSWD:ALL" > "/etc/sudoers.d/90-$DEPLOY_USER"
-  chmod 440 "/etc/sudoers.d/90-$DEPLOY_USER"
-fi
-
-# --- SSH hardening: key-only auth everywhere. Root keeps key-only login as a
-# fallback rather than being fully locked out, in case $DEPLOY_USER setup above
-# ever fails silently on a future image. ---
-sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
-sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config
-systemctl reload ssh 2>/dev/null || systemctl reload sshd
-
-# --- Unattended security upgrades ---
-apt-get install -y unattended-upgrades
-dpkg-reconfigure -f noninteractive unattended-upgrades
-systemctl enable --now unattended-upgrades

infra/ssh.tf

@@ -1,3 +1,4 @@
 data "hcloud_ssh_key" "main" {
-  name = var.ssh_key_name
+  for_each = toset(var.ssh_key_names)
+  name     = each.value
 }

infra/terraform.tfvars.example

@@ -2,24 +2,17 @@
 # never commit real values there.
 #
 # The Hetzner API token is NOT set here: export it as HCLOUD_TOKEN in your shell
-# before running tofu plan/apply.
+# before running tofu plan/apply. The same token and SSH key are used again by
+# Ansible afterwards - see ../ansible/README.md.
 
-# Name of an SSH key already uploaded to your Hetzner Cloud project
+# Names of SSH keys already uploaded to your Hetzner Cloud project
-# (Console > Security > SSH Keys). Required.
+# (Console > Security > SSH Keys). All of them are installed on every server. Required.
-ssh_key_name = "your-key-name"
+ssh_key_names = ["luke-else@PC", "luke-else@laptop"]
-
-# Local path to the private key matching ssh_key_name above. OpenTofu uses this
-# once per server to upload and run the post-install bootstrap script
-# (infra/scripts/bootstrap.sh.tftpl) immediately after creation. Required.
-ssh_private_key_path = "~/.ssh/id_ed25519"
 
 # Optional overrides - defaults live in variables.tf
 # location               = "nbg1"
-# dev_server_type        = "cx22"
+# dev_server_type        = "cx23"
-# prod_server_type       = "cx22"
+# prod_server_type       = "cx23"
-# vpn_server_type        = "cx22"
+# vpn_server_type        = "cx23"
-# dev_volume_size        = 20
+# allowed_ssh_source_ips = ["0.0.0.0/0"]
-# prod_volume_size       = 20
+# dns_zones              = ["luke-else.co.uk"]
-# allowed_ssh_source_ips = ["203.0.113.4/32"]
-# deploy_user             = "deploy"
-# dev_runner_count       = 3

infra/variables.tf

@@ -1,5 +1,5 @@
 variable "location" {
-  description = "Hetzner Cloud datacenter location for all servers and volumes."
+  description = "Hetzner Cloud datacenter location for all servers."
   type        = string
   default     = "nbg1"
 }
@@ -13,37 +13,25 @@ variable "network_zone" {
 variable "server_image" {
   description = "OS image used for all servers."
   type        = string
-  default     = "ubuntu-24.04"
+  default     = "ubuntu-26.04"
 }
 
 variable "dev_server_type" {
   description = "Server type for dev"
   type        = string
-  default     = "cx22"
+  default     = "cx23"
 }
 
 variable "prod_server_type" {
   description = "Server type for prod"
   type        = string
-  default     = "cx22"
+  default     = "cx23"
 }
 
 variable "vpn_server_type" {
   description = "Server type for vpn"
   type        = string
-  default     = "cx22"
+  default     = "cx23"
-}
-
-variable "dev_volume_size" {
-  description = "Size in GB of the volume attached to dev"
-  type        = number
-  default     = 20
-}
-
-variable "prod_volume_size" {
-  description = "Size in GB of the volume attached to prod (disk1 in docs/architecture.md)."
-  type        = number
-  default     = 20
 }
 
 variable "network_ip_range" {
@@ -70,9 +58,9 @@ variable "prod_private_ip" {
   default     = "10.0.1.11"
 }
 
-variable "ssh_key_name" {
+variable "ssh_key_names" {
-  description = "Name of an SSH key already uploaded to your Hetzner Cloud project (Console > Security > SSH Keys)."
+  description = "Names of SSH keys already uploaded to your Hetzner Cloud project (Console > Security > SSH Keys). All are installed on every server."
-  type        = string
+  type        = list(string)
 }
 
 variable "allowed_ssh_source_ips" {
@@ -81,19 +69,8 @@ variable "allowed_ssh_source_ips" {
   default     = ["0.0.0.0/0", "::/0"]
 }
 
-variable "ssh_private_key_path" {
+variable "dns_zones" {
-  description = "Local path to the private key matching var.ssh_key_name. Used once per server by OpenTofu to upload and run the post-install bootstrap script over SSH immediately after creation (see infra/scripts/bootstrap.sh.tftpl)."
+  description = "Domains to manage as Hetzner DNS zones. Point each domain's registrar NS records at tofu output dns_nameservers for Hetzner to actually become authoritative."
-  type        = string
+  type        = list(string)
-}
+  default     = ["luke-else.co.uk"]
-
-variable "deploy_user" {
-  description = "Non-root sudo user created on every server by the bootstrap script, with the same SSH key as root."
-  type        = string
-  default     = "deploy"
-}
-
-variable "dev_runner_count" {
-  description = "Number of Gitea Actions runner containers to run on dev. Rendered into services/dev/Runners/docker-compose.yml on every tofu apply - see infra/modules/dev/templates/runners-docker-compose.yml.tftpl."
-  type        = number
-  default     = 1
 }

infra/versions.tf

@@ -4,11 +4,7 @@ terraform {
   required_providers {
     hcloud = {
       source  = "hetznercloud/hcloud"
-      version = "~> 1.45"
+      version = "~> 1.54" # hcloud_zone / hcloud_zone_rrset (DNS) require >= 1.54.0
-    }
-    local = {
-      source  = "hashicorp/local"
-      version = "~> 2.5"
     }
   }
 }

readme.md

@@ -1,6 +1,6 @@
 # Server
 
-Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).
+Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), bootstrapped and deployed with [Ansible](https://www.ansible.com/), running Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).
 
 <p align="center">
   <img src="assets/images/main.png" width="70%">
@@ -10,12 +10,14 @@ Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted
 
 - [Architecture](#architecture)
 - [Repository layout](#repository-layout)
+- [Design choices](#design-choices)
 - [Prerequisites](#prerequisites)
+- [Quickstart](#quickstart)
 - [Provisioning the infrastructure](#provisioning-the-infrastructure-infra)
-- [Deploying the services](#deploying-the-services-services)
+- [Bootstrapping and deploying](#bootstrapping-and-deploying-ansible)
+- [Running the services](#running-the-services-services)
 - [Service inventory](#service-inventory)
-- [First-time setup](#first-time-setup)
+- [Caveats](#caveats)
-- [Development container](#development-container)
 - [Security notes](#security-notes)
 
 ## Architecture
@@ -28,9 +30,6 @@ architecture-beta
 
     group network(cloud)[network] in cloud
 
-    service disk1(mdi:disk)[Storage] in cloud
-    service disk2(mdi:disk)[Storage] in cloud
-
     service dev(mdi:server)[dev] in network
 
     service prod(mdi:server)[prod] in network
@@ -41,68 +40,86 @@ architecture-beta
 
     service gateway(mdi:web)[gateway] in cloud
 
+    service backups(mdi:bucket)[S3 Backups]
+
     dev:L -- R:prod
-    disk1:B -- T:prod
-    disk2:B -- T:dev
 
     prod:B -- T:prodfirewall
     vpn:B -- T:vpnfirewall
 
     prodfirewall: L -- R: gateway
     vpnfirewall: B -- T: gateway
+
+    dev:T -- B:backups
+    prod:T -- B:backups
+    vpn:R -- L:backups
 ```
 
-`gateway` represents the public internet, not a provisioned resource.
+`gateway` is the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from (see [Design choices](#design-choices)) — also not a Hetzner resource.
 
-| Server | Purpose | Network | Volume |
+| Server | Purpose | Network |
-|---|---|---|---|
+|---|---|---|
-| `dev`  | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) | `dev-storage` |
+| `dev`  | Gitea, CI runners, dev-facing Traefik | Private network only |
-| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | `prod-storage` |
+| `prod` | Public websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall |
-| `vpn`  | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | none |
+| `vpn`  | WireGuard (wg-easy) + its own Traefik | **Not** on the private network — isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` |
 
-`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach deployment targets on `prod` without exposing that traffic publicly. `vpn` is deliberately kept off this network. Each server has its own Hetzner Cloud Firewall (see `infra/modules/<host>/main.tf`) that only opens the ports actually used by the compose stacks running on it, plus SSH restricted to `var.allowed_ssh_source_ips`.
+`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach `prod` without exposing that traffic publicly. Each server has its own Hetzner Cloud Firewall (`infra/modules/<host>/main.tf`) opening only the ports its own compose stacks use, plus SSH restricted to `var.allowed_ssh_source_ips`.
 
 ## Repository layout
 
 ```
 .
-├── infra/                    # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes
+├── infra/                    # OpenTofu — provisions the 3 servers, network, firewalls, DNS
-│   ├── main.tf                # root module: wires network + dev/prod/vpn modules together
+│   ├── main.tf                # root module: wires network + dev/prod/vpn/dns modules together
-│   ├── variables.tf           # shared inputs (sizes, locations, IP ranges, SSH key name)
+│   ├── variables.tf           # shared inputs (sizes, locations, IP ranges, SSH key names)
-│   ├── outputs.tf             # pass-through outputs from each host module
-│   ├── ssh.tf                 # looks up the SSH key already uploaded to Hetzner Cloud
-│   ├── versions.tf            # provider requirements
 │   ├── terraform.tfvars.example
-│   ├── scripts/
-│   │   └── bootstrap.sh.tftpl # post-install script run on every host right after creation
 │   └── modules/
-│       ├── network/           # shared private network + subnet (used by dev and prod)
+│       ├── network/            # shared private network + subnet (dev + prod)
-│       ├── dev/                # dev server + firewall + volume + renders services/dev/Runners/docker-compose.yml
+│       ├── dev/ prod/ vpn/     # one server + firewall each, labeled role=<name>
-│       │   └── templates/
+│       └── dns/                # Hetzner DNS zones + records for every domain in var.dns_zones
-│       │       └── runners-docker-compose.yml.tftpl # shape of each generated runner service
+├── ansible/                   # bootstraps each server, deploys/starts services/<host>/ onto it
-│       ├── prod/               # prod server + firewall + volume
+│   ├── inventory/hcloud.yml    # dynamic inventory — queries the Hetzner API, groups by role label
-│       └── vpn/                # vpn server + firewall (no private network, no volume)
+│   ├── roles/
+│   │   ├── bootstrap/           # Docker, deploy user, sshd hardening, unattended-upgrades
+│   │   └── deploy/              # copies services/<host>/, renders .env + Runners/docker-compose.yml
+│   └── playbooks/               # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml
 ├── services/                 # Docker Compose stacks, grouped by which server they run on
-│   ├── dev/                   # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk)
+│   ├── dev/                    # Gitea + CI runners + Traefik + backup/restore
-│   ├── prod/                  # Websites, Bitwarden, RustDesk, status page + Traefik
+│   ├── prod/                   # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore
-│   ├── vpn/                    # OpenVPN + Traefik
+│   └── vpn/                     # WireGuard (wg-easy) + Traefik + backup/restore
-│   └── todo.md                # Outstanding manual setup/hardening tasks
+├── .devcontainer/             # git submodule: shared devcontainer for this repo (OpenTofu tooling)
-├── docs/
-│   └── architecture.md       # Source of the architecture diagram above
-├── .devcontainer/             # Git submodule: shared devcontainer for working on this repo (OpenTofu tooling)
 └── assets/
 ```
 
-Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` mirrors this same dev/prod/vpn split on the provisioning side, so a given host's cloud resources and its compose stacks are easy to find side by side.
+Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per service, a `backup-docker-compose.yml` + `restore.sh` pair, and a `spinup.sh` / `spindown.sh` pair that brings up or tears down everything on that host in order. `infra/modules/` and `ansible/roles/deploy` mirror the same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks sit side by side.
+
+## Design choices
+
+- **OpenTofu and Ansible have a clean split**: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches a server over SSH. Everything from "the server exists" onward — Docker, the `deploy` user, SSH hardening, copying `services/<host>/`, running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md).
+- **No Hetzner Cloud Volumes.** Every stateful service mounts a plain named Docker volume instead. Each host backs its volumes up nightly to S3 and restores any that are empty on first boot — see [Persistent data](#persistent-data).
+- **`vpn` is deliberately off the private network.** SSH to `dev`/`prod` is only reachable through it (see [Security notes](#security-notes)), but it can't reach either directly — containing a compromised VPN endpoint.
+- **DNS is provisioned, not just documented.** `module.dns` creates a Hetzner zone and every A/MX/TXT/CNAME record in `infra/main.tf` — see [Managing DNS](#managing-dns).
+- **Hosts are discovered dynamically** from the Hetzner API by role label, not a static inventory file — nothing in Ansible reads Terraform state.
+- **Gitea Actions runners are disposable.** Their `/data` is a local cache; runner identity re-registers with Gitea on every `spinup.sh`, so they're excluded from backup.
 
 ## Prerequisites
 
 - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token
-- An SSH key uploaded to that project (Console → Security → SSH Keys), and the matching private key available locally (OpenTofu uses it once per server to run the post-install bootstrap — see below)
+- One or more SSH keys uploaded to that project (Console → Security → SSH Keys), plus the matching private key available locally
 - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0`
-- DNS records for the domains in [Service inventory](#service-inventory) pointed at the relevant server's public IP
+- [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`)
+- Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner
+- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/)) and an access key/secret pair — not provisioned by OpenTofu, so create this yourself
 
-Docker + the Compose plugin no longer need installing by hand — the bootstrap script below handles that.
+## Quickstart
+
+[`control.sh`](control.sh) at the repo root is an interactive menu covering the whole setup:
+
+```sh
+./control.sh
+```
+
+It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, and the `BACKUP_S3_*` credentials, saved to gitignored `.control.env` so you only enter them once), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand.
 
 ## Provisioning the infrastructure (`infra/`)
 
@@ -110,54 +127,61 @@ Docker + the Compose plugin no longer need installing by hand — the bootstrap
 cd infra
 export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
 cp terraform.tfvars.example terraform.tfvars
-$EDITOR terraform.tfvars                     # set ssh_key_name and ssh_private_key_path at minimum
+$EDITOR terraform.tfvars                     # set ssh_key_names at minimum
 
 tofu init
 tofu plan
 tofu apply
 ```
 
-This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` in `infra/main.tf`:
+This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there.
-- `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`)
-- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses (`modules/dev`, `modules/prod`, `modules/vpn`)
-- one `hcloud_volume` each for `dev` and `prod` (`modules/dev`, `modules/prod`; `vpn` has none)
 
-Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`.
+Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`.
 
-`terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`.
+### Managing DNS
 
-### Post-install bootstrap
+Creating a zone doesn't make Hetzner authoritative by itself — run `tofu output dns_nameservers` and set those as the domain's NS records at its registrar. Propagation time depends on the registrar and the domain's previous NS TTL.
 
-Immediately after each `hcloud_server` is created, OpenTofu uploads and runs [`infra/scripts/bootstrap.sh.tftpl`](infra/scripts/bootstrap.sh.tftpl) over SSH as `root` (`connection` + `file`/`remote-exec` provisioners in each `modules/<host>/main.tf`). It:
+To add a subdomain, add an entry to `local.dns_records` in `infra/main.tf` (zone, name, target host) and re-run `tofu apply` — don't hand-create records in the Hetzner console, they'll drift from state. `var.records` supports `MX`/`TXT`/`CNAME`/etc. too (see `infra/modules/dns/variables.tf` for the value format each type expects); entries sharing zone/name/type merge into one RRSet.
 
-- installs Docker Engine + the Compose plugin
+`local.mail_records` tracks the Microsoft 365 records for `luke-else.co.uk` (MX, SPF, Google-site-verification, `autodiscover`). DMARC and DKIM aren't set up yet — see [Caveats](#caveats).
-- creates a non-root sudo user (`var.deploy_user`, default `deploy`) with the same SSH key as root, in the `sudo` and `docker` groups
-- hardens `sshd`: password authentication off, root login restricted to key-only (`PermitRootLogin prohibit-password`)
-- installs and enables `unattended-upgrades` for automatic security patches
 
-After bootstrap, SSH in as `var.deploy_user` for day-to-day work — root key-based login still works as a fallback.
+## Bootstrapping and deploying (`ansible/`)
 
-### Scaling Gitea Actions runners
+Once `tofu apply` has created the servers, Ansible installs Docker, creates the `deploy` user, hardens SSH, copies `services/<host>/` to each server, and runs `spinup.sh`/`spindown.sh`. Full detail in [`ansible/README.md`](ansible/README.md); the short version:
 
-The number of Gitea Actions runner containers on `dev` is an OpenTofu input, `var.dev_runner_count` (default `3`). On every `tofu apply`, `modules/dev`'s `local_file.runners_compose` resource renders [`modules/dev/templates/runners-docker-compose.yml.tftpl`](infra/modules/dev/templates/runners-docker-compose.yml.tftpl) straight into [`services/dev/Runners/docker-compose.yml`](services/dev/Runners/docker-compose.yml) in your working tree — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated: change `dev_runner_count` in `terraform.tfvars` and re-run `tofu apply` rather than hand-editing it.
+```sh
+cd ansible
+ansible-galaxy collection install -r requirements.yml
+export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
 
-This only updates your local working tree — `tofu apply` doesn't reach out and start containers on `dev` itself. Redeploy as usual (re-copy `services/dev/` to the server and re-run `spinup.sh`) to apply a count change.
+# vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly
+ansible-playbook playbooks/site.yml -l role_vpn
 
-Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore.
+# SSH to vpn as deploy and connect to the WireGuard tunnel it just started, then:
+ansible-playbook playbooks/site.yml -l role_dev,role_prod
+```
 
-### First apply: bootstrapping order matters
+`dev`/`prod`'s firewalls only accept SSH from `vpn`'s own public IP, so on a from-scratch estate they aren't reachable until `vpn` is up and you're tunneled through it — visit `https://vpn.luke-else.co.uk`, complete wg-easy's first-run setup, add a client peer, and connect before running the second command. If it's run before you're connected, Ansible just fails to connect — re-run once connected.
 
-`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch `tofu apply`, the `dev`/`prod` bootstrap provisioners can't connect yet. Bring the estate up in this order:
+### Persistent data
 
-1. `tofu apply -target=module.vpn` — creates `vpn` only; its firewall still allows SSH from `var.allowed_ssh_source_ips`, so its own bootstrap provisioner runs fine.
+Every stateful service mounts a named Docker volume (`gitea_data`, `bitwarden_data`, `letsencrypt_data`, etc.) rather than a Hetzner volume. Each host's `backup-docker-compose.yml` defines:
-2. Deploy and start the VPN service on `vpn` (see [Deploying the services](#deploying-the-services-services)), then connect to it with an OpenVPN client.
-3. `tofu apply` — creates `network`, `dev`, `prod`. Now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and the `dev`/`prod` bootstrap provisioners can connect.
 
-If step 3 is run before you're connected to the VPN, the `dev`/`prod` provisioners will fail and Terraform will mark those servers tainted — re-running `tofu apply` once connected will recreate and re-bootstrap them.
+- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), nightly cron, tars every mounted volume to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/<host>-<timestamp>.tar.gz`, 14 days of retention. Containers that need a consistent snapshot are stopped for the duration and restarted after.
+- **`restore`** — a one-shot container `spinup.sh` runs before anything else starts. Any volume that's already empty is populated from the most recent backup; anything with data is left untouched. On a fresh server (or no prior backup), it's a no-op.
 
-## Deploying the services (`services/`)
+The S3 credentials (`AWS_S3_BUCKET_NAME`, `AWS_ACCESS_KEY_ID`, etc.) are rendered into `services/<host>/.env` by Ansible from the `BACKUP_S3_*` env vars — `spinup.sh` refuses to start anything if `.env` is missing. To trigger either manually:
 
-Once a server exists, copy the relevant `services/<host>/` directory to it (e.g. `scp -r services/prod user@<prod_ipv4>:~/services`) and run the matching script from inside that directory:
+```sh
+cd services/<host>
+docker compose -f backup-docker-compose.yml exec backup backup   # force an immediate backup
+docker compose -f backup-docker-compose.yml run --rm restore     # restore any currently-empty volumes
+```
+
+## Running the services (`services/`)
+
+After `ansible-playbook playbooks/deploy.yml` has copied `services/<host>/` to the server, SSH in as `deploy` and run the matching script — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do this remotely:
 
 ```sh
 # on dev
@@ -165,25 +189,19 @@ Once a server exists, copy the relevant `services/<host>/` directory to it (e.g.
 ./spindown.sh   # reverse order, then prunes images/volumes
 
 # on prod
-./spinup.sh    # Traefik, → Watchtower → status → websites → Bitwarden → RustDesk
+./spinup.sh    # Traefik → Watchtower → status → websites → Bitwarden → RustDesk
 ./spindown.sh
 
 # on vpn
-./spinup.sh     # Traefik → OpenVPN → Watchtower
+./spinup.sh     # Traefik → wg-easy → Watchtower
 ./spindown.sh
 ```
 
-Each stack can also be managed individually with plain Compose, e.g.:
+Each stack can also be managed individually with plain Compose, e.g. `docker compose -f bitwarden-docker-compose.yml up -d` from inside `services/prod`.
 
-```sh
+All three hosts run [Watchtower](https://containrrr.dev/watchtower/), polling every 60s with cleanup enabled, so images stay current automatically once deployed — re-run the spinup scripts only after adding/removing a service or changing compose files. Every public-facing service is fronted by its host's own Traefik, terminating TLS via Let's Encrypt, joining an external `proxy` network and opting in via `traefik.enable=true` labels (RustDesk and the Gitea SSH port publish ports directly, since they aren't HTTP).
-cd services/prod
-docker compose -f bitwarden-docker-compose.yml up -d
-docker compose -f bitwarden-docker-compose.yml down
-```
 
-All three hosts run [Watchtower](https://containrrr.dev/watchtower/) polling every 60s with cleanup enabled, so images are kept current automatically once deployed — the spinup scripts only need to be re-run after adding/removing a service or changing compose files.
+`vpn` runs [wg-easy](https://github.com/wg-easy/wg-easy) — WireGuard plus a web UI, fronted by Traefik. It has no automated first-run setup: visit `https://vpn.luke-else.co.uk`, pick an admin username/password (enable 2FA — this account can view/rotate every client's private key and is reachable from the public internet), set the connect host/port (`vpn.luke-else.co.uk` / `51820`), then add a client per device and download its config or scan its QR code. wg-easy's own state lives in the `wg_easy_data` volume, backed up like everything else; client configs themselves are never stored anywhere but the UI.
-
-Every public-facing service is fronted by its host's own Traefik instance, terminating TLS via Let's Encrypt (`tlschallenge`, port 80/443). Each stack joins an external `proxy` Docker network and opts in via `traefik.enable=true` labels rather than publishing ports directly (RustDesk and the Gitea SSH port are the deliberate exceptions, since they aren't HTTP).
 
 ## Service inventory
 
@@ -193,8 +211,9 @@ Every public-facing service is fronted by its host's own Traefik instance, termi
 |---|---|---|
 | Traefik | `traefik-docker-compose.yml` | `traefik.cicd.luke-else.co.uk` |
 | Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` |
-| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | `cicd.luke-else.co.uk` |
+| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated, 3 by default) | N/A |
 | Watchtower | `watchtower-docker-compose.yml` | — |
+| Backup/restore | `backup-docker-compose.yml` | — |
 
 ### `prod`
 
@@ -206,25 +225,20 @@ Every public-facing service is fronted by its host's own Traefik instance, termi
 | Bitwarden (Vaultwarden) | `bitwarden-docker-compose.yml` | `bitwarden.luke-else.co.uk` |
 | RustDesk relay (hbbs/hbbr) | `rd-docker-compose.yml` | `rd.luke-else.co.uk`, ports `21115-21119` |
 | Watchtower | `watchtower-docker-compose.yml` | — |
+| Backup/restore | `backup-docker-compose.yml` | — |
 
 ### `vpn`
 
 | Service | Compose file | Domain(s) |
 |---|---|---|
 | Traefik | `traefik-docker-compose.yml` | `traefik.vpn.luke-else.co.uk` |
-| OpenVPN (Dockovpn) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `1194` |
+| WireGuard (wg-easy) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `51820` |
 | Watchtower | `watchtower-docker-compose.yml` | — |
-
+| Backup/restore | `backup-docker-compose.yml` | — |
-## First-time setup
-
-A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here:
-
-- **Traefik dashboard auth**: the committed basic-auth hash is a placeholder. Generate your own with `echo $(htpasswd -nb user password) | sed -e 's/\$/\$\$/g'` and replace the `traefik-auth` middleware value in each `traefik-docker-compose.yml`.
-- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by the [post-install bootstrap](#post-install-bootstrap); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)).
 
 ## Development container
 
-`.devcontainer` is a git submodule providing a ready-to-use OpenTofu development environment (VS Code + OpenTofu/Docker/Mermaid extensions). After cloning:
+`.devcontainer` is a git submodule providing a ready-to-use OpenTofu development environment (VS Code + OpenTofu/Docker/Mermaid extensions):
 
 ```sh
 git submodule update --init --recursive
@@ -232,10 +246,17 @@ git submodule update --init --recursive
 
 Then reopen the repo in VS Code with the Dev Containers extension.
 
+## Caveats
+
+- **UFW isn't configured** — the Hetzner Cloud Firewalls already allowlist per-host ports, but there's no host-level firewall as defense in depth yet.
+- **DMARC and DKIM aren't set up** for `luke-else.co.uk`. DKIM's targets are generated per-tenant by Microsoft 365 (Defender portal → Email & collaboration → Policies & rules → DKIM) and can't be guessed — enable it there first, then add the two CNAMEs it gives you to `local.mail_records`.
+- **Only IPv4 DNS is managed.** None of the infra modules track servers' IPv6 addresses, so no AAAA records are generated even though the firewalls already allow IPv6 traffic.
+- **`infra/modules/dns/moved.tf`** re-points pre-existing A records at their new resource address after the DNS module was generalized for non-A record types. It's a one-time migration aid — safe to delete once `tofu plan` shows no unexpected changes.
+
 ## Security notes
 
-- Real secrets (`HCLOUD_TOKEN`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`.
+- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_*`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. S3 credentials land in each host's `services/<host>/.env` (mode `0600`, gitignored, rendered by Ansible).
-- SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s).
+- SSH to `dev`/`prod` is restricted to `vpn`'s own public IP — you must be tunneled into the VPN to reach them. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your egress IP(s).
-- `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP.
+- Firewalls are per-host allowlists (`infra/modules/<host>/main.tf`) — only ports actually used by that host's compose stacks are open.
-- Firewalls are allowlists scoped per host in `infra/modules/<host>/main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open.
+- Ansible's bootstrap role disables SSH password auth, restricts root login to key-only, and creates a separate sudo user (`deploy_user`) for day-to-day access.
-- Every host's [post-install bootstrap](#post-install-bootstrap) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`var.deploy_user`) for day-to-day access.
+- DNS zones carry both Hetzner's `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, across all three managed domains.

services/dev/Runners/docker-compose.yml

@@ -1,28 +0,0 @@
-# Generated by OpenTofu from var.dev_runner_count - do not hand-edit.
-# To change the number of runners, edit dev_runner_count in terraform.tfvars and
-# run tofu apply. To change their shape, edit
-# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead.
-#
-# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable
-# (not baked in here) - services/dev/spinup.sh generates a fresh token and
-# writes it to Runners/.env immediately before starting these containers.
-services:
-  runner-1:
-    image: gitea/act_runner:latest
-    container_name: gitea_runner_1
-    volumes:
-      - ./config.yaml:/config.yaml
-      - ./gitea_runner_1:/data
-      - /var/run/docker.sock:/var/run/docker.sock
-    networks:
-      - proxy
-    environment:
-      CONFIG_FILE: /config.yaml
-      GITEA_INSTANCE_URL: "https://git.luke-else.co.uk"
-      GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}"
-      GITEA_RUNNER_NAME: "CICD#1"
-    restart: unless-stopped
-
-networks:
-  proxy:
-    external: true

services/dev/backup-docker-compose.yml

@@ -0,0 +1,48 @@
+services:
+  # Nightly backup of every named volume on this host to S3. Containers
+  # labeled docker-volume-backup.stop-during-backup=true (see
+  # gitea-docker-compose.yml) are stopped for the duration of the backup so
+  # their data is captured in a consistent state, then restarted.
+  backup:
+    image: offen/docker-volume-backup:v2
+    container_name: backup
+    environment:
+      BACKUP_FILENAME: "dev-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
+      BACKUP_CRON_EXPRESSION: "0 3 * * *"
+      BACKUP_RETENTION_DAYS: "14"
+      BACKUP_PRUNING_LEEWAY: "5m"
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - gitea_data:/backup/gitea_data:ro
+      - letsencrypt_data:/backup/letsencrypt_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+
+  # One-shot: restores empty volumes from the latest S3 backup, then exits.
+  # Run explicitly by spinup.sh before the services that own these volumes
+  # start - not started by `docker compose up`.
+  restore:
+    image: amazon/aws-cli:latest
+    container_name: backup-restore
+    entrypoint: ["/bin/bash", "/restore.sh"]
+    environment:
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - ./restore.sh:/restore.sh:ro
+      - gitea_data:/restore/gitea_data
+      - letsencrypt_data:/restore/letsencrypt_data
+    restart: "no"
+
+volumes:
+  gitea_data:
+    name: gitea_data
+  letsencrypt_data:
+    name: letsencrypt_data

services/dev/gitea-docker-compose.yml

@@ -4,7 +4,7 @@ services:
     image: gitea/gitea:latest
     container_name: gitea
     volumes:
-      - ./gitea:/data
+      - gitea_data:/data
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
     networks:
@@ -30,6 +30,10 @@ services:
       ## Expose Gitea Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
 
+      # Stopped for the duration of each backup run so the sqlite db/repos are
+      # captured in a consistent state - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowmethods=*"
       - "traefik.http.middlewares.cors-gitea.headers.accesscontrolalloworiginlist=*"
       - "traefik.http.middlewares.cors-gitea.headers.addvaryheader=true"
@@ -52,3 +56,7 @@ services:
 networks:
   proxy:
     external: true
+
+volumes:
+  gitea_data:
+    name: gitea_data

services/dev/restore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Restores this host's named volumes from the latest S3 backup, but only into
+# volumes that are currently empty - never overwrites data that's already
+# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml
+# run --rm restore` before any service that owns one of the mounted volumes
+# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh
+# estate): it just leaves the empty volumes alone.
+set -euo pipefail
+
+S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}"
+ENDPOINT_ARGS=()
+if [ -n "${AWS_ENDPOINT:-}" ]; then
+  ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}")
+fi
+
+empty_volumes=()
+for dir in /restore/*/; do
+  name="$(basename "$dir")"
+  if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+    empty_volumes+=("$name")
+  fi
+done
+
+if [ "${#empty_volumes[@]}" -eq 0 ]; then
+  echo "All volumes already contain data - skipping restore."
+  exit 0
+fi
+
+echo "Empty volumes: ${empty_volumes[*]}"
+
+latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)"
+if [ -z "$latest" ]; then
+  echo "No existing backup found at ${S3_URI} - starting with empty volumes."
+  exit 0
+fi
+
+echo "Restoring ${S3_URI}/${latest} into empty volumes only..."
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz"
+mkdir "$tmp/extracted"
+tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted"
+
+for name in "${empty_volumes[@]}"; do
+  if [ -d "$tmp/extracted/$name" ]; then
+    echo "Restoring $name..."
+    cp -a "$tmp/extracted/$name/." "/restore/$name/"
+  else
+    echo "No $name/ in backup - leaving volume empty."
+  fi
+done
+
+echo "Restore complete."

services/dev/spindown.sh

@@ -9,7 +9,9 @@ cd ..
 docker compose -f gitea-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
+docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-docker volume prune -f -a
+# Deliberately no `docker volume prune` - gitea_data/letsencrypt_data/etc. are
+# named volumes holding real data, backed up to S3 but not disposable.

services/dev/spinup.sh

@@ -3,13 +3,23 @@
 set -e
 cd "$(dirname "$0")"
 
+if [ ! -f .env ]; then
+  echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
+  exit 1
+fi
+
+echo "Restoring named volumes from S3 backup if any are empty..."
+docker compose -f backup-docker-compose.yml pull restore
+docker compose -f backup-docker-compose.yml run --rm restore
+
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 
 docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d
 
 echo "Waiting for Gitea to become ready..."
+sleep 20
 for i in $(seq 1 30); do
-  if docker exec gitea gitea admin user list >/dev/null 2>&1; then
+  if docker exec -u 1000:1000 gitea gitea admin user list >/dev/null 2>&1; then
     break
   fi
   sleep 2
@@ -18,7 +28,7 @@ done
 # Generate a fresh Actions runner registration token straight from Gitea and
 # hand it to the runner containers via .env (docker compose auto-loads this) -
 # no manual copy/paste from the admin UI needed.
-RUNNER_TOKEN=$(docker exec gitea gitea actions generate-runner-token)
+RUNNER_TOKEN=$(docker exec -u 1000:1000 gitea gitea actions generate-runner-token)
 echo "GITEA_RUNNER_REGISTRATION_TOKEN=$RUNNER_TOKEN" > Runners/.env
 
 cd Runners
@@ -26,3 +36,6 @@ docker compose pull && docker compose up -d
 cd ..
 
 docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d
+
+docker compose -f backup-docker-compose.yml pull backup
+docker compose -f backup-docker-compose.yml up -d backup

services/dev/traefik-docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - "80:80"
       - "443:443"
     volumes:
-      - "./letsencrypt:/letsencrypt"
+      - "letsencrypt_data:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
     networks:
       - proxy
@@ -31,10 +31,12 @@ services:
       - "traefik.http.routers.traefik.entrypoints=websecure"
       - "traefik.http.routers.traefik.service=api@internal"
       - "traefik.http.routers.traefik.tls.certresolver=myresolver"
-      - "traefik.http.routers.traefik.middlewares=traefik-auth"
-      - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS"
     restart: unless-stopped
 
 networks:
   proxy:
     name: proxy
+
+volumes:
+  letsencrypt_data:
+    name: letsencrypt_data

services/prod/backup-docker-compose.yml

@@ -0,0 +1,58 @@
+services:
+  # Nightly backup of every named volume on this host to S3. Containers
+  # labeled docker-volume-backup.stop-during-backup=true (see
+  # bitwarden-docker-compose.yml, rd-docker-compose.yml,
+  # status-docker-compose.yml, web-docker-compose.yml) are stopped for the
+  # duration of the backup so their data is captured in a consistent state,
+  # then restarted.
+  backup:
+    image: offen/docker-volume-backup:v2
+    container_name: backup
+    environment:
+      BACKUP_FILENAME: "prod-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
+      BACKUP_CRON_EXPRESSION: "0 3 * * *"
+      BACKUP_RETENTION_DAYS: "14"
+      BACKUP_PRUNING_LEEWAY: "5m"
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - bitwarden_data:/backup/bitwarden_data:ro
+      - rustdesk_data:/backup/rustdesk_data:ro
+      - uptime_kuma_data:/backup/uptime_kuma_data:ro
+      - letsencrypt_data:/backup/letsencrypt_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+
+  # One-shot: restores empty volumes from the latest S3 backup, then exits.
+  # Run explicitly by spinup.sh before the services that own these volumes
+  # start - not started by `docker compose up`.
+  restore:
+    image: amazon/aws-cli:latest
+    container_name: backup-restore
+    entrypoint: [ "/bin/bash", "/restore.sh" ]
+    environment:
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - ./restore.sh:/restore.sh:ro
+      - bitwarden_data:/restore/bitwarden_data
+      - rustdesk_data:/restore/rustdesk_data
+      - uptime_kuma_data:/restore/uptime_kuma_data
+      - letsencrypt_data:/restore/letsencrypt_data
+    restart: "no"
+
+volumes:
+  bitwarden_data:
+    name: bitwarden_data
+  rustdesk_data:
+    name: rustdesk_data
+  uptime_kuma_data:
+    name: uptime_kuma_data
+  letsencrypt_data:
+    name: letsencrypt_data

services/prod/bitwarden-docker-compose.yml

@@ -5,13 +5,16 @@ services:
     image: "vaultwarden/server:latest"
     container_name: vaultwarden
     volumes:
-      - ./bitwarden/:/data/
+      - bitwarden_data:/data/
     networks:
       - proxy
     labels:
       ## Expose Bitwarden Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
 
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.bitwarden-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.bitwarden-insecure.rule=Host(`bitwarden.luke-else.co.uk`)"
@@ -25,3 +28,7 @@ services:
 networks:
   proxy:
     external: true
+
+volumes:
+  bitwarden_data:
+    name: bitwarden_data

services/prod/rd-docker-compose.yml

@@ -6,7 +6,7 @@ services:
     restart: unless-stopped
 
     volumes:
-      - ./data:/root
+      - rustdesk_data:/root
 
     networks:
       - proxy
@@ -20,6 +20,9 @@ services:
     labels:
       - "traefik.enable=true"
 
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.rustdesk-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.rustdesk-insecure.rule=Host(`rd.luke-else.co.uk`)"
@@ -39,7 +42,7 @@ services:
     restart: unless-stopped
 
     volumes:
-      - ./data:/root
+      - rustdesk_data:/root
 
     networks:
       - proxy
@@ -51,3 +54,7 @@ services:
 networks:
   proxy:
     external: true
+
+volumes:
+  rustdesk_data:
+    name: rustdesk_data

services/prod/restore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Restores this host's named volumes from the latest S3 backup, but only into
+# volumes that are currently empty - never overwrites data that's already
+# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml
+# run --rm restore` before any service that owns one of the mounted volumes
+# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh
+# estate): it just leaves the empty volumes alone.
+set -euo pipefail
+
+S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}"
+ENDPOINT_ARGS=()
+if [ -n "${AWS_ENDPOINT:-}" ]; then
+  ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}")
+fi
+
+empty_volumes=()
+for dir in /restore/*/; do
+  name="$(basename "$dir")"
+  if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+    empty_volumes+=("$name")
+  fi
+done
+
+if [ "${#empty_volumes[@]}" -eq 0 ]; then
+  echo "All volumes already contain data - skipping restore."
+  exit 0
+fi
+
+echo "Empty volumes: ${empty_volumes[*]}"
+
+latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)"
+if [ -z "$latest" ]; then
+  echo "No existing backup found at ${S3_URI} - starting with empty volumes."
+  exit 0
+fi
+
+echo "Restoring ${S3_URI}/${latest} into empty volumes only..."
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz"
+mkdir "$tmp/extracted"
+tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted"
+
+for name in "${empty_volumes[@]}"; do
+  if [ -d "$tmp/extracted/$name" ]; then
+    echo "Restoring $name..."
+    cp -a "$tmp/extracted/$name/." "/restore/$name/"
+  else
+    echo "No $name/ in backup - leaving volume empty."
+  fi
+done
+
+echo "Restore complete."

services/prod/spindown.sh

@@ -8,7 +8,9 @@ docker compose -f bitwarden-docker-compose.yml down
 docker compose -f rd-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
+docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-docker volume prune -f -a
+# Deliberately no `docker volume prune` - bitwarden_data/rustdesk_data/etc. are
+# named volumes holding real data, backed up to S3 but not disposable.

services/prod/spinup.sh

@@ -3,6 +3,15 @@
 set -e
 cd "$(dirname "$0")"
 
+if [ ! -f .env ]; then
+  echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
+  exit 1
+fi
+
+echo "Restoring named volumes from S3 backup if any are empty..."
+docker compose -f backup-docker-compose.yml pull restore
+docker compose -f backup-docker-compose.yml run --rm restore
+
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 
 sleep 20 # Allow Traefik + registry auth to settle before starting the rest of the services
@@ -12,3 +21,6 @@ docker compose -f status-docker-compose.yml pull && docker compose -f status-doc
 docker compose -f web-docker-compose.yml pull && docker compose -f web-docker-compose.yml up -d
 docker compose -f bitwarden-docker-compose.yml pull && docker compose -f bitwarden-docker-compose.yml up -d
 docker compose -f rd-docker-compose.yml pull && docker compose -f rd-docker-compose.yml up -d
+
+docker compose -f backup-docker-compose.yml pull backup
+docker compose -f backup-docker-compose.yml up -d backup

services/prod/status-docker-compose.yml

@@ -3,13 +3,16 @@ services:
     image: louislam/uptime-kuma:latest
     container_name: status
     volumes:
-      - ./uptime-kuma/data:/app/data
+      - uptime_kuma_data:/app/data
     networks:
       - proxy
     labels:
       ## Expose uptime-kuma Through Trefik ##
       - "traefik.enable=true" # <== Enable traefik to proxy this container
 
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
+
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure"
       - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)"
@@ -23,3 +26,7 @@ services:
 networks:
   proxy:
     external: true
+
+volumes:
+  uptime_kuma_data:
+    name: uptime_kuma_data

services/prod/traefik-docker-compose.yml

@@ -20,7 +20,7 @@ services:
       - "443:443"
       - "27017:27017"
     volumes:
-      - "./letsencrypt:/letsencrypt"
+      - "letsencrypt_data:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
     networks:
       - proxy
@@ -36,10 +36,12 @@ services:
       - "traefik.http.routers.traefik.entrypoints=websecure"
       - "traefik.http.routers.traefik.service=api@internal"
       - "traefik.http.routers.traefik.tls.certresolver=myresolver"
-      - "traefik.http.routers.traefik.middlewares=traefik-auth"
-      - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS"
     restart: unless-stopped
 
 networks:
   proxy:
     name: proxy
+
+volumes:
+  letsencrypt_data:
+    name: letsencrypt_data

services/prod/web-docker-compose.yml

@@ -1,6 +1,6 @@
 services:
 
-#Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080)
+  #Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080)
   luke-else:
     image: git.luke-else.co.uk/luke-else/luke-else.co.uk
     container_name: luke-else
@@ -58,46 +58,6 @@ services:
       - "traefik.http.routers.metarius.tls.certresolver=myresolver"
     restart: unless-stopped
 
-  divine-couture:
-    image: git.luke-else.co.uk/luke-else/divine-couture.co.uk:latest
-    container_name: divine-couture
-    networks:
-      - proxy
-    labels:
-      ## Expose divine-couture Through Trefik ##
-      - "traefik.enable=true" # <== Enable traefik to proxy this container
-      
-      - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
-      - "traefik.http.routers.divine-couture-insecure.middlewares=redirect-web-secure"
-      - "traefik.http.routers.divine-couture-insecure.rule=Host(`www.divine-couture.co.uk`)"
-      - "traefik.http.routers.divine-couture-insecure.entrypoints=web"
-
-      - "traefik.http.routers.divine-couture.rule=Host(`www.divine-couture.co.uk`)"
-      - "traefik.http.routers.divine-couture.entrypoints=websecure"
-      - "traefik.http.routers.divine-couture.tls.certresolver=myresolver"
-    restart: unless-stopped
-
-  snexo:
-    image: "php:apache"
-    container_name: snexo
-    volumes:
-      - ./snexo.co.uk/:/var/www/html
-    networks:
-      - proxy
-    labels:
-      ## Expose Snexo Through Trefik ##
-      - "traefik.enable=true" # <== Enable traefik to proxy this container
-
-      - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
-      - "traefik.http.routers.snexo-insecure.middlewares=redirect-web-secure"
-      - "traefik.http.routers.snexo-insecure.rule=Host(`snexo.co.uk`)"
-      - "traefik.http.routers.snexo-insecure.entrypoints=web"
-
-      - "traefik.http.routers.snexo.rule=Host(`snexo.co.uk`)"
-      - "traefik.http.routers.snexo.entrypoints=websecure"
-      - "traefik.http.routers.snexo.tls.certresolver=myresolver"
-    restart: unless-stopped
-
 networks:
   proxy:
     external: true

services/todo.md

@@ -1,32 +0,0 @@
-# ToDo Items
-
-## General
-
-- Setup non root user
-
-   - "21115:21115/tcp"
-   - "21116:21116/tcp"
-   - "21116:21116/udp"
-   - "21117:21117/tcp"
-   - "21119:21119/tcp"
-
-- Install docker, docker-compose and apache utils.
-
-## Traefik + TraefikRunner + Traefik VPN
-
-- Setup htaccess -> `echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g`
-- Ensure email address is correct
-
-## Gitea
-
-- Ensure that ports are assigned correctly for the system
-
-## Websites
-
-- Ensure website files are copied over
-- Ensure that ports are assigned correctly for the system
-
-## Bitwarden
-
-- Ensure that all data is fully encrypted during transfer.
-- Ensure that ports are assigned correctly for the system

services/vpn/backup-docker-compose.yml

@@ -0,0 +1,48 @@
+services:
+  # Nightly backup of every named volume on this host to S3. Containers
+  # labeled docker-volume-backup.stop-during-backup=true (see
+  # vpn-docker-compose.yml) are stopped for the duration of the backup so
+  # their data is captured in a consistent state, then restarted.
+  backup:
+    image: offen/docker-volume-backup:v2
+    container_name: backup
+    environment:
+      BACKUP_FILENAME: "vpn-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"
+      BACKUP_CRON_EXPRESSION: "0 3 * * *"
+      BACKUP_RETENTION_DAYS: "14"
+      BACKUP_PRUNING_LEEWAY: "5m"
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - wg_easy_data:/backup/wg_easy_data:ro
+      - letsencrypt_data:/backup/letsencrypt_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    restart: unless-stopped
+
+  # One-shot: restores empty volumes from the latest S3 backup, then exits.
+  # Run explicitly by spinup.sh before the services that own these volumes
+  # start - not started by `docker compose up`.
+  restore:
+    image: amazon/aws-cli:latest
+    container_name: backup-restore
+    entrypoint: ["/bin/bash", "/restore.sh"]
+    environment:
+      AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+      AWS_S3_PATH: ${AWS_S3_PATH}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ENDPOINT: ${AWS_ENDPOINT}
+    volumes:
+      - ./restore.sh:/restore.sh:ro
+      - wg_easy_data:/restore/wg_easy_data
+      - letsencrypt_data:/restore/letsencrypt_data
+    restart: "no"
+
+volumes:
+  wg_easy_data:
+    name: wg_easy_data
+  letsencrypt_data:
+    name: letsencrypt_data

services/vpn/restore.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Restores this host's named volumes from the latest S3 backup, but only into
+# volumes that are currently empty - never overwrites data that's already
+# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml
+# run --rm restore` before any service that owns one of the mounted volumes
+# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh
+# estate): it just leaves the empty volumes alone.
+set -euo pipefail
+
+S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}"
+ENDPOINT_ARGS=()
+if [ -n "${AWS_ENDPOINT:-}" ]; then
+  ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}")
+fi
+
+empty_volumes=()
+for dir in /restore/*/; do
+  name="$(basename "$dir")"
+  if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+    empty_volumes+=("$name")
+  fi
+done
+
+if [ "${#empty_volumes[@]}" -eq 0 ]; then
+  echo "All volumes already contain data - skipping restore."
+  exit 0
+fi
+
+echo "Empty volumes: ${empty_volumes[*]}"
+
+latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)"
+if [ -z "$latest" ]; then
+  echo "No existing backup found at ${S3_URI} - starting with empty volumes."
+  exit 0
+fi
+
+echo "Restoring ${S3_URI}/${latest} into empty volumes only..."
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+
+aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz"
+mkdir "$tmp/extracted"
+tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted"
+
+for name in "${empty_volumes[@]}"; do
+  if [ -d "$tmp/extracted/$name" ]; then
+    echo "Restoring $name..."
+    cp -a "$tmp/extracted/$name/." "/restore/$name/"
+  else
+    echo "No $name/ in backup - leaving volume empty."
+  fi
+done
+
+echo "Restore complete."

services/vpn/spindown.sh

@@ -5,7 +5,9 @@ cd "$(dirname "$0")"
 docker compose -f vpn-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
+docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-docker volume prune -f -a
+# Deliberately no `docker volume prune` - wg_easy_data/letsencrypt_data are
+# named volumes holding real data, backed up to S3 but not disposable.

services/vpn/spinup.sh

@@ -3,6 +3,18 @@
 set -e
 cd "$(dirname "$0")"
 
+if [ ! -f .env ]; then
+  echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
+  exit 1
+fi
+
+echo "Restoring named volumes from S3 backup if any are empty..."
+docker compose -f backup-docker-compose.yml pull restore
+docker compose -f backup-docker-compose.yml run --rm restore
+
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 docker compose -f vpn-docker-compose.yml pull && docker compose -f vpn-docker-compose.yml up -d
 docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d
+
+docker compose -f backup-docker-compose.yml pull backup
+docker compose -f backup-docker-compose.yml up -d backup

services/vpn/traefik-docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - "80:80"
       - "443:443"
     volumes:
-      - "./letsencrypt:/letsencrypt"
+      - "letsencrypt_data:/letsencrypt"
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
     networks:
       - proxy
@@ -31,10 +31,12 @@ services:
       - "traefik.http.routers.traefik.entrypoints=websecure"
       - "traefik.http.routers.traefik.service=api@internal"
       - "traefik.http.routers.traefik.tls.certresolver=myresolver"
-      - "traefik.http.routers.traefik.middlewares=traefik-auth"
-      - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS"
     restart: unless-stopped
 
 networks:
   proxy:
     name: proxy
+
+volumes:
+  letsencrypt_data:
+    name: letsencrypt_data

services/vpn/vpn-docker-compose.yml

@@ -1,17 +1,28 @@
 services:
-  dockovpn:
+  wg-easy:
-    image: alekslitvinenk/openvpn
+    image: ghcr.io/wg-easy/wg-easy:15
     cap_add:
-        - NET_ADMIN
+      - NET_ADMIN
+      - SYS_MODULE # lets the container modprobe wireguard if the host kernel doesn't already have it loaded
+    sysctls:
+      - net.ipv4.ip_forward=1
+      - net.ipv4.conf.all.src_valid_mark=1
+      - net.ipv6.conf.all.disable_ipv6=0
+      - net.ipv6.conf.all.forwarding=1
+      - net.ipv6.conf.default.forwarding=1
     ports:
-        - 1194:1194/udp # Expose tcp if you defined HOST_TUN_PROTOCOL=tcp
+      - 51820:51820/udp # WireGuard tunnel - the web UI (51821) only needs to be reachable via Traefik below, not published directly
-    environment:
+    networks:
-        HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address 
+      - proxy
     volumes:
-        - ./openvpn_conf:/opt/Dockovpn_data
+      - /lib/modules:/lib/modules:ro
+      - wg_easy_data:/etc/wireguard
     labels:
-      ## Expose vpn Through Trefik ##
+      ## Expose the wg-easy web UI through Traefik ##
-      - "traefik.enable=true" # <== Enable traefik to proxy this container
+      - "traefik.enable=true"
+
+      # Stopped for the duration of each backup run - see backup-docker-compose.yml.
+      - "docker-volume-backup.stop-during-backup=true"
 
       - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
       - "traefik.http.routers.vpn-insecure.middlewares=redirect-web-secure"
@@ -21,4 +32,13 @@ services:
       - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)"
       - "traefik.http.routers.vpn.entrypoints=websecure"
       - "traefik.http.routers.vpn.tls.certresolver=myresolver"
-    restart: always
+      - "traefik.http.services.vpn.loadbalancer.server.port=51821"
+    restart: unless-stopped
+
+networks:
+  proxy:
+    external: true
+
+volumes:
+  wg_easy_data:
+    name: wg_easy_data