Merge pull request 'Feat/open tofu' (#8) from feat/open-tofu into main

Reviewed-on: #8

.gitignore

@@ -13,6 +13,10 @@
 # services/docker-compose.yml carve-out above.
 **/.env
 
+# Variables set through control.sh's "Set variables" menu (HCLOUD_TOKEN,
+# BACKUP_S3_*, etc.) - never committed.
+.control.env
+
 ### Ansible ###
 *.retry
 

ansible/README.md

@@ -14,16 +14,19 @@ to S3, not on a Hetzner volume - see `../readme.md`.
 - `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API
   directly (not Terraform state) and groups servers by their `role` label
   (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules/<host>/main.tf`).
-- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path,
-  `backup_s3_*` S3 backup credentials), `role_dev.yml` / `role_prod.yml` /
-  `role_vpn.yml` (per-role vars, e.g. `dev_runner_count`).
 - `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening,
   unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`.
 - `roles/deploy/` - copies `services/<host>/` to the server and renders `.env`
   (S3 backup credentials, see `../readme.md`) and, for `dev`,
   `Runners/docker-compose.yml`.
 - `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`,
-  and `site.yml` (all three in order).
+  `site.yml` (all three in order), and `group_vars/` (`all.yml` - shared:
+  `deploy_user`, SSH key path, `backup_s3_*` S3 backup credentials;
+  `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` - per-role vars, e.g.
+  `service_group`). `group_vars/` lives here rather than at the `ansible/`
+  root because Ansible only auto-loads `group_vars/`/`host_vars/` from
+  directories adjacent to the playbook file or the inventory source - not
+  from an arbitrary sibling directory.
 
 ## Prerequisites
 
@@ -37,7 +40,13 @@ export BACKUP_S3_SECRET_ACCESS_KEY=your-secret-access-key
 export BACKUP_S3_ENDPOINT=your-s3-endpoint   # omit for real AWS S3
 ```
 
-`ansible_ssh_private_key_file` (see `group_vars/all.yml`) defaults to
+`../control.sh`'s **Set variables** menu does the same thing interactively and
+persists your answers to `../.control.env` (gitignored) so you don't have to
+re-export them every session.
+
+`deploy_user` (see `playbooks/group_vars/all.yml`) defaults to `deploy` - override with
+the `DEPLOY_USER` env var if you want a different non-root user created on each
+host. `ansible_ssh_private_key_file` (same file) defaults to
 `~/.ssh/id_ed25519` - override with `ANSIBLE_SSH_PRIVATE_KEY_FILE` if you use a
 different key. It must match one of the SSH keys OpenTofu installed on the
 servers (`var.ssh_key_names`).
@@ -55,7 +64,8 @@ ansible-playbook playbooks/bootstrap.yml -l role_vpn
 ansible-playbook playbooks/deploy.yml    -l role_vpn
 ansible-playbook playbooks/spinup.yml    -l role_vpn
 
-# 2. connect to the VPN you just started with an OpenVPN client - now dev/prod
+# 2. visit https://vpn.luke-else.co.uk, complete wg-easy's first-run setup,
+#    add a client peer, and connect with a WireGuard client - now dev/prod
 #    are reachable, since your egress IP matches their firewall rule
 ansible-playbook playbooks/bootstrap.yml -l role_dev,role_prod
 ansible-playbook playbooks/deploy.yml    -l role_dev,role_prod
@@ -65,9 +75,10 @@ ansible-playbook playbooks/spinup.yml    -l role_dev,role_prod
 ansible-playbook playbooks/site.yml -l role_dev,role_prod
 ```
 
-Re-run `deploy.yml` + `spinup.yml` any time a compose file changes or
-`dev_runner_count` in `group_vars/role_dev.yml` is bumped - `spinup.sh` only
-starts/updates what's changed, same as before.
+Re-run `deploy.yml` + `spinup.yml` any time a compose file changes - `spinup.sh`
+only starts/updates what's changed, same as before. `dev` always runs three
+Gitea Actions runners (fixed in
+`roles/deploy/templates/runners-docker-compose.yml.j2`).
 
 `spindown.yml` runs `spindown.sh`, which does a full `docker system/volume
 prune -a` on the host - it prompts for a `yes` confirmation before running,

ansible/group_vars/role_dev.yml

@@ -1,7 +0,0 @@
----
-service_group: dev
-
-# Number of Gitea Actions runner containers rendered into
-# services/dev/Runners/docker-compose.yml by the deploy role - see
-# roles/deploy/templates/runners-docker-compose.yml.j2.
-dev_runner_count: 3

ansible/inventory/hcloud.yml

@@ -1,8 +1,7 @@
 ---
 # Dynamic inventory: queries the Hetzner Cloud API directly (HCLOUD_TOKEN),
 # grouping hosts by the `role` label set on each hcloud_server resource in
-# infra/modules/<host>/main.tf (role = dev/prod/vpn). Nothing here reads
-# Terraform state - this always reflects whatever actually exists in Hetzner.
+# infra/modules/<host>/main.tf (role = dev/prod/vpn).
 plugin: hetzner.hcloud.hcloud
 token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
 connect_with: public_ipv4

ansible/playbooks/group_vars/all.yml

@@ -1,8 +1,5 @@
 ---
-# Keep in sync with infra/variables.tf's var.deploy_user (default "deploy") -
-# the two aren't wired together automatically, since Ansible no longer reads
-# any Terraform state or output.
-deploy_user: deploy
+deploy_user: "{{ lookup('env', 'DEPLOY_USER') | default('deploy', true) }}"
 ansible_user: "{{ deploy_user }}"
 ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}"
 
@@ -10,11 +7,7 @@ ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE')
 services_root: "{{ playbook_dir }}/../../services"
 
 # S3-compatible bucket that services/<host>/backup-docker-compose.yml backs up
-# to and restores from - see readme.md. Not provisioned by OpenTofu (the
-# hcloud provider has no Object Storage resource), so these must point at a
-# bucket you've created yourself. AWS_ENDPOINT is only needed for a
-# non-AWS S3-compatible provider (e.g. Hetzner Object Storage); leave it unset
-# to use real AWS S3.
+# to and restores from
 backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}"
 backup_s3_access_key_id: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY_ID') }}"
 backup_s3_secret_access_key: "{{ lookup('env', 'BACKUP_S3_SECRET_ACCESS_KEY') }}"

ansible/playbooks/group_vars/role_dev.yml

@@ -0,0 +1,2 @@
+---
+service_group: dev

ansible/roles/bootstrap/tasks/main.yml

@@ -1,7 +1,5 @@
 ---
-# Post-install bootstrap - replaces infra/scripts/bootstrap.sh.tftpl. Runs as
-# root (see ansible_user override in playbooks/bootstrap.yml) since the deploy
-# user doesn't exist yet on a freshly created server. Idempotent: safe to
+# Post-install bootstrap - Idempotent: safe to
 # re-run against an already-bootstrapped host.
 
 - name: Update apt cache and upgrade packages
@@ -10,50 +8,18 @@
     upgrade: dist
     cache_valid_time: 3600
 
-- name: Install Docker prerequisites
+- name: Install Docker Engine and the Compose plugin
   ansible.builtin.apt:
     name:
-      - ca-certificates
-      - curl
-      - gnupg
+      - docker.io
+      - docker-compose-v2
     state: present
 
-- name: Create apt keyrings directory
-  ansible.builtin.file:
-    path: /etc/apt/keyrings
-    state: directory
-    mode: "0755"
-
-- name: Download Docker's GPG key
-  ansible.builtin.get_url:
-    url: https://download.docker.com/linux/ubuntu/gpg
-    dest: /etc/apt/keyrings/docker.asc
-    mode: "0644"
-    force: false
-
-- name: Determine dpkg architecture
-  ansible.builtin.command: dpkg --print-architecture
-  register: _dpkg_arch
-  changed_when: false
-
-- name: Add Docker apt repository
-  ansible.builtin.apt_repository:
-    repo: >-
-      deb [arch={{ _dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc]
-      https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable
-    filename: docker
-    state: present
-
-- name: Install Docker Engine and Compose plugin
-  ansible.builtin.apt:
-    update_cache: true
-    name:
-      - docker-ce
-      - docker-ce-cli
-      - containerd.io
-      - docker-buildx-plugin
-      - docker-compose-plugin
-    state: present
+- name: Ensure Docker is enabled and running
+  ansible.builtin.systemd:
+    name: docker
+    enabled: true
+    state: started
 
 - name: Create non-root sudo user, given the same key root was provisioned with
   ansible.builtin.user:

ansible/roles/deploy/tasks/main.yml

@@ -1,10 +1,8 @@
 ---
 # Copies services/<service_group>/ to the server and renders the files that
-# used to be generated by OpenTofu (services/<host>/.env and, for dev,
-# Runners/docker-compose.yml) - replaces the null_resource.deploy_services +
-# local_file provisioners previously in infra/modules/<host>/main.tf.
+# need templating (e.g. .env, docker-compose.yml).
 #
-# Persistent data lives in named Docker volumes now, not on a Hetzner volume -
+# Persistent data lives in named Docker volumes
 # see services/<host>/backup-docker-compose.yml. .env only carries the S3
 # credentials that backup/restore need, rendered for every host.
 

ansible/roles/deploy/templates/runners-docker-compose.yml.j2

@@ -1,7 +1,6 @@
-# Generated by Ansible from dev_runner_count - do not hand-edit.
-# To change the number of runners, edit dev_runner_count in
-# ansible/group_vars/role_dev.yml and re-run playbooks/deploy.yml.
-# To change their shape, edit this template instead.
+# Generated by Ansible - do not hand-edit. Three Gitea Actions runners, always.
+# To change their shape (image, volumes, env), edit this template and re-run
+# playbooks/deploy.yml.
 #
 # GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable
 # (not baked in here) - services/dev/spinup.sh generates a fresh token and
@@ -10,8 +9,9 @@
 # /data is a disposable local cache (runner identity re-registers on every
 # spinup.sh run anyway), so it's a plain named volume - not part of the S3
 # backup/restore stack in services/dev/backup-docker-compose.yml.
+{% set runner_count = 3 %}
 services:
-{% for i in range(1, dev_runner_count + 1) %}
+{% for i in range(1, runner_count + 1) %}
   runner-{{ i }}:
     image: gitea/act_runner:latest
     container_name: gitea_runner_{{ i }}
@@ -34,7 +34,7 @@ networks:
     external: true
 
 volumes:
-{% for i in range(1, dev_runner_count + 1) %}
+{% for i in range(1, runner_count + 1) %}
   gitea_runner_{{ i }}_data:
     name: gitea_runner_{{ i }}_data
 {% endfor %}

control.sh

@@ -0,0 +1,321 @@
+#!/usr/bin/env bash
+# control.sh - interactive control panel for this repo.
+#
+# Steps you through the pre-setup and day-to-day operations for both the
+# OpenTofu infra (infra/) and the Ansible config (ansible/), in the order the
+# firewall rules require (vpn first, then tunnel in, then dev/prod). It only
+# ever runs `tofu`/`ansible-playbook` for you - it's a menu wrapper, nothing
+# more, so anything it does you can also do by hand from infra/ or ansible/.
+
+# Re-exec under bash if invoked as `sh control.sh` (or another POSIX shell) -
+# this script relies on bash-only features (indirect expansion, arrays, [[ ]])
+# that fail with cryptic errors like "bad substitution" under dash/sh.
+if [ -z "${BASH_VERSION:-}" ]; then
+  exec bash "$0" "$@"
+fi
+
+set -uo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INFRA_DIR="$REPO_ROOT/infra"
+ANSIBLE_DIR="$REPO_ROOT/ansible"
+CONTROL_ENV="$REPO_ROOT/.control.env"
+
+# --- pretty output --------------------------------------------------------
+if [ -t 1 ]; then
+  BOLD=$(tput bold); DIM=$(tput dim); RESET=$(tput sgr0)
+  RED=$(tput setaf 1); GREEN=$(tput setaf 2); YELLOW=$(tput setaf 3); CYAN=$(tput setaf 6)
+else
+  BOLD=""; DIM=""; RESET=""; RED=""; GREEN=""; YELLOW=""; CYAN=""
+fi
+
+ok()    { printf "%s  ok %s %s\n"   "$GREEN" "$RESET" "$1"; }
+warn()  { printf "%s warn%s %s\n"   "$YELLOW" "$RESET" "$1"; }
+bad()   { printf "%s miss%s %s\n"   "$RED" "$RESET" "$1"; }
+title() { printf "\n%s%s%s\n" "$BOLD$CYAN" "$1" "$RESET"; }
+
+# Pick the OpenTofu binary: prefer `tofu`, fall back to `terraform`.
+TOFU=""
+detect_tofu() {
+  if command -v tofu >/dev/null 2>&1; then TOFU="tofu"
+  elif command -v terraform >/dev/null 2>&1; then TOFU="terraform"
+  fi
+}
+detect_tofu
+
+# --- variable persistence --------------------------------------------------
+# Variables set via the "Set variables" menu (HCLOUD_TOKEN, BACKUP_S3_*,
+# DEPLOY_USER, ANSIBLE_SSH_PRIVATE_KEY_FILE) live in .control.env, gitignored,
+# so they survive between runs of this script. They are NOT exported into your
+# normal shell - only into control.sh's own process and whatever it runs.
+load_control_env() {
+  if [ -f "$CONTROL_ENV" ]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$CONTROL_ENV"
+    set +a
+  fi
+}
+load_control_env
+
+# Upsert KEY=VALUE into .control.env and export it into this process.
+save_var() {
+  local key="$1" val="$2"
+  touch "$CONTROL_ENV"; chmod 600 "$CONTROL_ENV"
+  local tmp; tmp="$(mktemp)"
+  grep -v "^${key}=" "$CONTROL_ENV" > "$tmp" 2>/dev/null || true
+  mv "$tmp" "$CONTROL_ENV"
+  printf '%s=%q\n' "$key" "$val" >> "$CONTROL_ENV"
+  export "${key?}=${val}"
+}
+
+# Run a command inside a directory, echoing it first. Returns the command's exit code.
+run_in() {
+  local dir="$1"; shift
+  printf "%s\$ (cd %s && %s)%s\n" "$DIM" "${dir#$REPO_ROOT/}" "$*" "$RESET"
+  ( cd "$dir" && "$@" )
+}
+
+pause() { read -rp $'\nPress Enter to return to the menu... ' _; }
+
+confirm() {
+  local reply
+  read -rp "$1 [y/N] " reply
+  [[ "$reply" == "y" || "$reply" == "Y" ]]
+}
+
+# --- prerequisite checks --------------------------------------------------
+check_prereqs() {
+  title "Tools"
+  if [ -n "$TOFU" ]; then ok "OpenTofu ($TOFU: $("$TOFU" version 2>/dev/null | head -n1))"
+  else bad "Neither 'tofu' nor 'terraform' found on PATH"; fi
+  if command -v ansible-playbook >/dev/null 2>&1; then ok "Ansible ($(ansible --version 2>/dev/null | head -n1))"
+  else bad "'ansible-playbook' not found on PATH"; fi
+
+  title "Environment variables"
+  [ -n "${HCLOUD_TOKEN:-}" ]                 && ok "HCLOUD_TOKEN set"                 || bad "HCLOUD_TOKEN not set (needed by both tofu and the ansible inventory)"
+  [ -n "${BACKUP_S3_BUCKET:-}" ]             && ok "BACKUP_S3_BUCKET set"             || warn "BACKUP_S3_BUCKET not set (needed for deploy/spinup - S3 backup .env)"
+  [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ]      && ok "BACKUP_S3_ACCESS_KEY_ID set"      || warn "BACKUP_S3_ACCESS_KEY_ID not set"
+  [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ]  && ok "BACKUP_S3_SECRET_ACCESS_KEY set"  || warn "BACKUP_S3_SECRET_ACCESS_KEY not set"
+  [ -n "${BACKUP_S3_ENDPOINT:-}" ]           && ok "BACKUP_S3_ENDPOINT set"           || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)"
+  ok "DEPLOY_USER: ${DEPLOY_USER:-deploy} (default 'deploy' if unset)"
+
+  local key="${ANSIBLE_SSH_PRIVATE_KEY_FILE:-$HOME/.ssh/id_ed25519}"
+  [ -f "${key/#\~/$HOME}" ] && ok "SSH key present ($key)" || warn "SSH key not found at $key (set ANSIBLE_SSH_PRIVATE_KEY_FILE)"
+
+  [ -f "$CONTROL_ENV" ] && ok ".control.env present (loaded above)" || warn "No .control.env yet - use Configuration > Set variables to persist the above across runs"
+
+  title "Files"
+  [ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \
+    || bad "infra/terraform.tfvars missing (Configuration > Copy terraform.tfvars.example)"
+  [ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \
+    || warn "infra/.terraform missing - run OpenTofu > init"
+  if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed"
+  else warn "hetzner.hcloud collection not installed - run Ansible > Install collections"; fi
+}
+
+# --- configuration actions -------------------------------------------------
+# Prompts for one variable, keeping the current value if the user hits Enter.
+# $3 = true masks input and the shown current value, for tokens/secrets.
+prompt_var() {
+  local key="$1" desc="$2" secret="${3:-false}" current input shown
+  current="${!key:-}"
+  shown="$current"
+  [ "$secret" = "true" ] && [ -n "$current" ] && shown="(set, hidden)"
+  if [ "$secret" = "true" ]; then read -rsp "$desc [$shown]: " input; echo
+  else read -rp "$desc [$shown]: " input; fi
+  [ -n "$input" ] && save_var "$key" "$input"
+}
+
+set_variables() {
+  title "Set variables"
+  echo "Leave a prompt blank to keep its current value. Saved to .control.env" \
+       "(gitignored) and reloaded automatically next time you run control.sh -" \
+       "not exported to your normal shell outside of it."
+  echo
+  prompt_var HCLOUD_TOKEN                 "Hetzner Cloud API token" true
+  prompt_var DEPLOY_USER                  "Non-root deploy user Ansible creates (default: deploy)" false
+  prompt_var ANSIBLE_SSH_PRIVATE_KEY_FILE "SSH private key path (default: ~/.ssh/id_ed25519)" false
+  prompt_var BACKUP_S3_BUCKET             "S3 backup bucket name" false
+  prompt_var BACKUP_S3_ACCESS_KEY_ID      "S3 backup access key ID" true
+  prompt_var BACKUP_S3_SECRET_ACCESS_KEY  "S3 backup secret access key" true
+  prompt_var BACKUP_S3_ENDPOINT           "S3 backup endpoint (blank for real AWS S3)" false
+  ok "Variables saved."
+}
+
+copy_tfvars() {
+  local example="$INFRA_DIR/terraform.tfvars.example"
+  local target="$INFRA_DIR/terraform.tfvars"
+  if [ -f "$target" ]; then
+    warn "infra/terraform.tfvars already exists."
+    confirm "Overwrite it with a fresh copy of the example?" || return 0
+  fi
+  cp "$example" "$target"
+  ok "Copied terraform.tfvars.example -> terraform.tfvars"
+
+  local keys quoted k
+  read -rp "SSH key name(s) from Hetzner Console > Security > SSH Keys, comma-separated (blank = edit manually later): " keys
+  if [ -n "$keys" ]; then
+    quoted=""
+    IFS=',' read -ra _key_arr <<< "$keys"
+    for k in "${_key_arr[@]}"; do
+      k="$(echo "$k" | xargs)"
+      quoted+="\"$k\", "
+    done
+    quoted="${quoted%, }"
+    sed -i "s|^ssh_key_names = .*|ssh_key_names = [${quoted}]|" "$target"
+    ok "Set ssh_key_names = [${quoted}]"
+  else
+    warn "ssh_key_names still has the example's placeholder values - edit infra/terraform.tfvars before running tofu."
+  fi
+}
+
+# --- OpenTofu actions -----------------------------------------------------
+require_tofu() {
+  if [ -z "$TOFU" ]; then bad "No tofu/terraform binary found."; return 1; fi
+  if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then
+    bad "infra/terraform.tfvars is missing - copy terraform.tfvars.example first."; return 1
+  fi
+}
+tofu_init()    { require_tofu && run_in "$INFRA_DIR" "$TOFU" init; }
+tofu_plan()    { require_tofu && run_in "$INFRA_DIR" "$TOFU" plan; }
+tofu_apply()   { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply; }
+tofu_output()  { require_tofu && run_in "$INFRA_DIR" "$TOFU" output; }
+tofu_destroy() {
+  require_tofu || return 1
+  confirm "${RED}Destroy ALL infra managed by OpenTofu?${RESET}" && run_in "$INFRA_DIR" "$TOFU" destroy
+}
+
+# --- Ansible actions ------------------------------------------------------
+ansible_collections() { run_in "$ANSIBLE_DIR" ansible-galaxy collection install -r requirements.yml; }
+ansible_inventory()   { run_in "$ANSIBLE_DIR" ansible-inventory --graph; }
+
+# $1 = playbook, $2 = -l limit expression
+play() { run_in "$ANSIBLE_DIR" ansible-playbook "playbooks/$1" -l "$2"; }
+
+ansible_bootstrap_vpn()   { play bootstrap.yml role_vpn; }
+ansible_deploy_vpn()      { play deploy.yml role_vpn; }
+ansible_spinup_vpn()      { play spinup.yml role_vpn; }
+ansible_bootstrap_dev()   { play bootstrap.yml role_dev; }
+ansible_deploy_dev()      { play deploy.yml role_dev; }
+ansible_spinup_dev()      { play spinup.yml role_dev; }
+ansible_bootstrap_prod()  { play bootstrap.yml role_prod; }
+ansible_deploy_prod()     { play deploy.yml role_prod; }
+ansible_spinup_prod()     { play spinup.yml role_prod; }
+
+ansible_spindown() {
+  local target
+  read -rp "Spindown which hosts? (e.g. role_vpn, role_dev,role_prod, all): " target
+  [ -n "$target" ] && play spindown.yml "$target"
+}
+
+# --- guided walkthrough ---------------------------------------------------
+guided() {
+  title "Guided full setup"
+  cat <<EOF
+This walks the whole bring-up in the order the firewalls require. dev/prod only
+accept SSH from the vpn server's public IP, so vpn must exist and you must be
+connected to it (WireGuard, via wg-easy) before dev/prod are reachable.
+
+Steps, in order:
+  0. Configuration: variables + terraform.tfvars (skip if already done)
+  1. OpenTofu: init -> plan -> apply   (creates servers, network, firewalls, DNS)
+  2. Ansible:  bootstrap + deploy + spinup for role_vpn
+  3. YOU:      visit https://vpn.luke-else.co.uk, complete wg-easy's setup,
+               add a client peer, and connect with a WireGuard client
+  4. Ansible:  bootstrap + deploy + spinup for role_dev, then role_prod
+EOF
+  confirm "Start the guided walkthrough?" || return 0
+
+  echo; title "0/4  Configuration"
+  if confirm "Set/update variables now (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)?"; then
+    set_variables
+  fi
+  if [ ! -f "$INFRA_DIR/terraform.tfvars" ] && confirm "infra/terraform.tfvars is missing - create it from the example now?"; then
+    copy_tfvars
+  fi
+
+  echo; title "1/4  OpenTofu"
+  if confirm "Run $TOFU init/plan/apply now?"; then
+    tofu_init && tofu_plan && tofu_apply || { bad "OpenTofu step failed - stopping."; return 1; }
+  fi
+
+  echo; title "2/4  Ansible - vpn"
+  if confirm "Bootstrap + deploy + spinup role_vpn now?"; then
+    ansible_bootstrap_vpn && ansible_deploy_vpn && ansible_spinup_vpn || { bad "vpn step failed - stopping."; return 1; }
+  fi
+
+  echo; title "3/4  Connect the VPN"
+  warn "Visit https://vpn.luke-else.co.uk, complete wg-easy's setup wizard, add a client peer, and connect with a WireGuard client now."
+  confirm "Are you connected to the VPN?" || { warn "Stopping - reconnect and re-run this step."; return 0; }
+
+  echo; title "4/4  Ansible - dev + prod"
+  if confirm "Bootstrap + deploy + spinup role_dev now?"; then
+    ansible_bootstrap_dev && ansible_deploy_dev && ansible_spinup_dev || { bad "dev step failed."; return 1; }
+  fi
+  if confirm "Bootstrap + deploy + spinup role_prod now?"; then
+    ansible_bootstrap_prod && ansible_deploy_prod && ansible_spinup_prod || { bad "prod step failed."; return 1; }
+  fi
+  ok "Guided setup complete."
+}
+
+# --- menu -----------------------------------------------------------------
+menu() {
+  clear
+  printf "%s==== server control panel ====%s\n" "$BOLD$CYAN" "$RESET"
+  printf "%srepo: %s   tofu: %s%s\n" "$DIM" "$REPO_ROOT" "${TOFU:-none}" "$RESET"
+
+  title "Setup"
+  echo "  1) Check prerequisites"
+  echo "  2) Guided full setup (vpn -> connect -> dev/prod)"
+
+  title "Configuration"
+  echo "  3) Set variables (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)"
+  echo "  4) Copy terraform.tfvars.example -> terraform.tfvars"
+
+  title "OpenTofu (infra/)"
+  echo "  5) init        6) plan        7) apply"
+  echo "  8) output      9) destroy"
+
+  title "Ansible (ansible/)"
+  echo " 10) Install collections        11) Show inventory"
+  echo " 12) Bootstrap vpn              13) Deploy vpn          14) Spinup vpn"
+  echo " 15) Bootstrap dev               16) Deploy dev          17) Spinup dev"
+  echo " 18) Bootstrap prod             19) Deploy prod         20) Spinup prod"
+  echo " 21) Spindown (choose target)"
+
+  echo
+  echo "  0) Quit"
+  echo
+  read -rp "Select an option: " choice
+}
+
+while true; do
+  menu
+  case "$choice" in
+    1)  check_prereqs; pause ;;
+    2)  guided; pause ;;
+    3)  set_variables; pause ;;
+    4)  copy_tfvars; pause ;;
+    5)  tofu_init; pause ;;
+    6)  tofu_plan; pause ;;
+    7)  tofu_apply; pause ;;
+    8)  tofu_output; pause ;;
+    9)  tofu_destroy; pause ;;
+    10) ansible_collections; pause ;;
+    11) ansible_inventory; pause ;;
+    12) ansible_bootstrap_vpn; pause ;;
+    13) ansible_deploy_vpn; pause ;;
+    14) ansible_spinup_vpn; pause ;;
+    15) ansible_bootstrap_dev; pause ;;
+    16) ansible_deploy_dev; pause ;;
+    17) ansible_spinup_dev; pause ;;
+    18) ansible_bootstrap_prod; pause ;;
+    19) ansible_deploy_prod; pause ;;
+    20) ansible_spinup_prod; pause ;;
+    21) ansible_spindown; pause ;;
+    0)  echo "Bye."; exit 0 ;;
+    "") ;;
+    *)  warn "Unknown option: $choice"; pause ;;
+  esac
+done

infra/main.tf

@@ -31,6 +31,14 @@ locals {
     { zone = "luke-else.co.uk", name = "vpn", value = module.vpn.ipv4 },
     { zone = "luke-else.co.uk", name = "traefik.vpn", value = module.vpn.ipv4 },
   ]
+
+  # Microsoft 365 mail records for luke-else.co.uk
+  mail_records = [
+    { zone = "luke-else.co.uk", name = "@", type = "MX", value = "0 lukeelse-co-uk01b.mail.protection.outlook.com." },
+    { zone = "luke-else.co.uk", name = "@", type = "TXT", value = "\"v=spf1 include:spf.protection.outlook.com -all\"" },
+    { zone = "luke-else.co.uk", name = "@", type = "TXT", value = "\"google-site-verification=aKLk5BATowKUwNbFcU5zyfu-AIyCW9W9PBStePRjH3U\"" },
+    { zone = "luke-else.co.uk", name = "autodiscover", type = "CNAME", value = "autodiscover.outlook.com." },
+  ]
 }
 
 module "network" {
@@ -87,5 +95,5 @@ module "dns" {
   source = "./modules/dns"
 
   zone_names = var.dns_zones
-  records    = local.dns_records
+  records    = concat(local.dns_records, local.mail_records)
 }

infra/modules/dev/main.tf

@@ -1,26 +1,24 @@
 # dev: Gitea + Runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk)
 
-# Port sources: services/dev/*docker-compose.yml (published ports) and services/todo.md
-# (the documented UFW allow-list). dev has no firewall in docs/architecture.md, but we
-# add one anyway for baseline safety - see conversation history.
+# Port sources: services/dev/*docker-compose.yml (published ports)
 resource "hcloud_firewall" "this" {
   name = "dev-firewall"
 
-  rule { # server ssh - wired to the vpn server's public IP only, see infra/main.tf
+  rule { # server ssh - wired to the vpn server's public IP only
     direction  = "in"
     protocol   = "tcp"
     port       = "22"
     source_ips = var.allowed_ssh_source_ips
   }
 
-  rule { # gitea ssh (git.luke-else.co.uk, published as 222:22)
+  rule { # gitea ssh
     direction  = "in"
     protocol   = "tcp"
     port       = "222"
     source_ips = ["0.0.0.0/0", "::/0"]
   }
 
-  rule { # Traefik http/https (git.luke-else.co.uk, cicd.luke-else.co.uk)
+  rule { # Traefik http/https
     direction  = "in"
     protocol   = "tcp"
     port       = "80"
@@ -56,7 +54,7 @@ resource "hcloud_server" "this" {
   location     = var.location
   ssh_keys     = var.ssh_key_ids
   firewall_ids = [hcloud_firewall.this.id]
-  labels       = { role = "dev" } # picked up by ansible/inventory/hcloud.yml
+  labels       = { role = "dev" }
 
   network {
     network_id = var.network_id

infra/modules/dns/main.tf

@@ -5,24 +5,33 @@
 resource "hcloud_zone" "this" {
   for_each = toset(var.zone_names)
 
-  name              = each.value
-  mode              = "primary"
-  delete_protection = true # API-level guard, in addition to prevent_destroy below
+  name = each.value
+  mode = "primary"
+}
 
-  lifecycle {
-    prevent_destroy = true
+locals {
+  # Group flat var.records entries by zone/name/type into one set of values
+  # each, since a Hetzner RRSet is keyed on (zone, name, type) and can hold
+  # several values (e.g. an SPF and a google-site-verification TXT both live
+  # under the same "@" TXT RRSet).
+  rrsets = {
+    for key, items in { for r in var.records : "${r.zone}/${r.name}/${r.type}" => r... } : key => {
+      zone   = items[0].zone
+      name   = items[0].name
+      type   = items[0].type
+      ttl    = items[0].ttl
+      values = [for i in items : i.value]
+    }
   }
 }
 
-resource "hcloud_zone_rrset" "a" {
-  for_each = { for r in var.records : "${r.zone}/${r.name}" => r }
+resource "hcloud_zone_rrset" "this" {
+  for_each = local.rrsets
 
   zone = hcloud_zone.this[each.value.zone].id
   name = each.value.name
-  type = "A"
+  type = each.value.type
   ttl  = each.value.ttl
 
-  records = [
-    { value = each.value.value }
-  ]
+  records = [for v in each.value.values : { value = v }]
 }

infra/modules/dns/moved.tf

@@ -0,0 +1,60 @@
+# Generalizing hcloud_zone_rrset.a (for_each keyed "zone/name", A records
+# only) into hcloud_zone_rrset.this (keyed "zone/name/type", any record type)
+# changed every existing instance's resource address. Without these, `tofu
+# apply` would destroy and recreate every A record already in state instead of
+# just adding the new mail records - these moved blocks make it a no-op rename
+# for anything that already existed. Safe to delete once applied and merged.
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/@"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/@/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/dev"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/dev/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/metarius"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/metarius/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/traefik"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/traefik/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/status"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/status/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/bitwarden"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/bitwarden/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/rd"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/rd/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/git"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/git/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/traefik.cicd"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/traefik.cicd/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/vpn"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/vpn/A"]
+}
+
+moved {
+  from = hcloud_zone_rrset.a["luke-else.co.uk/traefik.vpn"]
+  to   = hcloud_zone_rrset.this["luke-else.co.uk/traefik.vpn/A"]
+}

infra/modules/dns/variables.tf

@@ -4,11 +4,20 @@ variable "zone_names" {
 }
 
 variable "records" {
-  description = "A records to create across the zones in var.zone_names. Use name = \"@\" for the zone apex."
+  description = <<-EOT
+    Records to create across the zones in var.zone_names. Use name = "@" for
+    the zone apex. Entries sharing the same zone/name/type become a single
+    Hetzner RRSet with multiple values (e.g. two TXT strings under the same
+    name) - Hetzner has no concept of separate same-type records otherwise.
+    For MX, embed the priority in value using standard zonefile syntax (e.g.
+    "10 mail.example.com."); Hetzner's API has no separate priority field.
+    For TXT, wrap value in escaped double quotes (e.g. "\"v=spf1 ...\"").
+  EOT
   type = list(object({
     zone  = string
     name  = string
     value = string
+    type  = optional(string, "A")
     ttl   = optional(number, 300)
   }))
   default = []

infra/modules/network/main.tf

@@ -1,5 +1,5 @@
-# Private network shared by dev and prod (see docs/architecture.md "network" group).
-# vpn is intentionally not attached to this network - it sits outside it in the diagram.
+# Private network shared by dev and prod (see readme.md "network" group).
+# vpn is intentionally not attached to this network
 resource "hcloud_network" "main" {
   name     = "server-network"
   ip_range = var.ip_range

infra/modules/prod/main.tf

@@ -1,18 +1,15 @@
-# prod: Traefik, Websites, Bitwarden, RustDesk, status page
-
-# Port sources: services/prod/*docker-compose.yml (published ports) and services/todo.md
-# (the documented UFW allow-list).
+# Port sources: services/prod/*docker-compose.yml (published ports)
 resource "hcloud_firewall" "this" {
   name = "prod-firewall"
 
-  rule { # server ssh - wired to the vpn server's public IP only, see infra/main.tf
+  rule { # server ssh - wired to the vpn server's public IP only
     direction  = "in"
     protocol   = "tcp"
     port       = "22"
     source_ips = var.allowed_ssh_source_ips
   }
 
-  rule { # Traefik http/https (Websites, Bitwarden, status page)
+  rule { # Traefik http/https
     direction  = "in"
     protocol   = "tcp"
     port       = "80"
@@ -61,7 +58,7 @@ resource "hcloud_firewall" "this" {
     source_ips = ["0.0.0.0/0", "::/0"]
   }
 
-  rule { # traffic from dev over the private network
+  rule { # traffic from over the private network
     direction  = "in"
     protocol   = "tcp"
     port       = "1-65535"
@@ -83,7 +80,7 @@ resource "hcloud_server" "this" {
   location     = var.location
   ssh_keys     = var.ssh_key_ids
   firewall_ids = [hcloud_firewall.this.id]
-  labels       = { role = "prod" } # picked up by ansible/inventory/hcloud.yml
+  labels       = { role = "prod" }
 
   network {
     network_id = var.network_id

infra/modules/vpn/main.tf

@@ -1,4 +1,4 @@
-# vpn: OpenVPN + Traefik. Not attached to the private network (see docs/architecture.md).
+# vpn: WireGuard (wg-easy) + Traefik. Not attached to the private network.
 resource "hcloud_firewall" "this" {
   name = "vpn-firewall"
 
@@ -9,7 +9,7 @@ resource "hcloud_firewall" "this" {
     source_ips = var.allowed_ssh_source_ips
   }
 
-  rule { # Traefik http/https (traefik.vpn.luke-else.co.uk)
+  rule { # Traefik http/https
     direction  = "in"
     protocol   = "tcp"
     port       = "80"
@@ -23,10 +23,10 @@ resource "hcloud_firewall" "this" {
     source_ips = ["0.0.0.0/0", "::/0"]
   }
 
-  rule { # OpenVPN tunnel - always direct to this server's public IP, never via a load balancer
+  rule { # WireGuard tunnel - always direct to this server's public IP, never via a load balancer
     direction  = "in"
     protocol   = "udp"
-    port       = "1194"
+    port       = "51820"
     source_ips = ["0.0.0.0/0", "::/0"]
   }
 }
@@ -38,5 +38,5 @@ resource "hcloud_server" "this" {
   location     = var.location
   ssh_keys     = var.ssh_key_ids
   firewall_ids = [hcloud_firewall.this.id]
-  labels       = { role = "vpn" } # picked up by ansible/inventory/hcloud.yml
+  labels       = { role = "vpn" }
 }

infra/modules/vpn/variables.tf

@@ -1,5 +1,5 @@
 variable "server_type" {
-  description = "Server type for vpn (OpenVPN + Traefik)."
+  description = "Server type for vpn (WireGuard/wg-easy + Traefik)."
   type        = string
 }
 

infra/variables.tf

@@ -13,25 +13,25 @@ variable "network_zone" {
 variable "server_image" {
   description = "OS image used for all servers."
   type        = string
-  default     = "ubuntu-24.04"
+  default     = "ubuntu-26.04"
 }
 
 variable "dev_server_type" {
   description = "Server type for dev"
   type        = string
-  default     = "cx22"
+  default     = "cx23"
 }
 
 variable "prod_server_type" {
   description = "Server type for prod"
   type        = string
-  default     = "cx22"
+  default     = "cx23"
 }
 
 variable "vpn_server_type" {
   description = "Server type for vpn"
   type        = string
-  default     = "cx22"
+  default     = "cx23"
 }
 
 variable "network_ip_range" {

readme.md

@@ -1,6 +1,6 @@
 # Server
 
-Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each bootstrapped and deployed with [Ansible](https://www.ansible.com/), running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).
+Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), bootstrapped and deployed with [Ansible](https://www.ansible.com/), running Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).
 
 <p align="center">
   <img src="assets/images/main.png" width="70%">
@@ -10,12 +10,14 @@ Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted
 
 - [Architecture](#architecture)
 - [Repository layout](#repository-layout)
+- [Design choices](#design-choices)
 - [Prerequisites](#prerequisites)
+- [Quickstart](#quickstart)
 - [Provisioning the infrastructure](#provisioning-the-infrastructure-infra)
-- [Bootstrapping and deploying with Ansible](#bootstrapping-and-deploying-with-ansible-ansible)
+- [Bootstrapping and deploying](#bootstrapping-and-deploying-ansible)
+- [Running the services](#running-the-services-services)
 - [Service inventory](#service-inventory)
-- [First-time setup](#first-time-setup)
-- [Development container](#development-container)
+- [Caveats](#caveats)
 - [Security notes](#security-notes)
 
 ## Architecture
@@ -53,67 +55,71 @@ architecture-beta
     vpn:R -- L:backups
 ```
 
-`gateway` represents the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from - not a Hetzner resource, see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3).
+`gateway` is the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from (see [Design choices](#design-choices)) — also not a Hetzner resource.
 
 | Server | Purpose | Network |
 |---|---|---|
-| `dev`  | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) |
-| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall |
-| `vpn`  | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` |
+| `dev`  | Gitea, CI runners, dev-facing Traefik | Private network only |
+| `prod` | Public websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall |
+| `vpn`  | WireGuard (wg-easy) + its own Traefik | **Not** on the private network — isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` |
 
-Each host's stateful data lives in local named Docker volumes, backed up nightly to a shared S3 bucket and restored automatically on first boot - see below. No host has a Hetzner Cloud Volume attached any more.
-
-`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach deployment targets on `prod` without exposing that traffic publicly. `vpn` is deliberately kept off this network. Each server has its own Hetzner Cloud Firewall (see `infra/modules/<host>/main.tf`) that only opens the ports actually used by the compose stacks running on it, plus SSH restricted to `var.allowed_ssh_source_ips`.
+`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach `prod` without exposing that traffic publicly. Each server has its own Hetzner Cloud Firewall (`infra/modules/<host>/main.tf`) opening only the ports its own compose stacks use, plus SSH restricted to `var.allowed_ssh_source_ips`.
 
 ## Repository layout
 
 ```
 .
-├── infra/                    # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, DNS
+├── infra/                    # OpenTofu — provisions the 3 servers, network, firewalls, DNS
 │   ├── main.tf                # root module: wires network + dev/prod/vpn/dns modules together
 │   ├── variables.tf           # shared inputs (sizes, locations, IP ranges, SSH key names)
-│   ├── outputs.tf             # pass-through outputs from each host module
-│   ├── ssh.tf                 # looks up each SSH key already uploaded to Hetzner Cloud
-│   ├── versions.tf            # provider requirements
 │   ├── terraform.tfvars.example
 │   └── modules/
-│       ├── network/           # shared private network + subnet (used by dev and prod)
-│       ├── dev/                # dev server + firewall, labeled role=dev
-│       ├── prod/               # prod server + firewall, labeled role=prod
-│       ├── vpn/                # vpn server + firewall (no private network), labeled role=vpn
-│       └── dns/                # Hetzner DNS zones + A records for every domain in var.dns_zones
-├── ansible/                   # Ansible: bootstraps each server and deploys/starts services/<host>/ onto it
-│   ├── inventory/hcloud.yml   # dynamic inventory — queries the Hetzner API, groups by the role label above
-│   ├── group_vars/            # deploy_user, dev_runner_count, backup_s3_*, etc.
+│       ├── network/            # shared private network + subnet (dev + prod)
+│       ├── dev/ prod/ vpn/     # one server + firewall each, labeled role=<name>
+│       └── dns/                # Hetzner DNS zones + records for every domain in var.dns_zones
+├── ansible/                   # bootstraps each server, deploys/starts services/<host>/ onto it
+│   ├── inventory/hcloud.yml    # dynamic inventory — queries the Hetzner API, groups by role label
 │   ├── roles/
-│   │   ├── bootstrap/          # Docker, deploy user, sshd hardening, unattended-upgrades
-│   │   └── deploy/             # copies services/<host>/, renders .env (S3 backup credentials) + Runners/docker-compose.yml
-│   └── playbooks/              # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml
+│   │   ├── bootstrap/           # Docker, deploy user, sshd hardening, unattended-upgrades
+│   │   └── deploy/              # copies services/<host>/, renders .env + Runners/docker-compose.yml
+│   └── playbooks/               # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml
 ├── services/                 # Docker Compose stacks, grouped by which server they run on
-│   ├── dev/                   # Gitea + CI runner + Traefik + backup/restore (git.luke-else.co.uk, cicd.luke-else.co.uk)
-│   ├── prod/                  # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore
-│   ├── vpn/                    # OpenVPN + Traefik + backup/restore
-│   └── todo.md                # Outstanding manual setup/hardening tasks
-├── docs/
-│   └── architecture.md       # Source of the architecture diagram above
-├── .devcontainer/             # Git submodule: shared devcontainer for working on this repo (OpenTofu tooling)
+│   ├── dev/                    # Gitea + CI runners + Traefik + backup/restore
+│   ├── prod/                   # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore
+│   └── vpn/                     # WireGuard (wg-easy) + Traefik + backup/restore
+├── .devcontainer/             # git submodule: shared devcontainer for this repo (OpenTofu tooling)
 └── assets/
 ```
 
-Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, a `backup-docker-compose.yml` + `restore.sh` pair for that host's S3 backup/restore (see below), plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side.
+Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per service, a `backup-docker-compose.yml` + `restore.sh` pair, and a `spinup.sh` / `spindown.sh` pair that brings up or tears down everything on that host in order. `infra/modules/` and `ansible/roles/deploy` mirror the same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks sit side by side.
 
-OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services/<host>/`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown.
+## Design choices
+
+- **OpenTofu and Ansible have a clean split**: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches a server over SSH. Everything from "the server exists" onward — Docker, the `deploy` user, SSH hardening, copying `services/<host>/`, running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md).
+- **No Hetzner Cloud Volumes.** Every stateful service mounts a plain named Docker volume instead. Each host backs its volumes up nightly to S3 and restores any that are empty on first boot — see [Persistent data](#persistent-data).
+- **`vpn` is deliberately off the private network.** SSH to `dev`/`prod` is only reachable through it (see [Security notes](#security-notes)), but it can't reach either directly — containing a compromised VPN endpoint.
+- **DNS is provisioned, not just documented.** `module.dns` creates a Hetzner zone and every A/MX/TXT/CNAME record in `infra/main.tf` — see [Managing DNS](#managing-dns).
+- **Hosts are discovered dynamically** from the Hetzner API by role label, not a static inventory file — nothing in Ansible reads Terraform state.
+- **Gitea Actions runners are disposable.** Their `/data` is a local cache; runner identity re-registers with Gitea on every `spinup.sh`, so they're excluded from backup.
 
 ## Prerequisites
 
 - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token
-- One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (Ansible uses it to bootstrap and deploy — see below)
+- One or more SSH keys uploaded to that project (Console → Security → SSH Keys), plus the matching private key available locally
 - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0`
 - [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`)
-- Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner (see [Managing DNS](#managing-dns) — the zones and records themselves are created for you)
-- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/), AWS S3, or any other S3-compatible provider) and an access key/secret pair with read/write access to it - not provisioned by OpenTofu (the `hcloud` provider has no Object Storage resource), so create this yourself and pass it to Ansible as `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` (and `BACKUP_S3_ENDPOINT` for anything other than real AWS S3) - see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3)
+- Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner
+- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/)) and an access key/secret pair — not provisioned by OpenTofu, so create this yourself
 
-Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below).
+## Quickstart
+
+[`control.sh`](control.sh) at the repo root is an interactive menu covering the whole setup:
+
+```sh
+./control.sh
+```
+
+It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, and the `BACKUP_S3_*` credentials, saved to gitignored `.control.env` so you only enter them once), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand.
 
 ## Provisioning the infrastructure (`infra/`)
 
@@ -128,32 +134,21 @@ tofu plan
 tofu apply
 ```
 
-This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` / `module.dns` in `infra/main.tf`:
-- `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`)
-- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses, each server labeled `role = "dev"/"prod"/"vpn"` for Ansible's dynamic inventory (`modules/dev`, `modules/prod`, `modules/vpn`)
-- one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns))
-
-No servers have an attached volume — persistent data lives in named Docker volumes backed up to S3 instead, see below.
+This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there.
 
 Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`.
 
-`terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`.
-
-OpenTofu never connects to the servers over SSH — no provisioners, no `bootstrap.sh`, no copying `services/` — that's all Ansible now. See below.
-
 ### Managing DNS
 
-`module.dns` (in `infra/modules/dns`) creates one Hetzner DNS zone per domain in `var.dns_zones` (default: `luke-else.co.uk`, `divine-couture.co.uk`, `snexo.co.uk`) and an `hcloud_zone_rrset` A record for every hostname currently referenced by a Traefik `Host()` rule anywhere under `services/` — kept in sync with `local.dns_records` in `infra/main.tf`, pointed at whichever of `dev`/`prod`/`vpn` actually serves it. Zones have `prevent_destroy = true`, matching the volumes — losing one deletes every record in it.
+Creating a zone doesn't make Hetzner authoritative by itself — run `tofu output dns_nameservers` and set those as the domain's NS records at its registrar. Propagation time depends on the registrar and the domain's previous NS TTL.
 
-Creating the zone doesn't make Hetzner authoritative for the domain by itself: you still need to point that domain's NS records at Hetzner's nameservers at whichever registrar it's registered through. Run `tofu output dns_nameservers` after applying to get the exact nameservers per domain, and set those as the domain's NS records at the registrar. Propagation can take a while depending on the registrar and the domain's previous NS TTL.
+To add a subdomain, add an entry to `local.dns_records` in `infra/main.tf` (zone, name, target host) and re-run `tofu apply` — don't hand-create records in the Hetzner console, they'll drift from state. `var.records` supports `MX`/`TXT`/`CNAME`/etc. too (see `infra/modules/dns/variables.tf` for the value format each type expects); entries sharing zone/name/type merge into one RRSet.
 
-Adding a new subdomain: add an entry to `local.dns_records` in `infra/main.tf` (zone, name, and the target `module.<host>.ipv4`) and re-run `tofu apply` — don't hand-create records in the Hetzner console, they'll drift from state. Note `name = "@"` is Hetzner's convention for a zone's apex record (e.g. bare `snexo.co.uk`), not an empty string.
+`local.mail_records` tracks the Microsoft 365 records for `luke-else.co.uk` (MX, SPF, Google-site-verification, `autodiscover`). DMARC and DKIM aren't set up yet — see [Caveats](#caveats).
 
-Only A records for IPv4 are managed here — none of the modules currently track servers' IPv6 addresses, so AAAA records aren't generated even though the firewalls already allow IPv6 traffic.
+## Bootstrapping and deploying (`ansible/`)
 
-## Bootstrapping and deploying with Ansible (`ansible/`)
-
-Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services/<host>/` to each server, rendering `.env` (S3 backup credentials) and, for `dev`, `Runners/docker-compose.yml`, and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version:
+Once `tofu apply` has created the servers, Ansible installs Docker, creates the `deploy` user, hardens SSH, copies `services/<host>/` to each server, and runs `spinup.sh`/`spindown.sh`. Full detail in [`ansible/README.md`](ansible/README.md); the short version:
 
 ```sh
 cd ansible
@@ -163,25 +158,20 @@ export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
 # vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly
 ansible-playbook playbooks/site.yml -l role_vpn
 
-# SSH to vpn as deploy and connect to the OpenVPN it just started, then:
+# SSH to vpn as deploy and connect to the WireGuard tunnel it just started, then:
 ansible-playbook playbooks/site.yml -l role_dev,role_prod
 ```
 
-Hosts are discovered dynamically from the Hetzner API (`ansible/inventory/hcloud.yml`), grouped into `role_dev`/`role_prod`/`role_vpn` by the `role` label OpenTofu sets on each server — there's no static inventory file to keep in sync, and nothing here reads Terraform state.
+`dev`/`prod`'s firewalls only accept SSH from `vpn`'s own public IP, so on a from-scratch estate they aren't reachable until `vpn` is up and you're tunneled through it — visit `https://vpn.luke-else.co.uk`, complete wg-easy's first-run setup, add a client peer, and connect before running the second command. If it's run before you're connected, Ansible just fails to connect — re-run once connected.
 
-### Persistent data lives in named Docker volumes, backed up to S3
+### Persistent data
 
-Every stateful service's compose file mounts a plain named Docker volume rather than a bind mount to a Hetzner volume - `gitea_data:/data`, `bitwarden_data:/data/`, `letsencrypt_data:/letsencrypt`, and so on across `services/dev/*.yml`, `services/prod/*.yml`, and `services/vpn/*.yml`. Each volume is declared with an explicit `name:` in every compose file that mounts it (the same pattern already used for the shared `proxy` network), so Compose creates it once and every stack that references it reuses the same underlying volume regardless of which compose file happens to run first.
+Every stateful service mounts a named Docker volume (`gitea_data`, `bitwarden_data`, `letsencrypt_data`, etc.) rather than a Hetzner volume. Each host's `backup-docker-compose.yml` defines:
 
-Each host's `backup-docker-compose.yml` defines two services against those same named volumes:
-- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), running on a nightly cron (`0 3 * * *`), tars up every mounted volume and uploads it to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/<host>-<timestamp>.tar.gz`, keeping 14 days of history (`BACKUP_RETENTION_DAYS`). Containers labeled `docker-volume-backup.stop-during-backup=true` (Gitea, Bitwarden, RustDesk, Uptime Kuma, snexo, OpenVPN) are stopped for the duration of the backup so their data is captured in a consistent state, then restarted automatically. Started by `spinup.sh` alongside everything else.
-- **`restore`** — a one-shot container (`amazon/aws-cli` running `restore.sh`) that `spinup.sh` runs *before* anything else starts. It checks each of the host's named volumes; any that are already empty are populated from the most recent object under that host's S3 prefix, any that already contain data are left completely untouched. On a fresh server with no prior backup (or a bucket that doesn't exist yet), it's a no-op - services just start with empty volumes as before.
+- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), nightly cron, tars every mounted volume to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/<host>-<timestamp>.tar.gz`, 14 days of retention. Containers that need a consistent snapshot are stopped for the duration and restarted after.
+- **`restore`** — a one-shot container `spinup.sh` runs before anything else starts. Any volume that's already empty is populated from the most recent backup; anything with data is left untouched. On a fresh server (or no prior backup), it's a no-op.
 
-`AWS_S3_BUCKET_NAME`, `AWS_S3_PATH` (set to the host name, e.g. `dev`), `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_ENDPOINT` are rendered into `services/<host>/.env` by Ansible's `deploy` role from `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`/`BACKUP_S3_ENDPOINT` in your shell environment (see [Prerequisites](#prerequisites)) — `docker compose` auto-loads `.env` from its working directory. `spinup.sh` on every host refuses to start anything if `.env` is missing.
-
-The Gitea Actions runners are the one exception: their `/data` is a disposable local cache (runner identity re-registers with Gitea on every `spinup.sh` run anyway), so it's a plain named volume per runner (`gitea_runner_N_data`, baked directly into the rendered `Runners/docker-compose.yml`) that's deliberately left out of the backup/restore stack.
-
-To trigger a backup or restore manually rather than waiting for cron or a restart:
+The S3 credentials (`AWS_S3_BUCKET_NAME`, `AWS_ACCESS_KEY_ID`, etc.) are rendered into `services/<host>/.env` by Ansible from the `BACKUP_S3_*` env vars — `spinup.sh` refuses to start anything if `.env` is missing. To trigger either manually:
 
 ```sh
 cd services/<host>
@@ -189,27 +179,9 @@ docker compose -f backup-docker-compose.yml exec backup backup   # force an imme
 docker compose -f backup-docker-compose.yml run --rm restore     # restore any currently-empty volumes
 ```
 
-### Scaling Gitea Actions runners
+## Running the services (`services/`)
 
-The number of Gitea Actions runner containers on `dev` is set by `dev_runner_count` in [`ansible/group_vars/role_dev.yml`](ansible/group_vars/role_dev.yml) (default `3`). Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host: change `dev_runner_count` and redeploy rather than hand-editing it.
-
-Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a count change.
-
-Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore.
-
-### First deploy: bootstrapping order matters
-
-`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch estate, Ansible can't reach `dev`/`prod` yet. Bring it up in this order:
-
-1. `ansible-playbook playbooks/site.yml -l role_vpn` — bootstraps, deploys, and starts `vpn` only; its firewall allows SSH from `var.allowed_ssh_source_ips` directly.
-2. SSH in as `deploy` and connect to the OpenVPN service you just started with an OpenVPN client.
-3. `ansible-playbook playbooks/site.yml -l role_dev,role_prod` — now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and `dev`/`prod` become reachable.
-
-If step 3 is run before you're connected to the VPN, Ansible will simply fail to connect — re-run it once connected.
-
-## Deploying the services (`services/`)
-
-After `ansible-playbook playbooks/deploy.yml` has copied `services/<host>/` to `/home/deploy/services/<host>` on the matching server, SSH in as `deploy` and run the matching script from inside that directory — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do exactly this remotely (see [`ansible/README.md`](ansible/README.md)):
+After `ansible-playbook playbooks/deploy.yml` has copied `services/<host>/` to the server, SSH in as `deploy` and run the matching script — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do this remotely:
 
 ```sh
 # on dev
@@ -217,27 +189,19 @@ After `ansible-playbook playbooks/deploy.yml` has copied `services/<host>/` to `
 ./spindown.sh   # reverse order, then prunes images/volumes
 
 # on prod
-./spinup.sh    # Traefik, → Watchtower → status → websites → Bitwarden → RustDesk
+./spinup.sh    # Traefik → Watchtower → status → websites → Bitwarden → RustDesk
 ./spindown.sh
 
 # on vpn
-./spinup.sh     # Traefik → OpenVPN → Watchtower
+./spinup.sh     # Traefik → wg-easy → Watchtower
 ./spindown.sh
 ```
 
-If you ever need to force a re-sync without going through Ansible (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@<prod_ipv4>:~/services` still works fine - `ansible-playbook playbooks/deploy.yml` will just overwrite it again next time it's run.
+Each stack can also be managed individually with plain Compose, e.g. `docker compose -f bitwarden-docker-compose.yml up -d` from inside `services/prod`.
 
-Each stack can also be managed individually with plain Compose, e.g.:
+All three hosts run [Watchtower](https://containrrr.dev/watchtower/), polling every 60s with cleanup enabled, so images stay current automatically once deployed — re-run the spinup scripts only after adding/removing a service or changing compose files. Every public-facing service is fronted by its host's own Traefik, terminating TLS via Let's Encrypt, joining an external `proxy` network and opting in via `traefik.enable=true` labels (RustDesk and the Gitea SSH port publish ports directly, since they aren't HTTP).
 
-```sh
-cd services/prod
-docker compose -f bitwarden-docker-compose.yml up -d
-docker compose -f bitwarden-docker-compose.yml down
-```
-
-All three hosts run [Watchtower](https://containrrr.dev/watchtower/) polling every 60s with cleanup enabled, so images are kept current automatically once deployed — the spinup scripts only need to be re-run after adding/removing a service or changing compose files.
-
-Every public-facing service is fronted by its host's own Traefik instance, terminating TLS via Let's Encrypt (`tlschallenge`, port 80/443). Each stack joins an external `proxy` Docker network and opts in via `traefik.enable=true` labels rather than publishing ports directly (RustDesk and the Gitea SSH port are the deliberate exceptions, since they aren't HTTP).
+`vpn` runs [wg-easy](https://github.com/wg-easy/wg-easy) — WireGuard plus a web UI, fronted by Traefik. It has no automated first-run setup: visit `https://vpn.luke-else.co.uk`, pick an admin username/password (enable 2FA — this account can view/rotate every client's private key and is reachable from the public internet), set the connect host/port (`vpn.luke-else.co.uk` / `51820`), then add a client per device and download its config or scan its QR code. wg-easy's own state lives in the `wg_easy_data` volume, backed up like everything else; client configs themselves are never stored anywhere but the UI.
 
 ## Service inventory
 
@@ -247,7 +211,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi
 |---|---|---|
 | Traefik | `traefik-docker-compose.yml` | `traefik.cicd.luke-else.co.uk` |
 | Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` |
-| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | N/A |
+| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated, 3 by default) | N/A |
 | Watchtower | `watchtower-docker-compose.yml` | — |
 | Backup/restore | `backup-docker-compose.yml` | — |
 
@@ -268,19 +232,13 @@ Every public-facing service is fronted by its host's own Traefik instance, termi
 | Service | Compose file | Domain(s) |
 |---|---|---|
 | Traefik | `traefik-docker-compose.yml` | `traefik.vpn.luke-else.co.uk` |
-| OpenVPN (Dockovpn) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `1194` |
+| WireGuard (wg-easy) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `51820` |
 | Watchtower | `watchtower-docker-compose.yml` | — |
 | Backup/restore | `backup-docker-compose.yml` | — |
 
-## First-time setup
-
-A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here:
-
-- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by [Ansible's `bootstrap` role](ansible/roles/bootstrap/tasks/main.yml); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)).
-
 ## Development container
 
-`.devcontainer` is a git submodule providing a ready-to-use OpenTofu development environment (VS Code + OpenTofu/Docker/Mermaid extensions). After cloning:
+`.devcontainer` is a git submodule providing a ready-to-use OpenTofu development environment (VS Code + OpenTofu/Docker/Mermaid extensions):
 
 ```sh
 git submodule update --init --recursive
@@ -288,11 +246,17 @@ git submodule update --init --recursive
 
 Then reopen the repo in VS Code with the Dev Containers extension.
 
+## Caveats
+
+- **UFW isn't configured** — the Hetzner Cloud Firewalls already allowlist per-host ports, but there's no host-level firewall as defense in depth yet.
+- **DMARC and DKIM aren't set up** for `luke-else.co.uk`. DKIM's targets are generated per-tenant by Microsoft 365 (Defender portal → Email & collaboration → Policies & rules → DKIM) and can't be guessed — enable it there first, then add the two CNAMEs it gives you to `local.mail_records`.
+- **Only IPv4 DNS is managed.** None of the infra modules track servers' IPv6 addresses, so no AAAA records are generated even though the firewalls already allow IPv6 traffic.
+- **`infra/modules/dns/moved.tf`** re-points pre-existing A records at their new resource address after the DNS module was generalized for non-A record types. It's a one-time migration aid — safe to delete once `tofu plan` shows no unexpected changes.
+
 ## Security notes
 
-- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. The S3 credentials end up in each host's `services/<host>/.env` (rendered by Ansible, mode `0600`, gitignored) - anyone with root on a host can read them, same trust boundary as everything else `deploy` can already do there.
-- SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s).
-- `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP.
-- Firewalls are allowlists scoped per host in `infra/modules/<host>/main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open.
-- Every host's [Ansible bootstrap role](ansible/roles/bootstrap/tasks/main.yml) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`deploy_user`, see `ansible/group_vars/all.yml`) for day-to-day access.
-- DNS zones (`modules/dns`) carry both Hetzner's own `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, including for `divine-couture.co.uk` and `snexo.co.uk`, not just `luke-else.co.uk`.
+- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_*`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. S3 credentials land in each host's `services/<host>/.env` (mode `0600`, gitignored, rendered by Ansible).
+- SSH to `dev`/`prod` is restricted to `vpn`'s own public IP — you must be tunneled into the VPN to reach them. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your egress IP(s).
+- Firewalls are per-host allowlists (`infra/modules/<host>/main.tf`) — only ports actually used by that host's compose stacks are open.
+- Ansible's bootstrap role disables SSH password auth, restricts root login to key-only, and creates a separate sudo user (`deploy_user`) for day-to-day access.
+- DNS zones carry both Hetzner's `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, across all three managed domains.

services/dev/backup-docker-compose.yml

@@ -26,7 +26,7 @@ services:
   # Run explicitly by spinup.sh before the services that own these volumes
   # start - not started by `docker compose up`.
   restore:
-    image: amazon/aws-cli:2
+    image: amazon/aws-cli:latest
     container_name: backup-restore
     entrypoint: ["/bin/bash", "/restore.sh"]
     environment:

services/dev/spinup.sh

@@ -17,8 +17,9 @@ docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-d
 docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d
 
 echo "Waiting for Gitea to become ready..."
+sleep 20
 for i in $(seq 1 30); do
-  if docker exec gitea gitea admin user list >/dev/null 2>&1; then
+  if docker exec -u 1000:1000 gitea gitea admin user list >/dev/null 2>&1; then
     break
   fi
   sleep 2
@@ -27,7 +28,7 @@ done
 # Generate a fresh Actions runner registration token straight from Gitea and
 # hand it to the runner containers via .env (docker compose auto-loads this) -
 # no manual copy/paste from the admin UI needed.
-RUNNER_TOKEN=$(docker exec gitea gitea actions generate-runner-token)
+RUNNER_TOKEN=$(docker exec -u 1000:1000 gitea gitea actions generate-runner-token)
 echo "GITEA_RUNNER_REGISTRATION_TOKEN=$RUNNER_TOKEN" > Runners/.env
 
 cd Runners

services/prod/backup-docker-compose.yml

@@ -22,7 +22,6 @@ services:
       - bitwarden_data:/backup/bitwarden_data:ro
       - rustdesk_data:/backup/rustdesk_data:ro
       - uptime_kuma_data:/backup/uptime_kuma_data:ro
-      - snexo_data:/backup/snexo_data:ro
       - letsencrypt_data:/backup/letsencrypt_data:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
     restart: unless-stopped
@@ -31,7 +30,7 @@ services:
   # Run explicitly by spinup.sh before the services that own these volumes
   # start - not started by `docker compose up`.
   restore:
-    image: amazon/aws-cli:2
+    image: amazon/aws-cli:latest
     container_name: backup-restore
     entrypoint: [ "/bin/bash", "/restore.sh" ]
     environment:

services/prod/web-docker-compose.yml

@@ -58,25 +58,6 @@ services:
       - "traefik.http.routers.metarius.tls.certresolver=myresolver"
     restart: unless-stopped
 
-  divine-couture:
-    image: git.luke-else.co.uk/luke-else/divine-couture.co.uk:latest
-    container_name: divine-couture
-    networks:
-      - proxy
-    labels:
-      ## Expose divine-couture Through Trefik ##
-      - "traefik.enable=true" # <== Enable traefik to proxy this container
-
-      - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
-      - "traefik.http.routers.divine-couture-insecure.middlewares=redirect-web-secure"
-      - "traefik.http.routers.divine-couture-insecure.rule=Host(`www.divine-couture.co.uk`)"
-      - "traefik.http.routers.divine-couture-insecure.entrypoints=web"
-
-      - "traefik.http.routers.divine-couture.rule=Host(`www.divine-couture.co.uk`)"
-      - "traefik.http.routers.divine-couture.entrypoints=websecure"
-      - "traefik.http.routers.divine-couture.tls.certresolver=myresolver"
-    restart: unless-stopped
-
 networks:
   proxy:
     external: true

services/todo.md

@@ -1,32 +0,0 @@
-# ToDo Items
-
-## General
-
-- Setup non root user
-
-   - "21115:21115/tcp"
-   - "21116:21116/tcp"
-   - "21116:21116/udp"
-   - "21117:21117/tcp"
-   - "21119:21119/tcp"
-
-- Install docker, docker-compose and apache utils.
-
-## Traefik + TraefikRunner + Traefik VPN
-
-- Setup htaccess -> `echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g`
-- Ensure email address is correct
-
-## Gitea
-
-- Ensure that ports are assigned correctly for the system
-
-## Websites
-
-- Ensure website files are copied over
-- Ensure that ports are assigned correctly for the system
-
-## Bitwarden
-
-- Ensure that all data is fully encrypted during transfer.
-- Ensure that ports are assigned correctly for the system

services/vpn/backup-docker-compose.yml

@@ -17,7 +17,7 @@ services:
       AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
       AWS_ENDPOINT: ${AWS_ENDPOINT}
     volumes:
-      - openvpn_data:/backup/openvpn_data:ro
+      - wg_easy_data:/backup/wg_easy_data:ro
       - letsencrypt_data:/backup/letsencrypt_data:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
     restart: unless-stopped
@@ -26,7 +26,7 @@ services:
   # Run explicitly by spinup.sh before the services that own these volumes
   # start - not started by `docker compose up`.
   restore:
-    image: amazon/aws-cli:2
+    image: amazon/aws-cli:latest
     container_name: backup-restore
     entrypoint: ["/bin/bash", "/restore.sh"]
     environment:
@@ -37,12 +37,12 @@ services:
       AWS_ENDPOINT: ${AWS_ENDPOINT}
     volumes:
       - ./restore.sh:/restore.sh:ro
-      - openvpn_data:/restore/openvpn_data
+      - wg_easy_data:/restore/wg_easy_data
       - letsencrypt_data:/restore/letsencrypt_data
     restart: "no"
 
 volumes:
-  openvpn_data:
-    name: openvpn_data
+  wg_easy_data:
+    name: wg_easy_data
   letsencrypt_data:
     name: letsencrypt_data

services/vpn/spindown.sh

@@ -9,5 +9,5 @@ docker compose -f backup-docker-compose.yml down
 
 docker rmi $(docker images -q)
 docker system prune -f -a
-# Deliberately no `docker volume prune` - openvpn_data/letsencrypt_data are
+# Deliberately no `docker volume prune` - wg_easy_data/letsencrypt_data are
 # named volumes holding real data, backed up to S3 but not disposable.

services/vpn/vpn-docker-compose.yml

@@ -1,17 +1,25 @@
 services:
-  dockovpn:
-    image: alekslitvinenk/openvpn
+  wg-easy:
+    image: ghcr.io/wg-easy/wg-easy:15
     cap_add:
-        - NET_ADMIN
+      - NET_ADMIN
+      - SYS_MODULE # lets the container modprobe wireguard if the host kernel doesn't already have it loaded
+    sysctls:
+      - net.ipv4.ip_forward=1
+      - net.ipv4.conf.all.src_valid_mark=1
+      - net.ipv6.conf.all.disable_ipv6=0
+      - net.ipv6.conf.all.forwarding=1
+      - net.ipv6.conf.default.forwarding=1
     ports:
-        - 1194:1194/udp # Expose tcp if you defined HOST_TUN_PROTOCOL=tcp
-    environment:
-        HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address 
+      - 51820:51820/udp # WireGuard tunnel - the web UI (51821) only needs to be reachable via Traefik below, not published directly
+    networks:
+      - proxy
     volumes:
-        - openvpn_data:/opt/Dockovpn_data
+      - /lib/modules:/lib/modules:ro
+      - wg_easy_data:/etc/wireguard
     labels:
-      ## Expose vpn Through Trefik ##
-      - "traefik.enable=true" # <== Enable traefik to proxy this container
+      ## Expose the wg-easy web UI through Traefik ##
+      - "traefik.enable=true"
 
       # Stopped for the duration of each backup run - see backup-docker-compose.yml.
       - "docker-volume-backup.stop-during-backup=true"
@@ -24,8 +32,13 @@ services:
       - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)"
       - "traefik.http.routers.vpn.entrypoints=websecure"
       - "traefik.http.routers.vpn.tls.certresolver=myresolver"
-    restart: always
+      - "traefik.http.services.vpn.loadbalancer.server.port=51821"
+    restart: unless-stopped
+
+networks:
+  proxy:
+    external: true
 
 volumes:
-  openvpn_data:
-    name: openvpn_data
+  wg_easy_data:
+    name: wg_easy_data