ansible/README.md

@@ -16,7 +16,7 @@ to S3, not on a Hetzner volume - see `../readme.md`.
   (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules/<host>/main.tf`).
 - `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path,
   `backup_s3_*` S3 backup credentials), `role_dev.yml` / `role_prod.yml` /
-  `role_vpn.yml` (per-role vars, e.g. `dev_runner_count`).
+  `role_vpn.yml` (per-role vars, e.g. `service_group`).
 - `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening,
   unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`.
 - `roles/deploy/` - copies `services/<host>/` to the server and renders `.env`
@@ -65,9 +65,10 @@ ansible-playbook playbooks/spinup.yml    -l role_dev,role_prod
 ansible-playbook playbooks/site.yml -l role_dev,role_prod
 ```
 
-Re-run `deploy.yml` + `spinup.yml` any time a compose file changes or
-`dev_runner_count` in `group_vars/role_dev.yml` is bumped - `spinup.sh` only
-starts/updates what's changed, same as before.
+Re-run `deploy.yml` + `spinup.yml` any time a compose file changes - `spinup.sh`
+only starts/updates what's changed, same as before. `dev` always runs three
+Gitea Actions runners (fixed in
+`roles/deploy/templates/runners-docker-compose.yml.j2`).
 
 `spindown.yml` runs `spindown.sh`, which does a full `docker system/volume
 prune -a` on the host - it prompts for a `yes` confirmation before running,

ansible/group_vars/role_dev.yml

@@ -1,7 +1,2 @@
 ---
 service_group: dev
-
-# Number of Gitea Actions runner containers rendered into
-# services/dev/Runners/docker-compose.yml by the deploy role - see
-# roles/deploy/templates/runners-docker-compose.yml.j2.
-dev_runner_count: 3

ansible/roles/bootstrap/tasks/main.yml

@@ -10,50 +10,22 @@
     upgrade: dist
     cache_valid_time: 3600
 
-- name: Install Docker prerequisites
+# Docker straight from Ubuntu's own repositories - no external apt repo, GPG
+# key or architecture juggling. docker.io is the engine, docker-compose-v2 is
+# the `docker compose` plugin. A little behind Docker's own channel, but far
+# simpler and plenty for these hosts.
+- name: Install Docker Engine and the Compose plugin
   ansible.builtin.apt:
     name:
-      - ca-certificates
-      - curl
-      - gnupg
+      - docker.io
+      - docker-compose-v2
     state: present
 
-- name: Create apt keyrings directory
-  ansible.builtin.file:
-    path: /etc/apt/keyrings
-    state: directory
-    mode: "0755"
-
-- name: Download Docker's GPG key
-  ansible.builtin.get_url:
-    url: https://download.docker.com/linux/ubuntu/gpg
-    dest: /etc/apt/keyrings/docker.asc
-    mode: "0644"
-    force: false
-
-- name: Determine dpkg architecture
-  ansible.builtin.command: dpkg --print-architecture
-  register: _dpkg_arch
-  changed_when: false
-
-- name: Add Docker apt repository
-  ansible.builtin.apt_repository:
-    repo: >-
-      deb [arch={{ _dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc]
-      https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable
-    filename: docker
-    state: present
-
-- name: Install Docker Engine and Compose plugin
-  ansible.builtin.apt:
-    update_cache: true
-    name:
-      - docker-ce
-      - docker-ce-cli
-      - containerd.io
-      - docker-buildx-plugin
-      - docker-compose-plugin
-    state: present
+- name: Ensure Docker is enabled and running
+  ansible.builtin.systemd:
+    name: docker
+    enabled: true
+    state: started
 
 - name: Create non-root sudo user, given the same key root was provisioned with
   ansible.builtin.user:

ansible/roles/deploy/templates/runners-docker-compose.yml.j2

@@ -1,7 +1,6 @@
-# Generated by Ansible from dev_runner_count - do not hand-edit.
-# To change the number of runners, edit dev_runner_count in
-# ansible/group_vars/role_dev.yml and re-run playbooks/deploy.yml.
-# To change their shape, edit this template instead.
+# Generated by Ansible - do not hand-edit. Three Gitea Actions runners, always.
+# To change their shape (image, volumes, env), edit this template and re-run
+# playbooks/deploy.yml.
 #
 # GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable
 # (not baked in here) - services/dev/spinup.sh generates a fresh token and
@@ -10,8 +9,9 @@
 # /data is a disposable local cache (runner identity re-registers on every
 # spinup.sh run anyway), so it's a plain named volume - not part of the S3
 # backup/restore stack in services/dev/backup-docker-compose.yml.
+{% set runner_count = 3 %}
 services:
-{% for i in range(1, dev_runner_count + 1) %}
+{% for i in range(1, runner_count + 1) %}
   runner-{{ i }}:
     image: gitea/act_runner:latest
     container_name: gitea_runner_{{ i }}
@@ -34,7 +34,7 @@ networks:
     external: true
 
 volumes:
-{% for i in range(1, dev_runner_count + 1) %}
+{% for i in range(1, runner_count + 1) %}
   gitea_runner_{{ i }}_data:
     name: gitea_runner_{{ i }}_data
 {% endfor %}

control.sh

@@ -0,0 +1,198 @@
+#!/usr/bin/env bash
+# control.sh - interactive control panel for this repo.
+#
+# Steps you through the pre-setup and day-to-day operations for both the
+# OpenTofu infra (infra/) and the Ansible config (ansible/), in the order the
+# firewall rules require (vpn first, then tunnel in, then dev/prod). It only
+# ever runs `tofu`/`ansible-playbook` for you - it's a menu wrapper, nothing
+# more, so anything it does you can also do by hand from infra/ or ansible/.
+
+set -uo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INFRA_DIR="$REPO_ROOT/infra"
+ANSIBLE_DIR="$REPO_ROOT/ansible"
+
+# --- pretty output --------------------------------------------------------
+if [ -t 1 ]; then
+  BOLD=$(tput bold); DIM=$(tput dim); RESET=$(tput sgr0)
+  RED=$(tput setaf 1); GREEN=$(tput setaf 2); YELLOW=$(tput setaf 3); CYAN=$(tput setaf 6)
+else
+  BOLD=""; DIM=""; RESET=""; RED=""; GREEN=""; YELLOW=""; CYAN=""
+fi
+
+ok()    { printf "%s  ok %s %s\n"   "$GREEN" "$RESET" "$1"; }
+warn()  { printf "%s warn%s %s\n"   "$YELLOW" "$RESET" "$1"; }
+bad()   { printf "%s miss%s %s\n"   "$RED" "$RESET" "$1"; }
+title() { printf "\n%s%s%s\n" "$BOLD$CYAN" "$1" "$RESET"; }
+
+# Pick the OpenTofu binary: prefer `tofu`, fall back to `terraform`.
+TOFU=""
+detect_tofu() {
+  if command -v tofu >/dev/null 2>&1; then TOFU="tofu"
+  elif command -v terraform >/dev/null 2>&1; then TOFU="terraform"
+  fi
+}
+detect_tofu
+
+# Run a command inside a directory, echoing it first. Returns the command's exit code.
+run_in() {
+  local dir="$1"; shift
+  printf "%s\$ (cd %s && %s)%s\n" "$DIM" "${dir#$REPO_ROOT/}" "$*" "$RESET"
+  ( cd "$dir" && "$@" )
+}
+
+pause() { read -rp $'\nPress Enter to return to the menu... ' _; }
+
+confirm() {
+  local reply
+  read -rp "$1 [y/N] " reply
+  [[ "$reply" == "y" || "$reply" == "Y" ]]
+}
+
+# --- prerequisite checks --------------------------------------------------
+check_prereqs() {
+  title "Tools"
+  if [ -n "$TOFU" ]; then ok "OpenTofu ($TOFU: $("$TOFU" version 2>/dev/null | head -n1))"
+  else bad "Neither 'tofu' nor 'terraform' found on PATH"; fi
+  if command -v ansible-playbook >/dev/null 2>&1; then ok "Ansible ($(ansible --version 2>/dev/null | head -n1))"
+  else bad "'ansible-playbook' not found on PATH"; fi
+
+  title "Environment variables"
+  [ -n "${HCLOUD_TOKEN:-}" ]                 && ok "HCLOUD_TOKEN set"                 || bad "HCLOUD_TOKEN not set (needed by both tofu and the ansible inventory)"
+  [ -n "${BACKUP_S3_BUCKET:-}" ]             && ok "BACKUP_S3_BUCKET set"             || warn "BACKUP_S3_BUCKET not set (needed for deploy/spinup - S3 backup .env)"
+  [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ]      && ok "BACKUP_S3_ACCESS_KEY_ID set"      || warn "BACKUP_S3_ACCESS_KEY_ID not set"
+  [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ]  && ok "BACKUP_S3_SECRET_ACCESS_KEY set"  || warn "BACKUP_S3_SECRET_ACCESS_KEY not set"
+  [ -n "${BACKUP_S3_ENDPOINT:-}" ]           && ok "BACKUP_S3_ENDPOINT set"           || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)"
+
+  local key="${ANSIBLE_SSH_PRIVATE_KEY_FILE:-$HOME/.ssh/id_ed25519}"
+  [ -f "${key/#\~/$HOME}" ] && ok "SSH key present ($key)" || warn "SSH key not found at $key (set ANSIBLE_SSH_PRIVATE_KEY_FILE)"
+
+  title "Files"
+  [ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \
+    || bad "infra/terraform.tfvars missing (copy terraform.tfvars.example and fill in ssh_key_names)"
+  [ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \
+    || warn "infra/.terraform missing - run OpenTofu > init"
+  if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed"
+  else warn "hetzner.hcloud collection not installed - run Ansible > Install collections"; fi
+}
+
+# --- OpenTofu actions -----------------------------------------------------
+require_tofu() {
+  if [ -z "$TOFU" ]; then bad "No tofu/terraform binary found."; return 1; fi
+  if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then
+    bad "infra/terraform.tfvars is missing - copy terraform.tfvars.example first."; return 1
+  fi
+}
+tofu_init()    { require_tofu && run_in "$INFRA_DIR" "$TOFU" init; }
+tofu_plan()    { require_tofu && run_in "$INFRA_DIR" "$TOFU" plan; }
+tofu_apply()   { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply; }
+tofu_output()  { require_tofu && run_in "$INFRA_DIR" "$TOFU" output; }
+tofu_destroy() {
+  require_tofu || return 1
+  confirm "${RED}Destroy ALL infra managed by OpenTofu?${RESET}" && run_in "$INFRA_DIR" "$TOFU" destroy
+}
+
+# --- Ansible actions ------------------------------------------------------
+ansible_collections() { run_in "$ANSIBLE_DIR" ansible-galaxy collection install -r requirements.yml; }
+ansible_inventory()   { run_in "$ANSIBLE_DIR" ansible-inventory --graph; }
+
+# $1 = playbook, $2 = -l limit expression
+play() { run_in "$ANSIBLE_DIR" ansible-playbook "playbooks/$1" -l "$2"; }
+
+ansible_bootstrap_vpn()   { play bootstrap.yml role_vpn; }
+ansible_deploy_vpn()      { play deploy.yml role_vpn && play spinup.yml role_vpn; }
+ansible_bootstrap_dp()    { play bootstrap.yml role_dev,role_prod; }
+ansible_deploy_dp()       { play deploy.yml role_dev,role_prod && play spinup.yml role_dev,role_prod; }
+
+ansible_spindown() {
+  local target
+  read -rp "Spindown which hosts? (e.g. role_vpn, role_dev,role_prod, all): " target
+  [ -n "$target" ] && play spindown.yml "$target"
+}
+
+# --- guided walkthrough ---------------------------------------------------
+guided() {
+  title "Guided full setup"
+  cat <<EOF
+This walks the whole bring-up in the order the firewalls require. dev/prod only
+accept SSH from the vpn server's public IP, so vpn must exist and you must be
+connected to it (OpenVPN) before dev/prod are reachable.
+
+Steps, in order:
+  1. OpenTofu: init -> plan -> apply   (creates servers, network, firewalls, DNS)
+  2. Ansible:  bootstrap + deploy + spinup for role_vpn
+  3. YOU:      connect to the VPN with an OpenVPN client
+  4. Ansible:  bootstrap + deploy + spinup for role_dev,role_prod
+EOF
+  confirm "Start the guided walkthrough?" || return 0
+
+  echo; title "1/4  OpenTofu"
+  if confirm "Run $TOFU init/plan/apply now?"; then
+    tofu_init && tofu_plan && tofu_apply || { bad "OpenTofu step failed - stopping."; return 1; }
+  fi
+
+  echo; title "2/4  Ansible - vpn"
+  if confirm "Bootstrap + deploy + spinup role_vpn now?"; then
+    ansible_bootstrap_vpn && ansible_deploy_vpn || { bad "vpn step failed - stopping."; return 1; }
+  fi
+
+  echo; title "3/4  Connect the VPN"
+  warn "Connect to the vpn server with your OpenVPN client now, then continue."
+  confirm "Are you connected to the VPN?" || { warn "Stopping - reconnect and re-run this step."; return 0; }
+
+  echo; title "4/4  Ansible - dev + prod"
+  if confirm "Bootstrap + deploy + spinup role_dev,role_prod now?"; then
+    ansible_bootstrap_dp && ansible_deploy_dp || { bad "dev/prod step failed."; return 1; }
+  fi
+  ok "Guided setup complete."
+}
+
+# --- menu -----------------------------------------------------------------
+menu() {
+  clear
+  printf "%s==== server control panel ====%s\n" "$BOLD$CYAN" "$RESET"
+  printf "%srepo: %s   tofu: %s%s\n" "$DIM" "$REPO_ROOT" "${TOFU:-none}" "$RESET"
+
+  title "Setup"
+  echo "  1) Check prerequisites"
+  echo "  2) Guided full setup (vpn -> connect -> dev/prod)"
+
+  title "OpenTofu (infra/)"
+  echo "  3) init        4) plan        5) apply"
+  echo "  6) output      7) destroy"
+
+  title "Ansible (ansible/)"
+  echo "  8) Install collections        9) Show inventory"
+  echo " 10) Bootstrap vpn             11) Deploy + spinup vpn"
+  echo " 12) Bootstrap dev+prod        13) Deploy + spinup dev+prod"
+  echo " 14) Spindown (choose target)"
+
+  echo
+  echo "  0) Quit"
+  echo
+  read -rp "Select an option: " choice
+}
+
+while true; do
+  menu
+  case "$choice" in
+    1)  check_prereqs; pause ;;
+    2)  guided; pause ;;
+    3)  tofu_init; pause ;;
+    4)  tofu_plan; pause ;;
+    5)  tofu_apply; pause ;;
+    6)  tofu_output; pause ;;
+    7)  tofu_destroy; pause ;;
+    8)  ansible_collections; pause ;;
+    9)  ansible_inventory; pause ;;
+    10) ansible_bootstrap_vpn; pause ;;
+    11) ansible_deploy_vpn; pause ;;
+    12) ansible_bootstrap_dp; pause ;;
+    13) ansible_deploy_dp; pause ;;
+    14) ansible_spindown; pause ;;
+    0)  echo "Bye."; exit 0 ;;
+    "") ;;
+    *)  warn "Unknown option: $choice"; pause ;;
+  esac
+done

readme.md

@@ -11,6 +11,7 @@ Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted
 - [Architecture](#architecture)
 - [Repository layout](#repository-layout)
 - [Prerequisites](#prerequisites)
+- [Control panel (`control.sh`)](#control-panel-controlsh)
 - [Provisioning the infrastructure](#provisioning-the-infrastructure-infra)
 - [Bootstrapping and deploying with Ansible](#bootstrapping-and-deploying-with-ansible-ansible)
 - [Service inventory](#service-inventory)
@@ -84,7 +85,7 @@ Each host's stateful data lives in local named Docker volumes, backed up nightly
 │       └── dns/                # Hetzner DNS zones + A records for every domain in var.dns_zones
 ├── ansible/                   # Ansible: bootstraps each server and deploys/starts services/<host>/ onto it
 │   ├── inventory/hcloud.yml   # dynamic inventory — queries the Hetzner API, groups by the role label above
-│   ├── group_vars/            # deploy_user, dev_runner_count, backup_s3_*, etc.
+│   ├── group_vars/            # deploy_user, service_group, backup_s3_*, etc.
 │   ├── roles/
 │   │   ├── bootstrap/          # Docker, deploy user, sshd hardening, unattended-upgrades
 │   │   └── deploy/             # copies services/<host>/, renders .env (S3 backup credentials) + Runners/docker-compose.yml
@@ -115,6 +116,16 @@ OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud res
 
 Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below).
 
+## Control panel (`control.sh`)
+
+If you'd rather not remember the exact command order, [`control.sh`](control.sh) at the repo root is an interactive menu that steps you through the whole pre-setup and bring-up. Run it from anywhere:
+
+```sh
+./control.sh
+```
+
+It offers a **Check prerequisites** option (tools, `HCLOUD_TOKEN`/`BACKUP_S3_*` env vars, `terraform.tfvars`, SSH key, the `hetzner.hcloud` collection), individual OpenTofu (`init`/`plan`/`apply`/`output`/`destroy`) and Ansible (bootstrap/deploy/spinup/spindown per role) actions, and a **Guided full setup** that runs everything in the firewall-imposed order — vpn first, pause for you to connect the VPN, then dev/prod. It's only a wrapper around the same `tofu` and `ansible-playbook` commands documented below, so you can always fall back to running them by hand.
+
 ## Provisioning the infrastructure (`infra/`)
 
 ```sh
@@ -189,11 +200,11 @@ docker compose -f backup-docker-compose.yml exec backup backup   # force an imme
 docker compose -f backup-docker-compose.yml run --rm restore     # restore any currently-empty volumes
 ```
 
-### Scaling Gitea Actions runners
+### Gitea Actions runners
 
-The number of Gitea Actions runner containers on `dev` is set by `dev_runner_count` in [`ansible/group_vars/role_dev.yml`](ansible/group_vars/role_dev.yml) (default `3`). Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host: change `dev_runner_count` and redeploy rather than hand-editing it.
+`dev` always runs three Gitea Actions runner containers, configured automatically on deploy. Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — three `runner-N` services, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host, so edit the template rather than hand-editing the rendered file (the count lives in a `{% raw %}{% set runner_count = 3 %}{% endraw %}` line at the top).
 
-Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a count change.
+Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a template change.
 
 Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore.