feat: Swapped tofu init scripting for ansible scripting

.gitignore

@@ -13,6 +13,9 @@
 # services/docker-compose.yml carve-out above.
 **/.env
 
+### Ansible ###
+*.retry
+
 ### Terraform ###
 # Local .terraform directories
 **/.terraform/*

ansible/README.md

@@ -0,0 +1,77 @@
+# Ansible
+
+Bootstraps, syncs, and starts/stops the Docker Compose stacks in `../services/`
+on the three servers OpenTofu provisions (see `../infra/`). OpenTofu still owns
+the actual cloud resources (servers, network, firewalls, volumes, DNS);
+everything downstream of "the server exists" - installing Docker, creating the
+`deploy` user, hardening SSH, copying `services/<host>/`, rendering `.env` /
+the Gitea runner compose file, and running `spinup.sh`/`spindown.sh` - lives
+here instead.
+
+## Contents
+
+- `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API
+  directly (not Terraform state) and groups servers by their `role` label
+  (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules/<host>/main.tf`).
+- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, `HCLOUD_TOKEN`
+  lookup), `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` (per-role vars,
+  e.g. `dev_runner_count`).
+- `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening,
+  unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`.
+- `roles/deploy/` - copies `services/<host>/` to the server, looks up the
+  host's data volume directly from the Hetzner API and renders `.env`
+  (`DATA_DIR=...`) and, for `dev`, `Runners/docker-compose.yml`.
+- `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`,
+  and `site.yml` (all three in order).
+
+## Prerequisites
+
+```sh
+cd ansible
+ansible-galaxy collection install -r requirements.yml
+export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
+```
+
+`ansible_ssh_private_key_file` (see `group_vars/all.yml`) defaults to
+`~/.ssh/id_ed25519` - override with `ANSIBLE_SSH_PRIVATE_KEY_FILE` if you use a
+different key. It must match one of the SSH keys OpenTofu installed on the
+servers (`var.ssh_key_names`).
+
+## Usage
+
+Same chicken-and-egg as OpenTofu's own first-apply ordering (see
+`../readme.md`): `dev`/`prod`'s firewalls only accept SSH from `vpn`'s own
+public IP, so you must bring `vpn` up and connect to it before `dev`/`prod`
+are reachable at all.
+
+```sh
+# 1. vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly
+ansible-playbook playbooks/bootstrap.yml -l role_vpn
+ansible-playbook playbooks/deploy.yml    -l role_vpn
+ansible-playbook playbooks/spinup.yml    -l role_vpn
+
+# 2. connect to the VPN you just started with an OpenVPN client - now dev/prod
+#    are reachable, since your egress IP matches their firewall rule
+ansible-playbook playbooks/bootstrap.yml -l role_dev,role_prod
+ansible-playbook playbooks/deploy.yml    -l role_dev,role_prod
+ansible-playbook playbooks/spinup.yml    -l role_dev,role_prod
+
+# ...or once already bootstrapped and connected, do everything in one go:
+ansible-playbook playbooks/site.yml -l role_dev,role_prod
+```
+
+Re-run `deploy.yml` + `spinup.yml` any time a compose file changes or
+`dev_runner_count` in `group_vars/role_dev.yml` is bumped - `spinup.sh` only
+starts/updates what's changed, same as before.
+
+`spindown.yml` runs `spindown.sh`, which does a full `docker system/volume
+prune -a` on the host - it prompts for a `yes` confirmation before running,
+since the dynamic inventory makes it trivial to target multiple hosts at once
+(`-l role_dev,role_prod`).
+
+Inspect what the dynamic inventory currently sees:
+
+```sh
+ansible-inventory --graph
+ansible-inventory --list
+```

ansible/ansible.cfg

@@ -0,0 +1,8 @@
+[defaults]
+inventory = inventory/hcloud.yml
+host_key_checking = False
+retry_files_enabled = False
+interpreter_python = auto_silent
+
+[inventory]
+enable_plugins = hetzner.hcloud.hcloud, auto

ansible/group_vars/all.yml

@@ -0,0 +1,14 @@
+---
+# Keep in sync with infra/variables.tf's var.deploy_user (default "deploy") -
+# the two aren't wired together automatically, since Ansible no longer reads
+# any Terraform state or output.
+deploy_user: deploy
+ansible_user: "{{ deploy_user }}"
+ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}"
+
+# Used by the deploy role to look up each host's data volume directly from the
+# Hetzner API (hcloud_volume_info) rather than from Terraform state/outputs.
+hcloud_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+
+# Repo root's services/ directory, relative to wherever a playbook lives under ansible/playbooks/.
+services_root: "{{ playbook_dir }}/../../services"

ansible/group_vars/role_dev.yml

@@ -0,0 +1,7 @@
+---
+service_group: dev
+
+# Number of Gitea Actions runner containers rendered into
+# services/dev/Runners/docker-compose.yml by the deploy role - see
+# roles/deploy/templates/runners-docker-compose.yml.j2.
+dev_runner_count: 3

ansible/group_vars/role_prod.yml

@@ -0,0 +1,2 @@
+---
+service_group: prod

ansible/group_vars/role_vpn.yml

@@ -0,0 +1,2 @@
+---
+service_group: vpn

ansible/inventory/hcloud.yml

@@ -0,0 +1,12 @@
+---
+# Dynamic inventory: queries the Hetzner Cloud API directly (HCLOUD_TOKEN),
+# grouping hosts by the `role` label set on each hcloud_server resource in
+# infra/modules/<host>/main.tf (role = dev/prod/vpn). Nothing here reads
+# Terraform state - this always reflects whatever actually exists in Hetzner.
+plugin: hetzner.hcloud.hcloud
+token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+connect_with: public_ipv4
+keyed_groups:
+  - key: labels.role
+    prefix: role
+    separator: "_"

ansible/playbooks/bootstrap.yml

@@ -0,0 +1,9 @@
+---
+- name: Bootstrap newly created servers (idempotent, safe to re-run)
+  hosts: all
+  gather_facts: true
+  become: false
+  vars:
+    ansible_user: root
+  roles:
+    - bootstrap

ansible/playbooks/deploy.yml

@@ -0,0 +1,6 @@
+---
+- name: Render templates and sync service definitions to each host
+  hosts: all
+  gather_facts: false
+  roles:
+    - deploy

ansible/playbooks/site.yml

@@ -0,0 +1,4 @@
+---
+- ansible.builtin.import_playbook: bootstrap.yml
+- ansible.builtin.import_playbook: deploy.yml
+- ansible.builtin.import_playbook: spinup.yml

ansible/playbooks/spindown.yml

@@ -0,0 +1,27 @@
+---
+- name: Stop every compose stack on each host and prune Docker resources
+  hosts: all
+  gather_facts: false
+  vars_prompt:
+    - name: confirm
+      prompt: >-
+        This runs spindown.sh (docker compose down, then a full
+        docker image/volume prune -a) on the targeted host(s).
+        Type 'yes' to continue
+      private: false
+  tasks:
+    - name: Abort unless confirmed
+      ansible.builtin.fail:
+        msg: Aborted.
+      when: confirm != 'yes'
+
+    - name: Run spindown.sh
+      ansible.builtin.command: ./spindown.sh
+      args:
+        chdir: "/home/{{ deploy_user }}/services/{{ service_group }}"
+      register: _spindown
+      changed_when: true
+
+    - name: Show spindown output
+      ansible.builtin.debug:
+        var: _spindown.stdout_lines

ansible/playbooks/spinup.yml

@@ -0,0 +1,15 @@
+---
+- name: Start every compose stack on each host, in the order spinup.sh defines
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Run spinup.sh
+      ansible.builtin.command: ./spinup.sh
+      args:
+        chdir: "/home/{{ deploy_user }}/services/{{ service_group }}"
+      register: _spinup
+      changed_when: true
+
+    - name: Show spinup output
+      ansible.builtin.debug:
+        var: _spinup.stdout_lines

ansible/requirements.yml

@@ -0,0 +1,4 @@
+---
+collections:
+  - name: hetzner.hcloud
+    version: ">=4.0.0"

ansible/roles/bootstrap/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Reload sshd
+  ansible.builtin.systemd:
+    name: ssh
+    state: reloaded

ansible/roles/bootstrap/tasks/main.yml

@@ -0,0 +1,123 @@
+---
+# Post-install bootstrap - replaces infra/scripts/bootstrap.sh.tftpl. Runs as
+# root (see ansible_user override in playbooks/bootstrap.yml) since the deploy
+# user doesn't exist yet on a freshly created server. Idempotent: safe to
+# re-run against an already-bootstrapped host.
+
+- name: Update apt cache and upgrade packages
+  ansible.builtin.apt:
+    update_cache: true
+    upgrade: dist
+    cache_valid_time: 3600
+
+- name: Install Docker prerequisites
+  ansible.builtin.apt:
+    name:
+      - ca-certificates
+      - curl
+      - gnupg
+    state: present
+
+- name: Create apt keyrings directory
+  ansible.builtin.file:
+    path: /etc/apt/keyrings
+    state: directory
+    mode: "0755"
+
+- name: Download Docker's GPG key
+  ansible.builtin.get_url:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    dest: /etc/apt/keyrings/docker.asc
+    mode: "0644"
+    force: false
+
+- name: Determine dpkg architecture
+  ansible.builtin.command: dpkg --print-architecture
+  register: _dpkg_arch
+  changed_when: false
+
+- name: Add Docker apt repository
+  ansible.builtin.apt_repository:
+    repo: >-
+      deb [arch={{ _dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc]
+      https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable
+    filename: docker
+    state: present
+
+- name: Install Docker Engine and Compose plugin
+  ansible.builtin.apt:
+    update_cache: true
+    name:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+      - docker-buildx-plugin
+      - docker-compose-plugin
+    state: present
+
+- name: Create non-root sudo user, given the same key root was provisioned with
+  ansible.builtin.user:
+    name: "{{ deploy_user }}"
+    shell: /bin/bash
+    groups: sudo,docker
+    append: true
+    create_home: true
+
+- name: Ensure deploy user's .ssh directory exists
+  ansible.builtin.file:
+    path: "/home/{{ deploy_user }}/.ssh"
+    state: directory
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+    mode: "0700"
+
+- name: Give deploy user the same authorized_keys as root
+  ansible.builtin.copy:
+    src: /root/.ssh/authorized_keys
+    dest: "/home/{{ deploy_user }}/.ssh/authorized_keys"
+    remote_src: true
+    owner: "{{ deploy_user }}"
+    group: "{{ deploy_user }}"
+    mode: "0600"
+
+- name: Grant deploy user passwordless sudo
+  ansible.builtin.copy:
+    content: "{{ deploy_user }} ALL=(ALL) NOPASSWD:ALL\n"
+    dest: "/etc/sudoers.d/90-{{ deploy_user }}"
+    validate: "visudo -cf %s"
+    mode: "0440"
+
+# Root keeps key-only login as a fallback rather than being fully locked out,
+# in case deploy user setup above ever fails silently on a future image.
+- name: Restrict root login to key-only
+  ansible.builtin.lineinfile:
+    path: /etc/ssh/sshd_config
+    regexp: "^#?PermitRootLogin"
+    line: "PermitRootLogin prohibit-password"
+  notify: Reload sshd
+
+- name: Disable SSH password authentication
+  ansible.builtin.lineinfile:
+    path: /etc/ssh/sshd_config
+    regexp: "^#?PasswordAuthentication"
+    line: "PasswordAuthentication no"
+  notify: Reload sshd
+
+- name: Install unattended-upgrades
+  ansible.builtin.apt:
+    name: unattended-upgrades
+    state: present
+
+- name: Enable automatic security upgrades
+  ansible.builtin.copy:
+    dest: /etc/apt/apt.conf.d/20auto-upgrades
+    content: |
+      APT::Periodic::Update-Package-Lists "1";
+      APT::Periodic::Unattended-Upgrade "1";
+    mode: "0644"
+
+- name: Enable and start unattended-upgrades
+  ansible.builtin.systemd:
+    name: unattended-upgrades
+    enabled: true
+    state: started

ansible/roles/deploy/tasks/main.yml

@@ -0,0 +1,55 @@
+---
+# Copies services/<service_group>/ to the server and renders the files that
+# used to be generated by OpenTofu (services/<host>/.env and, for dev,
+# Runners/docker-compose.yml) - replaces the null_resource.deploy_services +
+# local_file provisioners previously in infra/modules/<host>/main.tf.
+
+- name: Ensure services directory exists on the host
+  ansible.builtin.file:
+    path: "/home/{{ deploy_user }}/services/{{ service_group }}"
+    state: directory
+    mode: "0755"
+
+- name: Copy static service files (compose files, scripts, configs)
+  ansible.builtin.copy:
+    src: "{{ services_root }}/{{ service_group }}/"
+    dest: "/home/{{ deploy_user }}/services/{{ service_group }}/"
+    mode: preserve
+
+- name: Look up this host's data volume
+  when: service_group in ['dev', 'prod']
+  delegate_to: localhost
+  become: false
+  hetzner.hcloud.hcloud_volume_info:
+    api_token: "{{ hcloud_token }}"
+    name: "{{ service_group }}-storage"
+  register: _volume_info
+
+# Hetzner's automount tooling always mounts an auto-mounted volume at
+# /mnt/HC_Volume_<volume-id> - deterministic once the volume exists.
+- name: Compute DATA_DIR from the volume's id
+  when: service_group in ['dev', 'prod']
+  ansible.builtin.set_fact:
+    data_dir: "/mnt/HC_Volume_{{ _volume_info.hcloud_volume_info[0].id }}"
+
+- name: Render DATA_DIR into .env (auto-loaded by docker compose)
+  when: service_group in ['dev', 'prod']
+  ansible.builtin.template:
+    src: env.j2
+    dest: "/home/{{ deploy_user }}/services/{{ service_group }}/.env"
+    mode: "0644"
+
+- name: Render Gitea Actions runner compose file
+  when: service_group == 'dev'
+  ansible.builtin.template:
+    src: runners-docker-compose.yml.j2
+    dest: "/home/{{ deploy_user }}/services/dev/Runners/docker-compose.yml"
+    mode: "0644"
+
+- name: Make spinup/spindown scripts executable
+  ansible.builtin.file:
+    path: "/home/{{ deploy_user }}/services/{{ service_group }}/{{ item }}"
+    mode: "0755"
+  loop:
+    - spinup.sh
+    - spindown.sh

ansible/roles/deploy/templates/env.j2

@@ -0,0 +1 @@
+DATA_DIR={{ data_dir }}

ansible/roles/deploy/templates/runners-docker-compose.yml.j2

@@ -1,34 +1,34 @@
-# Generated by OpenTofu from var.dev_runner_count - do not hand-edit.
+# Generated by Ansible from dev_runner_count - do not hand-edit.
-# To change the number of runners, edit dev_runner_count in terraform.tfvars and
+# To change the number of runners, edit dev_runner_count in
-# run tofu apply. To change their shape, edit
+# ansible/group_vars/role_dev.yml and re-run playbooks/deploy.yml.
-# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead.
+# To change their shape, edit this template instead.
 #
 # GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable
 # (not baked in here) - services/dev/spinup.sh generates a fresh token and
 # writes it to Runners/.env immediately before starting these containers.
 #
-# /data lives on dev's volume (${data_dir}) so runner identity survives a
+# /data lives on dev's volume ({{ data_dir }}) so runner identity survives a
 # server rebuild - baked in directly rather than via .env, since Runners/.env
 # is already reserved for the registration token above and gets overwritten
 # on every deploy.
 services:
-%{ for i in range(runner_count) ~}
+{% for i in range(1, dev_runner_count + 1) %}
-  runner-${i + 1}:
+  runner-{{ i }}:
     image: gitea/act_runner:latest
-    container_name: gitea_runner_${i + 1}
+    container_name: gitea_runner_{{ i }}
     volumes:
       - ./config.yaml:/config.yaml
-      - ${data_dir}/gitea_runner_${i + 1}:/data
+      - {{ data_dir }}/gitea_runner_{{ i }}:/data
       - /var/run/docker.sock:/var/run/docker.sock
     networks:
       - proxy
     environment:
       CONFIG_FILE: /config.yaml
       GITEA_INSTANCE_URL: "https://git.luke-else.co.uk"
-      GITEA_RUNNER_REGISTRATION_TOKEN: "$${GITEA_RUNNER_REGISTRATION_TOKEN}"
+      GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}"
-      GITEA_RUNNER_NAME: "CICD#${i + 1}"
+      GITEA_RUNNER_NAME: "CICD#{{ i }}"
     restart: unless-stopped
-%{ endfor ~}
+{% endfor %}
 
 networks:
   proxy:

infra/main.tf

@@ -9,12 +9,6 @@ locals {
   # var.allowed_ssh_source_ips for direct SSH access.
   vpn_ssh_source_ips = ["${module.vpn.ipv4}/32"]
 
-  # Rendered once and uploaded+run by every host module immediately after its
-  # server is created - see scripts/bootstrap.sh.tftpl.
-  bootstrap_script = templatefile("${path.module}/scripts/bootstrap.sh.tftpl", {
-    deploy_user = var.deploy_user
-  })
-
   # IDs for every named key in var.ssh_key_names - installed on every server.
   ssh_key_ids = [for k in data.hcloud_ssh_key.main : k.id]
 
@@ -59,10 +53,6 @@ module "dev" {
   volume_size            = var.dev_volume_size
   allowed_ssh_source_ips = local.vpn_ssh_source_ips
   network_ip_range       = var.network_ip_range
-  bootstrap_script       = local.bootstrap_script
-  ssh_private_key_path   = var.ssh_private_key_path
-  deploy_user            = var.deploy_user
-  runner_count           = var.dev_runner_count
 
   # module.network.id alone doesn't guarantee the subnet exists yet, and a server
   # can't join a network before it has a subnet.
@@ -81,9 +71,6 @@ module "prod" {
   volume_size            = var.prod_volume_size
   allowed_ssh_source_ips = local.vpn_ssh_source_ips
   network_ip_range       = var.network_ip_range
-  bootstrap_script       = local.bootstrap_script
-  ssh_private_key_path   = var.ssh_private_key_path
-  deploy_user            = var.deploy_user
 
   depends_on = [module.network]
 }
@@ -96,9 +83,6 @@ module "vpn" {
   location               = var.location
   ssh_key_ids            = local.ssh_key_ids
   allowed_ssh_source_ips = var.allowed_ssh_source_ips
-  bootstrap_script       = local.bootstrap_script
-  ssh_private_key_path   = var.ssh_private_key_path
-  deploy_user            = var.deploy_user
 }
 
 module "dns" {

infra/modules/dev/main.tf

@@ -56,30 +56,12 @@ resource "hcloud_server" "this" {
   location     = var.location
   ssh_keys     = var.ssh_key_ids
   firewall_ids = [hcloud_firewall.this.id]
+  labels       = { role = "dev" } # picked up by ansible/inventory/hcloud.yml
 
   network {
     network_id = var.network_id
     ip         = var.private_ip
   }
-
-  connection {
-    type        = "ssh"
-    host        = self.ipv4_address
-    user        = "root"
-    private_key = file(var.ssh_private_key_path)
-  }
-
-  provisioner "file" {
-    content     = var.bootstrap_script
-    destination = "/root/bootstrap.sh"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "chmod +x /root/bootstrap.sh",
-      "/root/bootstrap.sh",
-    ]
-  }
 }
 
 resource "hcloud_volume" "storage" {
@@ -98,85 +80,8 @@ locals {
   # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is
   # always /mnt/HC_Volume_<id> - deterministic once the volume exists, since the
   # id is stable for the volume's lifetime regardless of which server it's
-  # attached to.
+  # attached to. Kept here only as a human-readable output (see outputs.tf) -
+  # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't
+  # read this value.
   data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}"
 }
-
-# Renders services/dev/Runners/docker-compose.yml directly into the repo, with
-# one runner service per var.runner_count. This only touches the local working
-# tree - deploying the change still goes through the normal scp + spinup.sh flow.
-resource "local_file" "runners_compose" {
-  filename = "${path.root}/../services/dev/Runners/docker-compose.yml"
-  content = templatefile("${path.module}/templates/runners-docker-compose.yml.tftpl", {
-    runner_count = var.runner_count
-    data_dir     = local.data_dir
-  })
-  file_permission = "0644"
-}
-
-# DATA_DIR is picked up automatically by `docker compose` (which auto-loads
-# .env from its working directory) for every dev/*-docker-compose.yml bind
-# mount - see services/dev/*.yml. Not committed (see .gitignore): it's tied to
-# the live volume's ID, so it's regenerated by tofu apply, not handed off via git.
-resource "local_file" "env" {
-  filename        = "${path.root}/../services/dev/.env"
-  content         = "DATA_DIR=${local.data_dir}\n"
-  file_permission = "0644"
-}
-
-locals {
-  # Everything under services/dev except the two files above: those are
-  # tracked via their own resource content (reading them back with fileset()/
-  # filesha1() before they exist would fail during `tofu plan`).
-  dev_static_files = sort([
-    for f in fileset("${path.root}/../services/dev", "**") :
-    f if f != ".env" && f != "Runners/docker-compose.yml"
-  ])
-
-  dev_static_files_hash = sha1(join("", [
-    for f in local.dev_static_files : filesha1("${path.root}/../services/dev/${f}")
-  ]))
-}
-
-# Copies services/dev to the server on every apply that changes it (a hand-edited
-# compose file, a new runner count, ...) or recreates the server - not just once
-# at first creation. Split from hcloud_server.this because it must run after the
-# generated files above, which in turn depend on the volume, which depends on
-# the server - so it can't be a provisioner on the server resource itself.
-resource "null_resource" "deploy_services" {
-  triggers = {
-    server_id    = hcloud_server.this.id
-    static_files = local.dev_static_files_hash
-    env          = local_file.env.content
-    runners      = local_file.runners_compose.content
-  }
-
-  connection {
-    type        = "ssh"
-    host        = hcloud_server.this.ipv4_address
-    user        = "root"
-    private_key = file(var.ssh_private_key_path)
-  }
-
-  provisioner "remote-exec" {
-    inline = ["mkdir -p /home/${var.deploy_user}/services"]
-  }
-
-  provisioner "file" {
-    source      = "${path.root}/../services/dev"
-    destination = "/home/${var.deploy_user}/services"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services",
-      "chmod +x /home/${var.deploy_user}/services/dev/*.sh",
-    ]
-  }
-
-  depends_on = [
-    hcloud_server.this,
-    local_file.env,
-    local_file.runners_compose,
-  ]
-}

infra/modules/dev/variables.tf

@@ -42,23 +42,3 @@ variable "network_ip_range" {
   description = "CIDR of the private network, allowed through the firewall for traffic from prod."
   type        = string
 }
-
-variable "bootstrap_script" {
-  description = "Rendered post-install script, uploaded and executed on the server immediately after creation."
-  type        = string
-}
-
-variable "ssh_private_key_path" {
-  description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH."
-  type        = string
-}
-
-variable "runner_count" {
-  description = "Number of Gitea Actions runner containers to render into services/dev/Runners/docker-compose.yml."
-  type        = number
-}
-
-variable "deploy_user" {
-  description = "Non-root sudo user created by the bootstrap script - services/dev is copied into this user's home directory."
-  type        = string
-}

infra/modules/prod/main.tf

@@ -83,30 +83,12 @@ resource "hcloud_server" "this" {
   location     = var.location
   ssh_keys     = var.ssh_key_ids
   firewall_ids = [hcloud_firewall.this.id]
+  labels       = { role = "prod" } # picked up by ansible/inventory/hcloud.yml
 
   network {
     network_id = var.network_id
     ip         = var.private_ip
   }
-
-  connection {
-    type        = "ssh"
-    host        = self.ipv4_address
-    user        = "root"
-    private_key = file(var.ssh_private_key_path)
-  }
-
-  provisioner "file" {
-    content     = var.bootstrap_script
-    destination = "/root/bootstrap.sh"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "chmod +x /root/bootstrap.sh",
-      "/root/bootstrap.sh",
-    ]
-  }
 }
 
 resource "hcloud_volume" "storage" {
@@ -125,70 +107,8 @@ locals {
   # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is
   # always /mnt/HC_Volume_<id> - deterministic once the volume exists, since the
   # id is stable for the volume's lifetime regardless of which server it's
-  # attached to.
+  # attached to. Kept here only as a human-readable output (see outputs.tf) -
+  # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't
+  # read this value.
   data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}"
 }
-
-# DATA_DIR is picked up automatically by `docker compose` (which auto-loads
-# .env from its working directory) for every prod/*-docker-compose.yml bind
-# mount - see services/prod/*.yml. Not committed (see .gitignore): it's tied to
-# the live volume's ID, so it's regenerated by tofu apply, not handed off via git.
-resource "local_file" "env" {
-  filename        = "${path.root}/../services/prod/.env"
-  content         = "DATA_DIR=${local.data_dir}\n"
-  file_permission = "0644"
-}
-
-locals {
-  # Everything under services/prod except .env: that's tracked via its own
-  # resource content (reading it back with fileset()/filesha1() before it
-  # exists would fail during `tofu plan`).
-  prod_static_files = sort([
-    for f in fileset("${path.root}/../services/prod", "**") : f if f != ".env"
-  ])
-
-  prod_static_files_hash = sha1(join("", [
-    for f in local.prod_static_files : filesha1("${path.root}/../services/prod/${f}")
-  ]))
-}
-
-# Copies services/prod to the server on every apply that changes it (a
-# hand-edited compose file, ...) or recreates the server - not just once at
-# first creation. Split from hcloud_server.this because it must run after
-# local_file.env, which in turn depends on the volume, which depends on the
-# server - so it can't be a provisioner on the server resource itself.
-resource "null_resource" "deploy_services" {
-  triggers = {
-    server_id    = hcloud_server.this.id
-    static_files = local.prod_static_files_hash
-    env          = local_file.env.content
-  }
-
-  connection {
-    type        = "ssh"
-    host        = hcloud_server.this.ipv4_address
-    user        = "root"
-    private_key = file(var.ssh_private_key_path)
-  }
-
-  provisioner "remote-exec" {
-    inline = ["mkdir -p /home/${var.deploy_user}/services"]
-  }
-
-  provisioner "file" {
-    source      = "${path.root}/../services/prod"
-    destination = "/home/${var.deploy_user}/services"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services",
-      "chmod +x /home/${var.deploy_user}/services/prod/*.sh",
-    ]
-  }
-
-  depends_on = [
-    hcloud_server.this,
-    local_file.env,
-  ]
-}

infra/modules/prod/variables.tf

@@ -42,18 +42,3 @@ variable "network_ip_range" {
   description = "CIDR of the private network, allowed through the firewall for traffic from dev."
   type        = string
 }
-
-variable "bootstrap_script" {
-  description = "Rendered post-install script, uploaded and executed on the server immediately after creation."
-  type        = string
-}
-
-variable "ssh_private_key_path" {
-  description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH."
-  type        = string
-}
-
-variable "deploy_user" {
-  description = "Non-root sudo user created by the bootstrap script - services/prod is copied into this user's home directory."
-  type        = string
-}

infra/modules/vpn/main.tf

@@ -38,67 +38,5 @@ resource "hcloud_server" "this" {
   location     = var.location
   ssh_keys     = var.ssh_key_ids
   firewall_ids = [hcloud_firewall.this.id]
-
+  labels       = { role = "vpn" } # picked up by ansible/inventory/hcloud.yml
-  connection {
-    type        = "ssh"
-    host        = self.ipv4_address
-    user        = "root"
-    private_key = file(var.ssh_private_key_path)
-  }
-
-  provisioner "file" {
-    content     = var.bootstrap_script
-    destination = "/root/bootstrap.sh"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "chmod +x /root/bootstrap.sh",
-      "/root/bootstrap.sh",
-    ]
-  }
-}
-
-locals {
-  vpn_static_files = sort(fileset("${path.root}/../services/vpn", "**"))
-
-  vpn_static_files_hash = sha1(join("", [
-    for f in local.vpn_static_files : filesha1("${path.root}/../services/vpn/${f}")
-  ]))
-}
-
-# Copies services/vpn to the server on every apply that changes it (a
-# hand-edited compose file, ...) or recreates the server - not just once at
-# first creation. Split from hcloud_server.this so it's triggered by content
-# changes independently of the bootstrap provisioners above.
-resource "null_resource" "deploy_services" {
-  triggers = {
-    server_id    = hcloud_server.this.id
-    static_files = local.vpn_static_files_hash
-  }
-
-  connection {
-    type        = "ssh"
-    host        = hcloud_server.this.ipv4_address
-    user        = "root"
-    private_key = file(var.ssh_private_key_path)
-  }
-
-  provisioner "remote-exec" {
-    inline = ["mkdir -p /home/${var.deploy_user}/services"]
-  }
-
-  provisioner "file" {
-    source      = "${path.root}/../services/vpn"
-    destination = "/home/${var.deploy_user}/services"
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services",
-      "chmod +x /home/${var.deploy_user}/services/vpn/*.sh",
-    ]
-  }
-
-  depends_on = [hcloud_server.this]
 }

infra/modules/vpn/variables.tf

@@ -22,18 +22,3 @@ variable "allowed_ssh_source_ips" {
   description = "CIDRs allowed to reach port 22 on vpn."
   type        = list(string)
 }
-
-variable "bootstrap_script" {
-  description = "Rendered post-install script, uploaded and executed on the server immediately after creation."
-  type        = string
-}
-
-variable "ssh_private_key_path" {
-  description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH."
-  type        = string
-}
-
-variable "deploy_user" {
-  description = "Non-root sudo user created by the bootstrap script - services/vpn is copied into this user's home directory."
-  type        = string
-}

infra/scripts/bootstrap.sh.tftpl

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-# Post-install bootstrap - uploaded and executed once by OpenTofu immediately
-# after server creation (see modules/<host>/main.tf). Idempotent-ish: safe to
-# re-run by hand, but only ever runs automatically on first create.
-set -euo pipefail
-
-export DEBIAN_FRONTEND=noninteractive
-DEPLOY_USER="${deploy_user}"
-
-apt-get update -y
-apt-get upgrade -y
-
-# --- Docker Engine + Compose plugin ---
-if ! command -v docker >/dev/null 2>&1; then
-  apt-get install -y ca-certificates curl gnupg
-  install -m 0755 -d /etc/apt/keyrings
-  curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
-  chmod a+r /etc/apt/keyrings/docker.asc
-  . /etc/os-release
-  echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $VERSION_CODENAME stable" \
-    > /etc/apt/sources.list.d/docker.list
-  apt-get update -y
-  apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-fi
-
-# --- Non-root sudo user, given the same key root was provisioned with ---
-if ! id -u "$DEPLOY_USER" >/dev/null 2>&1; then
-  useradd --create-home --shell /bin/bash "$DEPLOY_USER"
-  usermod -aG sudo,docker "$DEPLOY_USER"
-  mkdir -p "/home/$DEPLOY_USER/.ssh"
-  cp /root/.ssh/authorized_keys "/home/$DEPLOY_USER/.ssh/authorized_keys"
-  chown -R "$DEPLOY_USER:$DEPLOY_USER" "/home/$DEPLOY_USER/.ssh"
-  chmod 700 "/home/$DEPLOY_USER/.ssh"
-  chmod 600 "/home/$DEPLOY_USER/.ssh/authorized_keys"
-  echo "$DEPLOY_USER ALL=(ALL) NOPASSWD:ALL" > "/etc/sudoers.d/90-$DEPLOY_USER"
-  chmod 440 "/etc/sudoers.d/90-$DEPLOY_USER"
-fi
-
-# --- SSH hardening: key-only auth everywhere. Root keeps key-only login as a
-# fallback rather than being fully locked out, in case $DEPLOY_USER setup above
-# ever fails silently on a future image. ---
-sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
-sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config
-systemctl reload ssh 2>/dev/null || systemctl reload sshd
-
-# --- Unattended security upgrades ---
-apt-get install -y unattended-upgrades
-dpkg-reconfigure -f noninteractive unattended-upgrades
-systemctl enable --now unattended-upgrades

infra/terraform.tfvars.example

@@ -2,18 +2,13 @@
 # never commit real values there.
 #
 # The Hetzner API token is NOT set here: export it as HCLOUD_TOKEN in your shell
-# before running tofu plan/apply.
+# before running tofu plan/apply. The same token and SSH key are used again by
+# Ansible afterwards - see ../ansible/README.md.
 
 # Names of SSH keys already uploaded to your Hetzner Cloud project
 # (Console > Security > SSH Keys). All of them are installed on every server. Required.
 ssh_key_names = ["your-key-name", "laptop", "phone"]
 
-# Local path to the private key matching ONE of ssh_key_names above (doesn't need
-# to be all of them - just one you hold). OpenTofu uses this once per server to
-# upload and run the post-install bootstrap script
-# (infra/scripts/bootstrap.sh.tftpl) immediately after creation. Required.
-ssh_private_key_path = "~/.ssh/id_ed25519"
-
 # Optional overrides - defaults live in variables.tf
 # location               = "nbg1"
 # dev_server_type        = "cx22"
@@ -22,6 +17,4 @@ ssh_private_key_path = "~/.ssh/id_ed25519"
 # dev_volume_size        = 20
 # prod_volume_size       = 20
 # allowed_ssh_source_ips = ["203.0.113.4/32"]
-# deploy_user             = "deploy"
-# dev_runner_count       = 3
 # dns_zones              = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"]

infra/variables.tf

@@ -81,23 +81,6 @@ variable "allowed_ssh_source_ips" {
   default     = ["0.0.0.0/0", "::/0"]
 }
 
-variable "ssh_private_key_path" {
-  description = "Local path to the private key matching one of var.ssh_key_names - only needs to be a key you hold yourself, not all of them. Used once per server by OpenTofu to upload and run the post-install bootstrap script over SSH immediately after creation (see infra/scripts/bootstrap.sh.tftpl)."
-  type        = string
-}
-
-variable "deploy_user" {
-  description = "Non-root sudo user created on every server by the bootstrap script, with the same SSH key as root."
-  type        = string
-  default     = "deploy"
-}
-
-variable "dev_runner_count" {
-  description = "Number of Gitea Actions runner containers to run on dev. Rendered into services/dev/Runners/docker-compose.yml on every tofu apply - see infra/modules/dev/templates/runners-docker-compose.yml.tftpl."
-  type        = number
-  default     = 1
-}
-
 variable "dns_zones" {
   description = "Domains to manage as Hetzner DNS zones. Point each domain's registrar NS records at tofu output dns_nameservers for Hetzner to actually become authoritative."
   type        = list(string)

infra/versions.tf

@@ -6,14 +6,6 @@ terraform {
       source  = "hetznercloud/hcloud"
       version = "~> 1.54" # hcloud_zone / hcloud_zone_rrset (DNS) require >= 1.54.0
     }
-    local = {
-      source  = "hashicorp/local"
-      version = "~> 2.5"
-    }
-    null = {
-      source  = "hashicorp/null"
-      version = "~> 3.2"
-    }
   }
 }
 

readme.md

@@ -1,6 +1,6 @@
 # Server
 
-Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).
+Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each bootstrapped and deployed with [Ansible](https://www.ansible.com/), running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).
 
 <p align="center">
   <img src="assets/images/main.png" width="70%">
@@ -12,7 +12,7 @@ Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted
 - [Repository layout](#repository-layout)
 - [Prerequisites](#prerequisites)
 - [Provisioning the infrastructure](#provisioning-the-infrastructure-infra)
-- [Deploying the services](#deploying-the-services-services)
+- [Bootstrapping and deploying with Ansible](#bootstrapping-and-deploying-with-ansible-ansible)
 - [Service inventory](#service-inventory)
 - [First-time setup](#first-time-setup)
 - [Development container](#development-container)
@@ -66,23 +66,26 @@ architecture-beta
 
 ```
 .
-├── infra/                    # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes
+├── infra/                    # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes, DNS
-│   ├── main.tf                # root module: wires network + dev/prod/vpn modules together
+│   ├── main.tf                # root module: wires network + dev/prod/vpn/dns modules together
 │   ├── variables.tf           # shared inputs (sizes, locations, IP ranges, SSH key names)
 │   ├── outputs.tf             # pass-through outputs from each host module
 │   ├── ssh.tf                 # looks up each SSH key already uploaded to Hetzner Cloud
 │   ├── versions.tf            # provider requirements
 │   ├── terraform.tfvars.example
-│   ├── scripts/
-│   │   └── bootstrap.sh.tftpl # post-install script run on every host right after creation
 │   └── modules/
 │       ├── network/           # shared private network + subnet (used by dev and prod)
-│       ├── dev/                # dev server + firewall + volume, renders + deploys services/dev/
+│       ├── dev/                # dev server + firewall + volume, labeled role=dev
-│       │   └── templates/
+│       ├── prod/               # prod server + firewall + volume, labeled role=prod
-│       │       └── runners-docker-compose.yml.tftpl # shape of each generated runner service
+│       ├── vpn/                # vpn server + firewall (no private network, no volume), labeled role=vpn
-│       ├── prod/               # prod server + firewall + volume, deploys services/prod/
-│       ├── vpn/                # vpn server + firewall (no private network, no volume), deploys services/vpn/
 │       └── dns/                # Hetzner DNS zones + A records for every domain in var.dns_zones
+├── ansible/                   # Ansible: bootstraps each server and deploys/starts services/<host>/ onto it
+│   ├── inventory/hcloud.yml   # dynamic inventory — queries the Hetzner API, groups by the role label above
+│   ├── group_vars/            # deploy_user, dev_runner_count, etc.
+│   ├── roles/
+│   │   ├── bootstrap/          # Docker, deploy user, sshd hardening, unattended-upgrades
+│   │   └── deploy/             # copies services/<host>/, renders .env + Runners/docker-compose.yml
+│   └── playbooks/              # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml
 ├── services/                 # Docker Compose stacks, grouped by which server they run on
 │   ├── dev/                   # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk)
 │   ├── prod/                  # Websites, Bitwarden, RustDesk, status page + Traefik
@@ -94,16 +97,19 @@ architecture-beta
 └── assets/
 ```
 
-Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` mirrors this same dev/prod/vpn split on the provisioning side, so a given host's cloud resources and its compose stacks are easy to find side by side.
+Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side.
+
+OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, volumes, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services/<host>/`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown.
 
 ## Prerequisites
 
 - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token
-- One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (OpenTofu uses it once per server to run the post-install bootstrap — see below)
+- One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (Ansible uses it to bootstrap and deploy — see below)
 - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0`
+- [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`)
 - Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner (see [Managing DNS](#managing-dns) — the zones and records themselves are created for you)
 
-Docker + the Compose plugin no longer need installing by hand — the bootstrap script below handles that.
+Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below).
 
 ## Provisioning the infrastructure (`infra/`)
 
@@ -111,7 +117,7 @@ Docker + the Compose plugin no longer need installing by hand — the bootstrap
 cd infra
 export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
 cp terraform.tfvars.example terraform.tfvars
-$EDITOR terraform.tfvars                     # set ssh_key_names and ssh_private_key_path at minimum
+$EDITOR terraform.tfvars                     # set ssh_key_names at minimum
 
 tofu init
 tofu plan
@@ -120,52 +126,15 @@ tofu apply
 
 This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` / `module.dns` in `infra/main.tf`:
 - `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`)
-- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses (`modules/dev`, `modules/prod`, `modules/vpn`)
+- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses, each server labeled `role = "dev"/"prod"/"vpn"` for Ansible's dynamic inventory (`modules/dev`, `modules/prod`, `modules/vpn`)
 - one `hcloud_volume` each for `dev` and `prod` (`modules/dev`, `modules/prod`; `vpn` has none)
 - one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns))
 
-Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`, `tofu output dev_data_dir`, `tofu output prod_data_dir`.
+Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`, `tofu output dev_data_dir`, `tofu output prod_data_dir` (the last two are informational only now — Ansible looks the data directory up itself, see below).
 
 `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`.
 
-### Post-install bootstrap
+OpenTofu never connects to the servers over SSH — no provisioners, no `bootstrap.sh`, no copying `services/` — that's all Ansible now. See below.
-
-Immediately after each `hcloud_server` is created, OpenTofu uploads and runs [`infra/scripts/bootstrap.sh.tftpl`](infra/scripts/bootstrap.sh.tftpl) over SSH as `root` (`connection` + `file`/`remote-exec` provisioners in each `modules/<host>/main.tf`). It:
-
-- installs Docker Engine + the Compose plugin
-- creates a non-root sudo user (`var.deploy_user`, default `deploy`) with the same SSH keys as root, in the `sudo` and `docker` groups
-- hardens `sshd`: password authentication off, root login restricted to key-only (`PermitRootLogin prohibit-password`)
-- installs and enables `unattended-upgrades` for automatic security patches
-
-After bootstrap, SSH in as `var.deploy_user` for day-to-day work — root key-based login still works as a fallback.
-
-### Persistent data lives on the volumes, not the server disk
-
-`dev` and `prod`'s compose files used to bind-mount container data to paths relative to wherever `services/<host>/` happened to be copied on the server (e.g. `./gitea:/data`) - i.e. the server's local disk, which doesn't survive the server being destroyed and recreated. `dev-storage` and `prod-storage` (the `hcloud_volume`s) do survive that, so all stateful bind mounts now point there instead: `${DATA_DIR}/gitea:/data`, `${DATA_DIR}/bitwarden/:/data/`, and so on across `services/dev/*.yml` and `services/prod/*.yml`. `vpn` has no volume, so it's untouched - see the architecture table above.
-
-`DATA_DIR` needing to resolve to the right path is what makes this work, and that's answered deterministically: Hetzner's automount tooling always mounts an auto-mounted volume at `/mnt/HC_Volume_<volume-id>`, and since the volume is a Terraform resource, `modules/<host>/main.tf` already knows its `id` the moment it's created - `local.data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}"`. Every `tofu apply` writes that value straight into `services/dev/.env` / `services/prod/.env` as `DATA_DIR=...`, which `docker compose` auto-loads for variable substitution in any compose file run from that directory (the same mechanism the Gitea runner token already uses). Query it directly with `tofu output dev_data_dir` / `tofu output prod_data_dir`.
-
-These `.env` files are deliberately not committed (see `.gitignore`): the path is tied to the *live* volume's ID, so it's regenerated by `tofu apply`, not handed off via git. Run `tofu apply` at least once before the first deploy, and again if a volume is ever destroyed and recreated (new ID); `spinup.sh` on both hosts refuses to start anything if `.env` is missing, rather than silently falling back to a relative path.
-
-The Gitea Actions runners are the one exception to the `.env` mechanism: their `/data` path is baked directly into the generated `Runners/docker-compose.yml` at render time (via the same `templatefile()` call as `dev_runner_count`), because `Runners/.env` is already reserved for the registration token and gets overwritten on every deploy.
-
-### Scaling Gitea Actions runners
-
-The number of Gitea Actions runner containers on `dev` is an OpenTofu input, `var.dev_runner_count` (default `3`). On every `tofu apply`, `modules/dev`'s `local_file.runners_compose` resource renders [`modules/dev/templates/runners-docker-compose.yml.tftpl`](infra/modules/dev/templates/runners-docker-compose.yml.tftpl) straight into [`services/dev/Runners/docker-compose.yml`](services/dev/Runners/docker-compose.yml) in your working tree — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated: change `dev_runner_count` in `terraform.tfvars` and re-run `tofu apply` rather than hand-editing it.
-
-This only updates your local working tree — `tofu apply` doesn't reach out and start containers on `dev` itself. Redeploy as usual (re-copy `services/dev/` to the server and re-run `spinup.sh`) to apply a count change.
-
-Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore.
-
-### First apply: bootstrapping order matters
-
-`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch `tofu apply`, the `dev`/`prod` bootstrap provisioners can't connect yet. Bring the estate up in this order:
-
-1. `tofu apply -target=module.vpn` — creates `vpn` only; its firewall still allows SSH from `var.allowed_ssh_source_ips`, so its bootstrap provisioner and `null_resource.deploy_services` (which copies `services/vpn` over) both run fine.
-2. SSH in as `var.deploy_user` and run `~/services/vpn/spinup.sh` to start the VPN service, then connect to it with an OpenVPN client.
-3. `tofu apply` — creates `network`, `dev`, `prod`. Now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and the `dev`/`prod` bootstrap + deploy provisioners can connect.
-
-If step 3 is run before you're connected to the VPN, the `dev`/`prod` provisioners will fail and Terraform will mark those servers tainted — re-running `tofu apply` once connected will recreate and re-bootstrap them.
 
 ### Managing DNS
 
@@ -177,9 +146,53 @@ Adding a new subdomain: add an entry to `local.dns_records` in `infra/main.tf` (
 
 Only A records for IPv4 are managed here — none of the modules currently track servers' IPv6 addresses, so AAAA records aren't generated even though the firewalls already allow IPv6 traffic.
 
+## Bootstrapping and deploying with Ansible (`ansible/`)
+
+Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services/<host>/` to each server, rendering the files OpenTofu used to generate (`.env`'s `DATA_DIR`, `dev`'s `Runners/docker-compose.yml`), and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version:
+
+```sh
+cd ansible
+ansible-galaxy collection install -r requirements.yml
+export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
+
+# vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly
+ansible-playbook playbooks/site.yml -l role_vpn
+
+# SSH to vpn as deploy and connect to the OpenVPN it just started, then:
+ansible-playbook playbooks/site.yml -l role_dev,role_prod
+```
+
+Hosts are discovered dynamically from the Hetzner API (`ansible/inventory/hcloud.yml`), grouped into `role_dev`/`role_prod`/`role_vpn` by the `role` label OpenTofu sets on each server — there's no static inventory file to keep in sync, and nothing here reads Terraform state.
+
+### Persistent data lives on the volumes, not the server disk
+
+`dev` and `prod`'s compose files bind-mount container data to paths on the `dev-storage` / `prod-storage` volumes rather than the server's local disk, so it survives the server being destroyed and recreated: `${DATA_DIR}/gitea:/data`, `${DATA_DIR}/bitwarden/:/data/`, and so on across `services/dev/*.yml` and `services/prod/*.yml`. `vpn` has no volume, so it's untouched - see the architecture table above.
+
+`DATA_DIR` resolves deterministically: Hetzner always automounts a volume at `/mnt/HC_Volume_<volume-id>`. Ansible's `deploy` role looks the volume up by name (`hcloud_volume_info` for `dev-storage`/`prod-storage`) directly against the Hetzner API and renders that path into `services/<host>/.env` as `DATA_DIR=...` on the remote host — `docker compose` auto-loads `.env` from its working directory for variable substitution. Nothing is written locally; re-run `ansible-playbook playbooks/deploy.yml` if a volume is ever destroyed and recreated (new id). `spinup.sh` on both hosts refuses to start anything if `.env` is missing, rather than silently falling back to a relative path.
+
+The Gitea Actions runners are the one exception to the `.env` mechanism: their `/data` path is baked directly into the rendered `Runners/docker-compose.yml`, because `Runners/.env` is already reserved for the registration token and gets overwritten on every deploy.
+
+### Scaling Gitea Actions runners
+
+The number of Gitea Actions runner containers on `dev` is set by `dev_runner_count` in [`ansible/group_vars/role_dev.yml`](ansible/group_vars/role_dev.yml) (default `3`). Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host: change `dev_runner_count` and redeploy rather than hand-editing it.
+
+Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a count change.
+
+Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore.
+
+### First deploy: bootstrapping order matters
+
+`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch estate, Ansible can't reach `dev`/`prod` yet. Bring it up in this order:
+
+1. `ansible-playbook playbooks/site.yml -l role_vpn` — bootstraps, deploys, and starts `vpn` only; its firewall allows SSH from `var.allowed_ssh_source_ips` directly.
+2. SSH in as `deploy` and connect to the OpenVPN service you just started with an OpenVPN client.
+3. `ansible-playbook playbooks/site.yml -l role_dev,role_prod` — now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and `dev`/`prod` become reachable.
+
+If step 3 is run before you're connected to the VPN, Ansible will simply fail to connect — re-run it once connected.
+
 ## Deploying the services (`services/`)
 
-`tofu apply` already puts `services/<host>/` on the matching server for you — a `null_resource.deploy_services` in each `modules/<host>/main.tf` copies it to `/home/<var.deploy_user>/services/<host>` and `chown`s it to `var.deploy_user`, re-running whenever the server is recreated or the directory's contents change (a hand-edited compose file, a new `dev_runner_count`, a refreshed `DATA_DIR`, ...) - not just once at first creation. So after applying, just SSH in as `var.deploy_user` and run the matching script from inside that directory:
+After `ansible-playbook playbooks/deploy.yml` has copied `services/<host>/` to `/home/deploy/services/<host>` on the matching server, SSH in as `deploy` and run the matching script from inside that directory — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do exactly this remotely (see [`ansible/README.md`](ansible/README.md)):
 
 ```sh
 # on dev
@@ -195,7 +208,7 @@ Only A records for IPv4 are managed here — none of the modules currently track
 ./spindown.sh
 ```
 
-If you ever need to force a re-sync without going through Terraform (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@<prod_ipv4>:~/services` still works fine - `tofu apply` will just overwrite it again next time triggers change.
+If you ever need to force a re-sync without going through Ansible (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@<prod_ipv4>:~/services` still works fine - `ansible-playbook playbooks/deploy.yml` will just overwrite it again next time it's run.
 
 Each stack can also be managed individually with plain Compose, e.g.:
 
@@ -243,7 +256,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi
 
 A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here:
 
-- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by the [post-install bootstrap](#post-install-bootstrap); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)).
+- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by [Ansible's `bootstrap` role](ansible/roles/bootstrap/tasks/main.yml); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)).
 
 ## Development container
 
@@ -261,5 +274,5 @@ Then reopen the repo in VS Code with the Dev Containers extension.
 - SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s).
 - `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP.
 - Firewalls are allowlists scoped per host in `infra/modules/<host>/main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open.
-- Every host's [post-install bootstrap](#post-install-bootstrap) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`var.deploy_user`) for day-to-day access.
+- Every host's [Ansible bootstrap role](ansible/roles/bootstrap/tasks/main.yml) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`deploy_user`, see `ansible/group_vars/all.yml`) for day-to-day access.
 - DNS zones (`modules/dns`) carry both Hetzner's own `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, including for `divine-couture.co.uk` and `snexo.co.uk`, not just `luke-else.co.uk`.

services/dev/Runners/docker-compose.yml

@@ -1,33 +0,0 @@
-# Generated by OpenTofu from var.dev_runner_count - do not hand-edit.
-# To change the number of runners, edit dev_runner_count in terraform.tfvars and
-# run tofu apply. To change their shape, edit
-# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead.
-#
-# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable
-# (not baked in here) - services/dev/spinup.sh generates a fresh token and
-# writes it to Runners/.env immediately before starting these containers.
-#
-# /data lives on dev's volume (/mnt/HC_Volume_PENDING_TOFU_APPLY) so runner
-# identity survives a server rebuild - baked in directly rather than via .env,
-# since Runners/.env is already reserved for the registration token above and
-# gets overwritten on every deploy.
-services:
-  runner-1:
-    image: gitea/act_runner:latest
-    container_name: gitea_runner_1
-    volumes:
-      - ./config.yaml:/config.yaml
-      - /mnt/HC_Volume_PENDING_TOFU_APPLY/gitea_runner_1:/data
-      - /var/run/docker.sock:/var/run/docker.sock
-    networks:
-      - proxy
-    environment:
-      CONFIG_FILE: /config.yaml
-      GITEA_INSTANCE_URL: "https://git.luke-else.co.uk"
-      GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}"
-      GITEA_RUNNER_NAME: "CICD#1"
-    restart: unless-stopped
-
-networks:
-  proxy:
-    external: true

services/dev/spinup.sh

@@ -4,7 +4,7 @@ set -e
 cd "$(dirname "$0")"
 
 if [ ! -f .env ]; then
-  echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'tofu apply' in infra/ first, then re-copy this directory." >&2
+  echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
   exit 1
 fi
 

services/prod/spinup.sh

@@ -4,7 +4,7 @@ set -e
 cd "$(dirname "$0")"
 
 if [ ! -f .env ]; then
-  echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'tofu apply' in infra/ first, then re-copy this directory." >&2
+  echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2
   exit 1
 fi