chore: committing missing files from previous logic changes

.gitignore

@@ -8,6 +8,11 @@
 # Don't allow .env files, just env-exmaple files
 !.env-example
 
+# Secrets generated at deploy time (e.g. services/dev/Runners/.env, holding the
+# live Gitea Actions registration token) - never committed, regardless of the
+# services/docker-compose.yml carve-out above.
+**/.env
+
 ### Terraform ###
 # Local .terraform directories
 **/.terraform/*

infra/.terraform.lock.hcl

@@ -33,6 +33,43 @@ provider "registry.opentofu.org/hashicorp/hcloud" {
   ]
 }
 
+provider "registry.opentofu.org/hashicorp/local" {
+  version     = "2.9.0"
+  constraints = "~> 2.5"
+  hashes = [
+    "h1:1dtKYW/5a1qob3yneL6WzOlnSGfYtJ6a2XeejCk9yb4=",
+    "h1:5NseXq5wU8O20ersTtV4ocrLYFFtgFr7n0pRLO1W2Rw=",
+    "h1:5d22ZPPK4iiygPbwRz/PJF5Es/0axVpMlPRpCR0Padw=",
+    "h1:AnwyolirmIlBMjH6+tV8bKkvT+5axJNYxi2y2IguiX4=",
+    "h1:PBp+HeseY021Fw3sLznCG27idgwPoff4cBuNmKgPL2w=",
+    "h1:VDxIhe4GbzdOCdmt7mQaqdwERQW6GSI7Roonts42Gr0=",
+    "h1:ZO6eWWnf8LjjV1q/JNeL9WLtZ6fwIttOnyN5LjCNSEo=",
+    "h1:dPIAf8oUAz+vW2E0iZunMvpuPddRZIztRsPSY1u+VnY=",
+    "h1:fwTDVG9AhFVKQZIb1EXkHv4FqzsZNlLWgkyPGDmZZEE=",
+    "h1:kDc465XPC7/6XFCjrMC4mTqhA9ef0FHKuJ3ZgfGNfeg=",
+    "h1:kGbjxrI2P8MHeyVtE1U3Q1TbyF71ExnHxtkrE+Aj6UU=",
+    "h1:kcoK6Afbsj54u9zaEqpecWAFKytqjBijtguCNwV3d4M=",
+    "h1:rxomJjDwOo+YZ+WIPc25FqEgsz9orh/2MCyUcZmFjvw=",
+    "h1:t0CMn/Rkwquw8l2yQ+O4ApzbMZfY2UazbsDnZygzACA=",
+    "h1:tJwgm2BS4xCGlElCDQEFXQoefY9Y4t0JdSKTtsPBbBo=",
+    "zh:13ef7ecd1e397ec5b20ea588508dd3e3b8d6c50d809ae76b079abf9dd8d02e4b",
+    "zh:2190c9325980076489ce02b0f5dd2c0b91fc8711cefa99e714d8619a32827ad1",
+    "zh:2a0cfc5600730093705071707e4a4e4e953e7d9091859e0f66b46daa1060dd5d",
+    "zh:2ff53eac1af43ab9a2248a0e53c963d46e19cf04bc4c3f323591cfcebb218252",
+    "zh:4ebc3dee700f60af9da29970052fd02fa947813162b224716862dc9d7f1f7542",
+    "zh:5fe6dab84ceeaa8eb3f1567c5f05578333370c472240ca5c5bfc25e92d4d5586",
+    "zh:66bbec16367bbf440045502c9779b11f4ac5b022c8d8d17afe12d431950838b5",
+    "zh:7641e5c2e4b529e869cde29ab5b1de2fd1091489eb745b19ac2709bd7f4dfd84",
+    "zh:855bfba0756d17ce07595ff57d7cf664443d1495127cb88fb063362734b8b22a",
+    "zh:aaec10f237921d60c581d1b7a66f0a8a8019d9802dc04af11b5b981f6682e01d",
+    "zh:e460835a38ffa1e74f6929904bfd14ef473d217fd537b7ce834abe5ce5e2ce07",
+    "zh:ecc4295215db0e4aea3c9329611c31e09a853e1ae207d56742403bd4f5516703",
+    "zh:ee6d9fae63a612072e00402894e14826af7a3351c235b9c5b423b7629a77ca29",
+    "zh:f2b5c8db74aa7ebcf7cd423672358437d42401675069ef67b01ff910054e49d5",
+    "zh:f5aff74d3eb96d4592c7bca5cd3ea89b469e84efbf382944bd0f844a57059c09",
+  ]
+}
+
 provider "registry.opentofu.org/hetznercloud/hcloud" {
   version     = "1.66.0"
   constraints = "~> 1.45"

infra/main.tf

@@ -38,6 +38,7 @@ module "dev" {
   network_ip_range       = var.network_ip_range
   bootstrap_script       = local.bootstrap_script
   ssh_private_key_path   = var.ssh_private_key_path
+  runner_count           = var.dev_runner_count
 
   # module.network.id alone doesn't guarantee the subnet exists yet, and a server
   # can't join a network before it has a subnet.

infra/modules/dev/main.tf

@@ -88,4 +88,19 @@ resource "hcloud_volume" "storage" {
   server_id = hcloud_server.this.id
   automount = true
   format    = "ext4"
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+# Renders services/dev/Runners/docker-compose.yml directly into the repo, with
+# one runner service per var.runner_count. This only touches the local working
+# tree - deploying the change still goes through the normal scp + spinup.sh flow.
+resource "local_file" "runners_compose" {
+  filename = "${path.root}/../services/dev/Runners/docker-compose.yml"
+  content = templatefile("${path.module}/templates/runners-docker-compose.yml.tftpl", {
+    runner_count = var.runner_count
+  })
+  file_permission = "0644"
 }

infra/modules/dev/templates/runners-docker-compose.yml.tftpl

@@ -0,0 +1,30 @@
+# Generated by OpenTofu from var.dev_runner_count - do not hand-edit.
+# To change the number of runners, edit dev_runner_count in terraform.tfvars and
+# run tofu apply. To change their shape, edit
+# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead.
+#
+# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable
+# (not baked in here) - services/dev/spinup.sh generates a fresh token and
+# writes it to Runners/.env immediately before starting these containers.
+services:
+%{ for i in range(runner_count) ~}
+  runner-${i + 1}:
+    image: gitea/act_runner:latest
+    container_name: gitea_runner_${i + 1}
+    volumes:
+      - ./config.yaml:/config.yaml
+      - ./gitea_runner_${i + 1}:/data
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - proxy
+    environment:
+      CONFIG_FILE: /config.yaml
+      GITEA_INSTANCE_URL: "https://git.luke-else.co.uk"
+      GITEA_RUNNER_REGISTRATION_TOKEN: "$${GITEA_RUNNER_REGISTRATION_TOKEN}"
+      GITEA_RUNNER_NAME: "CICD#${i + 1}"
+    restart: unless-stopped
+%{ endfor ~}
+
+networks:
+  proxy:
+    external: true

infra/modules/dev/variables.tf

@@ -52,3 +52,8 @@ variable "ssh_private_key_path" {
   description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH."
   type        = string
 }
+
+variable "runner_count" {
+  description = "Number of Gitea Actions runner containers to render into services/dev/Runners/docker-compose.yml."
+  type        = number
+}

infra/modules/prod/main.tf

@@ -115,4 +115,8 @@ resource "hcloud_volume" "storage" {
   server_id = hcloud_server.this.id
   automount = true
   format    = "ext4"
+
+  lifecycle {
+    prevent_destroy = true
+  }
 }

infra/terraform.tfvars.example

@@ -22,3 +22,4 @@ ssh_private_key_path = "~/.ssh/id_ed25519"
 # prod_volume_size       = 20
 # allowed_ssh_source_ips = ["203.0.113.4/32"]
 # deploy_user             = "deploy"
+# dev_runner_count       = 3

infra/variables.tf

@@ -91,3 +91,9 @@ variable "deploy_user" {
   type        = string
   default     = "deploy"
 }
+
+variable "dev_runner_count" {
+  description = "Number of Gitea Actions runner containers to run on dev. Rendered into services/dev/Runners/docker-compose.yml on every tofu apply - see infra/modules/dev/templates/runners-docker-compose.yml.tftpl."
+  type        = number
+  default     = 1
+}

infra/versions.tf

@@ -6,6 +6,10 @@ terraform {
       source  = "hetznercloud/hcloud"
       version = "~> 1.45"
     }
+    local = {
+      source  = "hashicorp/local"
+      version = "~> 2.5"
+    }
   }
 }
 

readme.md

@@ -73,9 +73,13 @@ architecture-beta
 │   ├── ssh.tf                 # looks up the SSH key already uploaded to Hetzner Cloud
 │   ├── versions.tf            # provider requirements
 │   ├── terraform.tfvars.example
+│   ├── scripts/
+│   │   └── bootstrap.sh.tftpl # post-install script run on every host right after creation
 │   └── modules/
 │       ├── network/           # shared private network + subnet (used by dev and prod)
-│       ├── dev/                # dev server + firewall + volume
+│       ├── dev/                # dev server + firewall + volume + renders services/dev/Runners/docker-compose.yml
+│       │   └── templates/
+│       │       └── runners-docker-compose.yml.tftpl # shape of each generated runner service
 │       ├── prod/               # prod server + firewall + volume
 │       └── vpn/                # vpn server + firewall (no private network, no volume)
 ├── services/                 # Docker Compose stacks, grouped by which server they run on
@@ -94,18 +98,19 @@ Each of `services/dev`, `services/prod`, `services/vpn` follows the same convent
 ## Prerequisites
 
 - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token
-- An SSH key uploaded to that project (Console → Security → SSH Keys)
+- An SSH key uploaded to that project (Console → Security → SSH Keys), and the matching private key available locally (OpenTofu uses it once per server to run the post-install bootstrap — see below)
 - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0`
-- Docker + the Compose plugin on each target server
 - DNS records for the domains in [Service inventory](#service-inventory) pointed at the relevant server's public IP
 
+Docker + the Compose plugin no longer need installing by hand — the bootstrap script below handles that.
+
 ## Provisioning the infrastructure (`infra/`)
 
 ```sh
 cd infra
 export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
 cp terraform.tfvars.example terraform.tfvars
-$EDITOR terraform.tfvars                     # set ssh_key_name at minimum
+$EDITOR terraform.tfvars                     # set ssh_key_name and ssh_private_key_path at minimum
 
 tofu init
 tofu plan
@@ -121,17 +126,46 @@ Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vp
 
 `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`.
 
+### Post-install bootstrap
+
+Immediately after each `hcloud_server` is created, OpenTofu uploads and runs [`infra/scripts/bootstrap.sh.tftpl`](infra/scripts/bootstrap.sh.tftpl) over SSH as `root` (`connection` + `file`/`remote-exec` provisioners in each `modules/<host>/main.tf`). It:
+
+- installs Docker Engine + the Compose plugin
+- creates a non-root sudo user (`var.deploy_user`, default `deploy`) with the same SSH key as root, in the `sudo` and `docker` groups
+- hardens `sshd`: password authentication off, root login restricted to key-only (`PermitRootLogin prohibit-password`)
+- installs and enables `unattended-upgrades` for automatic security patches
+
+After bootstrap, SSH in as `var.deploy_user` for day-to-day work — root key-based login still works as a fallback.
+
+### Scaling Gitea Actions runners
+
+The number of Gitea Actions runner containers on `dev` is an OpenTofu input, `var.dev_runner_count` (default `3`). On every `tofu apply`, `modules/dev`'s `local_file.runners_compose` resource renders [`modules/dev/templates/runners-docker-compose.yml.tftpl`](infra/modules/dev/templates/runners-docker-compose.yml.tftpl) straight into [`services/dev/Runners/docker-compose.yml`](services/dev/Runners/docker-compose.yml) in your working tree — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated: change `dev_runner_count` in `terraform.tfvars` and re-run `tofu apply` rather than hand-editing it.
+
+This only updates your local working tree — `tofu apply` doesn't reach out and start containers on `dev` itself. Redeploy as usual (re-copy `services/dev/` to the server and re-run `spinup.sh`) to apply a count change.
+
+Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore.
+
+### First apply: bootstrapping order matters
+
+`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch `tofu apply`, the `dev`/`prod` bootstrap provisioners can't connect yet. Bring the estate up in this order:
+
+1. `tofu apply -target=module.vpn` — creates `vpn` only; its firewall still allows SSH from `var.allowed_ssh_source_ips`, so its own bootstrap provisioner runs fine.
+2. Deploy and start the VPN service on `vpn` (see [Deploying the services](#deploying-the-services-services)), then connect to it with an OpenVPN client.
+3. `tofu apply` — creates `network`, `dev`, `prod`. Now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and the `dev`/`prod` bootstrap provisioners can connect.
+
+If step 3 is run before you're connected to the VPN, the `dev`/`prod` provisioners will fail and Terraform will mark those servers tainted — re-running `tofu apply` once connected will recreate and re-bootstrap them.
+
 ## Deploying the services (`services/`)
 
 Once a server exists, copy the relevant `services/<host>/` directory to it (e.g. `scp -r services/prod user@<prod_ipv4>:~/services`) and run the matching script from inside that directory:
 
 ```sh
 # on dev
-./spinup.sh     # Traefik → Gitea Runner → Watchtower
+./spinup.sh     # Traefik → Gitea → (waits, generates a runner token) → Runners → Watchtower
 ./spindown.sh   # reverse order, then prunes images/volumes
 
 # on prod
-./spinup.sh    # Traefik, then (after a 20s cert/registry settle) Watchtower, status, websites, Bitwarden, RustDesk
+./spinup.sh    # Traefik, → Watchtower → status → websites → Bitwarden → RustDesk
 ./spindown.sh
 
 # on vpn
@@ -159,7 +193,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi
 |---|---|---|
 | Traefik | `traefik-docker-compose.yml` | `traefik.cicd.luke-else.co.uk` |
 | Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` |
-| Gitea Actions runner | `Runners/docker-compose.yml` | `cicd.luke-else.co.uk` |
+| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | `cicd.luke-else.co.uk` |
 | Watchtower | `watchtower-docker-compose.yml` | — |
 
 ### `prod`
@@ -185,9 +219,8 @@ Every public-facing service is fronted by its host's own Traefik instance, termi
 
 A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here:
 
-- **Gitea Actions runner**: set a real `GITEA_RUNNER_REGISTRATION_TOKEN` in `services/dev/Runners/docker-compose.yml` (generate one from the Gitea admin UI) before starting the runner.
 - **Traefik dashboard auth**: the committed basic-auth hash is a placeholder. Generate your own with `echo $(htpasswd -nb user password) | sed -e 's/\$/\$\$/g'` and replace the `traefik-auth` middleware value in each `traefik-docker-compose.yml`.
-- **General host hardening**: non-root user, UFW, unattended-upgrades, Docker install — see `services/todo.md`.
+- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by the [post-install bootstrap](#post-install-bootstrap); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)).
 
 ## Development container
 
@@ -201,7 +234,8 @@ Then reopen the repo in VS Code with the Dev Containers extension.
 
 ## Security notes
 
-- Real secrets (`HCLOUD_TOKEN`, `*.tfvars`, `.env` files) must never be committed — see `.gitignore`.
+- Real secrets (`HCLOUD_TOKEN`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`.
-- SSH is restricted to `var.allowed_ssh_source_ips` on every host; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s).
+- SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s).
-- `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly.
+- `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP.
 - Firewalls are allowlists scoped per host in `infra/modules/<host>/main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open.
+- Every host's [post-install bootstrap](#post-install-bootstrap) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`var.deploy_user`) for day-to-day access.

services/dev/Runners/docker-compose.yml

@@ -1,31 +1,26 @@
+# Generated by OpenTofu from var.dev_runner_count - do not hand-edit.
+# To change the number of runners, edit dev_runner_count in terraform.tfvars and
+# run tofu apply. To change their shape, edit
+# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead.
+#
+# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable
+# (not baked in here) - services/dev/spinup.sh generates a fresh token and
+# writes it to Runners/.env immediately before starting these containers.
 services:
-  # Runner
+  runner-1:
-  runner:
     image: gitea/act_runner:latest
-    container_name: gitea_runner
+    container_name: gitea_runner_1
     volumes:
       - ./config.yaml:/config.yaml
-      - ./gitea_runner:/data
+      - ./gitea_runner_1:/data
       - /var/run/docker.sock:/var/run/docker.sock
     networks:
       - proxy
     environment:
       CONFIG_FILE: /config.yaml
       GITEA_INSTANCE_URL: "https://git.luke-else.co.uk"
-      GITEA_RUNNER_REGISTRATION_TOKEN: "INSERT REGISTRATION TOKEN"
+      GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}"
       GITEA_RUNNER_NAME: "CICD#1"
-    labels:
-      ## Expose cicd Through Trefik ##
-      - "traefik.enable=true" # <== Enable traefik to proxy this container
-
-      - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https"
-      - "traefik.http.routers.cicd-insecure.middlewares=redirect-web-secure"
-      - "traefik.http.routers.cicd-insecure.rule=Host(`cicd.luke-else.co.uk`)"
-      - "traefik.http.routers.cicd-insecure.entrypoints=web"
-
-      - "traefik.http.routers.cicd.rule=Host(`cicd.luke-else.co.uk`)"
-      - "traefik.http.routers.cicd.entrypoints=websecure"
-      - "traefik.http.routers.cicd.tls.certresolver=myresolver"
     restart: unless-stopped
 
 networks:

services/dev/spindown.sh

@@ -6,6 +6,7 @@ cd Runners
 docker compose down
 cd ..
 
+docker compose -f gitea-docker-compose.yml down
 docker compose -f traefik-docker-compose.yml down
 docker compose -f watchtower-docker-compose.yml down
 

services/dev/spinup.sh

@@ -5,6 +5,22 @@ cd "$(dirname "$0")"
 
 docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d
 
+docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d
+
+echo "Waiting for Gitea to become ready..."
+for i in $(seq 1 30); do
+  if docker exec gitea gitea admin user list >/dev/null 2>&1; then
+    break
+  fi
+  sleep 2
+done
+
+# Generate a fresh Actions runner registration token straight from Gitea and
+# hand it to the runner containers via .env (docker compose auto-loads this) -
+# no manual copy/paste from the admin UI needed.
+RUNNER_TOKEN=$(docker exec gitea gitea actions generate-runner-token)
+echo "GITEA_RUNNER_REGISTRATION_TOKEN=$RUNNER_TOKEN" > Runners/.env
+
 cd Runners
 docker compose pull && docker compose up -d
 cd ..

services/todo.md

@@ -21,10 +21,6 @@
 
 - Ensure that ports are assigned correctly for the system
 
-# Gitea Runner
-
-- Ensure that a registration token has been setup before continuing
-
 ## Websites
 
 - Ensure website files are copied over