From 52aa4278bf3b85b774bba24ad2d6085d5e30df78 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Tue, 21 Jul 2026 20:11:23 +0100 Subject: [PATCH] chore: Refresh of repo --- ansible/inventory/hcloud.yml | 3 +- ansible/playbooks/group_vars/all.yml | 10 +- ansible/roles/bootstrap/tasks/main.yml | 8 +- ansible/roles/deploy/tasks/main.yml | 6 +- infra/main.tf | 5 +- infra/modules/dev/main.tf | 12 +- infra/modules/network/main.tf | 4 +- infra/modules/prod/main.tf | 13 +- infra/modules/vpn/main.tf | 6 +- readme.md | 215 ++++++++----------------- services/todo.md | 32 ---- 11 files changed, 92 insertions(+), 222 deletions(-) delete mode 100644 services/todo.md diff --git a/ansible/inventory/hcloud.yml b/ansible/inventory/hcloud.yml index aad0be5..d414f72 100644 --- a/ansible/inventory/hcloud.yml +++ b/ansible/inventory/hcloud.yml @@ -1,8 +1,7 @@ --- # Dynamic inventory: queries the Hetzner Cloud API directly (HCLOUD_TOKEN), # grouping hosts by the `role` label set on each hcloud_server resource in -# infra/modules//main.tf (role = dev/prod/vpn). Nothing here reads -# Terraform state - this always reflects whatever actually exists in Hetzner. +# infra/modules//main.tf (role = dev/prod/vpn). plugin: hetzner.hcloud.hcloud token: "{{ lookup('env', 'HCLOUD_TOKEN') }}" connect_with: public_ipv4 diff --git a/ansible/playbooks/group_vars/all.yml b/ansible/playbooks/group_vars/all.yml index ee4289f..a444ef2 100644 --- a/ansible/playbooks/group_vars/all.yml +++ b/ansible/playbooks/group_vars/all.yml @@ -1,8 +1,4 @@ --- -# Non-root sudo user the bootstrap role creates on each host - purely an -# Ansible-side concept, OpenTofu has no equivalent variable. Override with the -# DEPLOY_USER env var (control.sh's "Set variables" menu writes it) if you -# don't want the default. deploy_user: "{{ lookup('env', 'DEPLOY_USER') | default('deploy', true) }}" ansible_user: "{{ deploy_user }}" ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}" @@ -11,11 +7,7 @@ ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') services_root: "{{ playbook_dir }}/../../services" # S3-compatible bucket that services//backup-docker-compose.yml backs up -# to and restores from - see readme.md. Not provisioned by OpenTofu (the -# hcloud provider has no Object Storage resource), so these must point at a -# bucket you've created yourself. AWS_ENDPOINT is only needed for a -# non-AWS S3-compatible provider (e.g. Hetzner Object Storage); leave it unset -# to use real AWS S3. +# to and restores from backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}" backup_s3_access_key_id: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY_ID') }}" backup_s3_secret_access_key: "{{ lookup('env', 'BACKUP_S3_SECRET_ACCESS_KEY') }}" diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index 307f0cc..93bca70 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -1,7 +1,5 @@ --- -# Post-install bootstrap - replaces infra/scripts/bootstrap.sh.tftpl. Runs as -# root (see ansible_user override in playbooks/bootstrap.yml) since the deploy -# user doesn't exist yet on a freshly created server. Idempotent: safe to +# Post-install bootstrap - Idempotent: safe to # re-run against an already-bootstrapped host. - name: Update apt cache and upgrade packages @@ -10,10 +8,6 @@ upgrade: dist cache_valid_time: 3600 -# Docker straight from Ubuntu's own repositories - no external apt repo, GPG -# key or architecture juggling. docker.io is the engine, docker-compose-v2 is -# the `docker compose` plugin. A little behind Docker's own channel, but far -# simpler and plenty for these hosts. - name: Install Docker Engine and the Compose plugin ansible.builtin.apt: name: diff --git a/ansible/roles/deploy/tasks/main.yml b/ansible/roles/deploy/tasks/main.yml index 7ad5889..87be255 100644 --- a/ansible/roles/deploy/tasks/main.yml +++ b/ansible/roles/deploy/tasks/main.yml @@ -1,10 +1,8 @@ --- # Copies services// to the server and renders the files that -# used to be generated by OpenTofu (services//.env and, for dev, -# Runners/docker-compose.yml) - replaces the null_resource.deploy_services + -# local_file provisioners previously in infra/modules//main.tf. +# need templating (e.g. .env, docker-compose.yml). # -# Persistent data lives in named Docker volumes now, not on a Hetzner volume - +# Persistent data lives in named Docker volumes # see services//backup-docker-compose.yml. .env only carries the S3 # credentials that backup/restore need, rendered for every host. diff --git a/infra/main.tf b/infra/main.tf index 161587a..1b268c4 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -32,10 +32,7 @@ locals { { zone = "luke-else.co.uk", name = "traefik.vpn", value = module.vpn.ipv4 }, ] - # Microsoft 365 mail records for luke-else.co.uk - carried over from the - # domain's old DNS provider (pre-Hetzner), not previously applied here, so - # these are genuinely new to the Hetzner zone rather than something to - # import. + # Microsoft 365 mail records for luke-else.co.uk mail_records = [ { zone = "luke-else.co.uk", name = "@", type = "MX", value = "0 lukeelse-co-uk01b.mail.protection.outlook.com." }, { zone = "luke-else.co.uk", name = "@", type = "TXT", value = "\"v=spf1 include:spf.protection.outlook.com -all\"" }, diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf index cd811f1..9f9517f 100644 --- a/infra/modules/dev/main.tf +++ b/infra/modules/dev/main.tf @@ -1,26 +1,24 @@ # dev: Gitea + Runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) -# Port sources: services/dev/*docker-compose.yml (published ports) and services/todo.md -# (the documented UFW allow-list). dev has no firewall in docs/architecture.md, but we -# add one anyway for baseline safety - see conversation history. +# Port sources: services/dev/*docker-compose.yml (published ports) resource "hcloud_firewall" "this" { name = "dev-firewall" - rule { # server ssh - wired to the vpn server's public IP only, see infra/main.tf + rule { # server ssh - wired to the vpn server's public IP only direction = "in" protocol = "tcp" port = "22" source_ips = var.allowed_ssh_source_ips } - rule { # gitea ssh (git.luke-else.co.uk, published as 222:22) + rule { # gitea ssh direction = "in" protocol = "tcp" port = "222" source_ips = ["0.0.0.0/0", "::/0"] } - rule { # Traefik http/https (git.luke-else.co.uk, cicd.luke-else.co.uk) + rule { # Traefik http/https direction = "in" protocol = "tcp" port = "80" @@ -56,7 +54,7 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] - labels = { role = "dev" } # picked up by ansible/inventory/hcloud.yml + labels = { role = "dev" } network { network_id = var.network_id diff --git a/infra/modules/network/main.tf b/infra/modules/network/main.tf index 87abcb8..b094689 100644 --- a/infra/modules/network/main.tf +++ b/infra/modules/network/main.tf @@ -1,5 +1,5 @@ -# Private network shared by dev and prod (see docs/architecture.md "network" group). -# vpn is intentionally not attached to this network - it sits outside it in the diagram. +# Private network shared by dev and prod (see readme.md "network" group). +# vpn is intentionally not attached to this network resource "hcloud_network" "main" { name = "server-network" ip_range = var.ip_range diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index 65c2775..931c902 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -1,18 +1,15 @@ -# prod: Traefik, Websites, Bitwarden, RustDesk, status page - -# Port sources: services/prod/*docker-compose.yml (published ports) and services/todo.md -# (the documented UFW allow-list). +# Port sources: services/prod/*docker-compose.yml (published ports) resource "hcloud_firewall" "this" { name = "prod-firewall" - rule { # server ssh - wired to the vpn server's public IP only, see infra/main.tf + rule { # server ssh - wired to the vpn server's public IP only direction = "in" protocol = "tcp" port = "22" source_ips = var.allowed_ssh_source_ips } - rule { # Traefik http/https (Websites, Bitwarden, status page) + rule { # Traefik http/https direction = "in" protocol = "tcp" port = "80" @@ -61,7 +58,7 @@ resource "hcloud_firewall" "this" { source_ips = ["0.0.0.0/0", "::/0"] } - rule { # traffic from dev over the private network + rule { # traffic from over the private network direction = "in" protocol = "tcp" port = "1-65535" @@ -83,7 +80,7 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] - labels = { role = "prod" } # picked up by ansible/inventory/hcloud.yml + labels = { role = "prod" } network { network_id = var.network_id diff --git a/infra/modules/vpn/main.tf b/infra/modules/vpn/main.tf index 36ddc51..d7e5ef3 100644 --- a/infra/modules/vpn/main.tf +++ b/infra/modules/vpn/main.tf @@ -1,4 +1,4 @@ -# vpn: WireGuard (wg-easy) + Traefik. Not attached to the private network (see docs/architecture.md). +# vpn: WireGuard (wg-easy) + Traefik. Not attached to the private network. resource "hcloud_firewall" "this" { name = "vpn-firewall" @@ -9,7 +9,7 @@ resource "hcloud_firewall" "this" { source_ips = var.allowed_ssh_source_ips } - rule { # Traefik http/https (traefik.vpn.luke-else.co.uk) + rule { # Traefik http/https direction = "in" protocol = "tcp" port = "80" @@ -38,5 +38,5 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] - labels = { role = "vpn" } # picked up by ansible/inventory/hcloud.yml + labels = { role = "vpn" } } diff --git a/readme.md b/readme.md index 8167cf6..a5ae1cc 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # Server -Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each bootstrapped and deployed with [Ansible](https://www.ansible.com/), running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/). +Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), bootstrapped and deployed with [Ansible](https://www.ansible.com/), running Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).

@@ -10,13 +10,14 @@ Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted - [Architecture](#architecture) - [Repository layout](#repository-layout) +- [Design choices](#design-choices) - [Prerequisites](#prerequisites) -- [Control panel (`control.sh`)](#control-panel-controlsh) +- [Quickstart](#quickstart) - [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) -- [Bootstrapping and deploying with Ansible](#bootstrapping-and-deploying-with-ansible-ansible) +- [Bootstrapping and deploying](#bootstrapping-and-deploying-ansible) +- [Running the services](#running-the-services-services) - [Service inventory](#service-inventory) -- [First-time setup](#first-time-setup) -- [Development container](#development-container) +- [Caveats](#caveats) - [Security notes](#security-notes) ## Architecture @@ -54,83 +55,71 @@ architecture-beta vpn:R -- L:backups ``` -`gateway` represents the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from - not a Hetzner resource, see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3). +`gateway` is the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from (see [Design choices](#design-choices)) — also not a Hetzner resource. | Server | Purpose | Network | |---|---|---| -| `dev` | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) | -| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | -| `vpn` | WireGuard (wg-easy) + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | +| `dev` | Gitea, CI runners, dev-facing Traefik | Private network only | +| `prod` | Public websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | +| `vpn` | WireGuard (wg-easy) + its own Traefik | **Not** on the private network — isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | -Each host's stateful data lives in local named Docker volumes, backed up nightly to a shared S3 bucket and restored automatically on first boot - see below. No host has a Hetzner Cloud Volume attached any more. - -`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach deployment targets on `prod` without exposing that traffic publicly. `vpn` is deliberately kept off this network. Each server has its own Hetzner Cloud Firewall (see `infra/modules//main.tf`) that only opens the ports actually used by the compose stacks running on it, plus SSH restricted to `var.allowed_ssh_source_ips`. +`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach `prod` without exposing that traffic publicly. Each server has its own Hetzner Cloud Firewall (`infra/modules//main.tf`) opening only the ports its own compose stacks use, plus SSH restricted to `var.allowed_ssh_source_ips`. ## Repository layout ``` . -├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, DNS +├── infra/ # OpenTofu — provisions the 3 servers, network, firewalls, DNS │ ├── main.tf # root module: wires network + dev/prod/vpn/dns modules together │ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names) -│ ├── outputs.tf # pass-through outputs from each host module -│ ├── ssh.tf # looks up each SSH key already uploaded to Hetzner Cloud -│ ├── versions.tf # provider requirements │ ├── terraform.tfvars.example │ └── modules/ -│ ├── network/ # shared private network + subnet (used by dev and prod) -│ ├── dev/ # dev server + firewall, labeled role=dev -│ ├── prod/ # prod server + firewall, labeled role=prod -│ ├── vpn/ # vpn server + firewall (no private network), labeled role=vpn -│ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones -├── ansible/ # Ansible: bootstraps each server and deploys/starts services// onto it -│ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by the role label above +│ ├── network/ # shared private network + subnet (dev + prod) +│ ├── dev/ prod/ vpn/ # one server + firewall each, labeled role= +│ └── dns/ # Hetzner DNS zones + records for every domain in var.dns_zones +├── ansible/ # bootstraps each server, deploys/starts services// onto it +│ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by role label │ ├── roles/ -│ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades -│ │ └── deploy/ # copies services//, renders .env (S3 backup credentials) + Runners/docker-compose.yml -│ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml, -│ └── group_vars/ # deploy_user, service_group, backup_s3_*, etc. - must sit next to -│ # the playbooks for Ansible to auto-load it (see ansible/README.md) +│ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades +│ │ └── deploy/ # copies services//, renders .env + Runners/docker-compose.yml +│ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml ├── services/ # Docker Compose stacks, grouped by which server they run on -│ ├── dev/ # Gitea + CI runner + Traefik + backup/restore (git.luke-else.co.uk, cicd.luke-else.co.uk) -│ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore -│ ├── vpn/ # WireGuard (wg-easy) + Traefik + backup/restore -│ └── todo.md # Outstanding manual setup/hardening tasks -├── docs/ -│ └── architecture.md # Source of the architecture diagram above -├── .devcontainer/ # Git submodule: shared devcontainer for working on this repo (OpenTofu tooling) +│ ├── dev/ # Gitea + CI runners + Traefik + backup/restore +│ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore +│ └── vpn/ # WireGuard (wg-easy) + Traefik + backup/restore +├── .devcontainer/ # git submodule: shared devcontainer for this repo (OpenTofu tooling) └── assets/ ``` -Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, a `backup-docker-compose.yml` + `restore.sh` pair for that host's S3 backup/restore (see below), plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side. +Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per service, a `backup-docker-compose.yml` + `restore.sh` pair, and a `spinup.sh` / `spindown.sh` pair that brings up or tears down everything on that host in order. `infra/modules/` and `ansible/roles/deploy` mirror the same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks sit side by side. -OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services//`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown. +## Design choices + +- **OpenTofu and Ansible have a clean split**: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches a server over SSH. Everything from "the server exists" onward — Docker, the `deploy` user, SSH hardening, copying `services//`, running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md). +- **No Hetzner Cloud Volumes.** Every stateful service mounts a plain named Docker volume instead. Each host backs its volumes up nightly to S3 and restores any that are empty on first boot — see [Persistent data](#persistent-data). +- **`vpn` is deliberately off the private network.** SSH to `dev`/`prod` is only reachable through it (see [Security notes](#security-notes)), but it can't reach either directly — containing a compromised VPN endpoint. +- **DNS is provisioned, not just documented.** `module.dns` creates a Hetzner zone and every A/MX/TXT/CNAME record in `infra/main.tf` — see [Managing DNS](#managing-dns). +- **Hosts are discovered dynamically** from the Hetzner API by role label, not a static inventory file — nothing in Ansible reads Terraform state. +- **Gitea Actions runners are disposable.** Their `/data` is a local cache; runner identity re-registers with Gitea on every `spinup.sh`, so they're excluded from backup. ## Prerequisites - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token -- One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (Ansible uses it to bootstrap and deploy — see below) +- One or more SSH keys uploaded to that project (Console → Security → SSH Keys), plus the matching private key available locally - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` - [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`) -- Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner (see [Managing DNS](#managing-dns) — the zones and records themselves are created for you) -- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/), AWS S3, or any other S3-compatible provider) and an access key/secret pair with read/write access to it - not provisioned by OpenTofu (the `hcloud` provider has no Object Storage resource), so create this yourself and pass it to Ansible as `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` (and `BACKUP_S3_ENDPOINT` for anything other than real AWS S3) - see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3) +- Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner +- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/)) and an access key/secret pair — not provisioned by OpenTofu, so create this yourself -Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below). +## Quickstart -## Control panel (`control.sh`) - -If you'd rather not remember the exact command order, [`control.sh`](control.sh) at the repo root is an interactive menu that steps you through the whole pre-setup and bring-up. Run it from anywhere: +[`control.sh`](control.sh) at the repo root is an interactive menu covering the whole setup: ```sh ./control.sh ``` -It offers a **Check prerequisites** option (tools, `HCLOUD_TOKEN`/`DEPLOY_USER`/`BACKUP_S3_*` env vars, `terraform.tfvars`, SSH key, the `hetzner.hcloud` collection), a **Configuration** section, individual OpenTofu (`init`/`plan`/`apply`/`output`/`destroy`) and Ansible (bootstrap/deploy/spinup/spindown per role) actions, and a **Guided full setup** that runs everything in the firewall-imposed order — vpn first, pause for you to connect the VPN, then dev/prod. It's only a wrapper around the same `tofu` and `ansible-playbook` commands documented below, so you can always fall back to running them by hand. - -The **Configuration** section covers every variable both tools need: - -- **Set variables** prompts for `HCLOUD_TOKEN`, `DEPLOY_USER` (the non-root user Ansible creates — defaults to `deploy`), `ANSIBLE_SSH_PRIVATE_KEY_FILE`, and the `BACKUP_S3_*` credentials, one at a time (leave blank to keep the current value, secrets are input-masked). They're saved to `.control.env` at the repo root — gitignored, never committed — which is loaded automatically the next time you run `control.sh`, so you only enter them once. This is scoped to `control.sh` itself, not your regular shell. -- **Copy terraform.tfvars.example -> terraform.tfvars** creates `infra/terraform.tfvars` for you and prompts for `ssh_key_names` (the one required field with no default) so you don't leave the example's placeholder key names in place by accident. Everything else in that file (`location`, server types, `dns_zones`, etc.) still has sensible defaults you can edit by hand afterwards - see [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) below. +It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, and the `BACKUP_S3_*` credentials, saved to gitignored `.control.env` so you only enter them once), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand. ## Provisioning the infrastructure (`infra/`) @@ -145,42 +134,21 @@ tofu plan tofu apply ``` -This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` / `module.dns` in `infra/main.tf`: -- `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`) -- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses, each server labeled `role = "dev"/"prod"/"vpn"` for Ansible's dynamic inventory (`modules/dev`, `modules/prod`, `modules/vpn`) -- one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns)) - -No servers have an attached volume — persistent data lives in named Docker volumes backed up to S3 instead, see below. +This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`. -`terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`. - -OpenTofu never connects to the servers over SSH — no provisioners, no `bootstrap.sh`, no copying `services/` — that's all Ansible now. See below. - ### Managing DNS -`module.dns` (in `infra/modules/dns`) creates one Hetzner DNS zone per domain in `var.dns_zones` (default: `luke-else.co.uk`, `divine-couture.co.uk`, `snexo.co.uk`) and an `hcloud_zone_rrset` per record in `var.records` — kept in sync with `local.dns_records` (A records, one per hostname currently referenced by a Traefik `Host()` rule anywhere under `services/`, pointed at whichever of `dev`/`prod`/`vpn` actually serves it) and `local.mail_records` (see below) in `infra/main.tf`. Zones have `prevent_destroy = true`, matching the volumes — losing one deletes every record in it. +Creating a zone doesn't make Hetzner authoritative by itself — run `tofu output dns_nameservers` and set those as the domain's NS records at its registrar. Propagation time depends on the registrar and the domain's previous NS TTL. -Creating the zone doesn't make Hetzner authoritative for the domain by itself: you still need to point that domain's NS records at Hetzner's nameservers at whichever registrar it's registered through. Run `tofu output dns_nameservers` after applying to get the exact nameservers per domain, and set those as the domain's NS records at the registrar. Propagation can take a while depending on the registrar and the domain's previous NS TTL. +To add a subdomain, add an entry to `local.dns_records` in `infra/main.tf` (zone, name, target host) and re-run `tofu apply` — don't hand-create records in the Hetzner console, they'll drift from state. `var.records` supports `MX`/`TXT`/`CNAME`/etc. too (see `infra/modules/dns/variables.tf` for the value format each type expects); entries sharing zone/name/type merge into one RRSet. -Adding a new subdomain: add an entry to `local.dns_records` in `infra/main.tf` (zone, name, and the target `module..ipv4`) and re-run `tofu apply` — don't hand-create records in the Hetzner console, they'll drift from state. Note `name = "@"` is Hetzner's convention for a zone's apex record (e.g. bare `snexo.co.uk`), not an empty string. +`local.mail_records` tracks the Microsoft 365 records for `luke-else.co.uk` (MX, SPF, Google-site-verification, `autodiscover`). DMARC and DKIM aren't set up yet — see [Caveats](#caveats). -Only A records for IPv4 are managed for hosts — none of the modules currently track servers' IPv6 addresses, so AAAA records aren't generated even though the firewalls already allow IPv6 traffic. +## Bootstrapping and deploying (`ansible/`) -`var.records` isn't limited to A records: each entry takes an optional `type` (`A` by default, or `MX`/`TXT`/`CNAME`/etc.), and entries sharing the same `zone`/`name`/`type` are merged into a single Hetzner RRSet with multiple values (needed for e.g. two TXT strings under the same name) - see `infra/modules/dns/variables.tf` for the exact value format each type expects (MX embeds the priority in the value string, TXT values need escaped double quotes). - -#### Mail records (Microsoft 365) - -`local.mail_records` in `infra/main.tf` tracks the DNS records Microsoft 365 needs for `luke-else.co.uk` - the MX record, the SPF and Google-site-verification TXT records (merged into one `@` TXT RRSet), and the `autodiscover` CNAME. These carried over from the domain's DNS provider prior to Hetzner and were never applied to the Hetzner zone, so they're genuinely new records here (not something to `tofu import` - nothing to collide with). - -**Not yet included:** a DMARC policy (`_dmarc` TXT) and DKIM signing (two `selector1._domainkey`/`selector2._domainkey` CNAMEs). DKIM's targets are generated per-tenant by Microsoft 365 (Defender portal → Email & collaboration → Policies & rules → DKIM) and can't be guessed - enable DKIM there first, then add the two CNAMEs it gives you to `local.mail_records`. - -`infra/modules/dns/moved.tf` re-points every pre-existing A record at its new resource address, since generalizing the module to support non-A record types changed how they're keyed internally - without it, `tofu apply` would destroy and recreate every A record for no functional reason. It's a one-time migration aid: run `tofu plan` first to confirm the only real changes are the new mail records being created, then the file can be deleted after a successful apply. - -## Bootstrapping and deploying with Ansible (`ansible/`) - -Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services//` to each server, rendering `.env` (S3 backup credentials) and, for `dev`, `Runners/docker-compose.yml`, and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version: +Once `tofu apply` has created the servers, Ansible installs Docker, creates the `deploy` user, hardens SSH, copies `services//` to each server, and runs `spinup.sh`/`spindown.sh`. Full detail in [`ansible/README.md`](ansible/README.md); the short version: ```sh cd ansible @@ -194,21 +162,16 @@ ansible-playbook playbooks/site.yml -l role_vpn ansible-playbook playbooks/site.yml -l role_dev,role_prod ``` -Hosts are discovered dynamically from the Hetzner API (`ansible/inventory/hcloud.yml`), grouped into `role_dev`/`role_prod`/`role_vpn` by the `role` label OpenTofu sets on each server — there's no static inventory file to keep in sync, and nothing here reads Terraform state. +`dev`/`prod`'s firewalls only accept SSH from `vpn`'s own public IP, so on a from-scratch estate they aren't reachable until `vpn` is up and you're tunneled through it — visit `https://vpn.luke-else.co.uk`, complete wg-easy's first-run setup, add a client peer, and connect before running the second command. If it's run before you're connected, Ansible just fails to connect — re-run once connected. -### Persistent data lives in named Docker volumes, backed up to S3 +### Persistent data -Every stateful service's compose file mounts a plain named Docker volume rather than a bind mount to a Hetzner volume - `gitea_data:/data`, `bitwarden_data:/data/`, `letsencrypt_data:/letsencrypt`, and so on across `services/dev/*.yml`, `services/prod/*.yml`, and `services/vpn/*.yml`. Each volume is declared with an explicit `name:` in every compose file that mounts it (the same pattern already used for the shared `proxy` network), so Compose creates it once and every stack that references it reuses the same underlying volume regardless of which compose file happens to run first. +Every stateful service mounts a named Docker volume (`gitea_data`, `bitwarden_data`, `letsencrypt_data`, etc.) rather than a Hetzner volume. Each host's `backup-docker-compose.yml` defines: -Each host's `backup-docker-compose.yml` defines two services against those same named volumes: -- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), running on a nightly cron (`0 3 * * *`), tars up every mounted volume and uploads it to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/-.tar.gz`, keeping 14 days of history (`BACKUP_RETENTION_DAYS`). Containers labeled `docker-volume-backup.stop-during-backup=true` (Gitea, Bitwarden, RustDesk, Uptime Kuma, snexo, wg-easy) are stopped for the duration of the backup so their data is captured in a consistent state, then restarted automatically. Started by `spinup.sh` alongside everything else. -- **`restore`** — a one-shot container (`amazon/aws-cli` running `restore.sh`) that `spinup.sh` runs *before* anything else starts. It checks each of the host's named volumes; any that are already empty are populated from the most recent object under that host's S3 prefix, any that already contain data are left completely untouched. On a fresh server with no prior backup (or a bucket that doesn't exist yet), it's a no-op - services just start with empty volumes as before. +- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), nightly cron, tars every mounted volume to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/-.tar.gz`, 14 days of retention. Containers that need a consistent snapshot are stopped for the duration and restarted after. +- **`restore`** — a one-shot container `spinup.sh` runs before anything else starts. Any volume that's already empty is populated from the most recent backup; anything with data is left untouched. On a fresh server (or no prior backup), it's a no-op. -`AWS_S3_BUCKET_NAME`, `AWS_S3_PATH` (set to the host name, e.g. `dev`), `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_ENDPOINT` are rendered into `services//.env` by Ansible's `deploy` role from `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`/`BACKUP_S3_ENDPOINT` in your shell environment (see [Prerequisites](#prerequisites)) — `docker compose` auto-loads `.env` from its working directory. `spinup.sh` on every host refuses to start anything if `.env` is missing. - -The Gitea Actions runners are the one exception: their `/data` is a disposable local cache (runner identity re-registers with Gitea on every `spinup.sh` run anyway), so it's a plain named volume per runner (`gitea_runner_N_data`, baked directly into the rendered `Runners/docker-compose.yml`) that's deliberately left out of the backup/restore stack. - -To trigger a backup or restore manually rather than waiting for cron or a restart: +The S3 credentials (`AWS_S3_BUCKET_NAME`, `AWS_ACCESS_KEY_ID`, etc.) are rendered into `services//.env` by Ansible from the `BACKUP_S3_*` env vars — `spinup.sh` refuses to start anything if `.env` is missing. To trigger either manually: ```sh cd services/ @@ -216,27 +179,9 @@ docker compose -f backup-docker-compose.yml exec backup backup # force an imme docker compose -f backup-docker-compose.yml run --rm restore # restore any currently-empty volumes ``` -### Gitea Actions runners +## Running the services (`services/`) -`dev` always runs three Gitea Actions runner containers, configured automatically on deploy. Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — three `runner-N` services, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host, so edit the template rather than hand-editing the rendered file (the count lives in a `{% raw %}{% set runner_count = 3 %}{% endraw %}` line at the top). - -Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a template change. - -Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore. - -### First deploy: bootstrapping order matters - -`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own WireGuard service isn't running until you deploy it — so on a from-scratch estate, Ansible can't reach `dev`/`prod` yet. Bring it up in this order: - -1. `ansible-playbook playbooks/site.yml -l role_vpn` — bootstraps, deploys, and starts `vpn` only; its firewall allows SSH from `var.allowed_ssh_source_ips` directly. -2. Visit `https://vpn.luke-else.co.uk` and complete wg-easy's first-run setup wizard (admin username/password, then add a client peer and download its config/QR code) - see [Setting up WireGuard (wg-easy)](#setting-up-wireguard-wg-easy). Import that config into a WireGuard client and connect. -3. `ansible-playbook playbooks/site.yml -l role_dev,role_prod` — now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and `dev`/`prod` become reachable. - -If step 3 is run before you're connected to the VPN, Ansible will simply fail to connect — re-run it once connected. - -## Deploying the services (`services/`) - -After `ansible-playbook playbooks/deploy.yml` has copied `services//` to `/home/deploy/services/` on the matching server, SSH in as `deploy` and run the matching script from inside that directory — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do exactly this remotely (see [`ansible/README.md`](ansible/README.md)): +After `ansible-playbook playbooks/deploy.yml` has copied `services//` to the server, SSH in as `deploy` and run the matching script — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do this remotely: ```sh # on dev @@ -244,7 +189,7 @@ After `ansible-playbook playbooks/deploy.yml` has copied `services//` to ` ./spindown.sh # reverse order, then prunes images/volumes # on prod -./spinup.sh # Traefik, → Watchtower → status → websites → Bitwarden → RustDesk +./spinup.sh # Traefik → Watchtower → status → websites → Bitwarden → RustDesk ./spindown.sh # on vpn @@ -252,29 +197,11 @@ After `ansible-playbook playbooks/deploy.yml` has copied `services//` to ` ./spindown.sh ``` -### Setting up WireGuard (wg-easy) +Each stack can also be managed individually with plain Compose, e.g. `docker compose -f bitwarden-docker-compose.yml up -d` from inside `services/prod`. -`vpn` runs [wg-easy](https://github.com/wg-easy/wg-easy) - WireGuard plus a web UI for managing peers - fronted by Traefik like everything else. `spinup.sh` only starts the container; wg-easy itself has no automated first-run setup here (see `services/vpn/vpn-docker-compose.yml`), so once it's up: +All three hosts run [Watchtower](https://containrrr.dev/watchtower/), polling every 60s with cleanup enabled, so images stay current automatically once deployed — re-run the spinup scripts only after adding/removing a service or changing compose files. Every public-facing service is fronted by its host's own Traefik, terminating TLS via Let's Encrypt, joining an external `proxy` network and opting in via `traefik.enable=true` labels (RustDesk and the Gitea SSH port publish ports directly, since they aren't HTTP). -1. Visit `https://vpn.luke-else.co.uk` and complete the setup wizard - pick an admin username and a strong password (this account can view/rotate every client's private key, and it's reachable from the public internet). Enabling 2FA afterwards (wg-easy supports it) is worth doing since this isn't just a status page like the old OpenVPN setup was. -2. Set the host clients should connect to (`vpn.luke-else.co.uk`) and the port (`51820`, matching the firewall rule in `infra/modules/vpn/main.tf`) when prompted. -3. Add a client per device from the UI, then download its config file or scan the QR code directly into the WireGuard app. - -wg-easy's own state (admin account, peers, keys) lives in the `wg_easy_data` named volume, backed up to S3 like everything else - see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3). Client configs themselves are only ever downloaded through the UI, not stored anywhere else. - -If you ever need to force a re-sync without going through Ansible (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@:~/services` still works fine - `ansible-playbook playbooks/deploy.yml` will just overwrite it again next time it's run. - -Each stack can also be managed individually with plain Compose, e.g.: - -```sh -cd services/prod -docker compose -f bitwarden-docker-compose.yml up -d -docker compose -f bitwarden-docker-compose.yml down -``` - -All three hosts run [Watchtower](https://containrrr.dev/watchtower/) polling every 60s with cleanup enabled, so images are kept current automatically once deployed — the spinup scripts only need to be re-run after adding/removing a service or changing compose files. - -Every public-facing service is fronted by its host's own Traefik instance, terminating TLS via Let's Encrypt (`tlschallenge`, port 80/443). Each stack joins an external `proxy` Docker network and opts in via `traefik.enable=true` labels rather than publishing ports directly (RustDesk and the Gitea SSH port are the deliberate exceptions, since they aren't HTTP). +`vpn` runs [wg-easy](https://github.com/wg-easy/wg-easy) — WireGuard plus a web UI, fronted by Traefik. It has no automated first-run setup: visit `https://vpn.luke-else.co.uk`, pick an admin username/password (enable 2FA — this account can view/rotate every client's private key and is reachable from the public internet), set the connect host/port (`vpn.luke-else.co.uk` / `51820`), then add a client per device and download its config or scan its QR code. wg-easy's own state lives in the `wg_easy_data` volume, backed up like everything else; client configs themselves are never stored anywhere but the UI. ## Service inventory @@ -284,7 +211,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi |---|---|---| | Traefik | `traefik-docker-compose.yml` | `traefik.cicd.luke-else.co.uk` | | Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` | -| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | N/A | +| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated, 3 by default) | N/A | | Watchtower | `watchtower-docker-compose.yml` | — | | Backup/restore | `backup-docker-compose.yml` | — | @@ -309,15 +236,9 @@ Every public-facing service is fronted by its host's own Traefik instance, termi | Watchtower | `watchtower-docker-compose.yml` | — | | Backup/restore | `backup-docker-compose.yml` | — | -## First-time setup - -A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here: - -- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by [Ansible's `bootstrap` role](ansible/roles/bootstrap/tasks/main.yml); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)). - ## Development container -`.devcontainer` is a git submodule providing a ready-to-use OpenTofu development environment (VS Code + OpenTofu/Docker/Mermaid extensions). After cloning: +`.devcontainer` is a git submodule providing a ready-to-use OpenTofu development environment (VS Code + OpenTofu/Docker/Mermaid extensions): ```sh git submodule update --init --recursive @@ -325,11 +246,17 @@ git submodule update --init --recursive Then reopen the repo in VS Code with the Dev Containers extension. +## Caveats + +- **UFW isn't configured** — the Hetzner Cloud Firewalls already allowlist per-host ports, but there's no host-level firewall as defense in depth yet. +- **DMARC and DKIM aren't set up** for `luke-else.co.uk`. DKIM's targets are generated per-tenant by Microsoft 365 (Defender portal → Email & collaboration → Policies & rules → DKIM) and can't be guessed — enable it there first, then add the two CNAMEs it gives you to `local.mail_records`. +- **Only IPv4 DNS is managed.** None of the infra modules track servers' IPv6 addresses, so no AAAA records are generated even though the firewalls already allow IPv6 traffic. +- **`infra/modules/dns/moved.tf`** re-points pre-existing A records at their new resource address after the DNS module was generalized for non-A record types. It's a one-time migration aid — safe to delete once `tofu plan` shows no unexpected changes. + ## Security notes -- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. The S3 credentials end up in each host's `services//.env` (rendered by Ansible, mode `0600`, gitignored) - anyone with root on a host can read them, same trust boundary as everything else `deploy` can already do there. -- SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). -- `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP. -- Firewalls are allowlists scoped per host in `infra/modules//main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. -- Every host's [Ansible bootstrap role](ansible/roles/bootstrap/tasks/main.yml) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`deploy_user`, see `ansible/playbooks/group_vars/all.yml`) for day-to-day access. -- DNS zones (`modules/dns`) carry both Hetzner's own `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, including for `divine-couture.co.uk` and `snexo.co.uk`, not just `luke-else.co.uk`. +- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_*`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. S3 credentials land in each host's `services//.env` (mode `0600`, gitignored, rendered by Ansible). +- SSH to `dev`/`prod` is restricted to `vpn`'s own public IP — you must be tunneled into the VPN to reach them. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your egress IP(s). +- Firewalls are per-host allowlists (`infra/modules//main.tf`) — only ports actually used by that host's compose stacks are open. +- Ansible's bootstrap role disables SSH password auth, restricts root login to key-only, and creates a separate sudo user (`deploy_user`) for day-to-day access. +- DNS zones carry both Hetzner's `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, across all three managed domains. diff --git a/services/todo.md b/services/todo.md deleted file mode 100644 index 158ce2c..0000000 --- a/services/todo.md +++ /dev/null @@ -1,32 +0,0 @@ -# ToDo Items - -## General - -- Setup non root user - - - "21115:21115/tcp" - - "21116:21116/tcp" - - "21116:21116/udp" - - "21117:21117/tcp" - - "21119:21119/tcp" - -- Install docker, docker-compose and apache utils. - -## Traefik + TraefikRunner + Traefik VPN - -- Setup htaccess -> `echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g` -- Ensure email address is correct - -## Gitea - -- Ensure that ports are assigned correctly for the system - -## Websites - -- Ensure website files are copied over -- Ensure that ports are assigned correctly for the system - -## Bitwarden - -- Ensure that all data is fully encrypted during transfer. -- Ensure that ports are assigned correctly for the system