diff --git a/.gitignore b/.gitignore index 1aad0ee..6936989 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ # services/docker-compose.yml carve-out above. **/.env +### Ansible ### +*.retry + ### Terraform ### # Local .terraform directories **/.terraform/* diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 0000000..88f840b --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,77 @@ +# Ansible + +Bootstraps, syncs, and starts/stops the Docker Compose stacks in `../services/` +on the three servers OpenTofu provisions (see `../infra/`). OpenTofu still owns +the actual cloud resources (servers, network, firewalls, volumes, DNS); +everything downstream of "the server exists" - installing Docker, creating the +`deploy` user, hardening SSH, copying `services//`, rendering `.env` / +the Gitea runner compose file, and running `spinup.sh`/`spindown.sh` - lives +here instead. + +## Contents + +- `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API + directly (not Terraform state) and groups servers by their `role` label + (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules//main.tf`). +- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, `HCLOUD_TOKEN` + lookup), `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` (per-role vars, + e.g. `dev_runner_count`). +- `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening, + unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`. +- `roles/deploy/` - copies `services//` to the server, looks up the + host's data volume directly from the Hetzner API and renders `.env` + (`DATA_DIR=...`) and, for `dev`, `Runners/docker-compose.yml`. +- `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`, + and `site.yml` (all three in order). + +## Prerequisites + +```sh +cd ansible +ansible-galaxy collection install -r requirements.yml +export HCLOUD_TOKEN=your-hetzner-api-token # never commit this +``` + +`ansible_ssh_private_key_file` (see `group_vars/all.yml`) defaults to +`~/.ssh/id_ed25519` - override with `ANSIBLE_SSH_PRIVATE_KEY_FILE` if you use a +different key. It must match one of the SSH keys OpenTofu installed on the +servers (`var.ssh_key_names`). + +## Usage + +Same chicken-and-egg as OpenTofu's own first-apply ordering (see +`../readme.md`): `dev`/`prod`'s firewalls only accept SSH from `vpn`'s own +public IP, so you must bring `vpn` up and connect to it before `dev`/`prod` +are reachable at all. + +```sh +# 1. vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly +ansible-playbook playbooks/bootstrap.yml -l role_vpn +ansible-playbook playbooks/deploy.yml -l role_vpn +ansible-playbook playbooks/spinup.yml -l role_vpn + +# 2. connect to the VPN you just started with an OpenVPN client - now dev/prod +# are reachable, since your egress IP matches their firewall rule +ansible-playbook playbooks/bootstrap.yml -l role_dev,role_prod +ansible-playbook playbooks/deploy.yml -l role_dev,role_prod +ansible-playbook playbooks/spinup.yml -l role_dev,role_prod + +# ...or once already bootstrapped and connected, do everything in one go: +ansible-playbook playbooks/site.yml -l role_dev,role_prod +``` + +Re-run `deploy.yml` + `spinup.yml` any time a compose file changes or +`dev_runner_count` in `group_vars/role_dev.yml` is bumped - `spinup.sh` only +starts/updates what's changed, same as before. + +`spindown.yml` runs `spindown.sh`, which does a full `docker system/volume +prune -a` on the host - it prompts for a `yes` confirmation before running, +since the dynamic inventory makes it trivial to target multiple hosts at once +(`-l role_dev,role_prod`). + +Inspect what the dynamic inventory currently sees: + +```sh +ansible-inventory --graph +ansible-inventory --list +``` diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..c22a078 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,8 @@ +[defaults] +inventory = inventory/hcloud.yml +host_key_checking = False +retry_files_enabled = False +interpreter_python = auto_silent + +[inventory] +enable_plugins = hetzner.hcloud.hcloud, auto diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml new file mode 100644 index 0000000..5a6c35a --- /dev/null +++ b/ansible/group_vars/all.yml @@ -0,0 +1,14 @@ +--- +# Keep in sync with infra/variables.tf's var.deploy_user (default "deploy") - +# the two aren't wired together automatically, since Ansible no longer reads +# any Terraform state or output. +deploy_user: deploy +ansible_user: "{{ deploy_user }}" +ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}" + +# Used by the deploy role to look up each host's data volume directly from the +# Hetzner API (hcloud_volume_info) rather than from Terraform state/outputs. +hcloud_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}" + +# Repo root's services/ directory, relative to wherever a playbook lives under ansible/playbooks/. +services_root: "{{ playbook_dir }}/../../services" diff --git a/ansible/group_vars/role_dev.yml b/ansible/group_vars/role_dev.yml new file mode 100644 index 0000000..8e7a4ae --- /dev/null +++ b/ansible/group_vars/role_dev.yml @@ -0,0 +1,7 @@ +--- +service_group: dev + +# Number of Gitea Actions runner containers rendered into +# services/dev/Runners/docker-compose.yml by the deploy role - see +# roles/deploy/templates/runners-docker-compose.yml.j2. +dev_runner_count: 3 diff --git a/ansible/group_vars/role_prod.yml b/ansible/group_vars/role_prod.yml new file mode 100644 index 0000000..bf832e4 --- /dev/null +++ b/ansible/group_vars/role_prod.yml @@ -0,0 +1,2 @@ +--- +service_group: prod diff --git a/ansible/group_vars/role_vpn.yml b/ansible/group_vars/role_vpn.yml new file mode 100644 index 0000000..252118a --- /dev/null +++ b/ansible/group_vars/role_vpn.yml @@ -0,0 +1,2 @@ +--- +service_group: vpn diff --git a/ansible/inventory/hcloud.yml b/ansible/inventory/hcloud.yml new file mode 100644 index 0000000..17c6db5 --- /dev/null +++ b/ansible/inventory/hcloud.yml @@ -0,0 +1,12 @@ +--- +# Dynamic inventory: queries the Hetzner Cloud API directly (HCLOUD_TOKEN), +# grouping hosts by the `role` label set on each hcloud_server resource in +# infra/modules//main.tf (role = dev/prod/vpn). Nothing here reads +# Terraform state - this always reflects whatever actually exists in Hetzner. +plugin: hetzner.hcloud.hcloud +token: "{{ lookup('env', 'HCLOUD_TOKEN') }}" +connect_with: public_ipv4 +keyed_groups: + - key: labels.role + prefix: role + separator: "_" diff --git a/ansible/playbooks/bootstrap.yml b/ansible/playbooks/bootstrap.yml new file mode 100644 index 0000000..e015879 --- /dev/null +++ b/ansible/playbooks/bootstrap.yml @@ -0,0 +1,9 @@ +--- +- name: Bootstrap newly created servers (idempotent, safe to re-run) + hosts: all + gather_facts: true + become: false + vars: + ansible_user: root + roles: + - bootstrap diff --git a/ansible/playbooks/deploy.yml b/ansible/playbooks/deploy.yml new file mode 100644 index 0000000..996c578 --- /dev/null +++ b/ansible/playbooks/deploy.yml @@ -0,0 +1,6 @@ +--- +- name: Render templates and sync service definitions to each host + hosts: all + gather_facts: false + roles: + - deploy diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml new file mode 100644 index 0000000..286bee4 --- /dev/null +++ b/ansible/playbooks/site.yml @@ -0,0 +1,4 @@ +--- +- ansible.builtin.import_playbook: bootstrap.yml +- ansible.builtin.import_playbook: deploy.yml +- ansible.builtin.import_playbook: spinup.yml diff --git a/ansible/playbooks/spindown.yml b/ansible/playbooks/spindown.yml new file mode 100644 index 0000000..c09b3d7 --- /dev/null +++ b/ansible/playbooks/spindown.yml @@ -0,0 +1,27 @@ +--- +- name: Stop every compose stack on each host and prune Docker resources + hosts: all + gather_facts: false + vars_prompt: + - name: confirm + prompt: >- + This runs spindown.sh (docker compose down, then a full + docker image/volume prune -a) on the targeted host(s). + Type 'yes' to continue + private: false + tasks: + - name: Abort unless confirmed + ansible.builtin.fail: + msg: Aborted. + when: confirm != 'yes' + + - name: Run spindown.sh + ansible.builtin.command: ./spindown.sh + args: + chdir: "/home/{{ deploy_user }}/services/{{ service_group }}" + register: _spindown + changed_when: true + + - name: Show spindown output + ansible.builtin.debug: + var: _spindown.stdout_lines diff --git a/ansible/playbooks/spinup.yml b/ansible/playbooks/spinup.yml new file mode 100644 index 0000000..bff53f9 --- /dev/null +++ b/ansible/playbooks/spinup.yml @@ -0,0 +1,15 @@ +--- +- name: Start every compose stack on each host, in the order spinup.sh defines + hosts: all + gather_facts: false + tasks: + - name: Run spinup.sh + ansible.builtin.command: ./spinup.sh + args: + chdir: "/home/{{ deploy_user }}/services/{{ service_group }}" + register: _spinup + changed_when: true + + - name: Show spinup output + ansible.builtin.debug: + var: _spinup.stdout_lines diff --git a/ansible/requirements.yml b/ansible/requirements.yml new file mode 100644 index 0000000..c36d0d9 --- /dev/null +++ b/ansible/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - name: hetzner.hcloud + version: ">=4.0.0" diff --git a/ansible/roles/bootstrap/handlers/main.yml b/ansible/roles/bootstrap/handlers/main.yml new file mode 100644 index 0000000..9bb29ca --- /dev/null +++ b/ansible/roles/bootstrap/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload sshd + ansible.builtin.systemd: + name: ssh + state: reloaded diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml new file mode 100644 index 0000000..c476d32 --- /dev/null +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -0,0 +1,123 @@ +--- +# Post-install bootstrap - replaces infra/scripts/bootstrap.sh.tftpl. Runs as +# root (see ansible_user override in playbooks/bootstrap.yml) since the deploy +# user doesn't exist yet on a freshly created server. Idempotent: safe to +# re-run against an already-bootstrapped host. + +- name: Update apt cache and upgrade packages + ansible.builtin.apt: + update_cache: true + upgrade: dist + cache_valid_time: 3600 + +- name: Install Docker prerequisites + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + state: present + +- name: Create apt keyrings directory + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: "0755" + +- name: Download Docker's GPG key + ansible.builtin.get_url: + url: https://download.docker.com/linux/ubuntu/gpg + dest: /etc/apt/keyrings/docker.asc + mode: "0644" + force: false + +- name: Determine dpkg architecture + ansible.builtin.command: dpkg --print-architecture + register: _dpkg_arch + changed_when: false + +- name: Add Docker apt repository + ansible.builtin.apt_repository: + repo: >- + deb [arch={{ _dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] + https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable + filename: docker + state: present + +- name: Install Docker Engine and Compose plugin + ansible.builtin.apt: + update_cache: true + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + +- name: Create non-root sudo user, given the same key root was provisioned with + ansible.builtin.user: + name: "{{ deploy_user }}" + shell: /bin/bash + groups: sudo,docker + append: true + create_home: true + +- name: Ensure deploy user's .ssh directory exists + ansible.builtin.file: + path: "/home/{{ deploy_user }}/.ssh" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0700" + +- name: Give deploy user the same authorized_keys as root + ansible.builtin.copy: + src: /root/.ssh/authorized_keys + dest: "/home/{{ deploy_user }}/.ssh/authorized_keys" + remote_src: true + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0600" + +- name: Grant deploy user passwordless sudo + ansible.builtin.copy: + content: "{{ deploy_user }} ALL=(ALL) NOPASSWD:ALL\n" + dest: "/etc/sudoers.d/90-{{ deploy_user }}" + validate: "visudo -cf %s" + mode: "0440" + +# Root keeps key-only login as a fallback rather than being fully locked out, +# in case deploy user setup above ever fails silently on a future image. +- name: Restrict root login to key-only + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + regexp: "^#?PermitRootLogin" + line: "PermitRootLogin prohibit-password" + notify: Reload sshd + +- name: Disable SSH password authentication + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + regexp: "^#?PasswordAuthentication" + line: "PasswordAuthentication no" + notify: Reload sshd + +- name: Install unattended-upgrades + ansible.builtin.apt: + name: unattended-upgrades + state: present + +- name: Enable automatic security upgrades + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/20auto-upgrades + content: | + APT::Periodic::Update-Package-Lists "1"; + APT::Periodic::Unattended-Upgrade "1"; + mode: "0644" + +- name: Enable and start unattended-upgrades + ansible.builtin.systemd: + name: unattended-upgrades + enabled: true + state: started diff --git a/ansible/roles/deploy/tasks/main.yml b/ansible/roles/deploy/tasks/main.yml new file mode 100644 index 0000000..a24c27d --- /dev/null +++ b/ansible/roles/deploy/tasks/main.yml @@ -0,0 +1,55 @@ +--- +# Copies services// to the server and renders the files that +# used to be generated by OpenTofu (services//.env and, for dev, +# Runners/docker-compose.yml) - replaces the null_resource.deploy_services + +# local_file provisioners previously in infra/modules//main.tf. + +- name: Ensure services directory exists on the host + ansible.builtin.file: + path: "/home/{{ deploy_user }}/services/{{ service_group }}" + state: directory + mode: "0755" + +- name: Copy static service files (compose files, scripts, configs) + ansible.builtin.copy: + src: "{{ services_root }}/{{ service_group }}/" + dest: "/home/{{ deploy_user }}/services/{{ service_group }}/" + mode: preserve + +- name: Look up this host's data volume + when: service_group in ['dev', 'prod'] + delegate_to: localhost + become: false + hetzner.hcloud.hcloud_volume_info: + api_token: "{{ hcloud_token }}" + name: "{{ service_group }}-storage" + register: _volume_info + +# Hetzner's automount tooling always mounts an auto-mounted volume at +# /mnt/HC_Volume_ - deterministic once the volume exists. +- name: Compute DATA_DIR from the volume's id + when: service_group in ['dev', 'prod'] + ansible.builtin.set_fact: + data_dir: "/mnt/HC_Volume_{{ _volume_info.hcloud_volume_info[0].id }}" + +- name: Render DATA_DIR into .env (auto-loaded by docker compose) + when: service_group in ['dev', 'prod'] + ansible.builtin.template: + src: env.j2 + dest: "/home/{{ deploy_user }}/services/{{ service_group }}/.env" + mode: "0644" + +- name: Render Gitea Actions runner compose file + when: service_group == 'dev' + ansible.builtin.template: + src: runners-docker-compose.yml.j2 + dest: "/home/{{ deploy_user }}/services/dev/Runners/docker-compose.yml" + mode: "0644" + +- name: Make spinup/spindown scripts executable + ansible.builtin.file: + path: "/home/{{ deploy_user }}/services/{{ service_group }}/{{ item }}" + mode: "0755" + loop: + - spinup.sh + - spindown.sh diff --git a/ansible/roles/deploy/templates/env.j2 b/ansible/roles/deploy/templates/env.j2 new file mode 100644 index 0000000..a958fd1 --- /dev/null +++ b/ansible/roles/deploy/templates/env.j2 @@ -0,0 +1 @@ +DATA_DIR={{ data_dir }} diff --git a/infra/modules/dev/templates/runners-docker-compose.yml.tftpl b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 similarity index 54% rename from infra/modules/dev/templates/runners-docker-compose.yml.tftpl rename to ansible/roles/deploy/templates/runners-docker-compose.yml.j2 index bb42e52..f72c94f 100644 --- a/infra/modules/dev/templates/runners-docker-compose.yml.tftpl +++ b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 @@ -1,34 +1,34 @@ -# Generated by OpenTofu from var.dev_runner_count - do not hand-edit. -# To change the number of runners, edit dev_runner_count in terraform.tfvars and -# run tofu apply. To change their shape, edit -# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead. +# Generated by Ansible from dev_runner_count - do not hand-edit. +# To change the number of runners, edit dev_runner_count in +# ansible/group_vars/role_dev.yml and re-run playbooks/deploy.yml. +# To change their shape, edit this template instead. # # GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable # (not baked in here) - services/dev/spinup.sh generates a fresh token and # writes it to Runners/.env immediately before starting these containers. # -# /data lives on dev's volume (${data_dir}) so runner identity survives a +# /data lives on dev's volume ({{ data_dir }}) so runner identity survives a # server rebuild - baked in directly rather than via .env, since Runners/.env # is already reserved for the registration token above and gets overwritten # on every deploy. services: -%{ for i in range(runner_count) ~} - runner-${i + 1}: +{% for i in range(1, dev_runner_count + 1) %} + runner-{{ i }}: image: gitea/act_runner:latest - container_name: gitea_runner_${i + 1} + container_name: gitea_runner_{{ i }} volumes: - ./config.yaml:/config.yaml - - ${data_dir}/gitea_runner_${i + 1}:/data + - {{ data_dir }}/gitea_runner_{{ i }}:/data - /var/run/docker.sock:/var/run/docker.sock networks: - proxy environment: CONFIG_FILE: /config.yaml GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" - GITEA_RUNNER_REGISTRATION_TOKEN: "$${GITEA_RUNNER_REGISTRATION_TOKEN}" - GITEA_RUNNER_NAME: "CICD#${i + 1}" + GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}" + GITEA_RUNNER_NAME: "CICD#{{ i }}" restart: unless-stopped -%{ endfor ~} +{% endfor %} networks: proxy: diff --git a/infra/main.tf b/infra/main.tf index 328bd6b..9df4e16 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -9,12 +9,6 @@ locals { # var.allowed_ssh_source_ips for direct SSH access. vpn_ssh_source_ips = ["${module.vpn.ipv4}/32"] - # Rendered once and uploaded+run by every host module immediately after its - # server is created - see scripts/bootstrap.sh.tftpl. - bootstrap_script = templatefile("${path.module}/scripts/bootstrap.sh.tftpl", { - deploy_user = var.deploy_user - }) - # IDs for every named key in var.ssh_key_names - installed on every server. ssh_key_ids = [for k in data.hcloud_ssh_key.main : k.id] @@ -59,10 +53,6 @@ module "dev" { volume_size = var.dev_volume_size allowed_ssh_source_ips = local.vpn_ssh_source_ips network_ip_range = var.network_ip_range - bootstrap_script = local.bootstrap_script - ssh_private_key_path = var.ssh_private_key_path - deploy_user = var.deploy_user - runner_count = var.dev_runner_count # module.network.id alone doesn't guarantee the subnet exists yet, and a server # can't join a network before it has a subnet. @@ -81,9 +71,6 @@ module "prod" { volume_size = var.prod_volume_size allowed_ssh_source_ips = local.vpn_ssh_source_ips network_ip_range = var.network_ip_range - bootstrap_script = local.bootstrap_script - ssh_private_key_path = var.ssh_private_key_path - deploy_user = var.deploy_user depends_on = [module.network] } @@ -96,9 +83,6 @@ module "vpn" { location = var.location ssh_key_ids = local.ssh_key_ids allowed_ssh_source_ips = var.allowed_ssh_source_ips - bootstrap_script = local.bootstrap_script - ssh_private_key_path = var.ssh_private_key_path - deploy_user = var.deploy_user } module "dns" { diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf index aed7406..6bea690 100644 --- a/infra/modules/dev/main.tf +++ b/infra/modules/dev/main.tf @@ -56,30 +56,12 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] + labels = { role = "dev" } # picked up by ansible/inventory/hcloud.yml network { network_id = var.network_id ip = var.private_ip } - - connection { - type = "ssh" - host = self.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "file" { - content = var.bootstrap_script - destination = "/root/bootstrap.sh" - } - - provisioner "remote-exec" { - inline = [ - "chmod +x /root/bootstrap.sh", - "/root/bootstrap.sh", - ] - } } resource "hcloud_volume" "storage" { @@ -98,85 +80,8 @@ locals { # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is # always /mnt/HC_Volume_ - deterministic once the volume exists, since the # id is stable for the volume's lifetime regardless of which server it's - # attached to. + # attached to. Kept here only as a human-readable output (see outputs.tf) - + # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't + # read this value. data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}" } - -# Renders services/dev/Runners/docker-compose.yml directly into the repo, with -# one runner service per var.runner_count. This only touches the local working -# tree - deploying the change still goes through the normal scp + spinup.sh flow. -resource "local_file" "runners_compose" { - filename = "${path.root}/../services/dev/Runners/docker-compose.yml" - content = templatefile("${path.module}/templates/runners-docker-compose.yml.tftpl", { - runner_count = var.runner_count - data_dir = local.data_dir - }) - file_permission = "0644" -} - -# DATA_DIR is picked up automatically by `docker compose` (which auto-loads -# .env from its working directory) for every dev/*-docker-compose.yml bind -# mount - see services/dev/*.yml. Not committed (see .gitignore): it's tied to -# the live volume's ID, so it's regenerated by tofu apply, not handed off via git. -resource "local_file" "env" { - filename = "${path.root}/../services/dev/.env" - content = "DATA_DIR=${local.data_dir}\n" - file_permission = "0644" -} - -locals { - # Everything under services/dev except the two files above: those are - # tracked via their own resource content (reading them back with fileset()/ - # filesha1() before they exist would fail during `tofu plan`). - dev_static_files = sort([ - for f in fileset("${path.root}/../services/dev", "**") : - f if f != ".env" && f != "Runners/docker-compose.yml" - ]) - - dev_static_files_hash = sha1(join("", [ - for f in local.dev_static_files : filesha1("${path.root}/../services/dev/${f}") - ])) -} - -# Copies services/dev to the server on every apply that changes it (a hand-edited -# compose file, a new runner count, ...) or recreates the server - not just once -# at first creation. Split from hcloud_server.this because it must run after the -# generated files above, which in turn depend on the volume, which depends on -# the server - so it can't be a provisioner on the server resource itself. -resource "null_resource" "deploy_services" { - triggers = { - server_id = hcloud_server.this.id - static_files = local.dev_static_files_hash - env = local_file.env.content - runners = local_file.runners_compose.content - } - - connection { - type = "ssh" - host = hcloud_server.this.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "remote-exec" { - inline = ["mkdir -p /home/${var.deploy_user}/services"] - } - - provisioner "file" { - source = "${path.root}/../services/dev" - destination = "/home/${var.deploy_user}/services" - } - - provisioner "remote-exec" { - inline = [ - "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services", - "chmod +x /home/${var.deploy_user}/services/dev/*.sh", - ] - } - - depends_on = [ - hcloud_server.this, - local_file.env, - local_file.runners_compose, - ] -} diff --git a/infra/modules/dev/variables.tf b/infra/modules/dev/variables.tf index 49d475e..89dbb4e 100644 --- a/infra/modules/dev/variables.tf +++ b/infra/modules/dev/variables.tf @@ -42,23 +42,3 @@ variable "network_ip_range" { description = "CIDR of the private network, allowed through the firewall for traffic from prod." type = string } - -variable "bootstrap_script" { - description = "Rendered post-install script, uploaded and executed on the server immediately after creation." - type = string -} - -variable "ssh_private_key_path" { - description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." - type = string -} - -variable "runner_count" { - description = "Number of Gitea Actions runner containers to render into services/dev/Runners/docker-compose.yml." - type = number -} - -variable "deploy_user" { - description = "Non-root sudo user created by the bootstrap script - services/dev is copied into this user's home directory." - type = string -} diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index f3dc6f1..8135cae 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -83,30 +83,12 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] + labels = { role = "prod" } # picked up by ansible/inventory/hcloud.yml network { network_id = var.network_id ip = var.private_ip } - - connection { - type = "ssh" - host = self.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "file" { - content = var.bootstrap_script - destination = "/root/bootstrap.sh" - } - - provisioner "remote-exec" { - inline = [ - "chmod +x /root/bootstrap.sh", - "/root/bootstrap.sh", - ] - } } resource "hcloud_volume" "storage" { @@ -125,70 +107,8 @@ locals { # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is # always /mnt/HC_Volume_ - deterministic once the volume exists, since the # id is stable for the volume's lifetime regardless of which server it's - # attached to. + # attached to. Kept here only as a human-readable output (see outputs.tf) - + # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't + # read this value. data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}" } - -# DATA_DIR is picked up automatically by `docker compose` (which auto-loads -# .env from its working directory) for every prod/*-docker-compose.yml bind -# mount - see services/prod/*.yml. Not committed (see .gitignore): it's tied to -# the live volume's ID, so it's regenerated by tofu apply, not handed off via git. -resource "local_file" "env" { - filename = "${path.root}/../services/prod/.env" - content = "DATA_DIR=${local.data_dir}\n" - file_permission = "0644" -} - -locals { - # Everything under services/prod except .env: that's tracked via its own - # resource content (reading it back with fileset()/filesha1() before it - # exists would fail during `tofu plan`). - prod_static_files = sort([ - for f in fileset("${path.root}/../services/prod", "**") : f if f != ".env" - ]) - - prod_static_files_hash = sha1(join("", [ - for f in local.prod_static_files : filesha1("${path.root}/../services/prod/${f}") - ])) -} - -# Copies services/prod to the server on every apply that changes it (a -# hand-edited compose file, ...) or recreates the server - not just once at -# first creation. Split from hcloud_server.this because it must run after -# local_file.env, which in turn depends on the volume, which depends on the -# server - so it can't be a provisioner on the server resource itself. -resource "null_resource" "deploy_services" { - triggers = { - server_id = hcloud_server.this.id - static_files = local.prod_static_files_hash - env = local_file.env.content - } - - connection { - type = "ssh" - host = hcloud_server.this.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "remote-exec" { - inline = ["mkdir -p /home/${var.deploy_user}/services"] - } - - provisioner "file" { - source = "${path.root}/../services/prod" - destination = "/home/${var.deploy_user}/services" - } - - provisioner "remote-exec" { - inline = [ - "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services", - "chmod +x /home/${var.deploy_user}/services/prod/*.sh", - ] - } - - depends_on = [ - hcloud_server.this, - local_file.env, - ] -} diff --git a/infra/modules/prod/variables.tf b/infra/modules/prod/variables.tf index bab01c1..e105b70 100644 --- a/infra/modules/prod/variables.tf +++ b/infra/modules/prod/variables.tf @@ -42,18 +42,3 @@ variable "network_ip_range" { description = "CIDR of the private network, allowed through the firewall for traffic from dev." type = string } - -variable "bootstrap_script" { - description = "Rendered post-install script, uploaded and executed on the server immediately after creation." - type = string -} - -variable "ssh_private_key_path" { - description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." - type = string -} - -variable "deploy_user" { - description = "Non-root sudo user created by the bootstrap script - services/prod is copied into this user's home directory." - type = string -} diff --git a/infra/modules/vpn/main.tf b/infra/modules/vpn/main.tf index b62fb33..c73396e 100644 --- a/infra/modules/vpn/main.tf +++ b/infra/modules/vpn/main.tf @@ -38,67 +38,5 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] - - connection { - type = "ssh" - host = self.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "file" { - content = var.bootstrap_script - destination = "/root/bootstrap.sh" - } - - provisioner "remote-exec" { - inline = [ - "chmod +x /root/bootstrap.sh", - "/root/bootstrap.sh", - ] - } -} - -locals { - vpn_static_files = sort(fileset("${path.root}/../services/vpn", "**")) - - vpn_static_files_hash = sha1(join("", [ - for f in local.vpn_static_files : filesha1("${path.root}/../services/vpn/${f}") - ])) -} - -# Copies services/vpn to the server on every apply that changes it (a -# hand-edited compose file, ...) or recreates the server - not just once at -# first creation. Split from hcloud_server.this so it's triggered by content -# changes independently of the bootstrap provisioners above. -resource "null_resource" "deploy_services" { - triggers = { - server_id = hcloud_server.this.id - static_files = local.vpn_static_files_hash - } - - connection { - type = "ssh" - host = hcloud_server.this.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "remote-exec" { - inline = ["mkdir -p /home/${var.deploy_user}/services"] - } - - provisioner "file" { - source = "${path.root}/../services/vpn" - destination = "/home/${var.deploy_user}/services" - } - - provisioner "remote-exec" { - inline = [ - "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services", - "chmod +x /home/${var.deploy_user}/services/vpn/*.sh", - ] - } - - depends_on = [hcloud_server.this] + labels = { role = "vpn" } # picked up by ansible/inventory/hcloud.yml } diff --git a/infra/modules/vpn/variables.tf b/infra/modules/vpn/variables.tf index 384a4ad..022e909 100644 --- a/infra/modules/vpn/variables.tf +++ b/infra/modules/vpn/variables.tf @@ -22,18 +22,3 @@ variable "allowed_ssh_source_ips" { description = "CIDRs allowed to reach port 22 on vpn." type = list(string) } - -variable "bootstrap_script" { - description = "Rendered post-install script, uploaded and executed on the server immediately after creation." - type = string -} - -variable "ssh_private_key_path" { - description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." - type = string -} - -variable "deploy_user" { - description = "Non-root sudo user created by the bootstrap script - services/vpn is copied into this user's home directory." - type = string -} diff --git a/infra/scripts/bootstrap.sh.tftpl b/infra/scripts/bootstrap.sh.tftpl deleted file mode 100644 index 3227620..0000000 --- a/infra/scripts/bootstrap.sh.tftpl +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# Post-install bootstrap - uploaded and executed once by OpenTofu immediately -# after server creation (see modules//main.tf). Idempotent-ish: safe to -# re-run by hand, but only ever runs automatically on first create. -set -euo pipefail - -export DEBIAN_FRONTEND=noninteractive -DEPLOY_USER="${deploy_user}" - -apt-get update -y -apt-get upgrade -y - -# --- Docker Engine + Compose plugin --- -if ! command -v docker >/dev/null 2>&1; then - apt-get install -y ca-certificates curl gnupg - install -m 0755 -d /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - chmod a+r /etc/apt/keyrings/docker.asc - . /etc/os-release - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $VERSION_CODENAME stable" \ - > /etc/apt/sources.list.d/docker.list - apt-get update -y - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -fi - -# --- Non-root sudo user, given the same key root was provisioned with --- -if ! id -u "$DEPLOY_USER" >/dev/null 2>&1; then - useradd --create-home --shell /bin/bash "$DEPLOY_USER" - usermod -aG sudo,docker "$DEPLOY_USER" - mkdir -p "/home/$DEPLOY_USER/.ssh" - cp /root/.ssh/authorized_keys "/home/$DEPLOY_USER/.ssh/authorized_keys" - chown -R "$DEPLOY_USER:$DEPLOY_USER" "/home/$DEPLOY_USER/.ssh" - chmod 700 "/home/$DEPLOY_USER/.ssh" - chmod 600 "/home/$DEPLOY_USER/.ssh/authorized_keys" - echo "$DEPLOY_USER ALL=(ALL) NOPASSWD:ALL" > "/etc/sudoers.d/90-$DEPLOY_USER" - chmod 440 "/etc/sudoers.d/90-$DEPLOY_USER" -fi - -# --- SSH hardening: key-only auth everywhere. Root keeps key-only login as a -# fallback rather than being fully locked out, in case $DEPLOY_USER setup above -# ever fails silently on a future image. --- -sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config -sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config -systemctl reload ssh 2>/dev/null || systemctl reload sshd - -# --- Unattended security upgrades --- -apt-get install -y unattended-upgrades -dpkg-reconfigure -f noninteractive unattended-upgrades -systemctl enable --now unattended-upgrades diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 7ab14ec..3c8c22b 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -2,18 +2,13 @@ # never commit real values there. # # The Hetzner API token is NOT set here: export it as HCLOUD_TOKEN in your shell -# before running tofu plan/apply. +# before running tofu plan/apply. The same token and SSH key are used again by +# Ansible afterwards - see ../ansible/README.md. # Names of SSH keys already uploaded to your Hetzner Cloud project # (Console > Security > SSH Keys). All of them are installed on every server. Required. ssh_key_names = ["your-key-name", "laptop", "phone"] -# Local path to the private key matching ONE of ssh_key_names above (doesn't need -# to be all of them - just one you hold). OpenTofu uses this once per server to -# upload and run the post-install bootstrap script -# (infra/scripts/bootstrap.sh.tftpl) immediately after creation. Required. -ssh_private_key_path = "~/.ssh/id_ed25519" - # Optional overrides - defaults live in variables.tf # location = "nbg1" # dev_server_type = "cx22" @@ -22,6 +17,4 @@ ssh_private_key_path = "~/.ssh/id_ed25519" # dev_volume_size = 20 # prod_volume_size = 20 # allowed_ssh_source_ips = ["203.0.113.4/32"] -# deploy_user = "deploy" -# dev_runner_count = 3 # dns_zones = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"] diff --git a/infra/variables.tf b/infra/variables.tf index 10f6f43..95dc546 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -81,23 +81,6 @@ variable "allowed_ssh_source_ips" { default = ["0.0.0.0/0", "::/0"] } -variable "ssh_private_key_path" { - description = "Local path to the private key matching one of var.ssh_key_names - only needs to be a key you hold yourself, not all of them. Used once per server by OpenTofu to upload and run the post-install bootstrap script over SSH immediately after creation (see infra/scripts/bootstrap.sh.tftpl)." - type = string -} - -variable "deploy_user" { - description = "Non-root sudo user created on every server by the bootstrap script, with the same SSH key as root." - type = string - default = "deploy" -} - -variable "dev_runner_count" { - description = "Number of Gitea Actions runner containers to run on dev. Rendered into services/dev/Runners/docker-compose.yml on every tofu apply - see infra/modules/dev/templates/runners-docker-compose.yml.tftpl." - type = number - default = 1 -} - variable "dns_zones" { description = "Domains to manage as Hetzner DNS zones. Point each domain's registrar NS records at tofu output dns_nameservers for Hetzner to actually become authoritative." type = list(string) diff --git a/infra/versions.tf b/infra/versions.tf index 303b689..f817710 100644 --- a/infra/versions.tf +++ b/infra/versions.tf @@ -6,14 +6,6 @@ terraform { source = "hetznercloud/hcloud" version = "~> 1.54" # hcloud_zone / hcloud_zone_rrset (DNS) require >= 1.54.0 } - local = { - source = "hashicorp/local" - version = "~> 2.5" - } - null = { - source = "hashicorp/null" - version = "~> 3.2" - } } } diff --git a/readme.md b/readme.md index f35a312..2e017e4 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # Server -Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/). +Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each bootstrapped and deployed with [Ansible](https://www.ansible.com/), running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).

@@ -12,7 +12,7 @@ Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted - [Repository layout](#repository-layout) - [Prerequisites](#prerequisites) - [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) -- [Deploying the services](#deploying-the-services-services) +- [Bootstrapping and deploying with Ansible](#bootstrapping-and-deploying-with-ansible-ansible) - [Service inventory](#service-inventory) - [First-time setup](#first-time-setup) - [Development container](#development-container) @@ -66,23 +66,26 @@ architecture-beta ``` . -├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes -│ ├── main.tf # root module: wires network + dev/prod/vpn modules together +├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes, DNS +│ ├── main.tf # root module: wires network + dev/prod/vpn/dns modules together │ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names) │ ├── outputs.tf # pass-through outputs from each host module │ ├── ssh.tf # looks up each SSH key already uploaded to Hetzner Cloud │ ├── versions.tf # provider requirements │ ├── terraform.tfvars.example -│ ├── scripts/ -│ │ └── bootstrap.sh.tftpl # post-install script run on every host right after creation │ └── modules/ │ ├── network/ # shared private network + subnet (used by dev and prod) -│ ├── dev/ # dev server + firewall + volume, renders + deploys services/dev/ -│ │ └── templates/ -│ │ └── runners-docker-compose.yml.tftpl # shape of each generated runner service -│ ├── prod/ # prod server + firewall + volume, deploys services/prod/ -│ ├── vpn/ # vpn server + firewall (no private network, no volume), deploys services/vpn/ +│ ├── dev/ # dev server + firewall + volume, labeled role=dev +│ ├── prod/ # prod server + firewall + volume, labeled role=prod +│ ├── vpn/ # vpn server + firewall (no private network, no volume), labeled role=vpn │ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones +├── ansible/ # Ansible: bootstraps each server and deploys/starts services// onto it +│ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by the role label above +│ ├── group_vars/ # deploy_user, dev_runner_count, etc. +│ ├── roles/ +│ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades +│ │ └── deploy/ # copies services//, renders .env + Runners/docker-compose.yml +│ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml ├── services/ # Docker Compose stacks, grouped by which server they run on │ ├── dev/ # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) │ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik @@ -94,16 +97,19 @@ architecture-beta └── assets/ ``` -Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` mirrors this same dev/prod/vpn split on the provisioning side, so a given host's cloud resources and its compose stacks are easy to find side by side. +Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side. + +OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, volumes, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services//`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown. ## Prerequisites - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token -- One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (OpenTofu uses it once per server to run the post-install bootstrap — see below) +- One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (Ansible uses it to bootstrap and deploy — see below) - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` +- [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`) - Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner (see [Managing DNS](#managing-dns) — the zones and records themselves are created for you) -Docker + the Compose plugin no longer need installing by hand — the bootstrap script below handles that. +Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below). ## Provisioning the infrastructure (`infra/`) @@ -111,7 +117,7 @@ Docker + the Compose plugin no longer need installing by hand — the bootstrap cd infra export HCLOUD_TOKEN=your-hetzner-api-token # never commit this cp terraform.tfvars.example terraform.tfvars -$EDITOR terraform.tfvars # set ssh_key_names and ssh_private_key_path at minimum +$EDITOR terraform.tfvars # set ssh_key_names at minimum tofu init tofu plan @@ -120,52 +126,15 @@ tofu apply This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` / `module.dns` in `infra/main.tf`: - `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`) -- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses (`modules/dev`, `modules/prod`, `modules/vpn`) +- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses, each server labeled `role = "dev"/"prod"/"vpn"` for Ansible's dynamic inventory (`modules/dev`, `modules/prod`, `modules/vpn`) - one `hcloud_volume` each for `dev` and `prod` (`modules/dev`, `modules/prod`; `vpn` has none) - one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns)) -Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`, `tofu output dev_data_dir`, `tofu output prod_data_dir`. +Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`, `tofu output dev_data_dir`, `tofu output prod_data_dir` (the last two are informational only now — Ansible looks the data directory up itself, see below). `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`. -### Post-install bootstrap - -Immediately after each `hcloud_server` is created, OpenTofu uploads and runs [`infra/scripts/bootstrap.sh.tftpl`](infra/scripts/bootstrap.sh.tftpl) over SSH as `root` (`connection` + `file`/`remote-exec` provisioners in each `modules//main.tf`). It: - -- installs Docker Engine + the Compose plugin -- creates a non-root sudo user (`var.deploy_user`, default `deploy`) with the same SSH keys as root, in the `sudo` and `docker` groups -- hardens `sshd`: password authentication off, root login restricted to key-only (`PermitRootLogin prohibit-password`) -- installs and enables `unattended-upgrades` for automatic security patches - -After bootstrap, SSH in as `var.deploy_user` for day-to-day work — root key-based login still works as a fallback. - -### Persistent data lives on the volumes, not the server disk - -`dev` and `prod`'s compose files used to bind-mount container data to paths relative to wherever `services//` happened to be copied on the server (e.g. `./gitea:/data`) - i.e. the server's local disk, which doesn't survive the server being destroyed and recreated. `dev-storage` and `prod-storage` (the `hcloud_volume`s) do survive that, so all stateful bind mounts now point there instead: `${DATA_DIR}/gitea:/data`, `${DATA_DIR}/bitwarden/:/data/`, and so on across `services/dev/*.yml` and `services/prod/*.yml`. `vpn` has no volume, so it's untouched - see the architecture table above. - -`DATA_DIR` needing to resolve to the right path is what makes this work, and that's answered deterministically: Hetzner's automount tooling always mounts an auto-mounted volume at `/mnt/HC_Volume_`, and since the volume is a Terraform resource, `modules//main.tf` already knows its `id` the moment it's created - `local.data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}"`. Every `tofu apply` writes that value straight into `services/dev/.env` / `services/prod/.env` as `DATA_DIR=...`, which `docker compose` auto-loads for variable substitution in any compose file run from that directory (the same mechanism the Gitea runner token already uses). Query it directly with `tofu output dev_data_dir` / `tofu output prod_data_dir`. - -These `.env` files are deliberately not committed (see `.gitignore`): the path is tied to the *live* volume's ID, so it's regenerated by `tofu apply`, not handed off via git. Run `tofu apply` at least once before the first deploy, and again if a volume is ever destroyed and recreated (new ID); `spinup.sh` on both hosts refuses to start anything if `.env` is missing, rather than silently falling back to a relative path. - -The Gitea Actions runners are the one exception to the `.env` mechanism: their `/data` path is baked directly into the generated `Runners/docker-compose.yml` at render time (via the same `templatefile()` call as `dev_runner_count`), because `Runners/.env` is already reserved for the registration token and gets overwritten on every deploy. - -### Scaling Gitea Actions runners - -The number of Gitea Actions runner containers on `dev` is an OpenTofu input, `var.dev_runner_count` (default `3`). On every `tofu apply`, `modules/dev`'s `local_file.runners_compose` resource renders [`modules/dev/templates/runners-docker-compose.yml.tftpl`](infra/modules/dev/templates/runners-docker-compose.yml.tftpl) straight into [`services/dev/Runners/docker-compose.yml`](services/dev/Runners/docker-compose.yml) in your working tree — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated: change `dev_runner_count` in `terraform.tfvars` and re-run `tofu apply` rather than hand-editing it. - -This only updates your local working tree — `tofu apply` doesn't reach out and start containers on `dev` itself. Redeploy as usual (re-copy `services/dev/` to the server and re-run `spinup.sh`) to apply a count change. - -Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore. - -### First apply: bootstrapping order matters - -`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch `tofu apply`, the `dev`/`prod` bootstrap provisioners can't connect yet. Bring the estate up in this order: - -1. `tofu apply -target=module.vpn` — creates `vpn` only; its firewall still allows SSH from `var.allowed_ssh_source_ips`, so its bootstrap provisioner and `null_resource.deploy_services` (which copies `services/vpn` over) both run fine. -2. SSH in as `var.deploy_user` and run `~/services/vpn/spinup.sh` to start the VPN service, then connect to it with an OpenVPN client. -3. `tofu apply` — creates `network`, `dev`, `prod`. Now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and the `dev`/`prod` bootstrap + deploy provisioners can connect. - -If step 3 is run before you're connected to the VPN, the `dev`/`prod` provisioners will fail and Terraform will mark those servers tainted — re-running `tofu apply` once connected will recreate and re-bootstrap them. +OpenTofu never connects to the servers over SSH — no provisioners, no `bootstrap.sh`, no copying `services/` — that's all Ansible now. See below. ### Managing DNS @@ -177,9 +146,53 @@ Adding a new subdomain: add an entry to `local.dns_records` in `infra/main.tf` ( Only A records for IPv4 are managed here — none of the modules currently track servers' IPv6 addresses, so AAAA records aren't generated even though the firewalls already allow IPv6 traffic. +## Bootstrapping and deploying with Ansible (`ansible/`) + +Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services//` to each server, rendering the files OpenTofu used to generate (`.env`'s `DATA_DIR`, `dev`'s `Runners/docker-compose.yml`), and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version: + +```sh +cd ansible +ansible-galaxy collection install -r requirements.yml +export HCLOUD_TOKEN=your-hetzner-api-token # never commit this + +# vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly +ansible-playbook playbooks/site.yml -l role_vpn + +# SSH to vpn as deploy and connect to the OpenVPN it just started, then: +ansible-playbook playbooks/site.yml -l role_dev,role_prod +``` + +Hosts are discovered dynamically from the Hetzner API (`ansible/inventory/hcloud.yml`), grouped into `role_dev`/`role_prod`/`role_vpn` by the `role` label OpenTofu sets on each server — there's no static inventory file to keep in sync, and nothing here reads Terraform state. + +### Persistent data lives on the volumes, not the server disk + +`dev` and `prod`'s compose files bind-mount container data to paths on the `dev-storage` / `prod-storage` volumes rather than the server's local disk, so it survives the server being destroyed and recreated: `${DATA_DIR}/gitea:/data`, `${DATA_DIR}/bitwarden/:/data/`, and so on across `services/dev/*.yml` and `services/prod/*.yml`. `vpn` has no volume, so it's untouched - see the architecture table above. + +`DATA_DIR` resolves deterministically: Hetzner always automounts a volume at `/mnt/HC_Volume_`. Ansible's `deploy` role looks the volume up by name (`hcloud_volume_info` for `dev-storage`/`prod-storage`) directly against the Hetzner API and renders that path into `services//.env` as `DATA_DIR=...` on the remote host — `docker compose` auto-loads `.env` from its working directory for variable substitution. Nothing is written locally; re-run `ansible-playbook playbooks/deploy.yml` if a volume is ever destroyed and recreated (new id). `spinup.sh` on both hosts refuses to start anything if `.env` is missing, rather than silently falling back to a relative path. + +The Gitea Actions runners are the one exception to the `.env` mechanism: their `/data` path is baked directly into the rendered `Runners/docker-compose.yml`, because `Runners/.env` is already reserved for the registration token and gets overwritten on every deploy. + +### Scaling Gitea Actions runners + +The number of Gitea Actions runner containers on `dev` is set by `dev_runner_count` in [`ansible/group_vars/role_dev.yml`](ansible/group_vars/role_dev.yml) (default `3`). Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host: change `dev_runner_count` and redeploy rather than hand-editing it. + +Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a count change. + +Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore. + +### First deploy: bootstrapping order matters + +`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch estate, Ansible can't reach `dev`/`prod` yet. Bring it up in this order: + +1. `ansible-playbook playbooks/site.yml -l role_vpn` — bootstraps, deploys, and starts `vpn` only; its firewall allows SSH from `var.allowed_ssh_source_ips` directly. +2. SSH in as `deploy` and connect to the OpenVPN service you just started with an OpenVPN client. +3. `ansible-playbook playbooks/site.yml -l role_dev,role_prod` — now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and `dev`/`prod` become reachable. + +If step 3 is run before you're connected to the VPN, Ansible will simply fail to connect — re-run it once connected. + ## Deploying the services (`services/`) -`tofu apply` already puts `services//` on the matching server for you — a `null_resource.deploy_services` in each `modules//main.tf` copies it to `/home//services/` and `chown`s it to `var.deploy_user`, re-running whenever the server is recreated or the directory's contents change (a hand-edited compose file, a new `dev_runner_count`, a refreshed `DATA_DIR`, ...) - not just once at first creation. So after applying, just SSH in as `var.deploy_user` and run the matching script from inside that directory: +After `ansible-playbook playbooks/deploy.yml` has copied `services//` to `/home/deploy/services/` on the matching server, SSH in as `deploy` and run the matching script from inside that directory — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do exactly this remotely (see [`ansible/README.md`](ansible/README.md)): ```sh # on dev @@ -195,7 +208,7 @@ Only A records for IPv4 are managed here — none of the modules currently track ./spindown.sh ``` -If you ever need to force a re-sync without going through Terraform (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@:~/services` still works fine - `tofu apply` will just overwrite it again next time triggers change. +If you ever need to force a re-sync without going through Ansible (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@:~/services` still works fine - `ansible-playbook playbooks/deploy.yml` will just overwrite it again next time it's run. Each stack can also be managed individually with plain Compose, e.g.: @@ -243,7 +256,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here: -- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by the [post-install bootstrap](#post-install-bootstrap); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)). +- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by [Ansible's `bootstrap` role](ansible/roles/bootstrap/tasks/main.yml); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)). ## Development container @@ -261,5 +274,5 @@ Then reopen the repo in VS Code with the Dev Containers extension. - SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). - `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP. - Firewalls are allowlists scoped per host in `infra/modules//main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. -- Every host's [post-install bootstrap](#post-install-bootstrap) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`var.deploy_user`) for day-to-day access. +- Every host's [Ansible bootstrap role](ansible/roles/bootstrap/tasks/main.yml) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`deploy_user`, see `ansible/group_vars/all.yml`) for day-to-day access. - DNS zones (`modules/dns`) carry both Hetzner's own `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, including for `divine-couture.co.uk` and `snexo.co.uk`, not just `luke-else.co.uk`. diff --git a/services/dev/Runners/docker-compose.yml b/services/dev/Runners/docker-compose.yml deleted file mode 100644 index 6d0f9d4..0000000 --- a/services/dev/Runners/docker-compose.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by OpenTofu from var.dev_runner_count - do not hand-edit. -# To change the number of runners, edit dev_runner_count in terraform.tfvars and -# run tofu apply. To change their shape, edit -# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead. -# -# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable -# (not baked in here) - services/dev/spinup.sh generates a fresh token and -# writes it to Runners/.env immediately before starting these containers. -# -# /data lives on dev's volume (/mnt/HC_Volume_PENDING_TOFU_APPLY) so runner -# identity survives a server rebuild - baked in directly rather than via .env, -# since Runners/.env is already reserved for the registration token above and -# gets overwritten on every deploy. -services: - runner-1: - image: gitea/act_runner:latest - container_name: gitea_runner_1 - volumes: - - ./config.yaml:/config.yaml - - /mnt/HC_Volume_PENDING_TOFU_APPLY/gitea_runner_1:/data - - /var/run/docker.sock:/var/run/docker.sock - networks: - - proxy - environment: - CONFIG_FILE: /config.yaml - GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" - GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}" - GITEA_RUNNER_NAME: "CICD#1" - restart: unless-stopped - -networks: - proxy: - external: true diff --git a/services/dev/spinup.sh b/services/dev/spinup.sh index 0b4e466..0692a37 100755 --- a/services/dev/spinup.sh +++ b/services/dev/spinup.sh @@ -4,7 +4,7 @@ set -e cd "$(dirname "$0")" if [ ! -f .env ]; then - echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'tofu apply' in infra/ first, then re-copy this directory." >&2 + echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 exit 1 fi diff --git a/services/prod/spinup.sh b/services/prod/spinup.sh index d1e874e..ef7e9ee 100755 --- a/services/prod/spinup.sh +++ b/services/prod/spinup.sh @@ -4,7 +4,7 @@ set -e cd "$(dirname "$0")" if [ ! -f .env ]; then - echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'tofu apply' in infra/ first, then re-copy this directory." >&2 + echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 exit 1 fi