From 9c0dc78a2c73cec0d059f8fd4c3cb74d4b849863 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Tue, 21 Jul 2026 14:56:03 +0100 Subject: [PATCH] chore: Cleaned up ansible and added control script --- ansible/README.md | 9 +- ansible/group_vars/role_dev.yml | 5 - ansible/roles/bootstrap/tasks/main.yml | 52 ++--- .../templates/runners-docker-compose.yml.j2 | 12 +- control.sh | 198 ++++++++++++++++++ readme.md | 19 +- 6 files changed, 236 insertions(+), 59 deletions(-) create mode 100755 control.sh diff --git a/ansible/README.md b/ansible/README.md index 46c8890..289a2be 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -16,7 +16,7 @@ to S3, not on a Hetzner volume - see `../readme.md`. (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules//main.tf`). - `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, `backup_s3_*` S3 backup credentials), `role_dev.yml` / `role_prod.yml` / - `role_vpn.yml` (per-role vars, e.g. `dev_runner_count`). + `role_vpn.yml` (per-role vars, e.g. `service_group`). - `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening, unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`. - `roles/deploy/` - copies `services//` to the server and renders `.env` @@ -65,9 +65,10 @@ ansible-playbook playbooks/spinup.yml -l role_dev,role_prod ansible-playbook playbooks/site.yml -l role_dev,role_prod ``` -Re-run `deploy.yml` + `spinup.yml` any time a compose file changes or -`dev_runner_count` in `group_vars/role_dev.yml` is bumped - `spinup.sh` only -starts/updates what's changed, same as before. +Re-run `deploy.yml` + `spinup.yml` any time a compose file changes - `spinup.sh` +only starts/updates what's changed, same as before. `dev` always runs three +Gitea Actions runners (fixed in +`roles/deploy/templates/runners-docker-compose.yml.j2`). `spindown.yml` runs `spindown.sh`, which does a full `docker system/volume prune -a` on the host - it prompts for a `yes` confirmation before running, diff --git a/ansible/group_vars/role_dev.yml b/ansible/group_vars/role_dev.yml index 8e7a4ae..073d03d 100644 --- a/ansible/group_vars/role_dev.yml +++ b/ansible/group_vars/role_dev.yml @@ -1,7 +1,2 @@ --- service_group: dev - -# Number of Gitea Actions runner containers rendered into -# services/dev/Runners/docker-compose.yml by the deploy role - see -# roles/deploy/templates/runners-docker-compose.yml.j2. -dev_runner_count: 3 diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index c476d32..307f0cc 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -10,50 +10,22 @@ upgrade: dist cache_valid_time: 3600 -- name: Install Docker prerequisites +# Docker straight from Ubuntu's own repositories - no external apt repo, GPG +# key or architecture juggling. docker.io is the engine, docker-compose-v2 is +# the `docker compose` plugin. A little behind Docker's own channel, but far +# simpler and plenty for these hosts. +- name: Install Docker Engine and the Compose plugin ansible.builtin.apt: name: - - ca-certificates - - curl - - gnupg + - docker.io + - docker-compose-v2 state: present -- name: Create apt keyrings directory - ansible.builtin.file: - path: /etc/apt/keyrings - state: directory - mode: "0755" - -- name: Download Docker's GPG key - ansible.builtin.get_url: - url: https://download.docker.com/linux/ubuntu/gpg - dest: /etc/apt/keyrings/docker.asc - mode: "0644" - force: false - -- name: Determine dpkg architecture - ansible.builtin.command: dpkg --print-architecture - register: _dpkg_arch - changed_when: false - -- name: Add Docker apt repository - ansible.builtin.apt_repository: - repo: >- - deb [arch={{ _dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] - https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable - filename: docker - state: present - -- name: Install Docker Engine and Compose plugin - ansible.builtin.apt: - update_cache: true - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin - state: present +- name: Ensure Docker is enabled and running + ansible.builtin.systemd: + name: docker + enabled: true + state: started - name: Create non-root sudo user, given the same key root was provisioned with ansible.builtin.user: diff --git a/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 index a5bbcb4..802e82d 100644 --- a/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 +++ b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 @@ -1,7 +1,6 @@ -# Generated by Ansible from dev_runner_count - do not hand-edit. -# To change the number of runners, edit dev_runner_count in -# ansible/group_vars/role_dev.yml and re-run playbooks/deploy.yml. -# To change their shape, edit this template instead. +# Generated by Ansible - do not hand-edit. Three Gitea Actions runners, always. +# To change their shape (image, volumes, env), edit this template and re-run +# playbooks/deploy.yml. # # GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable # (not baked in here) - services/dev/spinup.sh generates a fresh token and @@ -10,8 +9,9 @@ # /data is a disposable local cache (runner identity re-registers on every # spinup.sh run anyway), so it's a plain named volume - not part of the S3 # backup/restore stack in services/dev/backup-docker-compose.yml. +{% set runner_count = 3 %} services: -{% for i in range(1, dev_runner_count + 1) %} +{% for i in range(1, runner_count + 1) %} runner-{{ i }}: image: gitea/act_runner:latest container_name: gitea_runner_{{ i }} @@ -34,7 +34,7 @@ networks: external: true volumes: -{% for i in range(1, dev_runner_count + 1) %} +{% for i in range(1, runner_count + 1) %} gitea_runner_{{ i }}_data: name: gitea_runner_{{ i }}_data {% endfor %} diff --git a/control.sh b/control.sh new file mode 100755 index 0000000..50cb4f6 --- /dev/null +++ b/control.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# control.sh - interactive control panel for this repo. +# +# Steps you through the pre-setup and day-to-day operations for both the +# OpenTofu infra (infra/) and the Ansible config (ansible/), in the order the +# firewall rules require (vpn first, then tunnel in, then dev/prod). It only +# ever runs `tofu`/`ansible-playbook` for you - it's a menu wrapper, nothing +# more, so anything it does you can also do by hand from infra/ or ansible/. + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INFRA_DIR="$REPO_ROOT/infra" +ANSIBLE_DIR="$REPO_ROOT/ansible" + +# --- pretty output -------------------------------------------------------- +if [ -t 1 ]; then + BOLD=$(tput bold); DIM=$(tput dim); RESET=$(tput sgr0) + RED=$(tput setaf 1); GREEN=$(tput setaf 2); YELLOW=$(tput setaf 3); CYAN=$(tput setaf 6) +else + BOLD=""; DIM=""; RESET=""; RED=""; GREEN=""; YELLOW=""; CYAN="" +fi + +ok() { printf "%s ok %s %s\n" "$GREEN" "$RESET" "$1"; } +warn() { printf "%s warn%s %s\n" "$YELLOW" "$RESET" "$1"; } +bad() { printf "%s miss%s %s\n" "$RED" "$RESET" "$1"; } +title() { printf "\n%s%s%s\n" "$BOLD$CYAN" "$1" "$RESET"; } + +# Pick the OpenTofu binary: prefer `tofu`, fall back to `terraform`. +TOFU="" +detect_tofu() { + if command -v tofu >/dev/null 2>&1; then TOFU="tofu" + elif command -v terraform >/dev/null 2>&1; then TOFU="terraform" + fi +} +detect_tofu + +# Run a command inside a directory, echoing it first. Returns the command's exit code. +run_in() { + local dir="$1"; shift + printf "%s\$ (cd %s && %s)%s\n" "$DIM" "${dir#$REPO_ROOT/}" "$*" "$RESET" + ( cd "$dir" && "$@" ) +} + +pause() { read -rp $'\nPress Enter to return to the menu... ' _; } + +confirm() { + local reply + read -rp "$1 [y/N] " reply + [[ "$reply" == "y" || "$reply" == "Y" ]] +} + +# --- prerequisite checks -------------------------------------------------- +check_prereqs() { + title "Tools" + if [ -n "$TOFU" ]; then ok "OpenTofu ($TOFU: $("$TOFU" version 2>/dev/null | head -n1))" + else bad "Neither 'tofu' nor 'terraform' found on PATH"; fi + if command -v ansible-playbook >/dev/null 2>&1; then ok "Ansible ($(ansible --version 2>/dev/null | head -n1))" + else bad "'ansible-playbook' not found on PATH"; fi + + title "Environment variables" + [ -n "${HCLOUD_TOKEN:-}" ] && ok "HCLOUD_TOKEN set" || bad "HCLOUD_TOKEN not set (needed by both tofu and the ansible inventory)" + [ -n "${BACKUP_S3_BUCKET:-}" ] && ok "BACKUP_S3_BUCKET set" || warn "BACKUP_S3_BUCKET not set (needed for deploy/spinup - S3 backup .env)" + [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ] && ok "BACKUP_S3_ACCESS_KEY_ID set" || warn "BACKUP_S3_ACCESS_KEY_ID not set" + [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && ok "BACKUP_S3_SECRET_ACCESS_KEY set" || warn "BACKUP_S3_SECRET_ACCESS_KEY not set" + [ -n "${BACKUP_S3_ENDPOINT:-}" ] && ok "BACKUP_S3_ENDPOINT set" || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)" + + local key="${ANSIBLE_SSH_PRIVATE_KEY_FILE:-$HOME/.ssh/id_ed25519}" + [ -f "${key/#\~/$HOME}" ] && ok "SSH key present ($key)" || warn "SSH key not found at $key (set ANSIBLE_SSH_PRIVATE_KEY_FILE)" + + title "Files" + [ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \ + || bad "infra/terraform.tfvars missing (copy terraform.tfvars.example and fill in ssh_key_names)" + [ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \ + || warn "infra/.terraform missing - run OpenTofu > init" + if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed" + else warn "hetzner.hcloud collection not installed - run Ansible > Install collections"; fi +} + +# --- OpenTofu actions ----------------------------------------------------- +require_tofu() { + if [ -z "$TOFU" ]; then bad "No tofu/terraform binary found."; return 1; fi + if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then + bad "infra/terraform.tfvars is missing - copy terraform.tfvars.example first."; return 1 + fi +} +tofu_init() { require_tofu && run_in "$INFRA_DIR" "$TOFU" init; } +tofu_plan() { require_tofu && run_in "$INFRA_DIR" "$TOFU" plan; } +tofu_apply() { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply; } +tofu_output() { require_tofu && run_in "$INFRA_DIR" "$TOFU" output; } +tofu_destroy() { + require_tofu || return 1 + confirm "${RED}Destroy ALL infra managed by OpenTofu?${RESET}" && run_in "$INFRA_DIR" "$TOFU" destroy +} + +# --- Ansible actions ------------------------------------------------------ +ansible_collections() { run_in "$ANSIBLE_DIR" ansible-galaxy collection install -r requirements.yml; } +ansible_inventory() { run_in "$ANSIBLE_DIR" ansible-inventory --graph; } + +# $1 = playbook, $2 = -l limit expression +play() { run_in "$ANSIBLE_DIR" ansible-playbook "playbooks/$1" -l "$2"; } + +ansible_bootstrap_vpn() { play bootstrap.yml role_vpn; } +ansible_deploy_vpn() { play deploy.yml role_vpn && play spinup.yml role_vpn; } +ansible_bootstrap_dp() { play bootstrap.yml role_dev,role_prod; } +ansible_deploy_dp() { play deploy.yml role_dev,role_prod && play spinup.yml role_dev,role_prod; } + +ansible_spindown() { + local target + read -rp "Spindown which hosts? (e.g. role_vpn, role_dev,role_prod, all): " target + [ -n "$target" ] && play spindown.yml "$target" +} + +# --- guided walkthrough --------------------------------------------------- +guided() { + title "Guided full setup" + cat < plan -> apply (creates servers, network, firewalls, DNS) + 2. Ansible: bootstrap + deploy + spinup for role_vpn + 3. YOU: connect to the VPN with an OpenVPN client + 4. Ansible: bootstrap + deploy + spinup for role_dev,role_prod +EOF + confirm "Start the guided walkthrough?" || return 0 + + echo; title "1/4 OpenTofu" + if confirm "Run $TOFU init/plan/apply now?"; then + tofu_init && tofu_plan && tofu_apply || { bad "OpenTofu step failed - stopping."; return 1; } + fi + + echo; title "2/4 Ansible - vpn" + if confirm "Bootstrap + deploy + spinup role_vpn now?"; then + ansible_bootstrap_vpn && ansible_deploy_vpn || { bad "vpn step failed - stopping."; return 1; } + fi + + echo; title "3/4 Connect the VPN" + warn "Connect to the vpn server with your OpenVPN client now, then continue." + confirm "Are you connected to the VPN?" || { warn "Stopping - reconnect and re-run this step."; return 0; } + + echo; title "4/4 Ansible - dev + prod" + if confirm "Bootstrap + deploy + spinup role_dev,role_prod now?"; then + ansible_bootstrap_dp && ansible_deploy_dp || { bad "dev/prod step failed."; return 1; } + fi + ok "Guided setup complete." +} + +# --- menu ----------------------------------------------------------------- +menu() { + clear + printf "%s==== server control panel ====%s\n" "$BOLD$CYAN" "$RESET" + printf "%srepo: %s tofu: %s%s\n" "$DIM" "$REPO_ROOT" "${TOFU:-none}" "$RESET" + + title "Setup" + echo " 1) Check prerequisites" + echo " 2) Guided full setup (vpn -> connect -> dev/prod)" + + title "OpenTofu (infra/)" + echo " 3) init 4) plan 5) apply" + echo " 6) output 7) destroy" + + title "Ansible (ansible/)" + echo " 8) Install collections 9) Show inventory" + echo " 10) Bootstrap vpn 11) Deploy + spinup vpn" + echo " 12) Bootstrap dev+prod 13) Deploy + spinup dev+prod" + echo " 14) Spindown (choose target)" + + echo + echo " 0) Quit" + echo + read -rp "Select an option: " choice +} + +while true; do + menu + case "$choice" in + 1) check_prereqs; pause ;; + 2) guided; pause ;; + 3) tofu_init; pause ;; + 4) tofu_plan; pause ;; + 5) tofu_apply; pause ;; + 6) tofu_output; pause ;; + 7) tofu_destroy; pause ;; + 8) ansible_collections; pause ;; + 9) ansible_inventory; pause ;; + 10) ansible_bootstrap_vpn; pause ;; + 11) ansible_deploy_vpn; pause ;; + 12) ansible_bootstrap_dp; pause ;; + 13) ansible_deploy_dp; pause ;; + 14) ansible_spindown; pause ;; + 0) echo "Bye."; exit 0 ;; + "") ;; + *) warn "Unknown option: $choice"; pause ;; + esac +done diff --git a/readme.md b/readme.md index ed8b055..483b19e 100644 --- a/readme.md +++ b/readme.md @@ -11,6 +11,7 @@ Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted - [Architecture](#architecture) - [Repository layout](#repository-layout) - [Prerequisites](#prerequisites) +- [Control panel (`control.sh`)](#control-panel-controlsh) - [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) - [Bootstrapping and deploying with Ansible](#bootstrapping-and-deploying-with-ansible-ansible) - [Service inventory](#service-inventory) @@ -84,7 +85,7 @@ Each host's stateful data lives in local named Docker volumes, backed up nightly │ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones ├── ansible/ # Ansible: bootstraps each server and deploys/starts services// onto it │ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by the role label above -│ ├── group_vars/ # deploy_user, dev_runner_count, backup_s3_*, etc. +│ ├── group_vars/ # deploy_user, service_group, backup_s3_*, etc. │ ├── roles/ │ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades │ │ └── deploy/ # copies services//, renders .env (S3 backup credentials) + Runners/docker-compose.yml @@ -115,6 +116,16 @@ OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud res Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below). +## Control panel (`control.sh`) + +If you'd rather not remember the exact command order, [`control.sh`](control.sh) at the repo root is an interactive menu that steps you through the whole pre-setup and bring-up. Run it from anywhere: + +```sh +./control.sh +``` + +It offers a **Check prerequisites** option (tools, `HCLOUD_TOKEN`/`BACKUP_S3_*` env vars, `terraform.tfvars`, SSH key, the `hetzner.hcloud` collection), individual OpenTofu (`init`/`plan`/`apply`/`output`/`destroy`) and Ansible (bootstrap/deploy/spinup/spindown per role) actions, and a **Guided full setup** that runs everything in the firewall-imposed order — vpn first, pause for you to connect the VPN, then dev/prod. It's only a wrapper around the same `tofu` and `ansible-playbook` commands documented below, so you can always fall back to running them by hand. + ## Provisioning the infrastructure (`infra/`) ```sh @@ -189,11 +200,11 @@ docker compose -f backup-docker-compose.yml exec backup backup # force an imme docker compose -f backup-docker-compose.yml run --rm restore # restore any currently-empty volumes ``` -### Scaling Gitea Actions runners +### Gitea Actions runners -The number of Gitea Actions runner containers on `dev` is set by `dev_runner_count` in [`ansible/group_vars/role_dev.yml`](ansible/group_vars/role_dev.yml) (default `3`). Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host: change `dev_runner_count` and redeploy rather than hand-editing it. +`dev` always runs three Gitea Actions runner containers, configured automatically on deploy. Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — three `runner-N` services, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host, so edit the template rather than hand-editing the rendered file (the count lives in a `{% raw %}{% set runner_count = 3 %}{% endraw %}` line at the top). -Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a count change. +Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a template change. Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore.