diff --git a/.gitignore b/.gitignore index c2bd0a0..f6c311e 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ # Local .terraform directories **/.terraform/* +# Real S3 backend config (bucket/endpoint) - see infra/backend.hcl.example +backend.hcl + # .tfstate files *.tfstate *.tfstate.* diff --git a/control.sh b/control.sh index a230171..939c103 100755 --- a/control.sh +++ b/control.sh @@ -45,9 +45,10 @@ detect_tofu # --- variable persistence -------------------------------------------------- # Variables set via the "Set variables" menu (HCLOUD_TOKEN, BACKUP_S3_*, -# DEPLOY_USER, ANSIBLE_SSH_PRIVATE_KEY_FILE) live in .control.env, gitignored, -# so they survive between runs of this script. They are NOT exported into your -# normal shell - only into control.sh's own process and whatever it runs. +# TF_VAR_state_encryption_passphrase, DEPLOY_USER, ANSIBLE_SSH_PRIVATE_KEY_FILE) +# live in .control.env, gitignored, so they survive between runs of this +# script. They are NOT exported into your normal shell - only into control.sh's +# own process and whatever it runs. load_control_env() { if [ -f "$CONTROL_ENV" ]; then set -a @@ -98,6 +99,8 @@ check_prereqs() { [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ] && ok "BACKUP_S3_ACCESS_KEY_ID set" || warn "BACKUP_S3_ACCESS_KEY_ID not set" [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && ok "BACKUP_S3_SECRET_ACCESS_KEY set" || warn "BACKUP_S3_SECRET_ACCESS_KEY not set" [ -n "${BACKUP_S3_ENDPOINT:-}" ] && ok "BACKUP_S3_ENDPOINT set" || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)" + [ -n "${TF_VAR_state_encryption_passphrase:-}" ] && ok "TF_VAR_state_encryption_passphrase set" \ + || bad "TF_VAR_state_encryption_passphrase not set (needed to read/write tofu state - see Configuration > Set variables)" ok "DEPLOY_USER: ${DEPLOY_USER:-deploy} (default 'deploy' if unset)" local key="${ANSIBLE_SSH_PRIVATE_KEY_FILE:-$HOME/.ssh/id_ed25519}" @@ -108,6 +111,8 @@ check_prereqs() { title "Files" [ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \ || bad "infra/terraform.tfvars missing (Configuration > Copy terraform.tfvars.example)" + [ -f "$INFRA_DIR/backend.hcl" ] && ok "infra/backend.hcl exists" \ + || bad "infra/backend.hcl missing (Configuration > Copy backend.hcl.example)" [ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \ || warn "infra/.terraform missing - run OpenTofu > init" if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed" @@ -140,6 +145,7 @@ set_variables() { prompt_var BACKUP_S3_ACCESS_KEY_ID "S3 backup access key ID" true prompt_var BACKUP_S3_SECRET_ACCESS_KEY "S3 backup secret access key" true prompt_var BACKUP_S3_ENDPOINT "S3 backup endpoint (blank for real AWS S3)" false + prompt_var TF_VAR_state_encryption_passphrase "OpenTofu state encryption passphrase (min 16 chars)" true ok "Variables saved." } @@ -170,17 +176,69 @@ copy_tfvars() { fi } +# backend.hcl holds the S3 backend's bucket/key/endpoint (see +# infra/backend.hcl.example) - it's the *same* bucket as BACKUP_S3_BUCKET, +# just a different object key, so state never collides with backup archives. +# Credentials aren't stored in it - see export_s3_backend_creds. +copy_backend_hcl() { + local example="$INFRA_DIR/backend.hcl.example" + local target="$INFRA_DIR/backend.hcl" + if [ -f "$target" ]; then + warn "infra/backend.hcl already exists." + confirm "Overwrite it with a fresh copy of the example?" || return 0 + fi + cp "$example" "$target" + ok "Copied backend.hcl.example -> backend.hcl" + + if [ -n "${BACKUP_S3_BUCKET:-}" ]; then + sed -i "s|^bucket = .*|bucket = \"${BACKUP_S3_BUCKET}\"|" "$target" + ok "Set bucket = \"${BACKUP_S3_BUCKET}\" (from BACKUP_S3_BUCKET)" + else + warn "BACKUP_S3_BUCKET not set - bucket still has the example's placeholder value." + fi + if [ -n "${BACKUP_S3_ENDPOINT:-}" ]; then + sed -i "s|^ s3 = .*| s3 = \"${BACKUP_S3_ENDPOINT}\"|" "$target" + ok "Set endpoints.s3 = \"${BACKUP_S3_ENDPOINT}\" (from BACKUP_S3_ENDPOINT)" + else + warn "BACKUP_S3_ENDPOINT not set - endpoints.s3 still has the example's placeholder value." + fi + + local key + read -rp "Object key/path for the state file within the bucket [tofu/terraform.tfstate]: " key + key="${key:-tofu/terraform.tfstate}" + sed -i "s|^key = .*|key = \"${key}\"|" "$target" + ok "Set key = \"${key}\"" + + warn "Check infra/backend.hcl's region still matches your bucket's actual location before running tofu init." +} + # --- OpenTofu actions ----------------------------------------------------- +# The s3 backend authenticates like the AWS CLI: standard AWS_ACCESS_KEY_ID / +# AWS_SECRET_ACCESS_KEY env vars, not backend-block attributes. Re-use the +# same backup bucket's credentials for both. +export_s3_backend_creds() { + [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ] && export AWS_ACCESS_KEY_ID="$BACKUP_S3_ACCESS_KEY_ID" + [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && export AWS_SECRET_ACCESS_KEY="$BACKUP_S3_SECRET_ACCESS_KEY" +} + require_tofu() { if [ -z "$TOFU" ]; then bad "No tofu/terraform binary found."; return 1; fi if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then bad "infra/terraform.tfvars is missing - copy terraform.tfvars.example first."; return 1 fi + if [ ! -f "$INFRA_DIR/backend.hcl" ]; then + bad "infra/backend.hcl is missing - copy backend.hcl.example first."; return 1 + fi + if [ -z "${TF_VAR_state_encryption_passphrase:-}" ]; then + bad "TF_VAR_state_encryption_passphrase is not set - Configuration > Set variables."; return 1 + fi + export_s3_backend_creds } -tofu_init() { require_tofu && run_in "$INFRA_DIR" "$TOFU" init; } +tofu_init() { require_tofu && run_in "$INFRA_DIR" "$TOFU" init -backend-config="$INFRA_DIR/backend.hcl"; } tofu_plan() { require_tofu && run_in "$INFRA_DIR" "$TOFU" plan; } tofu_apply() { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply; } tofu_output() { require_tofu && run_in "$INFRA_DIR" "$TOFU" output; } +tofu_refresh() { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply -refresh-only; } tofu_destroy() { require_tofu || return 1 confirm "${RED}Destroy ALL infra managed by OpenTofu?${RESET}" && run_in "$INFRA_DIR" "$TOFU" destroy @@ -218,7 +276,7 @@ accept SSH from the vpn server's public IP, so vpn must exist and you must be connected to it (WireGuard, via wg-easy) before dev/prod are reachable. Steps, in order: - 0. Configuration: variables + terraform.tfvars (skip if already done) + 0. Configuration: variables + terraform.tfvars + backend.hcl (skip if already done) 1. OpenTofu: init -> plan -> apply (creates servers, network, firewalls, DNS) 2. Ansible: bootstrap + deploy + spinup for role_vpn 3. YOU: visit https://vpn.luke-else.co.uk, complete wg-easy's setup, @@ -228,12 +286,15 @@ EOF confirm "Start the guided walkthrough?" || return 0 echo; title "0/4 Configuration" - if confirm "Set/update variables now (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)?"; then + if confirm "Set/update variables now (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, state passphrase, SSH key path)?"; then set_variables fi if [ ! -f "$INFRA_DIR/terraform.tfvars" ] && confirm "infra/terraform.tfvars is missing - create it from the example now?"; then copy_tfvars fi + if [ ! -f "$INFRA_DIR/backend.hcl" ] && confirm "infra/backend.hcl is missing - create it from the example now?"; then + copy_backend_hcl + fi echo; title "1/4 OpenTofu" if confirm "Run $TOFU init/plan/apply now?"; then @@ -270,19 +331,20 @@ menu() { echo " 2) Guided full setup (vpn -> connect -> dev/prod)" title "Configuration" - echo " 3) Set variables (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)" + echo " 3) Set variables (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, state passphrase, SSH key path)" echo " 4) Copy terraform.tfvars.example -> terraform.tfvars" + echo " 5) Copy backend.hcl.example -> backend.hcl" title "OpenTofu (infra/)" - echo " 5) init 6) plan 7) apply" - echo " 8) output 9) destroy" + echo " 6) init 7) plan 8) apply" + echo " 9) output 10) refresh 11) destroy" title "Ansible (ansible/)" - echo " 10) Install collections 11) Show inventory" - echo " 12) Bootstrap vpn 13) Deploy vpn 14) Spinup vpn" - echo " 15) Bootstrap dev 16) Deploy dev 17) Spinup dev" - echo " 18) Bootstrap prod 19) Deploy prod 20) Spinup prod" - echo " 21) Spindown (choose target)" + echo " 12) Install collections 13) Show inventory" + echo " 14) Bootstrap vpn 15) Deploy vpn 16) Spinup vpn" + echo " 17) Bootstrap dev 18) Deploy dev 19) Spinup dev" + echo " 20) Bootstrap prod 21) Deploy prod 22) Spinup prod" + echo " 23) Spindown (choose target)" echo echo " 0) Quit" @@ -297,23 +359,25 @@ while true; do 2) guided; pause ;; 3) set_variables; pause ;; 4) copy_tfvars; pause ;; - 5) tofu_init; pause ;; - 6) tofu_plan; pause ;; - 7) tofu_apply; pause ;; - 8) tofu_output; pause ;; - 9) tofu_destroy; pause ;; - 10) ansible_collections; pause ;; - 11) ansible_inventory; pause ;; - 12) ansible_bootstrap_vpn; pause ;; - 13) ansible_deploy_vpn; pause ;; - 14) ansible_spinup_vpn; pause ;; - 15) ansible_bootstrap_dev; pause ;; - 16) ansible_deploy_dev; pause ;; - 17) ansible_spinup_dev; pause ;; - 18) ansible_bootstrap_prod; pause ;; - 19) ansible_deploy_prod; pause ;; - 20) ansible_spinup_prod; pause ;; - 21) ansible_spindown; pause ;; + 5) copy_backend_hcl; pause ;; + 6) tofu_init; pause ;; + 7) tofu_plan; pause ;; + 8) tofu_apply; pause ;; + 9) tofu_output; pause ;; + 10) tofu_refresh; pause ;; + 11) tofu_destroy; pause ;; + 12) ansible_collections; pause ;; + 13) ansible_inventory; pause ;; + 14) ansible_bootstrap_vpn; pause ;; + 15) ansible_deploy_vpn; pause ;; + 16) ansible_spinup_vpn; pause ;; + 17) ansible_bootstrap_dev; pause ;; + 18) ansible_deploy_dev; pause ;; + 19) ansible_spinup_dev; pause ;; + 20) ansible_bootstrap_prod; pause ;; + 21) ansible_deploy_prod; pause ;; + 22) ansible_spinup_prod; pause ;; + 23) ansible_spindown; pause ;; 0) echo "Bye."; exit 0 ;; "") ;; *) warn "Unknown option: $choice"; pause ;; diff --git a/infra/backend.hcl.example b/infra/backend.hcl.example new file mode 100644 index 0000000..bb11e3f --- /dev/null +++ b/infra/backend.hcl.example @@ -0,0 +1,34 @@ +# Copy to backend.hcl and fill in - backend.hcl itself is gitignored, never +# commit real values there. Used as `tofu init -backend-config=backend.hcl` +# (control.sh's OpenTofu > init does this for you). +# +# This is the *same* S3-compatible bucket already used for service backups +# (BACKUP_S3_BUCKET/BACKUP_S3_ENDPOINT in control.sh > Set variables) - just a +# different object key so state never collides with backup archives. +# Credentials are NOT set here: the s3 backend picks up the standard +# AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables, which +# control.sh exports from BACKUP_S3_ACCESS_KEY_ID/BACKUP_S3_SECRET_ACCESS_KEY +# for you. + +bucket = "your-hetzner-bucket-name" # same bucket as BACKUP_S3_BUCKET +key = "tofu/terraform.tfstate" # path *within* the bucket - keep this distinct from the backup/ prefix + +# Hetzner Object Storage doesn't have "AWS regions" - this just has to be a +# syntactically valid region string, it isn't used to route the request (the +# endpoint below does that). Match it to your bucket's actual location. +region = "eu-central" + +endpoints = { + # e.g. https://nbg1.your-objectstorage.com - same value as BACKUP_S3_ENDPOINT + s3 = "https://your-location.your-objectstorage.com" +} + +# Hetzner Object Storage isn't AWS: skip the AWS-specific validation/lookup +# calls the backend would otherwise make, and address the bucket path-style +# (https://endpoint/bucket) rather than AWS's virtual-hosted style. +use_path_style = true +skip_credentials_validation = true +skip_region_validation = true +skip_requesting_account_id = true +skip_s3_checksum = true +skip_metadata_api_check = true diff --git a/infra/variables.tf b/infra/variables.tf index c8c811f..2acc995 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -74,3 +74,9 @@ variable "dns_zones" { type = list(string) default = ["luke-else.co.uk"] } + +variable "state_encryption_passphrase" { + description = "Passphrase (min 16 chars) used to encrypt the OpenTofu state file client-side before it's written to the S3 backend - see versions.tf. Set via the TF_VAR_state_encryption_passphrase environment variable (control.sh > Set variables), never as a literal default here." + type = string + sensitive = true +} diff --git a/infra/versions.tf b/infra/versions.tf index f817710..0709077 100644 --- a/infra/versions.tf +++ b/infra/versions.tf @@ -1,5 +1,5 @@ terraform { - required_version = ">= 1.6.0" + required_version = ">= 1.7.0" # state encryption (below) requires >= 1.7.0 required_providers { hcloud = { @@ -7,6 +7,35 @@ terraform { version = "~> 1.54" # hcloud_zone / hcloud_zone_rrset (DNS) require >= 1.54.0 } } + + # State lives in the same Hetzner Object Storage bucket used for service + # backups (see readme.md > Persistent data), under its own key so it never + # collides with backup objects. Deliberately empty (partial configuration): + # backend blocks can't reference variables, so bucket/key/endpoint are + # supplied at `tofu init` time via -backend-config=backend.hcl (gitignored, + # see backend.hcl.example) and credentials via the standard + # AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables - control.sh + # exports both from the BACKUP_S3_* variables before invoking tofu. + backend "s3" {} + + # State is encrypted client-side, independent of anything the bucket does - + # Hetzner Object Storage only supports SSE-C (a per-request customer key), + # not the SSE-S3 header the s3 backend's own `encrypt` option sends, so that + # option silently no-ops against Hetzner. `enforced` refuses to ever read or + # write unencrypted state, so a missing passphrase fails loudly instead of + # falling back to plaintext. + encryption { + key_provider "pbkdf2" "state" { + passphrase = var.state_encryption_passphrase + } + method "aes_gcm" "state" { + keys = key_provider.pbkdf2.state + } + state { + method = method.aes_gcm.state + enforced = true + } + } } # Reads the token from the HCLOUD_TOKEN environment variable. diff --git a/readme.md b/readme.md index 64565a1..bb219cf 100644 --- a/readme.md +++ b/readme.md @@ -72,7 +72,9 @@ architecture-beta ├── infra/ # OpenTofu — provisions the 3 servers, network, firewalls, DNS │ ├── main.tf # root module: wires network + dev/prod/vpn/dns modules together │ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names) +│ ├── versions.tf # provider/backend requirements + state encryption config │ ├── terraform.tfvars.example +│ ├── backend.hcl.example # S3 backend config (bucket/key/endpoint) - see Remote state │ └── modules/ │ ├── network/ # shared private network + subnet (dev + prod) │ ├── dev/ prod/ vpn/ # one server + firewall each, labeled role= @@ -107,10 +109,10 @@ Each of `services/dev`, `services/prod`, `services/vpn` follows the same convent - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token - One or more SSH keys uploaded to that project (Console → Security → SSH Keys), plus the matching private key available locally -- [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` +- [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.7.0` (state encryption, see [Remote state](#remote-state)) - [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`) - Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner -- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/)) and an access key/secret pair — not provisioned by OpenTofu, so create this yourself +- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/)) and an access key/secret pair — not provisioned by OpenTofu, so create this yourself. Used both for service backups (`BACKUP_S3_*`) and, under a separate object key, for OpenTofu's own remote state (see [Remote state](#remote-state)) ## Quickstart @@ -120,25 +122,36 @@ Each of `services/dev`, `services/prod`, `services/vpn` follows the same convent ./control.sh ``` -It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, and the `BACKUP_S3_*` credentials, saved to gitignored `.control.env` so you only enter them once), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand. +It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, the `BACKUP_S3_*` credentials, and the state encryption passphrase, saved to gitignored `.control.env` so you only enter them once; also copies `terraform.tfvars.example`/`backend.hcl.example` into place), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand. ## Provisioning the infrastructure (`infra/`) ```sh cd infra export HCLOUD_TOKEN=your-hetzner-api-token # never commit this +export AWS_ACCESS_KEY_ID=your-s3-access-key-id # for the state backend, below +export AWS_SECRET_ACCESS_KEY=your-s3-secret-key +export TF_VAR_state_encryption_passphrase=a-long-random-passphrase cp terraform.tfvars.example terraform.tfvars $EDITOR terraform.tfvars # set ssh_key_names at minimum +cp backend.hcl.example backend.hcl +$EDITOR backend.hcl # set bucket/key/region/endpoints.s3 -tofu init +tofu init -backend-config=backend.hcl tofu plan tofu apply ``` -This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. +This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars`, `backend.hcl`, and any `*.tfvars` file are gitignored — never commit real values there. Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`. +### Remote state + +State is stored remotely in the S3-compatible bucket from [Prerequisites](#prerequisites) (`infra/versions.tf`'s `backend "s3" {}`), under its own object key so it never collides with the `BACKUP_S3_*` backup archives living in the same bucket. `infra/backend.hcl` (copied from `backend.hcl.example`, gitignored) holds the non-secret bucket/key/region/endpoint; credentials come from the standard `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` environment variables (`control.sh` exports these from `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` for you) — backend blocks can't reference variables, which is why the split exists. + +The state itself is encrypted client-side (OpenTofu's built-in [state encryption](https://opentofu.org/docs/language/state/encryption/), PBKDF2 + AES-GCM) before it's ever written to the bucket, using `TF_VAR_state_encryption_passphrase`. This is deliberate rather than relying on the bucket's own server-side encryption: Hetzner Object Storage only supports SSE-C (a per-request customer-supplied key), not the SSE-S3 header the s3 backend's `encrypt` option sends, so that option wouldn't actually do anything here. `state { enforced = true }` in `infra/versions.tf` means every `tofu` command fails loudly if the passphrase is missing, rather than silently falling back to plaintext. **Losing the passphrase means losing the state** — back it up somewhere durable (a password manager, not the repo). + ### Managing DNS Creating a zone doesn't make Hetzner authoritative by itself — run `tofu output dns_nameservers` and set those as the domain's NS records at its registrar. Propagation time depends on the registrar and the domain's previous NS TTL.