From 7a61260a0743135ee61151b19fc59e4e0d00214c Mon Sep 17 00:00:00 2001 From: Luke Else Date: Mon, 27 Jul 2026 21:36:23 +0100 Subject: [PATCH] Fix: Fixed remote state storage in S3 Bucket --- .gitignore | 3 - control.sh | 104 +++++++------------------- infra/backend.hcl.example | 34 --------- infra/versions.tf | 36 +++++++-- readme.md | 13 ++-- services/prod/home-docker-compose.yml | 2 + 6 files changed, 65 insertions(+), 127 deletions(-) delete mode 100644 infra/backend.hcl.example diff --git a/.gitignore b/.gitignore index f6c311e..c2bd0a0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,9 +24,6 @@ # Local .terraform directories **/.terraform/* -# Real S3 backend config (bucket/endpoint) - see infra/backend.hcl.example -backend.hcl - # .tfstate files *.tfstate *.tfstate.* diff --git a/control.sh b/control.sh index 939c103..969c4ce 100755 --- a/control.sh +++ b/control.sh @@ -96,8 +96,8 @@ check_prereqs() { title "Environment variables" [ -n "${HCLOUD_TOKEN:-}" ] && ok "HCLOUD_TOKEN set" || bad "HCLOUD_TOKEN not set (needed by both tofu and the ansible inventory)" [ -n "${BACKUP_S3_BUCKET:-}" ] && ok "BACKUP_S3_BUCKET set" || warn "BACKUP_S3_BUCKET not set (needed for deploy/spinup - S3 backup .env)" - [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ] && ok "BACKUP_S3_ACCESS_KEY_ID set" || warn "BACKUP_S3_ACCESS_KEY_ID not set" - [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && ok "BACKUP_S3_SECRET_ACCESS_KEY set" || warn "BACKUP_S3_SECRET_ACCESS_KEY not set" + [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ] && ok "BACKUP_S3_ACCESS_KEY_ID set" || bad "BACKUP_S3_ACCESS_KEY_ID not set (needed for backups and to read/write tofu state)" + [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && ok "BACKUP_S3_SECRET_ACCESS_KEY set" || bad "BACKUP_S3_SECRET_ACCESS_KEY not set (needed for backups and to read/write tofu state)" [ -n "${BACKUP_S3_ENDPOINT:-}" ] && ok "BACKUP_S3_ENDPOINT set" || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)" [ -n "${TF_VAR_state_encryption_passphrase:-}" ] && ok "TF_VAR_state_encryption_passphrase set" \ || bad "TF_VAR_state_encryption_passphrase not set (needed to read/write tofu state - see Configuration > Set variables)" @@ -111,8 +111,6 @@ check_prereqs() { title "Files" [ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \ || bad "infra/terraform.tfvars missing (Configuration > Copy terraform.tfvars.example)" - [ -f "$INFRA_DIR/backend.hcl" ] && ok "infra/backend.hcl exists" \ - || bad "infra/backend.hcl missing (Configuration > Copy backend.hcl.example)" [ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \ || warn "infra/.terraform missing - run OpenTofu > init" if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed" @@ -176,42 +174,6 @@ copy_tfvars() { fi } -# backend.hcl holds the S3 backend's bucket/key/endpoint (see -# infra/backend.hcl.example) - it's the *same* bucket as BACKUP_S3_BUCKET, -# just a different object key, so state never collides with backup archives. -# Credentials aren't stored in it - see export_s3_backend_creds. -copy_backend_hcl() { - local example="$INFRA_DIR/backend.hcl.example" - local target="$INFRA_DIR/backend.hcl" - if [ -f "$target" ]; then - warn "infra/backend.hcl already exists." - confirm "Overwrite it with a fresh copy of the example?" || return 0 - fi - cp "$example" "$target" - ok "Copied backend.hcl.example -> backend.hcl" - - if [ -n "${BACKUP_S3_BUCKET:-}" ]; then - sed -i "s|^bucket = .*|bucket = \"${BACKUP_S3_BUCKET}\"|" "$target" - ok "Set bucket = \"${BACKUP_S3_BUCKET}\" (from BACKUP_S3_BUCKET)" - else - warn "BACKUP_S3_BUCKET not set - bucket still has the example's placeholder value." - fi - if [ -n "${BACKUP_S3_ENDPOINT:-}" ]; then - sed -i "s|^ s3 = .*| s3 = \"${BACKUP_S3_ENDPOINT}\"|" "$target" - ok "Set endpoints.s3 = \"${BACKUP_S3_ENDPOINT}\" (from BACKUP_S3_ENDPOINT)" - else - warn "BACKUP_S3_ENDPOINT not set - endpoints.s3 still has the example's placeholder value." - fi - - local key - read -rp "Object key/path for the state file within the bucket [tofu/terraform.tfstate]: " key - key="${key:-tofu/terraform.tfstate}" - sed -i "s|^key = .*|key = \"${key}\"|" "$target" - ok "Set key = \"${key}\"" - - warn "Check infra/backend.hcl's region still matches your bucket's actual location before running tofu init." -} - # --- OpenTofu actions ----------------------------------------------------- # The s3 backend authenticates like the AWS CLI: standard AWS_ACCESS_KEY_ID / # AWS_SECRET_ACCESS_KEY env vars, not backend-block attributes. Re-use the @@ -226,15 +188,12 @@ require_tofu() { if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then bad "infra/terraform.tfvars is missing - copy terraform.tfvars.example first."; return 1 fi - if [ ! -f "$INFRA_DIR/backend.hcl" ]; then - bad "infra/backend.hcl is missing - copy backend.hcl.example first."; return 1 - fi if [ -z "${TF_VAR_state_encryption_passphrase:-}" ]; then bad "TF_VAR_state_encryption_passphrase is not set - Configuration > Set variables."; return 1 fi export_s3_backend_creds } -tofu_init() { require_tofu && run_in "$INFRA_DIR" "$TOFU" init -backend-config="$INFRA_DIR/backend.hcl"; } +tofu_init() { require_tofu && run_in "$INFRA_DIR" "$TOFU" init; } tofu_plan() { require_tofu && run_in "$INFRA_DIR" "$TOFU" plan; } tofu_apply() { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply; } tofu_output() { require_tofu && run_in "$INFRA_DIR" "$TOFU" output; } @@ -276,7 +235,7 @@ accept SSH from the vpn server's public IP, so vpn must exist and you must be connected to it (WireGuard, via wg-easy) before dev/prod are reachable. Steps, in order: - 0. Configuration: variables + terraform.tfvars + backend.hcl (skip if already done) + 0. Configuration: variables + terraform.tfvars (skip if already done) 1. OpenTofu: init -> plan -> apply (creates servers, network, firewalls, DNS) 2. Ansible: bootstrap + deploy + spinup for role_vpn 3. YOU: visit https://vpn.luke-else.co.uk, complete wg-easy's setup, @@ -292,9 +251,6 @@ EOF if [ ! -f "$INFRA_DIR/terraform.tfvars" ] && confirm "infra/terraform.tfvars is missing - create it from the example now?"; then copy_tfvars fi - if [ ! -f "$INFRA_DIR/backend.hcl" ] && confirm "infra/backend.hcl is missing - create it from the example now?"; then - copy_backend_hcl - fi echo; title "1/4 OpenTofu" if confirm "Run $TOFU init/plan/apply now?"; then @@ -333,18 +289,17 @@ menu() { title "Configuration" echo " 3) Set variables (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, state passphrase, SSH key path)" echo " 4) Copy terraform.tfvars.example -> terraform.tfvars" - echo " 5) Copy backend.hcl.example -> backend.hcl" title "OpenTofu (infra/)" - echo " 6) init 7) plan 8) apply" - echo " 9) output 10) refresh 11) destroy" + echo " 5) init 6) plan 7) apply" + echo " 8) output 9) refresh 10) destroy" title "Ansible (ansible/)" - echo " 12) Install collections 13) Show inventory" - echo " 14) Bootstrap vpn 15) Deploy vpn 16) Spinup vpn" - echo " 17) Bootstrap dev 18) Deploy dev 19) Spinup dev" - echo " 20) Bootstrap prod 21) Deploy prod 22) Spinup prod" - echo " 23) Spindown (choose target)" + echo " 11) Install collections 12) Show inventory" + echo " 13) Bootstrap vpn 14) Deploy vpn 15) Spinup vpn" + echo " 16) Bootstrap dev 17) Deploy dev 18) Spinup dev" + echo " 19) Bootstrap prod 20) Deploy prod 21) Spinup prod" + echo " 22) Spindown (choose target)" echo echo " 0) Quit" @@ -359,25 +314,24 @@ while true; do 2) guided; pause ;; 3) set_variables; pause ;; 4) copy_tfvars; pause ;; - 5) copy_backend_hcl; pause ;; - 6) tofu_init; pause ;; - 7) tofu_plan; pause ;; - 8) tofu_apply; pause ;; - 9) tofu_output; pause ;; - 10) tofu_refresh; pause ;; - 11) tofu_destroy; pause ;; - 12) ansible_collections; pause ;; - 13) ansible_inventory; pause ;; - 14) ansible_bootstrap_vpn; pause ;; - 15) ansible_deploy_vpn; pause ;; - 16) ansible_spinup_vpn; pause ;; - 17) ansible_bootstrap_dev; pause ;; - 18) ansible_deploy_dev; pause ;; - 19) ansible_spinup_dev; pause ;; - 20) ansible_bootstrap_prod; pause ;; - 21) ansible_deploy_prod; pause ;; - 22) ansible_spinup_prod; pause ;; - 23) ansible_spindown; pause ;; + 5) tofu_init; pause ;; + 6) tofu_plan; pause ;; + 7) tofu_apply; pause ;; + 8) tofu_output; pause ;; + 9) tofu_refresh; pause ;; + 10) tofu_destroy; pause ;; + 11) ansible_collections; pause ;; + 12) ansible_inventory; pause ;; + 13) ansible_bootstrap_vpn; pause ;; + 14) ansible_deploy_vpn; pause ;; + 15) ansible_spinup_vpn; pause ;; + 16) ansible_bootstrap_dev; pause ;; + 17) ansible_deploy_dev; pause ;; + 18) ansible_spinup_dev; pause ;; + 19) ansible_bootstrap_prod; pause ;; + 20) ansible_deploy_prod; pause ;; + 21) ansible_spinup_prod; pause ;; + 22) ansible_spindown; pause ;; 0) echo "Bye."; exit 0 ;; "") ;; *) warn "Unknown option: $choice"; pause ;; diff --git a/infra/backend.hcl.example b/infra/backend.hcl.example deleted file mode 100644 index bb11e3f..0000000 --- a/infra/backend.hcl.example +++ /dev/null @@ -1,34 +0,0 @@ -# Copy to backend.hcl and fill in - backend.hcl itself is gitignored, never -# commit real values there. Used as `tofu init -backend-config=backend.hcl` -# (control.sh's OpenTofu > init does this for you). -# -# This is the *same* S3-compatible bucket already used for service backups -# (BACKUP_S3_BUCKET/BACKUP_S3_ENDPOINT in control.sh > Set variables) - just a -# different object key so state never collides with backup archives. -# Credentials are NOT set here: the s3 backend picks up the standard -# AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables, which -# control.sh exports from BACKUP_S3_ACCESS_KEY_ID/BACKUP_S3_SECRET_ACCESS_KEY -# for you. - -bucket = "your-hetzner-bucket-name" # same bucket as BACKUP_S3_BUCKET -key = "tofu/terraform.tfstate" # path *within* the bucket - keep this distinct from the backup/ prefix - -# Hetzner Object Storage doesn't have "AWS regions" - this just has to be a -# syntactically valid region string, it isn't used to route the request (the -# endpoint below does that). Match it to your bucket's actual location. -region = "eu-central" - -endpoints = { - # e.g. https://nbg1.your-objectstorage.com - same value as BACKUP_S3_ENDPOINT - s3 = "https://your-location.your-objectstorage.com" -} - -# Hetzner Object Storage isn't AWS: skip the AWS-specific validation/lookup -# calls the backend would otherwise make, and address the bucket path-style -# (https://endpoint/bucket) rather than AWS's virtual-hosted style. -use_path_style = true -skip_credentials_validation = true -skip_region_validation = true -skip_requesting_account_id = true -skip_s3_checksum = true -skip_metadata_api_check = true diff --git a/infra/versions.tf b/infra/versions.tf index 0709077..d9856a9 100644 --- a/infra/versions.tf +++ b/infra/versions.tf @@ -10,13 +10,35 @@ terraform { # State lives in the same Hetzner Object Storage bucket used for service # backups (see readme.md > Persistent data), under its own key so it never - # collides with backup objects. Deliberately empty (partial configuration): - # backend blocks can't reference variables, so bucket/key/endpoint are - # supplied at `tofu init` time via -backend-config=backend.hcl (gitignored, - # see backend.hcl.example) and credentials via the standard - # AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables - control.sh - # exports both from the BACKUP_S3_* variables before invoking tofu. - backend "s3" {} + # collides with backup objects. Backend blocks can't reference variables, so + # this is all non-secret - credentials come from the standard + # AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables, which + # control.sh exports from BACKUP_S3_ACCESS_KEY_ID/BACKUP_S3_SECRET_ACCESS_KEY + # before invoking tofu (same bucket, same credentials). + backend "s3" { + bucket = "luke-else" + key = "tofu/terraform.tfstate" + + # Hetzner Object Storage doesn't have "AWS regions" - this just has to be + # a syntactically valid region string, it isn't used to route the request + # (the endpoint below does that). Match it to the bucket's actual location. + region = "eu-central" + + endpoints = { + s3 = "https://nbg1.your-objectstorage.com" + } + + # Hetzner Object Storage isn't AWS: skip the AWS-specific validation/lookup + # calls the backend would otherwise make, and address the bucket + # path-style (https://endpoint/bucket) rather than AWS's virtual-hosted + # style. + use_path_style = true + skip_credentials_validation = true + skip_region_validation = true + skip_requesting_account_id = true + skip_s3_checksum = true + skip_metadata_api_check = true + } # State is encrypted client-side, independent of anything the bucket does - # Hetzner Object Storage only supports SSE-C (a per-request customer key), diff --git a/readme.md b/readme.md index bb219cf..05f4578 100644 --- a/readme.md +++ b/readme.md @@ -72,9 +72,8 @@ architecture-beta ├── infra/ # OpenTofu — provisions the 3 servers, network, firewalls, DNS │ ├── main.tf # root module: wires network + dev/prod/vpn/dns modules together │ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names) -│ ├── versions.tf # provider/backend requirements + state encryption config +│ ├── versions.tf # provider/backend requirements + S3 backend config + state encryption - see Remote state │ ├── terraform.tfvars.example -│ ├── backend.hcl.example # S3 backend config (bucket/key/endpoint) - see Remote state │ └── modules/ │ ├── network/ # shared private network + subnet (dev + prod) │ ├── dev/ prod/ vpn/ # one server + firewall each, labeled role= @@ -122,7 +121,7 @@ Each of `services/dev`, `services/prod`, `services/vpn` follows the same convent ./control.sh ``` -It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, the `BACKUP_S3_*` credentials, and the state encryption passphrase, saved to gitignored `.control.env` so you only enter them once; also copies `terraform.tfvars.example`/`backend.hcl.example` into place), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand. +It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, the `BACKUP_S3_*` credentials, and the state encryption passphrase, saved to gitignored `.control.env` so you only enter them once; also copies `terraform.tfvars.example` into place), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand. ## Provisioning the infrastructure (`infra/`) @@ -134,21 +133,19 @@ export AWS_SECRET_ACCESS_KEY=your-s3-secret-key export TF_VAR_state_encryption_passphrase=a-long-random-passphrase cp terraform.tfvars.example terraform.tfvars $EDITOR terraform.tfvars # set ssh_key_names at minimum -cp backend.hcl.example backend.hcl -$EDITOR backend.hcl # set bucket/key/region/endpoints.s3 -tofu init -backend-config=backend.hcl +tofu init tofu plan tofu apply ``` -This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars`, `backend.hcl`, and any `*.tfvars` file are gitignored — never commit real values there. +This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars` and any other `*.tfvars` file are gitignored — never commit real values there. Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`. ### Remote state -State is stored remotely in the S3-compatible bucket from [Prerequisites](#prerequisites) (`infra/versions.tf`'s `backend "s3" {}`), under its own object key so it never collides with the `BACKUP_S3_*` backup archives living in the same bucket. `infra/backend.hcl` (copied from `backend.hcl.example`, gitignored) holds the non-secret bucket/key/region/endpoint; credentials come from the standard `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` environment variables (`control.sh` exports these from `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` for you) — backend blocks can't reference variables, which is why the split exists. +State is stored remotely in the S3-compatible bucket from [Prerequisites](#prerequisites), under its own object key so it never collides with the `BACKUP_S3_*` backup archives living in the same bucket. The bucket/key/region/endpoint are hardcoded directly into `infra/versions.tf`'s `backend "s3" {}` block — backend blocks can't reference variables, but none of that is secret, so it just lives in the file. Credentials come from the standard `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` environment variables (`control.sh` exports these from `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` for you — same bucket, same credentials). The state itself is encrypted client-side (OpenTofu's built-in [state encryption](https://opentofu.org/docs/language/state/encryption/), PBKDF2 + AES-GCM) before it's ever written to the bucket, using `TF_VAR_state_encryption_passphrase`. This is deliberate rather than relying on the bucket's own server-side encryption: Hetzner Object Storage only supports SSE-C (a per-request customer-supplied key), not the SSE-S3 header the s3 backend's `encrypt` option sends, so that option wouldn't actually do anything here. `state { enforced = true }` in `infra/versions.tf` means every `tofu` command fails loudly if the passphrase is missing, rather than silently falling back to plaintext. **Losing the passphrase means losing the state** — back it up somewhere durable (a password manager, not the repo). diff --git a/services/prod/home-docker-compose.yml b/services/prod/home-docker-compose.yml index fce506e..2c48e71 100644 --- a/services/prod/home-docker-compose.yml +++ b/services/prod/home-docker-compose.yml @@ -2,6 +2,8 @@ services: home: image: ghcr.io/gethomepage/homepage:latest container_name: home + environment: + - HOMEPAGE_ALLOWED_HOSTS=home.luke-else.co.uk volumes: - ./home/config:/app/config - /var/run/docker.sock:/var/run/docker.sock:ro