feat: Added remote state storage in S3 bucket

.gitignore

@@ -24,6 +24,9 @@
 # Local .terraform directories
 **/.terraform/*
 
+# Real S3 backend config (bucket/endpoint) - see infra/backend.hcl.example
+backend.hcl
+
 # .tfstate files
 *.tfstate
 *.tfstate.*

control.sh

@@ -45,9 +45,10 @@ detect_tofu
 
 # --- variable persistence --------------------------------------------------
 # Variables set via the "Set variables" menu (HCLOUD_TOKEN, BACKUP_S3_*,
-# DEPLOY_USER, ANSIBLE_SSH_PRIVATE_KEY_FILE) live in .control.env, gitignored,
+# TF_VAR_state_encryption_passphrase, DEPLOY_USER, ANSIBLE_SSH_PRIVATE_KEY_FILE)
-# so they survive between runs of this script. They are NOT exported into your
+# live in .control.env, gitignored, so they survive between runs of this
-# normal shell - only into control.sh's own process and whatever it runs.
+# script. They are NOT exported into your normal shell - only into control.sh's
+# own process and whatever it runs.
 load_control_env() {
   if [ -f "$CONTROL_ENV" ]; then
     set -a
@@ -98,6 +99,8 @@ check_prereqs() {
   [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ]      && ok "BACKUP_S3_ACCESS_KEY_ID set"      || warn "BACKUP_S3_ACCESS_KEY_ID not set"
   [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ]  && ok "BACKUP_S3_SECRET_ACCESS_KEY set"  || warn "BACKUP_S3_SECRET_ACCESS_KEY not set"
   [ -n "${BACKUP_S3_ENDPOINT:-}" ]           && ok "BACKUP_S3_ENDPOINT set"           || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)"
+  [ -n "${TF_VAR_state_encryption_passphrase:-}" ] && ok "TF_VAR_state_encryption_passphrase set" \
+    || bad "TF_VAR_state_encryption_passphrase not set (needed to read/write tofu state - see Configuration > Set variables)"
   ok "DEPLOY_USER: ${DEPLOY_USER:-deploy} (default 'deploy' if unset)"
 
   local key="${ANSIBLE_SSH_PRIVATE_KEY_FILE:-$HOME/.ssh/id_ed25519}"
@@ -108,6 +111,8 @@ check_prereqs() {
   title "Files"
   [ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \
     || bad "infra/terraform.tfvars missing (Configuration > Copy terraform.tfvars.example)"
+  [ -f "$INFRA_DIR/backend.hcl" ] && ok "infra/backend.hcl exists" \
+    || bad "infra/backend.hcl missing (Configuration > Copy backend.hcl.example)"
   [ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \
     || warn "infra/.terraform missing - run OpenTofu > init"
   if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed"
@@ -140,6 +145,7 @@ set_variables() {
   prompt_var BACKUP_S3_ACCESS_KEY_ID      "S3 backup access key ID" true
   prompt_var BACKUP_S3_SECRET_ACCESS_KEY  "S3 backup secret access key" true
   prompt_var BACKUP_S3_ENDPOINT           "S3 backup endpoint (blank for real AWS S3)" false
+  prompt_var TF_VAR_state_encryption_passphrase "OpenTofu state encryption passphrase (min 16 chars)" true
   ok "Variables saved."
 }
 
@@ -170,17 +176,69 @@ copy_tfvars() {
   fi
 }
 
+# backend.hcl holds the S3 backend's bucket/key/endpoint (see
+# infra/backend.hcl.example) - it's the *same* bucket as BACKUP_S3_BUCKET,
+# just a different object key, so state never collides with backup archives.
+# Credentials aren't stored in it - see export_s3_backend_creds.
+copy_backend_hcl() {
+  local example="$INFRA_DIR/backend.hcl.example"
+  local target="$INFRA_DIR/backend.hcl"
+  if [ -f "$target" ]; then
+    warn "infra/backend.hcl already exists."
+    confirm "Overwrite it with a fresh copy of the example?" || return 0
+  fi
+  cp "$example" "$target"
+  ok "Copied backend.hcl.example -> backend.hcl"
+
+  if [ -n "${BACKUP_S3_BUCKET:-}" ]; then
+    sed -i "s|^bucket = .*|bucket = \"${BACKUP_S3_BUCKET}\"|" "$target"
+    ok "Set bucket = \"${BACKUP_S3_BUCKET}\" (from BACKUP_S3_BUCKET)"
+  else
+    warn "BACKUP_S3_BUCKET not set - bucket still has the example's placeholder value."
+  fi
+  if [ -n "${BACKUP_S3_ENDPOINT:-}" ]; then
+    sed -i "s|^  s3 = .*|  s3 = \"${BACKUP_S3_ENDPOINT}\"|" "$target"
+    ok "Set endpoints.s3 = \"${BACKUP_S3_ENDPOINT}\" (from BACKUP_S3_ENDPOINT)"
+  else
+    warn "BACKUP_S3_ENDPOINT not set - endpoints.s3 still has the example's placeholder value."
+  fi
+
+  local key
+  read -rp "Object key/path for the state file within the bucket [tofu/terraform.tfstate]: " key
+  key="${key:-tofu/terraform.tfstate}"
+  sed -i "s|^key    = .*|key    = \"${key}\"|" "$target"
+  ok "Set key = \"${key}\""
+
+  warn "Check infra/backend.hcl's region still matches your bucket's actual location before running tofu init."
+}
+
 # --- OpenTofu actions -----------------------------------------------------
+# The s3 backend authenticates like the AWS CLI: standard AWS_ACCESS_KEY_ID /
+# AWS_SECRET_ACCESS_KEY env vars, not backend-block attributes. Re-use the
+# same backup bucket's credentials for both.
+export_s3_backend_creds() {
+  [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ]     && export AWS_ACCESS_KEY_ID="$BACKUP_S3_ACCESS_KEY_ID"
+  [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && export AWS_SECRET_ACCESS_KEY="$BACKUP_S3_SECRET_ACCESS_KEY"
+}
+
 require_tofu() {
   if [ -z "$TOFU" ]; then bad "No tofu/terraform binary found."; return 1; fi
   if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then
     bad "infra/terraform.tfvars is missing - copy terraform.tfvars.example first."; return 1
   fi
+  if [ ! -f "$INFRA_DIR/backend.hcl" ]; then
+    bad "infra/backend.hcl is missing - copy backend.hcl.example first."; return 1
+  fi
+  if [ -z "${TF_VAR_state_encryption_passphrase:-}" ]; then
+    bad "TF_VAR_state_encryption_passphrase is not set - Configuration > Set variables."; return 1
+  fi
+  export_s3_backend_creds
 }
-tofu_init()    { require_tofu && run_in "$INFRA_DIR" "$TOFU" init; }
+tofu_init()    { require_tofu && run_in "$INFRA_DIR" "$TOFU" init -backend-config="$INFRA_DIR/backend.hcl"; }
 tofu_plan()    { require_tofu && run_in "$INFRA_DIR" "$TOFU" plan; }
 tofu_apply()   { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply; }
 tofu_output()  { require_tofu && run_in "$INFRA_DIR" "$TOFU" output; }
+tofu_refresh() { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply -refresh-only; }
 tofu_destroy() {
   require_tofu || return 1
   confirm "${RED}Destroy ALL infra managed by OpenTofu?${RESET}" && run_in "$INFRA_DIR" "$TOFU" destroy
@@ -218,7 +276,7 @@ accept SSH from the vpn server's public IP, so vpn must exist and you must be
 connected to it (WireGuard, via wg-easy) before dev/prod are reachable.
 
 Steps, in order:
-  0. Configuration: variables + terraform.tfvars (skip if already done)
+  0. Configuration: variables + terraform.tfvars + backend.hcl (skip if already done)
   1. OpenTofu: init -> plan -> apply   (creates servers, network, firewalls, DNS)
   2. Ansible:  bootstrap + deploy + spinup for role_vpn
   3. YOU:      visit https://vpn.luke-else.co.uk, complete wg-easy's setup,
@@ -228,12 +286,15 @@ EOF
   confirm "Start the guided walkthrough?" || return 0
 
   echo; title "0/4  Configuration"
-  if confirm "Set/update variables now (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)?"; then
+  if confirm "Set/update variables now (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, state passphrase, SSH key path)?"; then
     set_variables
   fi
   if [ ! -f "$INFRA_DIR/terraform.tfvars" ] && confirm "infra/terraform.tfvars is missing - create it from the example now?"; then
     copy_tfvars
   fi
+  if [ ! -f "$INFRA_DIR/backend.hcl" ] && confirm "infra/backend.hcl is missing - create it from the example now?"; then
+    copy_backend_hcl
+  fi
 
   echo; title "1/4  OpenTofu"
   if confirm "Run $TOFU init/plan/apply now?"; then
@@ -270,19 +331,20 @@ menu() {
   echo "  2) Guided full setup (vpn -> connect -> dev/prod)"
 
   title "Configuration"
-  echo "  3) Set variables (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)"
+  echo "  3) Set variables (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, state passphrase, SSH key path)"
   echo "  4) Copy terraform.tfvars.example -> terraform.tfvars"
+  echo "  5) Copy backend.hcl.example -> backend.hcl"
 
   title "OpenTofu (infra/)"
-  echo "  5) init        6) plan        7) apply"
+  echo "  6) init        7) plan        8) apply"
-  echo "  8) output      9) destroy"
+  echo "  9) output     10) refresh    11) destroy"
 
   title "Ansible (ansible/)"
-  echo " 10) Install collections        11) Show inventory"
+  echo " 12) Install collections        13) Show inventory"
-  echo " 12) Bootstrap vpn              13) Deploy vpn          14) Spinup vpn"
+  echo " 14) Bootstrap vpn              15) Deploy vpn          16) Spinup vpn"
-  echo " 15) Bootstrap dev               16) Deploy dev          17) Spinup dev"
+  echo " 17) Bootstrap dev               18) Deploy dev          19) Spinup dev"
-  echo " 18) Bootstrap prod             19) Deploy prod         20) Spinup prod"
+  echo " 20) Bootstrap prod             21) Deploy prod         22) Spinup prod"
-  echo " 21) Spindown (choose target)"
+  echo " 23) Spindown (choose target)"
 
   echo
   echo "  0) Quit"
@@ -297,23 +359,25 @@ while true; do
     2)  guided; pause ;;
     3)  set_variables; pause ;;
     4)  copy_tfvars; pause ;;
-    5)  tofu_init; pause ;;
+    5)  copy_backend_hcl; pause ;;
-    6)  tofu_plan; pause ;;
+    6)  tofu_init; pause ;;
-    7)  tofu_apply; pause ;;
+    7)  tofu_plan; pause ;;
-    8)  tofu_output; pause ;;
+    8)  tofu_apply; pause ;;
-    9)  tofu_destroy; pause ;;
+    9)  tofu_output; pause ;;
-    10) ansible_collections; pause ;;
+    10) tofu_refresh; pause ;;
-    11) ansible_inventory; pause ;;
+    11) tofu_destroy; pause ;;
-    12) ansible_bootstrap_vpn; pause ;;
+    12) ansible_collections; pause ;;
-    13) ansible_deploy_vpn; pause ;;
+    13) ansible_inventory; pause ;;
-    14) ansible_spinup_vpn; pause ;;
+    14) ansible_bootstrap_vpn; pause ;;
-    15) ansible_bootstrap_dev; pause ;;
+    15) ansible_deploy_vpn; pause ;;
-    16) ansible_deploy_dev; pause ;;
+    16) ansible_spinup_vpn; pause ;;
-    17) ansible_spinup_dev; pause ;;
+    17) ansible_bootstrap_dev; pause ;;
-    18) ansible_bootstrap_prod; pause ;;
+    18) ansible_deploy_dev; pause ;;
-    19) ansible_deploy_prod; pause ;;
+    19) ansible_spinup_dev; pause ;;
-    20) ansible_spinup_prod; pause ;;
+    20) ansible_bootstrap_prod; pause ;;
-    21) ansible_spindown; pause ;;
+    21) ansible_deploy_prod; pause ;;
+    22) ansible_spinup_prod; pause ;;
+    23) ansible_spindown; pause ;;
     0)  echo "Bye."; exit 0 ;;
     "") ;;
     *)  warn "Unknown option: $choice"; pause ;;

infra/backend.hcl.example

@@ -0,0 +1,34 @@
+# Copy to backend.hcl and fill in - backend.hcl itself is gitignored, never
+# commit real values there. Used as `tofu init -backend-config=backend.hcl`
+# (control.sh's OpenTofu > init does this for you).
+#
+# This is the *same* S3-compatible bucket already used for service backups
+# (BACKUP_S3_BUCKET/BACKUP_S3_ENDPOINT in control.sh > Set variables) - just a
+# different object key so state never collides with backup archives.
+# Credentials are NOT set here: the s3 backend picks up the standard
+# AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables, which
+# control.sh exports from BACKUP_S3_ACCESS_KEY_ID/BACKUP_S3_SECRET_ACCESS_KEY
+# for you.
+
+bucket = "your-hetzner-bucket-name" # same bucket as BACKUP_S3_BUCKET
+key    = "tofu/terraform.tfstate"   # path *within* the bucket - keep this distinct from the backup/ prefix
+
+# Hetzner Object Storage doesn't have "AWS regions" - this just has to be a
+# syntactically valid region string, it isn't used to route the request (the
+# endpoint below does that). Match it to your bucket's actual location.
+region = "eu-central"
+
+endpoints = {
+  # e.g. https://nbg1.your-objectstorage.com - same value as BACKUP_S3_ENDPOINT
+  s3 = "https://your-location.your-objectstorage.com"
+}
+
+# Hetzner Object Storage isn't AWS: skip the AWS-specific validation/lookup
+# calls the backend would otherwise make, and address the bucket path-style
+# (https://endpoint/bucket) rather than AWS's virtual-hosted style.
+use_path_style               = true
+skip_credentials_validation  = true
+skip_region_validation       = true
+skip_requesting_account_id   = true
+skip_s3_checksum             = true
+skip_metadata_api_check      = true

infra/variables.tf

@@ -74,3 +74,9 @@ variable "dns_zones" {
   type        = list(string)
   default     = ["luke-else.co.uk"]
 }
+
+variable "state_encryption_passphrase" {
+  description = "Passphrase (min 16 chars) used to encrypt the OpenTofu state file client-side before it's written to the S3 backend - see versions.tf. Set via the TF_VAR_state_encryption_passphrase environment variable (control.sh > Set variables), never as a literal default here."
+  type        = string
+  sensitive   = true
+}

infra/versions.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">= 1.6.0"
+  required_version = ">= 1.7.0" # state encryption (below) requires >= 1.7.0
 
   required_providers {
     hcloud = {
@@ -7,6 +7,35 @@ terraform {
       version = "~> 1.54" # hcloud_zone / hcloud_zone_rrset (DNS) require >= 1.54.0
     }
   }
+
+  # State lives in the same Hetzner Object Storage bucket used for service
+  # backups (see readme.md > Persistent data), under its own key so it never
+  # collides with backup objects. Deliberately empty (partial configuration):
+  # backend blocks can't reference variables, so bucket/key/endpoint are
+  # supplied at `tofu init` time via -backend-config=backend.hcl (gitignored,
+  # see backend.hcl.example) and credentials via the standard
+  # AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables - control.sh
+  # exports both from the BACKUP_S3_* variables before invoking tofu.
+  backend "s3" {}
+
+  # State is encrypted client-side, independent of anything the bucket does -
+  # Hetzner Object Storage only supports SSE-C (a per-request customer key),
+  # not the SSE-S3 header the s3 backend's own `encrypt` option sends, so that
+  # option silently no-ops against Hetzner. `enforced` refuses to ever read or
+  # write unencrypted state, so a missing passphrase fails loudly instead of
+  # falling back to plaintext.
+  encryption {
+    key_provider "pbkdf2" "state" {
+      passphrase = var.state_encryption_passphrase
+    }
+    method "aes_gcm" "state" {
+      keys = key_provider.pbkdf2.state
+    }
+    state {
+      method   = method.aes_gcm.state
+      enforced = true
+    }
+  }
 }
 
 # Reads the token from the HCLOUD_TOKEN environment variable.

readme.md

@@ -72,7 +72,9 @@ architecture-beta
 ├── infra/                    # OpenTofu — provisions the 3 servers, network, firewalls, DNS
 │   ├── main.tf                # root module: wires network + dev/prod/vpn/dns modules together
 │   ├── variables.tf           # shared inputs (sizes, locations, IP ranges, SSH key names)
+│   ├── versions.tf            # provider/backend requirements + state encryption config
 │   ├── terraform.tfvars.example
+│   ├── backend.hcl.example    # S3 backend config (bucket/key/endpoint) - see Remote state
 │   └── modules/
 │       ├── network/            # shared private network + subnet (dev + prod)
 │       ├── dev/ prod/ vpn/     # one server + firewall each, labeled role=<name>
@@ -107,10 +109,10 @@ Each of `services/dev`, `services/prod`, `services/vpn` follows the same convent
 
 - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token
 - One or more SSH keys uploaded to that project (Console → Security → SSH Keys), plus the matching private key available locally
-- [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0`
+- [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.7.0` (state encryption, see [Remote state](#remote-state))
 - [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`)
 - Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner
-- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/)) and an access key/secret pair — not provisioned by OpenTofu, so create this yourself
+- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/)) and an access key/secret pair — not provisioned by OpenTofu, so create this yourself. Used both for service backups (`BACKUP_S3_*`) and, under a separate object key, for OpenTofu's own remote state (see [Remote state](#remote-state))
 
 ## Quickstart
 
@@ -120,25 +122,36 @@ Each of `services/dev`, `services/prod`, `services/vpn` follows the same convent
 ./control.sh
 ```
 
-It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, and the `BACKUP_S3_*` credentials, saved to gitignored `.control.env` so you only enter them once), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand.
+It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, the `BACKUP_S3_*` credentials, and the state encryption passphrase, saved to gitignored `.control.env` so you only enter them once; also copies `terraform.tfvars.example`/`backend.hcl.example` into place), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand.
 
 ## Provisioning the infrastructure (`infra/`)
 
 ```sh
 cd infra
 export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
+export AWS_ACCESS_KEY_ID=your-s3-access-key-id       # for the state backend, below
+export AWS_SECRET_ACCESS_KEY=your-s3-secret-key
+export TF_VAR_state_encryption_passphrase=a-long-random-passphrase
 cp terraform.tfvars.example terraform.tfvars
 $EDITOR terraform.tfvars                     # set ssh_key_names at minimum
+cp backend.hcl.example backend.hcl
+$EDITOR backend.hcl                          # set bucket/key/region/endpoints.s3
 
-tofu init
+tofu init -backend-config=backend.hcl
 tofu plan
 tofu apply
 ```
 
-This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there.
+This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars`, `backend.hcl`, and any `*.tfvars` file are gitignored — never commit real values there.
 
 Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`.
 
+### Remote state
+
+State is stored remotely in the S3-compatible bucket from [Prerequisites](#prerequisites) (`infra/versions.tf`'s `backend "s3" {}`), under its own object key so it never collides with the `BACKUP_S3_*` backup archives living in the same bucket. `infra/backend.hcl` (copied from `backend.hcl.example`, gitignored) holds the non-secret bucket/key/region/endpoint; credentials come from the standard `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` environment variables (`control.sh` exports these from `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` for you) — backend blocks can't reference variables, which is why the split exists.
+
+The state itself is encrypted client-side (OpenTofu's built-in [state encryption](https://opentofu.org/docs/language/state/encryption/), PBKDF2 + AES-GCM) before it's ever written to the bucket, using `TF_VAR_state_encryption_passphrase`. This is deliberate rather than relying on the bucket's own server-side encryption: Hetzner Object Storage only supports SSE-C (a per-request customer-supplied key), not the SSE-S3 header the s3 backend's `encrypt` option sends, so that option wouldn't actually do anything here. `state { enforced = true }` in `infra/versions.tf` means every `tofu` command fails loudly if the passphrase is missing, rather than silently falling back to plaintext. **Losing the passphrase means losing the state** — back it up somewhere durable (a password manager, not the repo).
+
 ### Managing DNS
 
 Creating a zone doesn't make Hetzner authoritative by itself — run `tofu output dns_nameservers` and set those as the domain's NS records at its registrar. Propagation time depends on the registrar and the domain's previous NS TTL.