luke-else/server
1
0
Fork 0
Code Issues 5 Pull Requests Projects Releases Wiki Activity

Fix: Fixed remote state storage in S3 Bucket

Browse Source
This commit is contained in:
luke-else
2026-07-27 21:36:23 +01:00
parent a528867ef5
commit 7a61260a07
6 changed files with 65 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
-3
View File
@@ -24,9 +24,6 @@
# Local .terraform directories
**/.terraform/*
# Real S3 backend config (bucket/endpoint) - see infra/backend.hcl.example
backend.hcl
# .tfstate files
*.tfstate
*.tfstate.*
control.sh
+29 -75
View File
@@ -96,8 +96,8 @@ check_prereqs() {
title "Environment variables"
[ -n "${HCLOUD_TOKEN:-}" ] && ok "HCLOUD_TOKEN set" || bad "HCLOUD_TOKEN not set (needed by both tofu and the ansible inventory)"
[ -n "${BACKUP_S3_BUCKET:-}" ] && ok "BACKUP_S3_BUCKET set" || warn "BACKUP_S3_BUCKET not set (needed for deploy/spinup - S3 backup .env)"
[ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ] && ok "BACKUP_S3_ACCESS_KEY_ID set" || warn "BACKUP_S3_ACCESS_KEY_ID not set"
[ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && ok "BACKUP_S3_SECRET_ACCESS_KEY set" || warn "BACKUP_S3_SECRET_ACCESS_KEY not set"
[ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ] && ok "BACKUP_S3_ACCESS_KEY_ID set" || bad "BACKUP_S3_ACCESS_KEY_ID not set (needed for backups and to read/write tofu state)"
[ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && ok "BACKUP_S3_SECRET_ACCESS_KEY set" || bad "BACKUP_S3_SECRET_ACCESS_KEY not set (needed for backups and to read/write tofu state)"
[ -n "${BACKUP_S3_ENDPOINT:-}" ] && ok "BACKUP_S3_ENDPOINT set" || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)"
[ -n "${TF_VAR_state_encryption_passphrase:-}" ] && ok "TF_VAR_state_encryption_passphrase set" \
|| bad "TF_VAR_state_encryption_passphrase not set (needed to read/write tofu state - see Configuration > Set variables)"
@@ -111,8 +111,6 @@ check_prereqs() {
title "Files"
[ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \
|| bad "infra/terraform.tfvars missing (Configuration > Copy terraform.tfvars.example)"
[ -f "$INFRA_DIR/backend.hcl" ] && ok "infra/backend.hcl exists" \
|| bad "infra/backend.hcl missing (Configuration > Copy backend.hcl.example)"
[ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \
|| warn "infra/.terraform missing - run OpenTofu > init"
if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed"
@@ -176,42 +174,6 @@ copy_tfvars() {
fi
}
# backend.hcl holds the S3 backend's bucket/key/endpoint (see
# infra/backend.hcl.example) - it's the *same* bucket as BACKUP_S3_BUCKET,
# just a different object key, so state never collides with backup archives.
# Credentials aren't stored in it - see export_s3_backend_creds.
copy_backend_hcl() {
local example="$INFRA_DIR/backend.hcl.example"
local target="$INFRA_DIR/backend.hcl"
if [ -f "$target" ]; then
warn "infra/backend.hcl already exists."
confirm "Overwrite it with a fresh copy of the example?" || return 0
fi
cp "$example" "$target"
ok "Copied backend.hcl.example -> backend.hcl"
if [ -n "${BACKUP_S3_BUCKET:-}" ]; then
sed -i "s|^bucket = .*|bucket = \"${BACKUP_S3_BUCKET}\"|" "$target"
ok "Set bucket = \"${BACKUP_S3_BUCKET}\" (from BACKUP_S3_BUCKET)"
else
warn "BACKUP_S3_BUCKET not set - bucket still has the example's placeholder value."
fi
if [ -n "${BACKUP_S3_ENDPOINT:-}" ]; then
sed -i "s|^ s3 = .*| s3 = \"${BACKUP_S3_ENDPOINT}\"|" "$target"
ok "Set endpoints.s3 = \"${BACKUP_S3_ENDPOINT}\" (from BACKUP_S3_ENDPOINT)"
else
warn "BACKUP_S3_ENDPOINT not set - endpoints.s3 still has the example's placeholder value."
fi
local key
read -rp "Object key/path for the state file within the bucket [tofu/terraform.tfstate]: " key
key="${key:-tofu/terraform.tfstate}"
sed -i "s|^key = .*|key = \"${key}\"|" "$target"
ok "Set key = \"${key}\""
warn "Check infra/backend.hcl's region still matches your bucket's actual location before running tofu init."
}
# --- OpenTofu actions -----------------------------------------------------
# The s3 backend authenticates like the AWS CLI: standard AWS_ACCESS_KEY_ID /
# AWS_SECRET_ACCESS_KEY env vars, not backend-block attributes. Re-use the
@@ -226,15 +188,12 @@ require_tofu() {
if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then
bad "infra/terraform.tfvars is missing - copy terraform.tfvars.example first."; return 1
fi
if [ ! -f "$INFRA_DIR/backend.hcl" ]; then
bad "infra/backend.hcl is missing - copy backend.hcl.example first."; return 1
fi
if [ -z "${TF_VAR_state_encryption_passphrase:-}" ]; then
bad "TF_VAR_state_encryption_passphrase is not set - Configuration > Set variables."; return 1
fi
export_s3_backend_creds
}
tofu_init() { require_tofu && run_in "$INFRA_DIR" "$TOFU" init -backend-config="$INFRA_DIR/backend.hcl"; }
tofu_init() { require_tofu && run_in "$INFRA_DIR" "$TOFU" init; }
tofu_plan() { require_tofu && run_in "$INFRA_DIR" "$TOFU" plan; }
tofu_apply() { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply; }
tofu_output() { require_tofu && run_in "$INFRA_DIR" "$TOFU" output; }
@@ -276,7 +235,7 @@ accept SSH from the vpn server's public IP, so vpn must exist and you must be
connected to it (WireGuard, via wg-easy) before dev/prod are reachable.
Steps, in order:
0. Configuration: variables + terraform.tfvars + backend.hcl (skip if already done)
0. Configuration: variables + terraform.tfvars (skip if already done)
1. OpenTofu: init -> plan -> apply (creates servers, network, firewalls, DNS)
2. Ansible: bootstrap + deploy + spinup for role_vpn
3. YOU: visit https://vpn.luke-else.co.uk, complete wg-easy's setup,
@@ -292,9 +251,6 @@ EOF
if [ ! -f "$INFRA_DIR/terraform.tfvars" ] && confirm "infra/terraform.tfvars is missing - create it from the example now?"; then
copy_tfvars
fi
if [ ! -f "$INFRA_DIR/backend.hcl" ] && confirm "infra/backend.hcl is missing - create it from the example now?"; then
copy_backend_hcl
fi
echo; title "1/4 OpenTofu"
if confirm "Run $TOFU init/plan/apply now?"; then
@@ -333,18 +289,17 @@ menu() {
title "Configuration"
echo " 3) Set variables (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, state passphrase, SSH key path)"
echo " 4) Copy terraform.tfvars.example -> terraform.tfvars"
echo " 5) Copy backend.hcl.example -> backend.hcl"
title "OpenTofu (infra/)"
echo " 6) init 7) plan 8) apply"
echo " 9) output 10) refresh 11) destroy"
echo " 5) init 6) plan 7) apply"
echo " 8) output 9) refresh 10) destroy"
title "Ansible (ansible/)"
echo " 12) Install collections 13) Show inventory"
echo " 14) Bootstrap vpn 15) Deploy vpn 16) Spinup vpn"
echo " 17) Bootstrap dev 18) Deploy dev 19) Spinup dev"
echo " 20) Bootstrap prod 21) Deploy prod 22) Spinup prod"
echo " 23) Spindown (choose target)"
echo " 11) Install collections 12) Show inventory"
echo " 13) Bootstrap vpn 14) Deploy vpn 15) Spinup vpn"
echo " 16) Bootstrap dev 17) Deploy dev 18) Spinup dev"
echo " 19) Bootstrap prod 20) Deploy prod 21) Spinup prod"
echo " 22) Spindown (choose target)"
echo
echo " 0) Quit"
@@ -359,25 +314,24 @@ while true; do
2) guided; pause ;;
3) set_variables; pause ;;
4) copy_tfvars; pause ;;
5) copy_backend_hcl; pause ;;
6) tofu_init; pause ;;
7) tofu_plan; pause ;;
8) tofu_apply; pause ;;
9) tofu_output; pause ;;
10) tofu_refresh; pause ;;
11) tofu_destroy; pause ;;
12) ansible_collections; pause ;;
13) ansible_inventory; pause ;;
14) ansible_bootstrap_vpn; pause ;;
15) ansible_deploy_vpn; pause ;;
16) ansible_spinup_vpn; pause ;;
17) ansible_bootstrap_dev; pause ;;
18) ansible_deploy_dev; pause ;;
19) ansible_spinup_dev; pause ;;
20) ansible_bootstrap_prod; pause ;;
21) ansible_deploy_prod; pause ;;
22) ansible_spinup_prod; pause ;;
23) ansible_spindown; pause ;;
5) tofu_init; pause ;;
6) tofu_plan; pause ;;
7) tofu_apply; pause ;;
8) tofu_output; pause ;;
9) tofu_refresh; pause ;;
10) tofu_destroy; pause ;;
11) ansible_collections; pause ;;
12) ansible_inventory; pause ;;
13) ansible_bootstrap_vpn; pause ;;
14) ansible_deploy_vpn; pause ;;
15) ansible_spinup_vpn; pause ;;
16) ansible_bootstrap_dev; pause ;;
17) ansible_deploy_dev; pause ;;
18) ansible_spinup_dev; pause ;;
19) ansible_bootstrap_prod; pause ;;
20) ansible_deploy_prod; pause ;;
21) ansible_spinup_prod; pause ;;
22) ansible_spindown; pause ;;
0) echo "Bye."; exit 0 ;;
"") ;;
*) warn "Unknown option: $choice"; pause ;;
infra/backend.hcl.example
-34
View File
@@ -1,34 +0,0 @@
# Copy to backend.hcl and fill in - backend.hcl itself is gitignored, never
# commit real values there. Used as `tofu init -backend-config=backend.hcl`
# (control.sh's OpenTofu > init does this for you).
#
# This is the *same* S3-compatible bucket already used for service backups
# (BACKUP_S3_BUCKET/BACKUP_S3_ENDPOINT in control.sh > Set variables) - just a
# different object key so state never collides with backup archives.
# Credentials are NOT set here: the s3 backend picks up the standard
# AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables, which
# control.sh exports from BACKUP_S3_ACCESS_KEY_ID/BACKUP_S3_SECRET_ACCESS_KEY
# for you.
bucket = "your-hetzner-bucket-name" # same bucket as BACKUP_S3_BUCKET
key = "tofu/terraform.tfstate" # path *within* the bucket - keep this distinct from the backup/ prefix
# Hetzner Object Storage doesn't have "AWS regions" - this just has to be a
# syntactically valid region string, it isn't used to route the request (the
# endpoint below does that). Match it to your bucket's actual location.
region = "eu-central"
endpoints = {
# e.g. https://nbg1.your-objectstorage.com - same value as BACKUP_S3_ENDPOINT
s3 = "https://your-location.your-objectstorage.com"
}
# Hetzner Object Storage isn't AWS: skip the AWS-specific validation/lookup
# calls the backend would otherwise make, and address the bucket path-style
# (https://endpoint/bucket) rather than AWS's virtual-hosted style.
use_path_style = true
skip_credentials_validation = true
skip_region_validation = true
skip_requesting_account_id = true
skip_s3_checksum = true
skip_metadata_api_check = true
infra/versions.tf
+29 -7
View File
@@ -10,13 +10,35 @@ terraform {
# State lives in the same Hetzner Object Storage bucket used for service
# backups (see readme.md > Persistent data), under its own key so it never
# collides with backup objects. Deliberately empty (partial configuration):
# backend blocks can't reference variables, so bucket/key/endpoint are
# supplied at `tofu init` time via -backend-config=backend.hcl (gitignored,
# see backend.hcl.example) and credentials via the standard
# AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables - control.sh
# exports both from the BACKUP_S3_* variables before invoking tofu.
backend "s3" {}
# collides with backup objects. Backend blocks can't reference variables, so
# this is all non-secret - credentials come from the standard
# AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY environment variables, which
# control.sh exports from BACKUP_S3_ACCESS_KEY_ID/BACKUP_S3_SECRET_ACCESS_KEY
# before invoking tofu (same bucket, same credentials).
backend "s3" {
bucket = "luke-else"
key = "tofu/terraform.tfstate"
# Hetzner Object Storage doesn't have "AWS regions" - this just has to be
# a syntactically valid region string, it isn't used to route the request
# (the endpoint below does that). Match it to the bucket's actual location.
region = "eu-central"
endpoints = {
s3 = "https://nbg1.your-objectstorage.com"
}
# Hetzner Object Storage isn't AWS: skip the AWS-specific validation/lookup
# calls the backend would otherwise make, and address the bucket
# path-style (https://endpoint/bucket) rather than AWS's virtual-hosted
# style.
use_path_style = true
skip_credentials_validation = true
skip_region_validation = true
skip_requesting_account_id = true
skip_s3_checksum = true
skip_metadata_api_check = true
}
# State is encrypted client-side, independent of anything the bucket does -
# Hetzner Object Storage only supports SSE-C (a per-request customer key),
readme.md
+5 -8
View File
@@ -72,9 +72,8 @@ architecture-beta
├── infra/ # OpenTofu — provisions the 3 servers, network, firewalls, DNS
│ ├── main.tf # root module: wires network + dev/prod/vpn/dns modules together
│ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names)
│ ├── versions.tf # provider/backend requirements + state encryption config
│ ├── versions.tf # provider/backend requirements + S3 backend config + state encryption - see Remote state
│ ├── terraform.tfvars.example
│ ├── backend.hcl.example # S3 backend config (bucket/key/endpoint) - see Remote state
│ └── modules/
│ ├── network/ # shared private network + subnet (dev + prod)
│ ├── dev/ prod/ vpn/ # one server + firewall each, labeled role=<name>
@@ -122,7 +121,7 @@ Each of `services/dev`, `services/prod`, `services/vpn` follows the same convent
./control.sh
```
It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, the `BACKUP_S3_*` credentials, and the state encryption passphrase, saved to gitignored `.control.env` so you only enter them once; also copies `terraform.tfvars.example`/`backend.hcl.example` into place), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand.
It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, the `BACKUP_S3_*` credentials, and the state encryption passphrase, saved to gitignored `.control.env` so you only enter them once; also copies `terraform.tfvars.example` into place), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand.
## Provisioning the infrastructure (`infra/`)
@@ -134,21 +133,19 @@ export AWS_SECRET_ACCESS_KEY=your-s3-secret-key
export TF_VAR_state_encryption_passphrase=a-long-random-passphrase
cp terraform.tfvars.example terraform.tfvars
$EDITOR terraform.tfvars # set ssh_key_names at minimum
cp backend.hcl.example backend.hcl
$EDITOR backend.hcl # set bucket/key/region/endpoints.s3
tofu init -backend-config=backend.hcl
tofu init
tofu plan
tofu apply
```
This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars`, `backend.hcl`, and any `*.tfvars` file are gitignored — never commit real values there.
This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars` and any other `*.tfvars` file are gitignored — never commit real values there.
Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`.
### Remote state
State is stored remotely in the S3-compatible bucket from [Prerequisites](#prerequisites) (`infra/versions.tf`'s `backend "s3" {}`), under its own object key so it never collides with the `BACKUP_S3_*` backup archives living in the same bucket. `infra/backend.hcl` (copied from `backend.hcl.example`, gitignored) holds the non-secret bucket/key/region/endpoint; credentials come from the standard `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` environment variables (`control.sh` exports these from `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` for you) — backend blocks can't reference variables, which is why the split exists.
State is stored remotely in the S3-compatible bucket from [Prerequisites](#prerequisites), under its own object key so it never collides with the `BACKUP_S3_*` backup archives living in the same bucket. The bucket/key/region/endpoint are hardcoded directly into `infra/versions.tf`'s `backend "s3" {}` block — backend blocks can't reference variables, but none of that is secret, so it just lives in the file. Credentials come from the standard `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` environment variables (`control.sh` exports these from `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` for you — same bucket, same credentials).
The state itself is encrypted client-side (OpenTofu's built-in [state encryption](https://opentofu.org/docs/language/state/encryption/), PBKDF2 + AES-GCM) before it's ever written to the bucket, using `TF_VAR_state_encryption_passphrase`. This is deliberate rather than relying on the bucket's own server-side encryption: Hetzner Object Storage only supports SSE-C (a per-request customer-supplied key), not the SSE-S3 header the s3 backend's `encrypt` option sends, so that option wouldn't actually do anything here. `state { enforced = true }` in `infra/versions.tf` means every `tofu` command fails loudly if the passphrase is missing, rather than silently falling back to plaintext. **Losing the passphrase means losing the state** — back it up somewhere durable (a password manager, not the repo).
services/prod/home-docker-compose.yml
+2
View File
@@ -2,6 +2,8 @@ services:
home:
image: ghcr.io/gethomepage/homepage:latest
container_name: home
environment:
- HOMEPAGE_ALLOWED_HOSTS=home.luke-else.co.uk
volumes:
- ./home/config:/app/config
- /var/run/docker.sock:/var/run/docker.sock:ro