From 1aa340f7ce2f6712c568c2a103b6bace8200c777 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Mon, 20 Jul 2026 22:45:43 +0100 Subject: [PATCH] feat: backup and restore --- ansible/README.md | 21 +++-- ansible/group_vars/all.yml | 15 +++- ansible/roles/deploy/tasks/main.yml | 28 ++----- ansible/roles/deploy/templates/env.j2 | 6 +- .../templates/runners-docker-compose.yml.j2 | 15 ++-- infra/main.tf | 8 +- infra/modules/dev/main.tf | 21 ----- infra/modules/dev/outputs.tf | 5 -- infra/modules/dev/variables.tf | 5 -- infra/modules/prod/main.tf | 21 ----- infra/modules/prod/outputs.tf | 5 -- infra/modules/prod/variables.tf | 5 -- infra/outputs.tf | 10 --- infra/terraform.tfvars.example | 2 - infra/variables.tf | 16 +--- readme.md | 80 ++++++++++++------- services/dev/backup-docker-compose.yml | 48 +++++++++++ services/dev/gitea-docker-compose.yml | 14 +++- services/dev/restore.sh | 54 +++++++++++++ services/dev/spindown.sh | 4 +- services/dev/spinup.sh | 9 ++- services/dev/traefik-docker-compose.yml | 6 +- services/prod/backup-docker-compose.yml | 59 ++++++++++++++ services/prod/bitwarden-docker-compose.yml | 13 ++- services/prod/rd-docker-compose.yml | 13 ++- services/prod/restore.sh | 54 +++++++++++++ services/prod/spindown.sh | 4 +- services/prod/spinup.sh | 9 ++- services/prod/status-docker-compose.yml | 9 ++- services/prod/traefik-docker-compose.yml | 6 +- services/prod/web-docker-compose.yml | 31 ++----- services/vpn/backup-docker-compose.yml | 48 +++++++++++ services/vpn/restore.sh | 54 +++++++++++++ services/vpn/spindown.sh | 4 +- services/vpn/spinup.sh | 12 +++ services/vpn/traefik-docker-compose.yml | 6 +- services/vpn/vpn-docker-compose.yml | 11 ++- 37 files changed, 524 insertions(+), 207 deletions(-) create mode 100644 services/dev/backup-docker-compose.yml create mode 100755 services/dev/restore.sh create mode 100644 services/prod/backup-docker-compose.yml create mode 100755 services/prod/restore.sh create mode 100644 services/vpn/backup-docker-compose.yml create mode 100755 services/vpn/restore.sh diff --git a/ansible/README.md b/ansible/README.md index 88f840b..46c8890 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -2,25 +2,26 @@ Bootstraps, syncs, and starts/stops the Docker Compose stacks in `../services/` on the three servers OpenTofu provisions (see `../infra/`). OpenTofu still owns -the actual cloud resources (servers, network, firewalls, volumes, DNS); +the actual cloud resources (servers, network, firewalls, DNS); everything downstream of "the server exists" - installing Docker, creating the `deploy` user, hardening SSH, copying `services//`, rendering `.env` / the Gitea runner compose file, and running `spinup.sh`/`spindown.sh` - lives -here instead. +here instead. Persistent service data lives in named Docker volumes backed up +to S3, not on a Hetzner volume - see `../readme.md`. ## Contents - `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API directly (not Terraform state) and groups servers by their `role` label (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules//main.tf`). -- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, `HCLOUD_TOKEN` - lookup), `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` (per-role vars, - e.g. `dev_runner_count`). +- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, + `backup_s3_*` S3 backup credentials), `role_dev.yml` / `role_prod.yml` / + `role_vpn.yml` (per-role vars, e.g. `dev_runner_count`). - `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening, unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`. -- `roles/deploy/` - copies `services//` to the server, looks up the - host's data volume directly from the Hetzner API and renders `.env` - (`DATA_DIR=...`) and, for `dev`, `Runners/docker-compose.yml`. +- `roles/deploy/` - copies `services//` to the server and renders `.env` + (S3 backup credentials, see `../readme.md`) and, for `dev`, + `Runners/docker-compose.yml`. - `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`, and `site.yml` (all three in order). @@ -30,6 +31,10 @@ here instead. cd ansible ansible-galaxy collection install -r requirements.yml export HCLOUD_TOKEN=your-hetzner-api-token # never commit this +export BACKUP_S3_BUCKET=your-bucket-name +export BACKUP_S3_ACCESS_KEY_ID=your-access-key-id +export BACKUP_S3_SECRET_ACCESS_KEY=your-secret-access-key +export BACKUP_S3_ENDPOINT=your-s3-endpoint # omit for real AWS S3 ``` `ansible_ssh_private_key_file` (see `group_vars/all.yml`) defaults to diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 5a6c35a..d3308cf 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -6,9 +6,16 @@ deploy_user: deploy ansible_user: "{{ deploy_user }}" ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}" -# Used by the deploy role to look up each host's data volume directly from the -# Hetzner API (hcloud_volume_info) rather than from Terraform state/outputs. -hcloud_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}" - # Repo root's services/ directory, relative to wherever a playbook lives under ansible/playbooks/. services_root: "{{ playbook_dir }}/../../services" + +# S3-compatible bucket that services//backup-docker-compose.yml backs up +# to and restores from - see readme.md. Not provisioned by OpenTofu (the +# hcloud provider has no Object Storage resource), so these must point at a +# bucket you've created yourself. AWS_ENDPOINT is only needed for a +# non-AWS S3-compatible provider (e.g. Hetzner Object Storage); leave it unset +# to use real AWS S3. +backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}" +backup_s3_access_key_id: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY_ID') }}" +backup_s3_secret_access_key: "{{ lookup('env', 'BACKUP_S3_SECRET_ACCESS_KEY') }}" +backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}" diff --git a/ansible/roles/deploy/tasks/main.yml b/ansible/roles/deploy/tasks/main.yml index a24c27d..7ad5889 100644 --- a/ansible/roles/deploy/tasks/main.yml +++ b/ansible/roles/deploy/tasks/main.yml @@ -3,6 +3,10 @@ # used to be generated by OpenTofu (services//.env and, for dev, # Runners/docker-compose.yml) - replaces the null_resource.deploy_services + # local_file provisioners previously in infra/modules//main.tf. +# +# Persistent data lives in named Docker volumes now, not on a Hetzner volume - +# see services//backup-docker-compose.yml. .env only carries the S3 +# credentials that backup/restore need, rendered for every host. - name: Ensure services directory exists on the host ansible.builtin.file: @@ -16,28 +20,11 @@ dest: "/home/{{ deploy_user }}/services/{{ service_group }}/" mode: preserve -- name: Look up this host's data volume - when: service_group in ['dev', 'prod'] - delegate_to: localhost - become: false - hetzner.hcloud.hcloud_volume_info: - api_token: "{{ hcloud_token }}" - name: "{{ service_group }}-storage" - register: _volume_info - -# Hetzner's automount tooling always mounts an auto-mounted volume at -# /mnt/HC_Volume_ - deterministic once the volume exists. -- name: Compute DATA_DIR from the volume's id - when: service_group in ['dev', 'prod'] - ansible.builtin.set_fact: - data_dir: "/mnt/HC_Volume_{{ _volume_info.hcloud_volume_info[0].id }}" - -- name: Render DATA_DIR into .env (auto-loaded by docker compose) - when: service_group in ['dev', 'prod'] +- name: Render S3 backup credentials into .env (auto-loaded by docker compose) ansible.builtin.template: src: env.j2 dest: "/home/{{ deploy_user }}/services/{{ service_group }}/.env" - mode: "0644" + mode: "0600" - name: Render Gitea Actions runner compose file when: service_group == 'dev' @@ -46,10 +33,11 @@ dest: "/home/{{ deploy_user }}/services/dev/Runners/docker-compose.yml" mode: "0644" -- name: Make spinup/spindown scripts executable +- name: Make spinup/spindown/restore scripts executable ansible.builtin.file: path: "/home/{{ deploy_user }}/services/{{ service_group }}/{{ item }}" mode: "0755" loop: - spinup.sh - spindown.sh + - restore.sh diff --git a/ansible/roles/deploy/templates/env.j2 b/ansible/roles/deploy/templates/env.j2 index a958fd1..b44355b 100644 --- a/ansible/roles/deploy/templates/env.j2 +++ b/ansible/roles/deploy/templates/env.j2 @@ -1 +1,5 @@ -DATA_DIR={{ data_dir }} +AWS_ACCESS_KEY_ID={{ backup_s3_access_key_id }} +AWS_SECRET_ACCESS_KEY={{ backup_s3_secret_access_key }} +AWS_S3_BUCKET_NAME={{ backup_s3_bucket }} +AWS_S3_PATH={{ service_group }} +AWS_ENDPOINT={{ backup_s3_endpoint }} diff --git a/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 index f72c94f..a5bbcb4 100644 --- a/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 +++ b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 @@ -7,10 +7,9 @@ # (not baked in here) - services/dev/spinup.sh generates a fresh token and # writes it to Runners/.env immediately before starting these containers. # -# /data lives on dev's volume ({{ data_dir }}) so runner identity survives a -# server rebuild - baked in directly rather than via .env, since Runners/.env -# is already reserved for the registration token above and gets overwritten -# on every deploy. +# /data is a disposable local cache (runner identity re-registers on every +# spinup.sh run anyway), so it's a plain named volume - not part of the S3 +# backup/restore stack in services/dev/backup-docker-compose.yml. services: {% for i in range(1, dev_runner_count + 1) %} runner-{{ i }}: @@ -18,7 +17,7 @@ services: container_name: gitea_runner_{{ i }} volumes: - ./config.yaml:/config.yaml - - {{ data_dir }}/gitea_runner_{{ i }}:/data + - gitea_runner_{{ i }}_data:/data - /var/run/docker.sock:/var/run/docker.sock networks: - proxy @@ -33,3 +32,9 @@ services: networks: proxy: external: true + +volumes: +{% for i in range(1, dev_runner_count + 1) %} + gitea_runner_{{ i }}_data: + name: gitea_runner_{{ i }}_data +{% endfor %} diff --git a/infra/main.tf b/infra/main.tf index 9df4e16..fd6d20b 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -1,6 +1,8 @@ # Root module: wires together the shared network and the three host modules # (one per server in docs/architecture.md). Each host module owns its own -# firewall (and volume, where applicable) - see modules//main.tf. +# firewall - see modules//main.tf. Persistent data lives in named Docker +# volumes backed up to S3 (see services//backup-docker-compose.yml), not +# on Hetzner volumes. locals { # SSH to dev/prod is only permitted from the vpn server's public IP: admins must @@ -28,8 +30,6 @@ locals { { zone = "luke-else.co.uk", name = "traefik.cicd", value = module.dev.ipv4 }, { zone = "luke-else.co.uk", name = "vpn", value = module.vpn.ipv4 }, { zone = "luke-else.co.uk", name = "traefik.vpn", value = module.vpn.ipv4 }, - { zone = "divine-couture.co.uk", name = "www", value = module.prod.ipv4 }, - { zone = "snexo.co.uk", name = "@", value = module.prod.ipv4 }, ] } @@ -50,7 +50,6 @@ module "dev" { ssh_key_ids = local.ssh_key_ids network_id = module.network.id private_ip = var.dev_private_ip - volume_size = var.dev_volume_size allowed_ssh_source_ips = local.vpn_ssh_source_ips network_ip_range = var.network_ip_range @@ -68,7 +67,6 @@ module "prod" { ssh_key_ids = local.ssh_key_ids network_id = module.network.id private_ip = var.prod_private_ip - volume_size = var.prod_volume_size allowed_ssh_source_ips = local.vpn_ssh_source_ips network_ip_range = var.network_ip_range diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf index 6bea690..cd811f1 100644 --- a/infra/modules/dev/main.tf +++ b/infra/modules/dev/main.tf @@ -64,24 +64,3 @@ resource "hcloud_server" "this" { } } -resource "hcloud_volume" "storage" { - name = "dev-storage" - size = var.volume_size - server_id = hcloud_server.this.id - automount = true - format = "ext4" - - lifecycle { - prevent_destroy = true - } -} - -locals { - # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is - # always /mnt/HC_Volume_ - deterministic once the volume exists, since the - # id is stable for the volume's lifetime regardless of which server it's - # attached to. Kept here only as a human-readable output (see outputs.tf) - - # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't - # read this value. - data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}" -} diff --git a/infra/modules/dev/outputs.tf b/infra/modules/dev/outputs.tf index 86e4ee2..4897389 100644 --- a/infra/modules/dev/outputs.tf +++ b/infra/modules/dev/outputs.tf @@ -5,8 +5,3 @@ output "ipv4" { output "private_ipv4" { value = var.private_ip } - -output "data_dir" { - description = "Deterministic host path of the auto-mounted volume - see local.data_dir." - value = local.data_dir -} diff --git a/infra/modules/dev/variables.tf b/infra/modules/dev/variables.tf index 89dbb4e..a9d7899 100644 --- a/infra/modules/dev/variables.tf +++ b/infra/modules/dev/variables.tf @@ -28,11 +28,6 @@ variable "private_ip" { type = string } -variable "volume_size" { - description = "Size in GB of the volume attached to dev (disk2 in docs/architecture.md)." - type = number -} - variable "allowed_ssh_source_ips" { description = "CIDRs allowed to reach port 22 on dev. Set by the root module to the vpn server's public IP." type = list(string) diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index 8135cae..65c2775 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -91,24 +91,3 @@ resource "hcloud_server" "this" { } } -resource "hcloud_volume" "storage" { - name = "prod-storage" - size = var.volume_size - server_id = hcloud_server.this.id - automount = true - format = "ext4" - - lifecycle { - prevent_destroy = true - } -} - -locals { - # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is - # always /mnt/HC_Volume_ - deterministic once the volume exists, since the - # id is stable for the volume's lifetime regardless of which server it's - # attached to. Kept here only as a human-readable output (see outputs.tf) - - # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't - # read this value. - data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}" -} diff --git a/infra/modules/prod/outputs.tf b/infra/modules/prod/outputs.tf index 86e4ee2..4897389 100644 --- a/infra/modules/prod/outputs.tf +++ b/infra/modules/prod/outputs.tf @@ -5,8 +5,3 @@ output "ipv4" { output "private_ipv4" { value = var.private_ip } - -output "data_dir" { - description = "Deterministic host path of the auto-mounted volume - see local.data_dir." - value = local.data_dir -} diff --git a/infra/modules/prod/variables.tf b/infra/modules/prod/variables.tf index e105b70..860bb14 100644 --- a/infra/modules/prod/variables.tf +++ b/infra/modules/prod/variables.tf @@ -28,11 +28,6 @@ variable "private_ip" { type = string } -variable "volume_size" { - description = "Size in GB of the volume attached to prod (disk1 in docs/architecture.md)." - type = number -} - variable "allowed_ssh_source_ips" { description = "CIDRs allowed to reach port 22 on prod. Set by the root module to the vpn server's public IP." type = list(string) diff --git a/infra/outputs.tf b/infra/outputs.tf index d0b29b1..f8e427c 100644 --- a/infra/outputs.tf +++ b/infra/outputs.tf @@ -14,16 +14,6 @@ output "prod_private_ipv4" { value = module.prod.private_ipv4 } -output "dev_data_dir" { - description = "Deterministic host path of dev's auto-mounted volume - also written to services/dev/.env as DATA_DIR." - value = module.dev.data_dir -} - -output "prod_data_dir" { - description = "Deterministic host path of prod's auto-mounted volume - also written to services/prod/.env as DATA_DIR." - value = module.prod.data_dir -} - output "vpn_ipv4" { value = module.vpn.ipv4 } diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 3c8c22b..f5e7fdb 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -14,7 +14,5 @@ ssh_key_names = ["your-key-name", "laptop", "phone"] # dev_server_type = "cx22" # prod_server_type = "cx22" # vpn_server_type = "cx22" -# dev_volume_size = 20 -# prod_volume_size = 20 # allowed_ssh_source_ips = ["203.0.113.4/32"] # dns_zones = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"] diff --git a/infra/variables.tf b/infra/variables.tf index 95dc546..e974430 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -1,5 +1,5 @@ variable "location" { - description = "Hetzner Cloud datacenter location for all servers and volumes." + description = "Hetzner Cloud datacenter location for all servers." type = string default = "nbg1" } @@ -34,18 +34,6 @@ variable "vpn_server_type" { default = "cx22" } -variable "dev_volume_size" { - description = "Size in GB of the volume attached to dev" - type = number - default = 20 -} - -variable "prod_volume_size" { - description = "Size in GB of the volume attached to prod (disk1 in docs/architecture.md)." - type = number - default = 20 -} - variable "network_ip_range" { description = "IP range of the private network shared by dev and prod." type = string @@ -84,5 +72,5 @@ variable "allowed_ssh_source_ips" { variable "dns_zones" { description = "Domains to manage as Hetzner DNS zones. Point each domain's registrar NS records at tofu output dns_nameservers for Hetzner to actually become authoritative." type = list(string) - default = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"] + default = ["luke-else.co.uk"] } diff --git a/readme.md b/readme.md index 2e017e4..ed8b055 100644 --- a/readme.md +++ b/readme.md @@ -28,9 +28,6 @@ architecture-beta group network(cloud)[network] in cloud - service disk1(mdi:disk)[Storage] in cloud - service disk2(mdi:disk)[Storage] in cloud - service dev(mdi:server)[dev] in network service prod(mdi:server)[prod] in network @@ -41,24 +38,30 @@ architecture-beta service gateway(mdi:web)[gateway] in cloud + service backups(mdi:bucket)[S3 Backups] + dev:L -- R:prod - disk1:B -- T:prod - disk2:B -- T:dev prod:B -- T:prodfirewall vpn:B -- T:vpnfirewall prodfirewall: L -- R: gateway vpnfirewall: B -- T: gateway + + dev:T -- B:backups + prod:T -- B:backups + vpn:R -- L:backups ``` -`gateway` represents the public internet, not a provisioned resource. +`gateway` represents the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from - not a Hetzner resource, see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3). -| Server | Purpose | Network | Volume | -|---|---|---|---| -| `dev` | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) | `dev-storage` | -| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | `prod-storage` | -| `vpn` | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | none | +| Server | Purpose | Network | +|---|---|---| +| `dev` | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) | +| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | +| `vpn` | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | + +Each host's stateful data lives in local named Docker volumes, backed up nightly to a shared S3 bucket and restored automatically on first boot - see below. No host has a Hetzner Cloud Volume attached any more. `dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach deployment targets on `prod` without exposing that traffic publicly. `vpn` is deliberately kept off this network. Each server has its own Hetzner Cloud Firewall (see `infra/modules//main.tf`) that only opens the ports actually used by the compose stacks running on it, plus SSH restricted to `var.allowed_ssh_source_ips`. @@ -66,7 +69,7 @@ architecture-beta ``` . -├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes, DNS +├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, DNS │ ├── main.tf # root module: wires network + dev/prod/vpn/dns modules together │ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names) │ ├── outputs.tf # pass-through outputs from each host module @@ -75,21 +78,21 @@ architecture-beta │ ├── terraform.tfvars.example │ └── modules/ │ ├── network/ # shared private network + subnet (used by dev and prod) -│ ├── dev/ # dev server + firewall + volume, labeled role=dev -│ ├── prod/ # prod server + firewall + volume, labeled role=prod -│ ├── vpn/ # vpn server + firewall (no private network, no volume), labeled role=vpn +│ ├── dev/ # dev server + firewall, labeled role=dev +│ ├── prod/ # prod server + firewall, labeled role=prod +│ ├── vpn/ # vpn server + firewall (no private network), labeled role=vpn │ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones ├── ansible/ # Ansible: bootstraps each server and deploys/starts services// onto it │ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by the role label above -│ ├── group_vars/ # deploy_user, dev_runner_count, etc. +│ ├── group_vars/ # deploy_user, dev_runner_count, backup_s3_*, etc. │ ├── roles/ │ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades -│ │ └── deploy/ # copies services//, renders .env + Runners/docker-compose.yml +│ │ └── deploy/ # copies services//, renders .env (S3 backup credentials) + Runners/docker-compose.yml │ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml ├── services/ # Docker Compose stacks, grouped by which server they run on -│ ├── dev/ # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) -│ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik -│ ├── vpn/ # OpenVPN + Traefik +│ ├── dev/ # Gitea + CI runner + Traefik + backup/restore (git.luke-else.co.uk, cicd.luke-else.co.uk) +│ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore +│ ├── vpn/ # OpenVPN + Traefik + backup/restore │ └── todo.md # Outstanding manual setup/hardening tasks ├── docs/ │ └── architecture.md # Source of the architecture diagram above @@ -97,9 +100,9 @@ architecture-beta └── assets/ ``` -Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side. +Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, a `backup-docker-compose.yml` + `restore.sh` pair for that host's S3 backup/restore (see below), plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side. -OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, volumes, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services//`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown. +OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services//`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown. ## Prerequisites @@ -108,6 +111,7 @@ OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud res - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` - [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`) - Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner (see [Managing DNS](#managing-dns) — the zones and records themselves are created for you) +- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/), AWS S3, or any other S3-compatible provider) and an access key/secret pair with read/write access to it - not provisioned by OpenTofu (the `hcloud` provider has no Object Storage resource), so create this yourself and pass it to Ansible as `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` (and `BACKUP_S3_ENDPOINT` for anything other than real AWS S3) - see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3) Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below). @@ -127,10 +131,11 @@ tofu apply This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` / `module.dns` in `infra/main.tf`: - `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`) - one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses, each server labeled `role = "dev"/"prod"/"vpn"` for Ansible's dynamic inventory (`modules/dev`, `modules/prod`, `modules/vpn`) -- one `hcloud_volume` each for `dev` and `prod` (`modules/dev`, `modules/prod`; `vpn` has none) - one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns)) -Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`, `tofu output dev_data_dir`, `tofu output prod_data_dir` (the last two are informational only now — Ansible looks the data directory up itself, see below). +No servers have an attached volume — persistent data lives in named Docker volumes backed up to S3 instead, see below. + +Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`. `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`. @@ -148,7 +153,7 @@ Only A records for IPv4 are managed here — none of the modules currently track ## Bootstrapping and deploying with Ansible (`ansible/`) -Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services//` to each server, rendering the files OpenTofu used to generate (`.env`'s `DATA_DIR`, `dev`'s `Runners/docker-compose.yml`), and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version: +Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services//` to each server, rendering `.env` (S3 backup credentials) and, for `dev`, `Runners/docker-compose.yml`, and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version: ```sh cd ansible @@ -164,13 +169,25 @@ ansible-playbook playbooks/site.yml -l role_dev,role_prod Hosts are discovered dynamically from the Hetzner API (`ansible/inventory/hcloud.yml`), grouped into `role_dev`/`role_prod`/`role_vpn` by the `role` label OpenTofu sets on each server — there's no static inventory file to keep in sync, and nothing here reads Terraform state. -### Persistent data lives on the volumes, not the server disk +### Persistent data lives in named Docker volumes, backed up to S3 -`dev` and `prod`'s compose files bind-mount container data to paths on the `dev-storage` / `prod-storage` volumes rather than the server's local disk, so it survives the server being destroyed and recreated: `${DATA_DIR}/gitea:/data`, `${DATA_DIR}/bitwarden/:/data/`, and so on across `services/dev/*.yml` and `services/prod/*.yml`. `vpn` has no volume, so it's untouched - see the architecture table above. +Every stateful service's compose file mounts a plain named Docker volume rather than a bind mount to a Hetzner volume - `gitea_data:/data`, `bitwarden_data:/data/`, `letsencrypt_data:/letsencrypt`, and so on across `services/dev/*.yml`, `services/prod/*.yml`, and `services/vpn/*.yml`. Each volume is declared with an explicit `name:` in every compose file that mounts it (the same pattern already used for the shared `proxy` network), so Compose creates it once and every stack that references it reuses the same underlying volume regardless of which compose file happens to run first. -`DATA_DIR` resolves deterministically: Hetzner always automounts a volume at `/mnt/HC_Volume_`. Ansible's `deploy` role looks the volume up by name (`hcloud_volume_info` for `dev-storage`/`prod-storage`) directly against the Hetzner API and renders that path into `services//.env` as `DATA_DIR=...` on the remote host — `docker compose` auto-loads `.env` from its working directory for variable substitution. Nothing is written locally; re-run `ansible-playbook playbooks/deploy.yml` if a volume is ever destroyed and recreated (new id). `spinup.sh` on both hosts refuses to start anything if `.env` is missing, rather than silently falling back to a relative path. +Each host's `backup-docker-compose.yml` defines two services against those same named volumes: +- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), running on a nightly cron (`0 3 * * *`), tars up every mounted volume and uploads it to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/-.tar.gz`, keeping 14 days of history (`BACKUP_RETENTION_DAYS`). Containers labeled `docker-volume-backup.stop-during-backup=true` (Gitea, Bitwarden, RustDesk, Uptime Kuma, snexo, OpenVPN) are stopped for the duration of the backup so their data is captured in a consistent state, then restarted automatically. Started by `spinup.sh` alongside everything else. +- **`restore`** — a one-shot container (`amazon/aws-cli` running `restore.sh`) that `spinup.sh` runs *before* anything else starts. It checks each of the host's named volumes; any that are already empty are populated from the most recent object under that host's S3 prefix, any that already contain data are left completely untouched. On a fresh server with no prior backup (or a bucket that doesn't exist yet), it's a no-op - services just start with empty volumes as before. -The Gitea Actions runners are the one exception to the `.env` mechanism: their `/data` path is baked directly into the rendered `Runners/docker-compose.yml`, because `Runners/.env` is already reserved for the registration token and gets overwritten on every deploy. +`AWS_S3_BUCKET_NAME`, `AWS_S3_PATH` (set to the host name, e.g. `dev`), `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_ENDPOINT` are rendered into `services//.env` by Ansible's `deploy` role from `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`/`BACKUP_S3_ENDPOINT` in your shell environment (see [Prerequisites](#prerequisites)) — `docker compose` auto-loads `.env` from its working directory. `spinup.sh` on every host refuses to start anything if `.env` is missing. + +The Gitea Actions runners are the one exception: their `/data` is a disposable local cache (runner identity re-registers with Gitea on every `spinup.sh` run anyway), so it's a plain named volume per runner (`gitea_runner_N_data`, baked directly into the rendered `Runners/docker-compose.yml`) that's deliberately left out of the backup/restore stack. + +To trigger a backup or restore manually rather than waiting for cron or a restart: + +```sh +cd services/ +docker compose -f backup-docker-compose.yml exec backup backup # force an immediate backup +docker compose -f backup-docker-compose.yml run --rm restore # restore any currently-empty volumes +``` ### Scaling Gitea Actions runners @@ -232,6 +249,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi | Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` | | Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | N/A | | Watchtower | `watchtower-docker-compose.yml` | — | +| Backup/restore | `backup-docker-compose.yml` | — | ### `prod` @@ -243,6 +261,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi | Bitwarden (Vaultwarden) | `bitwarden-docker-compose.yml` | `bitwarden.luke-else.co.uk` | | RustDesk relay (hbbs/hbbr) | `rd-docker-compose.yml` | `rd.luke-else.co.uk`, ports `21115-21119` | | Watchtower | `watchtower-docker-compose.yml` | — | +| Backup/restore | `backup-docker-compose.yml` | — | ### `vpn` @@ -251,6 +270,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi | Traefik | `traefik-docker-compose.yml` | `traefik.vpn.luke-else.co.uk` | | OpenVPN (Dockovpn) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `1194` | | Watchtower | `watchtower-docker-compose.yml` | — | +| Backup/restore | `backup-docker-compose.yml` | — | ## First-time setup @@ -270,7 +290,7 @@ Then reopen the repo in VS Code with the Dev Containers extension. ## Security notes -- Real secrets (`HCLOUD_TOKEN`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. +- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. The S3 credentials end up in each host's `services//.env` (rendered by Ansible, mode `0600`, gitignored) - anyone with root on a host can read them, same trust boundary as everything else `deploy` can already do there. - SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). - `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP. - Firewalls are allowlists scoped per host in `infra/modules//main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. diff --git a/services/dev/backup-docker-compose.yml b/services/dev/backup-docker-compose.yml new file mode 100644 index 0000000..c9a18e1 --- /dev/null +++ b/services/dev/backup-docker-compose.yml @@ -0,0 +1,48 @@ +services: + # Nightly backup of every named volume on this host to S3. Containers + # labeled docker-volume-backup.stop-during-backup=true (see + # gitea-docker-compose.yml) are stopped for the duration of the backup so + # their data is captured in a consistent state, then restarted. + backup: + image: offen/docker-volume-backup:v2 + container_name: backup + environment: + BACKUP_FILENAME: "dev-%Y-%m-%dT%H-%M-%S.{{ .Extension }}" + BACKUP_CRON_EXPRESSION: "0 3 * * *" + BACKUP_RETENTION_DAYS: "14" + BACKUP_PRUNING_LEEWAY: "5m" + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - gitea_data:/backup/gitea_data:ro + - letsencrypt_data:/backup/letsencrypt_data:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + + # One-shot: restores empty volumes from the latest S3 backup, then exits. + # Run explicitly by spinup.sh before the services that own these volumes + # start - not started by `docker compose up`. + restore: + image: amazon/aws-cli:2 + container_name: backup-restore + entrypoint: ["/bin/bash", "/restore.sh"] + environment: + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - ./restore.sh:/restore.sh:ro + - gitea_data:/restore/gitea_data + - letsencrypt_data:/restore/letsencrypt_data + restart: "no" + +volumes: + gitea_data: + name: gitea_data + letsencrypt_data: + name: letsencrypt_data diff --git a/services/dev/gitea-docker-compose.yml b/services/dev/gitea-docker-compose.yml index 4354a68..836437a 100644 --- a/services/dev/gitea-docker-compose.yml +++ b/services/dev/gitea-docker-compose.yml @@ -4,7 +4,7 @@ services: image: gitea/gitea:latest container_name: gitea volumes: - - ${DATA_DIR}/gitea:/data + - gitea_data:/data - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro networks: @@ -29,7 +29,11 @@ services: labels: ## Expose Gitea Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + + # Stopped for the duration of each backup run so the sqlite db/repos are + # captured in a consistent state - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowmethods=*" - "traefik.http.middlewares.cors-gitea.headers.accesscontrolalloworiginlist=*" - "traefik.http.middlewares.cors-gitea.headers.addvaryheader=true" @@ -51,4 +55,8 @@ services: networks: proxy: - external: true \ No newline at end of file + external: true + +volumes: + gitea_data: + name: gitea_data \ No newline at end of file diff --git a/services/dev/restore.sh b/services/dev/restore.sh new file mode 100755 index 0000000..a7732cd --- /dev/null +++ b/services/dev/restore.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Restores this host's named volumes from the latest S3 backup, but only into +# volumes that are currently empty - never overwrites data that's already +# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml +# run --rm restore` before any service that owns one of the mounted volumes +# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh +# estate): it just leaves the empty volumes alone. +set -euo pipefail + +S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}" +ENDPOINT_ARGS=() +if [ -n "${AWS_ENDPOINT:-}" ]; then + ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}") +fi + +empty_volumes=() +for dir in /restore/*/; do + name="$(basename "$dir")" + if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then + empty_volumes+=("$name") + fi +done + +if [ "${#empty_volumes[@]}" -eq 0 ]; then + echo "All volumes already contain data - skipping restore." + exit 0 +fi + +echo "Empty volumes: ${empty_volumes[*]}" + +latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)" +if [ -z "$latest" ]; then + echo "No existing backup found at ${S3_URI} - starting with empty volumes." + exit 0 +fi + +echo "Restoring ${S3_URI}/${latest} into empty volumes only..." +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz" +mkdir "$tmp/extracted" +tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted" + +for name in "${empty_volumes[@]}"; do + if [ -d "$tmp/extracted/$name" ]; then + echo "Restoring $name..." + cp -a "$tmp/extracted/$name/." "/restore/$name/" + else + echo "No $name/ in backup - leaving volume empty." + fi +done + +echo "Restore complete." diff --git a/services/dev/spindown.sh b/services/dev/spindown.sh index be4e6a3..c495c7a 100755 --- a/services/dev/spindown.sh +++ b/services/dev/spindown.sh @@ -9,7 +9,9 @@ cd .. docker compose -f gitea-docker-compose.yml down docker compose -f traefik-docker-compose.yml down docker compose -f watchtower-docker-compose.yml down +docker compose -f backup-docker-compose.yml down docker rmi $(docker images -q) docker system prune -f -a -docker volume prune -f -a +# Deliberately no `docker volume prune` - gitea_data/letsencrypt_data/etc. are +# named volumes holding real data, backed up to S3 but not disposable. diff --git a/services/dev/spinup.sh b/services/dev/spinup.sh index 0692a37..22044b8 100755 --- a/services/dev/spinup.sh +++ b/services/dev/spinup.sh @@ -4,10 +4,14 @@ set -e cd "$(dirname "$0")" if [ ! -f .env ]; then - echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 + echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 exit 1 fi +echo "Restoring named volumes from S3 backup if any are empty..." +docker compose -f backup-docker-compose.yml pull restore +docker compose -f backup-docker-compose.yml run --rm restore + docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d @@ -31,3 +35,6 @@ docker compose pull && docker compose up -d cd .. docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d + +docker compose -f backup-docker-compose.yml pull backup +docker compose -f backup-docker-compose.yml up -d backup diff --git a/services/dev/traefik-docker-compose.yml b/services/dev/traefik-docker-compose.yml index 0a69b43..6e46ebc 100644 --- a/services/dev/traefik-docker-compose.yml +++ b/services/dev/traefik-docker-compose.yml @@ -15,7 +15,7 @@ services: - "80:80" - "443:443" volumes: - - "${DATA_DIR}/letsencrypt:/letsencrypt" + - "letsencrypt_data:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy @@ -36,3 +36,7 @@ services: networks: proxy: name: proxy + +volumes: + letsencrypt_data: + name: letsencrypt_data diff --git a/services/prod/backup-docker-compose.yml b/services/prod/backup-docker-compose.yml new file mode 100644 index 0000000..7e996f0 --- /dev/null +++ b/services/prod/backup-docker-compose.yml @@ -0,0 +1,59 @@ +services: + # Nightly backup of every named volume on this host to S3. Containers + # labeled docker-volume-backup.stop-during-backup=true (see + # bitwarden-docker-compose.yml, rd-docker-compose.yml, + # status-docker-compose.yml, web-docker-compose.yml) are stopped for the + # duration of the backup so their data is captured in a consistent state, + # then restarted. + backup: + image: offen/docker-volume-backup:v2 + container_name: backup + environment: + BACKUP_FILENAME: "prod-%Y-%m-%dT%H-%M-%S.{{ .Extension }}" + BACKUP_CRON_EXPRESSION: "0 3 * * *" + BACKUP_RETENTION_DAYS: "14" + BACKUP_PRUNING_LEEWAY: "5m" + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - bitwarden_data:/backup/bitwarden_data:ro + - rustdesk_data:/backup/rustdesk_data:ro + - uptime_kuma_data:/backup/uptime_kuma_data:ro + - snexo_data:/backup/snexo_data:ro + - letsencrypt_data:/backup/letsencrypt_data:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + + # One-shot: restores empty volumes from the latest S3 backup, then exits. + # Run explicitly by spinup.sh before the services that own these volumes + # start - not started by `docker compose up`. + restore: + image: amazon/aws-cli:2 + container_name: backup-restore + entrypoint: [ "/bin/bash", "/restore.sh" ] + environment: + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - ./restore.sh:/restore.sh:ro + - bitwarden_data:/restore/bitwarden_data + - rustdesk_data:/restore/rustdesk_data + - uptime_kuma_data:/restore/uptime_kuma_data + - letsencrypt_data:/restore/letsencrypt_data + restart: "no" + +volumes: + bitwarden_data: + name: bitwarden_data + rustdesk_data: + name: rustdesk_data + uptime_kuma_data: + name: uptime_kuma_data + letsencrypt_data: + name: letsencrypt_data diff --git a/services/prod/bitwarden-docker-compose.yml b/services/prod/bitwarden-docker-compose.yml index 8128bda..2cd48fa 100644 --- a/services/prod/bitwarden-docker-compose.yml +++ b/services/prod/bitwarden-docker-compose.yml @@ -5,13 +5,16 @@ services: image: "vaultwarden/server:latest" container_name: vaultwarden volumes: - - ${DATA_DIR}/bitwarden/:/data/ + - bitwarden_data:/data/ networks: - proxy labels: ## Expose Bitwarden Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.bitwarden-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.bitwarden-insecure.rule=Host(`bitwarden.luke-else.co.uk`)" @@ -24,4 +27,8 @@ services: networks: proxy: - external: true \ No newline at end of file + external: true + +volumes: + bitwarden_data: + name: bitwarden_data \ No newline at end of file diff --git a/services/prod/rd-docker-compose.yml b/services/prod/rd-docker-compose.yml index bab3865..5660ba1 100644 --- a/services/prod/rd-docker-compose.yml +++ b/services/prod/rd-docker-compose.yml @@ -6,7 +6,7 @@ services: restart: unless-stopped volumes: - - ${DATA_DIR}/rustdesk:/root + - rustdesk_data:/root networks: - proxy @@ -20,6 +20,9 @@ services: labels: - "traefik.enable=true" + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.rustdesk-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.rustdesk-insecure.rule=Host(`rd.luke-else.co.uk`)" @@ -39,7 +42,7 @@ services: restart: unless-stopped volumes: - - ${DATA_DIR}/rustdesk:/root + - rustdesk_data:/root networks: - proxy @@ -50,4 +53,8 @@ services: networks: proxy: - external: true \ No newline at end of file + external: true + +volumes: + rustdesk_data: + name: rustdesk_data \ No newline at end of file diff --git a/services/prod/restore.sh b/services/prod/restore.sh new file mode 100755 index 0000000..a7732cd --- /dev/null +++ b/services/prod/restore.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Restores this host's named volumes from the latest S3 backup, but only into +# volumes that are currently empty - never overwrites data that's already +# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml +# run --rm restore` before any service that owns one of the mounted volumes +# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh +# estate): it just leaves the empty volumes alone. +set -euo pipefail + +S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}" +ENDPOINT_ARGS=() +if [ -n "${AWS_ENDPOINT:-}" ]; then + ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}") +fi + +empty_volumes=() +for dir in /restore/*/; do + name="$(basename "$dir")" + if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then + empty_volumes+=("$name") + fi +done + +if [ "${#empty_volumes[@]}" -eq 0 ]; then + echo "All volumes already contain data - skipping restore." + exit 0 +fi + +echo "Empty volumes: ${empty_volumes[*]}" + +latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)" +if [ -z "$latest" ]; then + echo "No existing backup found at ${S3_URI} - starting with empty volumes." + exit 0 +fi + +echo "Restoring ${S3_URI}/${latest} into empty volumes only..." +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz" +mkdir "$tmp/extracted" +tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted" + +for name in "${empty_volumes[@]}"; do + if [ -d "$tmp/extracted/$name" ]; then + echo "Restoring $name..." + cp -a "$tmp/extracted/$name/." "/restore/$name/" + else + echo "No $name/ in backup - leaving volume empty." + fi +done + +echo "Restore complete." diff --git a/services/prod/spindown.sh b/services/prod/spindown.sh index 665ab2b..e6c19ad 100755 --- a/services/prod/spindown.sh +++ b/services/prod/spindown.sh @@ -8,7 +8,9 @@ docker compose -f bitwarden-docker-compose.yml down docker compose -f rd-docker-compose.yml down docker compose -f watchtower-docker-compose.yml down docker compose -f traefik-docker-compose.yml down +docker compose -f backup-docker-compose.yml down docker rmi $(docker images -q) docker system prune -f -a -docker volume prune -f -a +# Deliberately no `docker volume prune` - bitwarden_data/rustdesk_data/etc. are +# named volumes holding real data, backed up to S3 but not disposable. diff --git a/services/prod/spinup.sh b/services/prod/spinup.sh index ef7e9ee..ac5092b 100755 --- a/services/prod/spinup.sh +++ b/services/prod/spinup.sh @@ -4,10 +4,14 @@ set -e cd "$(dirname "$0")" if [ ! -f .env ]; then - echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 + echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 exit 1 fi +echo "Restoring named volumes from S3 backup if any are empty..." +docker compose -f backup-docker-compose.yml pull restore +docker compose -f backup-docker-compose.yml run --rm restore + docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d sleep 20 # Allow Traefik + registry auth to settle before starting the rest of the services @@ -17,3 +21,6 @@ docker compose -f status-docker-compose.yml pull && docker compose -f status-doc docker compose -f web-docker-compose.yml pull && docker compose -f web-docker-compose.yml up -d docker compose -f bitwarden-docker-compose.yml pull && docker compose -f bitwarden-docker-compose.yml up -d docker compose -f rd-docker-compose.yml pull && docker compose -f rd-docker-compose.yml up -d + +docker compose -f backup-docker-compose.yml pull backup +docker compose -f backup-docker-compose.yml up -d backup diff --git a/services/prod/status-docker-compose.yml b/services/prod/status-docker-compose.yml index 9595387..f9c5822 100644 --- a/services/prod/status-docker-compose.yml +++ b/services/prod/status-docker-compose.yml @@ -3,13 +3,16 @@ services: image: louislam/uptime-kuma:latest container_name: status volumes: - - ${DATA_DIR}/uptime-kuma/data:/app/data + - uptime_kuma_data:/app/data networks: - proxy labels: ## Expose uptime-kuma Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)" @@ -23,3 +26,7 @@ services: networks: proxy: external: true + +volumes: + uptime_kuma_data: + name: uptime_kuma_data diff --git a/services/prod/traefik-docker-compose.yml b/services/prod/traefik-docker-compose.yml index a4ed496..78b6483 100644 --- a/services/prod/traefik-docker-compose.yml +++ b/services/prod/traefik-docker-compose.yml @@ -20,7 +20,7 @@ services: - "443:443" - "27017:27017" volumes: - - "${DATA_DIR}/letsencrypt:/letsencrypt" + - "letsencrypt_data:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy @@ -41,3 +41,7 @@ services: networks: proxy: name: proxy + +volumes: + letsencrypt_data: + name: letsencrypt_data diff --git a/services/prod/web-docker-compose.yml b/services/prod/web-docker-compose.yml index af5adac..e4edb7f 100644 --- a/services/prod/web-docker-compose.yml +++ b/services/prod/web-docker-compose.yml @@ -1,6 +1,6 @@ services: -#Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080) + #Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080) luke-else: image: git.luke-else.co.uk/luke-else/luke-else.co.uk container_name: luke-else @@ -9,7 +9,7 @@ services: labels: ## Expose luke-else Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.personal-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.personal-insecure.rule=Host(`luke-else.co.uk`)" @@ -28,7 +28,7 @@ services: labels: ## Expose luke-else Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.personal-dev-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.personal-dev-insecure.rule=Host(`dev.luke-else.co.uk`)" @@ -47,7 +47,7 @@ services: labels: ## Expose metarius Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.metarius-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.metarius-insecure.rule=Host(`metarius.luke-else.co.uk`)" @@ -66,7 +66,7 @@ services: labels: ## Expose divine-couture Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.divine-couture-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.divine-couture-insecure.rule=Host(`www.divine-couture.co.uk`)" @@ -77,27 +77,6 @@ services: - "traefik.http.routers.divine-couture.tls.certresolver=myresolver" restart: unless-stopped - snexo: - image: "php:apache" - container_name: snexo - volumes: - - ${DATA_DIR}/snexo.co.uk/:/var/www/html - networks: - - proxy - labels: - ## Expose Snexo Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.snexo-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.snexo-insecure.rule=Host(`snexo.co.uk`)" - - "traefik.http.routers.snexo-insecure.entrypoints=web" - - - "traefik.http.routers.snexo.rule=Host(`snexo.co.uk`)" - - "traefik.http.routers.snexo.entrypoints=websecure" - - "traefik.http.routers.snexo.tls.certresolver=myresolver" - restart: unless-stopped - networks: proxy: external: true diff --git a/services/vpn/backup-docker-compose.yml b/services/vpn/backup-docker-compose.yml new file mode 100644 index 0000000..0c2ae8e --- /dev/null +++ b/services/vpn/backup-docker-compose.yml @@ -0,0 +1,48 @@ +services: + # Nightly backup of every named volume on this host to S3. Containers + # labeled docker-volume-backup.stop-during-backup=true (see + # vpn-docker-compose.yml) are stopped for the duration of the backup so + # their data is captured in a consistent state, then restarted. + backup: + image: offen/docker-volume-backup:v2 + container_name: backup + environment: + BACKUP_FILENAME: "vpn-%Y-%m-%dT%H-%M-%S.{{ .Extension }}" + BACKUP_CRON_EXPRESSION: "0 3 * * *" + BACKUP_RETENTION_DAYS: "14" + BACKUP_PRUNING_LEEWAY: "5m" + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - openvpn_data:/backup/openvpn_data:ro + - letsencrypt_data:/backup/letsencrypt_data:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + + # One-shot: restores empty volumes from the latest S3 backup, then exits. + # Run explicitly by spinup.sh before the services that own these volumes + # start - not started by `docker compose up`. + restore: + image: amazon/aws-cli:2 + container_name: backup-restore + entrypoint: ["/bin/bash", "/restore.sh"] + environment: + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - ./restore.sh:/restore.sh:ro + - openvpn_data:/restore/openvpn_data + - letsencrypt_data:/restore/letsencrypt_data + restart: "no" + +volumes: + openvpn_data: + name: openvpn_data + letsencrypt_data: + name: letsencrypt_data diff --git a/services/vpn/restore.sh b/services/vpn/restore.sh new file mode 100755 index 0000000..a7732cd --- /dev/null +++ b/services/vpn/restore.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Restores this host's named volumes from the latest S3 backup, but only into +# volumes that are currently empty - never overwrites data that's already +# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml +# run --rm restore` before any service that owns one of the mounted volumes +# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh +# estate): it just leaves the empty volumes alone. +set -euo pipefail + +S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}" +ENDPOINT_ARGS=() +if [ -n "${AWS_ENDPOINT:-}" ]; then + ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}") +fi + +empty_volumes=() +for dir in /restore/*/; do + name="$(basename "$dir")" + if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then + empty_volumes+=("$name") + fi +done + +if [ "${#empty_volumes[@]}" -eq 0 ]; then + echo "All volumes already contain data - skipping restore." + exit 0 +fi + +echo "Empty volumes: ${empty_volumes[*]}" + +latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)" +if [ -z "$latest" ]; then + echo "No existing backup found at ${S3_URI} - starting with empty volumes." + exit 0 +fi + +echo "Restoring ${S3_URI}/${latest} into empty volumes only..." +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz" +mkdir "$tmp/extracted" +tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted" + +for name in "${empty_volumes[@]}"; do + if [ -d "$tmp/extracted/$name" ]; then + echo "Restoring $name..." + cp -a "$tmp/extracted/$name/." "/restore/$name/" + else + echo "No $name/ in backup - leaving volume empty." + fi +done + +echo "Restore complete." diff --git a/services/vpn/spindown.sh b/services/vpn/spindown.sh index c30233f..c093daa 100755 --- a/services/vpn/spindown.sh +++ b/services/vpn/spindown.sh @@ -5,7 +5,9 @@ cd "$(dirname "$0")" docker compose -f vpn-docker-compose.yml down docker compose -f traefik-docker-compose.yml down docker compose -f watchtower-docker-compose.yml down +docker compose -f backup-docker-compose.yml down docker rmi $(docker images -q) docker system prune -f -a -docker volume prune -f -a +# Deliberately no `docker volume prune` - openvpn_data/letsencrypt_data are +# named volumes holding real data, backed up to S3 but not disposable. diff --git a/services/vpn/spinup.sh b/services/vpn/spinup.sh index 14250a8..1911994 100755 --- a/services/vpn/spinup.sh +++ b/services/vpn/spinup.sh @@ -3,6 +3,18 @@ set -e cd "$(dirname "$0")" +if [ ! -f .env ]; then + echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 + exit 1 +fi + +echo "Restoring named volumes from S3 backup if any are empty..." +docker compose -f backup-docker-compose.yml pull restore +docker compose -f backup-docker-compose.yml run --rm restore + docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d docker compose -f vpn-docker-compose.yml pull && docker compose -f vpn-docker-compose.yml up -d docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d + +docker compose -f backup-docker-compose.yml pull backup +docker compose -f backup-docker-compose.yml up -d backup diff --git a/services/vpn/traefik-docker-compose.yml b/services/vpn/traefik-docker-compose.yml index 50a0f18..234ad1c 100644 --- a/services/vpn/traefik-docker-compose.yml +++ b/services/vpn/traefik-docker-compose.yml @@ -15,7 +15,7 @@ services: - "80:80" - "443:443" volumes: - - "./letsencrypt:/letsencrypt" + - "letsencrypt_data:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy @@ -36,3 +36,7 @@ services: networks: proxy: name: proxy + +volumes: + letsencrypt_data: + name: letsencrypt_data diff --git a/services/vpn/vpn-docker-compose.yml b/services/vpn/vpn-docker-compose.yml index 502dcd8..f78e1af 100644 --- a/services/vpn/vpn-docker-compose.yml +++ b/services/vpn/vpn-docker-compose.yml @@ -8,11 +8,14 @@ services: environment: HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address volumes: - - ./openvpn_conf:/opt/Dockovpn_data + - openvpn_data:/opt/Dockovpn_data labels: ## Expose vpn Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.vpn-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.vpn-insecure.rule=Host(`vpn.luke-else.co.uk`)" @@ -21,4 +24,8 @@ services: - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)" - "traefik.http.routers.vpn.entrypoints=websecure" - "traefik.http.routers.vpn.tls.certresolver=myresolver" - restart: always \ No newline at end of file + restart: always + +volumes: + openvpn_data: + name: openvpn_data \ No newline at end of file