feat: backup and restore

ansible/README.md

@@ -2,25 +2,26 @@
 
 Bootstraps, syncs, and starts/stops the Docker Compose stacks in `../services/`
 on the three servers OpenTofu provisions (see `../infra/`). OpenTofu still owns
-the actual cloud resources (servers, network, firewalls, volumes, DNS);
+the actual cloud resources (servers, network, firewalls, DNS);
 everything downstream of "the server exists" - installing Docker, creating the
 `deploy` user, hardening SSH, copying `services/<host>/`, rendering `.env` /
 the Gitea runner compose file, and running `spinup.sh`/`spindown.sh` - lives
-here instead.
+here instead. Persistent service data lives in named Docker volumes backed up
+to S3, not on a Hetzner volume - see `../readme.md`.
 
 ## Contents
 
 - `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API
   directly (not Terraform state) and groups servers by their `role` label
   (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules/<host>/main.tf`).
-- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, `HCLOUD_TOKEN`
-  lookup), `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` (per-role vars,
-  e.g. `dev_runner_count`).
+- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path,
+  `backup_s3_*` S3 backup credentials), `role_dev.yml` / `role_prod.yml` /
+  `role_vpn.yml` (per-role vars, e.g. `dev_runner_count`).
 - `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening,
   unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`.
-- `roles/deploy/` - copies `services/<host>/` to the server, looks up the
-  host's data volume directly from the Hetzner API and renders `.env`
-  (`DATA_DIR=...`) and, for `dev`, `Runners/docker-compose.yml`.
+- `roles/deploy/` - copies `services/<host>/` to the server and renders `.env`
+  (S3 backup credentials, see `../readme.md`) and, for `dev`,
+  `Runners/docker-compose.yml`.
 - `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`,
   and `site.yml` (all three in order).
 
@@ -30,6 +31,10 @@ here instead.
 cd ansible
 ansible-galaxy collection install -r requirements.yml
 export HCLOUD_TOKEN=your-hetzner-api-token   # never commit this
+export BACKUP_S3_BUCKET=your-bucket-name
+export BACKUP_S3_ACCESS_KEY_ID=your-access-key-id
+export BACKUP_S3_SECRET_ACCESS_KEY=your-secret-access-key
+export BACKUP_S3_ENDPOINT=your-s3-endpoint   # omit for real AWS S3
 ```
 
 `ansible_ssh_private_key_file` (see `group_vars/all.yml`) defaults to

ansible/group_vars/all.yml

@@ -6,9 +6,16 @@ deploy_user: deploy
 ansible_user: "{{ deploy_user }}"
 ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}"
 
-# Used by the deploy role to look up each host's data volume directly from the
-# Hetzner API (hcloud_volume_info) rather than from Terraform state/outputs.
-hcloud_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
-
 # Repo root's services/ directory, relative to wherever a playbook lives under ansible/playbooks/.
 services_root: "{{ playbook_dir }}/../../services"
+
+# S3-compatible bucket that services/<host>/backup-docker-compose.yml backs up
+# to and restores from - see readme.md. Not provisioned by OpenTofu (the
+# hcloud provider has no Object Storage resource), so these must point at a
+# bucket you've created yourself. AWS_ENDPOINT is only needed for a
+# non-AWS S3-compatible provider (e.g. Hetzner Object Storage); leave it unset
+# to use real AWS S3.
+backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}"
+backup_s3_access_key_id: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY_ID') }}"
+backup_s3_secret_access_key: "{{ lookup('env', 'BACKUP_S3_SECRET_ACCESS_KEY') }}"
+backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}"

ansible/roles/deploy/tasks/main.yml

@@ -3,6 +3,10 @@
 # used to be generated by OpenTofu (services/<host>/.env and, for dev,
 # Runners/docker-compose.yml) - replaces the null_resource.deploy_services +
 # local_file provisioners previously in infra/modules/<host>/main.tf.
+#
+# Persistent data lives in named Docker volumes now, not on a Hetzner volume -
+# see services/<host>/backup-docker-compose.yml. .env only carries the S3
+# credentials that backup/restore need, rendered for every host.
 
 - name: Ensure services directory exists on the host
   ansible.builtin.file:
@@ -16,28 +20,11 @@
     dest: "/home/{{ deploy_user }}/services/{{ service_group }}/"
     mode: preserve
 
-- name: Look up this host's data volume
-  when: service_group in ['dev', 'prod']
-  delegate_to: localhost
-  become: false
-  hetzner.hcloud.hcloud_volume_info:
-    api_token: "{{ hcloud_token }}"
-    name: "{{ service_group }}-storage"
-  register: _volume_info
-
-# Hetzner's automount tooling always mounts an auto-mounted volume at
-# /mnt/HC_Volume_<volume-id> - deterministic once the volume exists.
-- name: Compute DATA_DIR from the volume's id
-  when: service_group in ['dev', 'prod']
-  ansible.builtin.set_fact:
-    data_dir: "/mnt/HC_Volume_{{ _volume_info.hcloud_volume_info[0].id }}"
-
-- name: Render DATA_DIR into .env (auto-loaded by docker compose)
-  when: service_group in ['dev', 'prod']
+- name: Render S3 backup credentials into .env (auto-loaded by docker compose)
   ansible.builtin.template:
     src: env.j2
     dest: "/home/{{ deploy_user }}/services/{{ service_group }}/.env"
-    mode: "0644"
+    mode: "0600"
 
 - name: Render Gitea Actions runner compose file
   when: service_group == 'dev'
@@ -46,10 +33,11 @@
     dest: "/home/{{ deploy_user }}/services/dev/Runners/docker-compose.yml"
     mode: "0644"
 
-- name: Make spinup/spindown scripts executable
+- name: Make spinup/spindown/restore scripts executable
   ansible.builtin.file:
     path: "/home/{{ deploy_user }}/services/{{ service_group }}/{{ item }}"
     mode: "0755"
   loop:
     - spinup.sh
     - spindown.sh
+    - restore.sh

ansible/roles/deploy/templates/env.j2

@@ -1 +1,5 @@
-DATA_DIR={{ data_dir }}
+AWS_ACCESS_KEY_ID={{ backup_s3_access_key_id }}
+AWS_SECRET_ACCESS_KEY={{ backup_s3_secret_access_key }}
+AWS_S3_BUCKET_NAME={{ backup_s3_bucket }}
+AWS_S3_PATH={{ service_group }}
+AWS_ENDPOINT={{ backup_s3_endpoint }}

ansible/roles/deploy/templates/runners-docker-compose.yml.j2

@@ -7,10 +7,9 @@
 # (not baked in here) - services/dev/spinup.sh generates a fresh token and
 # writes it to Runners/.env immediately before starting these containers.
 #
-# /data lives on dev's volume ({{ data_dir }}) so runner identity survives a
-# server rebuild - baked in directly rather than via .env, since Runners/.env
-# is already reserved for the registration token above and gets overwritten
-# on every deploy.
+# /data is a disposable local cache (runner identity re-registers on every
+# spinup.sh run anyway), so it's a plain named volume - not part of the S3
+# backup/restore stack in services/dev/backup-docker-compose.yml.
 services:
 {% for i in range(1, dev_runner_count + 1) %}
   runner-{{ i }}:
@@ -18,7 +17,7 @@ services:
     container_name: gitea_runner_{{ i }}
     volumes:
       - ./config.yaml:/config.yaml
-      - {{ data_dir }}/gitea_runner_{{ i }}:/data
+      - gitea_runner_{{ i }}_data:/data
       - /var/run/docker.sock:/var/run/docker.sock
     networks:
       - proxy
@@ -33,3 +32,9 @@ services:
 networks:
   proxy:
     external: true
+
+volumes:
+{% for i in range(1, dev_runner_count + 1) %}
+  gitea_runner_{{ i }}_data:
+    name: gitea_runner_{{ i }}_data
+{% endfor %}