diff --git a/.devcontainer b/.devcontainer new file mode 160000 index 0000000..7a54748 --- /dev/null +++ b/.devcontainer @@ -0,0 +1 @@ +Subproject commit 7a54748c0e189afb715d4409b24e7e6122e44dab diff --git a/.gitignore b/.gitignore index 8c4ebe1..c2bd0a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,57 @@ -# Ignore all files -** +# Ignore all files related to services +**/services # Allow docker-compose.yml !**/docker-compose.yml !/* -# Allow assets folder -!assets/** - # Don't allow .env files, just env-exmaple files !.env-example + +# Secrets generated at deploy time (e.g. services/dev/Runners/.env, holding the +# live Gitea Actions registration token) - never committed, regardless of the +# services/docker-compose.yml carve-out above. +**/.env + +# Variables set through control.sh's "Set variables" menu (HCLOUD_TOKEN, +# BACKUP_S3_*, etc.) - never committed. +.control.env + +### Ansible ### +*.retry + +### Terraform ### +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..070ca83 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule ".devcontainer"] + path = .devcontainer + url = ssh://git@git.luke-else.co.uk:222/luke-else/.devcontainer.git + branch = tf diff --git a/Database/docker-compose.yml b/Database/docker-compose.yml deleted file mode 100644 index 3d1a9a3..0000000 --- a/Database/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -services: - -#MongoDB (27017) - mongodb: - image: "mongo:latest" - container_name: mongoDB - volumes: - - ./mongo/:/data/db - networks: - - proxy - labels: - - "traefik.enable=true" - - "traefik.tcp.routers.mongodb.rule=HostSNI(`*`)" - - "traefik.tcp.routers.mongodb.entrypoints=mongo" - - "traefik.tcp.routers.mongodb.service=mongodb" - - "traefik.tcp.services.mongodb.loadbalancer.server.port=27017" - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: rootpassword - restart: unless-stopped - -networks: - proxy: - external: true \ No newline at end of file diff --git a/Development/Runners/docker-compose.yml b/Development/Runners/docker-compose.yml deleted file mode 100644 index f9f5acd..0000000 --- a/Development/Runners/docker-compose.yml +++ /dev/null @@ -1,33 +0,0 @@ -services: - # Runner - runner: - image: gitea/act_runner:latest - container_name: gitea_runner - volumes: - - ./config.yaml:/config.yaml - - ./gitea_runner:/data - - /var/run/docker.sock:/var/run/docker.sock - networks: - - proxy - environment: - CONFIG_FILE: /config.yaml - GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" - GITEA_RUNNER_REGISTRATION_TOKEN: "INSERT REGISTRATION TOKEN" - GITEA_RUNNER_NAME: "CICD" - labels: - ## Expose cicd Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.cicd-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.cicd-insecure.rule=Host(`cicd.luke-else.co.uk`)" - - "traefik.http.routers.cicd-insecure.entrypoints=web" - - - "traefik.http.routers.cicd.rule=Host(`cicd.luke-else.co.uk`)" - - "traefik.http.routers.cicd.entrypoints=websecure" - - "traefik.http.routers.cicd.tls.certresolver=myresolver" - restart: unless-stopped - -networks: - proxy: - external: true \ No newline at end of file diff --git a/Misc/docker-compose.yml b/Misc/docker-compose.yml deleted file mode 100644 index 7914b79..0000000 --- a/Misc/docker-compose.yml +++ /dev/null @@ -1,51 +0,0 @@ -services: - status: - image: louislam/uptime-kuma:latest - container_name: status - volumes: - - ./uptime-kuma/data:/app/data - networks: - - proxy - labels: - ## Expose uptime-kuma Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)" - - "traefik.http.routers.status-insecure.entrypoints=web" - - - "traefik.http.routers.status.rule=Host(`status.luke-else.co.uk`)" - - "traefik.http.routers.status.entrypoints=websecure" - - "traefik.http.routers.status.tls.certresolver=myresolver" - restart: unless-stopped - - portainer: - image: portainer/portainer-ce:latest - container_name: portainer - security_opt: - - no-new-privileges:true - volumes: - - /etc/localtime:/etc/localtime:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./portainer-data:/data - networks: - - proxy - labels: - ## Expose portainer Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.portainer-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.portainer-insecure.rule=Host(`portainer.luke-else.co.uk`)" - - "traefik.http.routers.portainer-insecure.entrypoints=web" - - - "traefik.http.routers.portainer.rule=Host(`portainer.luke-else.co.uk`)" - - "traefik.http.services.portainer.loadbalancer.server.port=9000" - - "traefik.http.routers.portainer.entrypoints=websecure" - - "traefik.http.routers.portainer.tls.certresolver=myresolver" - restart: unless-stopped - -networks: - proxy: - external: true \ No newline at end of file diff --git a/Tracking/docker-compose.yml b/Tracking/docker-compose.yml deleted file mode 100644 index 6fe612a..0000000 --- a/Tracking/docker-compose.yml +++ /dev/null @@ -1,52 +0,0 @@ -services: - ackee: - image: electerious/ackee - container_name: ackee - environment: - - WAIT_HOSTS=mongo:27017 - - ACKEE_MONGODB=mongodb://mongo-ackee:27017/ackee - env_file: - - .env - expose: - - 3000 - networks: - - tracking - - proxy - labels: - ## Expose Ackee Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - # Enable CORS headers - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowmethods=*" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolalloworiginlist=https://luke-else.co.uk" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowcredentials=true" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowheaders=Content-Type,Authorization" - - "traefik.http.middlewares.cors-tracking.headers.addvaryheader=true" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolmaxage=100" - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.ackee-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.ackee-insecure.rule=Host(`tracking.luke-else.co.uk`)" - - "traefik.http.routers.ackee-insecure.entrypoints=web" - - - "traefik.http.routers.ackee.rule=Host(`tracking.luke-else.co.uk`)" - - "traefik.http.routers.ackee.entrypoints=websecure" - - "traefik.http.routers.ackee.tls.certresolver=myresolver" - - "traefik.http.routers.ackee.middlewares=cors-tracking" - depends_on: - - mongo - restart: unless-stopped - - mongo: - image: mongo - container_name: mongo-ackee - volumes: - - ./data:/data/db - networks: - - tracking - restart: unless-stopped - -networks: - proxy: - external: true - tracking: \ No newline at end of file diff --git a/VPN/docker-compose.yml b/VPN/docker-compose.yml deleted file mode 100644 index 502dcd8..0000000 --- a/VPN/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -services: - dockovpn: - image: alekslitvinenk/openvpn - cap_add: - - NET_ADMIN - ports: - - 1194:1194/udp # Expose tcp if you defined HOST_TUN_PROTOCOL=tcp - environment: - HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address - volumes: - - ./openvpn_conf:/opt/Dockovpn_data - labels: - ## Expose vpn Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.vpn-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.vpn-insecure.rule=Host(`vpn.luke-else.co.uk`)" - - "traefik.http.routers.vpn-insecure.entrypoints=web" - - - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)" - - "traefik.http.routers.vpn.entrypoints=websecure" - - "traefik.http.routers.vpn.tls.certresolver=myresolver" - restart: always \ No newline at end of file diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 0000000..0555d70 --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,93 @@ +# Ansible + +Bootstraps, syncs, and starts/stops the Docker Compose stacks in `../services/` +on the three servers OpenTofu provisions (see `../infra/`). OpenTofu still owns +the actual cloud resources (servers, network, firewalls, DNS); +everything downstream of "the server exists" - installing Docker, creating the +`deploy` user, hardening SSH, copying `services//`, rendering `.env` / +the Gitea runner compose file, and running `spinup.sh`/`spindown.sh` - lives +here instead. Persistent service data lives in named Docker volumes backed up +to S3, not on a Hetzner volume - see `../readme.md`. + +## Contents + +- `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API + directly (not Terraform state) and groups servers by their `role` label + (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules//main.tf`). +- `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening, + unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`. +- `roles/deploy/` - copies `services//` to the server and renders `.env` + (S3 backup credentials, see `../readme.md`) and, for `dev`, + `Runners/docker-compose.yml`. +- `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`, + `site.yml` (all three in order), and `group_vars/` (`all.yml` - shared: + `deploy_user`, SSH key path, `backup_s3_*` S3 backup credentials; + `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` - per-role vars, e.g. + `service_group`). `group_vars/` lives here rather than at the `ansible/` + root because Ansible only auto-loads `group_vars/`/`host_vars/` from + directories adjacent to the playbook file or the inventory source - not + from an arbitrary sibling directory. + +## Prerequisites + +```sh +cd ansible +ansible-galaxy collection install -r requirements.yml +export HCLOUD_TOKEN=your-hetzner-api-token # never commit this +export BACKUP_S3_BUCKET=your-bucket-name +export BACKUP_S3_ACCESS_KEY_ID=your-access-key-id +export BACKUP_S3_SECRET_ACCESS_KEY=your-secret-access-key +export BACKUP_S3_ENDPOINT=your-s3-endpoint # omit for real AWS S3 +``` + +`../control.sh`'s **Set variables** menu does the same thing interactively and +persists your answers to `../.control.env` (gitignored) so you don't have to +re-export them every session. + +`deploy_user` (see `playbooks/group_vars/all.yml`) defaults to `deploy` - override with +the `DEPLOY_USER` env var if you want a different non-root user created on each +host. `ansible_ssh_private_key_file` (same file) defaults to +`~/.ssh/id_ed25519` - override with `ANSIBLE_SSH_PRIVATE_KEY_FILE` if you use a +different key. It must match one of the SSH keys OpenTofu installed on the +servers (`var.ssh_key_names`). + +## Usage + +Same chicken-and-egg as OpenTofu's own first-apply ordering (see +`../readme.md`): `dev`/`prod`'s firewalls only accept SSH from `vpn`'s own +public IP, so you must bring `vpn` up and connect to it before `dev`/`prod` +are reachable at all. + +```sh +# 1. vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly +ansible-playbook playbooks/bootstrap.yml -l role_vpn +ansible-playbook playbooks/deploy.yml -l role_vpn +ansible-playbook playbooks/spinup.yml -l role_vpn + +# 2. visit https://vpn.luke-else.co.uk, complete wg-easy's first-run setup, +# add a client peer, and connect with a WireGuard client - now dev/prod +# are reachable, since your egress IP matches their firewall rule +ansible-playbook playbooks/bootstrap.yml -l role_dev,role_prod +ansible-playbook playbooks/deploy.yml -l role_dev,role_prod +ansible-playbook playbooks/spinup.yml -l role_dev,role_prod + +# ...or once already bootstrapped and connected, do everything in one go: +ansible-playbook playbooks/site.yml -l role_dev,role_prod +``` + +Re-run `deploy.yml` + `spinup.yml` any time a compose file changes - `spinup.sh` +only starts/updates what's changed, same as before. `dev` always runs three +Gitea Actions runners (fixed in +`roles/deploy/templates/runners-docker-compose.yml.j2`). + +`spindown.yml` runs `spindown.sh`, which does a full `docker system/volume +prune -a` on the host - it prompts for a `yes` confirmation before running, +since the dynamic inventory makes it trivial to target multiple hosts at once +(`-l role_dev,role_prod`). + +Inspect what the dynamic inventory currently sees: + +```sh +ansible-inventory --graph +ansible-inventory --list +``` diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..0829ba6 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,9 @@ +[defaults] +inventory = inventory/hcloud.yml +roles_path = roles +host_key_checking = False +retry_files_enabled = False +interpreter_python = auto_silent + +[inventory] +enable_plugins = hetzner.hcloud.hcloud, auto diff --git a/ansible/inventory/hcloud.yml b/ansible/inventory/hcloud.yml new file mode 100644 index 0000000..d414f72 --- /dev/null +++ b/ansible/inventory/hcloud.yml @@ -0,0 +1,11 @@ +--- +# Dynamic inventory: queries the Hetzner Cloud API directly (HCLOUD_TOKEN), +# grouping hosts by the `role` label set on each hcloud_server resource in +# infra/modules//main.tf (role = dev/prod/vpn). +plugin: hetzner.hcloud.hcloud +token: "{{ lookup('env', 'HCLOUD_TOKEN') }}" +connect_with: public_ipv4 +keyed_groups: + - key: hcloud_labels.role + prefix: role + separator: "_" diff --git a/ansible/playbooks/bootstrap.yml b/ansible/playbooks/bootstrap.yml new file mode 100644 index 0000000..e015879 --- /dev/null +++ b/ansible/playbooks/bootstrap.yml @@ -0,0 +1,9 @@ +--- +- name: Bootstrap newly created servers (idempotent, safe to re-run) + hosts: all + gather_facts: true + become: false + vars: + ansible_user: root + roles: + - bootstrap diff --git a/ansible/playbooks/deploy.yml b/ansible/playbooks/deploy.yml new file mode 100644 index 0000000..996c578 --- /dev/null +++ b/ansible/playbooks/deploy.yml @@ -0,0 +1,6 @@ +--- +- name: Render templates and sync service definitions to each host + hosts: all + gather_facts: false + roles: + - deploy diff --git a/ansible/playbooks/group_vars/all.yml b/ansible/playbooks/group_vars/all.yml new file mode 100644 index 0000000..a444ef2 --- /dev/null +++ b/ansible/playbooks/group_vars/all.yml @@ -0,0 +1,14 @@ +--- +deploy_user: "{{ lookup('env', 'DEPLOY_USER') | default('deploy', true) }}" +ansible_user: "{{ deploy_user }}" +ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}" + +# Repo root's services/ directory, relative to wherever a playbook lives under ansible/playbooks/. +services_root: "{{ playbook_dir }}/../../services" + +# S3-compatible bucket that services//backup-docker-compose.yml backs up +# to and restores from +backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}" +backup_s3_access_key_id: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY_ID') }}" +backup_s3_secret_access_key: "{{ lookup('env', 'BACKUP_S3_SECRET_ACCESS_KEY') }}" +backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}" diff --git a/ansible/playbooks/group_vars/role_dev.yml b/ansible/playbooks/group_vars/role_dev.yml new file mode 100644 index 0000000..073d03d --- /dev/null +++ b/ansible/playbooks/group_vars/role_dev.yml @@ -0,0 +1,2 @@ +--- +service_group: dev diff --git a/ansible/playbooks/group_vars/role_prod.yml b/ansible/playbooks/group_vars/role_prod.yml new file mode 100644 index 0000000..bf832e4 --- /dev/null +++ b/ansible/playbooks/group_vars/role_prod.yml @@ -0,0 +1,2 @@ +--- +service_group: prod diff --git a/ansible/playbooks/group_vars/role_vpn.yml b/ansible/playbooks/group_vars/role_vpn.yml new file mode 100644 index 0000000..252118a --- /dev/null +++ b/ansible/playbooks/group_vars/role_vpn.yml @@ -0,0 +1,2 @@ +--- +service_group: vpn diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml new file mode 100644 index 0000000..286bee4 --- /dev/null +++ b/ansible/playbooks/site.yml @@ -0,0 +1,4 @@ +--- +- ansible.builtin.import_playbook: bootstrap.yml +- ansible.builtin.import_playbook: deploy.yml +- ansible.builtin.import_playbook: spinup.yml diff --git a/ansible/playbooks/spindown.yml b/ansible/playbooks/spindown.yml new file mode 100644 index 0000000..c09b3d7 --- /dev/null +++ b/ansible/playbooks/spindown.yml @@ -0,0 +1,27 @@ +--- +- name: Stop every compose stack on each host and prune Docker resources + hosts: all + gather_facts: false + vars_prompt: + - name: confirm + prompt: >- + This runs spindown.sh (docker compose down, then a full + docker image/volume prune -a) on the targeted host(s). + Type 'yes' to continue + private: false + tasks: + - name: Abort unless confirmed + ansible.builtin.fail: + msg: Aborted. + when: confirm != 'yes' + + - name: Run spindown.sh + ansible.builtin.command: ./spindown.sh + args: + chdir: "/home/{{ deploy_user }}/services/{{ service_group }}" + register: _spindown + changed_when: true + + - name: Show spindown output + ansible.builtin.debug: + var: _spindown.stdout_lines diff --git a/ansible/playbooks/spinup.yml b/ansible/playbooks/spinup.yml new file mode 100644 index 0000000..bff53f9 --- /dev/null +++ b/ansible/playbooks/spinup.yml @@ -0,0 +1,15 @@ +--- +- name: Start every compose stack on each host, in the order spinup.sh defines + hosts: all + gather_facts: false + tasks: + - name: Run spinup.sh + ansible.builtin.command: ./spinup.sh + args: + chdir: "/home/{{ deploy_user }}/services/{{ service_group }}" + register: _spinup + changed_when: true + + - name: Show spinup output + ansible.builtin.debug: + var: _spinup.stdout_lines diff --git a/ansible/requirements.yml b/ansible/requirements.yml new file mode 100644 index 0000000..c36d0d9 --- /dev/null +++ b/ansible/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - name: hetzner.hcloud + version: ">=4.0.0" diff --git a/ansible/roles/bootstrap/handlers/main.yml b/ansible/roles/bootstrap/handlers/main.yml new file mode 100644 index 0000000..9bb29ca --- /dev/null +++ b/ansible/roles/bootstrap/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload sshd + ansible.builtin.systemd: + name: ssh + state: reloaded diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml new file mode 100644 index 0000000..93bca70 --- /dev/null +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -0,0 +1,89 @@ +--- +# Post-install bootstrap - Idempotent: safe to +# re-run against an already-bootstrapped host. + +- name: Update apt cache and upgrade packages + ansible.builtin.apt: + update_cache: true + upgrade: dist + cache_valid_time: 3600 + +- name: Install Docker Engine and the Compose plugin + ansible.builtin.apt: + name: + - docker.io + - docker-compose-v2 + state: present + +- name: Ensure Docker is enabled and running + ansible.builtin.systemd: + name: docker + enabled: true + state: started + +- name: Create non-root sudo user, given the same key root was provisioned with + ansible.builtin.user: + name: "{{ deploy_user }}" + shell: /bin/bash + groups: sudo,docker + append: true + create_home: true + +- name: Ensure deploy user's .ssh directory exists + ansible.builtin.file: + path: "/home/{{ deploy_user }}/.ssh" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0700" + +- name: Give deploy user the same authorized_keys as root + ansible.builtin.copy: + src: /root/.ssh/authorized_keys + dest: "/home/{{ deploy_user }}/.ssh/authorized_keys" + remote_src: true + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0600" + +- name: Grant deploy user passwordless sudo + ansible.builtin.copy: + content: "{{ deploy_user }} ALL=(ALL) NOPASSWD:ALL\n" + dest: "/etc/sudoers.d/90-{{ deploy_user }}" + validate: "visudo -cf %s" + mode: "0440" + +# Root keeps key-only login as a fallback rather than being fully locked out, +# in case deploy user setup above ever fails silently on a future image. +- name: Restrict root login to key-only + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + regexp: "^#?PermitRootLogin" + line: "PermitRootLogin prohibit-password" + notify: Reload sshd + +- name: Disable SSH password authentication + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + regexp: "^#?PasswordAuthentication" + line: "PasswordAuthentication no" + notify: Reload sshd + +- name: Install unattended-upgrades + ansible.builtin.apt: + name: unattended-upgrades + state: present + +- name: Enable automatic security upgrades + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/20auto-upgrades + content: | + APT::Periodic::Update-Package-Lists "1"; + APT::Periodic::Unattended-Upgrade "1"; + mode: "0644" + +- name: Enable and start unattended-upgrades + ansible.builtin.systemd: + name: unattended-upgrades + enabled: true + state: started diff --git a/ansible/roles/deploy/tasks/main.yml b/ansible/roles/deploy/tasks/main.yml new file mode 100644 index 0000000..87be255 --- /dev/null +++ b/ansible/roles/deploy/tasks/main.yml @@ -0,0 +1,41 @@ +--- +# Copies services// to the server and renders the files that +# need templating (e.g. .env, docker-compose.yml). +# +# Persistent data lives in named Docker volumes +# see services//backup-docker-compose.yml. .env only carries the S3 +# credentials that backup/restore need, rendered for every host. + +- name: Ensure services directory exists on the host + ansible.builtin.file: + path: "/home/{{ deploy_user }}/services/{{ service_group }}" + state: directory + mode: "0755" + +- name: Copy static service files (compose files, scripts, configs) + ansible.builtin.copy: + src: "{{ services_root }}/{{ service_group }}/" + dest: "/home/{{ deploy_user }}/services/{{ service_group }}/" + mode: preserve + +- name: Render S3 backup credentials into .env (auto-loaded by docker compose) + ansible.builtin.template: + src: env.j2 + dest: "/home/{{ deploy_user }}/services/{{ service_group }}/.env" + mode: "0600" + +- name: Render Gitea Actions runner compose file + when: service_group == 'dev' + ansible.builtin.template: + src: runners-docker-compose.yml.j2 + dest: "/home/{{ deploy_user }}/services/dev/Runners/docker-compose.yml" + mode: "0644" + +- name: Make spinup/spindown/restore scripts executable + ansible.builtin.file: + path: "/home/{{ deploy_user }}/services/{{ service_group }}/{{ item }}" + mode: "0755" + loop: + - spinup.sh + - spindown.sh + - restore.sh diff --git a/ansible/roles/deploy/templates/env.j2 b/ansible/roles/deploy/templates/env.j2 new file mode 100644 index 0000000..b44355b --- /dev/null +++ b/ansible/roles/deploy/templates/env.j2 @@ -0,0 +1,5 @@ +AWS_ACCESS_KEY_ID={{ backup_s3_access_key_id }} +AWS_SECRET_ACCESS_KEY={{ backup_s3_secret_access_key }} +AWS_S3_BUCKET_NAME={{ backup_s3_bucket }} +AWS_S3_PATH={{ service_group }} +AWS_ENDPOINT={{ backup_s3_endpoint }} diff --git a/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 new file mode 100644 index 0000000..802e82d --- /dev/null +++ b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 @@ -0,0 +1,40 @@ +# Generated by Ansible - do not hand-edit. Three Gitea Actions runners, always. +# To change their shape (image, volumes, env), edit this template and re-run +# playbooks/deploy.yml. +# +# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable +# (not baked in here) - services/dev/spinup.sh generates a fresh token and +# writes it to Runners/.env immediately before starting these containers. +# +# /data is a disposable local cache (runner identity re-registers on every +# spinup.sh run anyway), so it's a plain named volume - not part of the S3 +# backup/restore stack in services/dev/backup-docker-compose.yml. +{% set runner_count = 3 %} +services: +{% for i in range(1, runner_count + 1) %} + runner-{{ i }}: + image: gitea/act_runner:latest + container_name: gitea_runner_{{ i }} + volumes: + - ./config.yaml:/config.yaml + - gitea_runner_{{ i }}_data:/data + - /var/run/docker.sock:/var/run/docker.sock + networks: + - proxy + environment: + CONFIG_FILE: /config.yaml + GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" + GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}" + GITEA_RUNNER_NAME: "CICD#{{ i }}" + restart: unless-stopped +{% endfor %} + +networks: + proxy: + external: true + +volumes: +{% for i in range(1, runner_count + 1) %} + gitea_runner_{{ i }}_data: + name: gitea_runner_{{ i }}_data +{% endfor %} diff --git a/cicd-spindown.sh b/cicd-spindown.sh deleted file mode 100755 index 1e3d49c..0000000 --- a/cicd-spindown.sh +++ /dev/null @@ -1,16 +0,0 @@ -#Script file for spinning down all CICD relevant docker-containers - cd ./Development/Runners/ - docker compose down - cd ../.. - -cd ./TraefikRunner/ -docker compose down -cd .. - -cd ./Watchtower/ -docker compose down -cd .. - -docker rmi $(docker images -q) -docker system prune -f -a -docker volume prune -f -a \ No newline at end of file diff --git a/cicd-spinup.sh b/cicd-spinup.sh deleted file mode 100755 index 6a3b139..0000000 --- a/cicd-spinup.sh +++ /dev/null @@ -1,13 +0,0 @@ -#Script file for spinning up all docker-containers - -cd ./TraefikRunner/ -docker compose pull && docker compose up -d -cd .. - - cd ./Development/Runners/ - docker compose pull && docker compose up -d - cd ../.. - -cd ./Watchtower/ -docker compose pull && docker compose up -d -cd .. \ No newline at end of file diff --git a/control.sh b/control.sh new file mode 100755 index 0000000..a230171 --- /dev/null +++ b/control.sh @@ -0,0 +1,321 @@ +#!/usr/bin/env bash +# control.sh - interactive control panel for this repo. +# +# Steps you through the pre-setup and day-to-day operations for both the +# OpenTofu infra (infra/) and the Ansible config (ansible/), in the order the +# firewall rules require (vpn first, then tunnel in, then dev/prod). It only +# ever runs `tofu`/`ansible-playbook` for you - it's a menu wrapper, nothing +# more, so anything it does you can also do by hand from infra/ or ansible/. + +# Re-exec under bash if invoked as `sh control.sh` (or another POSIX shell) - +# this script relies on bash-only features (indirect expansion, arrays, [[ ]]) +# that fail with cryptic errors like "bad substitution" under dash/sh. +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" +fi + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INFRA_DIR="$REPO_ROOT/infra" +ANSIBLE_DIR="$REPO_ROOT/ansible" +CONTROL_ENV="$REPO_ROOT/.control.env" + +# --- pretty output -------------------------------------------------------- +if [ -t 1 ]; then + BOLD=$(tput bold); DIM=$(tput dim); RESET=$(tput sgr0) + RED=$(tput setaf 1); GREEN=$(tput setaf 2); YELLOW=$(tput setaf 3); CYAN=$(tput setaf 6) +else + BOLD=""; DIM=""; RESET=""; RED=""; GREEN=""; YELLOW=""; CYAN="" +fi + +ok() { printf "%s ok %s %s\n" "$GREEN" "$RESET" "$1"; } +warn() { printf "%s warn%s %s\n" "$YELLOW" "$RESET" "$1"; } +bad() { printf "%s miss%s %s\n" "$RED" "$RESET" "$1"; } +title() { printf "\n%s%s%s\n" "$BOLD$CYAN" "$1" "$RESET"; } + +# Pick the OpenTofu binary: prefer `tofu`, fall back to `terraform`. +TOFU="" +detect_tofu() { + if command -v tofu >/dev/null 2>&1; then TOFU="tofu" + elif command -v terraform >/dev/null 2>&1; then TOFU="terraform" + fi +} +detect_tofu + +# --- variable persistence -------------------------------------------------- +# Variables set via the "Set variables" menu (HCLOUD_TOKEN, BACKUP_S3_*, +# DEPLOY_USER, ANSIBLE_SSH_PRIVATE_KEY_FILE) live in .control.env, gitignored, +# so they survive between runs of this script. They are NOT exported into your +# normal shell - only into control.sh's own process and whatever it runs. +load_control_env() { + if [ -f "$CONTROL_ENV" ]; then + set -a + # shellcheck disable=SC1090 + source "$CONTROL_ENV" + set +a + fi +} +load_control_env + +# Upsert KEY=VALUE into .control.env and export it into this process. +save_var() { + local key="$1" val="$2" + touch "$CONTROL_ENV"; chmod 600 "$CONTROL_ENV" + local tmp; tmp="$(mktemp)" + grep -v "^${key}=" "$CONTROL_ENV" > "$tmp" 2>/dev/null || true + mv "$tmp" "$CONTROL_ENV" + printf '%s=%q\n' "$key" "$val" >> "$CONTROL_ENV" + export "${key?}=${val}" +} + +# Run a command inside a directory, echoing it first. Returns the command's exit code. +run_in() { + local dir="$1"; shift + printf "%s\$ (cd %s && %s)%s\n" "$DIM" "${dir#$REPO_ROOT/}" "$*" "$RESET" + ( cd "$dir" && "$@" ) +} + +pause() { read -rp $'\nPress Enter to return to the menu... ' _; } + +confirm() { + local reply + read -rp "$1 [y/N] " reply + [[ "$reply" == "y" || "$reply" == "Y" ]] +} + +# --- prerequisite checks -------------------------------------------------- +check_prereqs() { + title "Tools" + if [ -n "$TOFU" ]; then ok "OpenTofu ($TOFU: $("$TOFU" version 2>/dev/null | head -n1))" + else bad "Neither 'tofu' nor 'terraform' found on PATH"; fi + if command -v ansible-playbook >/dev/null 2>&1; then ok "Ansible ($(ansible --version 2>/dev/null | head -n1))" + else bad "'ansible-playbook' not found on PATH"; fi + + title "Environment variables" + [ -n "${HCLOUD_TOKEN:-}" ] && ok "HCLOUD_TOKEN set" || bad "HCLOUD_TOKEN not set (needed by both tofu and the ansible inventory)" + [ -n "${BACKUP_S3_BUCKET:-}" ] && ok "BACKUP_S3_BUCKET set" || warn "BACKUP_S3_BUCKET not set (needed for deploy/spinup - S3 backup .env)" + [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ] && ok "BACKUP_S3_ACCESS_KEY_ID set" || warn "BACKUP_S3_ACCESS_KEY_ID not set" + [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && ok "BACKUP_S3_SECRET_ACCESS_KEY set" || warn "BACKUP_S3_SECRET_ACCESS_KEY not set" + [ -n "${BACKUP_S3_ENDPOINT:-}" ] && ok "BACKUP_S3_ENDPOINT set" || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)" + ok "DEPLOY_USER: ${DEPLOY_USER:-deploy} (default 'deploy' if unset)" + + local key="${ANSIBLE_SSH_PRIVATE_KEY_FILE:-$HOME/.ssh/id_ed25519}" + [ -f "${key/#\~/$HOME}" ] && ok "SSH key present ($key)" || warn "SSH key not found at $key (set ANSIBLE_SSH_PRIVATE_KEY_FILE)" + + [ -f "$CONTROL_ENV" ] && ok ".control.env present (loaded above)" || warn "No .control.env yet - use Configuration > Set variables to persist the above across runs" + + title "Files" + [ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \ + || bad "infra/terraform.tfvars missing (Configuration > Copy terraform.tfvars.example)" + [ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \ + || warn "infra/.terraform missing - run OpenTofu > init" + if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed" + else warn "hetzner.hcloud collection not installed - run Ansible > Install collections"; fi +} + +# --- configuration actions ------------------------------------------------- +# Prompts for one variable, keeping the current value if the user hits Enter. +# $3 = true masks input and the shown current value, for tokens/secrets. +prompt_var() { + local key="$1" desc="$2" secret="${3:-false}" current input shown + current="${!key:-}" + shown="$current" + [ "$secret" = "true" ] && [ -n "$current" ] && shown="(set, hidden)" + if [ "$secret" = "true" ]; then read -rsp "$desc [$shown]: " input; echo + else read -rp "$desc [$shown]: " input; fi + [ -n "$input" ] && save_var "$key" "$input" +} + +set_variables() { + title "Set variables" + echo "Leave a prompt blank to keep its current value. Saved to .control.env" \ + "(gitignored) and reloaded automatically next time you run control.sh -" \ + "not exported to your normal shell outside of it." + echo + prompt_var HCLOUD_TOKEN "Hetzner Cloud API token" true + prompt_var DEPLOY_USER "Non-root deploy user Ansible creates (default: deploy)" false + prompt_var ANSIBLE_SSH_PRIVATE_KEY_FILE "SSH private key path (default: ~/.ssh/id_ed25519)" false + prompt_var BACKUP_S3_BUCKET "S3 backup bucket name" false + prompt_var BACKUP_S3_ACCESS_KEY_ID "S3 backup access key ID" true + prompt_var BACKUP_S3_SECRET_ACCESS_KEY "S3 backup secret access key" true + prompt_var BACKUP_S3_ENDPOINT "S3 backup endpoint (blank for real AWS S3)" false + ok "Variables saved." +} + +copy_tfvars() { + local example="$INFRA_DIR/terraform.tfvars.example" + local target="$INFRA_DIR/terraform.tfvars" + if [ -f "$target" ]; then + warn "infra/terraform.tfvars already exists." + confirm "Overwrite it with a fresh copy of the example?" || return 0 + fi + cp "$example" "$target" + ok "Copied terraform.tfvars.example -> terraform.tfvars" + + local keys quoted k + read -rp "SSH key name(s) from Hetzner Console > Security > SSH Keys, comma-separated (blank = edit manually later): " keys + if [ -n "$keys" ]; then + quoted="" + IFS=',' read -ra _key_arr <<< "$keys" + for k in "${_key_arr[@]}"; do + k="$(echo "$k" | xargs)" + quoted+="\"$k\", " + done + quoted="${quoted%, }" + sed -i "s|^ssh_key_names = .*|ssh_key_names = [${quoted}]|" "$target" + ok "Set ssh_key_names = [${quoted}]" + else + warn "ssh_key_names still has the example's placeholder values - edit infra/terraform.tfvars before running tofu." + fi +} + +# --- OpenTofu actions ----------------------------------------------------- +require_tofu() { + if [ -z "$TOFU" ]; then bad "No tofu/terraform binary found."; return 1; fi + if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then + bad "infra/terraform.tfvars is missing - copy terraform.tfvars.example first."; return 1 + fi +} +tofu_init() { require_tofu && run_in "$INFRA_DIR" "$TOFU" init; } +tofu_plan() { require_tofu && run_in "$INFRA_DIR" "$TOFU" plan; } +tofu_apply() { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply; } +tofu_output() { require_tofu && run_in "$INFRA_DIR" "$TOFU" output; } +tofu_destroy() { + require_tofu || return 1 + confirm "${RED}Destroy ALL infra managed by OpenTofu?${RESET}" && run_in "$INFRA_DIR" "$TOFU" destroy +} + +# --- Ansible actions ------------------------------------------------------ +ansible_collections() { run_in "$ANSIBLE_DIR" ansible-galaxy collection install -r requirements.yml; } +ansible_inventory() { run_in "$ANSIBLE_DIR" ansible-inventory --graph; } + +# $1 = playbook, $2 = -l limit expression +play() { run_in "$ANSIBLE_DIR" ansible-playbook "playbooks/$1" -l "$2"; } + +ansible_bootstrap_vpn() { play bootstrap.yml role_vpn; } +ansible_deploy_vpn() { play deploy.yml role_vpn; } +ansible_spinup_vpn() { play spinup.yml role_vpn; } +ansible_bootstrap_dev() { play bootstrap.yml role_dev; } +ansible_deploy_dev() { play deploy.yml role_dev; } +ansible_spinup_dev() { play spinup.yml role_dev; } +ansible_bootstrap_prod() { play bootstrap.yml role_prod; } +ansible_deploy_prod() { play deploy.yml role_prod; } +ansible_spinup_prod() { play spinup.yml role_prod; } + +ansible_spindown() { + local target + read -rp "Spindown which hosts? (e.g. role_vpn, role_dev,role_prod, all): " target + [ -n "$target" ] && play spindown.yml "$target" +} + +# --- guided walkthrough --------------------------------------------------- +guided() { + title "Guided full setup" + cat < plan -> apply (creates servers, network, firewalls, DNS) + 2. Ansible: bootstrap + deploy + spinup for role_vpn + 3. YOU: visit https://vpn.luke-else.co.uk, complete wg-easy's setup, + add a client peer, and connect with a WireGuard client + 4. Ansible: bootstrap + deploy + spinup for role_dev, then role_prod +EOF + confirm "Start the guided walkthrough?" || return 0 + + echo; title "0/4 Configuration" + if confirm "Set/update variables now (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)?"; then + set_variables + fi + if [ ! -f "$INFRA_DIR/terraform.tfvars" ] && confirm "infra/terraform.tfvars is missing - create it from the example now?"; then + copy_tfvars + fi + + echo; title "1/4 OpenTofu" + if confirm "Run $TOFU init/plan/apply now?"; then + tofu_init && tofu_plan && tofu_apply || { bad "OpenTofu step failed - stopping."; return 1; } + fi + + echo; title "2/4 Ansible - vpn" + if confirm "Bootstrap + deploy + spinup role_vpn now?"; then + ansible_bootstrap_vpn && ansible_deploy_vpn && ansible_spinup_vpn || { bad "vpn step failed - stopping."; return 1; } + fi + + echo; title "3/4 Connect the VPN" + warn "Visit https://vpn.luke-else.co.uk, complete wg-easy's setup wizard, add a client peer, and connect with a WireGuard client now." + confirm "Are you connected to the VPN?" || { warn "Stopping - reconnect and re-run this step."; return 0; } + + echo; title "4/4 Ansible - dev + prod" + if confirm "Bootstrap + deploy + spinup role_dev now?"; then + ansible_bootstrap_dev && ansible_deploy_dev && ansible_spinup_dev || { bad "dev step failed."; return 1; } + fi + if confirm "Bootstrap + deploy + spinup role_prod now?"; then + ansible_bootstrap_prod && ansible_deploy_prod && ansible_spinup_prod || { bad "prod step failed."; return 1; } + fi + ok "Guided setup complete." +} + +# --- menu ----------------------------------------------------------------- +menu() { + clear + printf "%s==== server control panel ====%s\n" "$BOLD$CYAN" "$RESET" + printf "%srepo: %s tofu: %s%s\n" "$DIM" "$REPO_ROOT" "${TOFU:-none}" "$RESET" + + title "Setup" + echo " 1) Check prerequisites" + echo " 2) Guided full setup (vpn -> connect -> dev/prod)" + + title "Configuration" + echo " 3) Set variables (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)" + echo " 4) Copy terraform.tfvars.example -> terraform.tfvars" + + title "OpenTofu (infra/)" + echo " 5) init 6) plan 7) apply" + echo " 8) output 9) destroy" + + title "Ansible (ansible/)" + echo " 10) Install collections 11) Show inventory" + echo " 12) Bootstrap vpn 13) Deploy vpn 14) Spinup vpn" + echo " 15) Bootstrap dev 16) Deploy dev 17) Spinup dev" + echo " 18) Bootstrap prod 19) Deploy prod 20) Spinup prod" + echo " 21) Spindown (choose target)" + + echo + echo " 0) Quit" + echo + read -rp "Select an option: " choice +} + +while true; do + menu + case "$choice" in + 1) check_prereqs; pause ;; + 2) guided; pause ;; + 3) set_variables; pause ;; + 4) copy_tfvars; pause ;; + 5) tofu_init; pause ;; + 6) tofu_plan; pause ;; + 7) tofu_apply; pause ;; + 8) tofu_output; pause ;; + 9) tofu_destroy; pause ;; + 10) ansible_collections; pause ;; + 11) ansible_inventory; pause ;; + 12) ansible_bootstrap_vpn; pause ;; + 13) ansible_deploy_vpn; pause ;; + 14) ansible_spinup_vpn; pause ;; + 15) ansible_bootstrap_dev; pause ;; + 16) ansible_deploy_dev; pause ;; + 17) ansible_spinup_dev; pause ;; + 18) ansible_bootstrap_prod; pause ;; + 19) ansible_deploy_prod; pause ;; + 20) ansible_spinup_prod; pause ;; + 21) ansible_spindown; pause ;; + 0) echo "Bye."; exit 0 ;; + "") ;; + *) warn "Unknown option: $choice"; pause ;; + esac +done diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 0000000..df97a6f --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,67 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/hcloud" { + version = "1.66.0" + hashes = [ + "h1:+R3t/5wqCb2ics5bNZ4JgtfaAa08Z6uRzXzNtwovVEY=", + "h1:9c1ECk3N9xhdQKz4NVCfYEFxebJVVeQZS3E6wrR/Hx8=", + "h1:EtYB/KjPN2cIJa6hpEaVc8q/3yf6e9IT6/uAybUGqps=", + "h1:GUupu6Fg5zFY9i4/3esiy8uHZnUrt+u2B6NaE4+8YqQ=", + "h1:JHehRJySEFRGYK+D6pFS0MTTDDRlGSQdSaKdEo0AemA=", + "h1:XF+SwyO1ttQJ0WXJ5O628O7cGk9U37sGn0o3LwWUnDQ=", + "h1:czuvCXAd8CcszXh3k3E0N6tFfQkYSNbKXo99LG+yOIc=", + "h1:dPIwO6zTxYs6bHn5yf/w/1AExaHPppCLyTrWrdzH4RE=", + "h1:gqrhnYuIBzpUvgWHITNRUMLaDcKACGlrNfrbtbtO7KY=", + "h1:iVAGP8gRbZK0kJF7SiYJRt61wz0D5AF9q+WMsrAiBI0=", + "h1:kLi29SbGCU/Z/Ch0zuNdYFSKP3mHp8zfcOpZsIN/KIQ=", + "h1:mq2+1Q/5gWJYI8XOXQCeteUg0AP7VezPOGOjkkHRQso=", + "h1:rnWTWAOlJhHtD3kkU4Qfw0bg9ko/dlH822inxYLBmfQ=", + "zh:1286cee6fb63dbcb18f53077bbb5e5d132a4e4d9f006af4e8d8edfc08d6bcdc8", + "zh:204460dacc044bda019a4a18b398e094289500c36913c7c9457f432adf31b8b2", + "zh:214175d50773481cbeaf9c9004e4121a3a1c9686c79424ebdc8ff189dd057d3e", + "zh:22b17bceff61cc13ad04a399ba87521356a3a134d4687273727473ae9eccf5f1", + "zh:368867dac5525c411de7e38f2e27de0a71854d1750867322ff2b9321128c88fb", + "zh:5289b75f8370bdbc4c6051d55cf33d0b1bd25dc6d71bfbd39b360249a37f1501", + "zh:81cb676aa50c5777df8fc80d4e69c9012330ae751f5e6f12bf6074bfd2e7c496", + "zh:ab08aead10643b21aa6b51af562b50492e12b9dd0ab7dca27a05aa63209b7d66", + "zh:af25c210d0570cf61ef767b2545bf9f3fb909178135f0e5e14bec0c1c9d07a63", + "zh:bcad66f4830c97118fa793723e53f8a4d27ddd34ea969ff259408842c2238331", + "zh:ce3ed323d75ae905d975925fa98c7054a7514c81276a485fc37da8232b53e39f", + "zh:d481bc0ef0c87ab1969c17777f526b2f59f823432d676145134c41a6d29bd98e", + "zh:ea7ef88df2c3ca154d86238920636d52a3c9066c7467543d3fa45f1e52ec2f7b", + ] +} + +provider "registry.opentofu.org/hetznercloud/hcloud" { + version = "1.66.0" + constraints = "~> 1.54" + hashes = [ + "h1:+R3t/5wqCb2ics5bNZ4JgtfaAa08Z6uRzXzNtwovVEY=", + "h1:9c1ECk3N9xhdQKz4NVCfYEFxebJVVeQZS3E6wrR/Hx8=", + "h1:EtYB/KjPN2cIJa6hpEaVc8q/3yf6e9IT6/uAybUGqps=", + "h1:GUupu6Fg5zFY9i4/3esiy8uHZnUrt+u2B6NaE4+8YqQ=", + "h1:JHehRJySEFRGYK+D6pFS0MTTDDRlGSQdSaKdEo0AemA=", + "h1:XF+SwyO1ttQJ0WXJ5O628O7cGk9U37sGn0o3LwWUnDQ=", + "h1:czuvCXAd8CcszXh3k3E0N6tFfQkYSNbKXo99LG+yOIc=", + "h1:dPIwO6zTxYs6bHn5yf/w/1AExaHPppCLyTrWrdzH4RE=", + "h1:gqrhnYuIBzpUvgWHITNRUMLaDcKACGlrNfrbtbtO7KY=", + "h1:iVAGP8gRbZK0kJF7SiYJRt61wz0D5AF9q+WMsrAiBI0=", + "h1:kLi29SbGCU/Z/Ch0zuNdYFSKP3mHp8zfcOpZsIN/KIQ=", + "h1:mq2+1Q/5gWJYI8XOXQCeteUg0AP7VezPOGOjkkHRQso=", + "h1:rnWTWAOlJhHtD3kkU4Qfw0bg9ko/dlH822inxYLBmfQ=", + "zh:1286cee6fb63dbcb18f53077bbb5e5d132a4e4d9f006af4e8d8edfc08d6bcdc8", + "zh:204460dacc044bda019a4a18b398e094289500c36913c7c9457f432adf31b8b2", + "zh:214175d50773481cbeaf9c9004e4121a3a1c9686c79424ebdc8ff189dd057d3e", + "zh:22b17bceff61cc13ad04a399ba87521356a3a134d4687273727473ae9eccf5f1", + "zh:368867dac5525c411de7e38f2e27de0a71854d1750867322ff2b9321128c88fb", + "zh:5289b75f8370bdbc4c6051d55cf33d0b1bd25dc6d71bfbd39b360249a37f1501", + "zh:81cb676aa50c5777df8fc80d4e69c9012330ae751f5e6f12bf6074bfd2e7c496", + "zh:ab08aead10643b21aa6b51af562b50492e12b9dd0ab7dca27a05aa63209b7d66", + "zh:af25c210d0570cf61ef767b2545bf9f3fb909178135f0e5e14bec0c1c9d07a63", + "zh:bcad66f4830c97118fa793723e53f8a4d27ddd34ea969ff259408842c2238331", + "zh:ce3ed323d75ae905d975925fa98c7054a7514c81276a485fc37da8232b53e39f", + "zh:d481bc0ef0c87ab1969c17777f526b2f59f823432d676145134c41a6d29bd98e", + "zh:ea7ef88df2c3ca154d86238920636d52a3c9066c7467543d3fa45f1e52ec2f7b", + ] +} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 0000000..1b268c4 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,99 @@ +# Root module: wires together the shared network and the three host modules +# (one per server in docs/architecture.md). Each host module owns its own +# firewall - see modules//main.tf. Persistent data lives in named Docker +# volumes backed up to S3 (see services//backup-docker-compose.yml), not +# on Hetzner volumes. + +locals { + # SSH to dev/prod is only permitted from the vpn server's public IP: admins must + # tunnel through the VPN first, whose egress traffic is then sourced from this IP. + # vpn itself can't require tunneling through itself, so it keeps + # var.allowed_ssh_source_ips for direct SSH access. + vpn_ssh_source_ips = ["${module.vpn.ipv4}/32"] + + # IDs for every named key in var.ssh_key_names - installed on every server. + ssh_key_ids = [for k in data.hcloud_ssh_key.main : k.id] + + # Every A record currently referenced by a Traefik Host() rule anywhere under + # services/ - kept in sync with the actual compose files, not aspirational. + # "dev.luke-else.co.uk" is a *website* served from prod (a dev-tagged build of + # luke-else.co.uk), not the dev server - don't confuse the two. + dns_records = [ + { zone = "luke-else.co.uk", name = "@", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "dev", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "metarius", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "traefik", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "status", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "bitwarden", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "rd", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "git", value = module.dev.ipv4 }, + { zone = "luke-else.co.uk", name = "traefik.cicd", value = module.dev.ipv4 }, + { zone = "luke-else.co.uk", name = "vpn", value = module.vpn.ipv4 }, + { zone = "luke-else.co.uk", name = "traefik.vpn", value = module.vpn.ipv4 }, + ] + + # Microsoft 365 mail records for luke-else.co.uk + mail_records = [ + { zone = "luke-else.co.uk", name = "@", type = "MX", value = "0 lukeelse-co-uk01b.mail.protection.outlook.com." }, + { zone = "luke-else.co.uk", name = "@", type = "TXT", value = "\"v=spf1 include:spf.protection.outlook.com -all\"" }, + { zone = "luke-else.co.uk", name = "@", type = "TXT", value = "\"google-site-verification=aKLk5BATowKUwNbFcU5zyfu-AIyCW9W9PBStePRjH3U\"" }, + { zone = "luke-else.co.uk", name = "autodiscover", type = "CNAME", value = "autodiscover.outlook.com." }, + ] +} + +module "network" { + source = "./modules/network" + + ip_range = var.network_ip_range + network_zone = var.network_zone + subnet_ip_range = var.subnet_ip_range +} + +module "dev" { + source = "./modules/dev" + + server_type = var.dev_server_type + image = var.server_image + location = var.location + ssh_key_ids = local.ssh_key_ids + network_id = module.network.id + private_ip = var.dev_private_ip + allowed_ssh_source_ips = local.vpn_ssh_source_ips + network_ip_range = var.network_ip_range + + # module.network.id alone doesn't guarantee the subnet exists yet, and a server + # can't join a network before it has a subnet. + depends_on = [module.network] +} + +module "prod" { + source = "./modules/prod" + + server_type = var.prod_server_type + image = var.server_image + location = var.location + ssh_key_ids = local.ssh_key_ids + network_id = module.network.id + private_ip = var.prod_private_ip + allowed_ssh_source_ips = local.vpn_ssh_source_ips + network_ip_range = var.network_ip_range + + depends_on = [module.network] +} + +module "vpn" { + source = "./modules/vpn" + + server_type = var.vpn_server_type + image = var.server_image + location = var.location + ssh_key_ids = local.ssh_key_ids + allowed_ssh_source_ips = var.allowed_ssh_source_ips +} + +module "dns" { + source = "./modules/dns" + + zone_names = var.dns_zones + records = concat(local.dns_records, local.mail_records) +} diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf new file mode 100644 index 0000000..9f9517f --- /dev/null +++ b/infra/modules/dev/main.tf @@ -0,0 +1,64 @@ +# dev: Gitea + Runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) + +# Port sources: services/dev/*docker-compose.yml (published ports) +resource "hcloud_firewall" "this" { + name = "dev-firewall" + + rule { # server ssh - wired to the vpn server's public IP only + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.allowed_ssh_source_ips + } + + rule { # gitea ssh + direction = "in" + protocol = "tcp" + port = "222" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # Traefik http/https + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # traffic from prod over the private network + direction = "in" + protocol = "tcp" + port = "1-65535" + source_ips = [var.network_ip_range] + } + + rule { + direction = "in" + protocol = "udp" + port = "1-65535" + source_ips = [var.network_ip_range] + } +} + +resource "hcloud_server" "this" { + name = "dev" + server_type = var.server_type + image = var.image + location = var.location + ssh_keys = var.ssh_key_ids + firewall_ids = [hcloud_firewall.this.id] + labels = { role = "dev" } + + network { + network_id = var.network_id + ip = var.private_ip + } +} + diff --git a/infra/modules/dev/outputs.tf b/infra/modules/dev/outputs.tf new file mode 100644 index 0000000..4897389 --- /dev/null +++ b/infra/modules/dev/outputs.tf @@ -0,0 +1,7 @@ +output "ipv4" { + value = hcloud_server.this.ipv4_address +} + +output "private_ipv4" { + value = var.private_ip +} diff --git a/infra/modules/dev/variables.tf b/infra/modules/dev/variables.tf new file mode 100644 index 0000000..a9d7899 --- /dev/null +++ b/infra/modules/dev/variables.tf @@ -0,0 +1,39 @@ +variable "server_type" { + description = "Server type for dev (Gitea + Runner)." + type = string +} + +variable "image" { + description = "OS image used for the server." + type = string +} + +variable "location" { + description = "Hetzner Cloud datacenter location." + type = string +} + +variable "ssh_key_ids" { + description = "IDs of the Hetzner Cloud SSH keys to install on the server." + type = list(string) +} + +variable "network_id" { + description = "ID of the private network to attach dev to." + type = string +} + +variable "private_ip" { + description = "Private network IP for dev." + type = string +} + +variable "allowed_ssh_source_ips" { + description = "CIDRs allowed to reach port 22 on dev. Set by the root module to the vpn server's public IP." + type = list(string) +} + +variable "network_ip_range" { + description = "CIDR of the private network, allowed through the firewall for traffic from prod." + type = string +} diff --git a/infra/modules/dns/main.tf b/infra/modules/dns/main.tf new file mode 100644 index 0000000..3b4805e --- /dev/null +++ b/infra/modules/dns/main.tf @@ -0,0 +1,37 @@ +# One Hetzner DNS zone per domain in var.zone_names. Hetzner becomes +# authoritative for these once the domain's registrar NS records are pointed at +# the nameservers in the `nameservers` output - that step happens at whichever +# registrar the domain was bought through, and can't be done from here. +resource "hcloud_zone" "this" { + for_each = toset(var.zone_names) + + name = each.value + mode = "primary" +} + +locals { + # Group flat var.records entries by zone/name/type into one set of values + # each, since a Hetzner RRSet is keyed on (zone, name, type) and can hold + # several values (e.g. an SPF and a google-site-verification TXT both live + # under the same "@" TXT RRSet). + rrsets = { + for key, items in { for r in var.records : "${r.zone}/${r.name}/${r.type}" => r... } : key => { + zone = items[0].zone + name = items[0].name + type = items[0].type + ttl = items[0].ttl + values = [for i in items : i.value] + } + } +} + +resource "hcloud_zone_rrset" "this" { + for_each = local.rrsets + + zone = hcloud_zone.this[each.value.zone].id + name = each.value.name + type = each.value.type + ttl = each.value.ttl + + records = [for v in each.value.values : { value = v }] +} diff --git a/infra/modules/dns/moved.tf b/infra/modules/dns/moved.tf new file mode 100644 index 0000000..8bfb840 --- /dev/null +++ b/infra/modules/dns/moved.tf @@ -0,0 +1,60 @@ +# Generalizing hcloud_zone_rrset.a (for_each keyed "zone/name", A records +# only) into hcloud_zone_rrset.this (keyed "zone/name/type", any record type) +# changed every existing instance's resource address. Without these, `tofu +# apply` would destroy and recreate every A record already in state instead of +# just adding the new mail records - these moved blocks make it a no-op rename +# for anything that already existed. Safe to delete once applied and merged. +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/@"] + to = hcloud_zone_rrset.this["luke-else.co.uk/@/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/dev"] + to = hcloud_zone_rrset.this["luke-else.co.uk/dev/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/metarius"] + to = hcloud_zone_rrset.this["luke-else.co.uk/metarius/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/traefik"] + to = hcloud_zone_rrset.this["luke-else.co.uk/traefik/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/status"] + to = hcloud_zone_rrset.this["luke-else.co.uk/status/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/bitwarden"] + to = hcloud_zone_rrset.this["luke-else.co.uk/bitwarden/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/rd"] + to = hcloud_zone_rrset.this["luke-else.co.uk/rd/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/git"] + to = hcloud_zone_rrset.this["luke-else.co.uk/git/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/traefik.cicd"] + to = hcloud_zone_rrset.this["luke-else.co.uk/traefik.cicd/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/vpn"] + to = hcloud_zone_rrset.this["luke-else.co.uk/vpn/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/traefik.vpn"] + to = hcloud_zone_rrset.this["luke-else.co.uk/traefik.vpn/A"] +} diff --git a/infra/modules/dns/outputs.tf b/infra/modules/dns/outputs.tf new file mode 100644 index 0000000..2ef778d --- /dev/null +++ b/infra/modules/dns/outputs.tf @@ -0,0 +1,9 @@ +output "zone_ids" { + description = "Map of domain name to Hetzner zone ID." + value = { for name, z in hcloud_zone.this : name => z.id } +} + +output "nameservers" { + description = "Map of domain name to the nameservers Hetzner assigned it - point your registrar's NS records at these to delegate DNS." + value = { for name, z in hcloud_zone.this : name => z.authoritative_nameservers.assigned } +} diff --git a/infra/modules/dns/variables.tf b/infra/modules/dns/variables.tf new file mode 100644 index 0000000..7de9e57 --- /dev/null +++ b/infra/modules/dns/variables.tf @@ -0,0 +1,24 @@ +variable "zone_names" { + description = "Domains to create as Hetzner DNS zones." + type = list(string) +} + +variable "records" { + description = <<-EOT + Records to create across the zones in var.zone_names. Use name = "@" for + the zone apex. Entries sharing the same zone/name/type become a single + Hetzner RRSet with multiple values (e.g. two TXT strings under the same + name) - Hetzner has no concept of separate same-type records otherwise. + For MX, embed the priority in value using standard zonefile syntax (e.g. + "10 mail.example.com."); Hetzner's API has no separate priority field. + For TXT, wrap value in escaped double quotes (e.g. "\"v=spf1 ...\""). + EOT + type = list(object({ + zone = string + name = string + value = string + type = optional(string, "A") + ttl = optional(number, 300) + })) + default = [] +} diff --git a/infra/modules/network/main.tf b/infra/modules/network/main.tf new file mode 100644 index 0000000..b094689 --- /dev/null +++ b/infra/modules/network/main.tf @@ -0,0 +1,13 @@ +# Private network shared by dev and prod (see readme.md "network" group). +# vpn is intentionally not attached to this network +resource "hcloud_network" "main" { + name = "server-network" + ip_range = var.ip_range +} + +resource "hcloud_network_subnet" "main" { + network_id = hcloud_network.main.id + type = "cloud" + network_zone = var.network_zone + ip_range = var.subnet_ip_range +} diff --git a/infra/modules/network/outputs.tf b/infra/modules/network/outputs.tf new file mode 100644 index 0000000..622ad92 --- /dev/null +++ b/infra/modules/network/outputs.tf @@ -0,0 +1,9 @@ +output "id" { + value = hcloud_network.main.id +} + +# Exposed so dependents can depend_on the subnet existing, not just the network - +# attaching a server to a network fails if the network has no subnet yet. +output "subnet_id" { + value = hcloud_network_subnet.main.id +} diff --git a/infra/modules/network/variables.tf b/infra/modules/network/variables.tf new file mode 100644 index 0000000..ba05090 --- /dev/null +++ b/infra/modules/network/variables.tf @@ -0,0 +1,14 @@ +variable "ip_range" { + description = "IP range of the private network shared by dev and prod." + type = string +} + +variable "network_zone" { + description = "Hetzner Cloud network zone matching var.location." + type = string +} + +variable "subnet_ip_range" { + description = "IP range of the network subnet shared by dev and prod." + type = string +} diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf new file mode 100644 index 0000000..931c902 --- /dev/null +++ b/infra/modules/prod/main.tf @@ -0,0 +1,90 @@ +# Port sources: services/prod/*docker-compose.yml (published ports) +resource "hcloud_firewall" "this" { + name = "prod-firewall" + + rule { # server ssh - wired to the vpn server's public IP only + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.allowed_ssh_source_ips + } + + rule { # Traefik http/https + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # rustdesk hbbs + direction = "in" + protocol = "tcp" + port = "21115" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "21116" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "udp" + port = "21116" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # rustdesk hbbr + direction = "in" + protocol = "tcp" + port = "21117" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "21119" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # traffic from over the private network + direction = "in" + protocol = "tcp" + port = "1-65535" + source_ips = [var.network_ip_range] + } + + rule { + direction = "in" + protocol = "udp" + port = "1-65535" + source_ips = [var.network_ip_range] + } +} + +resource "hcloud_server" "this" { + name = "prod" + server_type = var.server_type + image = var.image + location = var.location + ssh_keys = var.ssh_key_ids + firewall_ids = [hcloud_firewall.this.id] + labels = { role = "prod" } + + network { + network_id = var.network_id + ip = var.private_ip + } +} + diff --git a/infra/modules/prod/outputs.tf b/infra/modules/prod/outputs.tf new file mode 100644 index 0000000..4897389 --- /dev/null +++ b/infra/modules/prod/outputs.tf @@ -0,0 +1,7 @@ +output "ipv4" { + value = hcloud_server.this.ipv4_address +} + +output "private_ipv4" { + value = var.private_ip +} diff --git a/infra/modules/prod/variables.tf b/infra/modules/prod/variables.tf new file mode 100644 index 0000000..860bb14 --- /dev/null +++ b/infra/modules/prod/variables.tf @@ -0,0 +1,39 @@ +variable "server_type" { + description = "Server type for prod (Traefik, Websites, Bitwarden, RustDesk, status page)." + type = string +} + +variable "image" { + description = "OS image used for the server." + type = string +} + +variable "location" { + description = "Hetzner Cloud datacenter location." + type = string +} + +variable "ssh_key_ids" { + description = "IDs of the Hetzner Cloud SSH keys to install on the server." + type = list(string) +} + +variable "network_id" { + description = "ID of the private network to attach prod to." + type = string +} + +variable "private_ip" { + description = "Private network IP for prod." + type = string +} + +variable "allowed_ssh_source_ips" { + description = "CIDRs allowed to reach port 22 on prod. Set by the root module to the vpn server's public IP." + type = list(string) +} + +variable "network_ip_range" { + description = "CIDR of the private network, allowed through the firewall for traffic from dev." + type = string +} diff --git a/infra/modules/vpn/main.tf b/infra/modules/vpn/main.tf new file mode 100644 index 0000000..d7e5ef3 --- /dev/null +++ b/infra/modules/vpn/main.tf @@ -0,0 +1,42 @@ +# vpn: WireGuard (wg-easy) + Traefik. Not attached to the private network. +resource "hcloud_firewall" "this" { + name = "vpn-firewall" + + rule { # server ssh + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.allowed_ssh_source_ips + } + + rule { # Traefik http/https + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # WireGuard tunnel - always direct to this server's public IP, never via a load balancer + direction = "in" + protocol = "udp" + port = "51820" + source_ips = ["0.0.0.0/0", "::/0"] + } +} + +resource "hcloud_server" "this" { + name = "vpn" + server_type = var.server_type + image = var.image + location = var.location + ssh_keys = var.ssh_key_ids + firewall_ids = [hcloud_firewall.this.id] + labels = { role = "vpn" } +} diff --git a/infra/modules/vpn/outputs.tf b/infra/modules/vpn/outputs.tf new file mode 100644 index 0000000..8952de2 --- /dev/null +++ b/infra/modules/vpn/outputs.tf @@ -0,0 +1,3 @@ +output "ipv4" { + value = hcloud_server.this.ipv4_address +} diff --git a/infra/modules/vpn/variables.tf b/infra/modules/vpn/variables.tf new file mode 100644 index 0000000..f2098f0 --- /dev/null +++ b/infra/modules/vpn/variables.tf @@ -0,0 +1,24 @@ +variable "server_type" { + description = "Server type for vpn (WireGuard/wg-easy + Traefik)." + type = string +} + +variable "image" { + description = "OS image used for the server." + type = string +} + +variable "location" { + description = "Hetzner Cloud datacenter location." + type = string +} + +variable "ssh_key_ids" { + description = "IDs of the Hetzner Cloud SSH keys to install on the server." + type = list(string) +} + +variable "allowed_ssh_source_ips" { + description = "CIDRs allowed to reach port 22 on vpn." + type = list(string) +} diff --git a/infra/outputs.tf b/infra/outputs.tf new file mode 100644 index 0000000..f8e427c --- /dev/null +++ b/infra/outputs.tf @@ -0,0 +1,24 @@ +output "dev_ipv4" { + value = module.dev.ipv4 +} + +output "dev_private_ipv4" { + value = module.dev.private_ipv4 +} + +output "prod_ipv4" { + value = module.prod.ipv4 +} + +output "prod_private_ipv4" { + value = module.prod.private_ipv4 +} + +output "vpn_ipv4" { + value = module.vpn.ipv4 +} + +output "dns_nameservers" { + description = "Nameservers Hetzner assigned each zone - point each domain's registrar NS records at these to delegate DNS to Hetzner." + value = module.dns.nameservers +} diff --git a/infra/ssh.tf b/infra/ssh.tf new file mode 100644 index 0000000..cffda71 --- /dev/null +++ b/infra/ssh.tf @@ -0,0 +1,4 @@ +data "hcloud_ssh_key" "main" { + for_each = toset(var.ssh_key_names) + name = each.value +} diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example new file mode 100644 index 0000000..e59d991 --- /dev/null +++ b/infra/terraform.tfvars.example @@ -0,0 +1,18 @@ +# Copy to terraform.tfvars and fill in - terraform.tfvars itself is gitignored, +# never commit real values there. +# +# The Hetzner API token is NOT set here: export it as HCLOUD_TOKEN in your shell +# before running tofu plan/apply. The same token and SSH key are used again by +# Ansible afterwards - see ../ansible/README.md. + +# Names of SSH keys already uploaded to your Hetzner Cloud project +# (Console > Security > SSH Keys). All of them are installed on every server. Required. +ssh_key_names = ["luke-else@PC", "luke-else@laptop"] + +# Optional overrides - defaults live in variables.tf +# location = "nbg1" +# dev_server_type = "cx23" +# prod_server_type = "cx23" +# vpn_server_type = "cx23" +# allowed_ssh_source_ips = ["0.0.0.0/0"] +# dns_zones = ["luke-else.co.uk"] \ No newline at end of file diff --git a/infra/variables.tf b/infra/variables.tf new file mode 100644 index 0000000..c8c811f --- /dev/null +++ b/infra/variables.tf @@ -0,0 +1,76 @@ +variable "location" { + description = "Hetzner Cloud datacenter location for all servers." + type = string + default = "nbg1" +} + +variable "network_zone" { + description = "Hetzner Cloud network zone matching var.location." + type = string + default = "eu-central" +} + +variable "server_image" { + description = "OS image used for all servers." + type = string + default = "ubuntu-26.04" +} + +variable "dev_server_type" { + description = "Server type for dev" + type = string + default = "cx23" +} + +variable "prod_server_type" { + description = "Server type for prod" + type = string + default = "cx23" +} + +variable "vpn_server_type" { + description = "Server type for vpn" + type = string + default = "cx23" +} + +variable "network_ip_range" { + description = "IP range of the private network shared by dev and prod." + type = string + default = "10.0.0.0/16" +} + +variable "subnet_ip_range" { + description = "IP range of the network subnet shared by dev and prod." + type = string + default = "10.0.1.0/24" +} + +variable "dev_private_ip" { + description = "Private network IP for dev." + type = string + default = "10.0.1.10" +} + +variable "prod_private_ip" { + description = "Private network IP for prod." + type = string + default = "10.0.1.11" +} + +variable "ssh_key_names" { + description = "Names of SSH keys already uploaded to your Hetzner Cloud project (Console > Security > SSH Keys). All are installed on every server." + type = list(string) +} + +variable "allowed_ssh_source_ips" { + description = "CIDRs allowed to reach port 22 on the vpn server. Narrow this to your own IP(s) once known. dev and prod don't use this - their SSH is restricted to the vpn server's own public IP instead (see infra/main.tf)." + type = list(string) + default = ["0.0.0.0/0", "::/0"] +} + +variable "dns_zones" { + description = "Domains to manage as Hetzner DNS zones. Point each domain's registrar NS records at tofu output dns_nameservers for Hetzner to actually become authoritative." + type = list(string) + default = ["luke-else.co.uk"] +} diff --git a/infra/versions.tf b/infra/versions.tf new file mode 100644 index 0000000..f817710 --- /dev/null +++ b/infra/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.6.0" + + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + version = "~> 1.54" # hcloud_zone / hcloud_zone_rrset (DNS) require >= 1.54.0 + } + } +} + +# Reads the token from the HCLOUD_TOKEN environment variable. +provider "hcloud" {} diff --git a/main-spindown.sh b/main-spindown.sh deleted file mode 100755 index 1012835..0000000 --- a/main-spindown.sh +++ /dev/null @@ -1,42 +0,0 @@ -#Script file for spinning down all docker-containers - -cd ./Websites/ -docker compose down -cd .. - -cd ./Tracking/ -docker compose down -cd .. - - - cd ./Development/Gitea/ - docker compose down - cd ../.. - -cd ./Database/ -docker compose down -cd .. - -cd ./Bitwarden/ -docker compose down -cd .. - -cd ./Rustdesk/ -docker compose down -cd .. - -cd ./Misc/ -docker compose down -cd .. - -cd ./Watchtower/ -docker compose down -cd .. - -cd ./Traefik/ -docker compose down -cd .. - -docker rmi $(docker images -q) -docker system prune -f -a -docker volume prune -f -a \ No newline at end of file diff --git a/main-spinup.sh b/main-spinup.sh deleted file mode 100755 index d8f8a03..0000000 --- a/main-spinup.sh +++ /dev/null @@ -1,39 +0,0 @@ -#Script file for spinning up all docker-containers - -cd ./Traefik/ -docker compose pull && docker compose up -d -cd .. - - cd ./Development/Gitea/ - docker compose pull && docker compose up -d - cd ../.. - -sleep 20 # Allow Gitea + registry to start up before starting the rest of the services - -cd ./Watchtower/ -docker compose pull && docker compose up -d -cd .. - -cd ./Tracking/ -docker compose pull && docker compose up -d -cd .. - -cd ./Websites/ -docker compose pull && docker compose up -d -cd .. - -cd ./Database/ -docker compose pull && docker compose up -d -cd .. - -cd ./Bitwarden/ -docker compose pull && docker compose up -d -cd .. - -cd ./Rustdesk/ -docker compose pull && docker compose up -d -cd .. - -cd ./Misc/ -docker compose pull && docker compose up -d -cd .. \ No newline at end of file diff --git a/readme.md b/readme.md index 00ffcbc..a5ae1cc 100644 --- a/readme.md +++ b/readme.md @@ -1,48 +1,262 @@ -# Server Repository +# Server -This repository contains various scripts and configurations for managing Docker containers and services. +Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), bootstrapped and deployed with [Ansible](https://www.ansible.com/), running Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).

-## Scripts +## Contents -- `cicd-spindown.sh`: Spins down all CICD relevant Docker containers. -- `cicd-spinup.sh`: Spins up all CICD relevant Docker containers. -- `main-spindown.sh`: Spins down all Docker containers. -- `main-spinup.sh`: Spins up all Docker containers. +- [Architecture](#architecture) +- [Repository layout](#repository-layout) +- [Design choices](#design-choices) +- [Prerequisites](#prerequisites) +- [Quickstart](#quickstart) +- [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) +- [Bootstrapping and deploying](#bootstrapping-and-deploying-ansible) +- [Running the services](#running-the-services-services) +- [Service inventory](#service-inventory) +- [Caveats](#caveats) +- [Security notes](#security-notes) -## Services +## Architecture -### Bitwarden +Three servers, one shared private network: -Configuration for Bitwarden is located in [Bitwarden/docker-compose.yml](Bitwarden/docker-compose.yml). +```mermaid +architecture-beta + group cloud(cloud)[Hetzner] -### Database + group network(cloud)[network] in cloud -Configuration for MongoDB is located in [Database/docker-compose.yml](Database/docker-compose.yml). + service dev(mdi:server)[dev] in network -### Misc + service prod(mdi:server)[prod] in network + service prodfirewall(mdi:firewall)[firewall] in cloud -Configuration for miscellaneous services is located in [Misc/docker-compose.yml](Misc/docker-compose.yml). + service vpn(mdi:server)[vpn] in cloud + service vpnfirewall(mdi:firewall)[firewall] in cloud -### Tracking + service gateway(mdi:web)[gateway] in cloud -Configuration for tracking services is located in [Tracking/docker-compose.yml](Tracking/docker-compose.yml). + service backups(mdi:bucket)[S3 Backups] -### Traefik + dev:L -- R:prod -Configuration for Traefik is located in [Traefik/docker-compose.yml](Traefik/docker-compose.yml). + prod:B -- T:prodfirewall + vpn:B -- T:vpnfirewall -### TraefikRunner + prodfirewall: L -- R: gateway + vpnfirewall: B -- T: gateway -Configuration for TraefikRunner is located in [TraefikRunner/docker-compose.yml](TraefikRunner/docker-compose.yml). + dev:T -- B:backups + prod:T -- B:backups + vpn:R -- L:backups +``` -### Websites +`gateway` is the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from (see [Design choices](#design-choices)) — also not a Hetzner resource. -Configuration for websites is located in [Websites/docker-compose.yml](Websites/docker-compose.yml). +| Server | Purpose | Network | +|---|---|---| +| `dev` | Gitea, CI runners, dev-facing Traefik | Private network only | +| `prod` | Public websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | +| `vpn` | WireGuard (wg-easy) + its own Traefik | **Not** on the private network — isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | -## ToDo +`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach `prod` without exposing that traffic publicly. Each server has its own Hetzner Cloud Firewall (`infra/modules//main.tf`) opening only the ports its own compose stacks use, plus SSH restricted to `var.allowed_ssh_source_ips`. -See [todo.md](todo.md) for a list of tasks and configurations that need to be completed. \ No newline at end of file +## Repository layout + +``` +. +├── infra/ # OpenTofu — provisions the 3 servers, network, firewalls, DNS +│ ├── main.tf # root module: wires network + dev/prod/vpn/dns modules together +│ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names) +│ ├── terraform.tfvars.example +│ └── modules/ +│ ├── network/ # shared private network + subnet (dev + prod) +│ ├── dev/ prod/ vpn/ # one server + firewall each, labeled role= +│ └── dns/ # Hetzner DNS zones + records for every domain in var.dns_zones +├── ansible/ # bootstraps each server, deploys/starts services// onto it +│ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by role label +│ ├── roles/ +│ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades +│ │ └── deploy/ # copies services//, renders .env + Runners/docker-compose.yml +│ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml +├── services/ # Docker Compose stacks, grouped by which server they run on +│ ├── dev/ # Gitea + CI runners + Traefik + backup/restore +│ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore +│ └── vpn/ # WireGuard (wg-easy) + Traefik + backup/restore +├── .devcontainer/ # git submodule: shared devcontainer for this repo (OpenTofu tooling) +└── assets/ +``` + +Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per service, a `backup-docker-compose.yml` + `restore.sh` pair, and a `spinup.sh` / `spindown.sh` pair that brings up or tears down everything on that host in order. `infra/modules/` and `ansible/roles/deploy` mirror the same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks sit side by side. + +## Design choices + +- **OpenTofu and Ansible have a clean split**: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches a server over SSH. Everything from "the server exists" onward — Docker, the `deploy` user, SSH hardening, copying `services//`, running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md). +- **No Hetzner Cloud Volumes.** Every stateful service mounts a plain named Docker volume instead. Each host backs its volumes up nightly to S3 and restores any that are empty on first boot — see [Persistent data](#persistent-data). +- **`vpn` is deliberately off the private network.** SSH to `dev`/`prod` is only reachable through it (see [Security notes](#security-notes)), but it can't reach either directly — containing a compromised VPN endpoint. +- **DNS is provisioned, not just documented.** `module.dns` creates a Hetzner zone and every A/MX/TXT/CNAME record in `infra/main.tf` — see [Managing DNS](#managing-dns). +- **Hosts are discovered dynamically** from the Hetzner API by role label, not a static inventory file — nothing in Ansible reads Terraform state. +- **Gitea Actions runners are disposable.** Their `/data` is a local cache; runner identity re-registers with Gitea on every `spinup.sh`, so they're excluded from backup. + +## Prerequisites + +- A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token +- One or more SSH keys uploaded to that project (Console → Security → SSH Keys), plus the matching private key available locally +- [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` +- [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`) +- Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner +- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/)) and an access key/secret pair — not provisioned by OpenTofu, so create this yourself + +## Quickstart + +[`control.sh`](control.sh) at the repo root is an interactive menu covering the whole setup: + +```sh +./control.sh +``` + +It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, and the `BACKUP_S3_*` credentials, saved to gitignored `.control.env` so you only enter them once), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand. + +## Provisioning the infrastructure (`infra/`) + +```sh +cd infra +export HCLOUD_TOKEN=your-hetzner-api-token # never commit this +cp terraform.tfvars.example terraform.tfvars +$EDITOR terraform.tfvars # set ssh_key_names at minimum + +tofu init +tofu plan +tofu apply +``` + +This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. + +Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`. + +### Managing DNS + +Creating a zone doesn't make Hetzner authoritative by itself — run `tofu output dns_nameservers` and set those as the domain's NS records at its registrar. Propagation time depends on the registrar and the domain's previous NS TTL. + +To add a subdomain, add an entry to `local.dns_records` in `infra/main.tf` (zone, name, target host) and re-run `tofu apply` — don't hand-create records in the Hetzner console, they'll drift from state. `var.records` supports `MX`/`TXT`/`CNAME`/etc. too (see `infra/modules/dns/variables.tf` for the value format each type expects); entries sharing zone/name/type merge into one RRSet. + +`local.mail_records` tracks the Microsoft 365 records for `luke-else.co.uk` (MX, SPF, Google-site-verification, `autodiscover`). DMARC and DKIM aren't set up yet — see [Caveats](#caveats). + +## Bootstrapping and deploying (`ansible/`) + +Once `tofu apply` has created the servers, Ansible installs Docker, creates the `deploy` user, hardens SSH, copies `services//` to each server, and runs `spinup.sh`/`spindown.sh`. Full detail in [`ansible/README.md`](ansible/README.md); the short version: + +```sh +cd ansible +ansible-galaxy collection install -r requirements.yml +export HCLOUD_TOKEN=your-hetzner-api-token # never commit this + +# vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly +ansible-playbook playbooks/site.yml -l role_vpn + +# SSH to vpn as deploy and connect to the WireGuard tunnel it just started, then: +ansible-playbook playbooks/site.yml -l role_dev,role_prod +``` + +`dev`/`prod`'s firewalls only accept SSH from `vpn`'s own public IP, so on a from-scratch estate they aren't reachable until `vpn` is up and you're tunneled through it — visit `https://vpn.luke-else.co.uk`, complete wg-easy's first-run setup, add a client peer, and connect before running the second command. If it's run before you're connected, Ansible just fails to connect — re-run once connected. + +### Persistent data + +Every stateful service mounts a named Docker volume (`gitea_data`, `bitwarden_data`, `letsencrypt_data`, etc.) rather than a Hetzner volume. Each host's `backup-docker-compose.yml` defines: + +- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), nightly cron, tars every mounted volume to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/-.tar.gz`, 14 days of retention. Containers that need a consistent snapshot are stopped for the duration and restarted after. +- **`restore`** — a one-shot container `spinup.sh` runs before anything else starts. Any volume that's already empty is populated from the most recent backup; anything with data is left untouched. On a fresh server (or no prior backup), it's a no-op. + +The S3 credentials (`AWS_S3_BUCKET_NAME`, `AWS_ACCESS_KEY_ID`, etc.) are rendered into `services//.env` by Ansible from the `BACKUP_S3_*` env vars — `spinup.sh` refuses to start anything if `.env` is missing. To trigger either manually: + +```sh +cd services/ +docker compose -f backup-docker-compose.yml exec backup backup # force an immediate backup +docker compose -f backup-docker-compose.yml run --rm restore # restore any currently-empty volumes +``` + +## Running the services (`services/`) + +After `ansible-playbook playbooks/deploy.yml` has copied `services//` to the server, SSH in as `deploy` and run the matching script — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do this remotely: + +```sh +# on dev +./spinup.sh # Traefik → Gitea → (waits, generates a runner token) → Runners → Watchtower +./spindown.sh # reverse order, then prunes images/volumes + +# on prod +./spinup.sh # Traefik → Watchtower → status → websites → Bitwarden → RustDesk +./spindown.sh + +# on vpn +./spinup.sh # Traefik → wg-easy → Watchtower +./spindown.sh +``` + +Each stack can also be managed individually with plain Compose, e.g. `docker compose -f bitwarden-docker-compose.yml up -d` from inside `services/prod`. + +All three hosts run [Watchtower](https://containrrr.dev/watchtower/), polling every 60s with cleanup enabled, so images stay current automatically once deployed — re-run the spinup scripts only after adding/removing a service or changing compose files. Every public-facing service is fronted by its host's own Traefik, terminating TLS via Let's Encrypt, joining an external `proxy` network and opting in via `traefik.enable=true` labels (RustDesk and the Gitea SSH port publish ports directly, since they aren't HTTP). + +`vpn` runs [wg-easy](https://github.com/wg-easy/wg-easy) — WireGuard plus a web UI, fronted by Traefik. It has no automated first-run setup: visit `https://vpn.luke-else.co.uk`, pick an admin username/password (enable 2FA — this account can view/rotate every client's private key and is reachable from the public internet), set the connect host/port (`vpn.luke-else.co.uk` / `51820`), then add a client per device and download its config or scan its QR code. wg-easy's own state lives in the `wg_easy_data` volume, backed up like everything else; client configs themselves are never stored anywhere but the UI. + +## Service inventory + +### `dev` + +| Service | Compose file | Domain(s) | +|---|---|---| +| Traefik | `traefik-docker-compose.yml` | `traefik.cicd.luke-else.co.uk` | +| Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` | +| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated, 3 by default) | N/A | +| Watchtower | `watchtower-docker-compose.yml` | — | +| Backup/restore | `backup-docker-compose.yml` | — | + +### `prod` + +| Service | Compose file | Domain(s) | +|---|---|---| +| Traefik | `traefik-docker-compose.yml` | `traefik.luke-else.co.uk` | +| Websites | `web-docker-compose.yml` | `luke-else.co.uk`, `dev.luke-else.co.uk`, `metarius.luke-else.co.uk`, `www.divine-couture.co.uk`, `snexo.co.uk` | +| Status page (Uptime Kuma) | `status-docker-compose.yml` | `status.luke-else.co.uk` | +| Bitwarden (Vaultwarden) | `bitwarden-docker-compose.yml` | `bitwarden.luke-else.co.uk` | +| RustDesk relay (hbbs/hbbr) | `rd-docker-compose.yml` | `rd.luke-else.co.uk`, ports `21115-21119` | +| Watchtower | `watchtower-docker-compose.yml` | — | +| Backup/restore | `backup-docker-compose.yml` | — | + +### `vpn` + +| Service | Compose file | Domain(s) | +|---|---|---| +| Traefik | `traefik-docker-compose.yml` | `traefik.vpn.luke-else.co.uk` | +| WireGuard (wg-easy) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `51820` | +| Watchtower | `watchtower-docker-compose.yml` | — | +| Backup/restore | `backup-docker-compose.yml` | — | + +## Development container + +`.devcontainer` is a git submodule providing a ready-to-use OpenTofu development environment (VS Code + OpenTofu/Docker/Mermaid extensions): + +```sh +git submodule update --init --recursive +``` + +Then reopen the repo in VS Code with the Dev Containers extension. + +## Caveats + +- **UFW isn't configured** — the Hetzner Cloud Firewalls already allowlist per-host ports, but there's no host-level firewall as defense in depth yet. +- **DMARC and DKIM aren't set up** for `luke-else.co.uk`. DKIM's targets are generated per-tenant by Microsoft 365 (Defender portal → Email & collaboration → Policies & rules → DKIM) and can't be guessed — enable it there first, then add the two CNAMEs it gives you to `local.mail_records`. +- **Only IPv4 DNS is managed.** None of the infra modules track servers' IPv6 addresses, so no AAAA records are generated even though the firewalls already allow IPv6 traffic. +- **`infra/modules/dns/moved.tf`** re-points pre-existing A records at their new resource address after the DNS module was generalized for non-A record types. It's a one-time migration aid — safe to delete once `tofu plan` shows no unexpected changes. + +## Security notes + +- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_*`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. S3 credentials land in each host's `services//.env` (mode `0600`, gitignored, rendered by Ansible). +- SSH to `dev`/`prod` is restricted to `vpn`'s own public IP — you must be tunneled into the VPN to reach them. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your egress IP(s). +- Firewalls are per-host allowlists (`infra/modules//main.tf`) — only ports actually used by that host's compose stacks are open. +- Ansible's bootstrap role disables SSH password auth, restricts root login to key-only, and creates a separate sudo user (`deploy_user`) for day-to-day access. +- DNS zones carry both Hetzner's `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, across all three managed domains. diff --git a/Development/Runners/config.yaml b/services/dev/Runners/config.yaml similarity index 100% rename from Development/Runners/config.yaml rename to services/dev/Runners/config.yaml diff --git a/services/dev/backup-docker-compose.yml b/services/dev/backup-docker-compose.yml new file mode 100644 index 0000000..26b676a --- /dev/null +++ b/services/dev/backup-docker-compose.yml @@ -0,0 +1,48 @@ +services: + # Nightly backup of every named volume on this host to S3. Containers + # labeled docker-volume-backup.stop-during-backup=true (see + # gitea-docker-compose.yml) are stopped for the duration of the backup so + # their data is captured in a consistent state, then restarted. + backup: + image: offen/docker-volume-backup:v2 + container_name: backup + environment: + BACKUP_FILENAME: "dev-%Y-%m-%dT%H-%M-%S.{{ .Extension }}" + BACKUP_CRON_EXPRESSION: "0 3 * * *" + BACKUP_RETENTION_DAYS: "14" + BACKUP_PRUNING_LEEWAY: "5m" + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - gitea_data:/backup/gitea_data:ro + - letsencrypt_data:/backup/letsencrypt_data:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + + # One-shot: restores empty volumes from the latest S3 backup, then exits. + # Run explicitly by spinup.sh before the services that own these volumes + # start - not started by `docker compose up`. + restore: + image: amazon/aws-cli:latest + container_name: backup-restore + entrypoint: ["/bin/bash", "/restore.sh"] + environment: + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - ./restore.sh:/restore.sh:ro + - gitea_data:/restore/gitea_data + - letsencrypt_data:/restore/letsencrypt_data + restart: "no" + +volumes: + gitea_data: + name: gitea_data + letsencrypt_data: + name: letsencrypt_data diff --git a/Development/Gitea/docker-compose.yml b/services/dev/gitea-docker-compose.yml similarity index 87% rename from Development/Gitea/docker-compose.yml rename to services/dev/gitea-docker-compose.yml index 3724a79..836437a 100644 --- a/Development/Gitea/docker-compose.yml +++ b/services/dev/gitea-docker-compose.yml @@ -4,7 +4,7 @@ services: image: gitea/gitea:latest container_name: gitea volumes: - - ./gitea:/data + - gitea_data:/data - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro networks: @@ -29,7 +29,11 @@ services: labels: ## Expose Gitea Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + + # Stopped for the duration of each backup run so the sqlite db/repos are + # captured in a consistent state - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowmethods=*" - "traefik.http.middlewares.cors-gitea.headers.accesscontrolalloworiginlist=*" - "traefik.http.middlewares.cors-gitea.headers.addvaryheader=true" @@ -51,4 +55,8 @@ services: networks: proxy: - external: true \ No newline at end of file + external: true + +volumes: + gitea_data: + name: gitea_data \ No newline at end of file diff --git a/services/dev/restore.sh b/services/dev/restore.sh new file mode 100755 index 0000000..a7732cd --- /dev/null +++ b/services/dev/restore.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Restores this host's named volumes from the latest S3 backup, but only into +# volumes that are currently empty - never overwrites data that's already +# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml +# run --rm restore` before any service that owns one of the mounted volumes +# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh +# estate): it just leaves the empty volumes alone. +set -euo pipefail + +S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}" +ENDPOINT_ARGS=() +if [ -n "${AWS_ENDPOINT:-}" ]; then + ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}") +fi + +empty_volumes=() +for dir in /restore/*/; do + name="$(basename "$dir")" + if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then + empty_volumes+=("$name") + fi +done + +if [ "${#empty_volumes[@]}" -eq 0 ]; then + echo "All volumes already contain data - skipping restore." + exit 0 +fi + +echo "Empty volumes: ${empty_volumes[*]}" + +latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)" +if [ -z "$latest" ]; then + echo "No existing backup found at ${S3_URI} - starting with empty volumes." + exit 0 +fi + +echo "Restoring ${S3_URI}/${latest} into empty volumes only..." +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz" +mkdir "$tmp/extracted" +tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted" + +for name in "${empty_volumes[@]}"; do + if [ -d "$tmp/extracted/$name" ]; then + echo "Restoring $name..." + cp -a "$tmp/extracted/$name/." "/restore/$name/" + else + echo "No $name/ in backup - leaving volume empty." + fi +done + +echo "Restore complete." diff --git a/services/dev/spindown.sh b/services/dev/spindown.sh new file mode 100755 index 0000000..c495c7a --- /dev/null +++ b/services/dev/spindown.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Script file for spinning down all dev docker-containers +cd "$(dirname "$0")" + +cd Runners +docker compose down +cd .. + +docker compose -f gitea-docker-compose.yml down +docker compose -f traefik-docker-compose.yml down +docker compose -f watchtower-docker-compose.yml down +docker compose -f backup-docker-compose.yml down + +docker rmi $(docker images -q) +docker system prune -f -a +# Deliberately no `docker volume prune` - gitea_data/letsencrypt_data/etc. are +# named volumes holding real data, backed up to S3 but not disposable. diff --git a/services/dev/spinup.sh b/services/dev/spinup.sh new file mode 100755 index 0000000..f25d991 --- /dev/null +++ b/services/dev/spinup.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Script file for spinning up all dev docker-containers +set -e +cd "$(dirname "$0")" + +if [ ! -f .env ]; then + echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 + exit 1 +fi + +echo "Restoring named volumes from S3 backup if any are empty..." +docker compose -f backup-docker-compose.yml pull restore +docker compose -f backup-docker-compose.yml run --rm restore + +docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d + +docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d + +echo "Waiting for Gitea to become ready..." +sleep 20 +for i in $(seq 1 30); do + if docker exec -u 1000:1000 gitea gitea admin user list >/dev/null 2>&1; then + break + fi + sleep 2 +done + +# Generate a fresh Actions runner registration token straight from Gitea and +# hand it to the runner containers via .env (docker compose auto-loads this) - +# no manual copy/paste from the admin UI needed. +RUNNER_TOKEN=$(docker exec -u 1000:1000 gitea gitea actions generate-runner-token) +echo "GITEA_RUNNER_REGISTRATION_TOKEN=$RUNNER_TOKEN" > Runners/.env + +cd Runners +docker compose pull && docker compose up -d +cd .. + +docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d + +docker compose -f backup-docker-compose.yml pull backup +docker compose -f backup-docker-compose.yml up -d backup diff --git a/TraefikRunner/docker-compose.yml b/services/dev/traefik-docker-compose.yml similarity index 84% rename from TraefikRunner/docker-compose.yml rename to services/dev/traefik-docker-compose.yml index e8970fd..6e46ebc 100644 --- a/TraefikRunner/docker-compose.yml +++ b/services/dev/traefik-docker-compose.yml @@ -15,7 +15,7 @@ services: - "80:80" - "443:443" volumes: - - "./letsencrypt:/letsencrypt" + - "letsencrypt_data:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy @@ -30,11 +30,13 @@ services: - "traefik.http.routers.traefik.rule=Host(`traefik.cicd.luke-else.co.uk`)" - "traefik.http.routers.traefik.entrypoints=websecure" - "traefik.http.routers.traefik.service=api@internal" - - "traefik.http.routers.traefik.tls.certresolver=myresolver" - - "traefik.http.routers.traefik.middlewares=traefik-auth" - - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" + - "traefik.http.routers.traefik.tls.certresolver=myresolver" restart: unless-stopped networks: proxy: - name: proxy \ No newline at end of file + name: proxy + +volumes: + letsencrypt_data: + name: letsencrypt_data diff --git a/Watchtower/docker-compose.yml b/services/dev/watchtower-docker-compose.yml similarity index 100% rename from Watchtower/docker-compose.yml rename to services/dev/watchtower-docker-compose.yml diff --git a/services/prod/backup-docker-compose.yml b/services/prod/backup-docker-compose.yml new file mode 100644 index 0000000..0515547 --- /dev/null +++ b/services/prod/backup-docker-compose.yml @@ -0,0 +1,58 @@ +services: + # Nightly backup of every named volume on this host to S3. Containers + # labeled docker-volume-backup.stop-during-backup=true (see + # bitwarden-docker-compose.yml, rd-docker-compose.yml, + # status-docker-compose.yml, web-docker-compose.yml) are stopped for the + # duration of the backup so their data is captured in a consistent state, + # then restarted. + backup: + image: offen/docker-volume-backup:v2 + container_name: backup + environment: + BACKUP_FILENAME: "prod-%Y-%m-%dT%H-%M-%S.{{ .Extension }}" + BACKUP_CRON_EXPRESSION: "0 3 * * *" + BACKUP_RETENTION_DAYS: "14" + BACKUP_PRUNING_LEEWAY: "5m" + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - bitwarden_data:/backup/bitwarden_data:ro + - rustdesk_data:/backup/rustdesk_data:ro + - uptime_kuma_data:/backup/uptime_kuma_data:ro + - letsencrypt_data:/backup/letsencrypt_data:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + + # One-shot: restores empty volumes from the latest S3 backup, then exits. + # Run explicitly by spinup.sh before the services that own these volumes + # start - not started by `docker compose up`. + restore: + image: amazon/aws-cli:latest + container_name: backup-restore + entrypoint: [ "/bin/bash", "/restore.sh" ] + environment: + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - ./restore.sh:/restore.sh:ro + - bitwarden_data:/restore/bitwarden_data + - rustdesk_data:/restore/rustdesk_data + - uptime_kuma_data:/restore/uptime_kuma_data + - letsencrypt_data:/restore/letsencrypt_data + restart: "no" + +volumes: + bitwarden_data: + name: bitwarden_data + rustdesk_data: + name: rustdesk_data + uptime_kuma_data: + name: uptime_kuma_data + letsencrypt_data: + name: letsencrypt_data diff --git a/Bitwarden/docker-compose.yml b/services/prod/bitwarden-docker-compose.yml similarity index 78% rename from Bitwarden/docker-compose.yml rename to services/prod/bitwarden-docker-compose.yml index 91c1ffe..2cd48fa 100644 --- a/Bitwarden/docker-compose.yml +++ b/services/prod/bitwarden-docker-compose.yml @@ -5,13 +5,16 @@ services: image: "vaultwarden/server:latest" container_name: vaultwarden volumes: - - ./bitwarden/:/data/ + - bitwarden_data:/data/ networks: - proxy labels: ## Expose Bitwarden Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.bitwarden-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.bitwarden-insecure.rule=Host(`bitwarden.luke-else.co.uk`)" @@ -24,4 +27,8 @@ services: networks: proxy: - external: true \ No newline at end of file + external: true + +volumes: + bitwarden_data: + name: bitwarden_data \ No newline at end of file diff --git a/Rustdesk/docker-compose.yml b/services/prod/rd-docker-compose.yml similarity index 81% rename from Rustdesk/docker-compose.yml rename to services/prod/rd-docker-compose.yml index 4dc4145..5660ba1 100644 --- a/Rustdesk/docker-compose.yml +++ b/services/prod/rd-docker-compose.yml @@ -6,7 +6,7 @@ services: restart: unless-stopped volumes: - - ./data:/root + - rustdesk_data:/root networks: - proxy @@ -20,6 +20,9 @@ services: labels: - "traefik.enable=true" + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.rustdesk-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.rustdesk-insecure.rule=Host(`rd.luke-else.co.uk`)" @@ -39,7 +42,7 @@ services: restart: unless-stopped volumes: - - ./data:/root + - rustdesk_data:/root networks: - proxy @@ -50,4 +53,8 @@ services: networks: proxy: - external: true \ No newline at end of file + external: true + +volumes: + rustdesk_data: + name: rustdesk_data \ No newline at end of file diff --git a/services/prod/restore.sh b/services/prod/restore.sh new file mode 100755 index 0000000..a7732cd --- /dev/null +++ b/services/prod/restore.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Restores this host's named volumes from the latest S3 backup, but only into +# volumes that are currently empty - never overwrites data that's already +# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml +# run --rm restore` before any service that owns one of the mounted volumes +# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh +# estate): it just leaves the empty volumes alone. +set -euo pipefail + +S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}" +ENDPOINT_ARGS=() +if [ -n "${AWS_ENDPOINT:-}" ]; then + ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}") +fi + +empty_volumes=() +for dir in /restore/*/; do + name="$(basename "$dir")" + if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then + empty_volumes+=("$name") + fi +done + +if [ "${#empty_volumes[@]}" -eq 0 ]; then + echo "All volumes already contain data - skipping restore." + exit 0 +fi + +echo "Empty volumes: ${empty_volumes[*]}" + +latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)" +if [ -z "$latest" ]; then + echo "No existing backup found at ${S3_URI} - starting with empty volumes." + exit 0 +fi + +echo "Restoring ${S3_URI}/${latest} into empty volumes only..." +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz" +mkdir "$tmp/extracted" +tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted" + +for name in "${empty_volumes[@]}"; do + if [ -d "$tmp/extracted/$name" ]; then + echo "Restoring $name..." + cp -a "$tmp/extracted/$name/." "/restore/$name/" + else + echo "No $name/ in backup - leaving volume empty." + fi +done + +echo "Restore complete." diff --git a/services/prod/spindown.sh b/services/prod/spindown.sh new file mode 100755 index 0000000..e6c19ad --- /dev/null +++ b/services/prod/spindown.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Script file for spinning down all prod docker-containers +cd "$(dirname "$0")" + +docker compose -f web-docker-compose.yml down +docker compose -f status-docker-compose.yml down +docker compose -f bitwarden-docker-compose.yml down +docker compose -f rd-docker-compose.yml down +docker compose -f watchtower-docker-compose.yml down +docker compose -f traefik-docker-compose.yml down +docker compose -f backup-docker-compose.yml down + +docker rmi $(docker images -q) +docker system prune -f -a +# Deliberately no `docker volume prune` - bitwarden_data/rustdesk_data/etc. are +# named volumes holding real data, backed up to S3 but not disposable. diff --git a/services/prod/spinup.sh b/services/prod/spinup.sh new file mode 100755 index 0000000..ac5092b --- /dev/null +++ b/services/prod/spinup.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Script file for spinning up all prod docker-containers +set -e +cd "$(dirname "$0")" + +if [ ! -f .env ]; then + echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 + exit 1 +fi + +echo "Restoring named volumes from S3 backup if any are empty..." +docker compose -f backup-docker-compose.yml pull restore +docker compose -f backup-docker-compose.yml run --rm restore + +docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d + +sleep 20 # Allow Traefik + registry auth to settle before starting the rest of the services + +docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d +docker compose -f status-docker-compose.yml pull && docker compose -f status-docker-compose.yml up -d +docker compose -f web-docker-compose.yml pull && docker compose -f web-docker-compose.yml up -d +docker compose -f bitwarden-docker-compose.yml pull && docker compose -f bitwarden-docker-compose.yml up -d +docker compose -f rd-docker-compose.yml pull && docker compose -f rd-docker-compose.yml up -d + +docker compose -f backup-docker-compose.yml pull backup +docker compose -f backup-docker-compose.yml up -d backup diff --git a/services/prod/status-docker-compose.yml b/services/prod/status-docker-compose.yml new file mode 100644 index 0000000..f9c5822 --- /dev/null +++ b/services/prod/status-docker-compose.yml @@ -0,0 +1,32 @@ +services: + status: + image: louislam/uptime-kuma:latest + container_name: status + volumes: + - uptime_kuma_data:/app/data + networks: + - proxy + labels: + ## Expose uptime-kuma Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)" + - "traefik.http.routers.status-insecure.entrypoints=web" + + - "traefik.http.routers.status.rule=Host(`status.luke-else.co.uk`)" + - "traefik.http.routers.status.entrypoints=websecure" + - "traefik.http.routers.status.tls.certresolver=myresolver" + restart: unless-stopped + +networks: + proxy: + external: true + +volumes: + uptime_kuma_data: + name: uptime_kuma_data diff --git a/Traefik/docker-compose.yml b/services/prod/traefik-docker-compose.yml similarity index 84% rename from Traefik/docker-compose.yml rename to services/prod/traefik-docker-compose.yml index c2b613a..78b6483 100644 --- a/Traefik/docker-compose.yml +++ b/services/prod/traefik-docker-compose.yml @@ -8,7 +8,6 @@ services: - "--providers.docker.exposedbydefault=false" - "--entrypoints.web.address=:80" - "--entrypoints.websecure.address=:443" - - "--entrypoints.mongo.address=:27017" - "--entrypoints.web.transport.respondingTimeouts.readTimeout=120s" - "--entrypoints.websecure.transport.respondingTimeouts.readTimeout=120s" - "--entrypoints.web.transport.respondingTimeouts.writeTimeout=120s" @@ -21,7 +20,7 @@ services: - "443:443" - "27017:27017" volumes: - - "./letsencrypt:/letsencrypt" + - "letsencrypt_data:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy @@ -36,11 +35,13 @@ services: - "traefik.http.routers.traefik.rule=Host(`traefik.luke-else.co.uk`)" - "traefik.http.routers.traefik.entrypoints=websecure" - "traefik.http.routers.traefik.service=api@internal" - - "traefik.http.routers.traefik.tls.certresolver=myresolver" - - "traefik.http.routers.traefik.middlewares=traefik-auth" - - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" + - "traefik.http.routers.traefik.tls.certresolver=myresolver" restart: unless-stopped networks: proxy: - name: proxy \ No newline at end of file + name: proxy + +volumes: + letsencrypt_data: + name: letsencrypt_data diff --git a/services/prod/watchtower-docker-compose.yml b/services/prod/watchtower-docker-compose.yml new file mode 100644 index 0000000..ee735d3 --- /dev/null +++ b/services/prod/watchtower-docker-compose.yml @@ -0,0 +1,8 @@ +services: + watchtower: + image: nickfedor/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - WATCHTOWER_CLEANUP=true + - WATCHTOWER_POLL_INTERVAL=60 diff --git a/Websites/docker-compose.yml b/services/prod/web-docker-compose.yml similarity index 58% rename from Websites/docker-compose.yml rename to services/prod/web-docker-compose.yml index ffc2c6b..93af14c 100644 --- a/Websites/docker-compose.yml +++ b/services/prod/web-docker-compose.yml @@ -1,6 +1,6 @@ services: -#Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080) + #Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080) luke-else: image: git.luke-else.co.uk/luke-else/luke-else.co.uk container_name: luke-else @@ -9,7 +9,7 @@ services: labels: ## Expose luke-else Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.personal-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.personal-insecure.rule=Host(`luke-else.co.uk`)" @@ -28,7 +28,7 @@ services: labels: ## Expose luke-else Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.personal-dev-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.personal-dev-insecure.rule=Host(`dev.luke-else.co.uk`)" @@ -47,7 +47,7 @@ services: labels: ## Expose metarius Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.metarius-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.metarius-insecure.rule=Host(`metarius.luke-else.co.uk`)" @@ -58,46 +58,6 @@ services: - "traefik.http.routers.metarius.tls.certresolver=myresolver" restart: unless-stopped - divine-couture: - image: git.luke-else.co.uk/luke-else/divine-couture.co.uk:latest - container_name: divine-couture - networks: - - proxy - labels: - ## Expose divine-couture Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.divine-couture-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.divine-couture-insecure.rule=Host(`www.divine-couture.co.uk`)" - - "traefik.http.routers.divine-couture-insecure.entrypoints=web" - - - "traefik.http.routers.divine-couture.rule=Host(`www.divine-couture.co.uk`)" - - "traefik.http.routers.divine-couture.entrypoints=websecure" - - "traefik.http.routers.divine-couture.tls.certresolver=myresolver" - restart: unless-stopped - - snexo: - image: "php:apache" - container_name: snexo - volumes: - - ./snexo.co.uk/:/var/www/html - networks: - - proxy - labels: - ## Expose Snexo Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.snexo-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.snexo-insecure.rule=Host(`snexo.co.uk`)" - - "traefik.http.routers.snexo-insecure.entrypoints=web" - - - "traefik.http.routers.snexo.rule=Host(`snexo.co.uk`)" - - "traefik.http.routers.snexo.entrypoints=websecure" - - "traefik.http.routers.snexo.tls.certresolver=myresolver" - restart: unless-stopped - networks: proxy: external: true diff --git a/services/vpn/backup-docker-compose.yml b/services/vpn/backup-docker-compose.yml new file mode 100644 index 0000000..2c72e03 --- /dev/null +++ b/services/vpn/backup-docker-compose.yml @@ -0,0 +1,48 @@ +services: + # Nightly backup of every named volume on this host to S3. Containers + # labeled docker-volume-backup.stop-during-backup=true (see + # vpn-docker-compose.yml) are stopped for the duration of the backup so + # their data is captured in a consistent state, then restarted. + backup: + image: offen/docker-volume-backup:v2 + container_name: backup + environment: + BACKUP_FILENAME: "vpn-%Y-%m-%dT%H-%M-%S.{{ .Extension }}" + BACKUP_CRON_EXPRESSION: "0 3 * * *" + BACKUP_RETENTION_DAYS: "14" + BACKUP_PRUNING_LEEWAY: "5m" + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - wg_easy_data:/backup/wg_easy_data:ro + - letsencrypt_data:/backup/letsencrypt_data:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + + # One-shot: restores empty volumes from the latest S3 backup, then exits. + # Run explicitly by spinup.sh before the services that own these volumes + # start - not started by `docker compose up`. + restore: + image: amazon/aws-cli:latest + container_name: backup-restore + entrypoint: ["/bin/bash", "/restore.sh"] + environment: + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - ./restore.sh:/restore.sh:ro + - wg_easy_data:/restore/wg_easy_data + - letsencrypt_data:/restore/letsencrypt_data + restart: "no" + +volumes: + wg_easy_data: + name: wg_easy_data + letsencrypt_data: + name: letsencrypt_data diff --git a/services/vpn/restore.sh b/services/vpn/restore.sh new file mode 100755 index 0000000..a7732cd --- /dev/null +++ b/services/vpn/restore.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Restores this host's named volumes from the latest S3 backup, but only into +# volumes that are currently empty - never overwrites data that's already +# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml +# run --rm restore` before any service that owns one of the mounted volumes +# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh +# estate): it just leaves the empty volumes alone. +set -euo pipefail + +S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}" +ENDPOINT_ARGS=() +if [ -n "${AWS_ENDPOINT:-}" ]; then + ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}") +fi + +empty_volumes=() +for dir in /restore/*/; do + name="$(basename "$dir")" + if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then + empty_volumes+=("$name") + fi +done + +if [ "${#empty_volumes[@]}" -eq 0 ]; then + echo "All volumes already contain data - skipping restore." + exit 0 +fi + +echo "Empty volumes: ${empty_volumes[*]}" + +latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)" +if [ -z "$latest" ]; then + echo "No existing backup found at ${S3_URI} - starting with empty volumes." + exit 0 +fi + +echo "Restoring ${S3_URI}/${latest} into empty volumes only..." +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz" +mkdir "$tmp/extracted" +tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted" + +for name in "${empty_volumes[@]}"; do + if [ -d "$tmp/extracted/$name" ]; then + echo "Restoring $name..." + cp -a "$tmp/extracted/$name/." "/restore/$name/" + else + echo "No $name/ in backup - leaving volume empty." + fi +done + +echo "Restore complete." diff --git a/services/vpn/spindown.sh b/services/vpn/spindown.sh new file mode 100755 index 0000000..8c365c4 --- /dev/null +++ b/services/vpn/spindown.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Script file for spinning down all vpn docker-containers +cd "$(dirname "$0")" + +docker compose -f vpn-docker-compose.yml down +docker compose -f traefik-docker-compose.yml down +docker compose -f watchtower-docker-compose.yml down +docker compose -f backup-docker-compose.yml down + +docker rmi $(docker images -q) +docker system prune -f -a +# Deliberately no `docker volume prune` - wg_easy_data/letsencrypt_data are +# named volumes holding real data, backed up to S3 but not disposable. diff --git a/services/vpn/spinup.sh b/services/vpn/spinup.sh new file mode 100755 index 0000000..1911994 --- /dev/null +++ b/services/vpn/spinup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Script file for spinning up all vpn docker-containers +set -e +cd "$(dirname "$0")" + +if [ ! -f .env ]; then + echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 + exit 1 +fi + +echo "Restoring named volumes from S3 backup if any are empty..." +docker compose -f backup-docker-compose.yml pull restore +docker compose -f backup-docker-compose.yml run --rm restore + +docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d +docker compose -f vpn-docker-compose.yml pull && docker compose -f vpn-docker-compose.yml up -d +docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d + +docker compose -f backup-docker-compose.yml pull backup +docker compose -f backup-docker-compose.yml up -d backup diff --git a/TraefikVPN/docker-compose.yml b/services/vpn/traefik-docker-compose.yml similarity index 84% rename from TraefikVPN/docker-compose.yml rename to services/vpn/traefik-docker-compose.yml index 7383006..234ad1c 100644 --- a/TraefikVPN/docker-compose.yml +++ b/services/vpn/traefik-docker-compose.yml @@ -15,7 +15,7 @@ services: - "80:80" - "443:443" volumes: - - "./letsencrypt:/letsencrypt" + - "letsencrypt_data:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy @@ -30,11 +30,13 @@ services: - "traefik.http.routers.traefik.rule=Host(`traefik.vpn.luke-else.co.uk`)" - "traefik.http.routers.traefik.entrypoints=websecure" - "traefik.http.routers.traefik.service=api@internal" - - "traefik.http.routers.traefik.tls.certresolver=myresolver" - - "traefik.http.routers.traefik.middlewares=traefik-auth" - - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" + - "traefik.http.routers.traefik.tls.certresolver=myresolver" restart: unless-stopped networks: proxy: - name: proxy \ No newline at end of file + name: proxy + +volumes: + letsencrypt_data: + name: letsencrypt_data diff --git a/services/vpn/vpn-docker-compose.yml b/services/vpn/vpn-docker-compose.yml new file mode 100644 index 0000000..8273abd --- /dev/null +++ b/services/vpn/vpn-docker-compose.yml @@ -0,0 +1,44 @@ +services: + wg-easy: + image: ghcr.io/wg-easy/wg-easy:15 + cap_add: + - NET_ADMIN + - SYS_MODULE # lets the container modprobe wireguard if the host kernel doesn't already have it loaded + sysctls: + - net.ipv4.ip_forward=1 + - net.ipv4.conf.all.src_valid_mark=1 + - net.ipv6.conf.all.disable_ipv6=0 + - net.ipv6.conf.all.forwarding=1 + - net.ipv6.conf.default.forwarding=1 + ports: + - 51820:51820/udp # WireGuard tunnel - the web UI (51821) only needs to be reachable via Traefik below, not published directly + networks: + - proxy + volumes: + - /lib/modules:/lib/modules:ro + - wg_easy_data:/etc/wireguard + labels: + ## Expose the wg-easy web UI through Traefik ## + - "traefik.enable=true" + + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.vpn-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.vpn-insecure.rule=Host(`vpn.luke-else.co.uk`)" + - "traefik.http.routers.vpn-insecure.entrypoints=web" + + - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)" + - "traefik.http.routers.vpn.entrypoints=websecure" + - "traefik.http.routers.vpn.tls.certresolver=myresolver" + - "traefik.http.services.vpn.loadbalancer.server.port=51821" + restart: unless-stopped + +networks: + proxy: + external: true + +volumes: + wg_easy_data: + name: wg_easy_data diff --git a/services/vpn/watchtower-docker-compose.yml b/services/vpn/watchtower-docker-compose.yml new file mode 100644 index 0000000..ee735d3 --- /dev/null +++ b/services/vpn/watchtower-docker-compose.yml @@ -0,0 +1,8 @@ +services: + watchtower: + image: nickfedor/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - WATCHTOWER_CLEANUP=true + - WATCHTOWER_POLL_INTERVAL=60 diff --git a/todo.md b/todo.md deleted file mode 100644 index a53d118..0000000 --- a/todo.md +++ /dev/null @@ -1,64 +0,0 @@ -# ToDo Items - -## General - -- Setup non root user -- UFW should be setup to keep VPS secure and only allow for: - - - https - - http - - ssh - - ftp - - 27017 - - - "21115:21115/tcp" - - "21116:21116/tcp" - - "21116:21116/udp" - - "21117:21117/tcp" - - "21119:21119/tcp" - -- Install SSH keys -- Setup unattended upgrades -- Install docker, docker-compose and apache utils. - -## Traefik + TraefikRunner + Traefik VPN - -- Setup htaccess -> `echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g` -- Ensure email address is correct - -## Gitea - -- Ensure that ports are assigned correctly for the system - -# Gitea Runner - -- Ensure that a registration token has been setup before continuing - -## Tracking - -Create a .env file with the following content: - -```sh -ACKEE_USERNAME=luke-else -ACKEE_PASSWORD=XXX -``` - -## Websites - -- Ensure website files are copied over -- Ensure that ports are assigned correctly for the system - -## Bitwarden - -- Ensure that all data is fully encrypted during transfer. -- Ensure that ports are assigned correctly for the system - -## Rustdesk - -- No additional setup required bar the ports - -## Database - -- Ensure that mysql root password, user and default database are updated. -- Ensure that mongo root password, and user are updated. -- Ensure database ports are correctly assigned and do not have to pass through traefik. diff --git a/vpn-spindown.sh b/vpn-spindown.sh deleted file mode 100755 index 7cd9ef1..0000000 --- a/vpn-spindown.sh +++ /dev/null @@ -1,17 +0,0 @@ -#Script file for spinning down all CICD relevant docker-containers - -cd ./VPN/ -docker compose down -cd .. - -cd ./TraefikVPN/ -docker compose down -cd .. - -cd ./Watchtower/ -docker compose down -cd .. - -docker rmi $(docker images -q) -docker system prune -f -a -docker volume prune -f -a \ No newline at end of file diff --git a/vpn-spinup.sh b/vpn-spinup.sh deleted file mode 100755 index 4cc5159..0000000 --- a/vpn-spinup.sh +++ /dev/null @@ -1,13 +0,0 @@ -#Script file for spinning up all docker-containers - -cd ./TraefikRunner/ -docker compose pull && docker compose up -d -cd .. - -cd ./VPN/ -docker compose pull && docker compose up -d -cd .. - -cd ./Watchtower/ -docker compose pull && docker compose up -d -cd .. \ No newline at end of file