From 40d3e203348ba8e532ea62bff892c56c5ca7860c Mon Sep 17 00:00:00 2001 From: Luke Else Date: Thu, 9 Jul 2026 20:52:25 +0100 Subject: [PATCH 01/24] chore: Restructured repo --- .devcontainer | 1 + .gitmodules | 4 + Bitwarden/docker-compose.yml | 27 ------- Database/docker-compose.yml | 24 ------ Development/Gitea/docker-compose.yml | 54 ------------- Development/Runners/config.yaml | 0 Development/Runners/docker-compose.yml | 33 -------- Misc/docker-compose.yml | 51 ------------ Rustdesk/docker-compose.yml | 53 ------------- Tracking/docker-compose.yml | 52 ------------- Traefik/docker-compose.yml | 46 ----------- TraefikRunner/docker-compose.yml | 40 ---------- TraefikVPN/docker-compose.yml | 40 ---------- VPN/docker-compose.yml | 24 ------ Watchtower/docker-compose.yml | 8 -- Websites/docker-compose.yml | 103 ------------------------- cicd-spindown.sh | 16 ---- cicd-spinup.sh | 13 ---- main-spindown.sh | 42 ---------- main-spinup.sh | 39 ---------- todo.md | 64 --------------- vpn-spindown.sh | 17 ---- vpn-spinup.sh | 13 ---- 23 files changed, 5 insertions(+), 759 deletions(-) create mode 160000 .devcontainer create mode 100644 .gitmodules delete mode 100644 Bitwarden/docker-compose.yml delete mode 100644 Database/docker-compose.yml delete mode 100644 Development/Gitea/docker-compose.yml delete mode 100644 Development/Runners/config.yaml delete mode 100644 Development/Runners/docker-compose.yml delete mode 100644 Misc/docker-compose.yml delete mode 100644 Rustdesk/docker-compose.yml delete mode 100644 Tracking/docker-compose.yml delete mode 100644 Traefik/docker-compose.yml delete mode 100644 TraefikRunner/docker-compose.yml delete mode 100644 TraefikVPN/docker-compose.yml delete mode 100644 VPN/docker-compose.yml delete mode 100644 Watchtower/docker-compose.yml delete mode 100644 Websites/docker-compose.yml delete mode 100755 cicd-spindown.sh delete mode 100755 cicd-spinup.sh delete mode 100755 main-spindown.sh delete mode 100755 main-spinup.sh delete mode 100644 todo.md delete mode 100755 vpn-spindown.sh delete mode 100755 vpn-spinup.sh diff --git a/.devcontainer b/.devcontainer new file mode 160000 index 0000000..192a20c --- /dev/null +++ b/.devcontainer @@ -0,0 +1 @@ +Subproject commit 192a20c759769b050f9efa89179baf7953bae65c diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..070ca83 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule ".devcontainer"] + path = .devcontainer + url = ssh://git@git.luke-else.co.uk:222/luke-else/.devcontainer.git + branch = tf diff --git a/Bitwarden/docker-compose.yml b/Bitwarden/docker-compose.yml deleted file mode 100644 index 91c1ffe..0000000 --- a/Bitwarden/docker-compose.yml +++ /dev/null @@ -1,27 +0,0 @@ -services: - -#Bitwarden () - bitwarden: - image: "vaultwarden/server:latest" - container_name: vaultwarden - volumes: - - ./bitwarden/:/data/ - networks: - - proxy - labels: - ## Expose Bitwarden Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.bitwarden-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.bitwarden-insecure.rule=Host(`bitwarden.luke-else.co.uk`)" - - "traefik.http.routers.bitwarden-insecure.entrypoints=web" - - - "traefik.http.routers.bitwarden.rule=Host(`bitwarden.luke-else.co.uk`)" - - "traefik.http.routers.bitwarden.entrypoints=websecure" - - "traefik.http.routers.bitwarden.tls.certresolver=myresolver" - restart: unless-stopped - -networks: - proxy: - external: true \ No newline at end of file diff --git a/Database/docker-compose.yml b/Database/docker-compose.yml deleted file mode 100644 index 3d1a9a3..0000000 --- a/Database/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -services: - -#MongoDB (27017) - mongodb: - image: "mongo:latest" - container_name: mongoDB - volumes: - - ./mongo/:/data/db - networks: - - proxy - labels: - - "traefik.enable=true" - - "traefik.tcp.routers.mongodb.rule=HostSNI(`*`)" - - "traefik.tcp.routers.mongodb.entrypoints=mongo" - - "traefik.tcp.routers.mongodb.service=mongodb" - - "traefik.tcp.services.mongodb.loadbalancer.server.port=27017" - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: rootpassword - restart: unless-stopped - -networks: - proxy: - external: true \ No newline at end of file diff --git a/Development/Gitea/docker-compose.yml b/Development/Gitea/docker-compose.yml deleted file mode 100644 index 3724a79..0000000 --- a/Development/Gitea/docker-compose.yml +++ /dev/null @@ -1,54 +0,0 @@ -services: - #gitea (222) - gitea: - image: gitea/gitea:latest - container_name: gitea - volumes: - - ./gitea:/data - - /etc/timezone:/etc/timezone:ro - - /etc/localtime:/etc/localtime:ro - networks: - - proxy - ports: - - "222:22" - environment: - - APP_NAME="gitea" - - USER_UID=1000 - - USER_GID=1000 - - USER=git - - RUN_MODE=prod - - DOMAIN=git.luke-else.co.uk - - SSH_DOMAIN=git.luke-else.co.uk - - HTTP_PORT=3000 - - ROOT_URL=https://git.luke-else.co.uk - - SSH_PORT=222 - - SSH_LISTEN_PORT=22 - - DB_TYPE=sqlite3 - - GITEA_service_DISABLE_REGISTRATION=true - - GITEA_server_LANDING_PAGE=/luke-else - labels: - ## Expose Gitea Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowmethods=*" - - "traefik.http.middlewares.cors-gitea.headers.accesscontrolalloworiginlist=*" - - "traefik.http.middlewares.cors-gitea.headers.addvaryheader=true" - - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowcredentials=true" - - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowheaders=Content-Type,Authorization" - - "traefik.http.middlewares.cors-gitea.headers.accesscontrolmaxage=100" - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.gitea-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.gitea-insecure.rule=Host(`git.luke-else.co.uk`)" - - "traefik.http.routers.gitea-insecure.entrypoints=web" - - - "traefik.http.services.gitea.loadbalancer.server.port=3000" - - "traefik.http.routers.gitea.rule=Host(`git.luke-else.co.uk`)" - - "traefik.http.routers.gitea.entrypoints=websecure" - - "traefik.http.routers.gitea.tls.certresolver=myresolver" - - "traefik.http.routers.gitea.middlewares=cors-gitea" - restart: unless-stopped - -networks: - proxy: - external: true \ No newline at end of file diff --git a/Development/Runners/config.yaml b/Development/Runners/config.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/Development/Runners/docker-compose.yml b/Development/Runners/docker-compose.yml deleted file mode 100644 index f9f5acd..0000000 --- a/Development/Runners/docker-compose.yml +++ /dev/null @@ -1,33 +0,0 @@ -services: - # Runner - runner: - image: gitea/act_runner:latest - container_name: gitea_runner - volumes: - - ./config.yaml:/config.yaml - - ./gitea_runner:/data - - /var/run/docker.sock:/var/run/docker.sock - networks: - - proxy - environment: - CONFIG_FILE: /config.yaml - GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" - GITEA_RUNNER_REGISTRATION_TOKEN: "INSERT REGISTRATION TOKEN" - GITEA_RUNNER_NAME: "CICD" - labels: - ## Expose cicd Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.cicd-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.cicd-insecure.rule=Host(`cicd.luke-else.co.uk`)" - - "traefik.http.routers.cicd-insecure.entrypoints=web" - - - "traefik.http.routers.cicd.rule=Host(`cicd.luke-else.co.uk`)" - - "traefik.http.routers.cicd.entrypoints=websecure" - - "traefik.http.routers.cicd.tls.certresolver=myresolver" - restart: unless-stopped - -networks: - proxy: - external: true \ No newline at end of file diff --git a/Misc/docker-compose.yml b/Misc/docker-compose.yml deleted file mode 100644 index 7914b79..0000000 --- a/Misc/docker-compose.yml +++ /dev/null @@ -1,51 +0,0 @@ -services: - status: - image: louislam/uptime-kuma:latest - container_name: status - volumes: - - ./uptime-kuma/data:/app/data - networks: - - proxy - labels: - ## Expose uptime-kuma Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)" - - "traefik.http.routers.status-insecure.entrypoints=web" - - - "traefik.http.routers.status.rule=Host(`status.luke-else.co.uk`)" - - "traefik.http.routers.status.entrypoints=websecure" - - "traefik.http.routers.status.tls.certresolver=myresolver" - restart: unless-stopped - - portainer: - image: portainer/portainer-ce:latest - container_name: portainer - security_opt: - - no-new-privileges:true - volumes: - - /etc/localtime:/etc/localtime:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./portainer-data:/data - networks: - - proxy - labels: - ## Expose portainer Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.portainer-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.portainer-insecure.rule=Host(`portainer.luke-else.co.uk`)" - - "traefik.http.routers.portainer-insecure.entrypoints=web" - - - "traefik.http.routers.portainer.rule=Host(`portainer.luke-else.co.uk`)" - - "traefik.http.services.portainer.loadbalancer.server.port=9000" - - "traefik.http.routers.portainer.entrypoints=websecure" - - "traefik.http.routers.portainer.tls.certresolver=myresolver" - restart: unless-stopped - -networks: - proxy: - external: true \ No newline at end of file diff --git a/Rustdesk/docker-compose.yml b/Rustdesk/docker-compose.yml deleted file mode 100644 index 4dc4145..0000000 --- a/Rustdesk/docker-compose.yml +++ /dev/null @@ -1,53 +0,0 @@ -services: - hbbs: - image: rustdesk/rustdesk-server:latest - container_name: rustdesk-hbbs - command: hbbs - restart: unless-stopped - - volumes: - - ./data:/root - - networks: - - proxy - - ports: - - "21115:21115/tcp" - - "21116:21116/tcp" - - "21116:21116/udp" - # - "21118:21118/tcp" - - labels: - - "traefik.enable=true" - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.rustdesk-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.rustdesk-insecure.rule=Host(`rd.luke-else.co.uk`)" - - "traefik.http.routers.rustdesk-insecure.entrypoints=web" - - - "traefik.http.routers.rustdesk.rule=Host(`rd.luke-else.co.uk`)" - - "traefik.http.routers.rustdesk.entrypoints=websecure" - - "traefik.http.routers.rustdesk.tls.certresolver=myresolver" - - # Service - - "traefik.http.services.rustdesk.loadbalancer.server.port=21118" - - hbbr: - image: rustdesk/rustdesk-server:latest - container_name: rustdesk-hbbr - command: hbbr - restart: unless-stopped - - volumes: - - ./data:/root - - networks: - - proxy - - ports: - - "21117:21117/tcp" - - "21119:21119/tcp" - -networks: - proxy: - external: true \ No newline at end of file diff --git a/Tracking/docker-compose.yml b/Tracking/docker-compose.yml deleted file mode 100644 index 6fe612a..0000000 --- a/Tracking/docker-compose.yml +++ /dev/null @@ -1,52 +0,0 @@ -services: - ackee: - image: electerious/ackee - container_name: ackee - environment: - - WAIT_HOSTS=mongo:27017 - - ACKEE_MONGODB=mongodb://mongo-ackee:27017/ackee - env_file: - - .env - expose: - - 3000 - networks: - - tracking - - proxy - labels: - ## Expose Ackee Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - # Enable CORS headers - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowmethods=*" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolalloworiginlist=https://luke-else.co.uk" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowcredentials=true" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowheaders=Content-Type,Authorization" - - "traefik.http.middlewares.cors-tracking.headers.addvaryheader=true" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolmaxage=100" - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.ackee-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.ackee-insecure.rule=Host(`tracking.luke-else.co.uk`)" - - "traefik.http.routers.ackee-insecure.entrypoints=web" - - - "traefik.http.routers.ackee.rule=Host(`tracking.luke-else.co.uk`)" - - "traefik.http.routers.ackee.entrypoints=websecure" - - "traefik.http.routers.ackee.tls.certresolver=myresolver" - - "traefik.http.routers.ackee.middlewares=cors-tracking" - depends_on: - - mongo - restart: unless-stopped - - mongo: - image: mongo - container_name: mongo-ackee - volumes: - - ./data:/data/db - networks: - - tracking - restart: unless-stopped - -networks: - proxy: - external: true - tracking: \ No newline at end of file diff --git a/Traefik/docker-compose.yml b/Traefik/docker-compose.yml deleted file mode 100644 index c2b613a..0000000 --- a/Traefik/docker-compose.yml +++ /dev/null @@ -1,46 +0,0 @@ -services: - traefik: - image: "traefik:latest" - container_name: "traefik" - command: - - "--api.dashboard=true" - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - - "--entrypoints.websecure.address=:443" - - "--entrypoints.mongo.address=:27017" - - "--entrypoints.web.transport.respondingTimeouts.readTimeout=120s" - - "--entrypoints.websecure.transport.respondingTimeouts.readTimeout=120s" - - "--entrypoints.web.transport.respondingTimeouts.writeTimeout=120s" - - "--entrypoints.websecure.transport.respondingTimeouts.writeTimeout=120s" - - "--certificatesresolvers.myresolver.acme.tlschallenge=true" - - "--certificatesresolvers.myresolver.acme.email=contact@luke-else.co.uk" - - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" - ports: - - "80:80" - - "443:443" - - "27017:27017" - volumes: - - "./letsencrypt:/letsencrypt" - - "/var/run/docker.sock:/var/run/docker.sock:ro" - networks: - - proxy - labels: - - "traefik.enable=true" - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.traefik-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.traefik-insecure.rule=Host(`traefik.luke-else.co.uk`)" - - "traefik.http.routers.traefik-insecure.entrypoints=web" - - - "traefik.http.routers.traefik.rule=Host(`traefik.luke-else.co.uk`)" - - "traefik.http.routers.traefik.entrypoints=websecure" - - "traefik.http.routers.traefik.service=api@internal" - - "traefik.http.routers.traefik.tls.certresolver=myresolver" - - "traefik.http.routers.traefik.middlewares=traefik-auth" - - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" - restart: unless-stopped - -networks: - proxy: - name: proxy \ No newline at end of file diff --git a/TraefikRunner/docker-compose.yml b/TraefikRunner/docker-compose.yml deleted file mode 100644 index e8970fd..0000000 --- a/TraefikRunner/docker-compose.yml +++ /dev/null @@ -1,40 +0,0 @@ -services: - traefik: - image: "traefik:latest" - container_name: "traefik" - command: - - "--api.dashboard=true" - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - - "--entrypoints.websecure.address=:443" - - "--certificatesresolvers.myresolver.acme.tlschallenge=true" - - "--certificatesresolvers.myresolver.acme.email=contact@luke-else.co.uk" - - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" - ports: - - "80:80" - - "443:443" - volumes: - - "./letsencrypt:/letsencrypt" - - "/var/run/docker.sock:/var/run/docker.sock:ro" - networks: - - proxy - labels: - - "traefik.enable=true" - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.traefik-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.traefik-insecure.rule=Host(`traefik.cicd.luke-else.co.uk`)" - - "traefik.http.routers.traefik-insecure.entrypoints=web" - - - "traefik.http.routers.traefik.rule=Host(`traefik.cicd.luke-else.co.uk`)" - - "traefik.http.routers.traefik.entrypoints=websecure" - - "traefik.http.routers.traefik.service=api@internal" - - "traefik.http.routers.traefik.tls.certresolver=myresolver" - - "traefik.http.routers.traefik.middlewares=traefik-auth" - - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" - restart: unless-stopped - -networks: - proxy: - name: proxy \ No newline at end of file diff --git a/TraefikVPN/docker-compose.yml b/TraefikVPN/docker-compose.yml deleted file mode 100644 index 7383006..0000000 --- a/TraefikVPN/docker-compose.yml +++ /dev/null @@ -1,40 +0,0 @@ -services: - traefik: - image: "traefik:latest" - container_name: "traefik" - command: - - "--api.dashboard=true" - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - - "--entrypoints.websecure.address=:443" - - "--certificatesresolvers.myresolver.acme.tlschallenge=true" - - "--certificatesresolvers.myresolver.acme.email=contact@luke-else.co.uk" - - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" - ports: - - "80:80" - - "443:443" - volumes: - - "./letsencrypt:/letsencrypt" - - "/var/run/docker.sock:/var/run/docker.sock:ro" - networks: - - proxy - labels: - - "traefik.enable=true" - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.traefik-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.traefik-insecure.rule=Host(`traefik.vpn.luke-else.co.uk`)" - - "traefik.http.routers.traefik-insecure.entrypoints=web" - - - "traefik.http.routers.traefik.rule=Host(`traefik.vpn.luke-else.co.uk`)" - - "traefik.http.routers.traefik.entrypoints=websecure" - - "traefik.http.routers.traefik.service=api@internal" - - "traefik.http.routers.traefik.tls.certresolver=myresolver" - - "traefik.http.routers.traefik.middlewares=traefik-auth" - - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" - restart: unless-stopped - -networks: - proxy: - name: proxy \ No newline at end of file diff --git a/VPN/docker-compose.yml b/VPN/docker-compose.yml deleted file mode 100644 index 502dcd8..0000000 --- a/VPN/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -services: - dockovpn: - image: alekslitvinenk/openvpn - cap_add: - - NET_ADMIN - ports: - - 1194:1194/udp # Expose tcp if you defined HOST_TUN_PROTOCOL=tcp - environment: - HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address - volumes: - - ./openvpn_conf:/opt/Dockovpn_data - labels: - ## Expose vpn Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.vpn-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.vpn-insecure.rule=Host(`vpn.luke-else.co.uk`)" - - "traefik.http.routers.vpn-insecure.entrypoints=web" - - - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)" - - "traefik.http.routers.vpn.entrypoints=websecure" - - "traefik.http.routers.vpn.tls.certresolver=myresolver" - restart: always \ No newline at end of file diff --git a/Watchtower/docker-compose.yml b/Watchtower/docker-compose.yml deleted file mode 100644 index ee735d3..0000000 --- a/Watchtower/docker-compose.yml +++ /dev/null @@ -1,8 +0,0 @@ -services: - watchtower: - image: nickfedor/watchtower - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - - WATCHTOWER_CLEANUP=true - - WATCHTOWER_POLL_INTERVAL=60 diff --git a/Websites/docker-compose.yml b/Websites/docker-compose.yml deleted file mode 100644 index ffc2c6b..0000000 --- a/Websites/docker-compose.yml +++ /dev/null @@ -1,103 +0,0 @@ -services: - -#Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080) - luke-else: - image: git.luke-else.co.uk/luke-else/luke-else.co.uk - container_name: luke-else - networks: - - proxy - labels: - ## Expose luke-else Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.personal-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.personal-insecure.rule=Host(`luke-else.co.uk`)" - - "traefik.http.routers.personal-insecure.entrypoints=web" - - - "traefik.http.routers.personal.rule=Host(`luke-else.co.uk`)" - - "traefik.http.routers.personal.entrypoints=websecure" - - "traefik.http.routers.personal.tls.certresolver=myresolver" - restart: unless-stopped - - luke-else-dev: - image: git.luke-else.co.uk/luke-else/luke-else.co.uk:dev - container_name: luke-else-dev - networks: - - proxy - labels: - ## Expose luke-else Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.personal-dev-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.personal-dev-insecure.rule=Host(`dev.luke-else.co.uk`)" - - "traefik.http.routers.personal-dev-insecure.entrypoints=web" - - - "traefik.http.routers.personal-dev.rule=Host(`dev.luke-else.co.uk`)" - - "traefik.http.routers.personal-dev.entrypoints=websecure" - - "traefik.http.routers.personal-dev.tls.certresolver=myresolver" - restart: unless-stopped - - metarius: - image: git.luke-else.co.uk/luke-else/metarius:latest - container_name: metarius - networks: - - proxy - labels: - ## Expose metarius Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.metarius-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.metarius-insecure.rule=Host(`metarius.luke-else.co.uk`)" - - "traefik.http.routers.metarius-insecure.entrypoints=web" - - - "traefik.http.routers.metarius.rule=Host(`metarius.luke-else.co.uk`)" - - "traefik.http.routers.metarius.entrypoints=websecure" - - "traefik.http.routers.metarius.tls.certresolver=myresolver" - restart: unless-stopped - - divine-couture: - image: git.luke-else.co.uk/luke-else/divine-couture.co.uk:latest - container_name: divine-couture - networks: - - proxy - labels: - ## Expose divine-couture Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.divine-couture-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.divine-couture-insecure.rule=Host(`www.divine-couture.co.uk`)" - - "traefik.http.routers.divine-couture-insecure.entrypoints=web" - - - "traefik.http.routers.divine-couture.rule=Host(`www.divine-couture.co.uk`)" - - "traefik.http.routers.divine-couture.entrypoints=websecure" - - "traefik.http.routers.divine-couture.tls.certresolver=myresolver" - restart: unless-stopped - - snexo: - image: "php:apache" - container_name: snexo - volumes: - - ./snexo.co.uk/:/var/www/html - networks: - - proxy - labels: - ## Expose Snexo Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.snexo-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.snexo-insecure.rule=Host(`snexo.co.uk`)" - - "traefik.http.routers.snexo-insecure.entrypoints=web" - - - "traefik.http.routers.snexo.rule=Host(`snexo.co.uk`)" - - "traefik.http.routers.snexo.entrypoints=websecure" - - "traefik.http.routers.snexo.tls.certresolver=myresolver" - restart: unless-stopped - -networks: - proxy: - external: true diff --git a/cicd-spindown.sh b/cicd-spindown.sh deleted file mode 100755 index 1e3d49c..0000000 --- a/cicd-spindown.sh +++ /dev/null @@ -1,16 +0,0 @@ -#Script file for spinning down all CICD relevant docker-containers - cd ./Development/Runners/ - docker compose down - cd ../.. - -cd ./TraefikRunner/ -docker compose down -cd .. - -cd ./Watchtower/ -docker compose down -cd .. - -docker rmi $(docker images -q) -docker system prune -f -a -docker volume prune -f -a \ No newline at end of file diff --git a/cicd-spinup.sh b/cicd-spinup.sh deleted file mode 100755 index 6a3b139..0000000 --- a/cicd-spinup.sh +++ /dev/null @@ -1,13 +0,0 @@ -#Script file for spinning up all docker-containers - -cd ./TraefikRunner/ -docker compose pull && docker compose up -d -cd .. - - cd ./Development/Runners/ - docker compose pull && docker compose up -d - cd ../.. - -cd ./Watchtower/ -docker compose pull && docker compose up -d -cd .. \ No newline at end of file diff --git a/main-spindown.sh b/main-spindown.sh deleted file mode 100755 index 1012835..0000000 --- a/main-spindown.sh +++ /dev/null @@ -1,42 +0,0 @@ -#Script file for spinning down all docker-containers - -cd ./Websites/ -docker compose down -cd .. - -cd ./Tracking/ -docker compose down -cd .. - - - cd ./Development/Gitea/ - docker compose down - cd ../.. - -cd ./Database/ -docker compose down -cd .. - -cd ./Bitwarden/ -docker compose down -cd .. - -cd ./Rustdesk/ -docker compose down -cd .. - -cd ./Misc/ -docker compose down -cd .. - -cd ./Watchtower/ -docker compose down -cd .. - -cd ./Traefik/ -docker compose down -cd .. - -docker rmi $(docker images -q) -docker system prune -f -a -docker volume prune -f -a \ No newline at end of file diff --git a/main-spinup.sh b/main-spinup.sh deleted file mode 100755 index d8f8a03..0000000 --- a/main-spinup.sh +++ /dev/null @@ -1,39 +0,0 @@ -#Script file for spinning up all docker-containers - -cd ./Traefik/ -docker compose pull && docker compose up -d -cd .. - - cd ./Development/Gitea/ - docker compose pull && docker compose up -d - cd ../.. - -sleep 20 # Allow Gitea + registry to start up before starting the rest of the services - -cd ./Watchtower/ -docker compose pull && docker compose up -d -cd .. - -cd ./Tracking/ -docker compose pull && docker compose up -d -cd .. - -cd ./Websites/ -docker compose pull && docker compose up -d -cd .. - -cd ./Database/ -docker compose pull && docker compose up -d -cd .. - -cd ./Bitwarden/ -docker compose pull && docker compose up -d -cd .. - -cd ./Rustdesk/ -docker compose pull && docker compose up -d -cd .. - -cd ./Misc/ -docker compose pull && docker compose up -d -cd .. \ No newline at end of file diff --git a/todo.md b/todo.md deleted file mode 100644 index a53d118..0000000 --- a/todo.md +++ /dev/null @@ -1,64 +0,0 @@ -# ToDo Items - -## General - -- Setup non root user -- UFW should be setup to keep VPS secure and only allow for: - - - https - - http - - ssh - - ftp - - 27017 - - - "21115:21115/tcp" - - "21116:21116/tcp" - - "21116:21116/udp" - - "21117:21117/tcp" - - "21119:21119/tcp" - -- Install SSH keys -- Setup unattended upgrades -- Install docker, docker-compose and apache utils. - -## Traefik + TraefikRunner + Traefik VPN - -- Setup htaccess -> `echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g` -- Ensure email address is correct - -## Gitea - -- Ensure that ports are assigned correctly for the system - -# Gitea Runner - -- Ensure that a registration token has been setup before continuing - -## Tracking - -Create a .env file with the following content: - -```sh -ACKEE_USERNAME=luke-else -ACKEE_PASSWORD=XXX -``` - -## Websites - -- Ensure website files are copied over -- Ensure that ports are assigned correctly for the system - -## Bitwarden - -- Ensure that all data is fully encrypted during transfer. -- Ensure that ports are assigned correctly for the system - -## Rustdesk - -- No additional setup required bar the ports - -## Database - -- Ensure that mysql root password, user and default database are updated. -- Ensure that mongo root password, and user are updated. -- Ensure database ports are correctly assigned and do not have to pass through traefik. diff --git a/vpn-spindown.sh b/vpn-spindown.sh deleted file mode 100755 index 7cd9ef1..0000000 --- a/vpn-spindown.sh +++ /dev/null @@ -1,17 +0,0 @@ -#Script file for spinning down all CICD relevant docker-containers - -cd ./VPN/ -docker compose down -cd .. - -cd ./TraefikVPN/ -docker compose down -cd .. - -cd ./Watchtower/ -docker compose down -cd .. - -docker rmi $(docker images -q) -docker system prune -f -a -docker volume prune -f -a \ No newline at end of file diff --git a/vpn-spinup.sh b/vpn-spinup.sh deleted file mode 100755 index 4cc5159..0000000 --- a/vpn-spinup.sh +++ /dev/null @@ -1,13 +0,0 @@ -#Script file for spinning up all docker-containers - -cd ./TraefikRunner/ -docker compose pull && docker compose up -d -cd .. - -cd ./VPN/ -docker compose pull && docker compose up -d -cd .. - -cd ./Watchtower/ -docker compose pull && docker compose up -d -cd .. \ No newline at end of file From 548671698125b914fa89841c2756b7106b907e87 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Thu, 9 Jul 2026 20:53:59 +0100 Subject: [PATCH 02/24] chore: Updated dependency on devcontainer --- .devcontainer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer b/.devcontainer index 192a20c..424e184 160000 --- a/.devcontainer +++ b/.devcontainer @@ -1 +1 @@ -Subproject commit 192a20c759769b050f9efa89179baf7953bae65c +Subproject commit 424e184c8fcdeb8b9b742568ee107bb0a9712aba From 3bfa0e6f3255f6773331d20a1039a2ee42f2e048 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Thu, 9 Jul 2026 21:55:50 +0100 Subject: [PATCH 03/24] feat: Added terraform config... just a basic one at first --- .gitignore | 43 ++++- docs/architecture.md | 33 ++++ infra/.terraform.lock.hcl | 35 ++++ infra/firewalls.tf | 162 ++++++++++++++++++ infra/network.tf | 13 ++ infra/outputs.tf | 19 ++ infra/servers.tf | 43 +++++ infra/ssh.tf | 3 + infra/terraform.tfvars.example | 18 ++ infra/variables.tf | 82 +++++++++ infra/versions.tf | 13 ++ infra/volumes.tf | 17 ++ services/Bitwarden/docker-compose.yml | 27 +++ services/Database/docker-compose.yml | 24 +++ services/Development/Gitea/docker-compose.yml | 54 ++++++ services/Development/Runners/config.yaml | 0 .../Development/Runners/docker-compose.yml | 33 ++++ services/Misc/docker-compose.yml | 51 ++++++ services/Rustdesk/docker-compose.yml | 53 ++++++ services/Tracking/docker-compose.yml | 52 ++++++ services/Traefik/docker-compose.yml | 46 +++++ services/TraefikRunner/docker-compose.yml | 40 +++++ services/TraefikVPN/docker-compose.yml | 40 +++++ services/VPN/docker-compose.yml | 24 +++ services/Watchtower/docker-compose.yml | 8 + services/Websites/docker-compose.yml | 103 +++++++++++ services/cicd-spindown.sh | 16 ++ services/cicd-spinup.sh | 13 ++ services/main-spindown.sh | 42 +++++ services/main-spinup.sh | 39 +++++ services/todo.md | 64 +++++++ services/vpn-spindown.sh | 17 ++ services/vpn-spinup.sh | 13 ++ 33 files changed, 1235 insertions(+), 5 deletions(-) create mode 100644 docs/architecture.md create mode 100644 infra/.terraform.lock.hcl create mode 100644 infra/firewalls.tf create mode 100644 infra/network.tf create mode 100644 infra/outputs.tf create mode 100644 infra/servers.tf create mode 100644 infra/ssh.tf create mode 100644 infra/terraform.tfvars.example create mode 100644 infra/variables.tf create mode 100644 infra/versions.tf create mode 100644 infra/volumes.tf create mode 100644 services/Bitwarden/docker-compose.yml create mode 100644 services/Database/docker-compose.yml create mode 100644 services/Development/Gitea/docker-compose.yml create mode 100644 services/Development/Runners/config.yaml create mode 100644 services/Development/Runners/docker-compose.yml create mode 100644 services/Misc/docker-compose.yml create mode 100644 services/Rustdesk/docker-compose.yml create mode 100644 services/Tracking/docker-compose.yml create mode 100644 services/Traefik/docker-compose.yml create mode 100644 services/TraefikRunner/docker-compose.yml create mode 100644 services/TraefikVPN/docker-compose.yml create mode 100644 services/VPN/docker-compose.yml create mode 100644 services/Watchtower/docker-compose.yml create mode 100644 services/Websites/docker-compose.yml create mode 100755 services/cicd-spindown.sh create mode 100755 services/cicd-spinup.sh create mode 100755 services/main-spindown.sh create mode 100755 services/main-spinup.sh create mode 100644 services/todo.md create mode 100755 services/vpn-spindown.sh create mode 100755 services/vpn-spinup.sh diff --git a/.gitignore b/.gitignore index 8c4ebe1..836aa13 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,45 @@ -# Ignore all files -** +# Ignore all files related to services +**/services # Allow docker-compose.yml !**/docker-compose.yml !/* -# Allow assets folder -!assets/** - # Don't allow .env files, just env-exmaple files !.env-example + +### Terraform ### +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc \ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..1395fb4 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,33 @@ +```mermaid +architecture-beta + group cloud(cloud)[Hetzner] + + group network(cloud)[network] in cloud + + service disk1(mdi:disk)[Storage] in cloud + service disk2(mdi:disk)[Storage] in cloud + + service dev(mdi:server)[dev] in network + + service prod(mdi:server)[prod] in network + service prodfirewall(mdi:firewall)[firewall] in cloud + + service vpn(mdi:server)[vpn] in cloud + service vpnfirewall(mdi:firewall)[firewall] in cloud + + + + service gateway(mdi:web)[gateway] in cloud + + dev:L -- R:prod + disk1:B -- T:prod + disk2:B -- T:dev + + + prod:B -- T:prodfirewall + vpn:B -- T:vpnfirewall + + prodfirewall: L -- R: gateway + vpnfirewall: B -- T: gateway + +``` \ No newline at end of file diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 0000000..37ba449 --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,35 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hetznercloud/hcloud" { + version = "1.66.0" + constraints = "~> 1.45" + hashes = [ + "h1:+R3t/5wqCb2ics5bNZ4JgtfaAa08Z6uRzXzNtwovVEY=", + "h1:9c1ECk3N9xhdQKz4NVCfYEFxebJVVeQZS3E6wrR/Hx8=", + "h1:EtYB/KjPN2cIJa6hpEaVc8q/3yf6e9IT6/uAybUGqps=", + "h1:GUupu6Fg5zFY9i4/3esiy8uHZnUrt+u2B6NaE4+8YqQ=", + "h1:JHehRJySEFRGYK+D6pFS0MTTDDRlGSQdSaKdEo0AemA=", + "h1:XF+SwyO1ttQJ0WXJ5O628O7cGk9U37sGn0o3LwWUnDQ=", + "h1:czuvCXAd8CcszXh3k3E0N6tFfQkYSNbKXo99LG+yOIc=", + "h1:dPIwO6zTxYs6bHn5yf/w/1AExaHPppCLyTrWrdzH4RE=", + "h1:gqrhnYuIBzpUvgWHITNRUMLaDcKACGlrNfrbtbtO7KY=", + "h1:iVAGP8gRbZK0kJF7SiYJRt61wz0D5AF9q+WMsrAiBI0=", + "h1:kLi29SbGCU/Z/Ch0zuNdYFSKP3mHp8zfcOpZsIN/KIQ=", + "h1:mq2+1Q/5gWJYI8XOXQCeteUg0AP7VezPOGOjkkHRQso=", + "h1:rnWTWAOlJhHtD3kkU4Qfw0bg9ko/dlH822inxYLBmfQ=", + "zh:1286cee6fb63dbcb18f53077bbb5e5d132a4e4d9f006af4e8d8edfc08d6bcdc8", + "zh:204460dacc044bda019a4a18b398e094289500c36913c7c9457f432adf31b8b2", + "zh:214175d50773481cbeaf9c9004e4121a3a1c9686c79424ebdc8ff189dd057d3e", + "zh:22b17bceff61cc13ad04a399ba87521356a3a134d4687273727473ae9eccf5f1", + "zh:368867dac5525c411de7e38f2e27de0a71854d1750867322ff2b9321128c88fb", + "zh:5289b75f8370bdbc4c6051d55cf33d0b1bd25dc6d71bfbd39b360249a37f1501", + "zh:81cb676aa50c5777df8fc80d4e69c9012330ae751f5e6f12bf6074bfd2e7c496", + "zh:ab08aead10643b21aa6b51af562b50492e12b9dd0ab7dca27a05aa63209b7d66", + "zh:af25c210d0570cf61ef767b2545bf9f3fb909178135f0e5e14bec0c1c9d07a63", + "zh:bcad66f4830c97118fa793723e53f8a4d27ddd34ea969ff259408842c2238331", + "zh:ce3ed323d75ae905d975925fa98c7054a7514c81276a485fc37da8232b53e39f", + "zh:d481bc0ef0c87ab1969c17777f526b2f59f823432d676145134c41a6d29bd98e", + "zh:ea7ef88df2c3ca154d86238920636d52a3c9066c7467543d3fa45f1e52ec2f7b", + ] +} diff --git a/infra/firewalls.tf b/infra/firewalls.tf new file mode 100644 index 0000000..807f33e --- /dev/null +++ b/infra/firewalls.tf @@ -0,0 +1,162 @@ +# Port sources: services/*/docker-compose.yml (published ports) and services/todo.md +# (the documented UFW allow-list). dev has no firewall in architecture.md, but we add +# one anyway for baseline safety - see conversation history. + +resource "hcloud_firewall" "dev" { + name = "dev-firewall" + + rule { # server ssh + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.allowed_ssh_source_ips + } + + rule { # gitea ssh (git.luke-else.co.uk, published as 222:22) + direction = "in" + protocol = "tcp" + port = "222" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # TraefikRunner http/https (git.luke-else.co.uk, cicd.luke-else.co.uk) + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # traffic from prod over the private network + direction = "in" + protocol = "tcp" + port = "1-65535" + source_ips = [var.network_ip_range] + } + + rule { + direction = "in" + protocol = "udp" + port = "1-65535" + source_ips = [var.network_ip_range] + } +} + +resource "hcloud_firewall" "prod" { + name = "prod-firewall" + + rule { # server ssh + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.allowed_ssh_source_ips + } + + rule { # Traefik http/https (Websites, Bitwarden, Misc, Tracking) + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # mongo via Traefik tcp entrypoint (Database) + direction = "in" + protocol = "tcp" + port = "27017" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # rustdesk hbbs + direction = "in" + protocol = "tcp" + port = "21115" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "21116" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "udp" + port = "21116" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # rustdesk hbbr + direction = "in" + protocol = "tcp" + port = "21117" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "21119" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # traffic from dev over the private network + direction = "in" + protocol = "tcp" + port = "1-65535" + source_ips = [var.network_ip_range] + } + + rule { + direction = "in" + protocol = "udp" + port = "1-65535" + source_ips = [var.network_ip_range] + } +} + +resource "hcloud_firewall" "vpn" { + name = "vpn-firewall" + + rule { # server ssh + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.allowed_ssh_source_ips + } + + rule { # TraefikVPN http/https (traefik.vpn.luke-else.co.uk) + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # OpenVPN tunnel - always direct to this server's public IP, never via a load balancer + direction = "in" + protocol = "udp" + port = "1194" + source_ips = ["0.0.0.0/0", "::/0"] + } +} diff --git a/infra/network.tf b/infra/network.tf new file mode 100644 index 0000000..29fad1b --- /dev/null +++ b/infra/network.tf @@ -0,0 +1,13 @@ +# Private network shared by dev and prod (see architecture.md "network" group). +# vpn is intentionally not attached to this network - it sits outside it in the diagram. +resource "hcloud_network" "main" { + name = "server-network" + ip_range = var.network_ip_range +} + +resource "hcloud_network_subnet" "main" { + network_id = hcloud_network.main.id + type = "cloud" + network_zone = var.network_zone + ip_range = var.subnet_ip_range +} diff --git a/infra/outputs.tf b/infra/outputs.tf new file mode 100644 index 0000000..68041a3 --- /dev/null +++ b/infra/outputs.tf @@ -0,0 +1,19 @@ +output "dev_ipv4" { + value = hcloud_server.dev.ipv4_address +} + +output "dev_private_ipv4" { + value = var.dev_private_ip +} + +output "prod_ipv4" { + value = hcloud_server.prod.ipv4_address +} + +output "prod_private_ipv4" { + value = var.prod_private_ip +} + +output "vpn_ipv4" { + value = hcloud_server.vpn.ipv4_address +} diff --git a/infra/servers.tf b/infra/servers.tf new file mode 100644 index 0000000..18bb96f --- /dev/null +++ b/infra/servers.tf @@ -0,0 +1,43 @@ +# dev: Gitea + Runner + TraefikRunner (git.luke-else.co.uk, cicd.luke-else.co.uk) +resource "hcloud_server" "dev" { + name = "dev" + server_type = var.dev_server_type + image = var.server_image + location = var.location + ssh_keys = [data.hcloud_ssh_key.main.id] + firewall_ids = [hcloud_firewall.dev.id] + + network { + network_id = hcloud_network.main.id + ip = var.dev_private_ip + } + + depends_on = [hcloud_network_subnet.main] +} + +# prod: Traefik, Websites, Database, Bitwarden, Misc, Tracking, Rustdesk +resource "hcloud_server" "prod" { + name = "prod" + server_type = var.prod_server_type + image = var.server_image + location = var.location + ssh_keys = [data.hcloud_ssh_key.main.id] + firewall_ids = [hcloud_firewall.prod.id] + + network { + network_id = hcloud_network.main.id + ip = var.prod_private_ip + } + + depends_on = [hcloud_network_subnet.main] +} + +# vpn: OpenVPN + TraefikVPN. Not attached to the private network (see architecture.md). +resource "hcloud_server" "vpn" { + name = "vpn" + server_type = var.vpn_server_type + image = var.server_image + location = var.location + ssh_keys = [data.hcloud_ssh_key.main.id] + firewall_ids = [hcloud_firewall.vpn.id] +} diff --git a/infra/ssh.tf b/infra/ssh.tf new file mode 100644 index 0000000..a6eea7e --- /dev/null +++ b/infra/ssh.tf @@ -0,0 +1,3 @@ +data "hcloud_ssh_key" "main" { + name = var.ssh_key_name +} diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example new file mode 100644 index 0000000..bed6ff2 --- /dev/null +++ b/infra/terraform.tfvars.example @@ -0,0 +1,18 @@ +# Copy to terraform.tfvars and fill in - terraform.tfvars itself is gitignored, +# never commit real values there. +# +# The Hetzner API token is NOT set here: export it as HCLOUD_TOKEN in your shell +# before running tofu plan/apply. + +# Name of an SSH key already uploaded to your Hetzner Cloud project +# (Console > Security > SSH Keys). Required. +ssh_key_name = "your-key-name" + +# Optional overrides - defaults live in variables.tf +# location = "nbg1" +# dev_server_type = "cx22" +# prod_server_type = "cx32" +# vpn_server_type = "cx22" +# dev_volume_size = 50 +# prod_volume_size = 50 +# allowed_ssh_source_ips = ["203.0.113.4/32"] diff --git a/infra/variables.tf b/infra/variables.tf new file mode 100644 index 0000000..53478de --- /dev/null +++ b/infra/variables.tf @@ -0,0 +1,82 @@ +variable "location" { + description = "Hetzner Cloud datacenter location for all servers and volumes." + type = string + default = "nbg1" +} + +variable "network_zone" { + description = "Hetzner Cloud network zone matching var.location." + type = string + default = "eu-central" +} + +variable "server_image" { + description = "OS image used for all servers." + type = string + default = "ubuntu-24.04" +} + +variable "dev_server_type" { + description = "Server type for dev (Gitea + Runner)." + type = string + default = "cx22" +} + +variable "prod_server_type" { + description = "Server type for prod (Traefik, Websites, Database, Bitwarden, Misc, Tracking, Rustdesk)." + type = string + default = "cx32" +} + +variable "vpn_server_type" { + description = "Server type for vpn (OpenVPN + TraefikVPN)." + type = string + default = "cx22" +} + +variable "dev_volume_size" { + description = "Size in GB of the volume attached to dev (disk2 in architecture.md)." + type = number + default = 50 +} + +variable "prod_volume_size" { + description = "Size in GB of the volume attached to prod (disk1 in architecture.md)." + type = number + default = 50 +} + +variable "network_ip_range" { + description = "IP range of the private network shared by dev and prod." + type = string + default = "10.0.0.0/16" +} + +variable "subnet_ip_range" { + description = "IP range of the network subnet shared by dev and prod." + type = string + default = "10.0.1.0/24" +} + +variable "dev_private_ip" { + description = "Private network IP for dev." + type = string + default = "10.0.1.10" +} + +variable "prod_private_ip" { + description = "Private network IP for prod." + type = string + default = "10.0.1.11" +} + +variable "ssh_key_name" { + description = "Name of an SSH key already uploaded to your Hetzner Cloud project (Console > Security > SSH Keys)." + type = string +} + +variable "allowed_ssh_source_ips" { + description = "CIDRs allowed to reach port 22 on every server. Narrow this to your own IP(s) once known." + type = list(string) + default = ["0.0.0.0/0", "::/0"] +} diff --git a/infra/versions.tf b/infra/versions.tf new file mode 100644 index 0000000..e01c365 --- /dev/null +++ b/infra/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.6.0" + + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + version = "~> 1.45" + } + } +} + +# Reads the token from the HCLOUD_TOKEN environment variable. +provider "hcloud" {} diff --git a/infra/volumes.tf b/infra/volumes.tf new file mode 100644 index 0000000..5a8863c --- /dev/null +++ b/infra/volumes.tf @@ -0,0 +1,17 @@ +# disk1 in architecture.md - attached to prod +resource "hcloud_volume" "prod_storage" { + name = "prod-storage" + size = var.prod_volume_size + server_id = hcloud_server.prod.id + automount = true + format = "ext4" +} + +# disk2 in architecture.md - attached to dev +resource "hcloud_volume" "dev_storage" { + name = "dev-storage" + size = var.dev_volume_size + server_id = hcloud_server.dev.id + automount = true + format = "ext4" +} diff --git a/services/Bitwarden/docker-compose.yml b/services/Bitwarden/docker-compose.yml new file mode 100644 index 0000000..91c1ffe --- /dev/null +++ b/services/Bitwarden/docker-compose.yml @@ -0,0 +1,27 @@ +services: + +#Bitwarden () + bitwarden: + image: "vaultwarden/server:latest" + container_name: vaultwarden + volumes: + - ./bitwarden/:/data/ + networks: + - proxy + labels: + ## Expose Bitwarden Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.bitwarden-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.bitwarden-insecure.rule=Host(`bitwarden.luke-else.co.uk`)" + - "traefik.http.routers.bitwarden-insecure.entrypoints=web" + + - "traefik.http.routers.bitwarden.rule=Host(`bitwarden.luke-else.co.uk`)" + - "traefik.http.routers.bitwarden.entrypoints=websecure" + - "traefik.http.routers.bitwarden.tls.certresolver=myresolver" + restart: unless-stopped + +networks: + proxy: + external: true \ No newline at end of file diff --git a/services/Database/docker-compose.yml b/services/Database/docker-compose.yml new file mode 100644 index 0000000..3d1a9a3 --- /dev/null +++ b/services/Database/docker-compose.yml @@ -0,0 +1,24 @@ +services: + +#MongoDB (27017) + mongodb: + image: "mongo:latest" + container_name: mongoDB + volumes: + - ./mongo/:/data/db + networks: + - proxy + labels: + - "traefik.enable=true" + - "traefik.tcp.routers.mongodb.rule=HostSNI(`*`)" + - "traefik.tcp.routers.mongodb.entrypoints=mongo" + - "traefik.tcp.routers.mongodb.service=mongodb" + - "traefik.tcp.services.mongodb.loadbalancer.server.port=27017" + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: rootpassword + restart: unless-stopped + +networks: + proxy: + external: true \ No newline at end of file diff --git a/services/Development/Gitea/docker-compose.yml b/services/Development/Gitea/docker-compose.yml new file mode 100644 index 0000000..3724a79 --- /dev/null +++ b/services/Development/Gitea/docker-compose.yml @@ -0,0 +1,54 @@ +services: + #gitea (222) + gitea: + image: gitea/gitea:latest + container_name: gitea + volumes: + - ./gitea:/data + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + networks: + - proxy + ports: + - "222:22" + environment: + - APP_NAME="gitea" + - USER_UID=1000 + - USER_GID=1000 + - USER=git + - RUN_MODE=prod + - DOMAIN=git.luke-else.co.uk + - SSH_DOMAIN=git.luke-else.co.uk + - HTTP_PORT=3000 + - ROOT_URL=https://git.luke-else.co.uk + - SSH_PORT=222 + - SSH_LISTEN_PORT=22 + - DB_TYPE=sqlite3 + - GITEA_service_DISABLE_REGISTRATION=true + - GITEA_server_LANDING_PAGE=/luke-else + labels: + ## Expose Gitea Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowmethods=*" + - "traefik.http.middlewares.cors-gitea.headers.accesscontrolalloworiginlist=*" + - "traefik.http.middlewares.cors-gitea.headers.addvaryheader=true" + - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowcredentials=true" + - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowheaders=Content-Type,Authorization" + - "traefik.http.middlewares.cors-gitea.headers.accesscontrolmaxage=100" + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.gitea-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.gitea-insecure.rule=Host(`git.luke-else.co.uk`)" + - "traefik.http.routers.gitea-insecure.entrypoints=web" + + - "traefik.http.services.gitea.loadbalancer.server.port=3000" + - "traefik.http.routers.gitea.rule=Host(`git.luke-else.co.uk`)" + - "traefik.http.routers.gitea.entrypoints=websecure" + - "traefik.http.routers.gitea.tls.certresolver=myresolver" + - "traefik.http.routers.gitea.middlewares=cors-gitea" + restart: unless-stopped + +networks: + proxy: + external: true \ No newline at end of file diff --git a/services/Development/Runners/config.yaml b/services/Development/Runners/config.yaml new file mode 100644 index 0000000..e69de29 diff --git a/services/Development/Runners/docker-compose.yml b/services/Development/Runners/docker-compose.yml new file mode 100644 index 0000000..f9f5acd --- /dev/null +++ b/services/Development/Runners/docker-compose.yml @@ -0,0 +1,33 @@ +services: + # Runner + runner: + image: gitea/act_runner:latest + container_name: gitea_runner + volumes: + - ./config.yaml:/config.yaml + - ./gitea_runner:/data + - /var/run/docker.sock:/var/run/docker.sock + networks: + - proxy + environment: + CONFIG_FILE: /config.yaml + GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" + GITEA_RUNNER_REGISTRATION_TOKEN: "INSERT REGISTRATION TOKEN" + GITEA_RUNNER_NAME: "CICD" + labels: + ## Expose cicd Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.cicd-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.cicd-insecure.rule=Host(`cicd.luke-else.co.uk`)" + - "traefik.http.routers.cicd-insecure.entrypoints=web" + + - "traefik.http.routers.cicd.rule=Host(`cicd.luke-else.co.uk`)" + - "traefik.http.routers.cicd.entrypoints=websecure" + - "traefik.http.routers.cicd.tls.certresolver=myresolver" + restart: unless-stopped + +networks: + proxy: + external: true \ No newline at end of file diff --git a/services/Misc/docker-compose.yml b/services/Misc/docker-compose.yml new file mode 100644 index 0000000..7914b79 --- /dev/null +++ b/services/Misc/docker-compose.yml @@ -0,0 +1,51 @@ +services: + status: + image: louislam/uptime-kuma:latest + container_name: status + volumes: + - ./uptime-kuma/data:/app/data + networks: + - proxy + labels: + ## Expose uptime-kuma Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)" + - "traefik.http.routers.status-insecure.entrypoints=web" + + - "traefik.http.routers.status.rule=Host(`status.luke-else.co.uk`)" + - "traefik.http.routers.status.entrypoints=websecure" + - "traefik.http.routers.status.tls.certresolver=myresolver" + restart: unless-stopped + + portainer: + image: portainer/portainer-ce:latest + container_name: portainer + security_opt: + - no-new-privileges:true + volumes: + - /etc/localtime:/etc/localtime:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./portainer-data:/data + networks: + - proxy + labels: + ## Expose portainer Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.portainer-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.portainer-insecure.rule=Host(`portainer.luke-else.co.uk`)" + - "traefik.http.routers.portainer-insecure.entrypoints=web" + + - "traefik.http.routers.portainer.rule=Host(`portainer.luke-else.co.uk`)" + - "traefik.http.services.portainer.loadbalancer.server.port=9000" + - "traefik.http.routers.portainer.entrypoints=websecure" + - "traefik.http.routers.portainer.tls.certresolver=myresolver" + restart: unless-stopped + +networks: + proxy: + external: true \ No newline at end of file diff --git a/services/Rustdesk/docker-compose.yml b/services/Rustdesk/docker-compose.yml new file mode 100644 index 0000000..4dc4145 --- /dev/null +++ b/services/Rustdesk/docker-compose.yml @@ -0,0 +1,53 @@ +services: + hbbs: + image: rustdesk/rustdesk-server:latest + container_name: rustdesk-hbbs + command: hbbs + restart: unless-stopped + + volumes: + - ./data:/root + + networks: + - proxy + + ports: + - "21115:21115/tcp" + - "21116:21116/tcp" + - "21116:21116/udp" + # - "21118:21118/tcp" + + labels: + - "traefik.enable=true" + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.rustdesk-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.rustdesk-insecure.rule=Host(`rd.luke-else.co.uk`)" + - "traefik.http.routers.rustdesk-insecure.entrypoints=web" + + - "traefik.http.routers.rustdesk.rule=Host(`rd.luke-else.co.uk`)" + - "traefik.http.routers.rustdesk.entrypoints=websecure" + - "traefik.http.routers.rustdesk.tls.certresolver=myresolver" + + # Service + - "traefik.http.services.rustdesk.loadbalancer.server.port=21118" + + hbbr: + image: rustdesk/rustdesk-server:latest + container_name: rustdesk-hbbr + command: hbbr + restart: unless-stopped + + volumes: + - ./data:/root + + networks: + - proxy + + ports: + - "21117:21117/tcp" + - "21119:21119/tcp" + +networks: + proxy: + external: true \ No newline at end of file diff --git a/services/Tracking/docker-compose.yml b/services/Tracking/docker-compose.yml new file mode 100644 index 0000000..6fe612a --- /dev/null +++ b/services/Tracking/docker-compose.yml @@ -0,0 +1,52 @@ +services: + ackee: + image: electerious/ackee + container_name: ackee + environment: + - WAIT_HOSTS=mongo:27017 + - ACKEE_MONGODB=mongodb://mongo-ackee:27017/ackee + env_file: + - .env + expose: + - 3000 + networks: + - tracking + - proxy + labels: + ## Expose Ackee Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + # Enable CORS headers + - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowmethods=*" + - "traefik.http.middlewares.cors-tracking.headers.accesscontrolalloworiginlist=https://luke-else.co.uk" + - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowcredentials=true" + - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowheaders=Content-Type,Authorization" + - "traefik.http.middlewares.cors-tracking.headers.addvaryheader=true" + - "traefik.http.middlewares.cors-tracking.headers.accesscontrolmaxage=100" + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.ackee-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.ackee-insecure.rule=Host(`tracking.luke-else.co.uk`)" + - "traefik.http.routers.ackee-insecure.entrypoints=web" + + - "traefik.http.routers.ackee.rule=Host(`tracking.luke-else.co.uk`)" + - "traefik.http.routers.ackee.entrypoints=websecure" + - "traefik.http.routers.ackee.tls.certresolver=myresolver" + - "traefik.http.routers.ackee.middlewares=cors-tracking" + depends_on: + - mongo + restart: unless-stopped + + mongo: + image: mongo + container_name: mongo-ackee + volumes: + - ./data:/data/db + networks: + - tracking + restart: unless-stopped + +networks: + proxy: + external: true + tracking: \ No newline at end of file diff --git a/services/Traefik/docker-compose.yml b/services/Traefik/docker-compose.yml new file mode 100644 index 0000000..c2b613a --- /dev/null +++ b/services/Traefik/docker-compose.yml @@ -0,0 +1,46 @@ +services: + traefik: + image: "traefik:latest" + container_name: "traefik" + command: + - "--api.dashboard=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--entrypoints.mongo.address=:27017" + - "--entrypoints.web.transport.respondingTimeouts.readTimeout=120s" + - "--entrypoints.websecure.transport.respondingTimeouts.readTimeout=120s" + - "--entrypoints.web.transport.respondingTimeouts.writeTimeout=120s" + - "--entrypoints.websecure.transport.respondingTimeouts.writeTimeout=120s" + - "--certificatesresolvers.myresolver.acme.tlschallenge=true" + - "--certificatesresolvers.myresolver.acme.email=contact@luke-else.co.uk" + - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" + ports: + - "80:80" + - "443:443" + - "27017:27017" + volumes: + - "./letsencrypt:/letsencrypt" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + networks: + - proxy + labels: + - "traefik.enable=true" + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.traefik-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.traefik-insecure.rule=Host(`traefik.luke-else.co.uk`)" + - "traefik.http.routers.traefik-insecure.entrypoints=web" + + - "traefik.http.routers.traefik.rule=Host(`traefik.luke-else.co.uk`)" + - "traefik.http.routers.traefik.entrypoints=websecure" + - "traefik.http.routers.traefik.service=api@internal" + - "traefik.http.routers.traefik.tls.certresolver=myresolver" + - "traefik.http.routers.traefik.middlewares=traefik-auth" + - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" + restart: unless-stopped + +networks: + proxy: + name: proxy \ No newline at end of file diff --git a/services/TraefikRunner/docker-compose.yml b/services/TraefikRunner/docker-compose.yml new file mode 100644 index 0000000..e8970fd --- /dev/null +++ b/services/TraefikRunner/docker-compose.yml @@ -0,0 +1,40 @@ +services: + traefik: + image: "traefik:latest" + container_name: "traefik" + command: + - "--api.dashboard=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--certificatesresolvers.myresolver.acme.tlschallenge=true" + - "--certificatesresolvers.myresolver.acme.email=contact@luke-else.co.uk" + - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" + ports: + - "80:80" + - "443:443" + volumes: + - "./letsencrypt:/letsencrypt" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + networks: + - proxy + labels: + - "traefik.enable=true" + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.traefik-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.traefik-insecure.rule=Host(`traefik.cicd.luke-else.co.uk`)" + - "traefik.http.routers.traefik-insecure.entrypoints=web" + + - "traefik.http.routers.traefik.rule=Host(`traefik.cicd.luke-else.co.uk`)" + - "traefik.http.routers.traefik.entrypoints=websecure" + - "traefik.http.routers.traefik.service=api@internal" + - "traefik.http.routers.traefik.tls.certresolver=myresolver" + - "traefik.http.routers.traefik.middlewares=traefik-auth" + - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" + restart: unless-stopped + +networks: + proxy: + name: proxy \ No newline at end of file diff --git a/services/TraefikVPN/docker-compose.yml b/services/TraefikVPN/docker-compose.yml new file mode 100644 index 0000000..7383006 --- /dev/null +++ b/services/TraefikVPN/docker-compose.yml @@ -0,0 +1,40 @@ +services: + traefik: + image: "traefik:latest" + container_name: "traefik" + command: + - "--api.dashboard=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--certificatesresolvers.myresolver.acme.tlschallenge=true" + - "--certificatesresolvers.myresolver.acme.email=contact@luke-else.co.uk" + - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" + ports: + - "80:80" + - "443:443" + volumes: + - "./letsencrypt:/letsencrypt" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + networks: + - proxy + labels: + - "traefik.enable=true" + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.traefik-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.traefik-insecure.rule=Host(`traefik.vpn.luke-else.co.uk`)" + - "traefik.http.routers.traefik-insecure.entrypoints=web" + + - "traefik.http.routers.traefik.rule=Host(`traefik.vpn.luke-else.co.uk`)" + - "traefik.http.routers.traefik.entrypoints=websecure" + - "traefik.http.routers.traefik.service=api@internal" + - "traefik.http.routers.traefik.tls.certresolver=myresolver" + - "traefik.http.routers.traefik.middlewares=traefik-auth" + - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" + restart: unless-stopped + +networks: + proxy: + name: proxy \ No newline at end of file diff --git a/services/VPN/docker-compose.yml b/services/VPN/docker-compose.yml new file mode 100644 index 0000000..502dcd8 --- /dev/null +++ b/services/VPN/docker-compose.yml @@ -0,0 +1,24 @@ +services: + dockovpn: + image: alekslitvinenk/openvpn + cap_add: + - NET_ADMIN + ports: + - 1194:1194/udp # Expose tcp if you defined HOST_TUN_PROTOCOL=tcp + environment: + HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address + volumes: + - ./openvpn_conf:/opt/Dockovpn_data + labels: + ## Expose vpn Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.vpn-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.vpn-insecure.rule=Host(`vpn.luke-else.co.uk`)" + - "traefik.http.routers.vpn-insecure.entrypoints=web" + + - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)" + - "traefik.http.routers.vpn.entrypoints=websecure" + - "traefik.http.routers.vpn.tls.certresolver=myresolver" + restart: always \ No newline at end of file diff --git a/services/Watchtower/docker-compose.yml b/services/Watchtower/docker-compose.yml new file mode 100644 index 0000000..ee735d3 --- /dev/null +++ b/services/Watchtower/docker-compose.yml @@ -0,0 +1,8 @@ +services: + watchtower: + image: nickfedor/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - WATCHTOWER_CLEANUP=true + - WATCHTOWER_POLL_INTERVAL=60 diff --git a/services/Websites/docker-compose.yml b/services/Websites/docker-compose.yml new file mode 100644 index 0000000..ffc2c6b --- /dev/null +++ b/services/Websites/docker-compose.yml @@ -0,0 +1,103 @@ +services: + +#Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080) + luke-else: + image: git.luke-else.co.uk/luke-else/luke-else.co.uk + container_name: luke-else + networks: + - proxy + labels: + ## Expose luke-else Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.personal-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.personal-insecure.rule=Host(`luke-else.co.uk`)" + - "traefik.http.routers.personal-insecure.entrypoints=web" + + - "traefik.http.routers.personal.rule=Host(`luke-else.co.uk`)" + - "traefik.http.routers.personal.entrypoints=websecure" + - "traefik.http.routers.personal.tls.certresolver=myresolver" + restart: unless-stopped + + luke-else-dev: + image: git.luke-else.co.uk/luke-else/luke-else.co.uk:dev + container_name: luke-else-dev + networks: + - proxy + labels: + ## Expose luke-else Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.personal-dev-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.personal-dev-insecure.rule=Host(`dev.luke-else.co.uk`)" + - "traefik.http.routers.personal-dev-insecure.entrypoints=web" + + - "traefik.http.routers.personal-dev.rule=Host(`dev.luke-else.co.uk`)" + - "traefik.http.routers.personal-dev.entrypoints=websecure" + - "traefik.http.routers.personal-dev.tls.certresolver=myresolver" + restart: unless-stopped + + metarius: + image: git.luke-else.co.uk/luke-else/metarius:latest + container_name: metarius + networks: + - proxy + labels: + ## Expose metarius Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.metarius-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.metarius-insecure.rule=Host(`metarius.luke-else.co.uk`)" + - "traefik.http.routers.metarius-insecure.entrypoints=web" + + - "traefik.http.routers.metarius.rule=Host(`metarius.luke-else.co.uk`)" + - "traefik.http.routers.metarius.entrypoints=websecure" + - "traefik.http.routers.metarius.tls.certresolver=myresolver" + restart: unless-stopped + + divine-couture: + image: git.luke-else.co.uk/luke-else/divine-couture.co.uk:latest + container_name: divine-couture + networks: + - proxy + labels: + ## Expose divine-couture Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.divine-couture-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.divine-couture-insecure.rule=Host(`www.divine-couture.co.uk`)" + - "traefik.http.routers.divine-couture-insecure.entrypoints=web" + + - "traefik.http.routers.divine-couture.rule=Host(`www.divine-couture.co.uk`)" + - "traefik.http.routers.divine-couture.entrypoints=websecure" + - "traefik.http.routers.divine-couture.tls.certresolver=myresolver" + restart: unless-stopped + + snexo: + image: "php:apache" + container_name: snexo + volumes: + - ./snexo.co.uk/:/var/www/html + networks: + - proxy + labels: + ## Expose Snexo Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.snexo-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.snexo-insecure.rule=Host(`snexo.co.uk`)" + - "traefik.http.routers.snexo-insecure.entrypoints=web" + + - "traefik.http.routers.snexo.rule=Host(`snexo.co.uk`)" + - "traefik.http.routers.snexo.entrypoints=websecure" + - "traefik.http.routers.snexo.tls.certresolver=myresolver" + restart: unless-stopped + +networks: + proxy: + external: true diff --git a/services/cicd-spindown.sh b/services/cicd-spindown.sh new file mode 100755 index 0000000..1e3d49c --- /dev/null +++ b/services/cicd-spindown.sh @@ -0,0 +1,16 @@ +#Script file for spinning down all CICD relevant docker-containers + cd ./Development/Runners/ + docker compose down + cd ../.. + +cd ./TraefikRunner/ +docker compose down +cd .. + +cd ./Watchtower/ +docker compose down +cd .. + +docker rmi $(docker images -q) +docker system prune -f -a +docker volume prune -f -a \ No newline at end of file diff --git a/services/cicd-spinup.sh b/services/cicd-spinup.sh new file mode 100755 index 0000000..6a3b139 --- /dev/null +++ b/services/cicd-spinup.sh @@ -0,0 +1,13 @@ +#Script file for spinning up all docker-containers + +cd ./TraefikRunner/ +docker compose pull && docker compose up -d +cd .. + + cd ./Development/Runners/ + docker compose pull && docker compose up -d + cd ../.. + +cd ./Watchtower/ +docker compose pull && docker compose up -d +cd .. \ No newline at end of file diff --git a/services/main-spindown.sh b/services/main-spindown.sh new file mode 100755 index 0000000..1012835 --- /dev/null +++ b/services/main-spindown.sh @@ -0,0 +1,42 @@ +#Script file for spinning down all docker-containers + +cd ./Websites/ +docker compose down +cd .. + +cd ./Tracking/ +docker compose down +cd .. + + + cd ./Development/Gitea/ + docker compose down + cd ../.. + +cd ./Database/ +docker compose down +cd .. + +cd ./Bitwarden/ +docker compose down +cd .. + +cd ./Rustdesk/ +docker compose down +cd .. + +cd ./Misc/ +docker compose down +cd .. + +cd ./Watchtower/ +docker compose down +cd .. + +cd ./Traefik/ +docker compose down +cd .. + +docker rmi $(docker images -q) +docker system prune -f -a +docker volume prune -f -a \ No newline at end of file diff --git a/services/main-spinup.sh b/services/main-spinup.sh new file mode 100755 index 0000000..d8f8a03 --- /dev/null +++ b/services/main-spinup.sh @@ -0,0 +1,39 @@ +#Script file for spinning up all docker-containers + +cd ./Traefik/ +docker compose pull && docker compose up -d +cd .. + + cd ./Development/Gitea/ + docker compose pull && docker compose up -d + cd ../.. + +sleep 20 # Allow Gitea + registry to start up before starting the rest of the services + +cd ./Watchtower/ +docker compose pull && docker compose up -d +cd .. + +cd ./Tracking/ +docker compose pull && docker compose up -d +cd .. + +cd ./Websites/ +docker compose pull && docker compose up -d +cd .. + +cd ./Database/ +docker compose pull && docker compose up -d +cd .. + +cd ./Bitwarden/ +docker compose pull && docker compose up -d +cd .. + +cd ./Rustdesk/ +docker compose pull && docker compose up -d +cd .. + +cd ./Misc/ +docker compose pull && docker compose up -d +cd .. \ No newline at end of file diff --git a/services/todo.md b/services/todo.md new file mode 100644 index 0000000..a53d118 --- /dev/null +++ b/services/todo.md @@ -0,0 +1,64 @@ +# ToDo Items + +## General + +- Setup non root user +- UFW should be setup to keep VPS secure and only allow for: + + - https + - http + - ssh + - ftp + - 27017 + + - "21115:21115/tcp" + - "21116:21116/tcp" + - "21116:21116/udp" + - "21117:21117/tcp" + - "21119:21119/tcp" + +- Install SSH keys +- Setup unattended upgrades +- Install docker, docker-compose and apache utils. + +## Traefik + TraefikRunner + Traefik VPN + +- Setup htaccess -> `echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g` +- Ensure email address is correct + +## Gitea + +- Ensure that ports are assigned correctly for the system + +# Gitea Runner + +- Ensure that a registration token has been setup before continuing + +## Tracking + +Create a .env file with the following content: + +```sh +ACKEE_USERNAME=luke-else +ACKEE_PASSWORD=XXX +``` + +## Websites + +- Ensure website files are copied over +- Ensure that ports are assigned correctly for the system + +## Bitwarden + +- Ensure that all data is fully encrypted during transfer. +- Ensure that ports are assigned correctly for the system + +## Rustdesk + +- No additional setup required bar the ports + +## Database + +- Ensure that mysql root password, user and default database are updated. +- Ensure that mongo root password, and user are updated. +- Ensure database ports are correctly assigned and do not have to pass through traefik. diff --git a/services/vpn-spindown.sh b/services/vpn-spindown.sh new file mode 100755 index 0000000..7cd9ef1 --- /dev/null +++ b/services/vpn-spindown.sh @@ -0,0 +1,17 @@ +#Script file for spinning down all CICD relevant docker-containers + +cd ./VPN/ +docker compose down +cd .. + +cd ./TraefikVPN/ +docker compose down +cd .. + +cd ./Watchtower/ +docker compose down +cd .. + +docker rmi $(docker images -q) +docker system prune -f -a +docker volume prune -f -a \ No newline at end of file diff --git a/services/vpn-spinup.sh b/services/vpn-spinup.sh new file mode 100755 index 0000000..4cc5159 --- /dev/null +++ b/services/vpn-spinup.sh @@ -0,0 +1,13 @@ +#Script file for spinning up all docker-containers + +cd ./TraefikRunner/ +docker compose pull && docker compose up -d +cd .. + +cd ./VPN/ +docker compose pull && docker compose up -d +cd .. + +cd ./Watchtower/ +docker compose pull && docker compose up -d +cd .. \ No newline at end of file From d0d1385bf42e46a533b0d88514617d0711240e75 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Fri, 10 Jul 2026 20:13:33 +0100 Subject: [PATCH 04/24] feat: Re-structured services and updated readme --- docs/architecture.md | 33 --- infra/firewalls.tf | 7 - readme.md | 206 ++++++++++++++++-- services/Database/docker-compose.yml | 24 -- services/Misc/docker-compose.yml | 51 ----- services/Tracking/docker-compose.yml | 52 ----- services/cicd-spindown.sh | 16 -- services/cicd-spinup.sh | 13 -- .../{Development => dev}/Runners/config.yaml | 0 .../Runners/docker-compose.yml | 4 +- .../gitea-docker-compose.yml} | 0 services/dev/spindown.sh | 14 ++ services/dev/spinup.sh | 12 + .../traefik-docker-compose.yml} | 0 .../watchtower-docker-compose.yml} | 0 services/main-spindown.sh | 42 ---- services/main-spinup.sh | 39 ---- .../bitwarden-docker-compose.yml} | 0 .../rd-docker-compose.yml} | 0 services/prod/spindown.sh | 14 ++ services/prod/spinup.sh | 14 ++ services/prod/status-docker-compose.yml | 25 +++ .../traefik-docker-compose.yml} | 5 +- services/prod/watchtower-docker-compose.yml | 8 + .../web-docker-compose.yml} | 0 services/todo.md | 28 --- services/vpn-spindown.sh | 17 -- services/vpn-spinup.sh | 13 -- services/vpn/spindown.sh | 11 + services/vpn/spinup.sh | 8 + .../traefik-docker-compose.yml} | 0 .../vpn-docker-compose.yml} | 0 services/vpn/watchtower-docker-compose.yml | 8 + 33 files changed, 300 insertions(+), 364 deletions(-) delete mode 100644 docs/architecture.md delete mode 100644 services/Database/docker-compose.yml delete mode 100644 services/Misc/docker-compose.yml delete mode 100644 services/Tracking/docker-compose.yml delete mode 100755 services/cicd-spindown.sh delete mode 100755 services/cicd-spinup.sh rename services/{Development => dev}/Runners/config.yaml (100%) rename services/{Development => dev}/Runners/docker-compose.yml (95%) rename services/{Development/Gitea/docker-compose.yml => dev/gitea-docker-compose.yml} (100%) create mode 100755 services/dev/spindown.sh create mode 100755 services/dev/spinup.sh rename services/{TraefikRunner/docker-compose.yml => dev/traefik-docker-compose.yml} (100%) rename services/{Watchtower/docker-compose.yml => dev/watchtower-docker-compose.yml} (100%) delete mode 100755 services/main-spindown.sh delete mode 100755 services/main-spinup.sh rename services/{Bitwarden/docker-compose.yml => prod/bitwarden-docker-compose.yml} (100%) rename services/{Rustdesk/docker-compose.yml => prod/rd-docker-compose.yml} (100%) create mode 100755 services/prod/spindown.sh create mode 100755 services/prod/spinup.sh create mode 100644 services/prod/status-docker-compose.yml rename services/{Traefik/docker-compose.yml => prod/traefik-docker-compose.yml} (96%) create mode 100644 services/prod/watchtower-docker-compose.yml rename services/{Websites/docker-compose.yml => prod/web-docker-compose.yml} (100%) delete mode 100755 services/vpn-spindown.sh delete mode 100755 services/vpn-spinup.sh create mode 100755 services/vpn/spindown.sh create mode 100755 services/vpn/spinup.sh rename services/{TraefikVPN/docker-compose.yml => vpn/traefik-docker-compose.yml} (100%) rename services/{VPN/docker-compose.yml => vpn/vpn-docker-compose.yml} (100%) create mode 100644 services/vpn/watchtower-docker-compose.yml diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index 1395fb4..0000000 --- a/docs/architecture.md +++ /dev/null @@ -1,33 +0,0 @@ -```mermaid -architecture-beta - group cloud(cloud)[Hetzner] - - group network(cloud)[network] in cloud - - service disk1(mdi:disk)[Storage] in cloud - service disk2(mdi:disk)[Storage] in cloud - - service dev(mdi:server)[dev] in network - - service prod(mdi:server)[prod] in network - service prodfirewall(mdi:firewall)[firewall] in cloud - - service vpn(mdi:server)[vpn] in cloud - service vpnfirewall(mdi:firewall)[firewall] in cloud - - - - service gateway(mdi:web)[gateway] in cloud - - dev:L -- R:prod - disk1:B -- T:prod - disk2:B -- T:dev - - - prod:B -- T:prodfirewall - vpn:B -- T:vpnfirewall - - prodfirewall: L -- R: gateway - vpnfirewall: B -- T: gateway - -``` \ No newline at end of file diff --git a/infra/firewalls.tf b/infra/firewalls.tf index 807f33e..55528ad 100644 --- a/infra/firewalls.tf +++ b/infra/firewalls.tf @@ -72,13 +72,6 @@ resource "hcloud_firewall" "prod" { source_ips = ["0.0.0.0/0", "::/0"] } - rule { # mongo via Traefik tcp entrypoint (Database) - direction = "in" - protocol = "tcp" - port = "27017" - source_ips = ["0.0.0.0/0", "::/0"] - } - rule { # rustdesk hbbs direction = "in" protocol = "tcp" diff --git a/readme.md b/readme.md index 00ffcbc..0a21744 100644 --- a/readme.md +++ b/readme.md @@ -1,48 +1,206 @@ -# Server Repository +# Server -This repository contains various scripts and configurations for managing Docker containers and services. +Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).

-## Scripts +## Contents -- `cicd-spindown.sh`: Spins down all CICD relevant Docker containers. -- `cicd-spinup.sh`: Spins up all CICD relevant Docker containers. -- `main-spindown.sh`: Spins down all Docker containers. -- `main-spinup.sh`: Spins up all Docker containers. +- [Architecture](#architecture) +- [Repository layout](#repository-layout) +- [Prerequisites](#prerequisites) +- [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) +- [Deploying the services](#deploying-the-services-services) +- [Service inventory](#service-inventory) +- [First-time setup](#first-time-setup) +- [Development container](#development-container) +- [Security notes](#security-notes) -## Services +## Architecture -### Bitwarden +Three servers, one shared private network: -Configuration for Bitwarden is located in [Bitwarden/docker-compose.yml](Bitwarden/docker-compose.yml). +```mermaid +architecture-beta + group cloud(cloud)[Hetzner] -### Database + group network(cloud)[network] in cloud -Configuration for MongoDB is located in [Database/docker-compose.yml](Database/docker-compose.yml). + service disk1(mdi:disk)[Storage] in cloud + service disk2(mdi:disk)[Storage] in cloud -### Misc + service dev(mdi:server)[dev] in network -Configuration for miscellaneous services is located in [Misc/docker-compose.yml](Misc/docker-compose.yml). + service prod(mdi:server)[prod] in network + service prodfirewall(mdi:firewall)[firewall] in cloud -### Tracking + service vpn(mdi:server)[vpn] in cloud + service vpnfirewall(mdi:firewall)[firewall] in cloud -Configuration for tracking services is located in [Tracking/docker-compose.yml](Tracking/docker-compose.yml). + service gateway(mdi:web)[gateway] in cloud -### Traefik + dev:L -- R:prod + disk1:B -- T:prod + disk2:B -- T:dev -Configuration for Traefik is located in [Traefik/docker-compose.yml](Traefik/docker-compose.yml). + prod:B -- T:prodfirewall + vpn:B -- T:vpnfirewall -### TraefikRunner + prodfirewall: L -- R: gateway + vpnfirewall: B -- T: gateway +``` -Configuration for TraefikRunner is located in [TraefikRunner/docker-compose.yml](TraefikRunner/docker-compose.yml). +`gateway` represents the public internet, not a provisioned resource. -### Websites +| Server | Purpose | Network | Volume | +|---|---|---|---| +| `dev` | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) | `dev-storage` | +| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | `prod-storage` | +| `vpn` | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | none | -Configuration for websites is located in [Websites/docker-compose.yml](Websites/docker-compose.yml). +`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach deployment targets on `prod` without exposing that traffic publicly. `vpn` is deliberately kept off this network. Each server has its own Hetzner Cloud Firewall (see `infra/firewalls.tf`) that only opens the ports actually used by the compose stacks running on it, plus SSH restricted to `var.allowed_ssh_source_ips`. -## ToDo +## Repository layout -See [todo.md](todo.md) for a list of tasks and configurations that need to be completed. \ No newline at end of file +``` +. +├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes +│ ├── servers.tf +│ ├── network.tf +│ ├── firewalls.tf +│ ├── volumes.tf +│ ├── ssh.tf +│ ├── variables.tf +│ ├── outputs.tf +│ ├── versions.tf +│ └── terraform.tfvars.example +├── services/ # Docker Compose stacks, grouped by which server they run on +│ ├── dev/ # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) +│ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik +│ ├── vpn/ # OpenVPN + Traefik +│ └── todo.md # Outstanding manual setup/hardening tasks +├── docs/ +│ └── architecture.md # Source of the architecture diagram above +├── .devcontainer/ # Git submodule: shared devcontainer for working on this repo (OpenTofu tooling) +└── assets/ +``` + +Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. + +## Prerequisites + +- A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token +- An SSH key uploaded to that project (Console → Security → SSH Keys) +- [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` +- Docker + the Compose plugin on each target server +- DNS records for the domains in [Service inventory](#service-inventory) pointed at the relevant server's public IP + +## Provisioning the infrastructure (`infra/`) + +```sh +cd infra +export HCLOUD_TOKEN=your-hetzner-api-token # never commit this +cp terraform.tfvars.example terraform.tfvars +$EDITOR terraform.tfvars # set ssh_key_name at minimum + +tofu init +tofu plan +tofu apply +``` + +This creates: +- `hcloud_server.dev`, `hcloud_server.prod`, `hcloud_server.vpn` +- `hcloud_network.main` + subnet, shared by `dev` and `prod` +- `hcloud_firewall.dev` / `.prod` / `.vpn`, scoped to the ports each host actually uses +- `hcloud_volume.dev_storage` / `.prod_storage` + +Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`. + +`terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and can be overridden per-environment via `terraform.tfvars`. + +## Deploying the services (`services/`) + +Once a server exists, copy the relevant `services//` directory to it (e.g. `scp -r services/prod user@:~/services`) and run the matching script from inside that directory: + +```sh +# on dev +./spinup.sh # Traefik → Gitea Runner → Watchtower +./spindown.sh # reverse order, then prunes images/volumes + +# on prod +./spinup.sh # Traefik, then (after a 20s cert/registry settle) Watchtower, status, websites, Bitwarden, RustDesk +./spindown.sh + +# on vpn +./spinup.sh # Traefik → OpenVPN → Watchtower +./spindown.sh +``` + +Each stack can also be managed individually with plain Compose, e.g.: + +```sh +cd services/prod +docker compose -f bitwarden-docker-compose.yml up -d +docker compose -f bitwarden-docker-compose.yml down +``` + +All three hosts run [Watchtower](https://containrrr.dev/watchtower/) polling every 60s with cleanup enabled, so images are kept current automatically once deployed — the spinup scripts only need to be re-run after adding/removing a service or changing compose files. + +Every public-facing service is fronted by its host's own Traefik instance, terminating TLS via Let's Encrypt (`tlschallenge`, port 80/443). Each stack joins an external `proxy` Docker network and opts in via `traefik.enable=true` labels rather than publishing ports directly (RustDesk and the Gitea SSH port are the deliberate exceptions, since they aren't HTTP). + +## Service inventory + +### `dev` + +| Service | Compose file | Domain(s) | +|---|---|---| +| Traefik | `traefik-docker-compose.yml` | `traefik.cicd.luke-else.co.uk` | +| Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` | +| Gitea Actions runner | `Runners/docker-compose.yml` | `cicd.luke-else.co.uk` | +| Watchtower | `watchtower-docker-compose.yml` | — | + +### `prod` + +| Service | Compose file | Domain(s) | +|---|---|---| +| Traefik | `traefik-docker-compose.yml` | `traefik.luke-else.co.uk` | +| Websites | `web-docker-compose.yml` | `luke-else.co.uk`, `dev.luke-else.co.uk`, `metarius.luke-else.co.uk`, `www.divine-couture.co.uk`, `snexo.co.uk` | +| Status page (Uptime Kuma) | `status-docker-compose.yml` | `status.luke-else.co.uk` | +| Bitwarden (Vaultwarden) | `bitwarden-docker-compose.yml` | `bitwarden.luke-else.co.uk` | +| RustDesk relay (hbbs/hbbr) | `rd-docker-compose.yml` | `rd.luke-else.co.uk`, ports `21115-21119` | +| Watchtower | `watchtower-docker-compose.yml` | — | + +### `vpn` + +| Service | Compose file | Domain(s) | +|---|---|---| +| Traefik | `traefik-docker-compose.yml` | `traefik.vpn.luke-else.co.uk` | +| OpenVPN (Dockovpn) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `1194` | +| Watchtower | `watchtower-docker-compose.yml` | — | + +## First-time setup + +A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here: + +- **Gitea Actions runner**: set a real `GITEA_RUNNER_REGISTRATION_TOKEN` in `services/dev/Runners/docker-compose.yml` (generate one from the Gitea admin UI) before starting the runner. +- **Traefik dashboard auth**: the committed basic-auth hash is a placeholder. Generate your own with `echo $(htpasswd -nb user password) | sed -e 's/\$/\$\$/g'` and replace the `traefik-auth` middleware value in each `traefik-docker-compose.yml`. +- **General host hardening**: non-root user, UFW, unattended-upgrades, Docker install — see `services/todo.md`. + +## Development container + +`.devcontainer` is a git submodule providing a ready-to-use OpenTofu development environment (VS Code + OpenTofu/Docker/Mermaid extensions). After cloning: + +```sh +git submodule update --init --recursive +``` + +Then reopen the repo in VS Code with the Dev Containers extension. + +## Security notes + +- Real secrets (`HCLOUD_TOKEN`, `*.tfvars`, `.env` files) must never be committed — see `.gitignore`. +- SSH is restricted to `var.allowed_ssh_source_ips` on every host; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). +- `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly. +- Firewalls are allowlists scoped per host in `infra/firewalls.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. diff --git a/services/Database/docker-compose.yml b/services/Database/docker-compose.yml deleted file mode 100644 index 3d1a9a3..0000000 --- a/services/Database/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -services: - -#MongoDB (27017) - mongodb: - image: "mongo:latest" - container_name: mongoDB - volumes: - - ./mongo/:/data/db - networks: - - proxy - labels: - - "traefik.enable=true" - - "traefik.tcp.routers.mongodb.rule=HostSNI(`*`)" - - "traefik.tcp.routers.mongodb.entrypoints=mongo" - - "traefik.tcp.routers.mongodb.service=mongodb" - - "traefik.tcp.services.mongodb.loadbalancer.server.port=27017" - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: rootpassword - restart: unless-stopped - -networks: - proxy: - external: true \ No newline at end of file diff --git a/services/Misc/docker-compose.yml b/services/Misc/docker-compose.yml deleted file mode 100644 index 7914b79..0000000 --- a/services/Misc/docker-compose.yml +++ /dev/null @@ -1,51 +0,0 @@ -services: - status: - image: louislam/uptime-kuma:latest - container_name: status - volumes: - - ./uptime-kuma/data:/app/data - networks: - - proxy - labels: - ## Expose uptime-kuma Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)" - - "traefik.http.routers.status-insecure.entrypoints=web" - - - "traefik.http.routers.status.rule=Host(`status.luke-else.co.uk`)" - - "traefik.http.routers.status.entrypoints=websecure" - - "traefik.http.routers.status.tls.certresolver=myresolver" - restart: unless-stopped - - portainer: - image: portainer/portainer-ce:latest - container_name: portainer - security_opt: - - no-new-privileges:true - volumes: - - /etc/localtime:/etc/localtime:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./portainer-data:/data - networks: - - proxy - labels: - ## Expose portainer Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.portainer-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.portainer-insecure.rule=Host(`portainer.luke-else.co.uk`)" - - "traefik.http.routers.portainer-insecure.entrypoints=web" - - - "traefik.http.routers.portainer.rule=Host(`portainer.luke-else.co.uk`)" - - "traefik.http.services.portainer.loadbalancer.server.port=9000" - - "traefik.http.routers.portainer.entrypoints=websecure" - - "traefik.http.routers.portainer.tls.certresolver=myresolver" - restart: unless-stopped - -networks: - proxy: - external: true \ No newline at end of file diff --git a/services/Tracking/docker-compose.yml b/services/Tracking/docker-compose.yml deleted file mode 100644 index 6fe612a..0000000 --- a/services/Tracking/docker-compose.yml +++ /dev/null @@ -1,52 +0,0 @@ -services: - ackee: - image: electerious/ackee - container_name: ackee - environment: - - WAIT_HOSTS=mongo:27017 - - ACKEE_MONGODB=mongodb://mongo-ackee:27017/ackee - env_file: - - .env - expose: - - 3000 - networks: - - tracking - - proxy - labels: - ## Expose Ackee Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - # Enable CORS headers - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowmethods=*" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolalloworiginlist=https://luke-else.co.uk" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowcredentials=true" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolallowheaders=Content-Type,Authorization" - - "traefik.http.middlewares.cors-tracking.headers.addvaryheader=true" - - "traefik.http.middlewares.cors-tracking.headers.accesscontrolmaxage=100" - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.ackee-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.ackee-insecure.rule=Host(`tracking.luke-else.co.uk`)" - - "traefik.http.routers.ackee-insecure.entrypoints=web" - - - "traefik.http.routers.ackee.rule=Host(`tracking.luke-else.co.uk`)" - - "traefik.http.routers.ackee.entrypoints=websecure" - - "traefik.http.routers.ackee.tls.certresolver=myresolver" - - "traefik.http.routers.ackee.middlewares=cors-tracking" - depends_on: - - mongo - restart: unless-stopped - - mongo: - image: mongo - container_name: mongo-ackee - volumes: - - ./data:/data/db - networks: - - tracking - restart: unless-stopped - -networks: - proxy: - external: true - tracking: \ No newline at end of file diff --git a/services/cicd-spindown.sh b/services/cicd-spindown.sh deleted file mode 100755 index 1e3d49c..0000000 --- a/services/cicd-spindown.sh +++ /dev/null @@ -1,16 +0,0 @@ -#Script file for spinning down all CICD relevant docker-containers - cd ./Development/Runners/ - docker compose down - cd ../.. - -cd ./TraefikRunner/ -docker compose down -cd .. - -cd ./Watchtower/ -docker compose down -cd .. - -docker rmi $(docker images -q) -docker system prune -f -a -docker volume prune -f -a \ No newline at end of file diff --git a/services/cicd-spinup.sh b/services/cicd-spinup.sh deleted file mode 100755 index 6a3b139..0000000 --- a/services/cicd-spinup.sh +++ /dev/null @@ -1,13 +0,0 @@ -#Script file for spinning up all docker-containers - -cd ./TraefikRunner/ -docker compose pull && docker compose up -d -cd .. - - cd ./Development/Runners/ - docker compose pull && docker compose up -d - cd ../.. - -cd ./Watchtower/ -docker compose pull && docker compose up -d -cd .. \ No newline at end of file diff --git a/services/Development/Runners/config.yaml b/services/dev/Runners/config.yaml similarity index 100% rename from services/Development/Runners/config.yaml rename to services/dev/Runners/config.yaml diff --git a/services/Development/Runners/docker-compose.yml b/services/dev/Runners/docker-compose.yml similarity index 95% rename from services/Development/Runners/docker-compose.yml rename to services/dev/Runners/docker-compose.yml index f9f5acd..0e1bb51 100644 --- a/services/Development/Runners/docker-compose.yml +++ b/services/dev/Runners/docker-compose.yml @@ -13,7 +13,7 @@ services: CONFIG_FILE: /config.yaml GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" GITEA_RUNNER_REGISTRATION_TOKEN: "INSERT REGISTRATION TOKEN" - GITEA_RUNNER_NAME: "CICD" + GITEA_RUNNER_NAME: "CICD#1" labels: ## Expose cicd Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container @@ -30,4 +30,4 @@ services: networks: proxy: - external: true \ No newline at end of file + external: true diff --git a/services/Development/Gitea/docker-compose.yml b/services/dev/gitea-docker-compose.yml similarity index 100% rename from services/Development/Gitea/docker-compose.yml rename to services/dev/gitea-docker-compose.yml diff --git a/services/dev/spindown.sh b/services/dev/spindown.sh new file mode 100755 index 0000000..5e71bd2 --- /dev/null +++ b/services/dev/spindown.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Script file for spinning down all dev docker-containers +cd "$(dirname "$0")" + +cd Runners +docker compose down +cd .. + +docker compose -f traefik-docker-compose.yml down +docker compose -f watchtower-docker-compose.yml down + +docker rmi $(docker images -q) +docker system prune -f -a +docker volume prune -f -a diff --git a/services/dev/spinup.sh b/services/dev/spinup.sh new file mode 100755 index 0000000..6d9bfcc --- /dev/null +++ b/services/dev/spinup.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Script file for spinning up all dev docker-containers +set -e +cd "$(dirname "$0")" + +docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d + +cd Runners +docker compose pull && docker compose up -d +cd .. + +docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d diff --git a/services/TraefikRunner/docker-compose.yml b/services/dev/traefik-docker-compose.yml similarity index 100% rename from services/TraefikRunner/docker-compose.yml rename to services/dev/traefik-docker-compose.yml diff --git a/services/Watchtower/docker-compose.yml b/services/dev/watchtower-docker-compose.yml similarity index 100% rename from services/Watchtower/docker-compose.yml rename to services/dev/watchtower-docker-compose.yml diff --git a/services/main-spindown.sh b/services/main-spindown.sh deleted file mode 100755 index 1012835..0000000 --- a/services/main-spindown.sh +++ /dev/null @@ -1,42 +0,0 @@ -#Script file for spinning down all docker-containers - -cd ./Websites/ -docker compose down -cd .. - -cd ./Tracking/ -docker compose down -cd .. - - - cd ./Development/Gitea/ - docker compose down - cd ../.. - -cd ./Database/ -docker compose down -cd .. - -cd ./Bitwarden/ -docker compose down -cd .. - -cd ./Rustdesk/ -docker compose down -cd .. - -cd ./Misc/ -docker compose down -cd .. - -cd ./Watchtower/ -docker compose down -cd .. - -cd ./Traefik/ -docker compose down -cd .. - -docker rmi $(docker images -q) -docker system prune -f -a -docker volume prune -f -a \ No newline at end of file diff --git a/services/main-spinup.sh b/services/main-spinup.sh deleted file mode 100755 index d8f8a03..0000000 --- a/services/main-spinup.sh +++ /dev/null @@ -1,39 +0,0 @@ -#Script file for spinning up all docker-containers - -cd ./Traefik/ -docker compose pull && docker compose up -d -cd .. - - cd ./Development/Gitea/ - docker compose pull && docker compose up -d - cd ../.. - -sleep 20 # Allow Gitea + registry to start up before starting the rest of the services - -cd ./Watchtower/ -docker compose pull && docker compose up -d -cd .. - -cd ./Tracking/ -docker compose pull && docker compose up -d -cd .. - -cd ./Websites/ -docker compose pull && docker compose up -d -cd .. - -cd ./Database/ -docker compose pull && docker compose up -d -cd .. - -cd ./Bitwarden/ -docker compose pull && docker compose up -d -cd .. - -cd ./Rustdesk/ -docker compose pull && docker compose up -d -cd .. - -cd ./Misc/ -docker compose pull && docker compose up -d -cd .. \ No newline at end of file diff --git a/services/Bitwarden/docker-compose.yml b/services/prod/bitwarden-docker-compose.yml similarity index 100% rename from services/Bitwarden/docker-compose.yml rename to services/prod/bitwarden-docker-compose.yml diff --git a/services/Rustdesk/docker-compose.yml b/services/prod/rd-docker-compose.yml similarity index 100% rename from services/Rustdesk/docker-compose.yml rename to services/prod/rd-docker-compose.yml diff --git a/services/prod/spindown.sh b/services/prod/spindown.sh new file mode 100755 index 0000000..665ab2b --- /dev/null +++ b/services/prod/spindown.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Script file for spinning down all prod docker-containers +cd "$(dirname "$0")" + +docker compose -f web-docker-compose.yml down +docker compose -f status-docker-compose.yml down +docker compose -f bitwarden-docker-compose.yml down +docker compose -f rd-docker-compose.yml down +docker compose -f watchtower-docker-compose.yml down +docker compose -f traefik-docker-compose.yml down + +docker rmi $(docker images -q) +docker system prune -f -a +docker volume prune -f -a diff --git a/services/prod/spinup.sh b/services/prod/spinup.sh new file mode 100755 index 0000000..8fa967e --- /dev/null +++ b/services/prod/spinup.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Script file for spinning up all prod docker-containers +set -e +cd "$(dirname "$0")" + +docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d + +sleep 20 # Allow Traefik + registry auth to settle before starting the rest of the services + +docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d +docker compose -f status-docker-compose.yml pull && docker compose -f status-docker-compose.yml up -d +docker compose -f web-docker-compose.yml pull && docker compose -f web-docker-compose.yml up -d +docker compose -f bitwarden-docker-compose.yml pull && docker compose -f bitwarden-docker-compose.yml up -d +docker compose -f rd-docker-compose.yml pull && docker compose -f rd-docker-compose.yml up -d diff --git a/services/prod/status-docker-compose.yml b/services/prod/status-docker-compose.yml new file mode 100644 index 0000000..0e833c2 --- /dev/null +++ b/services/prod/status-docker-compose.yml @@ -0,0 +1,25 @@ +services: + status: + image: louislam/uptime-kuma:latest + container_name: status + volumes: + - ./uptime-kuma/data:/app/data + networks: + - proxy + labels: + ## Expose uptime-kuma Through Trefik ## + - "traefik.enable=true" # <== Enable traefik to proxy this container + + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" + - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure" + - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)" + - "traefik.http.routers.status-insecure.entrypoints=web" + + - "traefik.http.routers.status.rule=Host(`status.luke-else.co.uk`)" + - "traefik.http.routers.status.entrypoints=websecure" + - "traefik.http.routers.status.tls.certresolver=myresolver" + restart: unless-stopped + +networks: + proxy: + external: true diff --git a/services/Traefik/docker-compose.yml b/services/prod/traefik-docker-compose.yml similarity index 96% rename from services/Traefik/docker-compose.yml rename to services/prod/traefik-docker-compose.yml index c2b613a..53e89ac 100644 --- a/services/Traefik/docker-compose.yml +++ b/services/prod/traefik-docker-compose.yml @@ -8,7 +8,6 @@ services: - "--providers.docker.exposedbydefault=false" - "--entrypoints.web.address=:80" - "--entrypoints.websecure.address=:443" - - "--entrypoints.mongo.address=:27017" - "--entrypoints.web.transport.respondingTimeouts.readTimeout=120s" - "--entrypoints.websecure.transport.respondingTimeouts.readTimeout=120s" - "--entrypoints.web.transport.respondingTimeouts.writeTimeout=120s" @@ -36,11 +35,11 @@ services: - "traefik.http.routers.traefik.rule=Host(`traefik.luke-else.co.uk`)" - "traefik.http.routers.traefik.entrypoints=websecure" - "traefik.http.routers.traefik.service=api@internal" - - "traefik.http.routers.traefik.tls.certresolver=myresolver" + - "traefik.http.routers.traefik.tls.certresolver=myresolver" - "traefik.http.routers.traefik.middlewares=traefik-auth" - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" restart: unless-stopped networks: proxy: - name: proxy \ No newline at end of file + name: proxy diff --git a/services/prod/watchtower-docker-compose.yml b/services/prod/watchtower-docker-compose.yml new file mode 100644 index 0000000..ee735d3 --- /dev/null +++ b/services/prod/watchtower-docker-compose.yml @@ -0,0 +1,8 @@ +services: + watchtower: + image: nickfedor/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - WATCHTOWER_CLEANUP=true + - WATCHTOWER_POLL_INTERVAL=60 diff --git a/services/Websites/docker-compose.yml b/services/prod/web-docker-compose.yml similarity index 100% rename from services/Websites/docker-compose.yml rename to services/prod/web-docker-compose.yml diff --git a/services/todo.md b/services/todo.md index a53d118..eccd34c 100644 --- a/services/todo.md +++ b/services/todo.md @@ -3,13 +3,6 @@ ## General - Setup non root user -- UFW should be setup to keep VPS secure and only allow for: - - - https - - http - - ssh - - ftp - - 27017 - "21115:21115/tcp" - "21116:21116/tcp" @@ -17,8 +10,6 @@ - "21117:21117/tcp" - "21119:21119/tcp" -- Install SSH keys -- Setup unattended upgrades - Install docker, docker-compose and apache utils. ## Traefik + TraefikRunner + Traefik VPN @@ -34,15 +25,6 @@ - Ensure that a registration token has been setup before continuing -## Tracking - -Create a .env file with the following content: - -```sh -ACKEE_USERNAME=luke-else -ACKEE_PASSWORD=XXX -``` - ## Websites - Ensure website files are copied over @@ -52,13 +34,3 @@ ACKEE_PASSWORD=XXX - Ensure that all data is fully encrypted during transfer. - Ensure that ports are assigned correctly for the system - -## Rustdesk - -- No additional setup required bar the ports - -## Database - -- Ensure that mysql root password, user and default database are updated. -- Ensure that mongo root password, and user are updated. -- Ensure database ports are correctly assigned and do not have to pass through traefik. diff --git a/services/vpn-spindown.sh b/services/vpn-spindown.sh deleted file mode 100755 index 7cd9ef1..0000000 --- a/services/vpn-spindown.sh +++ /dev/null @@ -1,17 +0,0 @@ -#Script file for spinning down all CICD relevant docker-containers - -cd ./VPN/ -docker compose down -cd .. - -cd ./TraefikVPN/ -docker compose down -cd .. - -cd ./Watchtower/ -docker compose down -cd .. - -docker rmi $(docker images -q) -docker system prune -f -a -docker volume prune -f -a \ No newline at end of file diff --git a/services/vpn-spinup.sh b/services/vpn-spinup.sh deleted file mode 100755 index 4cc5159..0000000 --- a/services/vpn-spinup.sh +++ /dev/null @@ -1,13 +0,0 @@ -#Script file for spinning up all docker-containers - -cd ./TraefikRunner/ -docker compose pull && docker compose up -d -cd .. - -cd ./VPN/ -docker compose pull && docker compose up -d -cd .. - -cd ./Watchtower/ -docker compose pull && docker compose up -d -cd .. \ No newline at end of file diff --git a/services/vpn/spindown.sh b/services/vpn/spindown.sh new file mode 100755 index 0000000..c30233f --- /dev/null +++ b/services/vpn/spindown.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Script file for spinning down all vpn docker-containers +cd "$(dirname "$0")" + +docker compose -f vpn-docker-compose.yml down +docker compose -f traefik-docker-compose.yml down +docker compose -f watchtower-docker-compose.yml down + +docker rmi $(docker images -q) +docker system prune -f -a +docker volume prune -f -a diff --git a/services/vpn/spinup.sh b/services/vpn/spinup.sh new file mode 100755 index 0000000..14250a8 --- /dev/null +++ b/services/vpn/spinup.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Script file for spinning up all vpn docker-containers +set -e +cd "$(dirname "$0")" + +docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d +docker compose -f vpn-docker-compose.yml pull && docker compose -f vpn-docker-compose.yml up -d +docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d diff --git a/services/TraefikVPN/docker-compose.yml b/services/vpn/traefik-docker-compose.yml similarity index 100% rename from services/TraefikVPN/docker-compose.yml rename to services/vpn/traefik-docker-compose.yml diff --git a/services/VPN/docker-compose.yml b/services/vpn/vpn-docker-compose.yml similarity index 100% rename from services/VPN/docker-compose.yml rename to services/vpn/vpn-docker-compose.yml diff --git a/services/vpn/watchtower-docker-compose.yml b/services/vpn/watchtower-docker-compose.yml new file mode 100644 index 0000000..ee735d3 --- /dev/null +++ b/services/vpn/watchtower-docker-compose.yml @@ -0,0 +1,8 @@ +services: + watchtower: + image: nickfedor/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - WATCHTOWER_CLEANUP=true + - WATCHTOWER_POLL_INTERVAL=60 From a7cfcc33cea1f286265aa63ae434acfcffba0e91 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Fri, 10 Jul 2026 20:54:22 +0100 Subject: [PATCH 05/24] chore: Restructured infra modules --- infra/.terraform.lock.hcl | 32 ++++ infra/firewalls.tf | 155 ------------------ infra/main.tf | 55 +++++++ infra/modules/dev/main.tf | 72 ++++++++ infra/modules/dev/outputs.tf | 7 + infra/modules/dev/variables.tf | 44 +++++ infra/{network.tf => modules/network/main.tf} | 4 +- infra/modules/network/outputs.tf | 9 + infra/modules/network/variables.tf | 14 ++ infra/modules/prod/main.tf | 106 ++++++++++++ infra/modules/prod/outputs.tf | 7 + infra/modules/prod/variables.tf | 44 +++++ infra/modules/vpn/main.tf | 41 +++++ infra/modules/vpn/outputs.tf | 3 + infra/modules/vpn/variables.tf | 24 +++ infra/outputs.tf | 10 +- infra/servers.tf | 43 ----- infra/variables.tf | 16 +- infra/volumes.tf | 17 -- 19 files changed, 473 insertions(+), 230 deletions(-) delete mode 100644 infra/firewalls.tf create mode 100644 infra/main.tf create mode 100644 infra/modules/dev/main.tf create mode 100644 infra/modules/dev/outputs.tf create mode 100644 infra/modules/dev/variables.tf rename infra/{network.tf => modules/network/main.tf} (74%) create mode 100644 infra/modules/network/outputs.tf create mode 100644 infra/modules/network/variables.tf create mode 100644 infra/modules/prod/main.tf create mode 100644 infra/modules/prod/outputs.tf create mode 100644 infra/modules/prod/variables.tf create mode 100644 infra/modules/vpn/main.tf create mode 100644 infra/modules/vpn/outputs.tf create mode 100644 infra/modules/vpn/variables.tf delete mode 100644 infra/servers.tf delete mode 100644 infra/volumes.tf diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl index 37ba449..8927fa3 100644 --- a/infra/.terraform.lock.hcl +++ b/infra/.terraform.lock.hcl @@ -1,6 +1,38 @@ # This file is maintained automatically by "tofu init". # Manual edits may be lost in future updates. +provider "registry.opentofu.org/hashicorp/hcloud" { + version = "1.66.0" + hashes = [ + "h1:+R3t/5wqCb2ics5bNZ4JgtfaAa08Z6uRzXzNtwovVEY=", + "h1:9c1ECk3N9xhdQKz4NVCfYEFxebJVVeQZS3E6wrR/Hx8=", + "h1:EtYB/KjPN2cIJa6hpEaVc8q/3yf6e9IT6/uAybUGqps=", + "h1:GUupu6Fg5zFY9i4/3esiy8uHZnUrt+u2B6NaE4+8YqQ=", + "h1:JHehRJySEFRGYK+D6pFS0MTTDDRlGSQdSaKdEo0AemA=", + "h1:XF+SwyO1ttQJ0WXJ5O628O7cGk9U37sGn0o3LwWUnDQ=", + "h1:czuvCXAd8CcszXh3k3E0N6tFfQkYSNbKXo99LG+yOIc=", + "h1:dPIwO6zTxYs6bHn5yf/w/1AExaHPppCLyTrWrdzH4RE=", + "h1:gqrhnYuIBzpUvgWHITNRUMLaDcKACGlrNfrbtbtO7KY=", + "h1:iVAGP8gRbZK0kJF7SiYJRt61wz0D5AF9q+WMsrAiBI0=", + "h1:kLi29SbGCU/Z/Ch0zuNdYFSKP3mHp8zfcOpZsIN/KIQ=", + "h1:mq2+1Q/5gWJYI8XOXQCeteUg0AP7VezPOGOjkkHRQso=", + "h1:rnWTWAOlJhHtD3kkU4Qfw0bg9ko/dlH822inxYLBmfQ=", + "zh:1286cee6fb63dbcb18f53077bbb5e5d132a4e4d9f006af4e8d8edfc08d6bcdc8", + "zh:204460dacc044bda019a4a18b398e094289500c36913c7c9457f432adf31b8b2", + "zh:214175d50773481cbeaf9c9004e4121a3a1c9686c79424ebdc8ff189dd057d3e", + "zh:22b17bceff61cc13ad04a399ba87521356a3a134d4687273727473ae9eccf5f1", + "zh:368867dac5525c411de7e38f2e27de0a71854d1750867322ff2b9321128c88fb", + "zh:5289b75f8370bdbc4c6051d55cf33d0b1bd25dc6d71bfbd39b360249a37f1501", + "zh:81cb676aa50c5777df8fc80d4e69c9012330ae751f5e6f12bf6074bfd2e7c496", + "zh:ab08aead10643b21aa6b51af562b50492e12b9dd0ab7dca27a05aa63209b7d66", + "zh:af25c210d0570cf61ef767b2545bf9f3fb909178135f0e5e14bec0c1c9d07a63", + "zh:bcad66f4830c97118fa793723e53f8a4d27ddd34ea969ff259408842c2238331", + "zh:ce3ed323d75ae905d975925fa98c7054a7514c81276a485fc37da8232b53e39f", + "zh:d481bc0ef0c87ab1969c17777f526b2f59f823432d676145134c41a6d29bd98e", + "zh:ea7ef88df2c3ca154d86238920636d52a3c9066c7467543d3fa45f1e52ec2f7b", + ] +} + provider "registry.opentofu.org/hetznercloud/hcloud" { version = "1.66.0" constraints = "~> 1.45" diff --git a/infra/firewalls.tf b/infra/firewalls.tf deleted file mode 100644 index 55528ad..0000000 --- a/infra/firewalls.tf +++ /dev/null @@ -1,155 +0,0 @@ -# Port sources: services/*/docker-compose.yml (published ports) and services/todo.md -# (the documented UFW allow-list). dev has no firewall in architecture.md, but we add -# one anyway for baseline safety - see conversation history. - -resource "hcloud_firewall" "dev" { - name = "dev-firewall" - - rule { # server ssh - direction = "in" - protocol = "tcp" - port = "22" - source_ips = var.allowed_ssh_source_ips - } - - rule { # gitea ssh (git.luke-else.co.uk, published as 222:22) - direction = "in" - protocol = "tcp" - port = "222" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { # TraefikRunner http/https (git.luke-else.co.uk, cicd.luke-else.co.uk) - direction = "in" - protocol = "tcp" - port = "80" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { - direction = "in" - protocol = "tcp" - port = "443" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { # traffic from prod over the private network - direction = "in" - protocol = "tcp" - port = "1-65535" - source_ips = [var.network_ip_range] - } - - rule { - direction = "in" - protocol = "udp" - port = "1-65535" - source_ips = [var.network_ip_range] - } -} - -resource "hcloud_firewall" "prod" { - name = "prod-firewall" - - rule { # server ssh - direction = "in" - protocol = "tcp" - port = "22" - source_ips = var.allowed_ssh_source_ips - } - - rule { # Traefik http/https (Websites, Bitwarden, Misc, Tracking) - direction = "in" - protocol = "tcp" - port = "80" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { - direction = "in" - protocol = "tcp" - port = "443" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { # rustdesk hbbs - direction = "in" - protocol = "tcp" - port = "21115" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { - direction = "in" - protocol = "tcp" - port = "21116" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { - direction = "in" - protocol = "udp" - port = "21116" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { # rustdesk hbbr - direction = "in" - protocol = "tcp" - port = "21117" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { - direction = "in" - protocol = "tcp" - port = "21119" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { # traffic from dev over the private network - direction = "in" - protocol = "tcp" - port = "1-65535" - source_ips = [var.network_ip_range] - } - - rule { - direction = "in" - protocol = "udp" - port = "1-65535" - source_ips = [var.network_ip_range] - } -} - -resource "hcloud_firewall" "vpn" { - name = "vpn-firewall" - - rule { # server ssh - direction = "in" - protocol = "tcp" - port = "22" - source_ips = var.allowed_ssh_source_ips - } - - rule { # TraefikVPN http/https (traefik.vpn.luke-else.co.uk) - direction = "in" - protocol = "tcp" - port = "80" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { - direction = "in" - protocol = "tcp" - port = "443" - source_ips = ["0.0.0.0/0", "::/0"] - } - - rule { # OpenVPN tunnel - always direct to this server's public IP, never via a load balancer - direction = "in" - protocol = "udp" - port = "1194" - source_ips = ["0.0.0.0/0", "::/0"] - } -} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 0000000..6821232 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,55 @@ +# Root module: wires together the shared network and the three host modules +# (one per server in docs/architecture.md). Each host module owns its own +# firewall (and volume, where applicable) - see modules//main.tf. + +module "network" { + source = "./modules/network" + + ip_range = var.network_ip_range + network_zone = var.network_zone + subnet_ip_range = var.subnet_ip_range +} + +module "dev" { + source = "./modules/dev" + + server_type = var.dev_server_type + image = var.server_image + location = var.location + ssh_key_id = data.hcloud_ssh_key.main.id + network_id = module.network.id + private_ip = var.dev_private_ip + volume_size = var.dev_volume_size + allowed_ssh_source_ips = var.allowed_ssh_source_ips + network_ip_range = var.network_ip_range + + # module.network.id alone doesn't guarantee the subnet exists yet, and a server + # can't join a network before it has a subnet. + depends_on = [module.network] +} + +module "prod" { + source = "./modules/prod" + + server_type = var.prod_server_type + image = var.server_image + location = var.location + ssh_key_id = data.hcloud_ssh_key.main.id + network_id = module.network.id + private_ip = var.prod_private_ip + volume_size = var.prod_volume_size + allowed_ssh_source_ips = var.allowed_ssh_source_ips + network_ip_range = var.network_ip_range + + depends_on = [module.network] +} + +module "vpn" { + source = "./modules/vpn" + + server_type = var.vpn_server_type + image = var.server_image + location = var.location + ssh_key_id = data.hcloud_ssh_key.main.id + allowed_ssh_source_ips = var.allowed_ssh_source_ips +} diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf new file mode 100644 index 0000000..90552be --- /dev/null +++ b/infra/modules/dev/main.tf @@ -0,0 +1,72 @@ +# dev: Gitea + Runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) + +# Port sources: services/dev/*docker-compose.yml (published ports) and services/todo.md +# (the documented UFW allow-list). dev has no firewall in docs/architecture.md, but we +# add one anyway for baseline safety - see conversation history. +resource "hcloud_firewall" "this" { + name = "dev-firewall" + + rule { # server ssh + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.allowed_ssh_source_ips + } + + rule { # gitea ssh (git.luke-else.co.uk, published as 222:22) + direction = "in" + protocol = "tcp" + port = "222" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # Traefik http/https (git.luke-else.co.uk, cicd.luke-else.co.uk) + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # traffic from prod over the private network + direction = "in" + protocol = "tcp" + port = "1-65535" + source_ips = [var.network_ip_range] + } + + rule { + direction = "in" + protocol = "udp" + port = "1-65535" + source_ips = [var.network_ip_range] + } +} + +resource "hcloud_server" "this" { + name = "dev" + server_type = var.server_type + image = var.image + location = var.location + ssh_keys = [var.ssh_key_id] + firewall_ids = [hcloud_firewall.this.id] + + network { + network_id = var.network_id + ip = var.private_ip + } +} + +resource "hcloud_volume" "storage" { + name = "dev-storage" + size = var.volume_size + server_id = hcloud_server.this.id + automount = true + format = "ext4" +} diff --git a/infra/modules/dev/outputs.tf b/infra/modules/dev/outputs.tf new file mode 100644 index 0000000..4897389 --- /dev/null +++ b/infra/modules/dev/outputs.tf @@ -0,0 +1,7 @@ +output "ipv4" { + value = hcloud_server.this.ipv4_address +} + +output "private_ipv4" { + value = var.private_ip +} diff --git a/infra/modules/dev/variables.tf b/infra/modules/dev/variables.tf new file mode 100644 index 0000000..7c0cd9f --- /dev/null +++ b/infra/modules/dev/variables.tf @@ -0,0 +1,44 @@ +variable "server_type" { + description = "Server type for dev (Gitea + Runner)." + type = string +} + +variable "image" { + description = "OS image used for the server." + type = string +} + +variable "location" { + description = "Hetzner Cloud datacenter location." + type = string +} + +variable "ssh_key_id" { + description = "ID of the Hetzner Cloud SSH key to install on the server." + type = string +} + +variable "network_id" { + description = "ID of the private network to attach dev to." + type = string +} + +variable "private_ip" { + description = "Private network IP for dev." + type = string +} + +variable "volume_size" { + description = "Size in GB of the volume attached to dev (disk2 in docs/architecture.md)." + type = number +} + +variable "allowed_ssh_source_ips" { + description = "CIDRs allowed to reach port 22 on dev." + type = list(string) +} + +variable "network_ip_range" { + description = "CIDR of the private network, allowed through the firewall for traffic from prod." + type = string +} diff --git a/infra/network.tf b/infra/modules/network/main.tf similarity index 74% rename from infra/network.tf rename to infra/modules/network/main.tf index 29fad1b..87abcb8 100644 --- a/infra/network.tf +++ b/infra/modules/network/main.tf @@ -1,8 +1,8 @@ -# Private network shared by dev and prod (see architecture.md "network" group). +# Private network shared by dev and prod (see docs/architecture.md "network" group). # vpn is intentionally not attached to this network - it sits outside it in the diagram. resource "hcloud_network" "main" { name = "server-network" - ip_range = var.network_ip_range + ip_range = var.ip_range } resource "hcloud_network_subnet" "main" { diff --git a/infra/modules/network/outputs.tf b/infra/modules/network/outputs.tf new file mode 100644 index 0000000..622ad92 --- /dev/null +++ b/infra/modules/network/outputs.tf @@ -0,0 +1,9 @@ +output "id" { + value = hcloud_network.main.id +} + +# Exposed so dependents can depend_on the subnet existing, not just the network - +# attaching a server to a network fails if the network has no subnet yet. +output "subnet_id" { + value = hcloud_network_subnet.main.id +} diff --git a/infra/modules/network/variables.tf b/infra/modules/network/variables.tf new file mode 100644 index 0000000..ba05090 --- /dev/null +++ b/infra/modules/network/variables.tf @@ -0,0 +1,14 @@ +variable "ip_range" { + description = "IP range of the private network shared by dev and prod." + type = string +} + +variable "network_zone" { + description = "Hetzner Cloud network zone matching var.location." + type = string +} + +variable "subnet_ip_range" { + description = "IP range of the network subnet shared by dev and prod." + type = string +} diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf new file mode 100644 index 0000000..4f1d733 --- /dev/null +++ b/infra/modules/prod/main.tf @@ -0,0 +1,106 @@ +# prod: Traefik, Websites, Bitwarden, RustDesk, status page + +# Port sources: services/prod/*docker-compose.yml (published ports) and services/todo.md +# (the documented UFW allow-list). +resource "hcloud_firewall" "this" { + name = "prod-firewall" + + rule { # server ssh + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.allowed_ssh_source_ips + } + + rule { # Traefik http/https (Websites, Bitwarden, status page) + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # mongo via Traefik tcp entrypoint + direction = "in" + protocol = "tcp" + port = "27017" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # rustdesk hbbs + direction = "in" + protocol = "tcp" + port = "21115" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "21116" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "udp" + port = "21116" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # rustdesk hbbr + direction = "in" + protocol = "tcp" + port = "21117" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "21119" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # traffic from dev over the private network + direction = "in" + protocol = "tcp" + port = "1-65535" + source_ips = [var.network_ip_range] + } + + rule { + direction = "in" + protocol = "udp" + port = "1-65535" + source_ips = [var.network_ip_range] + } +} + +resource "hcloud_server" "this" { + name = "prod" + server_type = var.server_type + image = var.image + location = var.location + ssh_keys = [var.ssh_key_id] + firewall_ids = [hcloud_firewall.this.id] + + network { + network_id = var.network_id + ip = var.private_ip + } +} + +resource "hcloud_volume" "storage" { + name = "prod-storage" + size = var.volume_size + server_id = hcloud_server.this.id + automount = true + format = "ext4" +} diff --git a/infra/modules/prod/outputs.tf b/infra/modules/prod/outputs.tf new file mode 100644 index 0000000..4897389 --- /dev/null +++ b/infra/modules/prod/outputs.tf @@ -0,0 +1,7 @@ +output "ipv4" { + value = hcloud_server.this.ipv4_address +} + +output "private_ipv4" { + value = var.private_ip +} diff --git a/infra/modules/prod/variables.tf b/infra/modules/prod/variables.tf new file mode 100644 index 0000000..6acee71 --- /dev/null +++ b/infra/modules/prod/variables.tf @@ -0,0 +1,44 @@ +variable "server_type" { + description = "Server type for prod (Traefik, Websites, Bitwarden, RustDesk, status page)." + type = string +} + +variable "image" { + description = "OS image used for the server." + type = string +} + +variable "location" { + description = "Hetzner Cloud datacenter location." + type = string +} + +variable "ssh_key_id" { + description = "ID of the Hetzner Cloud SSH key to install on the server." + type = string +} + +variable "network_id" { + description = "ID of the private network to attach prod to." + type = string +} + +variable "private_ip" { + description = "Private network IP for prod." + type = string +} + +variable "volume_size" { + description = "Size in GB of the volume attached to prod (disk1 in docs/architecture.md)." + type = number +} + +variable "allowed_ssh_source_ips" { + description = "CIDRs allowed to reach port 22 on prod." + type = list(string) +} + +variable "network_ip_range" { + description = "CIDR of the private network, allowed through the firewall for traffic from dev." + type = string +} diff --git a/infra/modules/vpn/main.tf b/infra/modules/vpn/main.tf new file mode 100644 index 0000000..e0b6953 --- /dev/null +++ b/infra/modules/vpn/main.tf @@ -0,0 +1,41 @@ +# vpn: OpenVPN + Traefik. Not attached to the private network (see docs/architecture.md). +resource "hcloud_firewall" "this" { + name = "vpn-firewall" + + rule { # server ssh + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.allowed_ssh_source_ips + } + + rule { # Traefik http/https (traefik.vpn.luke-else.co.uk) + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + + rule { # OpenVPN tunnel - always direct to this server's public IP, never via a load balancer + direction = "in" + protocol = "udp" + port = "1194" + source_ips = ["0.0.0.0/0", "::/0"] + } +} + +resource "hcloud_server" "this" { + name = "vpn" + server_type = var.server_type + image = var.image + location = var.location + ssh_keys = [var.ssh_key_id] + firewall_ids = [hcloud_firewall.this.id] +} diff --git a/infra/modules/vpn/outputs.tf b/infra/modules/vpn/outputs.tf new file mode 100644 index 0000000..8952de2 --- /dev/null +++ b/infra/modules/vpn/outputs.tf @@ -0,0 +1,3 @@ +output "ipv4" { + value = hcloud_server.this.ipv4_address +} diff --git a/infra/modules/vpn/variables.tf b/infra/modules/vpn/variables.tf new file mode 100644 index 0000000..75c7fb8 --- /dev/null +++ b/infra/modules/vpn/variables.tf @@ -0,0 +1,24 @@ +variable "server_type" { + description = "Server type for vpn (OpenVPN + Traefik)." + type = string +} + +variable "image" { + description = "OS image used for the server." + type = string +} + +variable "location" { + description = "Hetzner Cloud datacenter location." + type = string +} + +variable "ssh_key_id" { + description = "ID of the Hetzner Cloud SSH key to install on the server." + type = string +} + +variable "allowed_ssh_source_ips" { + description = "CIDRs allowed to reach port 22 on vpn." + type = list(string) +} diff --git a/infra/outputs.tf b/infra/outputs.tf index 68041a3..989bf4c 100644 --- a/infra/outputs.tf +++ b/infra/outputs.tf @@ -1,19 +1,19 @@ output "dev_ipv4" { - value = hcloud_server.dev.ipv4_address + value = module.dev.ipv4 } output "dev_private_ipv4" { - value = var.dev_private_ip + value = module.dev.private_ipv4 } output "prod_ipv4" { - value = hcloud_server.prod.ipv4_address + value = module.prod.ipv4 } output "prod_private_ipv4" { - value = var.prod_private_ip + value = module.prod.private_ipv4 } output "vpn_ipv4" { - value = hcloud_server.vpn.ipv4_address + value = module.vpn.ipv4 } diff --git a/infra/servers.tf b/infra/servers.tf deleted file mode 100644 index 18bb96f..0000000 --- a/infra/servers.tf +++ /dev/null @@ -1,43 +0,0 @@ -# dev: Gitea + Runner + TraefikRunner (git.luke-else.co.uk, cicd.luke-else.co.uk) -resource "hcloud_server" "dev" { - name = "dev" - server_type = var.dev_server_type - image = var.server_image - location = var.location - ssh_keys = [data.hcloud_ssh_key.main.id] - firewall_ids = [hcloud_firewall.dev.id] - - network { - network_id = hcloud_network.main.id - ip = var.dev_private_ip - } - - depends_on = [hcloud_network_subnet.main] -} - -# prod: Traefik, Websites, Database, Bitwarden, Misc, Tracking, Rustdesk -resource "hcloud_server" "prod" { - name = "prod" - server_type = var.prod_server_type - image = var.server_image - location = var.location - ssh_keys = [data.hcloud_ssh_key.main.id] - firewall_ids = [hcloud_firewall.prod.id] - - network { - network_id = hcloud_network.main.id - ip = var.prod_private_ip - } - - depends_on = [hcloud_network_subnet.main] -} - -# vpn: OpenVPN + TraefikVPN. Not attached to the private network (see architecture.md). -resource "hcloud_server" "vpn" { - name = "vpn" - server_type = var.vpn_server_type - image = var.server_image - location = var.location - ssh_keys = [data.hcloud_ssh_key.main.id] - firewall_ids = [hcloud_firewall.vpn.id] -} diff --git a/infra/variables.tf b/infra/variables.tf index 53478de..020d0c9 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -17,33 +17,33 @@ variable "server_image" { } variable "dev_server_type" { - description = "Server type for dev (Gitea + Runner)." + description = "Server type for dev" type = string default = "cx22" } variable "prod_server_type" { - description = "Server type for prod (Traefik, Websites, Database, Bitwarden, Misc, Tracking, Rustdesk)." + description = "Server type for prod" type = string - default = "cx32" + default = "cx22" } variable "vpn_server_type" { - description = "Server type for vpn (OpenVPN + TraefikVPN)." + description = "Server type for vpn" type = string default = "cx22" } variable "dev_volume_size" { - description = "Size in GB of the volume attached to dev (disk2 in architecture.md)." + description = "Size in GB of the volume attached to dev" type = number - default = 50 + default = 20 } variable "prod_volume_size" { - description = "Size in GB of the volume attached to prod (disk1 in architecture.md)." + description = "Size in GB of the volume attached to prod (disk1 in docs/architecture.md)." type = number - default = 50 + default = 20 } variable "network_ip_range" { diff --git a/infra/volumes.tf b/infra/volumes.tf deleted file mode 100644 index 5a8863c..0000000 --- a/infra/volumes.tf +++ /dev/null @@ -1,17 +0,0 @@ -# disk1 in architecture.md - attached to prod -resource "hcloud_volume" "prod_storage" { - name = "prod-storage" - size = var.prod_volume_size - server_id = hcloud_server.prod.id - automount = true - format = "ext4" -} - -# disk2 in architecture.md - attached to dev -resource "hcloud_volume" "dev_storage" { - name = "dev-storage" - size = var.dev_volume_size - server_id = hcloud_server.dev.id - automount = true - format = "ext4" -} From 242e8aa359500aefd9ea1747745b7b26f5dffc73 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Fri, 10 Jul 2026 20:59:55 +0100 Subject: [PATCH 06/24] chore: Added missing changes from last commit --- infra/modules/prod/main.tf | 7 ------- readme.md | 37 +++++++++++++++++++------------------ 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index 4f1d733..b9248ca 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -26,13 +26,6 @@ resource "hcloud_firewall" "this" { source_ips = ["0.0.0.0/0", "::/0"] } - rule { # mongo via Traefik tcp entrypoint - direction = "in" - protocol = "tcp" - port = "27017" - source_ips = ["0.0.0.0/0", "::/0"] - } - rule { # rustdesk hbbs direction = "in" protocol = "tcp" diff --git a/readme.md b/readme.md index 0a21744..dad3f3c 100644 --- a/readme.md +++ b/readme.md @@ -60,22 +60,24 @@ architecture-beta | `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | `prod-storage` | | `vpn` | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | none | -`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach deployment targets on `prod` without exposing that traffic publicly. `vpn` is deliberately kept off this network. Each server has its own Hetzner Cloud Firewall (see `infra/firewalls.tf`) that only opens the ports actually used by the compose stacks running on it, plus SSH restricted to `var.allowed_ssh_source_ips`. +`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach deployment targets on `prod` without exposing that traffic publicly. `vpn` is deliberately kept off this network. Each server has its own Hetzner Cloud Firewall (see `infra/modules//main.tf`) that only opens the ports actually used by the compose stacks running on it, plus SSH restricted to `var.allowed_ssh_source_ips`. ## Repository layout ``` . ├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes -│ ├── servers.tf -│ ├── network.tf -│ ├── firewalls.tf -│ ├── volumes.tf -│ ├── ssh.tf -│ ├── variables.tf -│ ├── outputs.tf -│ ├── versions.tf -│ └── terraform.tfvars.example +│ ├── main.tf # root module: wires network + dev/prod/vpn modules together +│ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key name) +│ ├── outputs.tf # pass-through outputs from each host module +│ ├── ssh.tf # looks up the SSH key already uploaded to Hetzner Cloud +│ ├── versions.tf # provider requirements +│ ├── terraform.tfvars.example +│ └── modules/ +│ ├── network/ # shared private network + subnet (used by dev and prod) +│ ├── dev/ # dev server + firewall + volume +│ ├── prod/ # prod server + firewall + volume +│ └── vpn/ # vpn server + firewall (no private network, no volume) ├── services/ # Docker Compose stacks, grouped by which server they run on │ ├── dev/ # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) │ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik @@ -87,7 +89,7 @@ architecture-beta └── assets/ ``` -Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. +Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` mirrors this same dev/prod/vpn split on the provisioning side, so a given host's cloud resources and its compose stacks are easy to find side by side. ## Prerequisites @@ -110,15 +112,14 @@ tofu plan tofu apply ``` -This creates: -- `hcloud_server.dev`, `hcloud_server.prod`, `hcloud_server.vpn` -- `hcloud_network.main` + subnet, shared by `dev` and `prod` -- `hcloud_firewall.dev` / `.prod` / `.vpn`, scoped to the ports each host actually uses -- `hcloud_volume.dev_storage` / `.prod_storage` +This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` in `infra/main.tf`: +- `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`) +- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses (`modules/dev`, `modules/prod`, `modules/vpn`) +- one `hcloud_volume` each for `dev` and `prod` (`modules/dev`, `modules/prod`; `vpn` has none) Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`. -`terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and can be overridden per-environment via `terraform.tfvars`. +`terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`. ## Deploying the services (`services/`) @@ -203,4 +204,4 @@ Then reopen the repo in VS Code with the Dev Containers extension. - Real secrets (`HCLOUD_TOKEN`, `*.tfvars`, `.env` files) must never be committed — see `.gitignore`. - SSH is restricted to `var.allowed_ssh_source_ips` on every host; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). - `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly. -- Firewalls are allowlists scoped per host in `infra/firewalls.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. +- Firewalls are allowlists scoped per host in `infra/modules//main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. From d4167d5d60b6f0f26857f2f1f3d7d2facc2169ac Mon Sep 17 00:00:00 2001 From: Luke Else Date: Fri, 10 Jul 2026 21:28:32 +0100 Subject: [PATCH 07/24] feat: Created post-install script --- infra/main.tf | 24 ++++++++++++++-- infra/modules/dev/main.tf | 21 +++++++++++++- infra/modules/dev/variables.tf | 12 +++++++- infra/modules/prod/main.tf | 21 +++++++++++++- infra/modules/prod/variables.tf | 12 +++++++- infra/modules/vpn/main.tf | 19 +++++++++++++ infra/modules/vpn/variables.tf | 10 +++++++ infra/scripts/bootstrap.sh.tftpl | 49 ++++++++++++++++++++++++++++++++ infra/terraform.tfvars.example | 12 ++++++-- infra/variables.tf | 13 ++++++++- 10 files changed, 183 insertions(+), 10 deletions(-) create mode 100644 infra/scripts/bootstrap.sh.tftpl diff --git a/infra/main.tf b/infra/main.tf index 6821232..0c96d20 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -2,6 +2,20 @@ # (one per server in docs/architecture.md). Each host module owns its own # firewall (and volume, where applicable) - see modules//main.tf. +locals { + # SSH to dev/prod is only permitted from the vpn server's public IP: admins must + # tunnel through the VPN first, whose egress traffic is then sourced from this IP. + # vpn itself can't require tunneling through itself, so it keeps + # var.allowed_ssh_source_ips for direct SSH access. + vpn_ssh_source_ips = ["${module.vpn.ipv4}/32"] + + # Rendered once and uploaded+run by every host module immediately after its + # server is created - see scripts/bootstrap.sh.tftpl. + bootstrap_script = templatefile("${path.module}/scripts/bootstrap.sh.tftpl", { + deploy_user = var.deploy_user + }) +} + module "network" { source = "./modules/network" @@ -20,8 +34,10 @@ module "dev" { network_id = module.network.id private_ip = var.dev_private_ip volume_size = var.dev_volume_size - allowed_ssh_source_ips = var.allowed_ssh_source_ips + allowed_ssh_source_ips = local.vpn_ssh_source_ips network_ip_range = var.network_ip_range + bootstrap_script = local.bootstrap_script + ssh_private_key_path = var.ssh_private_key_path # module.network.id alone doesn't guarantee the subnet exists yet, and a server # can't join a network before it has a subnet. @@ -38,8 +54,10 @@ module "prod" { network_id = module.network.id private_ip = var.prod_private_ip volume_size = var.prod_volume_size - allowed_ssh_source_ips = var.allowed_ssh_source_ips + allowed_ssh_source_ips = local.vpn_ssh_source_ips network_ip_range = var.network_ip_range + bootstrap_script = local.bootstrap_script + ssh_private_key_path = var.ssh_private_key_path depends_on = [module.network] } @@ -52,4 +70,6 @@ module "vpn" { location = var.location ssh_key_id = data.hcloud_ssh_key.main.id allowed_ssh_source_ips = var.allowed_ssh_source_ips + bootstrap_script = local.bootstrap_script + ssh_private_key_path = var.ssh_private_key_path } diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf index 90552be..7c117a0 100644 --- a/infra/modules/dev/main.tf +++ b/infra/modules/dev/main.tf @@ -6,7 +6,7 @@ resource "hcloud_firewall" "this" { name = "dev-firewall" - rule { # server ssh + rule { # server ssh - wired to the vpn server's public IP only, see infra/main.tf direction = "in" protocol = "tcp" port = "22" @@ -61,6 +61,25 @@ resource "hcloud_server" "this" { network_id = var.network_id ip = var.private_ip } + + connection { + type = "ssh" + host = self.ipv4_address + user = "root" + private_key = file(var.ssh_private_key_path) + } + + provisioner "file" { + content = var.bootstrap_script + destination = "/root/bootstrap.sh" + } + + provisioner "remote-exec" { + inline = [ + "chmod +x /root/bootstrap.sh", + "/root/bootstrap.sh", + ] + } } resource "hcloud_volume" "storage" { diff --git a/infra/modules/dev/variables.tf b/infra/modules/dev/variables.tf index 7c0cd9f..9a2dec0 100644 --- a/infra/modules/dev/variables.tf +++ b/infra/modules/dev/variables.tf @@ -34,7 +34,7 @@ variable "volume_size" { } variable "allowed_ssh_source_ips" { - description = "CIDRs allowed to reach port 22 on dev." + description = "CIDRs allowed to reach port 22 on dev. Set by the root module to the vpn server's public IP." type = list(string) } @@ -42,3 +42,13 @@ variable "network_ip_range" { description = "CIDR of the private network, allowed through the firewall for traffic from prod." type = string } + +variable "bootstrap_script" { + description = "Rendered post-install script, uploaded and executed on the server immediately after creation." + type = string +} + +variable "ssh_private_key_path" { + description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH." + type = string +} diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index b9248ca..1c5e5f0 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -5,7 +5,7 @@ resource "hcloud_firewall" "this" { name = "prod-firewall" - rule { # server ssh + rule { # server ssh - wired to the vpn server's public IP only, see infra/main.tf direction = "in" protocol = "tcp" port = "22" @@ -88,6 +88,25 @@ resource "hcloud_server" "this" { network_id = var.network_id ip = var.private_ip } + + connection { + type = "ssh" + host = self.ipv4_address + user = "root" + private_key = file(var.ssh_private_key_path) + } + + provisioner "file" { + content = var.bootstrap_script + destination = "/root/bootstrap.sh" + } + + provisioner "remote-exec" { + inline = [ + "chmod +x /root/bootstrap.sh", + "/root/bootstrap.sh", + ] + } } resource "hcloud_volume" "storage" { diff --git a/infra/modules/prod/variables.tf b/infra/modules/prod/variables.tf index 6acee71..ea50186 100644 --- a/infra/modules/prod/variables.tf +++ b/infra/modules/prod/variables.tf @@ -34,7 +34,7 @@ variable "volume_size" { } variable "allowed_ssh_source_ips" { - description = "CIDRs allowed to reach port 22 on prod." + description = "CIDRs allowed to reach port 22 on prod. Set by the root module to the vpn server's public IP." type = list(string) } @@ -42,3 +42,13 @@ variable "network_ip_range" { description = "CIDR of the private network, allowed through the firewall for traffic from dev." type = string } + +variable "bootstrap_script" { + description = "Rendered post-install script, uploaded and executed on the server immediately after creation." + type = string +} + +variable "ssh_private_key_path" { + description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH." + type = string +} diff --git a/infra/modules/vpn/main.tf b/infra/modules/vpn/main.tf index e0b6953..848eb7c 100644 --- a/infra/modules/vpn/main.tf +++ b/infra/modules/vpn/main.tf @@ -38,4 +38,23 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = [var.ssh_key_id] firewall_ids = [hcloud_firewall.this.id] + + connection { + type = "ssh" + host = self.ipv4_address + user = "root" + private_key = file(var.ssh_private_key_path) + } + + provisioner "file" { + content = var.bootstrap_script + destination = "/root/bootstrap.sh" + } + + provisioner "remote-exec" { + inline = [ + "chmod +x /root/bootstrap.sh", + "/root/bootstrap.sh", + ] + } } diff --git a/infra/modules/vpn/variables.tf b/infra/modules/vpn/variables.tf index 75c7fb8..31f9d31 100644 --- a/infra/modules/vpn/variables.tf +++ b/infra/modules/vpn/variables.tf @@ -22,3 +22,13 @@ variable "allowed_ssh_source_ips" { description = "CIDRs allowed to reach port 22 on vpn." type = list(string) } + +variable "bootstrap_script" { + description = "Rendered post-install script, uploaded and executed on the server immediately after creation." + type = string +} + +variable "ssh_private_key_path" { + description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH." + type = string +} diff --git a/infra/scripts/bootstrap.sh.tftpl b/infra/scripts/bootstrap.sh.tftpl new file mode 100644 index 0000000..3227620 --- /dev/null +++ b/infra/scripts/bootstrap.sh.tftpl @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Post-install bootstrap - uploaded and executed once by OpenTofu immediately +# after server creation (see modules//main.tf). Idempotent-ish: safe to +# re-run by hand, but only ever runs automatically on first create. +set -euo pipefail + +export DEBIAN_FRONTEND=noninteractive +DEPLOY_USER="${deploy_user}" + +apt-get update -y +apt-get upgrade -y + +# --- Docker Engine + Compose plugin --- +if ! command -v docker >/dev/null 2>&1; then + apt-get install -y ca-certificates curl gnupg + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + . /etc/os-release + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $VERSION_CODENAME stable" \ + > /etc/apt/sources.list.d/docker.list + apt-get update -y + apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +fi + +# --- Non-root sudo user, given the same key root was provisioned with --- +if ! id -u "$DEPLOY_USER" >/dev/null 2>&1; then + useradd --create-home --shell /bin/bash "$DEPLOY_USER" + usermod -aG sudo,docker "$DEPLOY_USER" + mkdir -p "/home/$DEPLOY_USER/.ssh" + cp /root/.ssh/authorized_keys "/home/$DEPLOY_USER/.ssh/authorized_keys" + chown -R "$DEPLOY_USER:$DEPLOY_USER" "/home/$DEPLOY_USER/.ssh" + chmod 700 "/home/$DEPLOY_USER/.ssh" + chmod 600 "/home/$DEPLOY_USER/.ssh/authorized_keys" + echo "$DEPLOY_USER ALL=(ALL) NOPASSWD:ALL" > "/etc/sudoers.d/90-$DEPLOY_USER" + chmod 440 "/etc/sudoers.d/90-$DEPLOY_USER" +fi + +# --- SSH hardening: key-only auth everywhere. Root keeps key-only login as a +# fallback rather than being fully locked out, in case $DEPLOY_USER setup above +# ever fails silently on a future image. --- +sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config +sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config +systemctl reload ssh 2>/dev/null || systemctl reload sshd + +# --- Unattended security upgrades --- +apt-get install -y unattended-upgrades +dpkg-reconfigure -f noninteractive unattended-upgrades +systemctl enable --now unattended-upgrades diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index bed6ff2..861f79c 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -8,11 +8,17 @@ # (Console > Security > SSH Keys). Required. ssh_key_name = "your-key-name" +# Local path to the private key matching ssh_key_name above. OpenTofu uses this +# once per server to upload and run the post-install bootstrap script +# (infra/scripts/bootstrap.sh.tftpl) immediately after creation. Required. +ssh_private_key_path = "~/.ssh/id_ed25519" + # Optional overrides - defaults live in variables.tf # location = "nbg1" # dev_server_type = "cx22" -# prod_server_type = "cx32" +# prod_server_type = "cx22" # vpn_server_type = "cx22" -# dev_volume_size = 50 -# prod_volume_size = 50 +# dev_volume_size = 20 +# prod_volume_size = 20 # allowed_ssh_source_ips = ["203.0.113.4/32"] +# deploy_user = "deploy" diff --git a/infra/variables.tf b/infra/variables.tf index 020d0c9..a19472c 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -76,7 +76,18 @@ variable "ssh_key_name" { } variable "allowed_ssh_source_ips" { - description = "CIDRs allowed to reach port 22 on every server. Narrow this to your own IP(s) once known." + description = "CIDRs allowed to reach port 22 on the vpn server. Narrow this to your own IP(s) once known. dev and prod don't use this - their SSH is restricted to the vpn server's own public IP instead (see infra/main.tf)." type = list(string) default = ["0.0.0.0/0", "::/0"] } + +variable "ssh_private_key_path" { + description = "Local path to the private key matching var.ssh_key_name. Used once per server by OpenTofu to upload and run the post-install bootstrap script over SSH immediately after creation (see infra/scripts/bootstrap.sh.tftpl)." + type = string +} + +variable "deploy_user" { + description = "Non-root sudo user created on every server by the bootstrap script, with the same SSH key as root." + type = string + default = "deploy" +} From 1ae71cbae332236f66b1c1da2d9c20fe330459f0 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Fri, 10 Jul 2026 21:54:42 +0100 Subject: [PATCH 08/24] feat: Added the ability to spin-up an abritrary number of runners at runtime --- infra/.terraform.lock.hcl | 37 +++++++++++++++++++ infra/main.tf | 1 + infra/modules/dev/main.tf | 15 ++++++++ .../runners-docker-compose.yml.tftpl | 30 +++++++++++++++ infra/modules/dev/variables.tf | 5 +++ infra/modules/prod/main.tf | 4 ++ infra/terraform.tfvars.example | 1 + infra/variables.tf | 6 +++ infra/versions.tf | 4 ++ 9 files changed, 103 insertions(+) create mode 100644 infra/modules/dev/templates/runners-docker-compose.yml.tftpl diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl index 8927fa3..9c18879 100644 --- a/infra/.terraform.lock.hcl +++ b/infra/.terraform.lock.hcl @@ -33,6 +33,43 @@ provider "registry.opentofu.org/hashicorp/hcloud" { ] } +provider "registry.opentofu.org/hashicorp/local" { + version = "2.9.0" + constraints = "~> 2.5" + hashes = [ + "h1:1dtKYW/5a1qob3yneL6WzOlnSGfYtJ6a2XeejCk9yb4=", + "h1:5NseXq5wU8O20ersTtV4ocrLYFFtgFr7n0pRLO1W2Rw=", + "h1:5d22ZPPK4iiygPbwRz/PJF5Es/0axVpMlPRpCR0Padw=", + "h1:AnwyolirmIlBMjH6+tV8bKkvT+5axJNYxi2y2IguiX4=", + "h1:PBp+HeseY021Fw3sLznCG27idgwPoff4cBuNmKgPL2w=", + "h1:VDxIhe4GbzdOCdmt7mQaqdwERQW6GSI7Roonts42Gr0=", + "h1:ZO6eWWnf8LjjV1q/JNeL9WLtZ6fwIttOnyN5LjCNSEo=", + "h1:dPIAf8oUAz+vW2E0iZunMvpuPddRZIztRsPSY1u+VnY=", + "h1:fwTDVG9AhFVKQZIb1EXkHv4FqzsZNlLWgkyPGDmZZEE=", + "h1:kDc465XPC7/6XFCjrMC4mTqhA9ef0FHKuJ3ZgfGNfeg=", + "h1:kGbjxrI2P8MHeyVtE1U3Q1TbyF71ExnHxtkrE+Aj6UU=", + "h1:kcoK6Afbsj54u9zaEqpecWAFKytqjBijtguCNwV3d4M=", + "h1:rxomJjDwOo+YZ+WIPc25FqEgsz9orh/2MCyUcZmFjvw=", + "h1:t0CMn/Rkwquw8l2yQ+O4ApzbMZfY2UazbsDnZygzACA=", + "h1:tJwgm2BS4xCGlElCDQEFXQoefY9Y4t0JdSKTtsPBbBo=", + "zh:13ef7ecd1e397ec5b20ea588508dd3e3b8d6c50d809ae76b079abf9dd8d02e4b", + "zh:2190c9325980076489ce02b0f5dd2c0b91fc8711cefa99e714d8619a32827ad1", + "zh:2a0cfc5600730093705071707e4a4e4e953e7d9091859e0f66b46daa1060dd5d", + "zh:2ff53eac1af43ab9a2248a0e53c963d46e19cf04bc4c3f323591cfcebb218252", + "zh:4ebc3dee700f60af9da29970052fd02fa947813162b224716862dc9d7f1f7542", + "zh:5fe6dab84ceeaa8eb3f1567c5f05578333370c472240ca5c5bfc25e92d4d5586", + "zh:66bbec16367bbf440045502c9779b11f4ac5b022c8d8d17afe12d431950838b5", + "zh:7641e5c2e4b529e869cde29ab5b1de2fd1091489eb745b19ac2709bd7f4dfd84", + "zh:855bfba0756d17ce07595ff57d7cf664443d1495127cb88fb063362734b8b22a", + "zh:aaec10f237921d60c581d1b7a66f0a8a8019d9802dc04af11b5b981f6682e01d", + "zh:e460835a38ffa1e74f6929904bfd14ef473d217fd537b7ce834abe5ce5e2ce07", + "zh:ecc4295215db0e4aea3c9329611c31e09a853e1ae207d56742403bd4f5516703", + "zh:ee6d9fae63a612072e00402894e14826af7a3351c235b9c5b423b7629a77ca29", + "zh:f2b5c8db74aa7ebcf7cd423672358437d42401675069ef67b01ff910054e49d5", + "zh:f5aff74d3eb96d4592c7bca5cd3ea89b469e84efbf382944bd0f844a57059c09", + ] +} + provider "registry.opentofu.org/hetznercloud/hcloud" { version = "1.66.0" constraints = "~> 1.45" diff --git a/infra/main.tf b/infra/main.tf index 0c96d20..03ead0e 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -38,6 +38,7 @@ module "dev" { network_ip_range = var.network_ip_range bootstrap_script = local.bootstrap_script ssh_private_key_path = var.ssh_private_key_path + runner_count = var.dev_runner_count # module.network.id alone doesn't guarantee the subnet exists yet, and a server # can't join a network before it has a subnet. diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf index 7c117a0..2aa4cbe 100644 --- a/infra/modules/dev/main.tf +++ b/infra/modules/dev/main.tf @@ -88,4 +88,19 @@ resource "hcloud_volume" "storage" { server_id = hcloud_server.this.id automount = true format = "ext4" + + lifecycle { + prevent_destroy = true + } +} + +# Renders services/dev/Runners/docker-compose.yml directly into the repo, with +# one runner service per var.runner_count. This only touches the local working +# tree - deploying the change still goes through the normal scp + spinup.sh flow. +resource "local_file" "runners_compose" { + filename = "${path.root}/../services/dev/Runners/docker-compose.yml" + content = templatefile("${path.module}/templates/runners-docker-compose.yml.tftpl", { + runner_count = var.runner_count + }) + file_permission = "0644" } diff --git a/infra/modules/dev/templates/runners-docker-compose.yml.tftpl b/infra/modules/dev/templates/runners-docker-compose.yml.tftpl new file mode 100644 index 0000000..1389842 --- /dev/null +++ b/infra/modules/dev/templates/runners-docker-compose.yml.tftpl @@ -0,0 +1,30 @@ +# Generated by OpenTofu from var.dev_runner_count - do not hand-edit. +# To change the number of runners, edit dev_runner_count in terraform.tfvars and +# run tofu apply. To change their shape, edit +# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead. +# +# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable +# (not baked in here) - services/dev/spinup.sh generates a fresh token and +# writes it to Runners/.env immediately before starting these containers. +services: +%{ for i in range(runner_count) ~} + runner-${i + 1}: + image: gitea/act_runner:latest + container_name: gitea_runner_${i + 1} + volumes: + - ./config.yaml:/config.yaml + - ./gitea_runner_${i + 1}:/data + - /var/run/docker.sock:/var/run/docker.sock + networks: + - proxy + environment: + CONFIG_FILE: /config.yaml + GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" + GITEA_RUNNER_REGISTRATION_TOKEN: "$${GITEA_RUNNER_REGISTRATION_TOKEN}" + GITEA_RUNNER_NAME: "CICD#${i + 1}" + restart: unless-stopped +%{ endfor ~} + +networks: + proxy: + external: true diff --git a/infra/modules/dev/variables.tf b/infra/modules/dev/variables.tf index 9a2dec0..9828bfb 100644 --- a/infra/modules/dev/variables.tf +++ b/infra/modules/dev/variables.tf @@ -52,3 +52,8 @@ variable "ssh_private_key_path" { description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH." type = string } + +variable "runner_count" { + description = "Number of Gitea Actions runner containers to render into services/dev/Runners/docker-compose.yml." + type = number +} diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index 1c5e5f0..f97ae0b 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -115,4 +115,8 @@ resource "hcloud_volume" "storage" { server_id = hcloud_server.this.id automount = true format = "ext4" + + lifecycle { + prevent_destroy = true + } } diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 861f79c..1338581 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -22,3 +22,4 @@ ssh_private_key_path = "~/.ssh/id_ed25519" # prod_volume_size = 20 # allowed_ssh_source_ips = ["203.0.113.4/32"] # deploy_user = "deploy" +# dev_runner_count = 3 diff --git a/infra/variables.tf b/infra/variables.tf index a19472c..b962133 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -91,3 +91,9 @@ variable "deploy_user" { type = string default = "deploy" } + +variable "dev_runner_count" { + description = "Number of Gitea Actions runner containers to run on dev. Rendered into services/dev/Runners/docker-compose.yml on every tofu apply - see infra/modules/dev/templates/runners-docker-compose.yml.tftpl." + type = number + default = 1 +} diff --git a/infra/versions.tf b/infra/versions.tf index e01c365..d991f1e 100644 --- a/infra/versions.tf +++ b/infra/versions.tf @@ -6,6 +6,10 @@ terraform { source = "hetznercloud/hcloud" version = "~> 1.45" } + local = { + source = "hashicorp/local" + version = "~> 2.5" + } } } From 547fee06fab80679922286d123c30a6c47832fd9 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Fri, 10 Jul 2026 21:55:37 +0100 Subject: [PATCH 09/24] chore: committing missing files from previous logic changes --- .gitignore | 5 +++ readme.md | 58 ++++++++++++++++++++----- services/dev/Runners/docker-compose.yml | 29 +++++-------- services/dev/spindown.sh | 1 + services/dev/spinup.sh | 16 +++++++ services/todo.md | 4 -- 6 files changed, 80 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 836aa13..1aad0ee 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,11 @@ # Don't allow .env files, just env-exmaple files !.env-example +# Secrets generated at deploy time (e.g. services/dev/Runners/.env, holding the +# live Gitea Actions registration token) - never committed, regardless of the +# services/docker-compose.yml carve-out above. +**/.env + ### Terraform ### # Local .terraform directories **/.terraform/* diff --git a/readme.md b/readme.md index dad3f3c..758f4a3 100644 --- a/readme.md +++ b/readme.md @@ -73,9 +73,13 @@ architecture-beta │ ├── ssh.tf # looks up the SSH key already uploaded to Hetzner Cloud │ ├── versions.tf # provider requirements │ ├── terraform.tfvars.example +│ ├── scripts/ +│ │ └── bootstrap.sh.tftpl # post-install script run on every host right after creation │ └── modules/ │ ├── network/ # shared private network + subnet (used by dev and prod) -│ ├── dev/ # dev server + firewall + volume +│ ├── dev/ # dev server + firewall + volume + renders services/dev/Runners/docker-compose.yml +│ │ └── templates/ +│ │ └── runners-docker-compose.yml.tftpl # shape of each generated runner service │ ├── prod/ # prod server + firewall + volume │ └── vpn/ # vpn server + firewall (no private network, no volume) ├── services/ # Docker Compose stacks, grouped by which server they run on @@ -94,18 +98,19 @@ Each of `services/dev`, `services/prod`, `services/vpn` follows the same convent ## Prerequisites - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token -- An SSH key uploaded to that project (Console → Security → SSH Keys) +- An SSH key uploaded to that project (Console → Security → SSH Keys), and the matching private key available locally (OpenTofu uses it once per server to run the post-install bootstrap — see below) - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` -- Docker + the Compose plugin on each target server - DNS records for the domains in [Service inventory](#service-inventory) pointed at the relevant server's public IP +Docker + the Compose plugin no longer need installing by hand — the bootstrap script below handles that. + ## Provisioning the infrastructure (`infra/`) ```sh cd infra export HCLOUD_TOKEN=your-hetzner-api-token # never commit this cp terraform.tfvars.example terraform.tfvars -$EDITOR terraform.tfvars # set ssh_key_name at minimum +$EDITOR terraform.tfvars # set ssh_key_name and ssh_private_key_path at minimum tofu init tofu plan @@ -121,17 +126,46 @@ Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vp `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`. +### Post-install bootstrap + +Immediately after each `hcloud_server` is created, OpenTofu uploads and runs [`infra/scripts/bootstrap.sh.tftpl`](infra/scripts/bootstrap.sh.tftpl) over SSH as `root` (`connection` + `file`/`remote-exec` provisioners in each `modules//main.tf`). It: + +- installs Docker Engine + the Compose plugin +- creates a non-root sudo user (`var.deploy_user`, default `deploy`) with the same SSH key as root, in the `sudo` and `docker` groups +- hardens `sshd`: password authentication off, root login restricted to key-only (`PermitRootLogin prohibit-password`) +- installs and enables `unattended-upgrades` for automatic security patches + +After bootstrap, SSH in as `var.deploy_user` for day-to-day work — root key-based login still works as a fallback. + +### Scaling Gitea Actions runners + +The number of Gitea Actions runner containers on `dev` is an OpenTofu input, `var.dev_runner_count` (default `3`). On every `tofu apply`, `modules/dev`'s `local_file.runners_compose` resource renders [`modules/dev/templates/runners-docker-compose.yml.tftpl`](infra/modules/dev/templates/runners-docker-compose.yml.tftpl) straight into [`services/dev/Runners/docker-compose.yml`](services/dev/Runners/docker-compose.yml) in your working tree — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated: change `dev_runner_count` in `terraform.tfvars` and re-run `tofu apply` rather than hand-editing it. + +This only updates your local working tree — `tofu apply` doesn't reach out and start containers on `dev` itself. Redeploy as usual (re-copy `services/dev/` to the server and re-run `spinup.sh`) to apply a count change. + +Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore. + +### First apply: bootstrapping order matters + +`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch `tofu apply`, the `dev`/`prod` bootstrap provisioners can't connect yet. Bring the estate up in this order: + +1. `tofu apply -target=module.vpn` — creates `vpn` only; its firewall still allows SSH from `var.allowed_ssh_source_ips`, so its own bootstrap provisioner runs fine. +2. Deploy and start the VPN service on `vpn` (see [Deploying the services](#deploying-the-services-services)), then connect to it with an OpenVPN client. +3. `tofu apply` — creates `network`, `dev`, `prod`. Now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and the `dev`/`prod` bootstrap provisioners can connect. + +If step 3 is run before you're connected to the VPN, the `dev`/`prod` provisioners will fail and Terraform will mark those servers tainted — re-running `tofu apply` once connected will recreate and re-bootstrap them. + ## Deploying the services (`services/`) Once a server exists, copy the relevant `services//` directory to it (e.g. `scp -r services/prod user@:~/services`) and run the matching script from inside that directory: ```sh # on dev -./spinup.sh # Traefik → Gitea Runner → Watchtower +./spinup.sh # Traefik → Gitea → (waits, generates a runner token) → Runners → Watchtower ./spindown.sh # reverse order, then prunes images/volumes # on prod -./spinup.sh # Traefik, then (after a 20s cert/registry settle) Watchtower, status, websites, Bitwarden, RustDesk +./spinup.sh # Traefik, → Watchtower → status → websites → Bitwarden → RustDesk ./spindown.sh # on vpn @@ -159,7 +193,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi |---|---|---| | Traefik | `traefik-docker-compose.yml` | `traefik.cicd.luke-else.co.uk` | | Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` | -| Gitea Actions runner | `Runners/docker-compose.yml` | `cicd.luke-else.co.uk` | +| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | `cicd.luke-else.co.uk` | | Watchtower | `watchtower-docker-compose.yml` | — | ### `prod` @@ -185,9 +219,8 @@ Every public-facing service is fronted by its host's own Traefik instance, termi A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here: -- **Gitea Actions runner**: set a real `GITEA_RUNNER_REGISTRATION_TOKEN` in `services/dev/Runners/docker-compose.yml` (generate one from the Gitea admin UI) before starting the runner. - **Traefik dashboard auth**: the committed basic-auth hash is a placeholder. Generate your own with `echo $(htpasswd -nb user password) | sed -e 's/\$/\$\$/g'` and replace the `traefik-auth` middleware value in each `traefik-docker-compose.yml`. -- **General host hardening**: non-root user, UFW, unattended-upgrades, Docker install — see `services/todo.md`. +- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by the [post-install bootstrap](#post-install-bootstrap); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)). ## Development container @@ -201,7 +234,8 @@ Then reopen the repo in VS Code with the Dev Containers extension. ## Security notes -- Real secrets (`HCLOUD_TOKEN`, `*.tfvars`, `.env` files) must never be committed — see `.gitignore`. -- SSH is restricted to `var.allowed_ssh_source_ips` on every host; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). -- `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly. +- Real secrets (`HCLOUD_TOKEN`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. +- SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). +- `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP. - Firewalls are allowlists scoped per host in `infra/modules//main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. +- Every host's [post-install bootstrap](#post-install-bootstrap) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`var.deploy_user`) for day-to-day access. diff --git a/services/dev/Runners/docker-compose.yml b/services/dev/Runners/docker-compose.yml index 0e1bb51..51f84a7 100644 --- a/services/dev/Runners/docker-compose.yml +++ b/services/dev/Runners/docker-compose.yml @@ -1,31 +1,26 @@ +# Generated by OpenTofu from var.dev_runner_count - do not hand-edit. +# To change the number of runners, edit dev_runner_count in terraform.tfvars and +# run tofu apply. To change their shape, edit +# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead. +# +# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable +# (not baked in here) - services/dev/spinup.sh generates a fresh token and +# writes it to Runners/.env immediately before starting these containers. services: - # Runner - runner: + runner-1: image: gitea/act_runner:latest - container_name: gitea_runner + container_name: gitea_runner_1 volumes: - ./config.yaml:/config.yaml - - ./gitea_runner:/data + - ./gitea_runner_1:/data - /var/run/docker.sock:/var/run/docker.sock networks: - proxy environment: CONFIG_FILE: /config.yaml GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" - GITEA_RUNNER_REGISTRATION_TOKEN: "INSERT REGISTRATION TOKEN" + GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}" GITEA_RUNNER_NAME: "CICD#1" - labels: - ## Expose cicd Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.cicd-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.cicd-insecure.rule=Host(`cicd.luke-else.co.uk`)" - - "traefik.http.routers.cicd-insecure.entrypoints=web" - - - "traefik.http.routers.cicd.rule=Host(`cicd.luke-else.co.uk`)" - - "traefik.http.routers.cicd.entrypoints=websecure" - - "traefik.http.routers.cicd.tls.certresolver=myresolver" restart: unless-stopped networks: diff --git a/services/dev/spindown.sh b/services/dev/spindown.sh index 5e71bd2..be4e6a3 100755 --- a/services/dev/spindown.sh +++ b/services/dev/spindown.sh @@ -6,6 +6,7 @@ cd Runners docker compose down cd .. +docker compose -f gitea-docker-compose.yml down docker compose -f traefik-docker-compose.yml down docker compose -f watchtower-docker-compose.yml down diff --git a/services/dev/spinup.sh b/services/dev/spinup.sh index 6d9bfcc..1aa5a7c 100755 --- a/services/dev/spinup.sh +++ b/services/dev/spinup.sh @@ -5,6 +5,22 @@ cd "$(dirname "$0")" docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d +docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d + +echo "Waiting for Gitea to become ready..." +for i in $(seq 1 30); do + if docker exec gitea gitea admin user list >/dev/null 2>&1; then + break + fi + sleep 2 +done + +# Generate a fresh Actions runner registration token straight from Gitea and +# hand it to the runner containers via .env (docker compose auto-loads this) - +# no manual copy/paste from the admin UI needed. +RUNNER_TOKEN=$(docker exec gitea gitea actions generate-runner-token) +echo "GITEA_RUNNER_REGISTRATION_TOKEN=$RUNNER_TOKEN" > Runners/.env + cd Runners docker compose pull && docker compose up -d cd .. diff --git a/services/todo.md b/services/todo.md index eccd34c..158ce2c 100644 --- a/services/todo.md +++ b/services/todo.md @@ -21,10 +21,6 @@ - Ensure that ports are assigned correctly for the system -# Gitea Runner - -- Ensure that a registration token has been setup before continuing - ## Websites - Ensure website files are copied over From e39e1637da81afcb41ffbef90a764eb68de5269b Mon Sep 17 00:00:00 2001 From: Luke Else Date: Fri, 10 Jul 2026 22:01:48 +0100 Subject: [PATCH 10/24] feat: Added multiple SSH Key uploads --- infra/main.tf | 9 ++++++--- infra/modules/dev/main.tf | 2 +- infra/modules/dev/variables.tf | 8 ++++---- infra/modules/prod/main.tf | 2 +- infra/modules/prod/variables.tf | 8 ++++---- infra/modules/vpn/main.tf | 2 +- infra/modules/vpn/variables.tf | 8 ++++---- infra/ssh.tf | 3 ++- infra/terraform.tfvars.example | 11 ++++++----- infra/variables.tf | 8 ++++---- readme.md | 10 +++++----- 11 files changed, 38 insertions(+), 33 deletions(-) diff --git a/infra/main.tf b/infra/main.tf index 03ead0e..760950f 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -14,6 +14,9 @@ locals { bootstrap_script = templatefile("${path.module}/scripts/bootstrap.sh.tftpl", { deploy_user = var.deploy_user }) + + # IDs for every named key in var.ssh_key_names - installed on every server. + ssh_key_ids = [for k in data.hcloud_ssh_key.main : k.id] } module "network" { @@ -30,7 +33,7 @@ module "dev" { server_type = var.dev_server_type image = var.server_image location = var.location - ssh_key_id = data.hcloud_ssh_key.main.id + ssh_key_ids = local.ssh_key_ids network_id = module.network.id private_ip = var.dev_private_ip volume_size = var.dev_volume_size @@ -51,7 +54,7 @@ module "prod" { server_type = var.prod_server_type image = var.server_image location = var.location - ssh_key_id = data.hcloud_ssh_key.main.id + ssh_key_ids = local.ssh_key_ids network_id = module.network.id private_ip = var.prod_private_ip volume_size = var.prod_volume_size @@ -69,7 +72,7 @@ module "vpn" { server_type = var.vpn_server_type image = var.server_image location = var.location - ssh_key_id = data.hcloud_ssh_key.main.id + ssh_key_ids = local.ssh_key_ids allowed_ssh_source_ips = var.allowed_ssh_source_ips bootstrap_script = local.bootstrap_script ssh_private_key_path = var.ssh_private_key_path diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf index 2aa4cbe..e2b2d25 100644 --- a/infra/modules/dev/main.tf +++ b/infra/modules/dev/main.tf @@ -54,7 +54,7 @@ resource "hcloud_server" "this" { server_type = var.server_type image = var.image location = var.location - ssh_keys = [var.ssh_key_id] + ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] network { diff --git a/infra/modules/dev/variables.tf b/infra/modules/dev/variables.tf index 9828bfb..40a9f0c 100644 --- a/infra/modules/dev/variables.tf +++ b/infra/modules/dev/variables.tf @@ -13,9 +13,9 @@ variable "location" { type = string } -variable "ssh_key_id" { - description = "ID of the Hetzner Cloud SSH key to install on the server." - type = string +variable "ssh_key_ids" { + description = "IDs of the Hetzner Cloud SSH keys to install on the server." + type = list(string) } variable "network_id" { @@ -49,7 +49,7 @@ variable "bootstrap_script" { } variable "ssh_private_key_path" { - description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH." + description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." type = string } diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index f97ae0b..99fbc09 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -81,7 +81,7 @@ resource "hcloud_server" "this" { server_type = var.server_type image = var.image location = var.location - ssh_keys = [var.ssh_key_id] + ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] network { diff --git a/infra/modules/prod/variables.tf b/infra/modules/prod/variables.tf index ea50186..3eb6544 100644 --- a/infra/modules/prod/variables.tf +++ b/infra/modules/prod/variables.tf @@ -13,9 +13,9 @@ variable "location" { type = string } -variable "ssh_key_id" { - description = "ID of the Hetzner Cloud SSH key to install on the server." - type = string +variable "ssh_key_ids" { + description = "IDs of the Hetzner Cloud SSH keys to install on the server." + type = list(string) } variable "network_id" { @@ -49,6 +49,6 @@ variable "bootstrap_script" { } variable "ssh_private_key_path" { - description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH." + description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." type = string } diff --git a/infra/modules/vpn/main.tf b/infra/modules/vpn/main.tf index 848eb7c..e1d6ffe 100644 --- a/infra/modules/vpn/main.tf +++ b/infra/modules/vpn/main.tf @@ -36,7 +36,7 @@ resource "hcloud_server" "this" { server_type = var.server_type image = var.image location = var.location - ssh_keys = [var.ssh_key_id] + ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] connection { diff --git a/infra/modules/vpn/variables.tf b/infra/modules/vpn/variables.tf index 31f9d31..1976e9a 100644 --- a/infra/modules/vpn/variables.tf +++ b/infra/modules/vpn/variables.tf @@ -13,9 +13,9 @@ variable "location" { type = string } -variable "ssh_key_id" { - description = "ID of the Hetzner Cloud SSH key to install on the server." - type = string +variable "ssh_key_ids" { + description = "IDs of the Hetzner Cloud SSH keys to install on the server." + type = list(string) } variable "allowed_ssh_source_ips" { @@ -29,6 +29,6 @@ variable "bootstrap_script" { } variable "ssh_private_key_path" { - description = "Local path to the private key matching var.ssh_key_id, used to run the bootstrap script over SSH." + description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." type = string } diff --git a/infra/ssh.tf b/infra/ssh.tf index a6eea7e..cffda71 100644 --- a/infra/ssh.tf +++ b/infra/ssh.tf @@ -1,3 +1,4 @@ data "hcloud_ssh_key" "main" { - name = var.ssh_key_name + for_each = toset(var.ssh_key_names) + name = each.value } diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 1338581..8be0e88 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -4,12 +4,13 @@ # The Hetzner API token is NOT set here: export it as HCLOUD_TOKEN in your shell # before running tofu plan/apply. -# Name of an SSH key already uploaded to your Hetzner Cloud project -# (Console > Security > SSH Keys). Required. -ssh_key_name = "your-key-name" +# Names of SSH keys already uploaded to your Hetzner Cloud project +# (Console > Security > SSH Keys). All of them are installed on every server. Required. +ssh_key_names = ["your-key-name", "laptop", "phone"] -# Local path to the private key matching ssh_key_name above. OpenTofu uses this -# once per server to upload and run the post-install bootstrap script +# Local path to the private key matching ONE of ssh_key_names above (doesn't need +# to be all of them - just one you hold). OpenTofu uses this once per server to +# upload and run the post-install bootstrap script # (infra/scripts/bootstrap.sh.tftpl) immediately after creation. Required. ssh_private_key_path = "~/.ssh/id_ed25519" diff --git a/infra/variables.tf b/infra/variables.tf index b962133..7c041c8 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -70,9 +70,9 @@ variable "prod_private_ip" { default = "10.0.1.11" } -variable "ssh_key_name" { - description = "Name of an SSH key already uploaded to your Hetzner Cloud project (Console > Security > SSH Keys)." - type = string +variable "ssh_key_names" { + description = "Names of SSH keys already uploaded to your Hetzner Cloud project (Console > Security > SSH Keys). All are installed on every server." + type = list(string) } variable "allowed_ssh_source_ips" { @@ -82,7 +82,7 @@ variable "allowed_ssh_source_ips" { } variable "ssh_private_key_path" { - description = "Local path to the private key matching var.ssh_key_name. Used once per server by OpenTofu to upload and run the post-install bootstrap script over SSH immediately after creation (see infra/scripts/bootstrap.sh.tftpl)." + description = "Local path to the private key matching one of var.ssh_key_names - only needs to be a key you hold yourself, not all of them. Used once per server by OpenTofu to upload and run the post-install bootstrap script over SSH immediately after creation (see infra/scripts/bootstrap.sh.tftpl)." type = string } diff --git a/readme.md b/readme.md index 758f4a3..eab589f 100644 --- a/readme.md +++ b/readme.md @@ -68,9 +68,9 @@ architecture-beta . ├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes │ ├── main.tf # root module: wires network + dev/prod/vpn modules together -│ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key name) +│ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names) │ ├── outputs.tf # pass-through outputs from each host module -│ ├── ssh.tf # looks up the SSH key already uploaded to Hetzner Cloud +│ ├── ssh.tf # looks up each SSH key already uploaded to Hetzner Cloud │ ├── versions.tf # provider requirements │ ├── terraform.tfvars.example │ ├── scripts/ @@ -98,7 +98,7 @@ Each of `services/dev`, `services/prod`, `services/vpn` follows the same convent ## Prerequisites - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token -- An SSH key uploaded to that project (Console → Security → SSH Keys), and the matching private key available locally (OpenTofu uses it once per server to run the post-install bootstrap — see below) +- One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (OpenTofu uses it once per server to run the post-install bootstrap — see below) - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` - DNS records for the domains in [Service inventory](#service-inventory) pointed at the relevant server's public IP @@ -110,7 +110,7 @@ Docker + the Compose plugin no longer need installing by hand — the bootstrap cd infra export HCLOUD_TOKEN=your-hetzner-api-token # never commit this cp terraform.tfvars.example terraform.tfvars -$EDITOR terraform.tfvars # set ssh_key_name and ssh_private_key_path at minimum +$EDITOR terraform.tfvars # set ssh_key_names and ssh_private_key_path at minimum tofu init tofu plan @@ -131,7 +131,7 @@ Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vp Immediately after each `hcloud_server` is created, OpenTofu uploads and runs [`infra/scripts/bootstrap.sh.tftpl`](infra/scripts/bootstrap.sh.tftpl) over SSH as `root` (`connection` + `file`/`remote-exec` provisioners in each `modules//main.tf`). It: - installs Docker Engine + the Compose plugin -- creates a non-root sudo user (`var.deploy_user`, default `deploy`) with the same SSH key as root, in the `sudo` and `docker` groups +- creates a non-root sudo user (`var.deploy_user`, default `deploy`) with the same SSH keys as root, in the `sudo` and `docker` groups - hardens `sshd`: password authentication off, root login restricted to key-only (`PermitRootLogin prohibit-password`) - installs and enables `unattended-upgrades` for automatic security patches From ae34a78a1da3ae7fafb6d78439712f6fdfd55afe Mon Sep 17 00:00:00 2001 From: Luke Else Date: Fri, 10 Jul 2026 22:19:08 +0100 Subject: [PATCH 11/24] feat: Added DNS control to tofu --- infra/main.tf | 27 +++++++++++++++++++++++++++ infra/modules/dns/main.tf | 28 ++++++++++++++++++++++++++++ infra/modules/dns/outputs.tf | 9 +++++++++ infra/modules/dns/variables.tf | 15 +++++++++++++++ infra/outputs.tf | 5 +++++ infra/terraform.tfvars.example | 1 + infra/variables.tf | 6 ++++++ infra/versions.tf | 2 +- readme.md | 21 +++++++++++++++++---- 9 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 infra/modules/dns/main.tf create mode 100644 infra/modules/dns/outputs.tf create mode 100644 infra/modules/dns/variables.tf diff --git a/infra/main.tf b/infra/main.tf index 760950f..da2e66c 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -17,6 +17,26 @@ locals { # IDs for every named key in var.ssh_key_names - installed on every server. ssh_key_ids = [for k in data.hcloud_ssh_key.main : k.id] + + # Every A record currently referenced by a Traefik Host() rule anywhere under + # services/ - kept in sync with the actual compose files, not aspirational. + # "dev.luke-else.co.uk" is a *website* served from prod (a dev-tagged build of + # luke-else.co.uk), not the dev server - don't confuse the two. + dns_records = [ + { zone = "luke-else.co.uk", name = "@", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "dev", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "metarius", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "traefik", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "status", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "bitwarden", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "rd", value = module.prod.ipv4 }, + { zone = "luke-else.co.uk", name = "git", value = module.dev.ipv4 }, + { zone = "luke-else.co.uk", name = "traefik.cicd", value = module.dev.ipv4 }, + { zone = "luke-else.co.uk", name = "vpn", value = module.vpn.ipv4 }, + { zone = "luke-else.co.uk", name = "traefik.vpn", value = module.vpn.ipv4 }, + { zone = "divine-couture.co.uk", name = "www", value = module.prod.ipv4 }, + { zone = "snexo.co.uk", name = "@", value = module.prod.ipv4 }, + ] } module "network" { @@ -77,3 +97,10 @@ module "vpn" { bootstrap_script = local.bootstrap_script ssh_private_key_path = var.ssh_private_key_path } + +module "dns" { + source = "./modules/dns" + + zone_names = var.dns_zones + records = local.dns_records +} diff --git a/infra/modules/dns/main.tf b/infra/modules/dns/main.tf new file mode 100644 index 0000000..08952ee --- /dev/null +++ b/infra/modules/dns/main.tf @@ -0,0 +1,28 @@ +# One Hetzner DNS zone per domain in var.zone_names. Hetzner becomes +# authoritative for these once the domain's registrar NS records are pointed at +# the nameservers in the `nameservers` output - that step happens at whichever +# registrar the domain was bought through, and can't be done from here. +resource "hcloud_zone" "this" { + for_each = toset(var.zone_names) + + name = each.value + mode = "primary" + delete_protection = true # API-level guard, in addition to prevent_destroy below + + lifecycle { + prevent_destroy = true + } +} + +resource "hcloud_zone_rrset" "a" { + for_each = { for r in var.records : "${r.zone}/${r.name}" => r } + + zone = hcloud_zone.this[each.value.zone].id + name = each.value.name + type = "A" + ttl = each.value.ttl + + records = [ + { value = each.value.value } + ] +} diff --git a/infra/modules/dns/outputs.tf b/infra/modules/dns/outputs.tf new file mode 100644 index 0000000..2ef778d --- /dev/null +++ b/infra/modules/dns/outputs.tf @@ -0,0 +1,9 @@ +output "zone_ids" { + description = "Map of domain name to Hetzner zone ID." + value = { for name, z in hcloud_zone.this : name => z.id } +} + +output "nameservers" { + description = "Map of domain name to the nameservers Hetzner assigned it - point your registrar's NS records at these to delegate DNS." + value = { for name, z in hcloud_zone.this : name => z.authoritative_nameservers.assigned } +} diff --git a/infra/modules/dns/variables.tf b/infra/modules/dns/variables.tf new file mode 100644 index 0000000..7f8265e --- /dev/null +++ b/infra/modules/dns/variables.tf @@ -0,0 +1,15 @@ +variable "zone_names" { + description = "Domains to create as Hetzner DNS zones." + type = list(string) +} + +variable "records" { + description = "A records to create across the zones in var.zone_names. Use name = \"@\" for the zone apex." + type = list(object({ + zone = string + name = string + value = string + ttl = optional(number, 300) + })) + default = [] +} diff --git a/infra/outputs.tf b/infra/outputs.tf index 989bf4c..f8e427c 100644 --- a/infra/outputs.tf +++ b/infra/outputs.tf @@ -17,3 +17,8 @@ output "prod_private_ipv4" { output "vpn_ipv4" { value = module.vpn.ipv4 } + +output "dns_nameservers" { + description = "Nameservers Hetzner assigned each zone - point each domain's registrar NS records at these to delegate DNS to Hetzner." + value = module.dns.nameservers +} diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 8be0e88..7ab14ec 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -24,3 +24,4 @@ ssh_private_key_path = "~/.ssh/id_ed25519" # allowed_ssh_source_ips = ["203.0.113.4/32"] # deploy_user = "deploy" # dev_runner_count = 3 +# dns_zones = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"] diff --git a/infra/variables.tf b/infra/variables.tf index 7c041c8..10f6f43 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -97,3 +97,9 @@ variable "dev_runner_count" { type = number default = 1 } + +variable "dns_zones" { + description = "Domains to manage as Hetzner DNS zones. Point each domain's registrar NS records at tofu output dns_nameservers for Hetzner to actually become authoritative." + type = list(string) + default = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"] +} diff --git a/infra/versions.tf b/infra/versions.tf index d991f1e..9552a41 100644 --- a/infra/versions.tf +++ b/infra/versions.tf @@ -4,7 +4,7 @@ terraform { required_providers { hcloud = { source = "hetznercloud/hcloud" - version = "~> 1.45" + version = "~> 1.54" # hcloud_zone / hcloud_zone_rrset (DNS) require >= 1.54.0 } local = { source = "hashicorp/local" diff --git a/readme.md b/readme.md index eab589f..72f2796 100644 --- a/readme.md +++ b/readme.md @@ -81,7 +81,8 @@ architecture-beta │ │ └── templates/ │ │ └── runners-docker-compose.yml.tftpl # shape of each generated runner service │ ├── prod/ # prod server + firewall + volume -│ └── vpn/ # vpn server + firewall (no private network, no volume) +│ ├── vpn/ # vpn server + firewall (no private network, no volume) +│ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones ├── services/ # Docker Compose stacks, grouped by which server they run on │ ├── dev/ # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) │ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik @@ -100,7 +101,7 @@ Each of `services/dev`, `services/prod`, `services/vpn` follows the same convent - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token - One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (OpenTofu uses it once per server to run the post-install bootstrap — see below) - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` -- DNS records for the domains in [Service inventory](#service-inventory) pointed at the relevant server's public IP +- Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner (see [Managing DNS](#managing-dns) — the zones and records themselves are created for you) Docker + the Compose plugin no longer need installing by hand — the bootstrap script below handles that. @@ -117,12 +118,13 @@ tofu plan tofu apply ``` -This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` in `infra/main.tf`: +This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` / `module.dns` in `infra/main.tf`: - `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`) - one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses (`modules/dev`, `modules/prod`, `modules/vpn`) - one `hcloud_volume` each for `dev` and `prod` (`modules/dev`, `modules/prod`; `vpn` has none) +- one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns)) -Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`. +Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`. `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`. @@ -155,6 +157,16 @@ Registration tokens are handled automatically, not baked into the generated file If step 3 is run before you're connected to the VPN, the `dev`/`prod` provisioners will fail and Terraform will mark those servers tainted — re-running `tofu apply` once connected will recreate and re-bootstrap them. +### Managing DNS + +`module.dns` (in `infra/modules/dns`) creates one Hetzner DNS zone per domain in `var.dns_zones` (default: `luke-else.co.uk`, `divine-couture.co.uk`, `snexo.co.uk`) and an `hcloud_zone_rrset` A record for every hostname currently referenced by a Traefik `Host()` rule anywhere under `services/` — kept in sync with `local.dns_records` in `infra/main.tf`, pointed at whichever of `dev`/`prod`/`vpn` actually serves it. Zones have `prevent_destroy = true`, matching the volumes — losing one deletes every record in it. + +Creating the zone doesn't make Hetzner authoritative for the domain by itself: you still need to point that domain's NS records at Hetzner's nameservers at whichever registrar it's registered through. Run `tofu output dns_nameservers` after applying to get the exact nameservers per domain, and set those as the domain's NS records at the registrar. Propagation can take a while depending on the registrar and the domain's previous NS TTL. + +Adding a new subdomain: add an entry to `local.dns_records` in `infra/main.tf` (zone, name, and the target `module..ipv4`) and re-run `tofu apply` — don't hand-create records in the Hetzner console, they'll drift from state. Note `name = "@"` is Hetzner's convention for a zone's apex record (e.g. bare `snexo.co.uk`), not an empty string. + +Only A records for IPv4 are managed here — none of the modules currently track servers' IPv6 addresses, so AAAA records aren't generated even though the firewalls already allow IPv6 traffic. + ## Deploying the services (`services/`) Once a server exists, copy the relevant `services//` directory to it (e.g. `scp -r services/prod user@:~/services`) and run the matching script from inside that directory: @@ -239,3 +251,4 @@ Then reopen the repo in VS Code with the Dev Containers extension. - `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP. - Firewalls are allowlists scoped per host in `infra/modules//main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. - Every host's [post-install bootstrap](#post-install-bootstrap) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`var.deploy_user`) for day-to-day access. +- DNS zones (`modules/dns`) carry both Hetzner's own `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, including for `divine-couture.co.uk` and `snexo.co.uk`, not just `luke-else.co.uk`. From 41d93563d0a61635bec8a3dac0c847ab89b30326 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Fri, 10 Jul 2026 22:24:58 +0100 Subject: [PATCH 12/24] chore: Removed password auth from traefik as not needed --- readme.md | 3 +-- services/dev/traefik-docker-compose.yml | 6 ++---- services/prod/traefik-docker-compose.yml | 2 -- services/vpn/traefik-docker-compose.yml | 6 ++---- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/readme.md b/readme.md index 72f2796..4e1883d 100644 --- a/readme.md +++ b/readme.md @@ -205,7 +205,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi |---|---|---| | Traefik | `traefik-docker-compose.yml` | `traefik.cicd.luke-else.co.uk` | | Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` | -| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | `cicd.luke-else.co.uk` | +| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | N/A | | Watchtower | `watchtower-docker-compose.yml` | — | ### `prod` @@ -231,7 +231,6 @@ Every public-facing service is fronted by its host's own Traefik instance, termi A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here: -- **Traefik dashboard auth**: the committed basic-auth hash is a placeholder. Generate your own with `echo $(htpasswd -nb user password) | sed -e 's/\$/\$\$/g'` and replace the `traefik-auth` middleware value in each `traefik-docker-compose.yml`. - **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by the [post-install bootstrap](#post-install-bootstrap); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)). ## Development container diff --git a/services/dev/traefik-docker-compose.yml b/services/dev/traefik-docker-compose.yml index e8970fd..41f308c 100644 --- a/services/dev/traefik-docker-compose.yml +++ b/services/dev/traefik-docker-compose.yml @@ -30,11 +30,9 @@ services: - "traefik.http.routers.traefik.rule=Host(`traefik.cicd.luke-else.co.uk`)" - "traefik.http.routers.traefik.entrypoints=websecure" - "traefik.http.routers.traefik.service=api@internal" - - "traefik.http.routers.traefik.tls.certresolver=myresolver" - - "traefik.http.routers.traefik.middlewares=traefik-auth" - - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" + - "traefik.http.routers.traefik.tls.certresolver=myresolver" restart: unless-stopped networks: proxy: - name: proxy \ No newline at end of file + name: proxy diff --git a/services/prod/traefik-docker-compose.yml b/services/prod/traefik-docker-compose.yml index 53e89ac..109a7eb 100644 --- a/services/prod/traefik-docker-compose.yml +++ b/services/prod/traefik-docker-compose.yml @@ -36,8 +36,6 @@ services: - "traefik.http.routers.traefik.entrypoints=websecure" - "traefik.http.routers.traefik.service=api@internal" - "traefik.http.routers.traefik.tls.certresolver=myresolver" - - "traefik.http.routers.traefik.middlewares=traefik-auth" - - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" restart: unless-stopped networks: diff --git a/services/vpn/traefik-docker-compose.yml b/services/vpn/traefik-docker-compose.yml index 7383006..50a0f18 100644 --- a/services/vpn/traefik-docker-compose.yml +++ b/services/vpn/traefik-docker-compose.yml @@ -30,11 +30,9 @@ services: - "traefik.http.routers.traefik.rule=Host(`traefik.vpn.luke-else.co.uk`)" - "traefik.http.routers.traefik.entrypoints=websecure" - "traefik.http.routers.traefik.service=api@internal" - - "traefik.http.routers.traefik.tls.certresolver=myresolver" - - "traefik.http.routers.traefik.middlewares=traefik-auth" - - "traefik.http.middlewares.traefik-auth.basicauth.users=user:$$2y$$05$$s/vPphFtSO2fWJR7SYkEb.90UwPDRM3aOKqgOF/rme/3fUQ5tvpTS" + - "traefik.http.routers.traefik.tls.certresolver=myresolver" restart: unless-stopped networks: proxy: - name: proxy \ No newline at end of file + name: proxy From bab789a34795954bbda25d0345c7474a70f6fd78 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Fri, 10 Jul 2026 22:58:56 +0100 Subject: [PATCH 13/24] feat: Added automatic script copying functionality --- infra/main.tf | 3 + infra/modules/dev/main.tf | 76 +++++++++++++++++++ infra/modules/dev/outputs.tf | 5 ++ .../runners-docker-compose.yml.tftpl | 7 +- infra/modules/dev/variables.tf | 5 ++ infra/modules/prod/main.tf | 72 ++++++++++++++++++ infra/modules/prod/outputs.tf | 5 ++ infra/modules/prod/variables.tf | 5 ++ infra/modules/vpn/main.tf | 44 +++++++++++ infra/modules/vpn/variables.tf | 5 ++ infra/outputs.tf | 10 +++ infra/versions.tf | 4 + readme.md | 28 +++++-- services/dev/Runners/docker-compose.yml | 7 +- services/dev/gitea-docker-compose.yml | 2 +- services/dev/spinup.sh | 5 ++ services/dev/traefik-docker-compose.yml | 2 +- services/prod/bitwarden-docker-compose.yml | 2 +- services/prod/rd-docker-compose.yml | 4 +- services/prod/spinup.sh | 5 ++ services/prod/status-docker-compose.yml | 2 +- services/prod/traefik-docker-compose.yml | 2 +- services/prod/web-docker-compose.yml | 2 +- 23 files changed, 284 insertions(+), 18 deletions(-) diff --git a/infra/main.tf b/infra/main.tf index da2e66c..328bd6b 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -61,6 +61,7 @@ module "dev" { network_ip_range = var.network_ip_range bootstrap_script = local.bootstrap_script ssh_private_key_path = var.ssh_private_key_path + deploy_user = var.deploy_user runner_count = var.dev_runner_count # module.network.id alone doesn't guarantee the subnet exists yet, and a server @@ -82,6 +83,7 @@ module "prod" { network_ip_range = var.network_ip_range bootstrap_script = local.bootstrap_script ssh_private_key_path = var.ssh_private_key_path + deploy_user = var.deploy_user depends_on = [module.network] } @@ -96,6 +98,7 @@ module "vpn" { allowed_ssh_source_ips = var.allowed_ssh_source_ips bootstrap_script = local.bootstrap_script ssh_private_key_path = var.ssh_private_key_path + deploy_user = var.deploy_user } module "dns" { diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf index e2b2d25..aed7406 100644 --- a/infra/modules/dev/main.tf +++ b/infra/modules/dev/main.tf @@ -94,6 +94,14 @@ resource "hcloud_volume" "storage" { } } +locals { + # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is + # always /mnt/HC_Volume_ - deterministic once the volume exists, since the + # id is stable for the volume's lifetime regardless of which server it's + # attached to. + data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}" +} + # Renders services/dev/Runners/docker-compose.yml directly into the repo, with # one runner service per var.runner_count. This only touches the local working # tree - deploying the change still goes through the normal scp + spinup.sh flow. @@ -101,6 +109,74 @@ resource "local_file" "runners_compose" { filename = "${path.root}/../services/dev/Runners/docker-compose.yml" content = templatefile("${path.module}/templates/runners-docker-compose.yml.tftpl", { runner_count = var.runner_count + data_dir = local.data_dir }) file_permission = "0644" } + +# DATA_DIR is picked up automatically by `docker compose` (which auto-loads +# .env from its working directory) for every dev/*-docker-compose.yml bind +# mount - see services/dev/*.yml. Not committed (see .gitignore): it's tied to +# the live volume's ID, so it's regenerated by tofu apply, not handed off via git. +resource "local_file" "env" { + filename = "${path.root}/../services/dev/.env" + content = "DATA_DIR=${local.data_dir}\n" + file_permission = "0644" +} + +locals { + # Everything under services/dev except the two files above: those are + # tracked via their own resource content (reading them back with fileset()/ + # filesha1() before they exist would fail during `tofu plan`). + dev_static_files = sort([ + for f in fileset("${path.root}/../services/dev", "**") : + f if f != ".env" && f != "Runners/docker-compose.yml" + ]) + + dev_static_files_hash = sha1(join("", [ + for f in local.dev_static_files : filesha1("${path.root}/../services/dev/${f}") + ])) +} + +# Copies services/dev to the server on every apply that changes it (a hand-edited +# compose file, a new runner count, ...) or recreates the server - not just once +# at first creation. Split from hcloud_server.this because it must run after the +# generated files above, which in turn depend on the volume, which depends on +# the server - so it can't be a provisioner on the server resource itself. +resource "null_resource" "deploy_services" { + triggers = { + server_id = hcloud_server.this.id + static_files = local.dev_static_files_hash + env = local_file.env.content + runners = local_file.runners_compose.content + } + + connection { + type = "ssh" + host = hcloud_server.this.ipv4_address + user = "root" + private_key = file(var.ssh_private_key_path) + } + + provisioner "remote-exec" { + inline = ["mkdir -p /home/${var.deploy_user}/services"] + } + + provisioner "file" { + source = "${path.root}/../services/dev" + destination = "/home/${var.deploy_user}/services" + } + + provisioner "remote-exec" { + inline = [ + "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services", + "chmod +x /home/${var.deploy_user}/services/dev/*.sh", + ] + } + + depends_on = [ + hcloud_server.this, + local_file.env, + local_file.runners_compose, + ] +} diff --git a/infra/modules/dev/outputs.tf b/infra/modules/dev/outputs.tf index 4897389..86e4ee2 100644 --- a/infra/modules/dev/outputs.tf +++ b/infra/modules/dev/outputs.tf @@ -5,3 +5,8 @@ output "ipv4" { output "private_ipv4" { value = var.private_ip } + +output "data_dir" { + description = "Deterministic host path of the auto-mounted volume - see local.data_dir." + value = local.data_dir +} diff --git a/infra/modules/dev/templates/runners-docker-compose.yml.tftpl b/infra/modules/dev/templates/runners-docker-compose.yml.tftpl index 1389842..bb42e52 100644 --- a/infra/modules/dev/templates/runners-docker-compose.yml.tftpl +++ b/infra/modules/dev/templates/runners-docker-compose.yml.tftpl @@ -6,6 +6,11 @@ # GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable # (not baked in here) - services/dev/spinup.sh generates a fresh token and # writes it to Runners/.env immediately before starting these containers. +# +# /data lives on dev's volume (${data_dir}) so runner identity survives a +# server rebuild - baked in directly rather than via .env, since Runners/.env +# is already reserved for the registration token above and gets overwritten +# on every deploy. services: %{ for i in range(runner_count) ~} runner-${i + 1}: @@ -13,7 +18,7 @@ services: container_name: gitea_runner_${i + 1} volumes: - ./config.yaml:/config.yaml - - ./gitea_runner_${i + 1}:/data + - ${data_dir}/gitea_runner_${i + 1}:/data - /var/run/docker.sock:/var/run/docker.sock networks: - proxy diff --git a/infra/modules/dev/variables.tf b/infra/modules/dev/variables.tf index 40a9f0c..49d475e 100644 --- a/infra/modules/dev/variables.tf +++ b/infra/modules/dev/variables.tf @@ -57,3 +57,8 @@ variable "runner_count" { description = "Number of Gitea Actions runner containers to render into services/dev/Runners/docker-compose.yml." type = number } + +variable "deploy_user" { + description = "Non-root sudo user created by the bootstrap script - services/dev is copied into this user's home directory." + type = string +} diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index 99fbc09..f3dc6f1 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -120,3 +120,75 @@ resource "hcloud_volume" "storage" { prevent_destroy = true } } + +locals { + # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is + # always /mnt/HC_Volume_ - deterministic once the volume exists, since the + # id is stable for the volume's lifetime regardless of which server it's + # attached to. + data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}" +} + +# DATA_DIR is picked up automatically by `docker compose` (which auto-loads +# .env from its working directory) for every prod/*-docker-compose.yml bind +# mount - see services/prod/*.yml. Not committed (see .gitignore): it's tied to +# the live volume's ID, so it's regenerated by tofu apply, not handed off via git. +resource "local_file" "env" { + filename = "${path.root}/../services/prod/.env" + content = "DATA_DIR=${local.data_dir}\n" + file_permission = "0644" +} + +locals { + # Everything under services/prod except .env: that's tracked via its own + # resource content (reading it back with fileset()/filesha1() before it + # exists would fail during `tofu plan`). + prod_static_files = sort([ + for f in fileset("${path.root}/../services/prod", "**") : f if f != ".env" + ]) + + prod_static_files_hash = sha1(join("", [ + for f in local.prod_static_files : filesha1("${path.root}/../services/prod/${f}") + ])) +} + +# Copies services/prod to the server on every apply that changes it (a +# hand-edited compose file, ...) or recreates the server - not just once at +# first creation. Split from hcloud_server.this because it must run after +# local_file.env, which in turn depends on the volume, which depends on the +# server - so it can't be a provisioner on the server resource itself. +resource "null_resource" "deploy_services" { + triggers = { + server_id = hcloud_server.this.id + static_files = local.prod_static_files_hash + env = local_file.env.content + } + + connection { + type = "ssh" + host = hcloud_server.this.ipv4_address + user = "root" + private_key = file(var.ssh_private_key_path) + } + + provisioner "remote-exec" { + inline = ["mkdir -p /home/${var.deploy_user}/services"] + } + + provisioner "file" { + source = "${path.root}/../services/prod" + destination = "/home/${var.deploy_user}/services" + } + + provisioner "remote-exec" { + inline = [ + "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services", + "chmod +x /home/${var.deploy_user}/services/prod/*.sh", + ] + } + + depends_on = [ + hcloud_server.this, + local_file.env, + ] +} diff --git a/infra/modules/prod/outputs.tf b/infra/modules/prod/outputs.tf index 4897389..86e4ee2 100644 --- a/infra/modules/prod/outputs.tf +++ b/infra/modules/prod/outputs.tf @@ -5,3 +5,8 @@ output "ipv4" { output "private_ipv4" { value = var.private_ip } + +output "data_dir" { + description = "Deterministic host path of the auto-mounted volume - see local.data_dir." + value = local.data_dir +} diff --git a/infra/modules/prod/variables.tf b/infra/modules/prod/variables.tf index 3eb6544..bab01c1 100644 --- a/infra/modules/prod/variables.tf +++ b/infra/modules/prod/variables.tf @@ -52,3 +52,8 @@ variable "ssh_private_key_path" { description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." type = string } + +variable "deploy_user" { + description = "Non-root sudo user created by the bootstrap script - services/prod is copied into this user's home directory." + type = string +} diff --git a/infra/modules/vpn/main.tf b/infra/modules/vpn/main.tf index e1d6ffe..b62fb33 100644 --- a/infra/modules/vpn/main.tf +++ b/infra/modules/vpn/main.tf @@ -58,3 +58,47 @@ resource "hcloud_server" "this" { ] } } + +locals { + vpn_static_files = sort(fileset("${path.root}/../services/vpn", "**")) + + vpn_static_files_hash = sha1(join("", [ + for f in local.vpn_static_files : filesha1("${path.root}/../services/vpn/${f}") + ])) +} + +# Copies services/vpn to the server on every apply that changes it (a +# hand-edited compose file, ...) or recreates the server - not just once at +# first creation. Split from hcloud_server.this so it's triggered by content +# changes independently of the bootstrap provisioners above. +resource "null_resource" "deploy_services" { + triggers = { + server_id = hcloud_server.this.id + static_files = local.vpn_static_files_hash + } + + connection { + type = "ssh" + host = hcloud_server.this.ipv4_address + user = "root" + private_key = file(var.ssh_private_key_path) + } + + provisioner "remote-exec" { + inline = ["mkdir -p /home/${var.deploy_user}/services"] + } + + provisioner "file" { + source = "${path.root}/../services/vpn" + destination = "/home/${var.deploy_user}/services" + } + + provisioner "remote-exec" { + inline = [ + "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services", + "chmod +x /home/${var.deploy_user}/services/vpn/*.sh", + ] + } + + depends_on = [hcloud_server.this] +} diff --git a/infra/modules/vpn/variables.tf b/infra/modules/vpn/variables.tf index 1976e9a..384a4ad 100644 --- a/infra/modules/vpn/variables.tf +++ b/infra/modules/vpn/variables.tf @@ -32,3 +32,8 @@ variable "ssh_private_key_path" { description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." type = string } + +variable "deploy_user" { + description = "Non-root sudo user created by the bootstrap script - services/vpn is copied into this user's home directory." + type = string +} diff --git a/infra/outputs.tf b/infra/outputs.tf index f8e427c..d0b29b1 100644 --- a/infra/outputs.tf +++ b/infra/outputs.tf @@ -14,6 +14,16 @@ output "prod_private_ipv4" { value = module.prod.private_ipv4 } +output "dev_data_dir" { + description = "Deterministic host path of dev's auto-mounted volume - also written to services/dev/.env as DATA_DIR." + value = module.dev.data_dir +} + +output "prod_data_dir" { + description = "Deterministic host path of prod's auto-mounted volume - also written to services/prod/.env as DATA_DIR." + value = module.prod.data_dir +} + output "vpn_ipv4" { value = module.vpn.ipv4 } diff --git a/infra/versions.tf b/infra/versions.tf index 9552a41..303b689 100644 --- a/infra/versions.tf +++ b/infra/versions.tf @@ -10,6 +10,10 @@ terraform { source = "hashicorp/local" version = "~> 2.5" } + null = { + source = "hashicorp/null" + version = "~> 3.2" + } } } diff --git a/readme.md b/readme.md index 4e1883d..f35a312 100644 --- a/readme.md +++ b/readme.md @@ -77,11 +77,11 @@ architecture-beta │ │ └── bootstrap.sh.tftpl # post-install script run on every host right after creation │ └── modules/ │ ├── network/ # shared private network + subnet (used by dev and prod) -│ ├── dev/ # dev server + firewall + volume + renders services/dev/Runners/docker-compose.yml +│ ├── dev/ # dev server + firewall + volume, renders + deploys services/dev/ │ │ └── templates/ │ │ └── runners-docker-compose.yml.tftpl # shape of each generated runner service -│ ├── prod/ # prod server + firewall + volume -│ ├── vpn/ # vpn server + firewall (no private network, no volume) +│ ├── prod/ # prod server + firewall + volume, deploys services/prod/ +│ ├── vpn/ # vpn server + firewall (no private network, no volume), deploys services/vpn/ │ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones ├── services/ # Docker Compose stacks, grouped by which server they run on │ ├── dev/ # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) @@ -124,7 +124,7 @@ This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` - one `hcloud_volume` each for `dev` and `prod` (`modules/dev`, `modules/prod`; `vpn` has none) - one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns)) -Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`. +Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`, `tofu output dev_data_dir`, `tofu output prod_data_dir`. `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`. @@ -139,6 +139,16 @@ Immediately after each `hcloud_server` is created, OpenTofu uploads and runs [`i After bootstrap, SSH in as `var.deploy_user` for day-to-day work — root key-based login still works as a fallback. +### Persistent data lives on the volumes, not the server disk + +`dev` and `prod`'s compose files used to bind-mount container data to paths relative to wherever `services//` happened to be copied on the server (e.g. `./gitea:/data`) - i.e. the server's local disk, which doesn't survive the server being destroyed and recreated. `dev-storage` and `prod-storage` (the `hcloud_volume`s) do survive that, so all stateful bind mounts now point there instead: `${DATA_DIR}/gitea:/data`, `${DATA_DIR}/bitwarden/:/data/`, and so on across `services/dev/*.yml` and `services/prod/*.yml`. `vpn` has no volume, so it's untouched - see the architecture table above. + +`DATA_DIR` needing to resolve to the right path is what makes this work, and that's answered deterministically: Hetzner's automount tooling always mounts an auto-mounted volume at `/mnt/HC_Volume_`, and since the volume is a Terraform resource, `modules//main.tf` already knows its `id` the moment it's created - `local.data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}"`. Every `tofu apply` writes that value straight into `services/dev/.env` / `services/prod/.env` as `DATA_DIR=...`, which `docker compose` auto-loads for variable substitution in any compose file run from that directory (the same mechanism the Gitea runner token already uses). Query it directly with `tofu output dev_data_dir` / `tofu output prod_data_dir`. + +These `.env` files are deliberately not committed (see `.gitignore`): the path is tied to the *live* volume's ID, so it's regenerated by `tofu apply`, not handed off via git. Run `tofu apply` at least once before the first deploy, and again if a volume is ever destroyed and recreated (new ID); `spinup.sh` on both hosts refuses to start anything if `.env` is missing, rather than silently falling back to a relative path. + +The Gitea Actions runners are the one exception to the `.env` mechanism: their `/data` path is baked directly into the generated `Runners/docker-compose.yml` at render time (via the same `templatefile()` call as `dev_runner_count`), because `Runners/.env` is already reserved for the registration token and gets overwritten on every deploy. + ### Scaling Gitea Actions runners The number of Gitea Actions runner containers on `dev` is an OpenTofu input, `var.dev_runner_count` (default `3`). On every `tofu apply`, `modules/dev`'s `local_file.runners_compose` resource renders [`modules/dev/templates/runners-docker-compose.yml.tftpl`](infra/modules/dev/templates/runners-docker-compose.yml.tftpl) straight into [`services/dev/Runners/docker-compose.yml`](services/dev/Runners/docker-compose.yml) in your working tree — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated: change `dev_runner_count` in `terraform.tfvars` and re-run `tofu apply` rather than hand-editing it. @@ -151,9 +161,9 @@ Registration tokens are handled automatically, not baked into the generated file `dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch `tofu apply`, the `dev`/`prod` bootstrap provisioners can't connect yet. Bring the estate up in this order: -1. `tofu apply -target=module.vpn` — creates `vpn` only; its firewall still allows SSH from `var.allowed_ssh_source_ips`, so its own bootstrap provisioner runs fine. -2. Deploy and start the VPN service on `vpn` (see [Deploying the services](#deploying-the-services-services)), then connect to it with an OpenVPN client. -3. `tofu apply` — creates `network`, `dev`, `prod`. Now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and the `dev`/`prod` bootstrap provisioners can connect. +1. `tofu apply -target=module.vpn` — creates `vpn` only; its firewall still allows SSH from `var.allowed_ssh_source_ips`, so its bootstrap provisioner and `null_resource.deploy_services` (which copies `services/vpn` over) both run fine. +2. SSH in as `var.deploy_user` and run `~/services/vpn/spinup.sh` to start the VPN service, then connect to it with an OpenVPN client. +3. `tofu apply` — creates `network`, `dev`, `prod`. Now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and the `dev`/`prod` bootstrap + deploy provisioners can connect. If step 3 is run before you're connected to the VPN, the `dev`/`prod` provisioners will fail and Terraform will mark those servers tainted — re-running `tofu apply` once connected will recreate and re-bootstrap them. @@ -169,7 +179,7 @@ Only A records for IPv4 are managed here — none of the modules currently track ## Deploying the services (`services/`) -Once a server exists, copy the relevant `services//` directory to it (e.g. `scp -r services/prod user@:~/services`) and run the matching script from inside that directory: +`tofu apply` already puts `services//` on the matching server for you — a `null_resource.deploy_services` in each `modules//main.tf` copies it to `/home//services/` and `chown`s it to `var.deploy_user`, re-running whenever the server is recreated or the directory's contents change (a hand-edited compose file, a new `dev_runner_count`, a refreshed `DATA_DIR`, ...) - not just once at first creation. So after applying, just SSH in as `var.deploy_user` and run the matching script from inside that directory: ```sh # on dev @@ -185,6 +195,8 @@ Once a server exists, copy the relevant `services//` directory to it (e.g. ./spindown.sh ``` +If you ever need to force a re-sync without going through Terraform (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@:~/services` still works fine - `tofu apply` will just overwrite it again next time triggers change. + Each stack can also be managed individually with plain Compose, e.g.: ```sh diff --git a/services/dev/Runners/docker-compose.yml b/services/dev/Runners/docker-compose.yml index 51f84a7..6d0f9d4 100644 --- a/services/dev/Runners/docker-compose.yml +++ b/services/dev/Runners/docker-compose.yml @@ -6,13 +6,18 @@ # GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable # (not baked in here) - services/dev/spinup.sh generates a fresh token and # writes it to Runners/.env immediately before starting these containers. +# +# /data lives on dev's volume (/mnt/HC_Volume_PENDING_TOFU_APPLY) so runner +# identity survives a server rebuild - baked in directly rather than via .env, +# since Runners/.env is already reserved for the registration token above and +# gets overwritten on every deploy. services: runner-1: image: gitea/act_runner:latest container_name: gitea_runner_1 volumes: - ./config.yaml:/config.yaml - - ./gitea_runner_1:/data + - /mnt/HC_Volume_PENDING_TOFU_APPLY/gitea_runner_1:/data - /var/run/docker.sock:/var/run/docker.sock networks: - proxy diff --git a/services/dev/gitea-docker-compose.yml b/services/dev/gitea-docker-compose.yml index 3724a79..4354a68 100644 --- a/services/dev/gitea-docker-compose.yml +++ b/services/dev/gitea-docker-compose.yml @@ -4,7 +4,7 @@ services: image: gitea/gitea:latest container_name: gitea volumes: - - ./gitea:/data + - ${DATA_DIR}/gitea:/data - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro networks: diff --git a/services/dev/spinup.sh b/services/dev/spinup.sh index 1aa5a7c..0b4e466 100755 --- a/services/dev/spinup.sh +++ b/services/dev/spinup.sh @@ -3,6 +3,11 @@ set -e cd "$(dirname "$0")" +if [ ! -f .env ]; then + echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'tofu apply' in infra/ first, then re-copy this directory." >&2 + exit 1 +fi + docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d diff --git a/services/dev/traefik-docker-compose.yml b/services/dev/traefik-docker-compose.yml index 41f308c..0a69b43 100644 --- a/services/dev/traefik-docker-compose.yml +++ b/services/dev/traefik-docker-compose.yml @@ -15,7 +15,7 @@ services: - "80:80" - "443:443" volumes: - - "./letsencrypt:/letsencrypt" + - "${DATA_DIR}/letsencrypt:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy diff --git a/services/prod/bitwarden-docker-compose.yml b/services/prod/bitwarden-docker-compose.yml index 91c1ffe..8128bda 100644 --- a/services/prod/bitwarden-docker-compose.yml +++ b/services/prod/bitwarden-docker-compose.yml @@ -5,7 +5,7 @@ services: image: "vaultwarden/server:latest" container_name: vaultwarden volumes: - - ./bitwarden/:/data/ + - ${DATA_DIR}/bitwarden/:/data/ networks: - proxy labels: diff --git a/services/prod/rd-docker-compose.yml b/services/prod/rd-docker-compose.yml index 4dc4145..bab3865 100644 --- a/services/prod/rd-docker-compose.yml +++ b/services/prod/rd-docker-compose.yml @@ -6,7 +6,7 @@ services: restart: unless-stopped volumes: - - ./data:/root + - ${DATA_DIR}/rustdesk:/root networks: - proxy @@ -39,7 +39,7 @@ services: restart: unless-stopped volumes: - - ./data:/root + - ${DATA_DIR}/rustdesk:/root networks: - proxy diff --git a/services/prod/spinup.sh b/services/prod/spinup.sh index 8fa967e..d1e874e 100755 --- a/services/prod/spinup.sh +++ b/services/prod/spinup.sh @@ -3,6 +3,11 @@ set -e cd "$(dirname "$0")" +if [ ! -f .env ]; then + echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'tofu apply' in infra/ first, then re-copy this directory." >&2 + exit 1 +fi + docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d sleep 20 # Allow Traefik + registry auth to settle before starting the rest of the services diff --git a/services/prod/status-docker-compose.yml b/services/prod/status-docker-compose.yml index 0e833c2..9595387 100644 --- a/services/prod/status-docker-compose.yml +++ b/services/prod/status-docker-compose.yml @@ -3,7 +3,7 @@ services: image: louislam/uptime-kuma:latest container_name: status volumes: - - ./uptime-kuma/data:/app/data + - ${DATA_DIR}/uptime-kuma/data:/app/data networks: - proxy labels: diff --git a/services/prod/traefik-docker-compose.yml b/services/prod/traefik-docker-compose.yml index 109a7eb..a4ed496 100644 --- a/services/prod/traefik-docker-compose.yml +++ b/services/prod/traefik-docker-compose.yml @@ -20,7 +20,7 @@ services: - "443:443" - "27017:27017" volumes: - - "./letsencrypt:/letsencrypt" + - "${DATA_DIR}/letsencrypt:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy diff --git a/services/prod/web-docker-compose.yml b/services/prod/web-docker-compose.yml index ffc2c6b..af5adac 100644 --- a/services/prod/web-docker-compose.yml +++ b/services/prod/web-docker-compose.yml @@ -81,7 +81,7 @@ services: image: "php:apache" container_name: snexo volumes: - - ./snexo.co.uk/:/var/www/html + - ${DATA_DIR}/snexo.co.uk/:/var/www/html networks: - proxy labels: From cf5504281c868cd38681026cd07f0d965d854d6c Mon Sep 17 00:00:00 2001 From: Luke Else Date: Mon, 13 Jul 2026 20:24:47 +0100 Subject: [PATCH 14/24] chore: Latest push --- .devcontainer | 2 +- infra/.terraform.lock.hcl | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.devcontainer b/.devcontainer index 424e184..fc4ccde 160000 --- a/.devcontainer +++ b/.devcontainer @@ -1 +1 @@ -Subproject commit 424e184c8fcdeb8b9b742568ee107bb0a9712aba +Subproject commit fc4ccde0acf64b80e074bcc07a69cc653e60b4d9 diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl index 9c18879..277330a 100644 --- a/infra/.terraform.lock.hcl +++ b/infra/.terraform.lock.hcl @@ -70,9 +70,46 @@ provider "registry.opentofu.org/hashicorp/local" { ] } +provider "registry.opentofu.org/hashicorp/null" { + version = "3.3.0" + constraints = "~> 3.2" + hashes = [ + "h1:0r7+t8CqzjfBgHgEiJGBCw+McEUdRXliMdF+Hk29d8o=", + "h1:EvvCOc4FJY3NitSm6BpzCcUPU53LayVCB/tPOxYmy7U=", + "h1:IDVnZXNCh0u4LfeSazc9z1v/kNz+92Eej7ePWV6SbyE=", + "h1:Iw2c0n9/4fS92N5WnJ3CCSwSUXZO953oHp9gj3pWCaM=", + "h1:JofS1og3hPN0ANjH+gNjxrJyyk6znodpC/F0qhp4eEk=", + "h1:QIBhsJ4+5+t0vFEgJwtezNLT31tsptFHOEyGAAhLR1o=", + "h1:RjjoL9qRPwNTwLdtJsYUaFvunbPM2/oujf2DcUcitOE=", + "h1:SSirA+z2VWTs1s+TCAx8vVKg9jh6cRjxqc8LYi2iQTI=", + "h1:U2XZc7hxcpcWp/C2S9LtuGUimhMOD2UT5xAEJJQQQaU=", + "h1:bPG+xE5UonkJv3y/Yn9Q7OfbP2qHU/QKiS31nwfe7S0=", + "h1:eODLdk/pARc4yxChAFtwseVmBr+r5fF9yGOvUhwGEyM=", + "h1:iFj1oM5ZPENspsPqK1kcvZzyP95jJE/CM0rlu0MfIss=", + "h1:mdu+qpyVmjDDLMrcL1JFy+cSyF58I3TFJwB5NssCZ58=", + "h1:tJmep6aoBeDH77XsYU65HAbi0RAjxtsmbCOXmnqT13U=", + "h1:tdMTn1evBLd6KCeLqWdQXCpF07hBu3n5rY6N3rXw3Rc=", + "zh:083dcc0bec53f8abfa3f2aa2ce9d732a9675338fd60ae7d61162e25db7cb08bf", + "zh:19f7456b5a2ad16595860974714bfdb25b87bc16356ea9d5c7453892aaa27864", + "zh:222c0ed1fed4e4c677ebe626104dbfdba66763e264de0d9c27c58ce60104ee69", + "zh:271711d6caa7dd5a4e9b79fe8c679fab61a840bcf80040a0f5ebb425d1b27d97", + "zh:5adcf35f30baaea13f80c2a2c774deb9369892719493049687e23476c9dff40f", + "zh:5bcfd19df16e73d7f0ad75bd09e2b3b86cf6700d09822d585d68304b71de1d97", + "zh:604edecf263e38674decb35bb4e0e048fdc951f26fa103c33065ff9728f0313b", + "zh:782acbfb4fa4807e273e588fe45b4aaea9dd0fd1136f76ec3200f6f4db3af8d6", + "zh:84411a596d528fe67294e5c1cfd0c2036b08802497bcc4215ce518924f3c9a4a", + "zh:85e79eecf3f5348975cffec3016b0eba3baf605646102d4348796ccd2df2e5f6", + "zh:95669535ca17aeefef307ebfd59ce6930953173baae5637e8cbbf0297ec7ad58", + "zh:d04d9b177747bfd66b4a45b5d911a2a7822aa8451f5e35621971fb7a4206b530", + "zh:e6d9c924475283e90833450a14a732f4deb6d9bb131db8f86ab856e894270836", + "zh:ebcab0c8a1334c86ed7cfa53f571a17ad6d27e9901f27a8854ea622a74b54bb6", + "zh:ef9c757bb2c83d2103811a3d86b6ec5be06b0ffc337b84db1582d023bce7cdcd", + ] +} + provider "registry.opentofu.org/hetznercloud/hcloud" { version = "1.66.0" - constraints = "~> 1.45" + constraints = "~> 1.54" hashes = [ "h1:+R3t/5wqCb2ics5bNZ4JgtfaAa08Z6uRzXzNtwovVEY=", "h1:9c1ECk3N9xhdQKz4NVCfYEFxebJVVeQZS3E6wrR/Hx8=", From c12a507fc1ccb8456037f6ef5fc7101827fc7b29 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Mon, 13 Jul 2026 21:23:31 +0100 Subject: [PATCH 15/24] feat: Swapped tofu init scripting for ansible scripting --- .gitignore | 3 + ansible/README.md | 77 ++++++++++ ansible/ansible.cfg | 8 ++ ansible/group_vars/all.yml | 14 ++ ansible/group_vars/role_dev.yml | 7 + ansible/group_vars/role_prod.yml | 2 + ansible/group_vars/role_vpn.yml | 2 + ansible/inventory/hcloud.yml | 12 ++ ansible/playbooks/bootstrap.yml | 9 ++ ansible/playbooks/deploy.yml | 6 + ansible/playbooks/site.yml | 4 + ansible/playbooks/spindown.yml | 27 ++++ ansible/playbooks/spinup.yml | 15 ++ ansible/requirements.yml | 4 + ansible/roles/bootstrap/handlers/main.yml | 5 + ansible/roles/bootstrap/tasks/main.yml | 123 ++++++++++++++++ ansible/roles/deploy/tasks/main.yml | 55 ++++++++ ansible/roles/deploy/templates/env.j2 | 1 + .../templates/runners-docker-compose.yml.j2 | 24 ++-- infra/main.tf | 16 --- infra/modules/dev/main.tf | 103 +------------- infra/modules/dev/variables.tf | 20 --- infra/modules/prod/main.tf | 88 +----------- infra/modules/prod/variables.tf | 15 -- infra/modules/vpn/main.tf | 64 +-------- infra/modules/vpn/variables.tf | 15 -- infra/scripts/bootstrap.sh.tftpl | 49 ------- infra/terraform.tfvars.example | 11 +- infra/variables.tf | 17 --- infra/versions.tf | 8 -- readme.md | 131 ++++++++++-------- services/dev/Runners/docker-compose.yml | 33 ----- services/dev/spinup.sh | 2 +- services/prod/spinup.sh | 2 +- 34 files changed, 471 insertions(+), 501 deletions(-) create mode 100644 ansible/README.md create mode 100644 ansible/ansible.cfg create mode 100644 ansible/group_vars/all.yml create mode 100644 ansible/group_vars/role_dev.yml create mode 100644 ansible/group_vars/role_prod.yml create mode 100644 ansible/group_vars/role_vpn.yml create mode 100644 ansible/inventory/hcloud.yml create mode 100644 ansible/playbooks/bootstrap.yml create mode 100644 ansible/playbooks/deploy.yml create mode 100644 ansible/playbooks/site.yml create mode 100644 ansible/playbooks/spindown.yml create mode 100644 ansible/playbooks/spinup.yml create mode 100644 ansible/requirements.yml create mode 100644 ansible/roles/bootstrap/handlers/main.yml create mode 100644 ansible/roles/bootstrap/tasks/main.yml create mode 100644 ansible/roles/deploy/tasks/main.yml create mode 100644 ansible/roles/deploy/templates/env.j2 rename infra/modules/dev/templates/runners-docker-compose.yml.tftpl => ansible/roles/deploy/templates/runners-docker-compose.yml.j2 (54%) delete mode 100644 infra/scripts/bootstrap.sh.tftpl delete mode 100644 services/dev/Runners/docker-compose.yml diff --git a/.gitignore b/.gitignore index 1aad0ee..6936989 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ # services/docker-compose.yml carve-out above. **/.env +### Ansible ### +*.retry + ### Terraform ### # Local .terraform directories **/.terraform/* diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 0000000..88f840b --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,77 @@ +# Ansible + +Bootstraps, syncs, and starts/stops the Docker Compose stacks in `../services/` +on the three servers OpenTofu provisions (see `../infra/`). OpenTofu still owns +the actual cloud resources (servers, network, firewalls, volumes, DNS); +everything downstream of "the server exists" - installing Docker, creating the +`deploy` user, hardening SSH, copying `services//`, rendering `.env` / +the Gitea runner compose file, and running `spinup.sh`/`spindown.sh` - lives +here instead. + +## Contents + +- `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API + directly (not Terraform state) and groups servers by their `role` label + (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules//main.tf`). +- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, `HCLOUD_TOKEN` + lookup), `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` (per-role vars, + e.g. `dev_runner_count`). +- `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening, + unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`. +- `roles/deploy/` - copies `services//` to the server, looks up the + host's data volume directly from the Hetzner API and renders `.env` + (`DATA_DIR=...`) and, for `dev`, `Runners/docker-compose.yml`. +- `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`, + and `site.yml` (all three in order). + +## Prerequisites + +```sh +cd ansible +ansible-galaxy collection install -r requirements.yml +export HCLOUD_TOKEN=your-hetzner-api-token # never commit this +``` + +`ansible_ssh_private_key_file` (see `group_vars/all.yml`) defaults to +`~/.ssh/id_ed25519` - override with `ANSIBLE_SSH_PRIVATE_KEY_FILE` if you use a +different key. It must match one of the SSH keys OpenTofu installed on the +servers (`var.ssh_key_names`). + +## Usage + +Same chicken-and-egg as OpenTofu's own first-apply ordering (see +`../readme.md`): `dev`/`prod`'s firewalls only accept SSH from `vpn`'s own +public IP, so you must bring `vpn` up and connect to it before `dev`/`prod` +are reachable at all. + +```sh +# 1. vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly +ansible-playbook playbooks/bootstrap.yml -l role_vpn +ansible-playbook playbooks/deploy.yml -l role_vpn +ansible-playbook playbooks/spinup.yml -l role_vpn + +# 2. connect to the VPN you just started with an OpenVPN client - now dev/prod +# are reachable, since your egress IP matches their firewall rule +ansible-playbook playbooks/bootstrap.yml -l role_dev,role_prod +ansible-playbook playbooks/deploy.yml -l role_dev,role_prod +ansible-playbook playbooks/spinup.yml -l role_dev,role_prod + +# ...or once already bootstrapped and connected, do everything in one go: +ansible-playbook playbooks/site.yml -l role_dev,role_prod +``` + +Re-run `deploy.yml` + `spinup.yml` any time a compose file changes or +`dev_runner_count` in `group_vars/role_dev.yml` is bumped - `spinup.sh` only +starts/updates what's changed, same as before. + +`spindown.yml` runs `spindown.sh`, which does a full `docker system/volume +prune -a` on the host - it prompts for a `yes` confirmation before running, +since the dynamic inventory makes it trivial to target multiple hosts at once +(`-l role_dev,role_prod`). + +Inspect what the dynamic inventory currently sees: + +```sh +ansible-inventory --graph +ansible-inventory --list +``` diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000..c22a078 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,8 @@ +[defaults] +inventory = inventory/hcloud.yml +host_key_checking = False +retry_files_enabled = False +interpreter_python = auto_silent + +[inventory] +enable_plugins = hetzner.hcloud.hcloud, auto diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml new file mode 100644 index 0000000..5a6c35a --- /dev/null +++ b/ansible/group_vars/all.yml @@ -0,0 +1,14 @@ +--- +# Keep in sync with infra/variables.tf's var.deploy_user (default "deploy") - +# the two aren't wired together automatically, since Ansible no longer reads +# any Terraform state or output. +deploy_user: deploy +ansible_user: "{{ deploy_user }}" +ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}" + +# Used by the deploy role to look up each host's data volume directly from the +# Hetzner API (hcloud_volume_info) rather than from Terraform state/outputs. +hcloud_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}" + +# Repo root's services/ directory, relative to wherever a playbook lives under ansible/playbooks/. +services_root: "{{ playbook_dir }}/../../services" diff --git a/ansible/group_vars/role_dev.yml b/ansible/group_vars/role_dev.yml new file mode 100644 index 0000000..8e7a4ae --- /dev/null +++ b/ansible/group_vars/role_dev.yml @@ -0,0 +1,7 @@ +--- +service_group: dev + +# Number of Gitea Actions runner containers rendered into +# services/dev/Runners/docker-compose.yml by the deploy role - see +# roles/deploy/templates/runners-docker-compose.yml.j2. +dev_runner_count: 3 diff --git a/ansible/group_vars/role_prod.yml b/ansible/group_vars/role_prod.yml new file mode 100644 index 0000000..bf832e4 --- /dev/null +++ b/ansible/group_vars/role_prod.yml @@ -0,0 +1,2 @@ +--- +service_group: prod diff --git a/ansible/group_vars/role_vpn.yml b/ansible/group_vars/role_vpn.yml new file mode 100644 index 0000000..252118a --- /dev/null +++ b/ansible/group_vars/role_vpn.yml @@ -0,0 +1,2 @@ +--- +service_group: vpn diff --git a/ansible/inventory/hcloud.yml b/ansible/inventory/hcloud.yml new file mode 100644 index 0000000..17c6db5 --- /dev/null +++ b/ansible/inventory/hcloud.yml @@ -0,0 +1,12 @@ +--- +# Dynamic inventory: queries the Hetzner Cloud API directly (HCLOUD_TOKEN), +# grouping hosts by the `role` label set on each hcloud_server resource in +# infra/modules//main.tf (role = dev/prod/vpn). Nothing here reads +# Terraform state - this always reflects whatever actually exists in Hetzner. +plugin: hetzner.hcloud.hcloud +token: "{{ lookup('env', 'HCLOUD_TOKEN') }}" +connect_with: public_ipv4 +keyed_groups: + - key: labels.role + prefix: role + separator: "_" diff --git a/ansible/playbooks/bootstrap.yml b/ansible/playbooks/bootstrap.yml new file mode 100644 index 0000000..e015879 --- /dev/null +++ b/ansible/playbooks/bootstrap.yml @@ -0,0 +1,9 @@ +--- +- name: Bootstrap newly created servers (idempotent, safe to re-run) + hosts: all + gather_facts: true + become: false + vars: + ansible_user: root + roles: + - bootstrap diff --git a/ansible/playbooks/deploy.yml b/ansible/playbooks/deploy.yml new file mode 100644 index 0000000..996c578 --- /dev/null +++ b/ansible/playbooks/deploy.yml @@ -0,0 +1,6 @@ +--- +- name: Render templates and sync service definitions to each host + hosts: all + gather_facts: false + roles: + - deploy diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml new file mode 100644 index 0000000..286bee4 --- /dev/null +++ b/ansible/playbooks/site.yml @@ -0,0 +1,4 @@ +--- +- ansible.builtin.import_playbook: bootstrap.yml +- ansible.builtin.import_playbook: deploy.yml +- ansible.builtin.import_playbook: spinup.yml diff --git a/ansible/playbooks/spindown.yml b/ansible/playbooks/spindown.yml new file mode 100644 index 0000000..c09b3d7 --- /dev/null +++ b/ansible/playbooks/spindown.yml @@ -0,0 +1,27 @@ +--- +- name: Stop every compose stack on each host and prune Docker resources + hosts: all + gather_facts: false + vars_prompt: + - name: confirm + prompt: >- + This runs spindown.sh (docker compose down, then a full + docker image/volume prune -a) on the targeted host(s). + Type 'yes' to continue + private: false + tasks: + - name: Abort unless confirmed + ansible.builtin.fail: + msg: Aborted. + when: confirm != 'yes' + + - name: Run spindown.sh + ansible.builtin.command: ./spindown.sh + args: + chdir: "/home/{{ deploy_user }}/services/{{ service_group }}" + register: _spindown + changed_when: true + + - name: Show spindown output + ansible.builtin.debug: + var: _spindown.stdout_lines diff --git a/ansible/playbooks/spinup.yml b/ansible/playbooks/spinup.yml new file mode 100644 index 0000000..bff53f9 --- /dev/null +++ b/ansible/playbooks/spinup.yml @@ -0,0 +1,15 @@ +--- +- name: Start every compose stack on each host, in the order spinup.sh defines + hosts: all + gather_facts: false + tasks: + - name: Run spinup.sh + ansible.builtin.command: ./spinup.sh + args: + chdir: "/home/{{ deploy_user }}/services/{{ service_group }}" + register: _spinup + changed_when: true + + - name: Show spinup output + ansible.builtin.debug: + var: _spinup.stdout_lines diff --git a/ansible/requirements.yml b/ansible/requirements.yml new file mode 100644 index 0000000..c36d0d9 --- /dev/null +++ b/ansible/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - name: hetzner.hcloud + version: ">=4.0.0" diff --git a/ansible/roles/bootstrap/handlers/main.yml b/ansible/roles/bootstrap/handlers/main.yml new file mode 100644 index 0000000..9bb29ca --- /dev/null +++ b/ansible/roles/bootstrap/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Reload sshd + ansible.builtin.systemd: + name: ssh + state: reloaded diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml new file mode 100644 index 0000000..c476d32 --- /dev/null +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -0,0 +1,123 @@ +--- +# Post-install bootstrap - replaces infra/scripts/bootstrap.sh.tftpl. Runs as +# root (see ansible_user override in playbooks/bootstrap.yml) since the deploy +# user doesn't exist yet on a freshly created server. Idempotent: safe to +# re-run against an already-bootstrapped host. + +- name: Update apt cache and upgrade packages + ansible.builtin.apt: + update_cache: true + upgrade: dist + cache_valid_time: 3600 + +- name: Install Docker prerequisites + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + state: present + +- name: Create apt keyrings directory + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: "0755" + +- name: Download Docker's GPG key + ansible.builtin.get_url: + url: https://download.docker.com/linux/ubuntu/gpg + dest: /etc/apt/keyrings/docker.asc + mode: "0644" + force: false + +- name: Determine dpkg architecture + ansible.builtin.command: dpkg --print-architecture + register: _dpkg_arch + changed_when: false + +- name: Add Docker apt repository + ansible.builtin.apt_repository: + repo: >- + deb [arch={{ _dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] + https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable + filename: docker + state: present + +- name: Install Docker Engine and Compose plugin + ansible.builtin.apt: + update_cache: true + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + +- name: Create non-root sudo user, given the same key root was provisioned with + ansible.builtin.user: + name: "{{ deploy_user }}" + shell: /bin/bash + groups: sudo,docker + append: true + create_home: true + +- name: Ensure deploy user's .ssh directory exists + ansible.builtin.file: + path: "/home/{{ deploy_user }}/.ssh" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0700" + +- name: Give deploy user the same authorized_keys as root + ansible.builtin.copy: + src: /root/.ssh/authorized_keys + dest: "/home/{{ deploy_user }}/.ssh/authorized_keys" + remote_src: true + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: "0600" + +- name: Grant deploy user passwordless sudo + ansible.builtin.copy: + content: "{{ deploy_user }} ALL=(ALL) NOPASSWD:ALL\n" + dest: "/etc/sudoers.d/90-{{ deploy_user }}" + validate: "visudo -cf %s" + mode: "0440" + +# Root keeps key-only login as a fallback rather than being fully locked out, +# in case deploy user setup above ever fails silently on a future image. +- name: Restrict root login to key-only + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + regexp: "^#?PermitRootLogin" + line: "PermitRootLogin prohibit-password" + notify: Reload sshd + +- name: Disable SSH password authentication + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + regexp: "^#?PasswordAuthentication" + line: "PasswordAuthentication no" + notify: Reload sshd + +- name: Install unattended-upgrades + ansible.builtin.apt: + name: unattended-upgrades + state: present + +- name: Enable automatic security upgrades + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/20auto-upgrades + content: | + APT::Periodic::Update-Package-Lists "1"; + APT::Periodic::Unattended-Upgrade "1"; + mode: "0644" + +- name: Enable and start unattended-upgrades + ansible.builtin.systemd: + name: unattended-upgrades + enabled: true + state: started diff --git a/ansible/roles/deploy/tasks/main.yml b/ansible/roles/deploy/tasks/main.yml new file mode 100644 index 0000000..a24c27d --- /dev/null +++ b/ansible/roles/deploy/tasks/main.yml @@ -0,0 +1,55 @@ +--- +# Copies services// to the server and renders the files that +# used to be generated by OpenTofu (services//.env and, for dev, +# Runners/docker-compose.yml) - replaces the null_resource.deploy_services + +# local_file provisioners previously in infra/modules//main.tf. + +- name: Ensure services directory exists on the host + ansible.builtin.file: + path: "/home/{{ deploy_user }}/services/{{ service_group }}" + state: directory + mode: "0755" + +- name: Copy static service files (compose files, scripts, configs) + ansible.builtin.copy: + src: "{{ services_root }}/{{ service_group }}/" + dest: "/home/{{ deploy_user }}/services/{{ service_group }}/" + mode: preserve + +- name: Look up this host's data volume + when: service_group in ['dev', 'prod'] + delegate_to: localhost + become: false + hetzner.hcloud.hcloud_volume_info: + api_token: "{{ hcloud_token }}" + name: "{{ service_group }}-storage" + register: _volume_info + +# Hetzner's automount tooling always mounts an auto-mounted volume at +# /mnt/HC_Volume_ - deterministic once the volume exists. +- name: Compute DATA_DIR from the volume's id + when: service_group in ['dev', 'prod'] + ansible.builtin.set_fact: + data_dir: "/mnt/HC_Volume_{{ _volume_info.hcloud_volume_info[0].id }}" + +- name: Render DATA_DIR into .env (auto-loaded by docker compose) + when: service_group in ['dev', 'prod'] + ansible.builtin.template: + src: env.j2 + dest: "/home/{{ deploy_user }}/services/{{ service_group }}/.env" + mode: "0644" + +- name: Render Gitea Actions runner compose file + when: service_group == 'dev' + ansible.builtin.template: + src: runners-docker-compose.yml.j2 + dest: "/home/{{ deploy_user }}/services/dev/Runners/docker-compose.yml" + mode: "0644" + +- name: Make spinup/spindown scripts executable + ansible.builtin.file: + path: "/home/{{ deploy_user }}/services/{{ service_group }}/{{ item }}" + mode: "0755" + loop: + - spinup.sh + - spindown.sh diff --git a/ansible/roles/deploy/templates/env.j2 b/ansible/roles/deploy/templates/env.j2 new file mode 100644 index 0000000..a958fd1 --- /dev/null +++ b/ansible/roles/deploy/templates/env.j2 @@ -0,0 +1 @@ +DATA_DIR={{ data_dir }} diff --git a/infra/modules/dev/templates/runners-docker-compose.yml.tftpl b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 similarity index 54% rename from infra/modules/dev/templates/runners-docker-compose.yml.tftpl rename to ansible/roles/deploy/templates/runners-docker-compose.yml.j2 index bb42e52..f72c94f 100644 --- a/infra/modules/dev/templates/runners-docker-compose.yml.tftpl +++ b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 @@ -1,34 +1,34 @@ -# Generated by OpenTofu from var.dev_runner_count - do not hand-edit. -# To change the number of runners, edit dev_runner_count in terraform.tfvars and -# run tofu apply. To change their shape, edit -# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead. +# Generated by Ansible from dev_runner_count - do not hand-edit. +# To change the number of runners, edit dev_runner_count in +# ansible/group_vars/role_dev.yml and re-run playbooks/deploy.yml. +# To change their shape, edit this template instead. # # GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable # (not baked in here) - services/dev/spinup.sh generates a fresh token and # writes it to Runners/.env immediately before starting these containers. # -# /data lives on dev's volume (${data_dir}) so runner identity survives a +# /data lives on dev's volume ({{ data_dir }}) so runner identity survives a # server rebuild - baked in directly rather than via .env, since Runners/.env # is already reserved for the registration token above and gets overwritten # on every deploy. services: -%{ for i in range(runner_count) ~} - runner-${i + 1}: +{% for i in range(1, dev_runner_count + 1) %} + runner-{{ i }}: image: gitea/act_runner:latest - container_name: gitea_runner_${i + 1} + container_name: gitea_runner_{{ i }} volumes: - ./config.yaml:/config.yaml - - ${data_dir}/gitea_runner_${i + 1}:/data + - {{ data_dir }}/gitea_runner_{{ i }}:/data - /var/run/docker.sock:/var/run/docker.sock networks: - proxy environment: CONFIG_FILE: /config.yaml GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" - GITEA_RUNNER_REGISTRATION_TOKEN: "$${GITEA_RUNNER_REGISTRATION_TOKEN}" - GITEA_RUNNER_NAME: "CICD#${i + 1}" + GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}" + GITEA_RUNNER_NAME: "CICD#{{ i }}" restart: unless-stopped -%{ endfor ~} +{% endfor %} networks: proxy: diff --git a/infra/main.tf b/infra/main.tf index 328bd6b..9df4e16 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -9,12 +9,6 @@ locals { # var.allowed_ssh_source_ips for direct SSH access. vpn_ssh_source_ips = ["${module.vpn.ipv4}/32"] - # Rendered once and uploaded+run by every host module immediately after its - # server is created - see scripts/bootstrap.sh.tftpl. - bootstrap_script = templatefile("${path.module}/scripts/bootstrap.sh.tftpl", { - deploy_user = var.deploy_user - }) - # IDs for every named key in var.ssh_key_names - installed on every server. ssh_key_ids = [for k in data.hcloud_ssh_key.main : k.id] @@ -59,10 +53,6 @@ module "dev" { volume_size = var.dev_volume_size allowed_ssh_source_ips = local.vpn_ssh_source_ips network_ip_range = var.network_ip_range - bootstrap_script = local.bootstrap_script - ssh_private_key_path = var.ssh_private_key_path - deploy_user = var.deploy_user - runner_count = var.dev_runner_count # module.network.id alone doesn't guarantee the subnet exists yet, and a server # can't join a network before it has a subnet. @@ -81,9 +71,6 @@ module "prod" { volume_size = var.prod_volume_size allowed_ssh_source_ips = local.vpn_ssh_source_ips network_ip_range = var.network_ip_range - bootstrap_script = local.bootstrap_script - ssh_private_key_path = var.ssh_private_key_path - deploy_user = var.deploy_user depends_on = [module.network] } @@ -96,9 +83,6 @@ module "vpn" { location = var.location ssh_key_ids = local.ssh_key_ids allowed_ssh_source_ips = var.allowed_ssh_source_ips - bootstrap_script = local.bootstrap_script - ssh_private_key_path = var.ssh_private_key_path - deploy_user = var.deploy_user } module "dns" { diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf index aed7406..6bea690 100644 --- a/infra/modules/dev/main.tf +++ b/infra/modules/dev/main.tf @@ -56,30 +56,12 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] + labels = { role = "dev" } # picked up by ansible/inventory/hcloud.yml network { network_id = var.network_id ip = var.private_ip } - - connection { - type = "ssh" - host = self.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "file" { - content = var.bootstrap_script - destination = "/root/bootstrap.sh" - } - - provisioner "remote-exec" { - inline = [ - "chmod +x /root/bootstrap.sh", - "/root/bootstrap.sh", - ] - } } resource "hcloud_volume" "storage" { @@ -98,85 +80,8 @@ locals { # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is # always /mnt/HC_Volume_ - deterministic once the volume exists, since the # id is stable for the volume's lifetime regardless of which server it's - # attached to. + # attached to. Kept here only as a human-readable output (see outputs.tf) - + # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't + # read this value. data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}" } - -# Renders services/dev/Runners/docker-compose.yml directly into the repo, with -# one runner service per var.runner_count. This only touches the local working -# tree - deploying the change still goes through the normal scp + spinup.sh flow. -resource "local_file" "runners_compose" { - filename = "${path.root}/../services/dev/Runners/docker-compose.yml" - content = templatefile("${path.module}/templates/runners-docker-compose.yml.tftpl", { - runner_count = var.runner_count - data_dir = local.data_dir - }) - file_permission = "0644" -} - -# DATA_DIR is picked up automatically by `docker compose` (which auto-loads -# .env from its working directory) for every dev/*-docker-compose.yml bind -# mount - see services/dev/*.yml. Not committed (see .gitignore): it's tied to -# the live volume's ID, so it's regenerated by tofu apply, not handed off via git. -resource "local_file" "env" { - filename = "${path.root}/../services/dev/.env" - content = "DATA_DIR=${local.data_dir}\n" - file_permission = "0644" -} - -locals { - # Everything under services/dev except the two files above: those are - # tracked via their own resource content (reading them back with fileset()/ - # filesha1() before they exist would fail during `tofu plan`). - dev_static_files = sort([ - for f in fileset("${path.root}/../services/dev", "**") : - f if f != ".env" && f != "Runners/docker-compose.yml" - ]) - - dev_static_files_hash = sha1(join("", [ - for f in local.dev_static_files : filesha1("${path.root}/../services/dev/${f}") - ])) -} - -# Copies services/dev to the server on every apply that changes it (a hand-edited -# compose file, a new runner count, ...) or recreates the server - not just once -# at first creation. Split from hcloud_server.this because it must run after the -# generated files above, which in turn depend on the volume, which depends on -# the server - so it can't be a provisioner on the server resource itself. -resource "null_resource" "deploy_services" { - triggers = { - server_id = hcloud_server.this.id - static_files = local.dev_static_files_hash - env = local_file.env.content - runners = local_file.runners_compose.content - } - - connection { - type = "ssh" - host = hcloud_server.this.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "remote-exec" { - inline = ["mkdir -p /home/${var.deploy_user}/services"] - } - - provisioner "file" { - source = "${path.root}/../services/dev" - destination = "/home/${var.deploy_user}/services" - } - - provisioner "remote-exec" { - inline = [ - "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services", - "chmod +x /home/${var.deploy_user}/services/dev/*.sh", - ] - } - - depends_on = [ - hcloud_server.this, - local_file.env, - local_file.runners_compose, - ] -} diff --git a/infra/modules/dev/variables.tf b/infra/modules/dev/variables.tf index 49d475e..89dbb4e 100644 --- a/infra/modules/dev/variables.tf +++ b/infra/modules/dev/variables.tf @@ -42,23 +42,3 @@ variable "network_ip_range" { description = "CIDR of the private network, allowed through the firewall for traffic from prod." type = string } - -variable "bootstrap_script" { - description = "Rendered post-install script, uploaded and executed on the server immediately after creation." - type = string -} - -variable "ssh_private_key_path" { - description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." - type = string -} - -variable "runner_count" { - description = "Number of Gitea Actions runner containers to render into services/dev/Runners/docker-compose.yml." - type = number -} - -variable "deploy_user" { - description = "Non-root sudo user created by the bootstrap script - services/dev is copied into this user's home directory." - type = string -} diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index f3dc6f1..8135cae 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -83,30 +83,12 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] + labels = { role = "prod" } # picked up by ansible/inventory/hcloud.yml network { network_id = var.network_id ip = var.private_ip } - - connection { - type = "ssh" - host = self.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "file" { - content = var.bootstrap_script - destination = "/root/bootstrap.sh" - } - - provisioner "remote-exec" { - inline = [ - "chmod +x /root/bootstrap.sh", - "/root/bootstrap.sh", - ] - } } resource "hcloud_volume" "storage" { @@ -125,70 +107,8 @@ locals { # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is # always /mnt/HC_Volume_ - deterministic once the volume exists, since the # id is stable for the volume's lifetime regardless of which server it's - # attached to. + # attached to. Kept here only as a human-readable output (see outputs.tf) - + # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't + # read this value. data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}" } - -# DATA_DIR is picked up automatically by `docker compose` (which auto-loads -# .env from its working directory) for every prod/*-docker-compose.yml bind -# mount - see services/prod/*.yml. Not committed (see .gitignore): it's tied to -# the live volume's ID, so it's regenerated by tofu apply, not handed off via git. -resource "local_file" "env" { - filename = "${path.root}/../services/prod/.env" - content = "DATA_DIR=${local.data_dir}\n" - file_permission = "0644" -} - -locals { - # Everything under services/prod except .env: that's tracked via its own - # resource content (reading it back with fileset()/filesha1() before it - # exists would fail during `tofu plan`). - prod_static_files = sort([ - for f in fileset("${path.root}/../services/prod", "**") : f if f != ".env" - ]) - - prod_static_files_hash = sha1(join("", [ - for f in local.prod_static_files : filesha1("${path.root}/../services/prod/${f}") - ])) -} - -# Copies services/prod to the server on every apply that changes it (a -# hand-edited compose file, ...) or recreates the server - not just once at -# first creation. Split from hcloud_server.this because it must run after -# local_file.env, which in turn depends on the volume, which depends on the -# server - so it can't be a provisioner on the server resource itself. -resource "null_resource" "deploy_services" { - triggers = { - server_id = hcloud_server.this.id - static_files = local.prod_static_files_hash - env = local_file.env.content - } - - connection { - type = "ssh" - host = hcloud_server.this.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "remote-exec" { - inline = ["mkdir -p /home/${var.deploy_user}/services"] - } - - provisioner "file" { - source = "${path.root}/../services/prod" - destination = "/home/${var.deploy_user}/services" - } - - provisioner "remote-exec" { - inline = [ - "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services", - "chmod +x /home/${var.deploy_user}/services/prod/*.sh", - ] - } - - depends_on = [ - hcloud_server.this, - local_file.env, - ] -} diff --git a/infra/modules/prod/variables.tf b/infra/modules/prod/variables.tf index bab01c1..e105b70 100644 --- a/infra/modules/prod/variables.tf +++ b/infra/modules/prod/variables.tf @@ -42,18 +42,3 @@ variable "network_ip_range" { description = "CIDR of the private network, allowed through the firewall for traffic from dev." type = string } - -variable "bootstrap_script" { - description = "Rendered post-install script, uploaded and executed on the server immediately after creation." - type = string -} - -variable "ssh_private_key_path" { - description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." - type = string -} - -variable "deploy_user" { - description = "Non-root sudo user created by the bootstrap script - services/prod is copied into this user's home directory." - type = string -} diff --git a/infra/modules/vpn/main.tf b/infra/modules/vpn/main.tf index b62fb33..c73396e 100644 --- a/infra/modules/vpn/main.tf +++ b/infra/modules/vpn/main.tf @@ -38,67 +38,5 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] - - connection { - type = "ssh" - host = self.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "file" { - content = var.bootstrap_script - destination = "/root/bootstrap.sh" - } - - provisioner "remote-exec" { - inline = [ - "chmod +x /root/bootstrap.sh", - "/root/bootstrap.sh", - ] - } -} - -locals { - vpn_static_files = sort(fileset("${path.root}/../services/vpn", "**")) - - vpn_static_files_hash = sha1(join("", [ - for f in local.vpn_static_files : filesha1("${path.root}/../services/vpn/${f}") - ])) -} - -# Copies services/vpn to the server on every apply that changes it (a -# hand-edited compose file, ...) or recreates the server - not just once at -# first creation. Split from hcloud_server.this so it's triggered by content -# changes independently of the bootstrap provisioners above. -resource "null_resource" "deploy_services" { - triggers = { - server_id = hcloud_server.this.id - static_files = local.vpn_static_files_hash - } - - connection { - type = "ssh" - host = hcloud_server.this.ipv4_address - user = "root" - private_key = file(var.ssh_private_key_path) - } - - provisioner "remote-exec" { - inline = ["mkdir -p /home/${var.deploy_user}/services"] - } - - provisioner "file" { - source = "${path.root}/../services/vpn" - destination = "/home/${var.deploy_user}/services" - } - - provisioner "remote-exec" { - inline = [ - "chown -R ${var.deploy_user}:${var.deploy_user} /home/${var.deploy_user}/services", - "chmod +x /home/${var.deploy_user}/services/vpn/*.sh", - ] - } - - depends_on = [hcloud_server.this] + labels = { role = "vpn" } # picked up by ansible/inventory/hcloud.yml } diff --git a/infra/modules/vpn/variables.tf b/infra/modules/vpn/variables.tf index 384a4ad..022e909 100644 --- a/infra/modules/vpn/variables.tf +++ b/infra/modules/vpn/variables.tf @@ -22,18 +22,3 @@ variable "allowed_ssh_source_ips" { description = "CIDRs allowed to reach port 22 on vpn." type = list(string) } - -variable "bootstrap_script" { - description = "Rendered post-install script, uploaded and executed on the server immediately after creation." - type = string -} - -variable "ssh_private_key_path" { - description = "Local path to the private key matching one of var.ssh_key_ids, used to run the bootstrap script over SSH." - type = string -} - -variable "deploy_user" { - description = "Non-root sudo user created by the bootstrap script - services/vpn is copied into this user's home directory." - type = string -} diff --git a/infra/scripts/bootstrap.sh.tftpl b/infra/scripts/bootstrap.sh.tftpl deleted file mode 100644 index 3227620..0000000 --- a/infra/scripts/bootstrap.sh.tftpl +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# Post-install bootstrap - uploaded and executed once by OpenTofu immediately -# after server creation (see modules//main.tf). Idempotent-ish: safe to -# re-run by hand, but only ever runs automatically on first create. -set -euo pipefail - -export DEBIAN_FRONTEND=noninteractive -DEPLOY_USER="${deploy_user}" - -apt-get update -y -apt-get upgrade -y - -# --- Docker Engine + Compose plugin --- -if ! command -v docker >/dev/null 2>&1; then - apt-get install -y ca-certificates curl gnupg - install -m 0755 -d /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - chmod a+r /etc/apt/keyrings/docker.asc - . /etc/os-release - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $VERSION_CODENAME stable" \ - > /etc/apt/sources.list.d/docker.list - apt-get update -y - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -fi - -# --- Non-root sudo user, given the same key root was provisioned with --- -if ! id -u "$DEPLOY_USER" >/dev/null 2>&1; then - useradd --create-home --shell /bin/bash "$DEPLOY_USER" - usermod -aG sudo,docker "$DEPLOY_USER" - mkdir -p "/home/$DEPLOY_USER/.ssh" - cp /root/.ssh/authorized_keys "/home/$DEPLOY_USER/.ssh/authorized_keys" - chown -R "$DEPLOY_USER:$DEPLOY_USER" "/home/$DEPLOY_USER/.ssh" - chmod 700 "/home/$DEPLOY_USER/.ssh" - chmod 600 "/home/$DEPLOY_USER/.ssh/authorized_keys" - echo "$DEPLOY_USER ALL=(ALL) NOPASSWD:ALL" > "/etc/sudoers.d/90-$DEPLOY_USER" - chmod 440 "/etc/sudoers.d/90-$DEPLOY_USER" -fi - -# --- SSH hardening: key-only auth everywhere. Root keeps key-only login as a -# fallback rather than being fully locked out, in case $DEPLOY_USER setup above -# ever fails silently on a future image. --- -sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config -sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config -systemctl reload ssh 2>/dev/null || systemctl reload sshd - -# --- Unattended security upgrades --- -apt-get install -y unattended-upgrades -dpkg-reconfigure -f noninteractive unattended-upgrades -systemctl enable --now unattended-upgrades diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 7ab14ec..3c8c22b 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -2,18 +2,13 @@ # never commit real values there. # # The Hetzner API token is NOT set here: export it as HCLOUD_TOKEN in your shell -# before running tofu plan/apply. +# before running tofu plan/apply. The same token and SSH key are used again by +# Ansible afterwards - see ../ansible/README.md. # Names of SSH keys already uploaded to your Hetzner Cloud project # (Console > Security > SSH Keys). All of them are installed on every server. Required. ssh_key_names = ["your-key-name", "laptop", "phone"] -# Local path to the private key matching ONE of ssh_key_names above (doesn't need -# to be all of them - just one you hold). OpenTofu uses this once per server to -# upload and run the post-install bootstrap script -# (infra/scripts/bootstrap.sh.tftpl) immediately after creation. Required. -ssh_private_key_path = "~/.ssh/id_ed25519" - # Optional overrides - defaults live in variables.tf # location = "nbg1" # dev_server_type = "cx22" @@ -22,6 +17,4 @@ ssh_private_key_path = "~/.ssh/id_ed25519" # dev_volume_size = 20 # prod_volume_size = 20 # allowed_ssh_source_ips = ["203.0.113.4/32"] -# deploy_user = "deploy" -# dev_runner_count = 3 # dns_zones = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"] diff --git a/infra/variables.tf b/infra/variables.tf index 10f6f43..95dc546 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -81,23 +81,6 @@ variable "allowed_ssh_source_ips" { default = ["0.0.0.0/0", "::/0"] } -variable "ssh_private_key_path" { - description = "Local path to the private key matching one of var.ssh_key_names - only needs to be a key you hold yourself, not all of them. Used once per server by OpenTofu to upload and run the post-install bootstrap script over SSH immediately after creation (see infra/scripts/bootstrap.sh.tftpl)." - type = string -} - -variable "deploy_user" { - description = "Non-root sudo user created on every server by the bootstrap script, with the same SSH key as root." - type = string - default = "deploy" -} - -variable "dev_runner_count" { - description = "Number of Gitea Actions runner containers to run on dev. Rendered into services/dev/Runners/docker-compose.yml on every tofu apply - see infra/modules/dev/templates/runners-docker-compose.yml.tftpl." - type = number - default = 1 -} - variable "dns_zones" { description = "Domains to manage as Hetzner DNS zones. Point each domain's registrar NS records at tofu output dns_nameservers for Hetzner to actually become authoritative." type = list(string) diff --git a/infra/versions.tf b/infra/versions.tf index 303b689..f817710 100644 --- a/infra/versions.tf +++ b/infra/versions.tf @@ -6,14 +6,6 @@ terraform { source = "hetznercloud/hcloud" version = "~> 1.54" # hcloud_zone / hcloud_zone_rrset (DNS) require >= 1.54.0 } - local = { - source = "hashicorp/local" - version = "~> 2.5" - } - null = { - source = "hashicorp/null" - version = "~> 3.2" - } } } diff --git a/readme.md b/readme.md index f35a312..2e017e4 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # Server -Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/). +Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each bootstrapped and deployed with [Ansible](https://www.ansible.com/), running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).

@@ -12,7 +12,7 @@ Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted - [Repository layout](#repository-layout) - [Prerequisites](#prerequisites) - [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) -- [Deploying the services](#deploying-the-services-services) +- [Bootstrapping and deploying with Ansible](#bootstrapping-and-deploying-with-ansible-ansible) - [Service inventory](#service-inventory) - [First-time setup](#first-time-setup) - [Development container](#development-container) @@ -66,23 +66,26 @@ architecture-beta ``` . -├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes -│ ├── main.tf # root module: wires network + dev/prod/vpn modules together +├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes, DNS +│ ├── main.tf # root module: wires network + dev/prod/vpn/dns modules together │ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names) │ ├── outputs.tf # pass-through outputs from each host module │ ├── ssh.tf # looks up each SSH key already uploaded to Hetzner Cloud │ ├── versions.tf # provider requirements │ ├── terraform.tfvars.example -│ ├── scripts/ -│ │ └── bootstrap.sh.tftpl # post-install script run on every host right after creation │ └── modules/ │ ├── network/ # shared private network + subnet (used by dev and prod) -│ ├── dev/ # dev server + firewall + volume, renders + deploys services/dev/ -│ │ └── templates/ -│ │ └── runners-docker-compose.yml.tftpl # shape of each generated runner service -│ ├── prod/ # prod server + firewall + volume, deploys services/prod/ -│ ├── vpn/ # vpn server + firewall (no private network, no volume), deploys services/vpn/ +│ ├── dev/ # dev server + firewall + volume, labeled role=dev +│ ├── prod/ # prod server + firewall + volume, labeled role=prod +│ ├── vpn/ # vpn server + firewall (no private network, no volume), labeled role=vpn │ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones +├── ansible/ # Ansible: bootstraps each server and deploys/starts services// onto it +│ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by the role label above +│ ├── group_vars/ # deploy_user, dev_runner_count, etc. +│ ├── roles/ +│ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades +│ │ └── deploy/ # copies services//, renders .env + Runners/docker-compose.yml +│ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml ├── services/ # Docker Compose stacks, grouped by which server they run on │ ├── dev/ # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) │ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik @@ -94,16 +97,19 @@ architecture-beta └── assets/ ``` -Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` mirrors this same dev/prod/vpn split on the provisioning side, so a given host's cloud resources and its compose stacks are easy to find side by side. +Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side. + +OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, volumes, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services//`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown. ## Prerequisites - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token -- One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (OpenTofu uses it once per server to run the post-install bootstrap — see below) +- One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (Ansible uses it to bootstrap and deploy — see below) - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` +- [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`) - Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner (see [Managing DNS](#managing-dns) — the zones and records themselves are created for you) -Docker + the Compose plugin no longer need installing by hand — the bootstrap script below handles that. +Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below). ## Provisioning the infrastructure (`infra/`) @@ -111,7 +117,7 @@ Docker + the Compose plugin no longer need installing by hand — the bootstrap cd infra export HCLOUD_TOKEN=your-hetzner-api-token # never commit this cp terraform.tfvars.example terraform.tfvars -$EDITOR terraform.tfvars # set ssh_key_names and ssh_private_key_path at minimum +$EDITOR terraform.tfvars # set ssh_key_names at minimum tofu init tofu plan @@ -120,52 +126,15 @@ tofu apply This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` / `module.dns` in `infra/main.tf`: - `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`) -- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses (`modules/dev`, `modules/prod`, `modules/vpn`) +- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses, each server labeled `role = "dev"/"prod"/"vpn"` for Ansible's dynamic inventory (`modules/dev`, `modules/prod`, `modules/vpn`) - one `hcloud_volume` each for `dev` and `prod` (`modules/dev`, `modules/prod`; `vpn` has none) - one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns)) -Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`, `tofu output dev_data_dir`, `tofu output prod_data_dir`. +Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`, `tofu output dev_data_dir`, `tofu output prod_data_dir` (the last two are informational only now — Ansible looks the data directory up itself, see below). `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`. -### Post-install bootstrap - -Immediately after each `hcloud_server` is created, OpenTofu uploads and runs [`infra/scripts/bootstrap.sh.tftpl`](infra/scripts/bootstrap.sh.tftpl) over SSH as `root` (`connection` + `file`/`remote-exec` provisioners in each `modules//main.tf`). It: - -- installs Docker Engine + the Compose plugin -- creates a non-root sudo user (`var.deploy_user`, default `deploy`) with the same SSH keys as root, in the `sudo` and `docker` groups -- hardens `sshd`: password authentication off, root login restricted to key-only (`PermitRootLogin prohibit-password`) -- installs and enables `unattended-upgrades` for automatic security patches - -After bootstrap, SSH in as `var.deploy_user` for day-to-day work — root key-based login still works as a fallback. - -### Persistent data lives on the volumes, not the server disk - -`dev` and `prod`'s compose files used to bind-mount container data to paths relative to wherever `services//` happened to be copied on the server (e.g. `./gitea:/data`) - i.e. the server's local disk, which doesn't survive the server being destroyed and recreated. `dev-storage` and `prod-storage` (the `hcloud_volume`s) do survive that, so all stateful bind mounts now point there instead: `${DATA_DIR}/gitea:/data`, `${DATA_DIR}/bitwarden/:/data/`, and so on across `services/dev/*.yml` and `services/prod/*.yml`. `vpn` has no volume, so it's untouched - see the architecture table above. - -`DATA_DIR` needing to resolve to the right path is what makes this work, and that's answered deterministically: Hetzner's automount tooling always mounts an auto-mounted volume at `/mnt/HC_Volume_`, and since the volume is a Terraform resource, `modules//main.tf` already knows its `id` the moment it's created - `local.data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}"`. Every `tofu apply` writes that value straight into `services/dev/.env` / `services/prod/.env` as `DATA_DIR=...`, which `docker compose` auto-loads for variable substitution in any compose file run from that directory (the same mechanism the Gitea runner token already uses). Query it directly with `tofu output dev_data_dir` / `tofu output prod_data_dir`. - -These `.env` files are deliberately not committed (see `.gitignore`): the path is tied to the *live* volume's ID, so it's regenerated by `tofu apply`, not handed off via git. Run `tofu apply` at least once before the first deploy, and again if a volume is ever destroyed and recreated (new ID); `spinup.sh` on both hosts refuses to start anything if `.env` is missing, rather than silently falling back to a relative path. - -The Gitea Actions runners are the one exception to the `.env` mechanism: their `/data` path is baked directly into the generated `Runners/docker-compose.yml` at render time (via the same `templatefile()` call as `dev_runner_count`), because `Runners/.env` is already reserved for the registration token and gets overwritten on every deploy. - -### Scaling Gitea Actions runners - -The number of Gitea Actions runner containers on `dev` is an OpenTofu input, `var.dev_runner_count` (default `3`). On every `tofu apply`, `modules/dev`'s `local_file.runners_compose` resource renders [`modules/dev/templates/runners-docker-compose.yml.tftpl`](infra/modules/dev/templates/runners-docker-compose.yml.tftpl) straight into [`services/dev/Runners/docker-compose.yml`](services/dev/Runners/docker-compose.yml) in your working tree — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated: change `dev_runner_count` in `terraform.tfvars` and re-run `tofu apply` rather than hand-editing it. - -This only updates your local working tree — `tofu apply` doesn't reach out and start containers on `dev` itself. Redeploy as usual (re-copy `services/dev/` to the server and re-run `spinup.sh`) to apply a count change. - -Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore. - -### First apply: bootstrapping order matters - -`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch `tofu apply`, the `dev`/`prod` bootstrap provisioners can't connect yet. Bring the estate up in this order: - -1. `tofu apply -target=module.vpn` — creates `vpn` only; its firewall still allows SSH from `var.allowed_ssh_source_ips`, so its bootstrap provisioner and `null_resource.deploy_services` (which copies `services/vpn` over) both run fine. -2. SSH in as `var.deploy_user` and run `~/services/vpn/spinup.sh` to start the VPN service, then connect to it with an OpenVPN client. -3. `tofu apply` — creates `network`, `dev`, `prod`. Now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and the `dev`/`prod` bootstrap + deploy provisioners can connect. - -If step 3 is run before you're connected to the VPN, the `dev`/`prod` provisioners will fail and Terraform will mark those servers tainted — re-running `tofu apply` once connected will recreate and re-bootstrap them. +OpenTofu never connects to the servers over SSH — no provisioners, no `bootstrap.sh`, no copying `services/` — that's all Ansible now. See below. ### Managing DNS @@ -177,9 +146,53 @@ Adding a new subdomain: add an entry to `local.dns_records` in `infra/main.tf` ( Only A records for IPv4 are managed here — none of the modules currently track servers' IPv6 addresses, so AAAA records aren't generated even though the firewalls already allow IPv6 traffic. +## Bootstrapping and deploying with Ansible (`ansible/`) + +Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services//` to each server, rendering the files OpenTofu used to generate (`.env`'s `DATA_DIR`, `dev`'s `Runners/docker-compose.yml`), and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version: + +```sh +cd ansible +ansible-galaxy collection install -r requirements.yml +export HCLOUD_TOKEN=your-hetzner-api-token # never commit this + +# vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly +ansible-playbook playbooks/site.yml -l role_vpn + +# SSH to vpn as deploy and connect to the OpenVPN it just started, then: +ansible-playbook playbooks/site.yml -l role_dev,role_prod +``` + +Hosts are discovered dynamically from the Hetzner API (`ansible/inventory/hcloud.yml`), grouped into `role_dev`/`role_prod`/`role_vpn` by the `role` label OpenTofu sets on each server — there's no static inventory file to keep in sync, and nothing here reads Terraform state. + +### Persistent data lives on the volumes, not the server disk + +`dev` and `prod`'s compose files bind-mount container data to paths on the `dev-storage` / `prod-storage` volumes rather than the server's local disk, so it survives the server being destroyed and recreated: `${DATA_DIR}/gitea:/data`, `${DATA_DIR}/bitwarden/:/data/`, and so on across `services/dev/*.yml` and `services/prod/*.yml`. `vpn` has no volume, so it's untouched - see the architecture table above. + +`DATA_DIR` resolves deterministically: Hetzner always automounts a volume at `/mnt/HC_Volume_`. Ansible's `deploy` role looks the volume up by name (`hcloud_volume_info` for `dev-storage`/`prod-storage`) directly against the Hetzner API and renders that path into `services//.env` as `DATA_DIR=...` on the remote host — `docker compose` auto-loads `.env` from its working directory for variable substitution. Nothing is written locally; re-run `ansible-playbook playbooks/deploy.yml` if a volume is ever destroyed and recreated (new id). `spinup.sh` on both hosts refuses to start anything if `.env` is missing, rather than silently falling back to a relative path. + +The Gitea Actions runners are the one exception to the `.env` mechanism: their `/data` path is baked directly into the rendered `Runners/docker-compose.yml`, because `Runners/.env` is already reserved for the registration token and gets overwritten on every deploy. + +### Scaling Gitea Actions runners + +The number of Gitea Actions runner containers on `dev` is set by `dev_runner_count` in [`ansible/group_vars/role_dev.yml`](ansible/group_vars/role_dev.yml) (default `3`). Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host: change `dev_runner_count` and redeploy rather than hand-editing it. + +Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a count change. + +Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore. + +### First deploy: bootstrapping order matters + +`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch estate, Ansible can't reach `dev`/`prod` yet. Bring it up in this order: + +1. `ansible-playbook playbooks/site.yml -l role_vpn` — bootstraps, deploys, and starts `vpn` only; its firewall allows SSH from `var.allowed_ssh_source_ips` directly. +2. SSH in as `deploy` and connect to the OpenVPN service you just started with an OpenVPN client. +3. `ansible-playbook playbooks/site.yml -l role_dev,role_prod` — now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and `dev`/`prod` become reachable. + +If step 3 is run before you're connected to the VPN, Ansible will simply fail to connect — re-run it once connected. + ## Deploying the services (`services/`) -`tofu apply` already puts `services//` on the matching server for you — a `null_resource.deploy_services` in each `modules//main.tf` copies it to `/home//services/` and `chown`s it to `var.deploy_user`, re-running whenever the server is recreated or the directory's contents change (a hand-edited compose file, a new `dev_runner_count`, a refreshed `DATA_DIR`, ...) - not just once at first creation. So after applying, just SSH in as `var.deploy_user` and run the matching script from inside that directory: +After `ansible-playbook playbooks/deploy.yml` has copied `services//` to `/home/deploy/services/` on the matching server, SSH in as `deploy` and run the matching script from inside that directory — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do exactly this remotely (see [`ansible/README.md`](ansible/README.md)): ```sh # on dev @@ -195,7 +208,7 @@ Only A records for IPv4 are managed here — none of the modules currently track ./spindown.sh ``` -If you ever need to force a re-sync without going through Terraform (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@:~/services` still works fine - `tofu apply` will just overwrite it again next time triggers change. +If you ever need to force a re-sync without going through Ansible (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@:~/services` still works fine - `ansible-playbook playbooks/deploy.yml` will just overwrite it again next time it's run. Each stack can also be managed individually with plain Compose, e.g.: @@ -243,7 +256,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here: -- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by the [post-install bootstrap](#post-install-bootstrap); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)). +- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by [Ansible's `bootstrap` role](ansible/roles/bootstrap/tasks/main.yml); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)). ## Development container @@ -261,5 +274,5 @@ Then reopen the repo in VS Code with the Dev Containers extension. - SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). - `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP. - Firewalls are allowlists scoped per host in `infra/modules//main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. -- Every host's [post-install bootstrap](#post-install-bootstrap) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`var.deploy_user`) for day-to-day access. +- Every host's [Ansible bootstrap role](ansible/roles/bootstrap/tasks/main.yml) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`deploy_user`, see `ansible/group_vars/all.yml`) for day-to-day access. - DNS zones (`modules/dns`) carry both Hetzner's own `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, including for `divine-couture.co.uk` and `snexo.co.uk`, not just `luke-else.co.uk`. diff --git a/services/dev/Runners/docker-compose.yml b/services/dev/Runners/docker-compose.yml deleted file mode 100644 index 6d0f9d4..0000000 --- a/services/dev/Runners/docker-compose.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by OpenTofu from var.dev_runner_count - do not hand-edit. -# To change the number of runners, edit dev_runner_count in terraform.tfvars and -# run tofu apply. To change their shape, edit -# infra/modules/dev/templates/runners-docker-compose.yml.tftpl instead. -# -# GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable -# (not baked in here) - services/dev/spinup.sh generates a fresh token and -# writes it to Runners/.env immediately before starting these containers. -# -# /data lives on dev's volume (/mnt/HC_Volume_PENDING_TOFU_APPLY) so runner -# identity survives a server rebuild - baked in directly rather than via .env, -# since Runners/.env is already reserved for the registration token above and -# gets overwritten on every deploy. -services: - runner-1: - image: gitea/act_runner:latest - container_name: gitea_runner_1 - volumes: - - ./config.yaml:/config.yaml - - /mnt/HC_Volume_PENDING_TOFU_APPLY/gitea_runner_1:/data - - /var/run/docker.sock:/var/run/docker.sock - networks: - - proxy - environment: - CONFIG_FILE: /config.yaml - GITEA_INSTANCE_URL: "https://git.luke-else.co.uk" - GITEA_RUNNER_REGISTRATION_TOKEN: "${GITEA_RUNNER_REGISTRATION_TOKEN}" - GITEA_RUNNER_NAME: "CICD#1" - restart: unless-stopped - -networks: - proxy: - external: true diff --git a/services/dev/spinup.sh b/services/dev/spinup.sh index 0b4e466..0692a37 100755 --- a/services/dev/spinup.sh +++ b/services/dev/spinup.sh @@ -4,7 +4,7 @@ set -e cd "$(dirname "$0")" if [ ! -f .env ]; then - echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'tofu apply' in infra/ first, then re-copy this directory." >&2 + echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 exit 1 fi diff --git a/services/prod/spinup.sh b/services/prod/spinup.sh index d1e874e..ef7e9ee 100755 --- a/services/prod/spinup.sh +++ b/services/prod/spinup.sh @@ -4,7 +4,7 @@ set -e cd "$(dirname "$0")" if [ ! -f .env ]; then - echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'tofu apply' in infra/ first, then re-copy this directory." >&2 + echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 exit 1 fi From 1aa340f7ce2f6712c568c2a103b6bace8200c777 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Mon, 20 Jul 2026 22:45:43 +0100 Subject: [PATCH 16/24] feat: backup and restore --- ansible/README.md | 21 +++-- ansible/group_vars/all.yml | 15 +++- ansible/roles/deploy/tasks/main.yml | 28 ++----- ansible/roles/deploy/templates/env.j2 | 6 +- .../templates/runners-docker-compose.yml.j2 | 15 ++-- infra/main.tf | 8 +- infra/modules/dev/main.tf | 21 ----- infra/modules/dev/outputs.tf | 5 -- infra/modules/dev/variables.tf | 5 -- infra/modules/prod/main.tf | 21 ----- infra/modules/prod/outputs.tf | 5 -- infra/modules/prod/variables.tf | 5 -- infra/outputs.tf | 10 --- infra/terraform.tfvars.example | 2 - infra/variables.tf | 16 +--- readme.md | 80 ++++++++++++------- services/dev/backup-docker-compose.yml | 48 +++++++++++ services/dev/gitea-docker-compose.yml | 14 +++- services/dev/restore.sh | 54 +++++++++++++ services/dev/spindown.sh | 4 +- services/dev/spinup.sh | 9 ++- services/dev/traefik-docker-compose.yml | 6 +- services/prod/backup-docker-compose.yml | 59 ++++++++++++++ services/prod/bitwarden-docker-compose.yml | 13 ++- services/prod/rd-docker-compose.yml | 13 ++- services/prod/restore.sh | 54 +++++++++++++ services/prod/spindown.sh | 4 +- services/prod/spinup.sh | 9 ++- services/prod/status-docker-compose.yml | 9 ++- services/prod/traefik-docker-compose.yml | 6 +- services/prod/web-docker-compose.yml | 31 ++----- services/vpn/backup-docker-compose.yml | 48 +++++++++++ services/vpn/restore.sh | 54 +++++++++++++ services/vpn/spindown.sh | 4 +- services/vpn/spinup.sh | 12 +++ services/vpn/traefik-docker-compose.yml | 6 +- services/vpn/vpn-docker-compose.yml | 11 ++- 37 files changed, 524 insertions(+), 207 deletions(-) create mode 100644 services/dev/backup-docker-compose.yml create mode 100755 services/dev/restore.sh create mode 100644 services/prod/backup-docker-compose.yml create mode 100755 services/prod/restore.sh create mode 100644 services/vpn/backup-docker-compose.yml create mode 100755 services/vpn/restore.sh diff --git a/ansible/README.md b/ansible/README.md index 88f840b..46c8890 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -2,25 +2,26 @@ Bootstraps, syncs, and starts/stops the Docker Compose stacks in `../services/` on the three servers OpenTofu provisions (see `../infra/`). OpenTofu still owns -the actual cloud resources (servers, network, firewalls, volumes, DNS); +the actual cloud resources (servers, network, firewalls, DNS); everything downstream of "the server exists" - installing Docker, creating the `deploy` user, hardening SSH, copying `services//`, rendering `.env` / the Gitea runner compose file, and running `spinup.sh`/`spindown.sh` - lives -here instead. +here instead. Persistent service data lives in named Docker volumes backed up +to S3, not on a Hetzner volume - see `../readme.md`. ## Contents - `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API directly (not Terraform state) and groups servers by their `role` label (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules//main.tf`). -- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, `HCLOUD_TOKEN` - lookup), `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` (per-role vars, - e.g. `dev_runner_count`). +- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, + `backup_s3_*` S3 backup credentials), `role_dev.yml` / `role_prod.yml` / + `role_vpn.yml` (per-role vars, e.g. `dev_runner_count`). - `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening, unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`. -- `roles/deploy/` - copies `services//` to the server, looks up the - host's data volume directly from the Hetzner API and renders `.env` - (`DATA_DIR=...`) and, for `dev`, `Runners/docker-compose.yml`. +- `roles/deploy/` - copies `services//` to the server and renders `.env` + (S3 backup credentials, see `../readme.md`) and, for `dev`, + `Runners/docker-compose.yml`. - `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`, and `site.yml` (all three in order). @@ -30,6 +31,10 @@ here instead. cd ansible ansible-galaxy collection install -r requirements.yml export HCLOUD_TOKEN=your-hetzner-api-token # never commit this +export BACKUP_S3_BUCKET=your-bucket-name +export BACKUP_S3_ACCESS_KEY_ID=your-access-key-id +export BACKUP_S3_SECRET_ACCESS_KEY=your-secret-access-key +export BACKUP_S3_ENDPOINT=your-s3-endpoint # omit for real AWS S3 ``` `ansible_ssh_private_key_file` (see `group_vars/all.yml`) defaults to diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 5a6c35a..d3308cf 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -6,9 +6,16 @@ deploy_user: deploy ansible_user: "{{ deploy_user }}" ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}" -# Used by the deploy role to look up each host's data volume directly from the -# Hetzner API (hcloud_volume_info) rather than from Terraform state/outputs. -hcloud_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}" - # Repo root's services/ directory, relative to wherever a playbook lives under ansible/playbooks/. services_root: "{{ playbook_dir }}/../../services" + +# S3-compatible bucket that services//backup-docker-compose.yml backs up +# to and restores from - see readme.md. Not provisioned by OpenTofu (the +# hcloud provider has no Object Storage resource), so these must point at a +# bucket you've created yourself. AWS_ENDPOINT is only needed for a +# non-AWS S3-compatible provider (e.g. Hetzner Object Storage); leave it unset +# to use real AWS S3. +backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}" +backup_s3_access_key_id: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY_ID') }}" +backup_s3_secret_access_key: "{{ lookup('env', 'BACKUP_S3_SECRET_ACCESS_KEY') }}" +backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}" diff --git a/ansible/roles/deploy/tasks/main.yml b/ansible/roles/deploy/tasks/main.yml index a24c27d..7ad5889 100644 --- a/ansible/roles/deploy/tasks/main.yml +++ b/ansible/roles/deploy/tasks/main.yml @@ -3,6 +3,10 @@ # used to be generated by OpenTofu (services//.env and, for dev, # Runners/docker-compose.yml) - replaces the null_resource.deploy_services + # local_file provisioners previously in infra/modules//main.tf. +# +# Persistent data lives in named Docker volumes now, not on a Hetzner volume - +# see services//backup-docker-compose.yml. .env only carries the S3 +# credentials that backup/restore need, rendered for every host. - name: Ensure services directory exists on the host ansible.builtin.file: @@ -16,28 +20,11 @@ dest: "/home/{{ deploy_user }}/services/{{ service_group }}/" mode: preserve -- name: Look up this host's data volume - when: service_group in ['dev', 'prod'] - delegate_to: localhost - become: false - hetzner.hcloud.hcloud_volume_info: - api_token: "{{ hcloud_token }}" - name: "{{ service_group }}-storage" - register: _volume_info - -# Hetzner's automount tooling always mounts an auto-mounted volume at -# /mnt/HC_Volume_ - deterministic once the volume exists. -- name: Compute DATA_DIR from the volume's id - when: service_group in ['dev', 'prod'] - ansible.builtin.set_fact: - data_dir: "/mnt/HC_Volume_{{ _volume_info.hcloud_volume_info[0].id }}" - -- name: Render DATA_DIR into .env (auto-loaded by docker compose) - when: service_group in ['dev', 'prod'] +- name: Render S3 backup credentials into .env (auto-loaded by docker compose) ansible.builtin.template: src: env.j2 dest: "/home/{{ deploy_user }}/services/{{ service_group }}/.env" - mode: "0644" + mode: "0600" - name: Render Gitea Actions runner compose file when: service_group == 'dev' @@ -46,10 +33,11 @@ dest: "/home/{{ deploy_user }}/services/dev/Runners/docker-compose.yml" mode: "0644" -- name: Make spinup/spindown scripts executable +- name: Make spinup/spindown/restore scripts executable ansible.builtin.file: path: "/home/{{ deploy_user }}/services/{{ service_group }}/{{ item }}" mode: "0755" loop: - spinup.sh - spindown.sh + - restore.sh diff --git a/ansible/roles/deploy/templates/env.j2 b/ansible/roles/deploy/templates/env.j2 index a958fd1..b44355b 100644 --- a/ansible/roles/deploy/templates/env.j2 +++ b/ansible/roles/deploy/templates/env.j2 @@ -1 +1,5 @@ -DATA_DIR={{ data_dir }} +AWS_ACCESS_KEY_ID={{ backup_s3_access_key_id }} +AWS_SECRET_ACCESS_KEY={{ backup_s3_secret_access_key }} +AWS_S3_BUCKET_NAME={{ backup_s3_bucket }} +AWS_S3_PATH={{ service_group }} +AWS_ENDPOINT={{ backup_s3_endpoint }} diff --git a/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 index f72c94f..a5bbcb4 100644 --- a/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 +++ b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 @@ -7,10 +7,9 @@ # (not baked in here) - services/dev/spinup.sh generates a fresh token and # writes it to Runners/.env immediately before starting these containers. # -# /data lives on dev's volume ({{ data_dir }}) so runner identity survives a -# server rebuild - baked in directly rather than via .env, since Runners/.env -# is already reserved for the registration token above and gets overwritten -# on every deploy. +# /data is a disposable local cache (runner identity re-registers on every +# spinup.sh run anyway), so it's a plain named volume - not part of the S3 +# backup/restore stack in services/dev/backup-docker-compose.yml. services: {% for i in range(1, dev_runner_count + 1) %} runner-{{ i }}: @@ -18,7 +17,7 @@ services: container_name: gitea_runner_{{ i }} volumes: - ./config.yaml:/config.yaml - - {{ data_dir }}/gitea_runner_{{ i }}:/data + - gitea_runner_{{ i }}_data:/data - /var/run/docker.sock:/var/run/docker.sock networks: - proxy @@ -33,3 +32,9 @@ services: networks: proxy: external: true + +volumes: +{% for i in range(1, dev_runner_count + 1) %} + gitea_runner_{{ i }}_data: + name: gitea_runner_{{ i }}_data +{% endfor %} diff --git a/infra/main.tf b/infra/main.tf index 9df4e16..fd6d20b 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -1,6 +1,8 @@ # Root module: wires together the shared network and the three host modules # (one per server in docs/architecture.md). Each host module owns its own -# firewall (and volume, where applicable) - see modules//main.tf. +# firewall - see modules//main.tf. Persistent data lives in named Docker +# volumes backed up to S3 (see services//backup-docker-compose.yml), not +# on Hetzner volumes. locals { # SSH to dev/prod is only permitted from the vpn server's public IP: admins must @@ -28,8 +30,6 @@ locals { { zone = "luke-else.co.uk", name = "traefik.cicd", value = module.dev.ipv4 }, { zone = "luke-else.co.uk", name = "vpn", value = module.vpn.ipv4 }, { zone = "luke-else.co.uk", name = "traefik.vpn", value = module.vpn.ipv4 }, - { zone = "divine-couture.co.uk", name = "www", value = module.prod.ipv4 }, - { zone = "snexo.co.uk", name = "@", value = module.prod.ipv4 }, ] } @@ -50,7 +50,6 @@ module "dev" { ssh_key_ids = local.ssh_key_ids network_id = module.network.id private_ip = var.dev_private_ip - volume_size = var.dev_volume_size allowed_ssh_source_ips = local.vpn_ssh_source_ips network_ip_range = var.network_ip_range @@ -68,7 +67,6 @@ module "prod" { ssh_key_ids = local.ssh_key_ids network_id = module.network.id private_ip = var.prod_private_ip - volume_size = var.prod_volume_size allowed_ssh_source_ips = local.vpn_ssh_source_ips network_ip_range = var.network_ip_range diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf index 6bea690..cd811f1 100644 --- a/infra/modules/dev/main.tf +++ b/infra/modules/dev/main.tf @@ -64,24 +64,3 @@ resource "hcloud_server" "this" { } } -resource "hcloud_volume" "storage" { - name = "dev-storage" - size = var.volume_size - server_id = hcloud_server.this.id - automount = true - format = "ext4" - - lifecycle { - prevent_destroy = true - } -} - -locals { - # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is - # always /mnt/HC_Volume_ - deterministic once the volume exists, since the - # id is stable for the volume's lifetime regardless of which server it's - # attached to. Kept here only as a human-readable output (see outputs.tf) - - # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't - # read this value. - data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}" -} diff --git a/infra/modules/dev/outputs.tf b/infra/modules/dev/outputs.tf index 86e4ee2..4897389 100644 --- a/infra/modules/dev/outputs.tf +++ b/infra/modules/dev/outputs.tf @@ -5,8 +5,3 @@ output "ipv4" { output "private_ipv4" { value = var.private_ip } - -output "data_dir" { - description = "Deterministic host path of the auto-mounted volume - see local.data_dir." - value = local.data_dir -} diff --git a/infra/modules/dev/variables.tf b/infra/modules/dev/variables.tf index 89dbb4e..a9d7899 100644 --- a/infra/modules/dev/variables.tf +++ b/infra/modules/dev/variables.tf @@ -28,11 +28,6 @@ variable "private_ip" { type = string } -variable "volume_size" { - description = "Size in GB of the volume attached to dev (disk2 in docs/architecture.md)." - type = number -} - variable "allowed_ssh_source_ips" { description = "CIDRs allowed to reach port 22 on dev. Set by the root module to the vpn server's public IP." type = list(string) diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index 8135cae..65c2775 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -91,24 +91,3 @@ resource "hcloud_server" "this" { } } -resource "hcloud_volume" "storage" { - name = "prod-storage" - size = var.volume_size - server_id = hcloud_server.this.id - automount = true - format = "ext4" - - lifecycle { - prevent_destroy = true - } -} - -locals { - # Hetzner's automount convention (hc-utils' 99-hc-volume-automount.rules) is - # always /mnt/HC_Volume_ - deterministic once the volume exists, since the - # id is stable for the volume's lifetime regardless of which server it's - # attached to. Kept here only as a human-readable output (see outputs.tf) - - # ansible/roles/deploy looks this up itself from the Hetzner API, it doesn't - # read this value. - data_dir = "/mnt/HC_Volume_${hcloud_volume.storage.id}" -} diff --git a/infra/modules/prod/outputs.tf b/infra/modules/prod/outputs.tf index 86e4ee2..4897389 100644 --- a/infra/modules/prod/outputs.tf +++ b/infra/modules/prod/outputs.tf @@ -5,8 +5,3 @@ output "ipv4" { output "private_ipv4" { value = var.private_ip } - -output "data_dir" { - description = "Deterministic host path of the auto-mounted volume - see local.data_dir." - value = local.data_dir -} diff --git a/infra/modules/prod/variables.tf b/infra/modules/prod/variables.tf index e105b70..860bb14 100644 --- a/infra/modules/prod/variables.tf +++ b/infra/modules/prod/variables.tf @@ -28,11 +28,6 @@ variable "private_ip" { type = string } -variable "volume_size" { - description = "Size in GB of the volume attached to prod (disk1 in docs/architecture.md)." - type = number -} - variable "allowed_ssh_source_ips" { description = "CIDRs allowed to reach port 22 on prod. Set by the root module to the vpn server's public IP." type = list(string) diff --git a/infra/outputs.tf b/infra/outputs.tf index d0b29b1..f8e427c 100644 --- a/infra/outputs.tf +++ b/infra/outputs.tf @@ -14,16 +14,6 @@ output "prod_private_ipv4" { value = module.prod.private_ipv4 } -output "dev_data_dir" { - description = "Deterministic host path of dev's auto-mounted volume - also written to services/dev/.env as DATA_DIR." - value = module.dev.data_dir -} - -output "prod_data_dir" { - description = "Deterministic host path of prod's auto-mounted volume - also written to services/prod/.env as DATA_DIR." - value = module.prod.data_dir -} - output "vpn_ipv4" { value = module.vpn.ipv4 } diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index 3c8c22b..f5e7fdb 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -14,7 +14,5 @@ ssh_key_names = ["your-key-name", "laptop", "phone"] # dev_server_type = "cx22" # prod_server_type = "cx22" # vpn_server_type = "cx22" -# dev_volume_size = 20 -# prod_volume_size = 20 # allowed_ssh_source_ips = ["203.0.113.4/32"] # dns_zones = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"] diff --git a/infra/variables.tf b/infra/variables.tf index 95dc546..e974430 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -1,5 +1,5 @@ variable "location" { - description = "Hetzner Cloud datacenter location for all servers and volumes." + description = "Hetzner Cloud datacenter location for all servers." type = string default = "nbg1" } @@ -34,18 +34,6 @@ variable "vpn_server_type" { default = "cx22" } -variable "dev_volume_size" { - description = "Size in GB of the volume attached to dev" - type = number - default = 20 -} - -variable "prod_volume_size" { - description = "Size in GB of the volume attached to prod (disk1 in docs/architecture.md)." - type = number - default = 20 -} - variable "network_ip_range" { description = "IP range of the private network shared by dev and prod." type = string @@ -84,5 +72,5 @@ variable "allowed_ssh_source_ips" { variable "dns_zones" { description = "Domains to manage as Hetzner DNS zones. Point each domain's registrar NS records at tofu output dns_nameservers for Hetzner to actually become authoritative." type = list(string) - default = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"] + default = ["luke-else.co.uk"] } diff --git a/readme.md b/readme.md index 2e017e4..ed8b055 100644 --- a/readme.md +++ b/readme.md @@ -28,9 +28,6 @@ architecture-beta group network(cloud)[network] in cloud - service disk1(mdi:disk)[Storage] in cloud - service disk2(mdi:disk)[Storage] in cloud - service dev(mdi:server)[dev] in network service prod(mdi:server)[prod] in network @@ -41,24 +38,30 @@ architecture-beta service gateway(mdi:web)[gateway] in cloud + service backups(mdi:bucket)[S3 Backups] + dev:L -- R:prod - disk1:B -- T:prod - disk2:B -- T:dev prod:B -- T:prodfirewall vpn:B -- T:vpnfirewall prodfirewall: L -- R: gateway vpnfirewall: B -- T: gateway + + dev:T -- B:backups + prod:T -- B:backups + vpn:R -- L:backups ``` -`gateway` represents the public internet, not a provisioned resource. +`gateway` represents the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from - not a Hetzner resource, see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3). -| Server | Purpose | Network | Volume | -|---|---|---|---| -| `dev` | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) | `dev-storage` | -| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | `prod-storage` | -| `vpn` | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | none | +| Server | Purpose | Network | +|---|---|---| +| `dev` | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) | +| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | +| `vpn` | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | + +Each host's stateful data lives in local named Docker volumes, backed up nightly to a shared S3 bucket and restored automatically on first boot - see below. No host has a Hetzner Cloud Volume attached any more. `dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach deployment targets on `prod` without exposing that traffic publicly. `vpn` is deliberately kept off this network. Each server has its own Hetzner Cloud Firewall (see `infra/modules//main.tf`) that only opens the ports actually used by the compose stacks running on it, plus SSH restricted to `var.allowed_ssh_source_ips`. @@ -66,7 +69,7 @@ architecture-beta ``` . -├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, volumes, DNS +├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, DNS │ ├── main.tf # root module: wires network + dev/prod/vpn/dns modules together │ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names) │ ├── outputs.tf # pass-through outputs from each host module @@ -75,21 +78,21 @@ architecture-beta │ ├── terraform.tfvars.example │ └── modules/ │ ├── network/ # shared private network + subnet (used by dev and prod) -│ ├── dev/ # dev server + firewall + volume, labeled role=dev -│ ├── prod/ # prod server + firewall + volume, labeled role=prod -│ ├── vpn/ # vpn server + firewall (no private network, no volume), labeled role=vpn +│ ├── dev/ # dev server + firewall, labeled role=dev +│ ├── prod/ # prod server + firewall, labeled role=prod +│ ├── vpn/ # vpn server + firewall (no private network), labeled role=vpn │ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones ├── ansible/ # Ansible: bootstraps each server and deploys/starts services// onto it │ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by the role label above -│ ├── group_vars/ # deploy_user, dev_runner_count, etc. +│ ├── group_vars/ # deploy_user, dev_runner_count, backup_s3_*, etc. │ ├── roles/ │ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades -│ │ └── deploy/ # copies services//, renders .env + Runners/docker-compose.yml +│ │ └── deploy/ # copies services//, renders .env (S3 backup credentials) + Runners/docker-compose.yml │ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml ├── services/ # Docker Compose stacks, grouped by which server they run on -│ ├── dev/ # Gitea + CI runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) -│ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik -│ ├── vpn/ # OpenVPN + Traefik +│ ├── dev/ # Gitea + CI runner + Traefik + backup/restore (git.luke-else.co.uk, cicd.luke-else.co.uk) +│ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore +│ ├── vpn/ # OpenVPN + Traefik + backup/restore │ └── todo.md # Outstanding manual setup/hardening tasks ├── docs/ │ └── architecture.md # Source of the architecture diagram above @@ -97,9 +100,9 @@ architecture-beta └── assets/ ``` -Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side. +Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, a `backup-docker-compose.yml` + `restore.sh` pair for that host's S3 backup/restore (see below), plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side. -OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, volumes, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services//`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown. +OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services//`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown. ## Prerequisites @@ -108,6 +111,7 @@ OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud res - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` - [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`) - Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner (see [Managing DNS](#managing-dns) — the zones and records themselves are created for you) +- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/), AWS S3, or any other S3-compatible provider) and an access key/secret pair with read/write access to it - not provisioned by OpenTofu (the `hcloud` provider has no Object Storage resource), so create this yourself and pass it to Ansible as `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` (and `BACKUP_S3_ENDPOINT` for anything other than real AWS S3) - see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3) Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below). @@ -127,10 +131,11 @@ tofu apply This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` / `module.dns` in `infra/main.tf`: - `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`) - one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses, each server labeled `role = "dev"/"prod"/"vpn"` for Ansible's dynamic inventory (`modules/dev`, `modules/prod`, `modules/vpn`) -- one `hcloud_volume` each for `dev` and `prod` (`modules/dev`, `modules/prod`; `vpn` has none) - one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns)) -Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`, `tofu output dev_data_dir`, `tofu output prod_data_dir` (the last two are informational only now — Ansible looks the data directory up itself, see below). +No servers have an attached volume — persistent data lives in named Docker volumes backed up to S3 instead, see below. + +Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`. `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`. @@ -148,7 +153,7 @@ Only A records for IPv4 are managed here — none of the modules currently track ## Bootstrapping and deploying with Ansible (`ansible/`) -Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services//` to each server, rendering the files OpenTofu used to generate (`.env`'s `DATA_DIR`, `dev`'s `Runners/docker-compose.yml`), and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version: +Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services//` to each server, rendering `.env` (S3 backup credentials) and, for `dev`, `Runners/docker-compose.yml`, and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version: ```sh cd ansible @@ -164,13 +169,25 @@ ansible-playbook playbooks/site.yml -l role_dev,role_prod Hosts are discovered dynamically from the Hetzner API (`ansible/inventory/hcloud.yml`), grouped into `role_dev`/`role_prod`/`role_vpn` by the `role` label OpenTofu sets on each server — there's no static inventory file to keep in sync, and nothing here reads Terraform state. -### Persistent data lives on the volumes, not the server disk +### Persistent data lives in named Docker volumes, backed up to S3 -`dev` and `prod`'s compose files bind-mount container data to paths on the `dev-storage` / `prod-storage` volumes rather than the server's local disk, so it survives the server being destroyed and recreated: `${DATA_DIR}/gitea:/data`, `${DATA_DIR}/bitwarden/:/data/`, and so on across `services/dev/*.yml` and `services/prod/*.yml`. `vpn` has no volume, so it's untouched - see the architecture table above. +Every stateful service's compose file mounts a plain named Docker volume rather than a bind mount to a Hetzner volume - `gitea_data:/data`, `bitwarden_data:/data/`, `letsencrypt_data:/letsencrypt`, and so on across `services/dev/*.yml`, `services/prod/*.yml`, and `services/vpn/*.yml`. Each volume is declared with an explicit `name:` in every compose file that mounts it (the same pattern already used for the shared `proxy` network), so Compose creates it once and every stack that references it reuses the same underlying volume regardless of which compose file happens to run first. -`DATA_DIR` resolves deterministically: Hetzner always automounts a volume at `/mnt/HC_Volume_`. Ansible's `deploy` role looks the volume up by name (`hcloud_volume_info` for `dev-storage`/`prod-storage`) directly against the Hetzner API and renders that path into `services//.env` as `DATA_DIR=...` on the remote host — `docker compose` auto-loads `.env` from its working directory for variable substitution. Nothing is written locally; re-run `ansible-playbook playbooks/deploy.yml` if a volume is ever destroyed and recreated (new id). `spinup.sh` on both hosts refuses to start anything if `.env` is missing, rather than silently falling back to a relative path. +Each host's `backup-docker-compose.yml` defines two services against those same named volumes: +- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), running on a nightly cron (`0 3 * * *`), tars up every mounted volume and uploads it to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/-.tar.gz`, keeping 14 days of history (`BACKUP_RETENTION_DAYS`). Containers labeled `docker-volume-backup.stop-during-backup=true` (Gitea, Bitwarden, RustDesk, Uptime Kuma, snexo, OpenVPN) are stopped for the duration of the backup so their data is captured in a consistent state, then restarted automatically. Started by `spinup.sh` alongside everything else. +- **`restore`** — a one-shot container (`amazon/aws-cli` running `restore.sh`) that `spinup.sh` runs *before* anything else starts. It checks each of the host's named volumes; any that are already empty are populated from the most recent object under that host's S3 prefix, any that already contain data are left completely untouched. On a fresh server with no prior backup (or a bucket that doesn't exist yet), it's a no-op - services just start with empty volumes as before. -The Gitea Actions runners are the one exception to the `.env` mechanism: their `/data` path is baked directly into the rendered `Runners/docker-compose.yml`, because `Runners/.env` is already reserved for the registration token and gets overwritten on every deploy. +`AWS_S3_BUCKET_NAME`, `AWS_S3_PATH` (set to the host name, e.g. `dev`), `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_ENDPOINT` are rendered into `services//.env` by Ansible's `deploy` role from `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`/`BACKUP_S3_ENDPOINT` in your shell environment (see [Prerequisites](#prerequisites)) — `docker compose` auto-loads `.env` from its working directory. `spinup.sh` on every host refuses to start anything if `.env` is missing. + +The Gitea Actions runners are the one exception: their `/data` is a disposable local cache (runner identity re-registers with Gitea on every `spinup.sh` run anyway), so it's a plain named volume per runner (`gitea_runner_N_data`, baked directly into the rendered `Runners/docker-compose.yml`) that's deliberately left out of the backup/restore stack. + +To trigger a backup or restore manually rather than waiting for cron or a restart: + +```sh +cd services/ +docker compose -f backup-docker-compose.yml exec backup backup # force an immediate backup +docker compose -f backup-docker-compose.yml run --rm restore # restore any currently-empty volumes +``` ### Scaling Gitea Actions runners @@ -232,6 +249,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi | Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` | | Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | N/A | | Watchtower | `watchtower-docker-compose.yml` | — | +| Backup/restore | `backup-docker-compose.yml` | — | ### `prod` @@ -243,6 +261,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi | Bitwarden (Vaultwarden) | `bitwarden-docker-compose.yml` | `bitwarden.luke-else.co.uk` | | RustDesk relay (hbbs/hbbr) | `rd-docker-compose.yml` | `rd.luke-else.co.uk`, ports `21115-21119` | | Watchtower | `watchtower-docker-compose.yml` | — | +| Backup/restore | `backup-docker-compose.yml` | — | ### `vpn` @@ -251,6 +270,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi | Traefik | `traefik-docker-compose.yml` | `traefik.vpn.luke-else.co.uk` | | OpenVPN (Dockovpn) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `1194` | | Watchtower | `watchtower-docker-compose.yml` | — | +| Backup/restore | `backup-docker-compose.yml` | — | ## First-time setup @@ -270,7 +290,7 @@ Then reopen the repo in VS Code with the Dev Containers extension. ## Security notes -- Real secrets (`HCLOUD_TOKEN`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. +- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. The S3 credentials end up in each host's `services//.env` (rendered by Ansible, mode `0600`, gitignored) - anyone with root on a host can read them, same trust boundary as everything else `deploy` can already do there. - SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). - `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP. - Firewalls are allowlists scoped per host in `infra/modules//main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. diff --git a/services/dev/backup-docker-compose.yml b/services/dev/backup-docker-compose.yml new file mode 100644 index 0000000..c9a18e1 --- /dev/null +++ b/services/dev/backup-docker-compose.yml @@ -0,0 +1,48 @@ +services: + # Nightly backup of every named volume on this host to S3. Containers + # labeled docker-volume-backup.stop-during-backup=true (see + # gitea-docker-compose.yml) are stopped for the duration of the backup so + # their data is captured in a consistent state, then restarted. + backup: + image: offen/docker-volume-backup:v2 + container_name: backup + environment: + BACKUP_FILENAME: "dev-%Y-%m-%dT%H-%M-%S.{{ .Extension }}" + BACKUP_CRON_EXPRESSION: "0 3 * * *" + BACKUP_RETENTION_DAYS: "14" + BACKUP_PRUNING_LEEWAY: "5m" + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - gitea_data:/backup/gitea_data:ro + - letsencrypt_data:/backup/letsencrypt_data:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + + # One-shot: restores empty volumes from the latest S3 backup, then exits. + # Run explicitly by spinup.sh before the services that own these volumes + # start - not started by `docker compose up`. + restore: + image: amazon/aws-cli:2 + container_name: backup-restore + entrypoint: ["/bin/bash", "/restore.sh"] + environment: + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - ./restore.sh:/restore.sh:ro + - gitea_data:/restore/gitea_data + - letsencrypt_data:/restore/letsencrypt_data + restart: "no" + +volumes: + gitea_data: + name: gitea_data + letsencrypt_data: + name: letsencrypt_data diff --git a/services/dev/gitea-docker-compose.yml b/services/dev/gitea-docker-compose.yml index 4354a68..836437a 100644 --- a/services/dev/gitea-docker-compose.yml +++ b/services/dev/gitea-docker-compose.yml @@ -4,7 +4,7 @@ services: image: gitea/gitea:latest container_name: gitea volumes: - - ${DATA_DIR}/gitea:/data + - gitea_data:/data - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro networks: @@ -29,7 +29,11 @@ services: labels: ## Expose Gitea Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + + # Stopped for the duration of each backup run so the sqlite db/repos are + # captured in a consistent state - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.cors-gitea.headers.accesscontrolallowmethods=*" - "traefik.http.middlewares.cors-gitea.headers.accesscontrolalloworiginlist=*" - "traefik.http.middlewares.cors-gitea.headers.addvaryheader=true" @@ -51,4 +55,8 @@ services: networks: proxy: - external: true \ No newline at end of file + external: true + +volumes: + gitea_data: + name: gitea_data \ No newline at end of file diff --git a/services/dev/restore.sh b/services/dev/restore.sh new file mode 100755 index 0000000..a7732cd --- /dev/null +++ b/services/dev/restore.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Restores this host's named volumes from the latest S3 backup, but only into +# volumes that are currently empty - never overwrites data that's already +# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml +# run --rm restore` before any service that owns one of the mounted volumes +# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh +# estate): it just leaves the empty volumes alone. +set -euo pipefail + +S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}" +ENDPOINT_ARGS=() +if [ -n "${AWS_ENDPOINT:-}" ]; then + ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}") +fi + +empty_volumes=() +for dir in /restore/*/; do + name="$(basename "$dir")" + if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then + empty_volumes+=("$name") + fi +done + +if [ "${#empty_volumes[@]}" -eq 0 ]; then + echo "All volumes already contain data - skipping restore." + exit 0 +fi + +echo "Empty volumes: ${empty_volumes[*]}" + +latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)" +if [ -z "$latest" ]; then + echo "No existing backup found at ${S3_URI} - starting with empty volumes." + exit 0 +fi + +echo "Restoring ${S3_URI}/${latest} into empty volumes only..." +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz" +mkdir "$tmp/extracted" +tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted" + +for name in "${empty_volumes[@]}"; do + if [ -d "$tmp/extracted/$name" ]; then + echo "Restoring $name..." + cp -a "$tmp/extracted/$name/." "/restore/$name/" + else + echo "No $name/ in backup - leaving volume empty." + fi +done + +echo "Restore complete." diff --git a/services/dev/spindown.sh b/services/dev/spindown.sh index be4e6a3..c495c7a 100755 --- a/services/dev/spindown.sh +++ b/services/dev/spindown.sh @@ -9,7 +9,9 @@ cd .. docker compose -f gitea-docker-compose.yml down docker compose -f traefik-docker-compose.yml down docker compose -f watchtower-docker-compose.yml down +docker compose -f backup-docker-compose.yml down docker rmi $(docker images -q) docker system prune -f -a -docker volume prune -f -a +# Deliberately no `docker volume prune` - gitea_data/letsencrypt_data/etc. are +# named volumes holding real data, backed up to S3 but not disposable. diff --git a/services/dev/spinup.sh b/services/dev/spinup.sh index 0692a37..22044b8 100755 --- a/services/dev/spinup.sh +++ b/services/dev/spinup.sh @@ -4,10 +4,14 @@ set -e cd "$(dirname "$0")" if [ ! -f .env ]; then - echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 + echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 exit 1 fi +echo "Restoring named volumes from S3 backup if any are empty..." +docker compose -f backup-docker-compose.yml pull restore +docker compose -f backup-docker-compose.yml run --rm restore + docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d @@ -31,3 +35,6 @@ docker compose pull && docker compose up -d cd .. docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d + +docker compose -f backup-docker-compose.yml pull backup +docker compose -f backup-docker-compose.yml up -d backup diff --git a/services/dev/traefik-docker-compose.yml b/services/dev/traefik-docker-compose.yml index 0a69b43..6e46ebc 100644 --- a/services/dev/traefik-docker-compose.yml +++ b/services/dev/traefik-docker-compose.yml @@ -15,7 +15,7 @@ services: - "80:80" - "443:443" volumes: - - "${DATA_DIR}/letsencrypt:/letsencrypt" + - "letsencrypt_data:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy @@ -36,3 +36,7 @@ services: networks: proxy: name: proxy + +volumes: + letsencrypt_data: + name: letsencrypt_data diff --git a/services/prod/backup-docker-compose.yml b/services/prod/backup-docker-compose.yml new file mode 100644 index 0000000..7e996f0 --- /dev/null +++ b/services/prod/backup-docker-compose.yml @@ -0,0 +1,59 @@ +services: + # Nightly backup of every named volume on this host to S3. Containers + # labeled docker-volume-backup.stop-during-backup=true (see + # bitwarden-docker-compose.yml, rd-docker-compose.yml, + # status-docker-compose.yml, web-docker-compose.yml) are stopped for the + # duration of the backup so their data is captured in a consistent state, + # then restarted. + backup: + image: offen/docker-volume-backup:v2 + container_name: backup + environment: + BACKUP_FILENAME: "prod-%Y-%m-%dT%H-%M-%S.{{ .Extension }}" + BACKUP_CRON_EXPRESSION: "0 3 * * *" + BACKUP_RETENTION_DAYS: "14" + BACKUP_PRUNING_LEEWAY: "5m" + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - bitwarden_data:/backup/bitwarden_data:ro + - rustdesk_data:/backup/rustdesk_data:ro + - uptime_kuma_data:/backup/uptime_kuma_data:ro + - snexo_data:/backup/snexo_data:ro + - letsencrypt_data:/backup/letsencrypt_data:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + + # One-shot: restores empty volumes from the latest S3 backup, then exits. + # Run explicitly by spinup.sh before the services that own these volumes + # start - not started by `docker compose up`. + restore: + image: amazon/aws-cli:2 + container_name: backup-restore + entrypoint: [ "/bin/bash", "/restore.sh" ] + environment: + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - ./restore.sh:/restore.sh:ro + - bitwarden_data:/restore/bitwarden_data + - rustdesk_data:/restore/rustdesk_data + - uptime_kuma_data:/restore/uptime_kuma_data + - letsencrypt_data:/restore/letsencrypt_data + restart: "no" + +volumes: + bitwarden_data: + name: bitwarden_data + rustdesk_data: + name: rustdesk_data + uptime_kuma_data: + name: uptime_kuma_data + letsencrypt_data: + name: letsencrypt_data diff --git a/services/prod/bitwarden-docker-compose.yml b/services/prod/bitwarden-docker-compose.yml index 8128bda..2cd48fa 100644 --- a/services/prod/bitwarden-docker-compose.yml +++ b/services/prod/bitwarden-docker-compose.yml @@ -5,13 +5,16 @@ services: image: "vaultwarden/server:latest" container_name: vaultwarden volumes: - - ${DATA_DIR}/bitwarden/:/data/ + - bitwarden_data:/data/ networks: - proxy labels: ## Expose Bitwarden Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.bitwarden-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.bitwarden-insecure.rule=Host(`bitwarden.luke-else.co.uk`)" @@ -24,4 +27,8 @@ services: networks: proxy: - external: true \ No newline at end of file + external: true + +volumes: + bitwarden_data: + name: bitwarden_data \ No newline at end of file diff --git a/services/prod/rd-docker-compose.yml b/services/prod/rd-docker-compose.yml index bab3865..5660ba1 100644 --- a/services/prod/rd-docker-compose.yml +++ b/services/prod/rd-docker-compose.yml @@ -6,7 +6,7 @@ services: restart: unless-stopped volumes: - - ${DATA_DIR}/rustdesk:/root + - rustdesk_data:/root networks: - proxy @@ -20,6 +20,9 @@ services: labels: - "traefik.enable=true" + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.rustdesk-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.rustdesk-insecure.rule=Host(`rd.luke-else.co.uk`)" @@ -39,7 +42,7 @@ services: restart: unless-stopped volumes: - - ${DATA_DIR}/rustdesk:/root + - rustdesk_data:/root networks: - proxy @@ -50,4 +53,8 @@ services: networks: proxy: - external: true \ No newline at end of file + external: true + +volumes: + rustdesk_data: + name: rustdesk_data \ No newline at end of file diff --git a/services/prod/restore.sh b/services/prod/restore.sh new file mode 100755 index 0000000..a7732cd --- /dev/null +++ b/services/prod/restore.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Restores this host's named volumes from the latest S3 backup, but only into +# volumes that are currently empty - never overwrites data that's already +# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml +# run --rm restore` before any service that owns one of the mounted volumes +# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh +# estate): it just leaves the empty volumes alone. +set -euo pipefail + +S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}" +ENDPOINT_ARGS=() +if [ -n "${AWS_ENDPOINT:-}" ]; then + ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}") +fi + +empty_volumes=() +for dir in /restore/*/; do + name="$(basename "$dir")" + if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then + empty_volumes+=("$name") + fi +done + +if [ "${#empty_volumes[@]}" -eq 0 ]; then + echo "All volumes already contain data - skipping restore." + exit 0 +fi + +echo "Empty volumes: ${empty_volumes[*]}" + +latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)" +if [ -z "$latest" ]; then + echo "No existing backup found at ${S3_URI} - starting with empty volumes." + exit 0 +fi + +echo "Restoring ${S3_URI}/${latest} into empty volumes only..." +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz" +mkdir "$tmp/extracted" +tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted" + +for name in "${empty_volumes[@]}"; do + if [ -d "$tmp/extracted/$name" ]; then + echo "Restoring $name..." + cp -a "$tmp/extracted/$name/." "/restore/$name/" + else + echo "No $name/ in backup - leaving volume empty." + fi +done + +echo "Restore complete." diff --git a/services/prod/spindown.sh b/services/prod/spindown.sh index 665ab2b..e6c19ad 100755 --- a/services/prod/spindown.sh +++ b/services/prod/spindown.sh @@ -8,7 +8,9 @@ docker compose -f bitwarden-docker-compose.yml down docker compose -f rd-docker-compose.yml down docker compose -f watchtower-docker-compose.yml down docker compose -f traefik-docker-compose.yml down +docker compose -f backup-docker-compose.yml down docker rmi $(docker images -q) docker system prune -f -a -docker volume prune -f -a +# Deliberately no `docker volume prune` - bitwarden_data/rustdesk_data/etc. are +# named volumes holding real data, backed up to S3 but not disposable. diff --git a/services/prod/spinup.sh b/services/prod/spinup.sh index ef7e9ee..ac5092b 100755 --- a/services/prod/spinup.sh +++ b/services/prod/spinup.sh @@ -4,10 +4,14 @@ set -e cd "$(dirname "$0")" if [ ! -f .env ]; then - echo "Missing .env (expected DATA_DIR=/mnt/HC_Volume_...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 + echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 exit 1 fi +echo "Restoring named volumes from S3 backup if any are empty..." +docker compose -f backup-docker-compose.yml pull restore +docker compose -f backup-docker-compose.yml run --rm restore + docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d sleep 20 # Allow Traefik + registry auth to settle before starting the rest of the services @@ -17,3 +21,6 @@ docker compose -f status-docker-compose.yml pull && docker compose -f status-doc docker compose -f web-docker-compose.yml pull && docker compose -f web-docker-compose.yml up -d docker compose -f bitwarden-docker-compose.yml pull && docker compose -f bitwarden-docker-compose.yml up -d docker compose -f rd-docker-compose.yml pull && docker compose -f rd-docker-compose.yml up -d + +docker compose -f backup-docker-compose.yml pull backup +docker compose -f backup-docker-compose.yml up -d backup diff --git a/services/prod/status-docker-compose.yml b/services/prod/status-docker-compose.yml index 9595387..f9c5822 100644 --- a/services/prod/status-docker-compose.yml +++ b/services/prod/status-docker-compose.yml @@ -3,13 +3,16 @@ services: image: louislam/uptime-kuma:latest container_name: status volumes: - - ${DATA_DIR}/uptime-kuma/data:/app/data + - uptime_kuma_data:/app/data networks: - proxy labels: ## Expose uptime-kuma Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.status-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.status-insecure.rule=Host(`status.luke-else.co.uk`)" @@ -23,3 +26,7 @@ services: networks: proxy: external: true + +volumes: + uptime_kuma_data: + name: uptime_kuma_data diff --git a/services/prod/traefik-docker-compose.yml b/services/prod/traefik-docker-compose.yml index a4ed496..78b6483 100644 --- a/services/prod/traefik-docker-compose.yml +++ b/services/prod/traefik-docker-compose.yml @@ -20,7 +20,7 @@ services: - "443:443" - "27017:27017" volumes: - - "${DATA_DIR}/letsencrypt:/letsencrypt" + - "letsencrypt_data:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy @@ -41,3 +41,7 @@ services: networks: proxy: name: proxy + +volumes: + letsencrypt_data: + name: letsencrypt_data diff --git a/services/prod/web-docker-compose.yml b/services/prod/web-docker-compose.yml index af5adac..e4edb7f 100644 --- a/services/prod/web-docker-compose.yml +++ b/services/prod/web-docker-compose.yml @@ -1,6 +1,6 @@ services: -#Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080) + #Websites luke-else.co.uk (8000) snexo.co.uk (8001) divine-couture.co.uk (80) wmgzon.luke-else.co.uk (8080) luke-else: image: git.luke-else.co.uk/luke-else/luke-else.co.uk container_name: luke-else @@ -9,7 +9,7 @@ services: labels: ## Expose luke-else Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.personal-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.personal-insecure.rule=Host(`luke-else.co.uk`)" @@ -28,7 +28,7 @@ services: labels: ## Expose luke-else Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.personal-dev-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.personal-dev-insecure.rule=Host(`dev.luke-else.co.uk`)" @@ -47,7 +47,7 @@ services: labels: ## Expose metarius Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.metarius-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.metarius-insecure.rule=Host(`metarius.luke-else.co.uk`)" @@ -66,7 +66,7 @@ services: labels: ## Expose divine-couture Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container - + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.divine-couture-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.divine-couture-insecure.rule=Host(`www.divine-couture.co.uk`)" @@ -77,27 +77,6 @@ services: - "traefik.http.routers.divine-couture.tls.certresolver=myresolver" restart: unless-stopped - snexo: - image: "php:apache" - container_name: snexo - volumes: - - ${DATA_DIR}/snexo.co.uk/:/var/www/html - networks: - - proxy - labels: - ## Expose Snexo Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.snexo-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.snexo-insecure.rule=Host(`snexo.co.uk`)" - - "traefik.http.routers.snexo-insecure.entrypoints=web" - - - "traefik.http.routers.snexo.rule=Host(`snexo.co.uk`)" - - "traefik.http.routers.snexo.entrypoints=websecure" - - "traefik.http.routers.snexo.tls.certresolver=myresolver" - restart: unless-stopped - networks: proxy: external: true diff --git a/services/vpn/backup-docker-compose.yml b/services/vpn/backup-docker-compose.yml new file mode 100644 index 0000000..0c2ae8e --- /dev/null +++ b/services/vpn/backup-docker-compose.yml @@ -0,0 +1,48 @@ +services: + # Nightly backup of every named volume on this host to S3. Containers + # labeled docker-volume-backup.stop-during-backup=true (see + # vpn-docker-compose.yml) are stopped for the duration of the backup so + # their data is captured in a consistent state, then restarted. + backup: + image: offen/docker-volume-backup:v2 + container_name: backup + environment: + BACKUP_FILENAME: "vpn-%Y-%m-%dT%H-%M-%S.{{ .Extension }}" + BACKUP_CRON_EXPRESSION: "0 3 * * *" + BACKUP_RETENTION_DAYS: "14" + BACKUP_PRUNING_LEEWAY: "5m" + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - openvpn_data:/backup/openvpn_data:ro + - letsencrypt_data:/backup/letsencrypt_data:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + + # One-shot: restores empty volumes from the latest S3 backup, then exits. + # Run explicitly by spinup.sh before the services that own these volumes + # start - not started by `docker compose up`. + restore: + image: amazon/aws-cli:2 + container_name: backup-restore + entrypoint: ["/bin/bash", "/restore.sh"] + environment: + AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME} + AWS_S3_PATH: ${AWS_S3_PATH} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_ENDPOINT: ${AWS_ENDPOINT} + volumes: + - ./restore.sh:/restore.sh:ro + - openvpn_data:/restore/openvpn_data + - letsencrypt_data:/restore/letsencrypt_data + restart: "no" + +volumes: + openvpn_data: + name: openvpn_data + letsencrypt_data: + name: letsencrypt_data diff --git a/services/vpn/restore.sh b/services/vpn/restore.sh new file mode 100755 index 0000000..a7732cd --- /dev/null +++ b/services/vpn/restore.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Restores this host's named volumes from the latest S3 backup, but only into +# volumes that are currently empty - never overwrites data that's already +# there. Run by spinup.sh via `docker compose -f backup-docker-compose.yml +# run --rm restore` before any service that owns one of the mounted volumes +# starts. Safe to run on a bucket/prefix that doesn't exist yet (fresh +# estate): it just leaves the empty volumes alone. +set -euo pipefail + +S3_URI="s3://${AWS_S3_BUCKET_NAME}/${AWS_S3_PATH}" +ENDPOINT_ARGS=() +if [ -n "${AWS_ENDPOINT:-}" ]; then + ENDPOINT_ARGS=(--endpoint-url "https://${AWS_ENDPOINT}") +fi + +empty_volumes=() +for dir in /restore/*/; do + name="$(basename "$dir")" + if [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then + empty_volumes+=("$name") + fi +done + +if [ "${#empty_volumes[@]}" -eq 0 ]; then + echo "All volumes already contain data - skipping restore." + exit 0 +fi + +echo "Empty volumes: ${empty_volumes[*]}" + +latest="$(aws "${ENDPOINT_ARGS[@]}" s3 ls "${S3_URI}/" 2>/dev/null | awk '{print $4}' | sort | tail -n1 || true)" +if [ -z "$latest" ]; then + echo "No existing backup found at ${S3_URI} - starting with empty volumes." + exit 0 +fi + +echo "Restoring ${S3_URI}/${latest} into empty volumes only..." +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +aws "${ENDPOINT_ARGS[@]}" s3 cp "${S3_URI}/${latest}" "$tmp/backup.tar.gz" +mkdir "$tmp/extracted" +tar -xzf "$tmp/backup.tar.gz" -C "$tmp/extracted" + +for name in "${empty_volumes[@]}"; do + if [ -d "$tmp/extracted/$name" ]; then + echo "Restoring $name..." + cp -a "$tmp/extracted/$name/." "/restore/$name/" + else + echo "No $name/ in backup - leaving volume empty." + fi +done + +echo "Restore complete." diff --git a/services/vpn/spindown.sh b/services/vpn/spindown.sh index c30233f..c093daa 100755 --- a/services/vpn/spindown.sh +++ b/services/vpn/spindown.sh @@ -5,7 +5,9 @@ cd "$(dirname "$0")" docker compose -f vpn-docker-compose.yml down docker compose -f traefik-docker-compose.yml down docker compose -f watchtower-docker-compose.yml down +docker compose -f backup-docker-compose.yml down docker rmi $(docker images -q) docker system prune -f -a -docker volume prune -f -a +# Deliberately no `docker volume prune` - openvpn_data/letsencrypt_data are +# named volumes holding real data, backed up to S3 but not disposable. diff --git a/services/vpn/spinup.sh b/services/vpn/spinup.sh index 14250a8..1911994 100755 --- a/services/vpn/spinup.sh +++ b/services/vpn/spinup.sh @@ -3,6 +3,18 @@ set -e cd "$(dirname "$0")" +if [ ! -f .env ]; then + echo "Missing .env (expected AWS_S3_BUCKET_NAME/AWS_ACCESS_KEY_ID/...) - run 'ansible-playbook playbooks/deploy.yml' in ansible/ first." >&2 + exit 1 +fi + +echo "Restoring named volumes from S3 backup if any are empty..." +docker compose -f backup-docker-compose.yml pull restore +docker compose -f backup-docker-compose.yml run --rm restore + docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-docker-compose.yml up -d docker compose -f vpn-docker-compose.yml pull && docker compose -f vpn-docker-compose.yml up -d docker compose -f watchtower-docker-compose.yml pull && docker compose -f watchtower-docker-compose.yml up -d + +docker compose -f backup-docker-compose.yml pull backup +docker compose -f backup-docker-compose.yml up -d backup diff --git a/services/vpn/traefik-docker-compose.yml b/services/vpn/traefik-docker-compose.yml index 50a0f18..234ad1c 100644 --- a/services/vpn/traefik-docker-compose.yml +++ b/services/vpn/traefik-docker-compose.yml @@ -15,7 +15,7 @@ services: - "80:80" - "443:443" volumes: - - "./letsencrypt:/letsencrypt" + - "letsencrypt_data:/letsencrypt" - "/var/run/docker.sock:/var/run/docker.sock:ro" networks: - proxy @@ -36,3 +36,7 @@ services: networks: proxy: name: proxy + +volumes: + letsencrypt_data: + name: letsencrypt_data diff --git a/services/vpn/vpn-docker-compose.yml b/services/vpn/vpn-docker-compose.yml index 502dcd8..f78e1af 100644 --- a/services/vpn/vpn-docker-compose.yml +++ b/services/vpn/vpn-docker-compose.yml @@ -8,11 +8,14 @@ services: environment: HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address volumes: - - ./openvpn_conf:/opt/Dockovpn_data + - openvpn_data:/opt/Dockovpn_data labels: ## Expose vpn Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container + # Stopped for the duration of each backup run - see backup-docker-compose.yml. + - "docker-volume-backup.stop-during-backup=true" + - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - "traefik.http.routers.vpn-insecure.middlewares=redirect-web-secure" - "traefik.http.routers.vpn-insecure.rule=Host(`vpn.luke-else.co.uk`)" @@ -21,4 +24,8 @@ services: - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)" - "traefik.http.routers.vpn.entrypoints=websecure" - "traefik.http.routers.vpn.tls.certresolver=myresolver" - restart: always \ No newline at end of file + restart: always + +volumes: + openvpn_data: + name: openvpn_data \ No newline at end of file From 538d425d4aa1042fefde1a472da63992d987f97a Mon Sep 17 00:00:00 2001 From: Luke Else Date: Mon, 20 Jul 2026 23:42:11 +0100 Subject: [PATCH 17/24] feat: Made some small updated to deploy --- ansible/ansible.cfg | 1 + ansible/inventory/hcloud.yml | 2 +- infra/.terraform.lock.hcl | 74 ---------------------------------- infra/terraform.tfvars.example | 12 +++--- 4 files changed, 8 insertions(+), 81 deletions(-) diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index c22a078..0829ba6 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -1,5 +1,6 @@ [defaults] inventory = inventory/hcloud.yml +roles_path = roles host_key_checking = False retry_files_enabled = False interpreter_python = auto_silent diff --git a/ansible/inventory/hcloud.yml b/ansible/inventory/hcloud.yml index 17c6db5..aad0be5 100644 --- a/ansible/inventory/hcloud.yml +++ b/ansible/inventory/hcloud.yml @@ -7,6 +7,6 @@ plugin: hetzner.hcloud.hcloud token: "{{ lookup('env', 'HCLOUD_TOKEN') }}" connect_with: public_ipv4 keyed_groups: - - key: labels.role + - key: hcloud_labels.role prefix: role separator: "_" diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl index 277330a..df97a6f 100644 --- a/infra/.terraform.lock.hcl +++ b/infra/.terraform.lock.hcl @@ -33,80 +33,6 @@ provider "registry.opentofu.org/hashicorp/hcloud" { ] } -provider "registry.opentofu.org/hashicorp/local" { - version = "2.9.0" - constraints = "~> 2.5" - hashes = [ - "h1:1dtKYW/5a1qob3yneL6WzOlnSGfYtJ6a2XeejCk9yb4=", - "h1:5NseXq5wU8O20ersTtV4ocrLYFFtgFr7n0pRLO1W2Rw=", - "h1:5d22ZPPK4iiygPbwRz/PJF5Es/0axVpMlPRpCR0Padw=", - "h1:AnwyolirmIlBMjH6+tV8bKkvT+5axJNYxi2y2IguiX4=", - "h1:PBp+HeseY021Fw3sLznCG27idgwPoff4cBuNmKgPL2w=", - "h1:VDxIhe4GbzdOCdmt7mQaqdwERQW6GSI7Roonts42Gr0=", - "h1:ZO6eWWnf8LjjV1q/JNeL9WLtZ6fwIttOnyN5LjCNSEo=", - "h1:dPIAf8oUAz+vW2E0iZunMvpuPddRZIztRsPSY1u+VnY=", - "h1:fwTDVG9AhFVKQZIb1EXkHv4FqzsZNlLWgkyPGDmZZEE=", - "h1:kDc465XPC7/6XFCjrMC4mTqhA9ef0FHKuJ3ZgfGNfeg=", - "h1:kGbjxrI2P8MHeyVtE1U3Q1TbyF71ExnHxtkrE+Aj6UU=", - "h1:kcoK6Afbsj54u9zaEqpecWAFKytqjBijtguCNwV3d4M=", - "h1:rxomJjDwOo+YZ+WIPc25FqEgsz9orh/2MCyUcZmFjvw=", - "h1:t0CMn/Rkwquw8l2yQ+O4ApzbMZfY2UazbsDnZygzACA=", - "h1:tJwgm2BS4xCGlElCDQEFXQoefY9Y4t0JdSKTtsPBbBo=", - "zh:13ef7ecd1e397ec5b20ea588508dd3e3b8d6c50d809ae76b079abf9dd8d02e4b", - "zh:2190c9325980076489ce02b0f5dd2c0b91fc8711cefa99e714d8619a32827ad1", - "zh:2a0cfc5600730093705071707e4a4e4e953e7d9091859e0f66b46daa1060dd5d", - "zh:2ff53eac1af43ab9a2248a0e53c963d46e19cf04bc4c3f323591cfcebb218252", - "zh:4ebc3dee700f60af9da29970052fd02fa947813162b224716862dc9d7f1f7542", - "zh:5fe6dab84ceeaa8eb3f1567c5f05578333370c472240ca5c5bfc25e92d4d5586", - "zh:66bbec16367bbf440045502c9779b11f4ac5b022c8d8d17afe12d431950838b5", - "zh:7641e5c2e4b529e869cde29ab5b1de2fd1091489eb745b19ac2709bd7f4dfd84", - "zh:855bfba0756d17ce07595ff57d7cf664443d1495127cb88fb063362734b8b22a", - "zh:aaec10f237921d60c581d1b7a66f0a8a8019d9802dc04af11b5b981f6682e01d", - "zh:e460835a38ffa1e74f6929904bfd14ef473d217fd537b7ce834abe5ce5e2ce07", - "zh:ecc4295215db0e4aea3c9329611c31e09a853e1ae207d56742403bd4f5516703", - "zh:ee6d9fae63a612072e00402894e14826af7a3351c235b9c5b423b7629a77ca29", - "zh:f2b5c8db74aa7ebcf7cd423672358437d42401675069ef67b01ff910054e49d5", - "zh:f5aff74d3eb96d4592c7bca5cd3ea89b469e84efbf382944bd0f844a57059c09", - ] -} - -provider "registry.opentofu.org/hashicorp/null" { - version = "3.3.0" - constraints = "~> 3.2" - hashes = [ - "h1:0r7+t8CqzjfBgHgEiJGBCw+McEUdRXliMdF+Hk29d8o=", - "h1:EvvCOc4FJY3NitSm6BpzCcUPU53LayVCB/tPOxYmy7U=", - "h1:IDVnZXNCh0u4LfeSazc9z1v/kNz+92Eej7ePWV6SbyE=", - "h1:Iw2c0n9/4fS92N5WnJ3CCSwSUXZO953oHp9gj3pWCaM=", - "h1:JofS1og3hPN0ANjH+gNjxrJyyk6znodpC/F0qhp4eEk=", - "h1:QIBhsJ4+5+t0vFEgJwtezNLT31tsptFHOEyGAAhLR1o=", - "h1:RjjoL9qRPwNTwLdtJsYUaFvunbPM2/oujf2DcUcitOE=", - "h1:SSirA+z2VWTs1s+TCAx8vVKg9jh6cRjxqc8LYi2iQTI=", - "h1:U2XZc7hxcpcWp/C2S9LtuGUimhMOD2UT5xAEJJQQQaU=", - "h1:bPG+xE5UonkJv3y/Yn9Q7OfbP2qHU/QKiS31nwfe7S0=", - "h1:eODLdk/pARc4yxChAFtwseVmBr+r5fF9yGOvUhwGEyM=", - "h1:iFj1oM5ZPENspsPqK1kcvZzyP95jJE/CM0rlu0MfIss=", - "h1:mdu+qpyVmjDDLMrcL1JFy+cSyF58I3TFJwB5NssCZ58=", - "h1:tJmep6aoBeDH77XsYU65HAbi0RAjxtsmbCOXmnqT13U=", - "h1:tdMTn1evBLd6KCeLqWdQXCpF07hBu3n5rY6N3rXw3Rc=", - "zh:083dcc0bec53f8abfa3f2aa2ce9d732a9675338fd60ae7d61162e25db7cb08bf", - "zh:19f7456b5a2ad16595860974714bfdb25b87bc16356ea9d5c7453892aaa27864", - "zh:222c0ed1fed4e4c677ebe626104dbfdba66763e264de0d9c27c58ce60104ee69", - "zh:271711d6caa7dd5a4e9b79fe8c679fab61a840bcf80040a0f5ebb425d1b27d97", - "zh:5adcf35f30baaea13f80c2a2c774deb9369892719493049687e23476c9dff40f", - "zh:5bcfd19df16e73d7f0ad75bd09e2b3b86cf6700d09822d585d68304b71de1d97", - "zh:604edecf263e38674decb35bb4e0e048fdc951f26fa103c33065ff9728f0313b", - "zh:782acbfb4fa4807e273e588fe45b4aaea9dd0fd1136f76ec3200f6f4db3af8d6", - "zh:84411a596d528fe67294e5c1cfd0c2036b08802497bcc4215ce518924f3c9a4a", - "zh:85e79eecf3f5348975cffec3016b0eba3baf605646102d4348796ccd2df2e5f6", - "zh:95669535ca17aeefef307ebfd59ce6930953173baae5637e8cbbf0297ec7ad58", - "zh:d04d9b177747bfd66b4a45b5d911a2a7822aa8451f5e35621971fb7a4206b530", - "zh:e6d9c924475283e90833450a14a732f4deb6d9bb131db8f86ab856e894270836", - "zh:ebcab0c8a1334c86ed7cfa53f571a17ad6d27e9901f27a8854ea622a74b54bb6", - "zh:ef9c757bb2c83d2103811a3d86b6ec5be06b0ffc337b84db1582d023bce7cdcd", - ] -} - provider "registry.opentofu.org/hetznercloud/hcloud" { version = "1.66.0" constraints = "~> 1.54" diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example index f5e7fdb..e59d991 100644 --- a/infra/terraform.tfvars.example +++ b/infra/terraform.tfvars.example @@ -7,12 +7,12 @@ # Names of SSH keys already uploaded to your Hetzner Cloud project # (Console > Security > SSH Keys). All of them are installed on every server. Required. -ssh_key_names = ["your-key-name", "laptop", "phone"] +ssh_key_names = ["luke-else@PC", "luke-else@laptop"] # Optional overrides - defaults live in variables.tf # location = "nbg1" -# dev_server_type = "cx22" -# prod_server_type = "cx22" -# vpn_server_type = "cx22" -# allowed_ssh_source_ips = ["203.0.113.4/32"] -# dns_zones = ["luke-else.co.uk", "divine-couture.co.uk", "snexo.co.uk"] +# dev_server_type = "cx23" +# prod_server_type = "cx23" +# vpn_server_type = "cx23" +# allowed_ssh_source_ips = ["0.0.0.0/0"] +# dns_zones = ["luke-else.co.uk"] \ No newline at end of file From 9c0dc78a2c73cec0d059f8fd4c3cb74d4b849863 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Tue, 21 Jul 2026 14:56:03 +0100 Subject: [PATCH 18/24] chore: Cleaned up ansible and added control script --- ansible/README.md | 9 +- ansible/group_vars/role_dev.yml | 5 - ansible/roles/bootstrap/tasks/main.yml | 52 ++--- .../templates/runners-docker-compose.yml.j2 | 12 +- control.sh | 198 ++++++++++++++++++ readme.md | 19 +- 6 files changed, 236 insertions(+), 59 deletions(-) create mode 100755 control.sh diff --git a/ansible/README.md b/ansible/README.md index 46c8890..289a2be 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -16,7 +16,7 @@ to S3, not on a Hetzner volume - see `../readme.md`. (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules//main.tf`). - `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, `backup_s3_*` S3 backup credentials), `role_dev.yml` / `role_prod.yml` / - `role_vpn.yml` (per-role vars, e.g. `dev_runner_count`). + `role_vpn.yml` (per-role vars, e.g. `service_group`). - `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening, unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`. - `roles/deploy/` - copies `services//` to the server and renders `.env` @@ -65,9 +65,10 @@ ansible-playbook playbooks/spinup.yml -l role_dev,role_prod ansible-playbook playbooks/site.yml -l role_dev,role_prod ``` -Re-run `deploy.yml` + `spinup.yml` any time a compose file changes or -`dev_runner_count` in `group_vars/role_dev.yml` is bumped - `spinup.sh` only -starts/updates what's changed, same as before. +Re-run `deploy.yml` + `spinup.yml` any time a compose file changes - `spinup.sh` +only starts/updates what's changed, same as before. `dev` always runs three +Gitea Actions runners (fixed in +`roles/deploy/templates/runners-docker-compose.yml.j2`). `spindown.yml` runs `spindown.sh`, which does a full `docker system/volume prune -a` on the host - it prompts for a `yes` confirmation before running, diff --git a/ansible/group_vars/role_dev.yml b/ansible/group_vars/role_dev.yml index 8e7a4ae..073d03d 100644 --- a/ansible/group_vars/role_dev.yml +++ b/ansible/group_vars/role_dev.yml @@ -1,7 +1,2 @@ --- service_group: dev - -# Number of Gitea Actions runner containers rendered into -# services/dev/Runners/docker-compose.yml by the deploy role - see -# roles/deploy/templates/runners-docker-compose.yml.j2. -dev_runner_count: 3 diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index c476d32..307f0cc 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -10,50 +10,22 @@ upgrade: dist cache_valid_time: 3600 -- name: Install Docker prerequisites +# Docker straight from Ubuntu's own repositories - no external apt repo, GPG +# key or architecture juggling. docker.io is the engine, docker-compose-v2 is +# the `docker compose` plugin. A little behind Docker's own channel, but far +# simpler and plenty for these hosts. +- name: Install Docker Engine and the Compose plugin ansible.builtin.apt: name: - - ca-certificates - - curl - - gnupg + - docker.io + - docker-compose-v2 state: present -- name: Create apt keyrings directory - ansible.builtin.file: - path: /etc/apt/keyrings - state: directory - mode: "0755" - -- name: Download Docker's GPG key - ansible.builtin.get_url: - url: https://download.docker.com/linux/ubuntu/gpg - dest: /etc/apt/keyrings/docker.asc - mode: "0644" - force: false - -- name: Determine dpkg architecture - ansible.builtin.command: dpkg --print-architecture - register: _dpkg_arch - changed_when: false - -- name: Add Docker apt repository - ansible.builtin.apt_repository: - repo: >- - deb [arch={{ _dpkg_arch.stdout }} signed-by=/etc/apt/keyrings/docker.asc] - https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable - filename: docker - state: present - -- name: Install Docker Engine and Compose plugin - ansible.builtin.apt: - update_cache: true - name: - - docker-ce - - docker-ce-cli - - containerd.io - - docker-buildx-plugin - - docker-compose-plugin - state: present +- name: Ensure Docker is enabled and running + ansible.builtin.systemd: + name: docker + enabled: true + state: started - name: Create non-root sudo user, given the same key root was provisioned with ansible.builtin.user: diff --git a/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 index a5bbcb4..802e82d 100644 --- a/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 +++ b/ansible/roles/deploy/templates/runners-docker-compose.yml.j2 @@ -1,7 +1,6 @@ -# Generated by Ansible from dev_runner_count - do not hand-edit. -# To change the number of runners, edit dev_runner_count in -# ansible/group_vars/role_dev.yml and re-run playbooks/deploy.yml. -# To change their shape, edit this template instead. +# Generated by Ansible - do not hand-edit. Three Gitea Actions runners, always. +# To change their shape (image, volumes, env), edit this template and re-run +# playbooks/deploy.yml. # # GITEA_RUNNER_REGISTRATION_TOKEN is intentionally left as a compose variable # (not baked in here) - services/dev/spinup.sh generates a fresh token and @@ -10,8 +9,9 @@ # /data is a disposable local cache (runner identity re-registers on every # spinup.sh run anyway), so it's a plain named volume - not part of the S3 # backup/restore stack in services/dev/backup-docker-compose.yml. +{% set runner_count = 3 %} services: -{% for i in range(1, dev_runner_count + 1) %} +{% for i in range(1, runner_count + 1) %} runner-{{ i }}: image: gitea/act_runner:latest container_name: gitea_runner_{{ i }} @@ -34,7 +34,7 @@ networks: external: true volumes: -{% for i in range(1, dev_runner_count + 1) %} +{% for i in range(1, runner_count + 1) %} gitea_runner_{{ i }}_data: name: gitea_runner_{{ i }}_data {% endfor %} diff --git a/control.sh b/control.sh new file mode 100755 index 0000000..50cb4f6 --- /dev/null +++ b/control.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# control.sh - interactive control panel for this repo. +# +# Steps you through the pre-setup and day-to-day operations for both the +# OpenTofu infra (infra/) and the Ansible config (ansible/), in the order the +# firewall rules require (vpn first, then tunnel in, then dev/prod). It only +# ever runs `tofu`/`ansible-playbook` for you - it's a menu wrapper, nothing +# more, so anything it does you can also do by hand from infra/ or ansible/. + +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INFRA_DIR="$REPO_ROOT/infra" +ANSIBLE_DIR="$REPO_ROOT/ansible" + +# --- pretty output -------------------------------------------------------- +if [ -t 1 ]; then + BOLD=$(tput bold); DIM=$(tput dim); RESET=$(tput sgr0) + RED=$(tput setaf 1); GREEN=$(tput setaf 2); YELLOW=$(tput setaf 3); CYAN=$(tput setaf 6) +else + BOLD=""; DIM=""; RESET=""; RED=""; GREEN=""; YELLOW=""; CYAN="" +fi + +ok() { printf "%s ok %s %s\n" "$GREEN" "$RESET" "$1"; } +warn() { printf "%s warn%s %s\n" "$YELLOW" "$RESET" "$1"; } +bad() { printf "%s miss%s %s\n" "$RED" "$RESET" "$1"; } +title() { printf "\n%s%s%s\n" "$BOLD$CYAN" "$1" "$RESET"; } + +# Pick the OpenTofu binary: prefer `tofu`, fall back to `terraform`. +TOFU="" +detect_tofu() { + if command -v tofu >/dev/null 2>&1; then TOFU="tofu" + elif command -v terraform >/dev/null 2>&1; then TOFU="terraform" + fi +} +detect_tofu + +# Run a command inside a directory, echoing it first. Returns the command's exit code. +run_in() { + local dir="$1"; shift + printf "%s\$ (cd %s && %s)%s\n" "$DIM" "${dir#$REPO_ROOT/}" "$*" "$RESET" + ( cd "$dir" && "$@" ) +} + +pause() { read -rp $'\nPress Enter to return to the menu... ' _; } + +confirm() { + local reply + read -rp "$1 [y/N] " reply + [[ "$reply" == "y" || "$reply" == "Y" ]] +} + +# --- prerequisite checks -------------------------------------------------- +check_prereqs() { + title "Tools" + if [ -n "$TOFU" ]; then ok "OpenTofu ($TOFU: $("$TOFU" version 2>/dev/null | head -n1))" + else bad "Neither 'tofu' nor 'terraform' found on PATH"; fi + if command -v ansible-playbook >/dev/null 2>&1; then ok "Ansible ($(ansible --version 2>/dev/null | head -n1))" + else bad "'ansible-playbook' not found on PATH"; fi + + title "Environment variables" + [ -n "${HCLOUD_TOKEN:-}" ] && ok "HCLOUD_TOKEN set" || bad "HCLOUD_TOKEN not set (needed by both tofu and the ansible inventory)" + [ -n "${BACKUP_S3_BUCKET:-}" ] && ok "BACKUP_S3_BUCKET set" || warn "BACKUP_S3_BUCKET not set (needed for deploy/spinup - S3 backup .env)" + [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ] && ok "BACKUP_S3_ACCESS_KEY_ID set" || warn "BACKUP_S3_ACCESS_KEY_ID not set" + [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && ok "BACKUP_S3_SECRET_ACCESS_KEY set" || warn "BACKUP_S3_SECRET_ACCESS_KEY not set" + [ -n "${BACKUP_S3_ENDPOINT:-}" ] && ok "BACKUP_S3_ENDPOINT set" || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)" + + local key="${ANSIBLE_SSH_PRIVATE_KEY_FILE:-$HOME/.ssh/id_ed25519}" + [ -f "${key/#\~/$HOME}" ] && ok "SSH key present ($key)" || warn "SSH key not found at $key (set ANSIBLE_SSH_PRIVATE_KEY_FILE)" + + title "Files" + [ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \ + || bad "infra/terraform.tfvars missing (copy terraform.tfvars.example and fill in ssh_key_names)" + [ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \ + || warn "infra/.terraform missing - run OpenTofu > init" + if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed" + else warn "hetzner.hcloud collection not installed - run Ansible > Install collections"; fi +} + +# --- OpenTofu actions ----------------------------------------------------- +require_tofu() { + if [ -z "$TOFU" ]; then bad "No tofu/terraform binary found."; return 1; fi + if [ ! -f "$INFRA_DIR/terraform.tfvars" ]; then + bad "infra/terraform.tfvars is missing - copy terraform.tfvars.example first."; return 1 + fi +} +tofu_init() { require_tofu && run_in "$INFRA_DIR" "$TOFU" init; } +tofu_plan() { require_tofu && run_in "$INFRA_DIR" "$TOFU" plan; } +tofu_apply() { require_tofu && run_in "$INFRA_DIR" "$TOFU" apply; } +tofu_output() { require_tofu && run_in "$INFRA_DIR" "$TOFU" output; } +tofu_destroy() { + require_tofu || return 1 + confirm "${RED}Destroy ALL infra managed by OpenTofu?${RESET}" && run_in "$INFRA_DIR" "$TOFU" destroy +} + +# --- Ansible actions ------------------------------------------------------ +ansible_collections() { run_in "$ANSIBLE_DIR" ansible-galaxy collection install -r requirements.yml; } +ansible_inventory() { run_in "$ANSIBLE_DIR" ansible-inventory --graph; } + +# $1 = playbook, $2 = -l limit expression +play() { run_in "$ANSIBLE_DIR" ansible-playbook "playbooks/$1" -l "$2"; } + +ansible_bootstrap_vpn() { play bootstrap.yml role_vpn; } +ansible_deploy_vpn() { play deploy.yml role_vpn && play spinup.yml role_vpn; } +ansible_bootstrap_dp() { play bootstrap.yml role_dev,role_prod; } +ansible_deploy_dp() { play deploy.yml role_dev,role_prod && play spinup.yml role_dev,role_prod; } + +ansible_spindown() { + local target + read -rp "Spindown which hosts? (e.g. role_vpn, role_dev,role_prod, all): " target + [ -n "$target" ] && play spindown.yml "$target" +} + +# --- guided walkthrough --------------------------------------------------- +guided() { + title "Guided full setup" + cat < plan -> apply (creates servers, network, firewalls, DNS) + 2. Ansible: bootstrap + deploy + spinup for role_vpn + 3. YOU: connect to the VPN with an OpenVPN client + 4. Ansible: bootstrap + deploy + spinup for role_dev,role_prod +EOF + confirm "Start the guided walkthrough?" || return 0 + + echo; title "1/4 OpenTofu" + if confirm "Run $TOFU init/plan/apply now?"; then + tofu_init && tofu_plan && tofu_apply || { bad "OpenTofu step failed - stopping."; return 1; } + fi + + echo; title "2/4 Ansible - vpn" + if confirm "Bootstrap + deploy + spinup role_vpn now?"; then + ansible_bootstrap_vpn && ansible_deploy_vpn || { bad "vpn step failed - stopping."; return 1; } + fi + + echo; title "3/4 Connect the VPN" + warn "Connect to the vpn server with your OpenVPN client now, then continue." + confirm "Are you connected to the VPN?" || { warn "Stopping - reconnect and re-run this step."; return 0; } + + echo; title "4/4 Ansible - dev + prod" + if confirm "Bootstrap + deploy + spinup role_dev,role_prod now?"; then + ansible_bootstrap_dp && ansible_deploy_dp || { bad "dev/prod step failed."; return 1; } + fi + ok "Guided setup complete." +} + +# --- menu ----------------------------------------------------------------- +menu() { + clear + printf "%s==== server control panel ====%s\n" "$BOLD$CYAN" "$RESET" + printf "%srepo: %s tofu: %s%s\n" "$DIM" "$REPO_ROOT" "${TOFU:-none}" "$RESET" + + title "Setup" + echo " 1) Check prerequisites" + echo " 2) Guided full setup (vpn -> connect -> dev/prod)" + + title "OpenTofu (infra/)" + echo " 3) init 4) plan 5) apply" + echo " 6) output 7) destroy" + + title "Ansible (ansible/)" + echo " 8) Install collections 9) Show inventory" + echo " 10) Bootstrap vpn 11) Deploy + spinup vpn" + echo " 12) Bootstrap dev+prod 13) Deploy + spinup dev+prod" + echo " 14) Spindown (choose target)" + + echo + echo " 0) Quit" + echo + read -rp "Select an option: " choice +} + +while true; do + menu + case "$choice" in + 1) check_prereqs; pause ;; + 2) guided; pause ;; + 3) tofu_init; pause ;; + 4) tofu_plan; pause ;; + 5) tofu_apply; pause ;; + 6) tofu_output; pause ;; + 7) tofu_destroy; pause ;; + 8) ansible_collections; pause ;; + 9) ansible_inventory; pause ;; + 10) ansible_bootstrap_vpn; pause ;; + 11) ansible_deploy_vpn; pause ;; + 12) ansible_bootstrap_dp; pause ;; + 13) ansible_deploy_dp; pause ;; + 14) ansible_spindown; pause ;; + 0) echo "Bye."; exit 0 ;; + "") ;; + *) warn "Unknown option: $choice"; pause ;; + esac +done diff --git a/readme.md b/readme.md index ed8b055..483b19e 100644 --- a/readme.md +++ b/readme.md @@ -11,6 +11,7 @@ Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted - [Architecture](#architecture) - [Repository layout](#repository-layout) - [Prerequisites](#prerequisites) +- [Control panel (`control.sh`)](#control-panel-controlsh) - [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) - [Bootstrapping and deploying with Ansible](#bootstrapping-and-deploying-with-ansible-ansible) - [Service inventory](#service-inventory) @@ -84,7 +85,7 @@ Each host's stateful data lives in local named Docker volumes, backed up nightly │ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones ├── ansible/ # Ansible: bootstraps each server and deploys/starts services// onto it │ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by the role label above -│ ├── group_vars/ # deploy_user, dev_runner_count, backup_s3_*, etc. +│ ├── group_vars/ # deploy_user, service_group, backup_s3_*, etc. │ ├── roles/ │ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades │ │ └── deploy/ # copies services//, renders .env (S3 backup credentials) + Runners/docker-compose.yml @@ -115,6 +116,16 @@ OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud res Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below). +## Control panel (`control.sh`) + +If you'd rather not remember the exact command order, [`control.sh`](control.sh) at the repo root is an interactive menu that steps you through the whole pre-setup and bring-up. Run it from anywhere: + +```sh +./control.sh +``` + +It offers a **Check prerequisites** option (tools, `HCLOUD_TOKEN`/`BACKUP_S3_*` env vars, `terraform.tfvars`, SSH key, the `hetzner.hcloud` collection), individual OpenTofu (`init`/`plan`/`apply`/`output`/`destroy`) and Ansible (bootstrap/deploy/spinup/spindown per role) actions, and a **Guided full setup** that runs everything in the firewall-imposed order — vpn first, pause for you to connect the VPN, then dev/prod. It's only a wrapper around the same `tofu` and `ansible-playbook` commands documented below, so you can always fall back to running them by hand. + ## Provisioning the infrastructure (`infra/`) ```sh @@ -189,11 +200,11 @@ docker compose -f backup-docker-compose.yml exec backup backup # force an imme docker compose -f backup-docker-compose.yml run --rm restore # restore any currently-empty volumes ``` -### Scaling Gitea Actions runners +### Gitea Actions runners -The number of Gitea Actions runner containers on `dev` is set by `dev_runner_count` in [`ansible/group_vars/role_dev.yml`](ansible/group_vars/role_dev.yml) (default `3`). Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — one `runner-N` service per count, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host: change `dev_runner_count` and redeploy rather than hand-editing it. +`dev` always runs three Gitea Actions runner containers, configured automatically on deploy. Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — three `runner-N` services, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host, so edit the template rather than hand-editing the rendered file (the count lives in a `{% raw %}{% set runner_count = 3 %}{% endraw %}` line at the top). -Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a count change. +Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a template change. Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore. From aa3ea39ddf156e6ed354825573d8090362512912 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Tue, 21 Jul 2026 15:30:51 +0100 Subject: [PATCH 19/24] chore: refined control script --- .devcontainer | 2 +- .gitignore | 4 + ansible/README.md | 8 +- ansible/group_vars/all.yml | 9 ++- control.sh | 146 ++++++++++++++++++++++++++++++++----- readme.md | 7 +- 6 files changed, 150 insertions(+), 26 deletions(-) diff --git a/.devcontainer b/.devcontainer index fc4ccde..7a54748 160000 --- a/.devcontainer +++ b/.devcontainer @@ -1 +1 @@ -Subproject commit fc4ccde0acf64b80e074bcc07a69cc653e60b4d9 +Subproject commit 7a54748c0e189afb715d4409b24e7e6122e44dab diff --git a/.gitignore b/.gitignore index 6936989..c2bd0a0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,10 @@ # services/docker-compose.yml carve-out above. **/.env +# Variables set through control.sh's "Set variables" menu (HCLOUD_TOKEN, +# BACKUP_S3_*, etc.) - never committed. +.control.env + ### Ansible ### *.retry diff --git a/ansible/README.md b/ansible/README.md index 289a2be..f0ff35a 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -37,7 +37,13 @@ export BACKUP_S3_SECRET_ACCESS_KEY=your-secret-access-key export BACKUP_S3_ENDPOINT=your-s3-endpoint # omit for real AWS S3 ``` -`ansible_ssh_private_key_file` (see `group_vars/all.yml`) defaults to +`../control.sh`'s **Set variables** menu does the same thing interactively and +persists your answers to `../.control.env` (gitignored) so you don't have to +re-export them every session. + +`deploy_user` (see `group_vars/all.yml`) defaults to `deploy` - override with +the `DEPLOY_USER` env var if you want a different non-root user created on each +host. `ansible_ssh_private_key_file` (same file) defaults to `~/.ssh/id_ed25519` - override with `ANSIBLE_SSH_PRIVATE_KEY_FILE` if you use a different key. It must match one of the SSH keys OpenTofu installed on the servers (`var.ssh_key_names`). diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index d3308cf..ee4289f 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -1,8 +1,9 @@ --- -# Keep in sync with infra/variables.tf's var.deploy_user (default "deploy") - -# the two aren't wired together automatically, since Ansible no longer reads -# any Terraform state or output. -deploy_user: deploy +# Non-root sudo user the bootstrap role creates on each host - purely an +# Ansible-side concept, OpenTofu has no equivalent variable. Override with the +# DEPLOY_USER env var (control.sh's "Set variables" menu writes it) if you +# don't want the default. +deploy_user: "{{ lookup('env', 'DEPLOY_USER') | default('deploy', true) }}" ansible_user: "{{ deploy_user }}" ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}" diff --git a/control.sh b/control.sh index 50cb4f6..aaf05d4 100755 --- a/control.sh +++ b/control.sh @@ -7,11 +7,19 @@ # ever runs `tofu`/`ansible-playbook` for you - it's a menu wrapper, nothing # more, so anything it does you can also do by hand from infra/ or ansible/. +# Re-exec under bash if invoked as `sh control.sh` (or another POSIX shell) - +# this script relies on bash-only features (indirect expansion, arrays, [[ ]]) +# that fail with cryptic errors like "bad substitution" under dash/sh. +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" +fi + set -uo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" INFRA_DIR="$REPO_ROOT/infra" ANSIBLE_DIR="$REPO_ROOT/ansible" +CONTROL_ENV="$REPO_ROOT/.control.env" # --- pretty output -------------------------------------------------------- if [ -t 1 ]; then @@ -35,6 +43,32 @@ detect_tofu() { } detect_tofu +# --- variable persistence -------------------------------------------------- +# Variables set via the "Set variables" menu (HCLOUD_TOKEN, BACKUP_S3_*, +# DEPLOY_USER, ANSIBLE_SSH_PRIVATE_KEY_FILE) live in .control.env, gitignored, +# so they survive between runs of this script. They are NOT exported into your +# normal shell - only into control.sh's own process and whatever it runs. +load_control_env() { + if [ -f "$CONTROL_ENV" ]; then + set -a + # shellcheck disable=SC1090 + source "$CONTROL_ENV" + set +a + fi +} +load_control_env + +# Upsert KEY=VALUE into .control.env and export it into this process. +save_var() { + local key="$1" val="$2" + touch "$CONTROL_ENV"; chmod 600 "$CONTROL_ENV" + local tmp; tmp="$(mktemp)" + grep -v "^${key}=" "$CONTROL_ENV" > "$tmp" 2>/dev/null || true + mv "$tmp" "$CONTROL_ENV" + printf '%s=%q\n' "$key" "$val" >> "$CONTROL_ENV" + export "${key?}=${val}" +} + # Run a command inside a directory, echoing it first. Returns the command's exit code. run_in() { local dir="$1"; shift @@ -64,19 +98,78 @@ check_prereqs() { [ -n "${BACKUP_S3_ACCESS_KEY_ID:-}" ] && ok "BACKUP_S3_ACCESS_KEY_ID set" || warn "BACKUP_S3_ACCESS_KEY_ID not set" [ -n "${BACKUP_S3_SECRET_ACCESS_KEY:-}" ] && ok "BACKUP_S3_SECRET_ACCESS_KEY set" || warn "BACKUP_S3_SECRET_ACCESS_KEY not set" [ -n "${BACKUP_S3_ENDPOINT:-}" ] && ok "BACKUP_S3_ENDPOINT set" || warn "BACKUP_S3_ENDPOINT not set (omit only for real AWS S3)" + ok "DEPLOY_USER: ${DEPLOY_USER:-deploy} (default 'deploy' if unset)" local key="${ANSIBLE_SSH_PRIVATE_KEY_FILE:-$HOME/.ssh/id_ed25519}" [ -f "${key/#\~/$HOME}" ] && ok "SSH key present ($key)" || warn "SSH key not found at $key (set ANSIBLE_SSH_PRIVATE_KEY_FILE)" + [ -f "$CONTROL_ENV" ] && ok ".control.env present (loaded above)" || warn "No .control.env yet - use Configuration > Set variables to persist the above across runs" + title "Files" [ -f "$INFRA_DIR/terraform.tfvars" ] && ok "infra/terraform.tfvars exists" \ - || bad "infra/terraform.tfvars missing (copy terraform.tfvars.example and fill in ssh_key_names)" + || bad "infra/terraform.tfvars missing (Configuration > Copy terraform.tfvars.example)" [ -d "$INFRA_DIR/.terraform" ] && ok "infra/.terraform exists (tofu init has run)" \ || warn "infra/.terraform missing - run OpenTofu > init" if ansible-galaxy collection list hetzner.hcloud >/dev/null 2>&1; then ok "hetzner.hcloud collection installed" else warn "hetzner.hcloud collection not installed - run Ansible > Install collections"; fi } +# --- configuration actions ------------------------------------------------- +# Prompts for one variable, keeping the current value if the user hits Enter. +# $3 = true masks input and the shown current value, for tokens/secrets. +prompt_var() { + local key="$1" desc="$2" secret="${3:-false}" current input shown + current="${!key:-}" + shown="$current" + [ "$secret" = "true" ] && [ -n "$current" ] && shown="(set, hidden)" + if [ "$secret" = "true" ]; then read -rsp "$desc [$shown]: " input; echo + else read -rp "$desc [$shown]: " input; fi + [ -n "$input" ] && save_var "$key" "$input" +} + +set_variables() { + title "Set variables" + echo "Leave a prompt blank to keep its current value. Saved to .control.env" \ + "(gitignored) and reloaded automatically next time you run control.sh -" \ + "not exported to your normal shell outside of it." + echo + prompt_var HCLOUD_TOKEN "Hetzner Cloud API token" true + prompt_var DEPLOY_USER "Non-root deploy user Ansible creates (default: deploy)" false + prompt_var ANSIBLE_SSH_PRIVATE_KEY_FILE "SSH private key path (default: ~/.ssh/id_ed25519)" false + prompt_var BACKUP_S3_BUCKET "S3 backup bucket name" false + prompt_var BACKUP_S3_ACCESS_KEY_ID "S3 backup access key ID" true + prompt_var BACKUP_S3_SECRET_ACCESS_KEY "S3 backup secret access key" true + prompt_var BACKUP_S3_ENDPOINT "S3 backup endpoint (blank for real AWS S3)" false + ok "Variables saved." +} + +copy_tfvars() { + local example="$INFRA_DIR/terraform.tfvars.example" + local target="$INFRA_DIR/terraform.tfvars" + if [ -f "$target" ]; then + warn "infra/terraform.tfvars already exists." + confirm "Overwrite it with a fresh copy of the example?" || return 0 + fi + cp "$example" "$target" + ok "Copied terraform.tfvars.example -> terraform.tfvars" + + local keys quoted k + read -rp "SSH key name(s) from Hetzner Console > Security > SSH Keys, comma-separated (blank = edit manually later): " keys + if [ -n "$keys" ]; then + quoted="" + IFS=',' read -ra _key_arr <<< "$keys" + for k in "${_key_arr[@]}"; do + k="$(echo "$k" | xargs)" + quoted+="\"$k\", " + done + quoted="${quoted%, }" + sed -i "s|^ssh_key_names = .*|ssh_key_names = [${quoted}]|" "$target" + ok "Set ssh_key_names = [${quoted}]" + else + warn "ssh_key_names still has the example's placeholder values - edit infra/terraform.tfvars before running tofu." + fi +} + # --- OpenTofu actions ----------------------------------------------------- require_tofu() { if [ -z "$TOFU" ]; then bad "No tofu/terraform binary found."; return 1; fi @@ -120,6 +213,7 @@ accept SSH from the vpn server's public IP, so vpn must exist and you must be connected to it (OpenVPN) before dev/prod are reachable. Steps, in order: + 0. Configuration: variables + terraform.tfvars (skip if already done) 1. OpenTofu: init -> plan -> apply (creates servers, network, firewalls, DNS) 2. Ansible: bootstrap + deploy + spinup for role_vpn 3. YOU: connect to the VPN with an OpenVPN client @@ -127,6 +221,14 @@ Steps, in order: EOF confirm "Start the guided walkthrough?" || return 0 + echo; title "0/4 Configuration" + if confirm "Set/update variables now (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)?"; then + set_variables + fi + if [ ! -f "$INFRA_DIR/terraform.tfvars" ] && confirm "infra/terraform.tfvars is missing - create it from the example now?"; then + copy_tfvars + fi + echo; title "1/4 OpenTofu" if confirm "Run $TOFU init/plan/apply now?"; then tofu_init && tofu_plan && tofu_apply || { bad "OpenTofu step failed - stopping."; return 1; } @@ -158,15 +260,19 @@ menu() { echo " 1) Check prerequisites" echo " 2) Guided full setup (vpn -> connect -> dev/prod)" + title "Configuration" + echo " 3) Set variables (HCLOUD_TOKEN, DEPLOY_USER, BACKUP_S3_*, SSH key path)" + echo " 4) Copy terraform.tfvars.example -> terraform.tfvars" + title "OpenTofu (infra/)" - echo " 3) init 4) plan 5) apply" - echo " 6) output 7) destroy" + echo " 5) init 6) plan 7) apply" + echo " 8) output 9) destroy" title "Ansible (ansible/)" - echo " 8) Install collections 9) Show inventory" - echo " 10) Bootstrap vpn 11) Deploy + spinup vpn" - echo " 12) Bootstrap dev+prod 13) Deploy + spinup dev+prod" - echo " 14) Spindown (choose target)" + echo " 10) Install collections 11) Show inventory" + echo " 12) Bootstrap vpn 13) Deploy + spinup vpn" + echo " 14) Bootstrap dev+prod 15) Deploy + spinup dev+prod" + echo " 16) Spindown (choose target)" echo echo " 0) Quit" @@ -179,18 +285,20 @@ while true; do case "$choice" in 1) check_prereqs; pause ;; 2) guided; pause ;; - 3) tofu_init; pause ;; - 4) tofu_plan; pause ;; - 5) tofu_apply; pause ;; - 6) tofu_output; pause ;; - 7) tofu_destroy; pause ;; - 8) ansible_collections; pause ;; - 9) ansible_inventory; pause ;; - 10) ansible_bootstrap_vpn; pause ;; - 11) ansible_deploy_vpn; pause ;; - 12) ansible_bootstrap_dp; pause ;; - 13) ansible_deploy_dp; pause ;; - 14) ansible_spindown; pause ;; + 3) set_variables; pause ;; + 4) copy_tfvars; pause ;; + 5) tofu_init; pause ;; + 6) tofu_plan; pause ;; + 7) tofu_apply; pause ;; + 8) tofu_output; pause ;; + 9) tofu_destroy; pause ;; + 10) ansible_collections; pause ;; + 11) ansible_inventory; pause ;; + 12) ansible_bootstrap_vpn; pause ;; + 13) ansible_deploy_vpn; pause ;; + 14) ansible_bootstrap_dp; pause ;; + 15) ansible_deploy_dp; pause ;; + 16) ansible_spindown; pause ;; 0) echo "Bye."; exit 0 ;; "") ;; *) warn "Unknown option: $choice"; pause ;; diff --git a/readme.md b/readme.md index 483b19e..d2bb5a7 100644 --- a/readme.md +++ b/readme.md @@ -124,7 +124,12 @@ If you'd rather not remember the exact command order, [`control.sh`](control.sh) ./control.sh ``` -It offers a **Check prerequisites** option (tools, `HCLOUD_TOKEN`/`BACKUP_S3_*` env vars, `terraform.tfvars`, SSH key, the `hetzner.hcloud` collection), individual OpenTofu (`init`/`plan`/`apply`/`output`/`destroy`) and Ansible (bootstrap/deploy/spinup/spindown per role) actions, and a **Guided full setup** that runs everything in the firewall-imposed order — vpn first, pause for you to connect the VPN, then dev/prod. It's only a wrapper around the same `tofu` and `ansible-playbook` commands documented below, so you can always fall back to running them by hand. +It offers a **Check prerequisites** option (tools, `HCLOUD_TOKEN`/`DEPLOY_USER`/`BACKUP_S3_*` env vars, `terraform.tfvars`, SSH key, the `hetzner.hcloud` collection), a **Configuration** section, individual OpenTofu (`init`/`plan`/`apply`/`output`/`destroy`) and Ansible (bootstrap/deploy/spinup/spindown per role) actions, and a **Guided full setup** that runs everything in the firewall-imposed order — vpn first, pause for you to connect the VPN, then dev/prod. It's only a wrapper around the same `tofu` and `ansible-playbook` commands documented below, so you can always fall back to running them by hand. + +The **Configuration** section covers every variable both tools need: + +- **Set variables** prompts for `HCLOUD_TOKEN`, `DEPLOY_USER` (the non-root user Ansible creates — defaults to `deploy`), `ANSIBLE_SSH_PRIVATE_KEY_FILE`, and the `BACKUP_S3_*` credentials, one at a time (leave blank to keep the current value, secrets are input-masked). They're saved to `.control.env` at the repo root — gitignored, never committed — which is loaded automatically the next time you run `control.sh`, so you only enter them once. This is scoped to `control.sh` itself, not your regular shell. +- **Copy terraform.tfvars.example -> terraform.tfvars** creates `infra/terraform.tfvars` for you and prompts for `ssh_key_names` (the one required field with no default) so you don't leave the example's placeholder key names in place by accident. Everything else in that file (`location`, server types, `dns_zones`, etc.) still has sensible defaults you can edit by hand afterwards - see [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) below. ## Provisioning the infrastructure (`infra/`) From 7ebd02f6b8aea1ab754a30a6db7630ebe008ed2a Mon Sep 17 00:00:00 2001 From: Luke Else Date: Tue, 21 Jul 2026 16:53:15 +0100 Subject: [PATCH 20/24] chore: refined control script --- ansible/README.md | 13 ++-- ansible/{ => playbooks}/group_vars/all.yml | 0 .../{ => playbooks}/group_vars/role_dev.yml | 0 .../{ => playbooks}/group_vars/role_prod.yml | 0 .../{ => playbooks}/group_vars/role_vpn.yml | 0 infra/main.tf | 13 +++- infra/modules/dns/main.tf | 26 ++++++-- infra/modules/dns/moved.tf | 60 +++++++++++++++++++ infra/modules/dns/variables.tf | 11 +++- infra/variables.tf | 8 +-- readme.md | 21 +++++-- services/dev/backup-docker-compose.yml | 2 +- services/prod/backup-docker-compose.yml | 2 +- services/vpn/backup-docker-compose.yml | 2 +- services/vpn/vpn-docker-compose.yml | 16 +++-- 15 files changed, 144 insertions(+), 30 deletions(-) rename ansible/{ => playbooks}/group_vars/all.yml (100%) rename ansible/{ => playbooks}/group_vars/role_dev.yml (100%) rename ansible/{ => playbooks}/group_vars/role_prod.yml (100%) rename ansible/{ => playbooks}/group_vars/role_vpn.yml (100%) create mode 100644 infra/modules/dns/moved.tf diff --git a/ansible/README.md b/ansible/README.md index f0ff35a..1c32d8e 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -14,16 +14,19 @@ to S3, not on a Hetzner volume - see `../readme.md`. - `inventory/hcloud.yml` - dynamic inventory, queries the Hetzner Cloud API directly (not Terraform state) and groups servers by their `role` label (`role_dev`, `role_prod`, `role_vpn` - set in `infra/modules//main.tf`). -- `group_vars/` - `all.yml` (shared: `deploy_user`, SSH key path, - `backup_s3_*` S3 backup credentials), `role_dev.yml` / `role_prod.yml` / - `role_vpn.yml` (per-role vars, e.g. `service_group`). - `roles/bootstrap/` - Docker install, `deploy` user creation, SSH hardening, unattended-upgrades. Replaces `infra/scripts/bootstrap.sh.tftpl`. - `roles/deploy/` - copies `services//` to the server and renders `.env` (S3 backup credentials, see `../readme.md`) and, for `dev`, `Runners/docker-compose.yml`. - `playbooks/` - `bootstrap.yml`, `deploy.yml`, `spinup.yml`, `spindown.yml`, - and `site.yml` (all three in order). + `site.yml` (all three in order), and `group_vars/` (`all.yml` - shared: + `deploy_user`, SSH key path, `backup_s3_*` S3 backup credentials; + `role_dev.yml` / `role_prod.yml` / `role_vpn.yml` - per-role vars, e.g. + `service_group`). `group_vars/` lives here rather than at the `ansible/` + root because Ansible only auto-loads `group_vars/`/`host_vars/` from + directories adjacent to the playbook file or the inventory source - not + from an arbitrary sibling directory. ## Prerequisites @@ -41,7 +44,7 @@ export BACKUP_S3_ENDPOINT=your-s3-endpoint # omit for real AWS S3 persists your answers to `../.control.env` (gitignored) so you don't have to re-export them every session. -`deploy_user` (see `group_vars/all.yml`) defaults to `deploy` - override with +`deploy_user` (see `playbooks/group_vars/all.yml`) defaults to `deploy` - override with the `DEPLOY_USER` env var if you want a different non-root user created on each host. `ansible_ssh_private_key_file` (same file) defaults to `~/.ssh/id_ed25519` - override with `ANSIBLE_SSH_PRIVATE_KEY_FILE` if you use a diff --git a/ansible/group_vars/all.yml b/ansible/playbooks/group_vars/all.yml similarity index 100% rename from ansible/group_vars/all.yml rename to ansible/playbooks/group_vars/all.yml diff --git a/ansible/group_vars/role_dev.yml b/ansible/playbooks/group_vars/role_dev.yml similarity index 100% rename from ansible/group_vars/role_dev.yml rename to ansible/playbooks/group_vars/role_dev.yml diff --git a/ansible/group_vars/role_prod.yml b/ansible/playbooks/group_vars/role_prod.yml similarity index 100% rename from ansible/group_vars/role_prod.yml rename to ansible/playbooks/group_vars/role_prod.yml diff --git a/ansible/group_vars/role_vpn.yml b/ansible/playbooks/group_vars/role_vpn.yml similarity index 100% rename from ansible/group_vars/role_vpn.yml rename to ansible/playbooks/group_vars/role_vpn.yml diff --git a/infra/main.tf b/infra/main.tf index fd6d20b..161587a 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -31,6 +31,17 @@ locals { { zone = "luke-else.co.uk", name = "vpn", value = module.vpn.ipv4 }, { zone = "luke-else.co.uk", name = "traefik.vpn", value = module.vpn.ipv4 }, ] + + # Microsoft 365 mail records for luke-else.co.uk - carried over from the + # domain's old DNS provider (pre-Hetzner), not previously applied here, so + # these are genuinely new to the Hetzner zone rather than something to + # import. + mail_records = [ + { zone = "luke-else.co.uk", name = "@", type = "MX", value = "0 lukeelse-co-uk01b.mail.protection.outlook.com." }, + { zone = "luke-else.co.uk", name = "@", type = "TXT", value = "\"v=spf1 include:spf.protection.outlook.com -all\"" }, + { zone = "luke-else.co.uk", name = "@", type = "TXT", value = "\"google-site-verification=aKLk5BATowKUwNbFcU5zyfu-AIyCW9W9PBStePRjH3U\"" }, + { zone = "luke-else.co.uk", name = "autodiscover", type = "CNAME", value = "autodiscover.outlook.com." }, + ] } module "network" { @@ -87,5 +98,5 @@ module "dns" { source = "./modules/dns" zone_names = var.dns_zones - records = local.dns_records + records = concat(local.dns_records, local.mail_records) } diff --git a/infra/modules/dns/main.tf b/infra/modules/dns/main.tf index 08952ee..7e8d92e 100644 --- a/infra/modules/dns/main.tf +++ b/infra/modules/dns/main.tf @@ -14,15 +14,29 @@ resource "hcloud_zone" "this" { } } -resource "hcloud_zone_rrset" "a" { - for_each = { for r in var.records : "${r.zone}/${r.name}" => r } +locals { + # Group flat var.records entries by zone/name/type into one set of values + # each, since a Hetzner RRSet is keyed on (zone, name, type) and can hold + # several values (e.g. an SPF and a google-site-verification TXT both live + # under the same "@" TXT RRSet). + rrsets = { + for key, items in { for r in var.records : "${r.zone}/${r.name}/${r.type}" => r... } : key => { + zone = items[0].zone + name = items[0].name + type = items[0].type + ttl = items[0].ttl + values = [for i in items : i.value] + } + } +} + +resource "hcloud_zone_rrset" "this" { + for_each = local.rrsets zone = hcloud_zone.this[each.value.zone].id name = each.value.name - type = "A" + type = each.value.type ttl = each.value.ttl - records = [ - { value = each.value.value } - ] + records = [for v in each.value.values : { value = v }] } diff --git a/infra/modules/dns/moved.tf b/infra/modules/dns/moved.tf new file mode 100644 index 0000000..8bfb840 --- /dev/null +++ b/infra/modules/dns/moved.tf @@ -0,0 +1,60 @@ +# Generalizing hcloud_zone_rrset.a (for_each keyed "zone/name", A records +# only) into hcloud_zone_rrset.this (keyed "zone/name/type", any record type) +# changed every existing instance's resource address. Without these, `tofu +# apply` would destroy and recreate every A record already in state instead of +# just adding the new mail records - these moved blocks make it a no-op rename +# for anything that already existed. Safe to delete once applied and merged. +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/@"] + to = hcloud_zone_rrset.this["luke-else.co.uk/@/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/dev"] + to = hcloud_zone_rrset.this["luke-else.co.uk/dev/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/metarius"] + to = hcloud_zone_rrset.this["luke-else.co.uk/metarius/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/traefik"] + to = hcloud_zone_rrset.this["luke-else.co.uk/traefik/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/status"] + to = hcloud_zone_rrset.this["luke-else.co.uk/status/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/bitwarden"] + to = hcloud_zone_rrset.this["luke-else.co.uk/bitwarden/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/rd"] + to = hcloud_zone_rrset.this["luke-else.co.uk/rd/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/git"] + to = hcloud_zone_rrset.this["luke-else.co.uk/git/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/traefik.cicd"] + to = hcloud_zone_rrset.this["luke-else.co.uk/traefik.cicd/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/vpn"] + to = hcloud_zone_rrset.this["luke-else.co.uk/vpn/A"] +} + +moved { + from = hcloud_zone_rrset.a["luke-else.co.uk/traefik.vpn"] + to = hcloud_zone_rrset.this["luke-else.co.uk/traefik.vpn/A"] +} diff --git a/infra/modules/dns/variables.tf b/infra/modules/dns/variables.tf index 7f8265e..7de9e57 100644 --- a/infra/modules/dns/variables.tf +++ b/infra/modules/dns/variables.tf @@ -4,11 +4,20 @@ variable "zone_names" { } variable "records" { - description = "A records to create across the zones in var.zone_names. Use name = \"@\" for the zone apex." + description = <<-EOT + Records to create across the zones in var.zone_names. Use name = "@" for + the zone apex. Entries sharing the same zone/name/type become a single + Hetzner RRSet with multiple values (e.g. two TXT strings under the same + name) - Hetzner has no concept of separate same-type records otherwise. + For MX, embed the priority in value using standard zonefile syntax (e.g. + "10 mail.example.com."); Hetzner's API has no separate priority field. + For TXT, wrap value in escaped double quotes (e.g. "\"v=spf1 ...\""). + EOT type = list(object({ zone = string name = string value = string + type = optional(string, "A") ttl = optional(number, 300) })) default = [] diff --git a/infra/variables.tf b/infra/variables.tf index e974430..c8c811f 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -13,25 +13,25 @@ variable "network_zone" { variable "server_image" { description = "OS image used for all servers." type = string - default = "ubuntu-24.04" + default = "ubuntu-26.04" } variable "dev_server_type" { description = "Server type for dev" type = string - default = "cx22" + default = "cx23" } variable "prod_server_type" { description = "Server type for prod" type = string - default = "cx22" + default = "cx23" } variable "vpn_server_type" { description = "Server type for vpn" type = string - default = "cx22" + default = "cx23" } variable "network_ip_range" { diff --git a/readme.md b/readme.md index d2bb5a7..d16673e 100644 --- a/readme.md +++ b/readme.md @@ -85,11 +85,12 @@ Each host's stateful data lives in local named Docker volumes, backed up nightly │ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones ├── ansible/ # Ansible: bootstraps each server and deploys/starts services// onto it │ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by the role label above -│ ├── group_vars/ # deploy_user, service_group, backup_s3_*, etc. │ ├── roles/ │ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades │ │ └── deploy/ # copies services//, renders .env (S3 backup credentials) + Runners/docker-compose.yml -│ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml +│ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml, +│ └── group_vars/ # deploy_user, service_group, backup_s3_*, etc. - must sit next to +│ # the playbooks for Ansible to auto-load it (see ansible/README.md) ├── services/ # Docker Compose stacks, grouped by which server they run on │ ├── dev/ # Gitea + CI runner + Traefik + backup/restore (git.luke-else.co.uk, cicd.luke-else.co.uk) │ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore @@ -159,13 +160,23 @@ OpenTofu never connects to the servers over SSH — no provisioners, no `bootstr ### Managing DNS -`module.dns` (in `infra/modules/dns`) creates one Hetzner DNS zone per domain in `var.dns_zones` (default: `luke-else.co.uk`, `divine-couture.co.uk`, `snexo.co.uk`) and an `hcloud_zone_rrset` A record for every hostname currently referenced by a Traefik `Host()` rule anywhere under `services/` — kept in sync with `local.dns_records` in `infra/main.tf`, pointed at whichever of `dev`/`prod`/`vpn` actually serves it. Zones have `prevent_destroy = true`, matching the volumes — losing one deletes every record in it. +`module.dns` (in `infra/modules/dns`) creates one Hetzner DNS zone per domain in `var.dns_zones` (default: `luke-else.co.uk`, `divine-couture.co.uk`, `snexo.co.uk`) and an `hcloud_zone_rrset` per record in `var.records` — kept in sync with `local.dns_records` (A records, one per hostname currently referenced by a Traefik `Host()` rule anywhere under `services/`, pointed at whichever of `dev`/`prod`/`vpn` actually serves it) and `local.mail_records` (see below) in `infra/main.tf`. Zones have `prevent_destroy = true`, matching the volumes — losing one deletes every record in it. Creating the zone doesn't make Hetzner authoritative for the domain by itself: you still need to point that domain's NS records at Hetzner's nameservers at whichever registrar it's registered through. Run `tofu output dns_nameservers` after applying to get the exact nameservers per domain, and set those as the domain's NS records at the registrar. Propagation can take a while depending on the registrar and the domain's previous NS TTL. Adding a new subdomain: add an entry to `local.dns_records` in `infra/main.tf` (zone, name, and the target `module..ipv4`) and re-run `tofu apply` — don't hand-create records in the Hetzner console, they'll drift from state. Note `name = "@"` is Hetzner's convention for a zone's apex record (e.g. bare `snexo.co.uk`), not an empty string. -Only A records for IPv4 are managed here — none of the modules currently track servers' IPv6 addresses, so AAAA records aren't generated even though the firewalls already allow IPv6 traffic. +Only A records for IPv4 are managed for hosts — none of the modules currently track servers' IPv6 addresses, so AAAA records aren't generated even though the firewalls already allow IPv6 traffic. + +`var.records` isn't limited to A records: each entry takes an optional `type` (`A` by default, or `MX`/`TXT`/`CNAME`/etc.), and entries sharing the same `zone`/`name`/`type` are merged into a single Hetzner RRSet with multiple values (needed for e.g. two TXT strings under the same name) - see `infra/modules/dns/variables.tf` for the exact value format each type expects (MX embeds the priority in the value string, TXT values need escaped double quotes). + +#### Mail records (Microsoft 365) + +`local.mail_records` in `infra/main.tf` tracks the DNS records Microsoft 365 needs for `luke-else.co.uk` - the MX record, the SPF and Google-site-verification TXT records (merged into one `@` TXT RRSet), and the `autodiscover` CNAME. These carried over from the domain's DNS provider prior to Hetzner and were never applied to the Hetzner zone, so they're genuinely new records here (not something to `tofu import` - nothing to collide with). + +**Not yet included:** a DMARC policy (`_dmarc` TXT) and DKIM signing (two `selector1._domainkey`/`selector2._domainkey` CNAMEs). DKIM's targets are generated per-tenant by Microsoft 365 (Defender portal → Email & collaboration → Policies & rules → DKIM) and can't be guessed - enable DKIM there first, then add the two CNAMEs it gives you to `local.mail_records`. + +`infra/modules/dns/moved.tf` re-points every pre-existing A record at its new resource address, since generalizing the module to support non-A record types changed how they're keyed internally - without it, `tofu apply` would destroy and recreate every A record for no functional reason. It's a one-time migration aid: run `tofu plan` first to confirm the only real changes are the new mail records being created, then the file can be deleted after a successful apply. ## Bootstrapping and deploying with Ansible (`ansible/`) @@ -310,5 +321,5 @@ Then reopen the repo in VS Code with the Dev Containers extension. - SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). - `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP. - Firewalls are allowlists scoped per host in `infra/modules//main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. -- Every host's [Ansible bootstrap role](ansible/roles/bootstrap/tasks/main.yml) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`deploy_user`, see `ansible/group_vars/all.yml`) for day-to-day access. +- Every host's [Ansible bootstrap role](ansible/roles/bootstrap/tasks/main.yml) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`deploy_user`, see `ansible/playbooks/group_vars/all.yml`) for day-to-day access. - DNS zones (`modules/dns`) carry both Hetzner's own `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, including for `divine-couture.co.uk` and `snexo.co.uk`, not just `luke-else.co.uk`. diff --git a/services/dev/backup-docker-compose.yml b/services/dev/backup-docker-compose.yml index c9a18e1..26b676a 100644 --- a/services/dev/backup-docker-compose.yml +++ b/services/dev/backup-docker-compose.yml @@ -26,7 +26,7 @@ services: # Run explicitly by spinup.sh before the services that own these volumes # start - not started by `docker compose up`. restore: - image: amazon/aws-cli:2 + image: amazon/aws-cli:latest container_name: backup-restore entrypoint: ["/bin/bash", "/restore.sh"] environment: diff --git a/services/prod/backup-docker-compose.yml b/services/prod/backup-docker-compose.yml index 7e996f0..7263c2b 100644 --- a/services/prod/backup-docker-compose.yml +++ b/services/prod/backup-docker-compose.yml @@ -31,7 +31,7 @@ services: # Run explicitly by spinup.sh before the services that own these volumes # start - not started by `docker compose up`. restore: - image: amazon/aws-cli:2 + image: amazon/aws-cli:latest container_name: backup-restore entrypoint: [ "/bin/bash", "/restore.sh" ] environment: diff --git a/services/vpn/backup-docker-compose.yml b/services/vpn/backup-docker-compose.yml index 0c2ae8e..92d40c7 100644 --- a/services/vpn/backup-docker-compose.yml +++ b/services/vpn/backup-docker-compose.yml @@ -26,7 +26,7 @@ services: # Run explicitly by spinup.sh before the services that own these volumes # start - not started by `docker compose up`. restore: - image: amazon/aws-cli:2 + image: amazon/aws-cli:latest container_name: backup-restore entrypoint: ["/bin/bash", "/restore.sh"] environment: diff --git a/services/vpn/vpn-docker-compose.yml b/services/vpn/vpn-docker-compose.yml index f78e1af..9d0739c 100644 --- a/services/vpn/vpn-docker-compose.yml +++ b/services/vpn/vpn-docker-compose.yml @@ -2,13 +2,15 @@ services: dockovpn: image: alekslitvinenk/openvpn cap_add: - - NET_ADMIN + - NET_ADMIN ports: - - 1194:1194/udp # Expose tcp if you defined HOST_TUN_PROTOCOL=tcp + - 1194:1194/udp # Expose tcp if you defined HOST_TUN_PROTOCOL=tcp + networks: + - proxy environment: - HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address + HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address volumes: - - openvpn_data:/opt/Dockovpn_data + - openvpn_data:/opt/Dockovpn_data labels: ## Expose vpn Through Trefik ## - "traefik.enable=true" # <== Enable traefik to proxy this container @@ -26,6 +28,10 @@ services: - "traefik.http.routers.vpn.tls.certresolver=myresolver" restart: always +networks: + proxy: + external: true + volumes: openvpn_data: - name: openvpn_data \ No newline at end of file + name: openvpn_data From a9d602f35e18c878ed9e726079bd66c3006d2e71 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Tue, 21 Jul 2026 17:17:33 +0100 Subject: [PATCH 21/24] feat: Changed openvpn for wireguard --- ansible/README.md | 3 ++- control.sh | 7 ++++--- infra/modules/dns/main.tf | 9 ++------ infra/modules/vpn/main.tf | 6 +++--- infra/modules/vpn/variables.tf | 2 +- readme.md | 26 ++++++++++++++++------- services/vpn/backup-docker-compose.yml | 8 +++---- services/vpn/spindown.sh | 2 +- services/vpn/vpn-docker-compose.yml | 29 ++++++++++++++++---------- 9 files changed, 53 insertions(+), 39 deletions(-) diff --git a/ansible/README.md b/ansible/README.md index 1c32d8e..0555d70 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -64,7 +64,8 @@ ansible-playbook playbooks/bootstrap.yml -l role_vpn ansible-playbook playbooks/deploy.yml -l role_vpn ansible-playbook playbooks/spinup.yml -l role_vpn -# 2. connect to the VPN you just started with an OpenVPN client - now dev/prod +# 2. visit https://vpn.luke-else.co.uk, complete wg-easy's first-run setup, +# add a client peer, and connect with a WireGuard client - now dev/prod # are reachable, since your egress IP matches their firewall rule ansible-playbook playbooks/bootstrap.yml -l role_dev,role_prod ansible-playbook playbooks/deploy.yml -l role_dev,role_prod diff --git a/control.sh b/control.sh index aaf05d4..761fa58 100755 --- a/control.sh +++ b/control.sh @@ -210,13 +210,14 @@ guided() { cat < plan -> apply (creates servers, network, firewalls, DNS) 2. Ansible: bootstrap + deploy + spinup for role_vpn - 3. YOU: connect to the VPN with an OpenVPN client + 3. YOU: visit https://vpn.luke-else.co.uk, complete wg-easy's setup, + add a client peer, and connect with a WireGuard client 4. Ansible: bootstrap + deploy + spinup for role_dev,role_prod EOF confirm "Start the guided walkthrough?" || return 0 @@ -240,7 +241,7 @@ EOF fi echo; title "3/4 Connect the VPN" - warn "Connect to the vpn server with your OpenVPN client now, then continue." + warn "Visit https://vpn.luke-else.co.uk, complete wg-easy's setup wizard, add a client peer, and connect with a WireGuard client now." confirm "Are you connected to the VPN?" || { warn "Stopping - reconnect and re-run this step."; return 0; } echo; title "4/4 Ansible - dev + prod" diff --git a/infra/modules/dns/main.tf b/infra/modules/dns/main.tf index 7e8d92e..3b4805e 100644 --- a/infra/modules/dns/main.tf +++ b/infra/modules/dns/main.tf @@ -5,13 +5,8 @@ resource "hcloud_zone" "this" { for_each = toset(var.zone_names) - name = each.value - mode = "primary" - delete_protection = true # API-level guard, in addition to prevent_destroy below - - lifecycle { - prevent_destroy = true - } + name = each.value + mode = "primary" } locals { diff --git a/infra/modules/vpn/main.tf b/infra/modules/vpn/main.tf index c73396e..36ddc51 100644 --- a/infra/modules/vpn/main.tf +++ b/infra/modules/vpn/main.tf @@ -1,4 +1,4 @@ -# vpn: OpenVPN + Traefik. Not attached to the private network (see docs/architecture.md). +# vpn: WireGuard (wg-easy) + Traefik. Not attached to the private network (see docs/architecture.md). resource "hcloud_firewall" "this" { name = "vpn-firewall" @@ -23,10 +23,10 @@ resource "hcloud_firewall" "this" { source_ips = ["0.0.0.0/0", "::/0"] } - rule { # OpenVPN tunnel - always direct to this server's public IP, never via a load balancer + rule { # WireGuard tunnel - always direct to this server's public IP, never via a load balancer direction = "in" protocol = "udp" - port = "1194" + port = "51820" source_ips = ["0.0.0.0/0", "::/0"] } } diff --git a/infra/modules/vpn/variables.tf b/infra/modules/vpn/variables.tf index 022e909..f2098f0 100644 --- a/infra/modules/vpn/variables.tf +++ b/infra/modules/vpn/variables.tf @@ -1,5 +1,5 @@ variable "server_type" { - description = "Server type for vpn (OpenVPN + Traefik)." + description = "Server type for vpn (WireGuard/wg-easy + Traefik)." type = string } diff --git a/readme.md b/readme.md index d16673e..8167cf6 100644 --- a/readme.md +++ b/readme.md @@ -60,7 +60,7 @@ architecture-beta |---|---|---| | `dev` | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) | | `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | -| `vpn` | OpenVPN + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | +| `vpn` | WireGuard (wg-easy) + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | Each host's stateful data lives in local named Docker volumes, backed up nightly to a shared S3 bucket and restored automatically on first boot - see below. No host has a Hetzner Cloud Volume attached any more. @@ -94,7 +94,7 @@ Each host's stateful data lives in local named Docker volumes, backed up nightly ├── services/ # Docker Compose stacks, grouped by which server they run on │ ├── dev/ # Gitea + CI runner + Traefik + backup/restore (git.luke-else.co.uk, cicd.luke-else.co.uk) │ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore -│ ├── vpn/ # OpenVPN + Traefik + backup/restore +│ ├── vpn/ # WireGuard (wg-easy) + Traefik + backup/restore │ └── todo.md # Outstanding manual setup/hardening tasks ├── docs/ │ └── architecture.md # Source of the architecture diagram above @@ -190,7 +190,7 @@ export HCLOUD_TOKEN=your-hetzner-api-token # never commit this # vpn first - its firewall accepts SSH from var.allowed_ssh_source_ips directly ansible-playbook playbooks/site.yml -l role_vpn -# SSH to vpn as deploy and connect to the OpenVPN it just started, then: +# SSH to vpn as deploy and connect to the WireGuard tunnel it just started, then: ansible-playbook playbooks/site.yml -l role_dev,role_prod ``` @@ -201,7 +201,7 @@ Hosts are discovered dynamically from the Hetzner API (`ansible/inventory/hcloud Every stateful service's compose file mounts a plain named Docker volume rather than a bind mount to a Hetzner volume - `gitea_data:/data`, `bitwarden_data:/data/`, `letsencrypt_data:/letsencrypt`, and so on across `services/dev/*.yml`, `services/prod/*.yml`, and `services/vpn/*.yml`. Each volume is declared with an explicit `name:` in every compose file that mounts it (the same pattern already used for the shared `proxy` network), so Compose creates it once and every stack that references it reuses the same underlying volume regardless of which compose file happens to run first. Each host's `backup-docker-compose.yml` defines two services against those same named volumes: -- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), running on a nightly cron (`0 3 * * *`), tars up every mounted volume and uploads it to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/-.tar.gz`, keeping 14 days of history (`BACKUP_RETENTION_DAYS`). Containers labeled `docker-volume-backup.stop-during-backup=true` (Gitea, Bitwarden, RustDesk, Uptime Kuma, snexo, OpenVPN) are stopped for the duration of the backup so their data is captured in a consistent state, then restarted automatically. Started by `spinup.sh` alongside everything else. +- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), running on a nightly cron (`0 3 * * *`), tars up every mounted volume and uploads it to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/-.tar.gz`, keeping 14 days of history (`BACKUP_RETENTION_DAYS`). Containers labeled `docker-volume-backup.stop-during-backup=true` (Gitea, Bitwarden, RustDesk, Uptime Kuma, snexo, wg-easy) are stopped for the duration of the backup so their data is captured in a consistent state, then restarted automatically. Started by `spinup.sh` alongside everything else. - **`restore`** — a one-shot container (`amazon/aws-cli` running `restore.sh`) that `spinup.sh` runs *before* anything else starts. It checks each of the host's named volumes; any that are already empty are populated from the most recent object under that host's S3 prefix, any that already contain data are left completely untouched. On a fresh server with no prior backup (or a bucket that doesn't exist yet), it's a no-op - services just start with empty volumes as before. `AWS_S3_BUCKET_NAME`, `AWS_S3_PATH` (set to the host name, e.g. `dev`), `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_ENDPOINT` are rendered into `services//.env` by Ansible's `deploy` role from `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`/`BACKUP_S3_ENDPOINT` in your shell environment (see [Prerequisites](#prerequisites)) — `docker compose` auto-loads `.env` from its working directory. `spinup.sh` on every host refuses to start anything if `.env` is missing. @@ -226,10 +226,10 @@ Registration tokens are handled automatically, not baked into the generated file ### First deploy: bootstrapping order matters -`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own OpenVPN service isn't running until you deploy it — so on a from-scratch estate, Ansible can't reach `dev`/`prod` yet. Bring it up in this order: +`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own WireGuard service isn't running until you deploy it — so on a from-scratch estate, Ansible can't reach `dev`/`prod` yet. Bring it up in this order: 1. `ansible-playbook playbooks/site.yml -l role_vpn` — bootstraps, deploys, and starts `vpn` only; its firewall allows SSH from `var.allowed_ssh_source_ips` directly. -2. SSH in as `deploy` and connect to the OpenVPN service you just started with an OpenVPN client. +2. Visit `https://vpn.luke-else.co.uk` and complete wg-easy's first-run setup wizard (admin username/password, then add a client peer and download its config/QR code) - see [Setting up WireGuard (wg-easy)](#setting-up-wireguard-wg-easy). Import that config into a WireGuard client and connect. 3. `ansible-playbook playbooks/site.yml -l role_dev,role_prod` — now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and `dev`/`prod` become reachable. If step 3 is run before you're connected to the VPN, Ansible will simply fail to connect — re-run it once connected. @@ -248,10 +248,20 @@ After `ansible-playbook playbooks/deploy.yml` has copied `services//` to ` ./spindown.sh # on vpn -./spinup.sh # Traefik → OpenVPN → Watchtower +./spinup.sh # Traefik → wg-easy → Watchtower ./spindown.sh ``` +### Setting up WireGuard (wg-easy) + +`vpn` runs [wg-easy](https://github.com/wg-easy/wg-easy) - WireGuard plus a web UI for managing peers - fronted by Traefik like everything else. `spinup.sh` only starts the container; wg-easy itself has no automated first-run setup here (see `services/vpn/vpn-docker-compose.yml`), so once it's up: + +1. Visit `https://vpn.luke-else.co.uk` and complete the setup wizard - pick an admin username and a strong password (this account can view/rotate every client's private key, and it's reachable from the public internet). Enabling 2FA afterwards (wg-easy supports it) is worth doing since this isn't just a status page like the old OpenVPN setup was. +2. Set the host clients should connect to (`vpn.luke-else.co.uk`) and the port (`51820`, matching the firewall rule in `infra/modules/vpn/main.tf`) when prompted. +3. Add a client per device from the UI, then download its config file or scan the QR code directly into the WireGuard app. + +wg-easy's own state (admin account, peers, keys) lives in the `wg_easy_data` named volume, backed up to S3 like everything else - see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3). Client configs themselves are only ever downloaded through the UI, not stored anywhere else. + If you ever need to force a re-sync without going through Ansible (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@:~/services` still works fine - `ansible-playbook playbooks/deploy.yml` will just overwrite it again next time it's run. Each stack can also be managed individually with plain Compose, e.g.: @@ -295,7 +305,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi | Service | Compose file | Domain(s) | |---|---|---| | Traefik | `traefik-docker-compose.yml` | `traefik.vpn.luke-else.co.uk` | -| OpenVPN (Dockovpn) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `1194` | +| WireGuard (wg-easy) | `vpn-docker-compose.yml` | `vpn.luke-else.co.uk`, UDP `51820` | | Watchtower | `watchtower-docker-compose.yml` | — | | Backup/restore | `backup-docker-compose.yml` | — | diff --git a/services/vpn/backup-docker-compose.yml b/services/vpn/backup-docker-compose.yml index 92d40c7..2c72e03 100644 --- a/services/vpn/backup-docker-compose.yml +++ b/services/vpn/backup-docker-compose.yml @@ -17,7 +17,7 @@ services: AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} AWS_ENDPOINT: ${AWS_ENDPOINT} volumes: - - openvpn_data:/backup/openvpn_data:ro + - wg_easy_data:/backup/wg_easy_data:ro - letsencrypt_data:/backup/letsencrypt_data:ro - /var/run/docker.sock:/var/run/docker.sock:ro restart: unless-stopped @@ -37,12 +37,12 @@ services: AWS_ENDPOINT: ${AWS_ENDPOINT} volumes: - ./restore.sh:/restore.sh:ro - - openvpn_data:/restore/openvpn_data + - wg_easy_data:/restore/wg_easy_data - letsencrypt_data:/restore/letsencrypt_data restart: "no" volumes: - openvpn_data: - name: openvpn_data + wg_easy_data: + name: wg_easy_data letsencrypt_data: name: letsencrypt_data diff --git a/services/vpn/spindown.sh b/services/vpn/spindown.sh index c093daa..8c365c4 100755 --- a/services/vpn/spindown.sh +++ b/services/vpn/spindown.sh @@ -9,5 +9,5 @@ docker compose -f backup-docker-compose.yml down docker rmi $(docker images -q) docker system prune -f -a -# Deliberately no `docker volume prune` - openvpn_data/letsencrypt_data are +# Deliberately no `docker volume prune` - wg_easy_data/letsencrypt_data are # named volumes holding real data, backed up to S3 but not disposable. diff --git a/services/vpn/vpn-docker-compose.yml b/services/vpn/vpn-docker-compose.yml index 9d0739c..8273abd 100644 --- a/services/vpn/vpn-docker-compose.yml +++ b/services/vpn/vpn-docker-compose.yml @@ -1,19 +1,25 @@ services: - dockovpn: - image: alekslitvinenk/openvpn + wg-easy: + image: ghcr.io/wg-easy/wg-easy:15 cap_add: - NET_ADMIN + - SYS_MODULE # lets the container modprobe wireguard if the host kernel doesn't already have it loaded + sysctls: + - net.ipv4.ip_forward=1 + - net.ipv4.conf.all.src_valid_mark=1 + - net.ipv6.conf.all.disable_ipv6=0 + - net.ipv6.conf.all.forwarding=1 + - net.ipv6.conf.default.forwarding=1 ports: - - 1194:1194/udp # Expose tcp if you defined HOST_TUN_PROTOCOL=tcp + - 51820:51820/udp # WireGuard tunnel - the web UI (51821) only needs to be reachable via Traefik below, not published directly networks: - proxy - environment: - HOST_ADDR: vpn.luke-else.co.uk # Your VPN server address volumes: - - openvpn_data:/opt/Dockovpn_data + - /lib/modules:/lib/modules:ro + - wg_easy_data:/etc/wireguard labels: - ## Expose vpn Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container + ## Expose the wg-easy web UI through Traefik ## + - "traefik.enable=true" # Stopped for the duration of each backup run - see backup-docker-compose.yml. - "docker-volume-backup.stop-during-backup=true" @@ -26,12 +32,13 @@ services: - "traefik.http.routers.vpn.rule=Host(`vpn.luke-else.co.uk`)" - "traefik.http.routers.vpn.entrypoints=websecure" - "traefik.http.routers.vpn.tls.certresolver=myresolver" - restart: always + - "traefik.http.services.vpn.loadbalancer.server.port=51821" + restart: unless-stopped networks: proxy: external: true volumes: - openvpn_data: - name: openvpn_data + wg_easy_data: + name: wg_easy_data From 52aa4278bf3b85b774bba24ad2d6085d5e30df78 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Tue, 21 Jul 2026 20:11:23 +0100 Subject: [PATCH 22/24] chore: Refresh of repo --- ansible/inventory/hcloud.yml | 3 +- ansible/playbooks/group_vars/all.yml | 10 +- ansible/roles/bootstrap/tasks/main.yml | 8 +- ansible/roles/deploy/tasks/main.yml | 6 +- infra/main.tf | 5 +- infra/modules/dev/main.tf | 12 +- infra/modules/network/main.tf | 4 +- infra/modules/prod/main.tf | 13 +- infra/modules/vpn/main.tf | 6 +- readme.md | 215 ++++++++----------------- services/todo.md | 32 ---- 11 files changed, 92 insertions(+), 222 deletions(-) delete mode 100644 services/todo.md diff --git a/ansible/inventory/hcloud.yml b/ansible/inventory/hcloud.yml index aad0be5..d414f72 100644 --- a/ansible/inventory/hcloud.yml +++ b/ansible/inventory/hcloud.yml @@ -1,8 +1,7 @@ --- # Dynamic inventory: queries the Hetzner Cloud API directly (HCLOUD_TOKEN), # grouping hosts by the `role` label set on each hcloud_server resource in -# infra/modules//main.tf (role = dev/prod/vpn). Nothing here reads -# Terraform state - this always reflects whatever actually exists in Hetzner. +# infra/modules//main.tf (role = dev/prod/vpn). plugin: hetzner.hcloud.hcloud token: "{{ lookup('env', 'HCLOUD_TOKEN') }}" connect_with: public_ipv4 diff --git a/ansible/playbooks/group_vars/all.yml b/ansible/playbooks/group_vars/all.yml index ee4289f..a444ef2 100644 --- a/ansible/playbooks/group_vars/all.yml +++ b/ansible/playbooks/group_vars/all.yml @@ -1,8 +1,4 @@ --- -# Non-root sudo user the bootstrap role creates on each host - purely an -# Ansible-side concept, OpenTofu has no equivalent variable. Override with the -# DEPLOY_USER env var (control.sh's "Set variables" menu writes it) if you -# don't want the default. deploy_user: "{{ lookup('env', 'DEPLOY_USER') | default('deploy', true) }}" ansible_user: "{{ deploy_user }}" ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') | default('~/.ssh/id_ed25519', true) }}" @@ -11,11 +7,7 @@ ansible_ssh_private_key_file: "{{ lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY_FILE') services_root: "{{ playbook_dir }}/../../services" # S3-compatible bucket that services//backup-docker-compose.yml backs up -# to and restores from - see readme.md. Not provisioned by OpenTofu (the -# hcloud provider has no Object Storage resource), so these must point at a -# bucket you've created yourself. AWS_ENDPOINT is only needed for a -# non-AWS S3-compatible provider (e.g. Hetzner Object Storage); leave it unset -# to use real AWS S3. +# to and restores from backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}" backup_s3_access_key_id: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY_ID') }}" backup_s3_secret_access_key: "{{ lookup('env', 'BACKUP_S3_SECRET_ACCESS_KEY') }}" diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index 307f0cc..93bca70 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -1,7 +1,5 @@ --- -# Post-install bootstrap - replaces infra/scripts/bootstrap.sh.tftpl. Runs as -# root (see ansible_user override in playbooks/bootstrap.yml) since the deploy -# user doesn't exist yet on a freshly created server. Idempotent: safe to +# Post-install bootstrap - Idempotent: safe to # re-run against an already-bootstrapped host. - name: Update apt cache and upgrade packages @@ -10,10 +8,6 @@ upgrade: dist cache_valid_time: 3600 -# Docker straight from Ubuntu's own repositories - no external apt repo, GPG -# key or architecture juggling. docker.io is the engine, docker-compose-v2 is -# the `docker compose` plugin. A little behind Docker's own channel, but far -# simpler and plenty for these hosts. - name: Install Docker Engine and the Compose plugin ansible.builtin.apt: name: diff --git a/ansible/roles/deploy/tasks/main.yml b/ansible/roles/deploy/tasks/main.yml index 7ad5889..87be255 100644 --- a/ansible/roles/deploy/tasks/main.yml +++ b/ansible/roles/deploy/tasks/main.yml @@ -1,10 +1,8 @@ --- # Copies services// to the server and renders the files that -# used to be generated by OpenTofu (services//.env and, for dev, -# Runners/docker-compose.yml) - replaces the null_resource.deploy_services + -# local_file provisioners previously in infra/modules//main.tf. +# need templating (e.g. .env, docker-compose.yml). # -# Persistent data lives in named Docker volumes now, not on a Hetzner volume - +# Persistent data lives in named Docker volumes # see services//backup-docker-compose.yml. .env only carries the S3 # credentials that backup/restore need, rendered for every host. diff --git a/infra/main.tf b/infra/main.tf index 161587a..1b268c4 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -32,10 +32,7 @@ locals { { zone = "luke-else.co.uk", name = "traefik.vpn", value = module.vpn.ipv4 }, ] - # Microsoft 365 mail records for luke-else.co.uk - carried over from the - # domain's old DNS provider (pre-Hetzner), not previously applied here, so - # these are genuinely new to the Hetzner zone rather than something to - # import. + # Microsoft 365 mail records for luke-else.co.uk mail_records = [ { zone = "luke-else.co.uk", name = "@", type = "MX", value = "0 lukeelse-co-uk01b.mail.protection.outlook.com." }, { zone = "luke-else.co.uk", name = "@", type = "TXT", value = "\"v=spf1 include:spf.protection.outlook.com -all\"" }, diff --git a/infra/modules/dev/main.tf b/infra/modules/dev/main.tf index cd811f1..9f9517f 100644 --- a/infra/modules/dev/main.tf +++ b/infra/modules/dev/main.tf @@ -1,26 +1,24 @@ # dev: Gitea + Runner + Traefik (git.luke-else.co.uk, cicd.luke-else.co.uk) -# Port sources: services/dev/*docker-compose.yml (published ports) and services/todo.md -# (the documented UFW allow-list). dev has no firewall in docs/architecture.md, but we -# add one anyway for baseline safety - see conversation history. +# Port sources: services/dev/*docker-compose.yml (published ports) resource "hcloud_firewall" "this" { name = "dev-firewall" - rule { # server ssh - wired to the vpn server's public IP only, see infra/main.tf + rule { # server ssh - wired to the vpn server's public IP only direction = "in" protocol = "tcp" port = "22" source_ips = var.allowed_ssh_source_ips } - rule { # gitea ssh (git.luke-else.co.uk, published as 222:22) + rule { # gitea ssh direction = "in" protocol = "tcp" port = "222" source_ips = ["0.0.0.0/0", "::/0"] } - rule { # Traefik http/https (git.luke-else.co.uk, cicd.luke-else.co.uk) + rule { # Traefik http/https direction = "in" protocol = "tcp" port = "80" @@ -56,7 +54,7 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] - labels = { role = "dev" } # picked up by ansible/inventory/hcloud.yml + labels = { role = "dev" } network { network_id = var.network_id diff --git a/infra/modules/network/main.tf b/infra/modules/network/main.tf index 87abcb8..b094689 100644 --- a/infra/modules/network/main.tf +++ b/infra/modules/network/main.tf @@ -1,5 +1,5 @@ -# Private network shared by dev and prod (see docs/architecture.md "network" group). -# vpn is intentionally not attached to this network - it sits outside it in the diagram. +# Private network shared by dev and prod (see readme.md "network" group). +# vpn is intentionally not attached to this network resource "hcloud_network" "main" { name = "server-network" ip_range = var.ip_range diff --git a/infra/modules/prod/main.tf b/infra/modules/prod/main.tf index 65c2775..931c902 100644 --- a/infra/modules/prod/main.tf +++ b/infra/modules/prod/main.tf @@ -1,18 +1,15 @@ -# prod: Traefik, Websites, Bitwarden, RustDesk, status page - -# Port sources: services/prod/*docker-compose.yml (published ports) and services/todo.md -# (the documented UFW allow-list). +# Port sources: services/prod/*docker-compose.yml (published ports) resource "hcloud_firewall" "this" { name = "prod-firewall" - rule { # server ssh - wired to the vpn server's public IP only, see infra/main.tf + rule { # server ssh - wired to the vpn server's public IP only direction = "in" protocol = "tcp" port = "22" source_ips = var.allowed_ssh_source_ips } - rule { # Traefik http/https (Websites, Bitwarden, status page) + rule { # Traefik http/https direction = "in" protocol = "tcp" port = "80" @@ -61,7 +58,7 @@ resource "hcloud_firewall" "this" { source_ips = ["0.0.0.0/0", "::/0"] } - rule { # traffic from dev over the private network + rule { # traffic from over the private network direction = "in" protocol = "tcp" port = "1-65535" @@ -83,7 +80,7 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] - labels = { role = "prod" } # picked up by ansible/inventory/hcloud.yml + labels = { role = "prod" } network { network_id = var.network_id diff --git a/infra/modules/vpn/main.tf b/infra/modules/vpn/main.tf index 36ddc51..d7e5ef3 100644 --- a/infra/modules/vpn/main.tf +++ b/infra/modules/vpn/main.tf @@ -1,4 +1,4 @@ -# vpn: WireGuard (wg-easy) + Traefik. Not attached to the private network (see docs/architecture.md). +# vpn: WireGuard (wg-easy) + Traefik. Not attached to the private network. resource "hcloud_firewall" "this" { name = "vpn-firewall" @@ -9,7 +9,7 @@ resource "hcloud_firewall" "this" { source_ips = var.allowed_ssh_source_ips } - rule { # Traefik http/https (traefik.vpn.luke-else.co.uk) + rule { # Traefik http/https direction = "in" protocol = "tcp" port = "80" @@ -38,5 +38,5 @@ resource "hcloud_server" "this" { location = var.location ssh_keys = var.ssh_key_ids firewall_ids = [hcloud_firewall.this.id] - labels = { role = "vpn" } # picked up by ansible/inventory/hcloud.yml + labels = { role = "vpn" } } diff --git a/readme.md b/readme.md index 8167cf6..a5ae1cc 100644 --- a/readme.md +++ b/readme.md @@ -1,6 +1,6 @@ # Server -Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), each bootstrapped and deployed with [Ansible](https://www.ansible.com/), running a set of Docker Compose stacks behind [Traefik](https://traefik.io/traefik/). +Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted server estate: three [Hetzner Cloud](https://www.hetzner.com/cloud/) VPS instances provisioned with [OpenTofu](https://opentofu.org/), bootstrapped and deployed with [Ansible](https://www.ansible.com/), running Docker Compose stacks behind [Traefik](https://traefik.io/traefik/).

@@ -10,13 +10,14 @@ Infrastructure-as-code and service definitions for luke-else.co.uk's self-hosted - [Architecture](#architecture) - [Repository layout](#repository-layout) +- [Design choices](#design-choices) - [Prerequisites](#prerequisites) -- [Control panel (`control.sh`)](#control-panel-controlsh) +- [Quickstart](#quickstart) - [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) -- [Bootstrapping and deploying with Ansible](#bootstrapping-and-deploying-with-ansible-ansible) +- [Bootstrapping and deploying](#bootstrapping-and-deploying-ansible) +- [Running the services](#running-the-services-services) - [Service inventory](#service-inventory) -- [First-time setup](#first-time-setup) -- [Development container](#development-container) +- [Caveats](#caveats) - [Security notes](#security-notes) ## Architecture @@ -54,83 +55,71 @@ architecture-beta vpn:R -- L:backups ``` -`gateway` represents the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from - not a Hetzner resource, see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3). +`gateway` is the public internet, not a provisioned resource. `backups` is the S3-compatible bucket every host's `backup-docker-compose.yml` pushes to and restores from (see [Design choices](#design-choices)) — also not a Hetzner resource. | Server | Purpose | Network | |---|---|---| -| `dev` | Gitea, CI runner, dev-facing Traefik | Private network only (no public firewall exposure beyond CI/CD) | -| `prod` | Public-facing websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | -| `vpn` | WireGuard (wg-easy) + its own Traefik | **Not** attached to the private network — kept isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | +| `dev` | Gitea, CI runners, dev-facing Traefik | Private network only | +| `prod` | Public websites, Bitwarden, RustDesk, status page, prod Traefik | Private network + public firewall | +| `vpn` | WireGuard (wg-easy) + its own Traefik | **Not** on the private network — isolated so a compromised VPN endpoint can't pivot to `dev`/`prod` | -Each host's stateful data lives in local named Docker volumes, backed up nightly to a shared S3 bucket and restored automatically on first boot - see below. No host has a Hetzner Cloud Volume attached any more. - -`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach deployment targets on `prod` without exposing that traffic publicly. `vpn` is deliberately kept off this network. Each server has its own Hetzner Cloud Firewall (see `infra/modules//main.tf`) that only opens the ports actually used by the compose stacks running on it, plus SSH restricted to `var.allowed_ssh_source_ips`. +`dev` and `prod` share a private Hetzner network (`10.0.1.0/24` by default) so CI/CD on `dev` can reach `prod` without exposing that traffic publicly. Each server has its own Hetzner Cloud Firewall (`infra/modules//main.tf`) opening only the ports its own compose stacks use, plus SSH restricted to `var.allowed_ssh_source_ips`. ## Repository layout ``` . -├── infra/ # OpenTofu (Terraform-compatible) config — provisions the 3 servers, network, firewalls, DNS +├── infra/ # OpenTofu — provisions the 3 servers, network, firewalls, DNS │ ├── main.tf # root module: wires network + dev/prod/vpn/dns modules together │ ├── variables.tf # shared inputs (sizes, locations, IP ranges, SSH key names) -│ ├── outputs.tf # pass-through outputs from each host module -│ ├── ssh.tf # looks up each SSH key already uploaded to Hetzner Cloud -│ ├── versions.tf # provider requirements │ ├── terraform.tfvars.example │ └── modules/ -│ ├── network/ # shared private network + subnet (used by dev and prod) -│ ├── dev/ # dev server + firewall, labeled role=dev -│ ├── prod/ # prod server + firewall, labeled role=prod -│ ├── vpn/ # vpn server + firewall (no private network), labeled role=vpn -│ └── dns/ # Hetzner DNS zones + A records for every domain in var.dns_zones -├── ansible/ # Ansible: bootstraps each server and deploys/starts services// onto it -│ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by the role label above +│ ├── network/ # shared private network + subnet (dev + prod) +│ ├── dev/ prod/ vpn/ # one server + firewall each, labeled role= +│ └── dns/ # Hetzner DNS zones + records for every domain in var.dns_zones +├── ansible/ # bootstraps each server, deploys/starts services// onto it +│ ├── inventory/hcloud.yml # dynamic inventory — queries the Hetzner API, groups by role label │ ├── roles/ -│ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades -│ │ └── deploy/ # copies services//, renders .env (S3 backup credentials) + Runners/docker-compose.yml -│ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml, -│ └── group_vars/ # deploy_user, service_group, backup_s3_*, etc. - must sit next to -│ # the playbooks for Ansible to auto-load it (see ansible/README.md) +│ │ ├── bootstrap/ # Docker, deploy user, sshd hardening, unattended-upgrades +│ │ └── deploy/ # copies services//, renders .env + Runners/docker-compose.yml +│ └── playbooks/ # bootstrap.yml, deploy.yml, spinup.yml, spindown.yml, site.yml ├── services/ # Docker Compose stacks, grouped by which server they run on -│ ├── dev/ # Gitea + CI runner + Traefik + backup/restore (git.luke-else.co.uk, cicd.luke-else.co.uk) -│ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore -│ ├── vpn/ # WireGuard (wg-easy) + Traefik + backup/restore -│ └── todo.md # Outstanding manual setup/hardening tasks -├── docs/ -│ └── architecture.md # Source of the architecture diagram above -├── .devcontainer/ # Git submodule: shared devcontainer for working on this repo (OpenTofu tooling) +│ ├── dev/ # Gitea + CI runners + Traefik + backup/restore +│ ├── prod/ # Websites, Bitwarden, RustDesk, status page + Traefik + backup/restore +│ └── vpn/ # WireGuard (wg-easy) + Traefik + backup/restore +├── .devcontainer/ # git submodule: shared devcontainer for this repo (OpenTofu tooling) └── assets/ ``` -Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per logical service, a `backup-docker-compose.yml` + `restore.sh` pair for that host's S3 backup/restore (see below), plus a `spinup.sh` / `spindown.sh` pair that brings up or tears down every stack on that host in the right order. `infra/modules/` and `ansible/roles/deploy` both mirror this same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks are easy to find side by side. +Each of `services/dev`, `services/prod`, `services/vpn` follows the same convention: one `*-docker-compose.yml` (or subdirectory, e.g. `Runners/`) per service, a `backup-docker-compose.yml` + `restore.sh` pair, and a `spinup.sh` / `spindown.sh` pair that brings up or tears down everything on that host in order. `infra/modules/` and `ansible/roles/deploy` mirror the same dev/prod/vpn split, so a given host's cloud resources, bootstrap/deploy logic, and compose stacks sit side by side. -OpenTofu and Ansible have a clean split: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches anything over SSH. Everything from "the server exists" onward — installing Docker, creating the `deploy` user, hardening SSH, copying `services//`, and running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md) for the full rundown. +## Design choices + +- **OpenTofu and Ansible have a clean split**: OpenTofu only ever provisions cloud resources (servers, network, firewalls, DNS) and never touches a server over SSH. Everything from "the server exists" onward — Docker, the `deploy` user, SSH hardening, copying `services//`, running `spinup.sh`/`spindown.sh` — is Ansible's job. See [`ansible/README.md`](ansible/README.md). +- **No Hetzner Cloud Volumes.** Every stateful service mounts a plain named Docker volume instead. Each host backs its volumes up nightly to S3 and restores any that are empty on first boot — see [Persistent data](#persistent-data). +- **`vpn` is deliberately off the private network.** SSH to `dev`/`prod` is only reachable through it (see [Security notes](#security-notes)), but it can't reach either directly — containing a compromised VPN endpoint. +- **DNS is provisioned, not just documented.** `module.dns` creates a Hetzner zone and every A/MX/TXT/CNAME record in `infra/main.tf` — see [Managing DNS](#managing-dns). +- **Hosts are discovered dynamically** from the Hetzner API by role label, not a static inventory file — nothing in Ansible reads Terraform state. +- **Gitea Actions runners are disposable.** Their `/data` is a local cache; runner identity re-registers with Gitea on every `spinup.sh`, so they're excluded from backup. ## Prerequisites - A [Hetzner Cloud](https://console.hetzner.cloud/) project and API token -- One or more SSH keys uploaded to that project (Console → Security → SSH Keys) - all are installed on every server - plus the private key matching one of them available locally (Ansible uses it to bootstrap and deploy — see below) +- One or more SSH keys uploaded to that project (Console → Security → SSH Keys), plus the matching private key available locally - [OpenTofu](https://opentofu.org/docs/intro/install/) `>= 1.6.0` - [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/index.html) `>= 2.15` and the `hetzner.hcloud` collection (`ansible-galaxy collection install -r ansible/requirements.yml`) -- Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner (see [Managing DNS](#managing-dns) — the zones and records themselves are created for you) -- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/), AWS S3, or any other S3-compatible provider) and an access key/secret pair with read/write access to it - not provisioned by OpenTofu (the `hcloud` provider has no Object Storage resource), so create this yourself and pass it to Ansible as `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY` (and `BACKUP_S3_ENDPOINT` for anything other than real AWS S3) - see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3) +- Ownership of the domains in `var.dns_zones` at whatever registrar they're bought through, so you can point their NS records at Hetzner +- An S3-compatible bucket (e.g. [Hetzner Object Storage](https://www.hetzner.com/storage/object-storage/)) and an access key/secret pair — not provisioned by OpenTofu, so create this yourself -Docker + the Compose plugin, the non-root `deploy` user, and SSH hardening no longer need doing by hand — Ansible's `bootstrap` role handles all of that (see below). +## Quickstart -## Control panel (`control.sh`) - -If you'd rather not remember the exact command order, [`control.sh`](control.sh) at the repo root is an interactive menu that steps you through the whole pre-setup and bring-up. Run it from anywhere: +[`control.sh`](control.sh) at the repo root is an interactive menu covering the whole setup: ```sh ./control.sh ``` -It offers a **Check prerequisites** option (tools, `HCLOUD_TOKEN`/`DEPLOY_USER`/`BACKUP_S3_*` env vars, `terraform.tfvars`, SSH key, the `hetzner.hcloud` collection), a **Configuration** section, individual OpenTofu (`init`/`plan`/`apply`/`output`/`destroy`) and Ansible (bootstrap/deploy/spinup/spindown per role) actions, and a **Guided full setup** that runs everything in the firewall-imposed order — vpn first, pause for you to connect the VPN, then dev/prod. It's only a wrapper around the same `tofu` and `ansible-playbook` commands documented below, so you can always fall back to running them by hand. - -The **Configuration** section covers every variable both tools need: - -- **Set variables** prompts for `HCLOUD_TOKEN`, `DEPLOY_USER` (the non-root user Ansible creates — defaults to `deploy`), `ANSIBLE_SSH_PRIVATE_KEY_FILE`, and the `BACKUP_S3_*` credentials, one at a time (leave blank to keep the current value, secrets are input-masked). They're saved to `.control.env` at the repo root — gitignored, never committed — which is loaded automatically the next time you run `control.sh`, so you only enter them once. This is scoped to `control.sh` itself, not your regular shell. -- **Copy terraform.tfvars.example -> terraform.tfvars** creates `infra/terraform.tfvars` for you and prompts for `ssh_key_names` (the one required field with no default) so you don't leave the example's placeholder key names in place by accident. Everything else in that file (`location`, server types, `dns_zones`, etc.) still has sensible defaults you can edit by hand afterwards - see [Provisioning the infrastructure](#provisioning-the-infrastructure-infra) below. +It offers **Check prerequisites**, a **Configuration** section (prompts for `HCLOUD_TOKEN`, `DEPLOY_USER`, `ANSIBLE_SSH_PRIVATE_KEY_FILE`, and the `BACKUP_S3_*` credentials, saved to gitignored `.control.env` so you only enter them once), individual OpenTofu/Ansible actions, and a **Guided full setup** that runs everything in the firewall-imposed order — `vpn` first, pause for you to connect, then `dev`/`prod`. It's just a wrapper around the `tofu`/`ansible-playbook` commands below, so you can always drop to running them by hand. ## Provisioning the infrastructure (`infra/`) @@ -145,42 +134,21 @@ tofu plan tofu apply ``` -This creates, via `module.network` / `module.dev` / `module.prod` / `module.vpn` / `module.dns` in `infra/main.tf`: -- `hcloud_network` + subnet, shared by `dev` and `prod` (`modules/network`) -- one `hcloud_server` + `hcloud_firewall` per host, scoped to the ports each host actually uses, each server labeled `role = "dev"/"prod"/"vpn"` for Ansible's dynamic inventory (`modules/dev`, `modules/prod`, `modules/vpn`) -- one Hetzner DNS zone per domain in `var.dns_zones`, plus every A record in [Service inventory](#service-inventory) (`modules/dns` — see [Managing DNS](#managing-dns)) - -No servers have an attached volume — persistent data lives in named Docker volumes backed up to S3 instead, see below. +This creates the private network + subnet, one server + scoped firewall per host (labeled `role = dev/prod/vpn`), and one DNS zone per domain in `var.dns_zones` plus every record in [Service inventory](#service-inventory). `terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Useful outputs: `tofu output dev_ipv4`, `tofu output prod_ipv4`, `tofu output vpn_ipv4`, `tofu output dns_nameservers`. -`terraform.tfvars` and any `*.tfvars` file are gitignored — never commit real values there. Defaults for server sizes, locations, and IP ranges live in `infra/variables.tf` and are passed down into the modules from `infra/main.tf`; override them per-environment via `terraform.tfvars`. - -OpenTofu never connects to the servers over SSH — no provisioners, no `bootstrap.sh`, no copying `services/` — that's all Ansible now. See below. - ### Managing DNS -`module.dns` (in `infra/modules/dns`) creates one Hetzner DNS zone per domain in `var.dns_zones` (default: `luke-else.co.uk`, `divine-couture.co.uk`, `snexo.co.uk`) and an `hcloud_zone_rrset` per record in `var.records` — kept in sync with `local.dns_records` (A records, one per hostname currently referenced by a Traefik `Host()` rule anywhere under `services/`, pointed at whichever of `dev`/`prod`/`vpn` actually serves it) and `local.mail_records` (see below) in `infra/main.tf`. Zones have `prevent_destroy = true`, matching the volumes — losing one deletes every record in it. +Creating a zone doesn't make Hetzner authoritative by itself — run `tofu output dns_nameservers` and set those as the domain's NS records at its registrar. Propagation time depends on the registrar and the domain's previous NS TTL. -Creating the zone doesn't make Hetzner authoritative for the domain by itself: you still need to point that domain's NS records at Hetzner's nameservers at whichever registrar it's registered through. Run `tofu output dns_nameservers` after applying to get the exact nameservers per domain, and set those as the domain's NS records at the registrar. Propagation can take a while depending on the registrar and the domain's previous NS TTL. +To add a subdomain, add an entry to `local.dns_records` in `infra/main.tf` (zone, name, target host) and re-run `tofu apply` — don't hand-create records in the Hetzner console, they'll drift from state. `var.records` supports `MX`/`TXT`/`CNAME`/etc. too (see `infra/modules/dns/variables.tf` for the value format each type expects); entries sharing zone/name/type merge into one RRSet. -Adding a new subdomain: add an entry to `local.dns_records` in `infra/main.tf` (zone, name, and the target `module..ipv4`) and re-run `tofu apply` — don't hand-create records in the Hetzner console, they'll drift from state. Note `name = "@"` is Hetzner's convention for a zone's apex record (e.g. bare `snexo.co.uk`), not an empty string. +`local.mail_records` tracks the Microsoft 365 records for `luke-else.co.uk` (MX, SPF, Google-site-verification, `autodiscover`). DMARC and DKIM aren't set up yet — see [Caveats](#caveats). -Only A records for IPv4 are managed for hosts — none of the modules currently track servers' IPv6 addresses, so AAAA records aren't generated even though the firewalls already allow IPv6 traffic. +## Bootstrapping and deploying (`ansible/`) -`var.records` isn't limited to A records: each entry takes an optional `type` (`A` by default, or `MX`/`TXT`/`CNAME`/etc.), and entries sharing the same `zone`/`name`/`type` are merged into a single Hetzner RRSet with multiple values (needed for e.g. two TXT strings under the same name) - see `infra/modules/dns/variables.tf` for the exact value format each type expects (MX embeds the priority in the value string, TXT values need escaped double quotes). - -#### Mail records (Microsoft 365) - -`local.mail_records` in `infra/main.tf` tracks the DNS records Microsoft 365 needs for `luke-else.co.uk` - the MX record, the SPF and Google-site-verification TXT records (merged into one `@` TXT RRSet), and the `autodiscover` CNAME. These carried over from the domain's DNS provider prior to Hetzner and were never applied to the Hetzner zone, so they're genuinely new records here (not something to `tofu import` - nothing to collide with). - -**Not yet included:** a DMARC policy (`_dmarc` TXT) and DKIM signing (two `selector1._domainkey`/`selector2._domainkey` CNAMEs). DKIM's targets are generated per-tenant by Microsoft 365 (Defender portal → Email & collaboration → Policies & rules → DKIM) and can't be guessed - enable DKIM there first, then add the two CNAMEs it gives you to `local.mail_records`. - -`infra/modules/dns/moved.tf` re-points every pre-existing A record at its new resource address, since generalizing the module to support non-A record types changed how they're keyed internally - without it, `tofu apply` would destroy and recreate every A record for no functional reason. It's a one-time migration aid: run `tofu plan` first to confirm the only real changes are the new mail records being created, then the file can be deleted after a successful apply. - -## Bootstrapping and deploying with Ansible (`ansible/`) - -Once `tofu apply` has created the servers, [`ansible/`](ansible/README.md) takes over everything else: installing Docker, creating the `deploy` user, hardening SSH, copying `services//` to each server, rendering `.env` (S3 backup credentials) and, for `dev`, `Runners/docker-compose.yml`, and running `spinup.sh`/`spindown.sh`. Full detail lives in [`ansible/README.md`](ansible/README.md); the short version: +Once `tofu apply` has created the servers, Ansible installs Docker, creates the `deploy` user, hardens SSH, copies `services//` to each server, and runs `spinup.sh`/`spindown.sh`. Full detail in [`ansible/README.md`](ansible/README.md); the short version: ```sh cd ansible @@ -194,21 +162,16 @@ ansible-playbook playbooks/site.yml -l role_vpn ansible-playbook playbooks/site.yml -l role_dev,role_prod ``` -Hosts are discovered dynamically from the Hetzner API (`ansible/inventory/hcloud.yml`), grouped into `role_dev`/`role_prod`/`role_vpn` by the `role` label OpenTofu sets on each server — there's no static inventory file to keep in sync, and nothing here reads Terraform state. +`dev`/`prod`'s firewalls only accept SSH from `vpn`'s own public IP, so on a from-scratch estate they aren't reachable until `vpn` is up and you're tunneled through it — visit `https://vpn.luke-else.co.uk`, complete wg-easy's first-run setup, add a client peer, and connect before running the second command. If it's run before you're connected, Ansible just fails to connect — re-run once connected. -### Persistent data lives in named Docker volumes, backed up to S3 +### Persistent data -Every stateful service's compose file mounts a plain named Docker volume rather than a bind mount to a Hetzner volume - `gitea_data:/data`, `bitwarden_data:/data/`, `letsencrypt_data:/letsencrypt`, and so on across `services/dev/*.yml`, `services/prod/*.yml`, and `services/vpn/*.yml`. Each volume is declared with an explicit `name:` in every compose file that mounts it (the same pattern already used for the shared `proxy` network), so Compose creates it once and every stack that references it reuses the same underlying volume regardless of which compose file happens to run first. +Every stateful service mounts a named Docker volume (`gitea_data`, `bitwarden_data`, `letsencrypt_data`, etc.) rather than a Hetzner volume. Each host's `backup-docker-compose.yml` defines: -Each host's `backup-docker-compose.yml` defines two services against those same named volumes: -- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), running on a nightly cron (`0 3 * * *`), tars up every mounted volume and uploads it to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/-.tar.gz`, keeping 14 days of history (`BACKUP_RETENTION_DAYS`). Containers labeled `docker-volume-backup.stop-during-backup=true` (Gitea, Bitwarden, RustDesk, Uptime Kuma, snexo, wg-easy) are stopped for the duration of the backup so their data is captured in a consistent state, then restarted automatically. Started by `spinup.sh` alongside everything else. -- **`restore`** — a one-shot container (`amazon/aws-cli` running `restore.sh`) that `spinup.sh` runs *before* anything else starts. It checks each of the host's named volumes; any that are already empty are populated from the most recent object under that host's S3 prefix, any that already contain data are left completely untouched. On a fresh server with no prior backup (or a bucket that doesn't exist yet), it's a no-op - services just start with empty volumes as before. +- **`backup`** — [`offen/docker-volume-backup`](https://github.com/offen/docker-volume-backup), nightly cron, tars every mounted volume to `s3://$AWS_S3_BUCKET_NAME/$AWS_S3_PATH/-.tar.gz`, 14 days of retention. Containers that need a consistent snapshot are stopped for the duration and restarted after. +- **`restore`** — a one-shot container `spinup.sh` runs before anything else starts. Any volume that's already empty is populated from the most recent backup; anything with data is left untouched. On a fresh server (or no prior backup), it's a no-op. -`AWS_S3_BUCKET_NAME`, `AWS_S3_PATH` (set to the host name, e.g. `dev`), `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_ENDPOINT` are rendered into `services//.env` by Ansible's `deploy` role from `BACKUP_S3_BUCKET`/`BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`/`BACKUP_S3_ENDPOINT` in your shell environment (see [Prerequisites](#prerequisites)) — `docker compose` auto-loads `.env` from its working directory. `spinup.sh` on every host refuses to start anything if `.env` is missing. - -The Gitea Actions runners are the one exception: their `/data` is a disposable local cache (runner identity re-registers with Gitea on every `spinup.sh` run anyway), so it's a plain named volume per runner (`gitea_runner_N_data`, baked directly into the rendered `Runners/docker-compose.yml`) that's deliberately left out of the backup/restore stack. - -To trigger a backup or restore manually rather than waiting for cron or a restart: +The S3 credentials (`AWS_S3_BUCKET_NAME`, `AWS_ACCESS_KEY_ID`, etc.) are rendered into `services//.env` by Ansible from the `BACKUP_S3_*` env vars — `spinup.sh` refuses to start anything if `.env` is missing. To trigger either manually: ```sh cd services/ @@ -216,27 +179,9 @@ docker compose -f backup-docker-compose.yml exec backup backup # force an imme docker compose -f backup-docker-compose.yml run --rm restore # restore any currently-empty volumes ``` -### Gitea Actions runners +## Running the services (`services/`) -`dev` always runs three Gitea Actions runner containers, configured automatically on deploy. Re-running `ansible-playbook playbooks/deploy.yml -l role_dev` renders [`roles/deploy/templates/runners-docker-compose.yml.j2`](ansible/roles/deploy/templates/runners-docker-compose.yml.j2) straight onto the server as `services/dev/Runners/docker-compose.yml` — three `runner-N` services, each with its own container name and `/data` volume so their registrations don't collide. That file is generated on the remote host, so edit the template rather than hand-editing the rendered file (the count lives in a `{% raw %}{% set runner_count = 3 %}{% endraw %}` line at the top). - -Rendering it doesn't start anything by itself — re-run `ansible-playbook playbooks/spinup.yml -l role_dev` afterwards to apply a template change. - -Registration tokens are handled automatically, not baked into the generated file: `services/dev/spinup.sh` waits for Gitea to come up, runs `gitea actions generate-runner-token` inside the Gitea container, and writes the result to `Runners/.env`, which Compose loads automatically. There's no manual admin-UI step for this anymore. - -### First deploy: bootstrapping order matters - -`dev` and `prod`'s firewalls only accept SSH from `vpn`'s public IP (see [Security notes](#security-notes)), but `vpn`'s own WireGuard service isn't running until you deploy it — so on a from-scratch estate, Ansible can't reach `dev`/`prod` yet. Bring it up in this order: - -1. `ansible-playbook playbooks/site.yml -l role_vpn` — bootstraps, deploys, and starts `vpn` only; its firewall allows SSH from `var.allowed_ssh_source_ips` directly. -2. Visit `https://vpn.luke-else.co.uk` and complete wg-easy's first-run setup wizard (admin username/password, then add a client peer and download its config/QR code) - see [Setting up WireGuard (wg-easy)](#setting-up-wireguard-wg-easy). Import that config into a WireGuard client and connect. -3. `ansible-playbook playbooks/site.yml -l role_dev,role_prod` — now that your machine is tunneled through `vpn`, its NATed egress IP matches the firewall rule and `dev`/`prod` become reachable. - -If step 3 is run before you're connected to the VPN, Ansible will simply fail to connect — re-run it once connected. - -## Deploying the services (`services/`) - -After `ansible-playbook playbooks/deploy.yml` has copied `services//` to `/home/deploy/services/` on the matching server, SSH in as `deploy` and run the matching script from inside that directory — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do exactly this remotely (see [`ansible/README.md`](ansible/README.md)): +After `ansible-playbook playbooks/deploy.yml` has copied `services//` to the server, SSH in as `deploy` and run the matching script — or just use `ansible-playbook playbooks/spinup.yml` / `spindown.yml`, which do this remotely: ```sh # on dev @@ -244,7 +189,7 @@ After `ansible-playbook playbooks/deploy.yml` has copied `services//` to ` ./spindown.sh # reverse order, then prunes images/volumes # on prod -./spinup.sh # Traefik, → Watchtower → status → websites → Bitwarden → RustDesk +./spinup.sh # Traefik → Watchtower → status → websites → Bitwarden → RustDesk ./spindown.sh # on vpn @@ -252,29 +197,11 @@ After `ansible-playbook playbooks/deploy.yml` has copied `services//` to ` ./spindown.sh ``` -### Setting up WireGuard (wg-easy) +Each stack can also be managed individually with plain Compose, e.g. `docker compose -f bitwarden-docker-compose.yml up -d` from inside `services/prod`. -`vpn` runs [wg-easy](https://github.com/wg-easy/wg-easy) - WireGuard plus a web UI for managing peers - fronted by Traefik like everything else. `spinup.sh` only starts the container; wg-easy itself has no automated first-run setup here (see `services/vpn/vpn-docker-compose.yml`), so once it's up: +All three hosts run [Watchtower](https://containrrr.dev/watchtower/), polling every 60s with cleanup enabled, so images stay current automatically once deployed — re-run the spinup scripts only after adding/removing a service or changing compose files. Every public-facing service is fronted by its host's own Traefik, terminating TLS via Let's Encrypt, joining an external `proxy` network and opting in via `traefik.enable=true` labels (RustDesk and the Gitea SSH port publish ports directly, since they aren't HTTP). -1. Visit `https://vpn.luke-else.co.uk` and complete the setup wizard - pick an admin username and a strong password (this account can view/rotate every client's private key, and it's reachable from the public internet). Enabling 2FA afterwards (wg-easy supports it) is worth doing since this isn't just a status page like the old OpenVPN setup was. -2. Set the host clients should connect to (`vpn.luke-else.co.uk`) and the port (`51820`, matching the firewall rule in `infra/modules/vpn/main.tf`) when prompted. -3. Add a client per device from the UI, then download its config file or scan the QR code directly into the WireGuard app. - -wg-easy's own state (admin account, peers, keys) lives in the `wg_easy_data` named volume, backed up to S3 like everything else - see [Persistent data lives in named Docker volumes, backed up to S3](#persistent-data-lives-in-named-docker-volumes-backed-up-to-s3). Client configs themselves are only ever downloaded through the UI, not stored anywhere else. - -If you ever need to force a re-sync without going through Ansible (e.g. testing a local edit before committing it), a manual `scp -r services/prod deploy@:~/services` still works fine - `ansible-playbook playbooks/deploy.yml` will just overwrite it again next time it's run. - -Each stack can also be managed individually with plain Compose, e.g.: - -```sh -cd services/prod -docker compose -f bitwarden-docker-compose.yml up -d -docker compose -f bitwarden-docker-compose.yml down -``` - -All three hosts run [Watchtower](https://containrrr.dev/watchtower/) polling every 60s with cleanup enabled, so images are kept current automatically once deployed — the spinup scripts only need to be re-run after adding/removing a service or changing compose files. - -Every public-facing service is fronted by its host's own Traefik instance, terminating TLS via Let's Encrypt (`tlschallenge`, port 80/443). Each stack joins an external `proxy` Docker network and opts in via `traefik.enable=true` labels rather than publishing ports directly (RustDesk and the Gitea SSH port are the deliberate exceptions, since they aren't HTTP). +`vpn` runs [wg-easy](https://github.com/wg-easy/wg-easy) — WireGuard plus a web UI, fronted by Traefik. It has no automated first-run setup: visit `https://vpn.luke-else.co.uk`, pick an admin username/password (enable 2FA — this account can view/rotate every client's private key and is reachable from the public internet), set the connect host/port (`vpn.luke-else.co.uk` / `51820`), then add a client per device and download its config or scan its QR code. wg-easy's own state lives in the `wg_easy_data` volume, backed up like everything else; client configs themselves are never stored anywhere but the UI. ## Service inventory @@ -284,7 +211,7 @@ Every public-facing service is fronted by its host's own Traefik instance, termi |---|---|---| | Traefik | `traefik-docker-compose.yml` | `traefik.cicd.luke-else.co.uk` | | Gitea | `gitea-docker-compose.yml` | `git.luke-else.co.uk` (HTTP), SSH on `222` | -| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated — see [Scaling Gitea Actions runners](#scaling-gitea-actions-runners)) | N/A | +| Gitea Actions runner(s) | `Runners/docker-compose.yml` (generated, 3 by default) | N/A | | Watchtower | `watchtower-docker-compose.yml` | — | | Backup/restore | `backup-docker-compose.yml` | — | @@ -309,15 +236,9 @@ Every public-facing service is fronted by its host's own Traefik instance, termi | Watchtower | `watchtower-docker-compose.yml` | — | | Backup/restore | `backup-docker-compose.yml` | — | -## First-time setup - -A few things need manual attention before a stack is fully live — tracked in [`services/todo.md`](services/todo.md), summarized here: - -- **General host hardening**: non-root user, Docker, and unattended-upgrades are now handled automatically by [Ansible's `bootstrap` role](ansible/roles/bootstrap/tasks/main.yml); UFW is the remaining manual item in `services/todo.md` (the Hetzner Cloud Firewalls already allowlist per-host ports — see [Security notes](#security-notes)). - ## Development container -`.devcontainer` is a git submodule providing a ready-to-use OpenTofu development environment (VS Code + OpenTofu/Docker/Mermaid extensions). After cloning: +`.devcontainer` is a git submodule providing a ready-to-use OpenTofu development environment (VS Code + OpenTofu/Docker/Mermaid extensions): ```sh git submodule update --init --recursive @@ -325,11 +246,17 @@ git submodule update --init --recursive Then reopen the repo in VS Code with the Dev Containers extension. +## Caveats + +- **UFW isn't configured** — the Hetzner Cloud Firewalls already allowlist per-host ports, but there's no host-level firewall as defense in depth yet. +- **DMARC and DKIM aren't set up** for `luke-else.co.uk`. DKIM's targets are generated per-tenant by Microsoft 365 (Defender portal → Email & collaboration → Policies & rules → DKIM) and can't be guessed — enable it there first, then add the two CNAMEs it gives you to `local.mail_records`. +- **Only IPv4 DNS is managed.** None of the infra modules track servers' IPv6 addresses, so no AAAA records are generated even though the firewalls already allow IPv6 traffic. +- **`infra/modules/dns/moved.tf`** re-points pre-existing A records at their new resource address after the DNS module was generalized for non-A record types. It's a one-time migration aid — safe to delete once `tofu plan` shows no unexpected changes. + ## Security notes -- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_ACCESS_KEY_ID`/`BACKUP_S3_SECRET_ACCESS_KEY`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. The S3 credentials end up in each host's `services//.env` (rendered by Ansible, mode `0600`, gitignored) - anyone with root on a host can read them, same trust boundary as everything else `deploy` can already do there. -- SSH to `dev` and `prod` is restricted to the `vpn` server's own public IP — you must be tunneled into the VPN to reach them over SSH. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your own egress IP(s). -- `vpn` is intentionally excluded from the private network so that a compromised VPN endpoint cannot reach `dev` or `prod` directly over it — SSH access still works because the VPN's egress traffic is NATed through its own public IP. -- Firewalls are allowlists scoped per host in `infra/modules//main.tf` — only ports actually used by that host's compose stacks (plus the cross-host private network range) are open. -- Every host's [Ansible bootstrap role](ansible/roles/bootstrap/tasks/main.yml) disables SSH password authentication, restricts root login to key-only, and creates a separate sudo user (`deploy_user`, see `ansible/playbooks/group_vars/all.yml`) for day-to-day access. -- DNS zones (`modules/dns`) carry both Hetzner's own `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, including for `divine-couture.co.uk` and `snexo.co.uk`, not just `luke-else.co.uk`. +- Real secrets (`HCLOUD_TOKEN`, `BACKUP_S3_*`, `*.tfvars`, your SSH private key) must never be committed — see `.gitignore`. S3 credentials land in each host's `services//.env` (mode `0600`, gitignored, rendered by Ansible). +- SSH to `dev`/`prod` is restricted to `vpn`'s own public IP — you must be tunneled into the VPN to reach them. SSH to `vpn` itself is gated by `var.allowed_ssh_source_ips`; narrow this from the default `0.0.0.0/0` once you know your egress IP(s). +- Firewalls are per-host allowlists (`infra/modules//main.tf`) — only ports actually used by that host's compose stacks are open. +- Ansible's bootstrap role disables SSH password auth, restricts root login to key-only, and creates a separate sudo user (`deploy_user`) for day-to-day access. +- DNS zones carry both Hetzner's `delete_protection` and Terraform's `prevent_destroy` — losing a zone takes every record in it with it, across all three managed domains. diff --git a/services/todo.md b/services/todo.md deleted file mode 100644 index 158ce2c..0000000 --- a/services/todo.md +++ /dev/null @@ -1,32 +0,0 @@ -# ToDo Items - -## General - -- Setup non root user - - - "21115:21115/tcp" - - "21116:21116/tcp" - - "21116:21116/udp" - - "21117:21117/tcp" - - "21119:21119/tcp" - -- Install docker, docker-compose and apache utils. - -## Traefik + TraefikRunner + Traefik VPN - -- Setup htaccess -> `echo $(htpasswd -nb user password) | sed -e s/\\$/\\$\\$/g` -- Ensure email address is correct - -## Gitea - -- Ensure that ports are assigned correctly for the system - -## Websites - -- Ensure website files are copied over -- Ensure that ports are assigned correctly for the system - -## Bitwarden - -- Ensure that all data is fully encrypted during transfer. -- Ensure that ports are assigned correctly for the system From 2a45a7f68058c32212eeb1a20107ceb326205f2a Mon Sep 17 00:00:00 2001 From: Luke Else Date: Tue, 21 Jul 2026 20:44:54 +0100 Subject: [PATCH 23/24] chore: Cleaned up control script by seperating servers apart --- control.sh | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/control.sh b/control.sh index 761fa58..a230171 100755 --- a/control.sh +++ b/control.sh @@ -194,9 +194,14 @@ ansible_inventory() { run_in "$ANSIBLE_DIR" ansible-inventory --graph; } play() { run_in "$ANSIBLE_DIR" ansible-playbook "playbooks/$1" -l "$2"; } ansible_bootstrap_vpn() { play bootstrap.yml role_vpn; } -ansible_deploy_vpn() { play deploy.yml role_vpn && play spinup.yml role_vpn; } -ansible_bootstrap_dp() { play bootstrap.yml role_dev,role_prod; } -ansible_deploy_dp() { play deploy.yml role_dev,role_prod && play spinup.yml role_dev,role_prod; } +ansible_deploy_vpn() { play deploy.yml role_vpn; } +ansible_spinup_vpn() { play spinup.yml role_vpn; } +ansible_bootstrap_dev() { play bootstrap.yml role_dev; } +ansible_deploy_dev() { play deploy.yml role_dev; } +ansible_spinup_dev() { play spinup.yml role_dev; } +ansible_bootstrap_prod() { play bootstrap.yml role_prod; } +ansible_deploy_prod() { play deploy.yml role_prod; } +ansible_spinup_prod() { play spinup.yml role_prod; } ansible_spindown() { local target @@ -218,7 +223,7 @@ Steps, in order: 2. Ansible: bootstrap + deploy + spinup for role_vpn 3. YOU: visit https://vpn.luke-else.co.uk, complete wg-easy's setup, add a client peer, and connect with a WireGuard client - 4. Ansible: bootstrap + deploy + spinup for role_dev,role_prod + 4. Ansible: bootstrap + deploy + spinup for role_dev, then role_prod EOF confirm "Start the guided walkthrough?" || return 0 @@ -237,7 +242,7 @@ EOF echo; title "2/4 Ansible - vpn" if confirm "Bootstrap + deploy + spinup role_vpn now?"; then - ansible_bootstrap_vpn && ansible_deploy_vpn || { bad "vpn step failed - stopping."; return 1; } + ansible_bootstrap_vpn && ansible_deploy_vpn && ansible_spinup_vpn || { bad "vpn step failed - stopping."; return 1; } fi echo; title "3/4 Connect the VPN" @@ -245,8 +250,11 @@ EOF confirm "Are you connected to the VPN?" || { warn "Stopping - reconnect and re-run this step."; return 0; } echo; title "4/4 Ansible - dev + prod" - if confirm "Bootstrap + deploy + spinup role_dev,role_prod now?"; then - ansible_bootstrap_dp && ansible_deploy_dp || { bad "dev/prod step failed."; return 1; } + if confirm "Bootstrap + deploy + spinup role_dev now?"; then + ansible_bootstrap_dev && ansible_deploy_dev && ansible_spinup_dev || { bad "dev step failed."; return 1; } + fi + if confirm "Bootstrap + deploy + spinup role_prod now?"; then + ansible_bootstrap_prod && ansible_deploy_prod && ansible_spinup_prod || { bad "prod step failed."; return 1; } fi ok "Guided setup complete." } @@ -271,9 +279,10 @@ menu() { title "Ansible (ansible/)" echo " 10) Install collections 11) Show inventory" - echo " 12) Bootstrap vpn 13) Deploy + spinup vpn" - echo " 14) Bootstrap dev+prod 15) Deploy + spinup dev+prod" - echo " 16) Spindown (choose target)" + echo " 12) Bootstrap vpn 13) Deploy vpn 14) Spinup vpn" + echo " 15) Bootstrap dev 16) Deploy dev 17) Spinup dev" + echo " 18) Bootstrap prod 19) Deploy prod 20) Spinup prod" + echo " 21) Spindown (choose target)" echo echo " 0) Quit" @@ -297,9 +306,14 @@ while true; do 11) ansible_inventory; pause ;; 12) ansible_bootstrap_vpn; pause ;; 13) ansible_deploy_vpn; pause ;; - 14) ansible_bootstrap_dp; pause ;; - 15) ansible_deploy_dp; pause ;; - 16) ansible_spindown; pause ;; + 14) ansible_spinup_vpn; pause ;; + 15) ansible_bootstrap_dev; pause ;; + 16) ansible_deploy_dev; pause ;; + 17) ansible_spinup_dev; pause ;; + 18) ansible_bootstrap_prod; pause ;; + 19) ansible_deploy_prod; pause ;; + 20) ansible_spinup_prod; pause ;; + 21) ansible_spindown; pause ;; 0) echo "Bye."; exit 0 ;; "") ;; *) warn "Unknown option: $choice"; pause ;; From 9738b59543c2003c355f949e0bbed39711697397 Mon Sep 17 00:00:00 2001 From: Luke Else Date: Wed, 22 Jul 2026 00:23:42 +0100 Subject: [PATCH 24/24] chore: Refined final deployment stats --- services/dev/spinup.sh | 5 +++-- services/prod/backup-docker-compose.yml | 1 - services/prod/web-docker-compose.yml | 19 ------------------- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/services/dev/spinup.sh b/services/dev/spinup.sh index 22044b8..f25d991 100755 --- a/services/dev/spinup.sh +++ b/services/dev/spinup.sh @@ -17,8 +17,9 @@ docker compose -f traefik-docker-compose.yml pull && docker compose -f traefik-d docker compose -f gitea-docker-compose.yml pull && docker compose -f gitea-docker-compose.yml up -d echo "Waiting for Gitea to become ready..." +sleep 20 for i in $(seq 1 30); do - if docker exec gitea gitea admin user list >/dev/null 2>&1; then + if docker exec -u 1000:1000 gitea gitea admin user list >/dev/null 2>&1; then break fi sleep 2 @@ -27,7 +28,7 @@ done # Generate a fresh Actions runner registration token straight from Gitea and # hand it to the runner containers via .env (docker compose auto-loads this) - # no manual copy/paste from the admin UI needed. -RUNNER_TOKEN=$(docker exec gitea gitea actions generate-runner-token) +RUNNER_TOKEN=$(docker exec -u 1000:1000 gitea gitea actions generate-runner-token) echo "GITEA_RUNNER_REGISTRATION_TOKEN=$RUNNER_TOKEN" > Runners/.env cd Runners diff --git a/services/prod/backup-docker-compose.yml b/services/prod/backup-docker-compose.yml index 7263c2b..0515547 100644 --- a/services/prod/backup-docker-compose.yml +++ b/services/prod/backup-docker-compose.yml @@ -22,7 +22,6 @@ services: - bitwarden_data:/backup/bitwarden_data:ro - rustdesk_data:/backup/rustdesk_data:ro - uptime_kuma_data:/backup/uptime_kuma_data:ro - - snexo_data:/backup/snexo_data:ro - letsencrypt_data:/backup/letsencrypt_data:ro - /var/run/docker.sock:/var/run/docker.sock:ro restart: unless-stopped diff --git a/services/prod/web-docker-compose.yml b/services/prod/web-docker-compose.yml index e4edb7f..93af14c 100644 --- a/services/prod/web-docker-compose.yml +++ b/services/prod/web-docker-compose.yml @@ -58,25 +58,6 @@ services: - "traefik.http.routers.metarius.tls.certresolver=myresolver" restart: unless-stopped - divine-couture: - image: git.luke-else.co.uk/luke-else/divine-couture.co.uk:latest - container_name: divine-couture - networks: - - proxy - labels: - ## Expose divine-couture Through Trefik ## - - "traefik.enable=true" # <== Enable traefik to proxy this container - - - "traefik.http.middlewares.redirect-web-secure.redirectscheme.scheme=https" - - "traefik.http.routers.divine-couture-insecure.middlewares=redirect-web-secure" - - "traefik.http.routers.divine-couture-insecure.rule=Host(`www.divine-couture.co.uk`)" - - "traefik.http.routers.divine-couture-insecure.entrypoints=web" - - - "traefik.http.routers.divine-couture.rule=Host(`www.divine-couture.co.uk`)" - - "traefik.http.routers.divine-couture.entrypoints=websecure" - - "traefik.http.routers.divine-couture.tls.certresolver=myresolver" - restart: unless-stopped - networks: proxy: external: true