From 5c5ce535f17a026c1397650e6159ec93f38bd858 Mon Sep 17 00:00:00 2001 From: Wouter Meetsma Date: Tue, 30 Jun 2026 15:11:17 +0200 Subject: [PATCH 1/3] fix: observability stack log paths and self-hosted Supabase wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs found while debugging an empty Grafana dashboard on a HestiaCP deployment: - RUBRICMAKER_LOG_DIR examples used a shell glob (/home//web/*/log), which Docker bind mounts don't expand — it silently created an empty literal directory instead. Now points at the parent dir; Promtail's own ** glob already recurses into each domain's logs/ folder. - HestiaCP's per-domain web//logs/*.log files are typically symlinks to the real webserver log dir (e.g. /var/log/apache2/domains), which lives outside the RUBRICMAKER_LOG_DIR mount and so couldn't be resolved. Added RUBRICMAKER_WEBSERVER_LOG_DIR to mount that target dir at the same path. - The self-hosted-db network join example assumed this repo's combined docker-compose.yml stack (network rubricmaker_default). Documented the official self-hosted Supabase CLI stack as well (network supabase_default, container supabase-db), which is what most self-hosted deployments actually run. Also wired VITE_STRESS_TEST_LOGGING into the HestiaCP deploy workflow as an opt-in repo variable, so the client_logs diagnostic stream can be toggled for a pilot window without a manual build. --- .env.observability.example | 15 +++++++++++- .github/workflows/deploy-hestiacp.yml | 5 ++++ docker-compose.observability.yml | 22 ++++++++++++++---- docs/OBSERVABILITY_HESTIACP.md | 33 ++++++++++++++++++++------- 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/.env.observability.example b/.env.observability.example index b0eff2ed..5bf68a74 100644 --- a/.env.observability.example +++ b/.env.observability.example @@ -20,15 +20,28 @@ GRAFANA_ROOT_URL= # ── Web server logs (Promtail) ─────────────────────────────────────────────────── # Directory Promtail scans (recursively) for *access*.log / *error*.log files. +# Must be a real, literal directory — this is a Docker bind-mount source, so +# shell globs (e.g. a trailing `/*/log`) do NOT work; Docker creates an empty +# directory with that literal name instead of expanding it. # Examples: # /var/log ← default; covers /var/log/virtualmin, # /var/log/nginx, /var/log/apache2, etc. -# /home/rubricmaker/web/*/log ← HestiaCP per-domain logs +# /home/rubricmaker/web ← HestiaCP per-domain logs (Promtail's +# own ** glob recurses into each +# domain's logs/ subfolder) # /var/log/virtualmin ← Virtualmin vhost logs # Combined docker-compose.yml stack: leave as default — Promtail's Docker scrape # job picks up the `app` container's stdout logs directly (see nginx.prod.conf). RUBRICMAKER_LOG_DIR=/var/log +# HestiaCP only: web//logs/*.log under RUBRICMAKER_LOG_DIR are usually +# symlinks to the panel's real webserver log dir, e.g. /var/log/apache2/domains +# (Apache) or /var/log/nginx/domains (Nginx) — check with +# `ls -la /home//web//logs/`. Set this to that real target dir so +# the symlinks resolve inside the container; it's mounted at the same absolute +# path. Leave unset/blank on non-HestiaCP setups. +RUBRICMAKER_WEBSERVER_LOG_DIR=/var/log/apache2/domains + # ── client_logs (Part 1 app-level diagnostics) ─────────────────────────────────── # Optional: lets Grafana query the `client_logs` table (see README → "Stress-test diff --git a/.github/workflows/deploy-hestiacp.yml b/.github/workflows/deploy-hestiacp.yml index 2bdfdb77..9d9f2aca 100644 --- a/.github/workflows/deploy-hestiacp.yml +++ b/.github/workflows/deploy-hestiacp.yml @@ -31,6 +31,11 @@ jobs: VITE_SUPABASE_URL: ${{ secrets.VITE_SUPABASE_URL }} VITE_SUPABASE_ANON_KEY: ${{ secrets.VITE_SUPABASE_ANON_KEY }} SUPABASE_PUBLIC_URL: ${{ secrets.SUPABASE_PUBLIC_URL }} + # Optional diagnostic event stream to `client_logs` — see README → + # "Stress-test logging". Toggle via repo Settings → Secrets and + # variables → Actions → Variables (not a secret: it's a feature + # flag, not sensitive data). + VITE_STRESS_TEST_LOGGING: ${{ vars.VITE_STRESS_TEST_LOGGING }} - name: Deploy via rsync uses: burnett01/rsync-deployments@9.0.0 diff --git a/docker-compose.observability.yml b/docker-compose.observability.yml index 3fab11d9..31531679 100644 --- a/docker-compose.observability.yml +++ b/docker-compose.observability.yml @@ -33,6 +33,12 @@ services: # Host log directory for traditional Apache/Nginx + HestiaCP/Virtualmin # deployments (set RUBRICMAKER_LOG_DIR to the panel's log path). - ${RUBRICMAKER_LOG_DIR:-/var/log}:/var/host-logs:ro + # HestiaCP's per-domain web//logs/*.log files are symlinks to + # the panel's real webserver log dir (e.g. /var/log/apache2/domains) — + # mount that dir too, at the *same* path, so the symlinks resolve + # inside the container instead of "no such file or directory". + # Leave RUBRICMAKER_WEBSERVER_LOG_DIR unset on non-HestiaCP setups. + - ${RUBRICMAKER_WEBSERVER_LOG_DIR:-/var/log/apache2/domains}:${RUBRICMAKER_WEBSERVER_LOG_DIR:-/var/log/apache2/domains}:ro # Optional: only relevant if the combined docker-compose.yml stack also # runs on this host — harmless if these paths don't exist. - /var/lib/docker/containers:/var/lib/docker/containers:ro @@ -76,13 +82,21 @@ volumes: grafana-data: # persists Grafana dashboards/users # ── Optional: self-hosted db on the same host ──────────────────────────────── -# If the combined docker-compose.yml stack (db, auth, rest, ...) also runs on -# this host and you want Grafana to reach its `db` container by name, join its -# network and set SUPABASE_DB_HOST=db: +# If a self-hosted Supabase stack also runs on this host and you want Grafana +# to reach its db container by name, join its Docker network. The network and +# container names depend on which self-hosted stack this is: +# - This repo's combined docker-compose.yml (`name: rubricmaker`): +# network rubricmaker_default, SUPABASE_DB_HOST=db:5432 +# - The official Supabase CLI / self-hosted compose stack: +# network supabase_default, SUPABASE_DB_HOST=supabase-db:5432 +# (connect to the `supabase-db` container directly, not the +# `supabase-pooler`/Supavisor container also on 5432/6543) +# Confirm with `docker network ls` / `docker ps` on the host — names can be +# customized in either stack's own compose file. # # networks: # default: -# name: rubricmaker_default +# name: supabase_default # external: true # # Otherwise (managed Supabase, or Supabase on another host), leave this diff --git a/docs/OBSERVABILITY_HESTIACP.md b/docs/OBSERVABILITY_HESTIACP.md index c41e7aed..8bf4fb4a 100644 --- a/docs/OBSERVABILITY_HESTIACP.md +++ b/docs/OBSERVABILITY_HESTIACP.md @@ -102,14 +102,25 @@ Edit `.env.observability`: GRAFANA_ADMIN_PASSWORD= # HestiaCP per-domain log layout (see .env.observability.example for the -# Virtualmin equivalent). Promtail recurses into this path looking for -# *access*.log / *error*.log files. -RUBRICMAKER_LOG_DIR=/home/rubricmaker/web/*/log +# Virtualmin equivalent). This is a Docker bind-mount source, so it must be a +# real, literal directory — no shell globs (Docker won't expand a `*` in a +# host path; it just creates an empty dir with that literal name and mounts +# that). Point at the parent `web` dir; Promtail's own `**` glob in +# promtail-config.yml recurses into each domain's log/ folder from there. +RUBRICMAKER_LOG_DIR=/home/rubricmaker/web # Set in step 7, once the subdomain is live. GRAFANA_DOMAIN=observability.rubricmaker.example.com GRAFANA_ROOT_URL=https://observability.rubricmaker.example.com/ +# HestiaCP's web//logs/*.log files are usually symlinks to the +# panel's real webserver log dir — check with +# `ls -la /home/rubricmaker/web//logs/`. Mount that real target +# dir too (same absolute path) so the symlinks resolve inside the container. +# Apache (most HestiaCP installs): +RUBRICMAKER_WEBSERVER_LOG_DIR=/var/log/apache2/domains +# Nginx-only HestiaCP installs would instead use /var/log/nginx/domains. + # Optional — only if you also want client_logs queryable from Grafana # (see README → "Stress-test logging"). # SUPABASE_DB_HOST= @@ -118,11 +129,17 @@ GRAFANA_ROOT_URL=https://observability.rubricmaker.example.com/ # SUPABASE_DB_PASSWORD= ``` -> **`RUBRICMAKER_LOG_DIR=/home/rubricmaker/web/*/log` vs `/var/log`:** -> HestiaCP writes per-domain logs to `/home//web//log/` *and* -> aggregates them under `/var/log/$WEB_SYSTEM/domains/`. Either path works; -> the `/home/.../web/*/log` glob is narrower if this HestiaCP user hosts -> multiple unrelated domains and you only want RubricMaker's logs. +> **`RUBRICMAKER_LOG_DIR=/home/rubricmaker/web` vs `/var/log`:** +> HestiaCP writes per-domain logs to `/home//web//logs/` *and* +> aggregates them under `/var/log/$WEB_SYSTEM/domains/`. Either parent path +> works as `RUBRICMAKER_LOG_DIR`, but the per-domain `logs/` entries are +> typically symlinks pointing at the latter — see `RUBRICMAKER_WEBSERVER_LOG_DIR` +> above, which mounts that real target dir so the symlinks don't dangle inside +> the container. To narrow to a single domain if this HestiaCP user hosts +> multiple unrelated ones, point `RUBRICMAKER_LOG_DIR` directly at that +> domain's own `logs/` dir (e.g. +> `/home/rubricmaker/web/rubricmaker.example.com/logs`) — but it must be a +> literal, existing path; wildcards don't work in a bind-mount source. --- From af102ca7da904467683b252cc31c32b98da0397a Mon Sep 17 00:00:00 2001 From: Wouter Meetsma Date: Tue, 30 Jun 2026 17:14:24 +0200 Subject: [PATCH 2/3] fix: make RUBRICMAKER_WEBSERVER_LOG_DIR truly optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address CodeRabbit review on #244: - .env.observability.example example line contradicted its own "leave unset" guidance — comment it out. - docker-compose.observability.yml defaulted the bind-mount source to /var/log/apache2/domains even when unset, which Docker would silently create as an empty directory on non-HestiaCP hosts. Default to /var/log instead, which always exists, so leaving the var unset is a true no-op. - Clarified in the HestiaCP doc that the second mount isn't independently scraped by Promtail (only one webserver job globs RUBRICMAKER_LOG_DIR), so it doesn't cause duplicate log ingestion as flagged. --- .env.observability.example | 5 +++-- docker-compose.observability.yml | 8 +++++--- docs/OBSERVABILITY_HESTIACP.md | 4 ++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.env.observability.example b/.env.observability.example index 5bf68a74..c0e5495f 100644 --- a/.env.observability.example +++ b/.env.observability.example @@ -39,8 +39,9 @@ RUBRICMAKER_LOG_DIR=/var/log # (Apache) or /var/log/nginx/domains (Nginx) — check with # `ls -la /home//web//logs/`. Set this to that real target dir so # the symlinks resolve inside the container; it's mounted at the same absolute -# path. Leave unset/blank on non-HestiaCP setups. -RUBRICMAKER_WEBSERVER_LOG_DIR=/var/log/apache2/domains +# path. Leave unset/blank on non-HestiaCP setups — when blank, no extra mount +# is added (see docker-compose.observability.yml). +# RUBRICMAKER_WEBSERVER_LOG_DIR=/var/log/apache2/domains # ── client_logs (Part 1 app-level diagnostics) ─────────────────────────────────── diff --git a/docker-compose.observability.yml b/docker-compose.observability.yml index 31531679..dd34cb0f 100644 --- a/docker-compose.observability.yml +++ b/docker-compose.observability.yml @@ -36,9 +36,11 @@ services: # HestiaCP's per-domain web//logs/*.log files are symlinks to # the panel's real webserver log dir (e.g. /var/log/apache2/domains) — # mount that dir too, at the *same* path, so the symlinks resolve - # inside the container instead of "no such file or directory". - # Leave RUBRICMAKER_WEBSERVER_LOG_DIR unset on non-HestiaCP setups. - - ${RUBRICMAKER_WEBSERVER_LOG_DIR:-/var/log/apache2/domains}:${RUBRICMAKER_WEBSERVER_LOG_DIR:-/var/log/apache2/domains}:ro + # inside the container instead of "no such file or directory". Falls + # back to /var/log (already guaranteed to exist, unlike an Apache-only + # path) so leaving RUBRICMAKER_WEBSERVER_LOG_DIR unset on non-HestiaCP + # setups is a harmless no-op rather than creating a bogus empty dir. + - ${RUBRICMAKER_WEBSERVER_LOG_DIR:-/var/log}:${RUBRICMAKER_WEBSERVER_LOG_DIR:-/var/log}:ro # Optional: only relevant if the combined docker-compose.yml stack also # runs on this host — harmless if these paths don't exist. - /var/lib/docker/containers:/var/lib/docker/containers:ro diff --git a/docs/OBSERVABILITY_HESTIACP.md b/docs/OBSERVABILITY_HESTIACP.md index 8bf4fb4a..9728dcd3 100644 --- a/docs/OBSERVABILITY_HESTIACP.md +++ b/docs/OBSERVABILITY_HESTIACP.md @@ -117,6 +117,10 @@ GRAFANA_ROOT_URL=https://observability.rubricmaker.example.com/ # panel's real webserver log dir — check with # `ls -la /home/rubricmaker/web//logs/`. Mount that real target # dir too (same absolute path) so the symlinks resolve inside the container. +# This doesn't cause duplicate log ingestion: Promtail's only webserver scrape +# path is RUBRICMAKER_LOG_DIR (`/var/host-logs/**` in promtail-config.yml) — +# this second mount isn't separately scraped, it just makes the symlink +# targets resolvable when Promtail follows them. # Apache (most HestiaCP installs): RUBRICMAKER_WEBSERVER_LOG_DIR=/var/log/apache2/domains # Nginx-only HestiaCP installs would instead use /var/log/nginx/domains. From 5902714b398d6fd9851d82deb7325e0dd9ef1245 Mon Sep 17 00:00:00 2001 From: Wouter Meetsma Date: Tue, 30 Jun 2026 18:42:38 +0200 Subject: [PATCH 3/3] fix: correct misleading comment on RUBRICMAKER_WEBSERVER_LOG_DIR fallback The comment claimed leaving it unset adds no mount; it actually falls back to mounting /var/log read-only, which is harmless since Promtail only scrapes RUBRICMAKER_LOG_DIR. Address CodeRabbit follow-up on #244. --- .env.observability.example | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.env.observability.example b/.env.observability.example index c0e5495f..ca23ec37 100644 --- a/.env.observability.example +++ b/.env.observability.example @@ -39,8 +39,9 @@ RUBRICMAKER_LOG_DIR=/var/log # (Apache) or /var/log/nginx/domains (Nginx) — check with # `ls -la /home//web//logs/`. Set this to that real target dir so # the symlinks resolve inside the container; it's mounted at the same absolute -# path. Leave unset/blank on non-HestiaCP setups — when blank, no extra mount -# is added (see docker-compose.observability.yml). +# path. Leave unset/blank on non-HestiaCP setups — when blank, the compose +# file falls back to mounting `/var/log` read-only at the same path, which +# is harmless since Promtail only scrapes `/var/host-logs`. # RUBRICMAKER_WEBSERVER_LOG_DIR=/var/log/apache2/domains