Refine public docs generation and simplify topology diagrams

scripts/docs/generate-all.sh

@@ -6,7 +6,6 @@ mkdir -p docs/generated docs/diagrams docs/public
 scripts/docs/render-compose-config.sh
 python3 scripts/docs/generate-compose-inventory.py docs/generated/docker-compose.resolved.yml docs/generated/compose-inventory.md
 python3 scripts/docs/generate-traefik-routes.py docs/generated/docker-compose.resolved.yml docs/generated/traefik-routes.md
-python3 scripts/docs/generate-prometheus-rules.py docs/generated/prometheus-rules.md
 python3 scripts/docs/generate-docs-index.py docs/generated/index.md
 python3 scripts/docs/generate-diagrams.py --compose docs/generated/docker-compose.resolved.yml --out-dir docs/diagrams
 python3 scripts/docs/sanitize-public-docs.py docs/generated docs/diagrams docs/public

scripts/docs/generate-diagrams.py

@@ -77,38 +77,67 @@ def infer_host(service_name: str, service: dict) -> str:
     return "docker"
 
 
+def categorize_service(service_name: str) -> str:
+    s = service_name.lower()
+    if any(k in s for k in ["traefik", "authelia", "oauth", "auth", "proxy", "nginx", "caddy"]):
+        return "edge/proxy/auth"
+    if any(k in s for k in ["prometheus", "grafana", "loki", "promtail", "alert", "node-exporter", "cadvisor"]):
+        return "monitoring"
+    if any(k in s for k in ["watchtower", "diun", "ansible", "cron", "runner", "backup"]):
+        return "automation"
+    if any(k in s for k in ["postgres", "mariadb", "mysql", "redis", "minio", "nfs", "storage", "db", "queue"]):
+        return "storage/database/support"
+    return "apps"
+
+
 def generate_physical_topology(compose: dict, out_dot: Path, out_svg: Path) -> None:
     services = compose.get("services") or {}
-    hosts: dict[str, list[str]] = {}
+    hosts: dict[str, dict[str, list[str]]] = {}
     for name, svc in services.items():
-        hosts.setdefault(infer_host(name, svc), []).append(name)
+        host = infer_host(name, svc)
+        cat = categorize_service(name)
+        hosts.setdefault(host, {}).setdefault(cat, []).append(name)
 
     lines = [
         "digraph PhysicalTopology {",
-        "  graph [rankdir=LR, compound=true, splines=ortho, nodesep=0.45, ranksep=0.75, fontname=\"Helvetica\"];",
+        "  graph [rankdir=LR, compound=true, splines=polyline, nodesep=0.7, ranksep=1.2, fontname=\"Helvetica\", concentrate=true];",
         "  node [fontname=\"Helvetica\", fontsize=10, style=\"rounded,filled\", fillcolor=\"#ffffff\"];",
-        "  edge [fontname=\"Helvetica\", fontsize=9];",
+        "  edge [fontname=\"Helvetica\", fontsize=9, color=\"#64748b\"];",
     ]
 
-    for host, host_services in sorted(hosts.items()):
-        lines.extend(
-            [
-                f'  subgraph "cluster_{host}" {{',
-                f'    label="{host}";',
-                '    style="rounded,filled";',
-                '    color="#c7d6f5";',
-                '    fillcolor="#eef3ff";',
-                f'    "host:{host}" [label="{host}", shape=box3d, fillcolor="#d4e3ff"];',
-            ]
-        )
-        for service in sorted(host_services):
-            lines.append(
-                f'    "svc:{service}" [label="{service}", shape=box, fillcolor="#dff2e1"];'
-            )
-            lines.append(f'    "host:{host}" -> "svc:{service}" [style=dashed, color="#6b7280"];')
-        lines.append("  }")
+    for host, cat_map in sorted(hosts.items()):
+        lines.extend([
+            f'  subgraph "cluster_{host}" {{',
+            f'    label="{host} host";',
+            '    style="rounded,filled";',
+            '    color="#93c5fd";',
+            '    fillcolor="#eff6ff";',
+        ])
+        for category, svcs in sorted(cat_map.items()):
+            cluster_id = f"cluster_{host}_{re.sub(r'[^a-zA-Z0-9]+', '_', category)}"
+            lines.extend([
+                f'    subgraph "{cluster_id}" {{',
+                f'      label="{category}";',
+                '      style="rounded,dashed";',
+                '      color="#bfdbfe";',
+                '      fillcolor="#f8fbff";',
+            ])
+            for service in sorted(svcs):
+                lines.append(f'      "svc:{service}" [label="{service}", shape=box, fillcolor="#dcfce7"];')
+            lines.append('    }')
+        lines.append('  }')
 
-    lines.append("}")
+    lines.extend([
+        '  subgraph "cluster_legend" {',
+        '    label="Legend"; style="rounded"; color="#d1d5db";',
+        '    "leg_host" [label="Host cluster", shape=box3d, fillcolor="#eff6ff"];',
+        '    "leg_cat" [label="Service category", shape=folder, fillcolor="#f8fbff"];',
+        '    "leg_svc" [label="Container/service", shape=box, fillcolor="#dcfce7"];',
+        '    "leg_host" -> "leg_cat" [style=dashed];',
+        '    "leg_cat" -> "leg_svc" [style=dashed];',
+        '  }',
+        "}",
+    ])
     write_dot(out_dot, lines)
     render_svg(out_dot, out_svg)
 
@@ -120,62 +149,83 @@ def generate_docker_traefik_dynu(compose: dict, out_dot: Path, out_svg: Path) ->
 
     lines = [
         "digraph DockerTraefikDynu {",
-        "  graph [rankdir=LR, compound=true, splines=true, nodesep=0.5, ranksep=1.0, fontname=\"Helvetica\"];",
+        "  graph [rankdir=LR, compound=true, splines=ortho, nodesep=0.7, ranksep=1.2, fontname=\"Helvetica\", concentrate=true];",
         "  node [fontname=\"Helvetica\", fontsize=10, style=\"rounded,filled\"];",
-        "  edge [fontname=\"Helvetica\", fontsize=9];",
-        '  "ext:dynu" [label="Dynu / Public DNS", shape=oval, fillcolor="#fde68a"];',
-        '  "svc:traefik" [label="traefik", shape=box, fillcolor="#bfdbfe"];',
+        "  edge [fontname=\"Helvetica\", fontsize=9, color=\"#334155\"];",
+        '  "svc:traefik" [label="Traefik\n(entrypoint)", shape=box, fillcolor="#bfdbfe"];',
     ]
 
-    for net in sorted(networks.keys()):
-        lines.append(f'  "net:{net}" [label="{net}", shape=ellipse, fillcolor="#f3f4f6"];')
+    routes: dict[str, dict] = {}
+    dns_nodes: set[str] = set()
 
     for svc_name, svc in sorted(services.items()):
-        lines.append(f'  "svc:{svc_name}" [label="{svc_name}", shape=box, fillcolor="#dcfce7"];')
-        svc_nets = svc.get("networks") or []
-        if isinstance(svc_nets, dict):
-            svc_nets = svc_nets.keys()
-        for net in svc_nets:
-            lines.append(f'  "svc:{svc_name}" -> "net:{net}" [color="#6b7280"];')
-
         labels = parse_labels(svc)
         router_prefix = "traefik.http.routers."
         service_prefix = "traefik.http.services."
+        lb_ports = {}
+        for k, v in labels.items():
+            if k.startswith(service_prefix) and k.endswith(".loadbalancer.server.port"):
+                lb_ports[k[len(service_prefix):].split(".", 1)[0]] = v
 
         routers = sorted({k[len(router_prefix):].split(".", 1)[0] for k in labels if k.startswith(router_prefix)})
         for router in routers:
             rule = labels.get(f"{router_prefix}{router}.rule", "")
-            router_service = labels.get(f"{router_prefix}{router}.service", svc_name)
+            target = labels.get(f"{router_prefix}{router}.service", svc_name)
             middlewares = labels.get(f"{router_prefix}{router}.middlewares", "")
-            entrypoints = labels.get(f"{router_prefix}{router}.entrypoints", "")
             tls = labels.get(f"{router_prefix}{router}.tls", "false")
-            lines.append(f'  "router:{router}" [label="router:{router}\\nentry:{entrypoints} tls:{tls}", shape=diamond, fillcolor="#fbcfe8"];')
-            lines.append(f'  "svc:traefik" -> "router:{router}";')
-            target_node = f"traefik-service:{router_service}"
-            lines.append(f'  "{target_node}" [label="service:{router_service}", shape=component, fillcolor="#fecaca"];')
-            lines.append(f'  "router:{router}" -> "{target_node}";')
+            port = lb_ports.get(target, "")
+            badges = []
+            if str(tls).lower() in ("true", "1"):
+                badges.append("TLS")
+            mw_low = middlewares.lower()
+            if "authelia" in mw_low:
+                badges.append("authelia")
+            if "mtls" in mw_low:
+                badges.append("mTLS")
+            hosts = [sanitize_domain(h, known_domains) for h in extract_hosts(rule)]
+            if not hosts:
+                continue
+            info = routes.setdefault(svc_name, {"hosts": set(), "port": port, "badges": set()})
+            info["hosts"].update(hosts)
+            if port:
+                info["port"] = port
+            info["badges"].update(badges)
 
-            for host in extract_hosts(rule):
-                clean = sanitize_domain(host, known_domains)
-                lines.append(f'  "dns:{clean}" [label="{clean}", shape=note, fillcolor="#fde68a"];')
-                lines.append(f'  "ext:dynu" -> "dns:{clean}";')
-                lines.append(f'  "dns:{clean}" -> "router:{router}";')
+    for svc_name, info in sorted(routes.items()):
+        label = svc_name
+        if info.get("port"):
+            label += f"\n:{info['port']}"
+        if info.get("badges"):
+            label += "\n[" + ", ".join(sorted(info["badges"])) + "]"
+        lines.append(f'  "svc:{svc_name}" [label="{label}", shape=box, fillcolor="#dcfce7"];')
+        lines.append(f'  "svc:traefik" -> "svc:{svc_name}" [penwidth=1.4];')
+        for host in sorted(info["hosts"]):
+            dns_nodes.add(host)
+            lines.append(f'  "dns:{host}" [label="{host}", shape=note, fillcolor="#fef3c7"];')
+            lines.append(f'  "dns:{host}" -> "svc:traefik";')
 
-            for middleware in [m.strip() for m in middlewares.split(",") if m.strip()]:
-                lines.append(f'  "mw:{middleware}" [label="{middleware}", shape=hexagon, fillcolor="#ddd6fe"];')
-                lines.append(f'  "router:{router}" -> "mw:{middleware}" [style=dashed];')
+    lines.append('  { rank=same; ' + '; '.join([f'"dns:{d}"' for d in sorted(dns_nodes)]) + '; }' if dns_nodes else '')
 
-        lb_services = sorted({k[len(service_prefix):].split(".", 1)[0] for k in labels if k.startswith(service_prefix)})
-        for lb in lb_services:
-            port = labels.get(f"{service_prefix}{lb}.loadbalancer.server.port", "")
-            lines.append(f'  "traefik-service:{lb}" [label="service:{lb}\\nport:{port}", shape=component, fillcolor="#fecaca"];')
-            lines.append(f'  "traefik-service:{lb}" -> "svc:{svc_name}";')
+    lines.append('  subgraph "cluster_networks" {')
+    lines.append('    label="Docker backend networks"; style="rounded,dashed"; color="#d1d5db";')
+    for net in sorted(networks.keys()):
+        lines.append(f'    "net:{net}" [label="{net}", shape=ellipse, fillcolor="#f8fafc"];')
+    lines.append('  }')
+
+    for svc_name in sorted(routes.keys()):
+        svc = services.get(svc_name, {})
+        svc_nets = svc.get("networks") or []
+        if isinstance(svc_nets, dict):
+            svc_nets = svc_nets.keys()
+        for net in svc_nets:
+            lines.append(f'  "svc:{svc_name}" -> "net:{net}" [style=dashed, color="#94a3b8", arrowsize=0.7];')
 
     lines.append("}")
     write_dot(out_dot, lines)
     render_svg(out_dot, out_svg)
 
 
+
 def generate_compose_topology(compose: dict, out_dot: Path, out_svg: Path) -> None:
     services = compose.get("services") or {}
     networks = compose.get("networks") or {}

scripts/docs/generate-docs-index.py

@@ -15,7 +15,6 @@ This directory contains documentation generated automatically from repository co
 - [Resolved Docker Compose config](docker-compose.resolved.yml)
 - [Compose inventory](compose-inventory.md)
 - [Traefik routes](traefik-routes.md)
-- [Prometheus rules](prometheus-rules.md)
 - [Docker Compose diagram](../diagrams/docker-compose.svg)
 """
 )

scripts/docs/sanitize-public-docs.py

@@ -21,7 +21,7 @@ def sanitize_text(content: str) -> str:
     return content
 
 
-for name in ['compose-inventory.md', 'traefik-routes.md', 'prometheus-rules.md']:
+for name in ['compose-inventory.md', 'traefik-routes.md']:
     src = src_generated / name
     if src.exists():
         (out_dir / name).write_text(sanitize_text(src.read_text(errors='ignore')))
@@ -36,6 +36,8 @@ for svg_name in ['docker-compose.svg', 'physical-topology.svg', 'docker-traefik-
 
 This documentation is generated from the infrastructure repository. Sensitive values are redacted.
 
+> Generated docs are sanitised/redacted before publishing to GitHub Pages.
+
 ## Infrastructure diagrams
 
 ### Physical / virtual topology
@@ -51,7 +53,6 @@ This documentation is generated from the infrastructure repository. Sensitive va
 - [Diagrams](diagrams.md)
 - [Compose Inventory](compose-inventory.md)
 - [Traefik Routes](traefik-routes.md)
-- [Prometheus Rules](prometheus-rules.md)
 """
 )
 
@@ -60,10 +61,16 @@ This documentation is generated from the infrastructure repository. Sensitive va
 
 ## Physical / virtual topology
 
+This view groups containers by inferred host and service role (edge/proxy/auth, monitoring, automation, apps, and supporting storage/services).
+
 ![Physical topology](physical-topology.svg)
 
 ## Docker, Traefik and Dynu routing
 
+This view shows sanitised public DNS names flowing to Traefik, then to exposed Docker services, with backend Docker network membership shown as secondary context.
+
+_Diagrams are generated from Compose data and Traefik labels._
+
 ![Docker Traefik Dynu](docker-traefik-dynu.svg)
 """
 )