Fix public topology/routing diagram generation and layout

scripts/docs/generate-all.sh

@@ -7,5 +7,16 @@ scripts/docs/render-compose-config.sh
 python3 scripts/docs/generate-compose-inventory.py docs/generated/docker-compose.resolved.yml docs/generated/compose-inventory.md
 python3 scripts/docs/generate-traefik-routes.py docs/generated/docker-compose.resolved.yml docs/generated/traefik-routes.md
 python3 scripts/docs/generate-docs-index.py docs/generated/index.md
-python3 scripts/docs/generate-diagrams.py --compose docs/generated/docker-compose.resolved.yml --out-dir docs/diagrams
+HOST_INVENTORY=""
+for p in data/terraform/proxmox-inventory.json infrastructure/terraform/proxmox/generated/infrastructure_inventory.json; do
+  if [[ -f "$p" ]]; then
+    HOST_INVENTORY="$p"
+    break
+  fi
+done
+DNS_INVENTORY="infrastructure/terraform/dynu/generated/dynu_dns_records_inventory.json"
+GEN_ARGS=(--compose docs/generated/docker-compose.resolved.yml --out-dir docs/diagrams --domain-display redacted-label)
+[[ -n "$HOST_INVENTORY" ]] && GEN_ARGS+=(--host-inventory "$HOST_INVENTORY")
+[[ -f "$DNS_INVENTORY" ]] && GEN_ARGS+=(--dns-inventory "$DNS_INVENTORY")
+python3 scripts/docs/generate-diagrams.py "${GEN_ARGS[@]}"
 python3 scripts/docs/sanitize-public-docs.py docs/generated docs/diagrams docs/public

scripts/docs/generate-diagrams.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import argparse
+import json
 import re
 import subprocess
 import shutil
@@ -23,13 +24,17 @@ def load_compose(path: Path) -> dict:
         return yaml.safe_load(handle) or {}
 
 
-def sanitize_domain(value: str, known: dict[str, str]) -> str:
-    if value in known:
-        return known[value]
+PARENT_KEYS = ("node", "node_name", "host", "physical_host", "hypervisor_host", "proxmox_node")
+
+
+def display_domain(value: str, mode: str) -> str:
+    if mode == "full":
+        return value
+    if mode == "placeholder":
+        return "<internal-domain>" if INTERNAL_DOMAIN_RE.search(value) else value
     if INTERNAL_DOMAIN_RE.search(value):
-        mapped = f"service-{len(known)+1}.<internal-domain>"
-        known[value] = mapped
-        return mapped
+        label = re.sub(r"\.lan\.ddnsgeek\.com$", "", value)
+        return f"{label}.<domain>"
     return value
 
 
@@ -90,51 +95,109 @@ def categorize_service(service_name: str) -> str:
     return "apps"
 
 
-def generate_physical_topology(compose: dict, out_dot: Path, out_svg: Path) -> None:
-    services = compose.get("services") or {}
-    hosts: dict[str, dict[str, list[str]]] = {}
-    for name, svc in services.items():
-        host = infer_host(name, svc)
-        cat = categorize_service(name)
-        hosts.setdefault(host, {}).setdefault(cat, []).append(name)
+def load_inventory(path: Path | None) -> dict:
+    if not path or not path.exists():
+        return {}
+    payload = json.loads(path.read_text())
+    return payload.get("value", payload) if isinstance(payload, dict) else {}
+
+
+def to_records(data) -> list[dict]:
+    if not isinstance(data, dict):
+        return []
+    out = []
+    for key, value in data.items():
+        if isinstance(value, dict):
+            rec = dict(value)
+            rec.setdefault("_key", str(key))
+            rec.setdefault("name", rec.get("hostname") or rec.get("vm_name") or str(key))
+            out.append(rec)
+    return out
+
+
+def parent_name(item: dict) -> str:
+    for k in PARENT_KEYS:
+        v = item.get(k)
+        if v:
+            return str(v)
+    return ""
+
+
+def generate_physical_topology(compose: dict, inventory: dict, out_dot: Path, out_svg: Path) -> None:
+    physical = to_records(inventory.get("physical_hosts", {}))
+    virtual = to_records(inventory.get("virtual_hosts", {})) + to_records(inventory.get("vms", {}))
+    if not physical and not virtual:
+        lines = [
+            "digraph PhysicalTopology {",
+            "  graph [rankdir=LR, fontname=\"Helvetica\", nodesep=1.0, ranksep=1.5];",
+            '  "placeholder:inventory" [shape=note, style="filled", fillcolor="#fef3c7", label="Host inventory JSON not found.\\nGenerate terraform inventory and rerun scripts/docs/generate-all.sh\\n(--host-inventory <path>)."];',
+        ]
+        lines.append("}")
+        write_dot(out_dot, lines)
+        render_svg(out_dot, out_svg)
+        return
 
     lines = [
         "digraph PhysicalTopology {",
-        "  graph [rankdir=LR, compound=true, splines=polyline, nodesep=0.7, ranksep=1.2, fontname=\"Helvetica\", concentrate=true];",
-        "  node [fontname=\"Helvetica\", fontsize=10, style=\"rounded,filled\", fillcolor=\"#ffffff\"];",
-        "  edge [fontname=\"Helvetica\", fontsize=9, color=\"#64748b\"];",
+        "  graph [rankdir=LR, compound=true, splines=polyline, nodesep=0.95, ranksep=1.7, ratio=compress, fontname=\"Helvetica\", fontsize=13, concentrate=true, newrank=true];",
+        "  node [fontname=\"Helvetica\", fontsize=12, style=\"rounded,filled\", fillcolor=\"#ffffff\"];",
+        "  edge [fontname=\"Helvetica\", fontsize=10, color=\"#64748b\"];",
     ]
-
-    for host, cat_map in sorted(hosts.items()):
+    phys_names = {str(p.get("name")): p for p in physical}
+    children: dict[str, list[dict]] = {k: [] for k in phys_names}
+    orphans: list[dict] = []
+    for vm in virtual:
+        parent = parent_name(vm)
+        if parent in children:
+            children[parent].append(vm)
+        else:
+            orphans.append(vm)
+    for host, record in sorted(phys_names.items()):
+        host_role = record.get("role", "")
         lines.extend([
             f'  subgraph "cluster_{host}" {{',
-            f'    label="{host} host";',
+            f'    label="{host}\\n{host_role}".strip();',
             '    style="rounded,filled";',
-            '    color="#93c5fd";',
+            '    color="#60a5fa";',
             '    fillcolor="#eff6ff";',
+            f'    "phys:{host}" [label="{host}", shape=box3d, fillcolor="#bfdbfe"];',
         ])
-        for category, svcs in sorted(cat_map.items()):
-            cluster_id = f"cluster_{host}_{re.sub(r'[^a-zA-Z0-9]+', '_', category)}"
+        for vm in sorted(children.get(host, []), key=lambda x: str(x.get("name", "")).lower()):
+            vm_name = str(vm.get("name"))
+            vm_role = str(vm.get("role", "") or "virtual host")
+            cluster_id = f"cluster_{host}_{re.sub(r'[^a-zA-Z0-9]+', '_', vm_name)}"
             lines.extend([
                 f'    subgraph "{cluster_id}" {{',
-                f'      label="{category}";',
+                f'      label="{vm_name}";',
                 '      style="rounded,dashed";',
                 '      color="#bfdbfe";',
                 '      fillcolor="#f8fbff";',
+                f'      "vm:{vm_name}" [label="{vm_name}\\n{vm_role}", shape=component, fillcolor="#dcfce7"];',
             ])
-            for service in sorted(svcs):
-                lines.append(f'      "svc:{service}" [label="{service}", shape=box, fillcolor="#dcfce7"];')
+            if "docker" in vm_role.lower() or "docker" in vm_name.lower():
+                lines.append(f'      "role:{vm_name}" [label="Docker host", shape=box, fillcolor="#fef3c7"];')
+                lines.append(f'      "vm:{vm_name}" -> "role:{vm_name}" [style=dashed, label="runs"];')
             lines.append('    }')
+            lines.append(f'    "phys:{host}" -> "vm:{vm_name}" [label="hosts"];')
         lines.append('  }')
+    if orphans:
+        lines.extend([
+            '  subgraph "cluster_orphans" {',
+            '    label="Unmapped virtual hosts"; style="rounded,dashed"; color="#d1d5db";',
+        ])
+        for vm in sorted(orphans, key=lambda x: str(x.get("name", "")).lower()):
+            vm_name = str(vm.get("name"))
+            lines.append(f'    "vm:{vm_name}" [label="{vm_name}", shape=component, fillcolor="#fee2e2"];')
+        lines.append("  }")
 
     lines.extend([
         '  subgraph "cluster_legend" {',
         '    label="Legend"; style="rounded"; color="#d1d5db";',
-        '    "leg_host" [label="Host cluster", shape=box3d, fillcolor="#eff6ff"];',
-        '    "leg_cat" [label="Service category", shape=folder, fillcolor="#f8fbff"];',
-        '    "leg_svc" [label="Container/service", shape=box, fillcolor="#dcfce7"];',
-        '    "leg_host" -> "leg_cat" [style=dashed];',
-        '    "leg_cat" -> "leg_svc" [style=dashed];',
+        '    "leg_host" [label="Physical host", shape=box3d, fillcolor="#eff6ff"];',
+        '    "leg_vm" [label="Virtual machine", shape=component, fillcolor="#dcfce7"];',
+        '    "leg_role" [label="Hosted role", shape=box, fillcolor="#fef3c7"];',
+        '    "leg_host" -> "leg_vm" [label="hosts"];',
+        '    "leg_vm" -> "leg_role" [style=dashed, label="runs"];',
         '  }',
         "}",
     ])
@@ -142,17 +205,17 @@ def generate_physical_topology(compose: dict, out_dot: Path, out_svg: Path) -> N
     render_svg(out_dot, out_svg)
 
 
-def generate_docker_traefik_dynu(compose: dict, out_dot: Path, out_svg: Path) -> None:
+def generate_docker_traefik_dynu(compose: dict, dns_inventory: dict, domain_mode: str, out_dot: Path, out_svg: Path) -> None:
     services = compose.get("services") or {}
     networks = compose.get("networks") or {}
-    known_domains: dict[str, str] = {}
-
     lines = [
         "digraph DockerTraefikDynu {",
-        "  graph [rankdir=LR, compound=true, splines=ortho, nodesep=0.7, ranksep=1.2, fontname=\"Helvetica\", concentrate=true];",
-        "  node [fontname=\"Helvetica\", fontsize=10, style=\"rounded,filled\"];",
+        "  graph [rankdir=LR, compound=true, splines=polyline, nodesep=0.9, ranksep=1.6, fontname=\"Helvetica\", concentrate=true, newrank=true];",
+        "  node [fontname=\"Helvetica\", fontsize=11, style=\"rounded,filled\"];",
         "  edge [fontname=\"Helvetica\", fontsize=9, color=\"#334155\"];",
-        '  "svc:traefik" [label="Traefik\n(entrypoint)", shape=box, fillcolor="#bfdbfe"];',
+        '  "dynu" [label="Dynu / Public DNS", shape=box, fillcolor="#fde68a"];',
+        '  "svc:traefik" [label="Traefik", shape=box, fillcolor="#bfdbfe"];',
+        '  "dynu" -> "svc:traefik" [penwidth=1.6];',
     ]
 
     routes: dict[str, dict] = {}
@@ -182,7 +245,7 @@ def generate_docker_traefik_dynu(compose: dict, out_dot: Path, out_svg: Path) ->
                 badges.append("authelia")
             if "mtls" in mw_low:
                 badges.append("mTLS")
-            hosts = [sanitize_domain(h, known_domains) for h in extract_hosts(rule)]
+            hosts = [display_domain(h, domain_mode) for h in extract_hosts(rule)]
             if not hosts:
                 continue
             info = routes.setdefault(svc_name, {"hosts": set(), "port": port, "badges": set()})
@@ -202,7 +265,17 @@ def generate_docker_traefik_dynu(compose: dict, out_dot: Path, out_svg: Path) ->
         for host in sorted(info["hosts"]):
             dns_nodes.add(host)
             lines.append(f'  "dns:{host}" [label="{host}", shape=note, fillcolor="#fef3c7"];')
-            lines.append(f'  "dns:{host}" -> "svc:traefik";')
+            lines.append(f'  "dns:{host}" -> "dynu";')
+    for record in (dns_inventory.get("records", []) if isinstance(dns_inventory, dict) else []):
+        host = record.get("hostname") or record.get("name")
+        if not host:
+            continue
+        host_disp = display_domain(str(host), domain_mode)
+        if host_disp in dns_nodes:
+            continue
+        dns_nodes.add(host_disp)
+        lines.append(f'  "dns:{host_disp}" [label="{host_disp}", shape=note, fillcolor="#fef3c7"];')
+        lines.append(f'  "dns:{host_disp}" -> "dynu" [style=dashed, color=\"#94a3b8\"];')
 
     lines.append('  { rank=same; ' + '; '.join([f'"dns:{d}"' for d in sorted(dns_nodes)]) + '; }' if dns_nodes else '')
 
@@ -256,6 +329,9 @@ def main() -> None:
     parser.add_argument("legacy", nargs="*")
     parser.add_argument("--compose")
     parser.add_argument("--out-dir")
+    parser.add_argument("--host-inventory")
+    parser.add_argument("--dns-inventory")
+    parser.add_argument("--domain-display", choices=["full", "redacted-label", "placeholder"], default="redacted-label")
     args = parser.parse_args()
 
     require_dot()
@@ -271,9 +347,11 @@ def main() -> None:
 
     out_dir.mkdir(parents=True, exist_ok=True)
     compose = load_compose(compose_path)
+    host_inventory = load_inventory(Path(args.host_inventory)) if args.host_inventory else {}
+    dns_inventory = load_inventory(Path(args.dns_inventory)) if args.dns_inventory else {}
 
-    generate_docker_traefik_dynu(compose, out_dir / "docker-traefik-dynu.dot", out_dir / "docker-traefik-dynu.svg")
-    generate_physical_topology(compose, out_dir / "physical-topology.dot", out_dir / "physical-topology.svg")
+    generate_docker_traefik_dynu(compose, dns_inventory, args.domain_display, out_dir / "docker-traefik-dynu.dot", out_dir / "docker-traefik-dynu.svg")
+    generate_physical_topology(compose, host_inventory, out_dir / "physical-topology.dot", out_dir / "physical-topology.svg")
     generate_compose_topology(compose, out_dir / "docker-compose.dot", out_dir / "docker-compose.svg")
 
 

scripts/docs/sanitize-public-docs.py

@@ -10,7 +10,7 @@ out_dir.mkdir(parents=True, exist_ok=True)
 
 
 def sanitize_text(content: str) -> str:
-    content = re.sub(r'\b[a-zA-Z0-9.-]+\.lan\.ddnsgeek\.com\b', '<internal-domain>', content)
+    content = re.sub(r'\b([a-zA-Z0-9-]+)\.lan\.ddnsgeek\.com\b', r'\1.<domain>', content)
     content = re.sub(
         r'\b(?:10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(?:1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})\b',
         '<private-ip>',
@@ -63,7 +63,9 @@ This documentation is generated from the infrastructure repository. Sensitive va
 
 This view groups containers by inferred host and service role (edge/proxy/auth, monitoring, automation, apps, and supporting storage/services).
 
-![Physical topology](physical-topology.svg)
+<div class="diagram-wrap">
+  <img src="physical-topology.svg" alt="Physical topology">
+</div>
 
 ## Docker, Traefik and Dynu routing
 
@@ -71,6 +73,8 @@ This view shows sanitised public DNS names flowing to Traefik, then to exposed D
 
 _Diagrams are generated from Compose data and Traefik labels._
 
-![Docker Traefik Dynu](docker-traefik-dynu.svg)
+<div class="diagram-wrap">
+  <img src="docker-traefik-dynu.svg" alt="Docker Traefik Dynu">
+</div>
 """
 )