Improve docs pipeline with generated topology diagrams
This commit is contained in:
beatz174-bit
@@ -8,5 +8,5 @@ python3 scripts/docs/generate-compose-inventory.py docs/generated/docker-compose
|
||||
python3 scripts/docs/generate-traefik-routes.py docs/generated/docker-compose.resolved.yml docs/generated/traefik-routes.md
|
||||
python3 scripts/docs/generate-prometheus-rules.py docs/generated/prometheus-rules.md
|
||||
python3 scripts/docs/generate-docs-index.py docs/generated/index.md
|
||||
python3 scripts/docs/generate-diagrams.py docs/generated/docker-compose.resolved.yml docs/diagrams/docker-compose.dot docs/diagrams/docker-compose.svg
|
||||
python3 scripts/docs/generate-diagrams.py --compose docs/generated/docker-compose.resolved.yml --out-dir docs/diagrams
|
||||
python3 scripts/docs/sanitize-public-docs.py docs/generated docs/diagrams docs/public
|
||||
|
||||
@@ -1,18 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys,yaml,subprocess,shutil
|
||||
inp,dotf,svgf=sys.argv[1],sys.argv[2],sys.argv[3]
|
||||
with open(inp) as f:c=yaml.safe_load(f) or {}
|
||||
svcs=c.get('services') or {}
|
||||
lines=["digraph Compose {"," rankdir=LR;"," node [fontname=Helvetica];"]
|
||||
for s in svcs: lines.append(f' "svc:{s}" [label="{s}", shape=box, style=filled, fillcolor="#dfefff"];')
|
||||
for n in (c.get('networks') or {}).keys(): lines.append(f' "net:{n}" [label="{n}", shape=ellipse, style=filled, fillcolor="#f4f4f4"];')
|
||||
for s,sv in svcs.items():
|
||||
ns=sv.get('networks') or []
|
||||
if isinstance(ns,dict): ns=ns.keys()
|
||||
for n in ns: lines.append(f' "svc:{s}" -> "net:{n}";')
|
||||
lines.append("}")
|
||||
open(dotf,'w').write('\n'.join(lines)+'\n')
|
||||
if shutil.which('dot'):
|
||||
subprocess.run(['dot','-Tsvg',dotf,'-o',svgf],check=True)
|
||||
else:
|
||||
open(svgf,'w').write('<svg xmlns="http://www.w3.org/2000/svg" width="640" height="80"><text x="10" y="40">Graphviz dot not found in environment.</text></svg>\n')
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
import yaml
|
||||
|
||||
INTERNAL_DOMAIN_RE = re.compile(r"\b[a-zA-Z0-9.-]+\.lan\.ddnsgeek\.com\b")
|
||||
HOST_MATCH_RE = re.compile(r"Host\(([^)]*)\)")
|
||||
TICKED_HOST_RE = re.compile(r"`([^`]+)`")
|
||||
|
||||
|
||||
def require_dot() -> None:
|
||||
if not shutil.which("dot"):
|
||||
raise SystemExit(
|
||||
"Graphviz 'dot' not found in environment. Install graphviz before running docs generation."
|
||||
)
|
||||
|
||||
|
||||
def load_compose(path: Path) -> dict:
|
||||
with path.open() as handle:
|
||||
return yaml.safe_load(handle) or {}
|
||||
|
||||
|
||||
def sanitize_domain(value: str, known: dict[str, str]) -> str:
|
||||
if value in known:
|
||||
return known[value]
|
||||
if INTERNAL_DOMAIN_RE.search(value):
|
||||
mapped = f"service-{len(known)+1}.<internal-domain>"
|
||||
known[value] = mapped
|
||||
return mapped
|
||||
return value
|
||||
|
||||
|
||||
def parse_labels(service: dict) -> dict[str, str]:
|
||||
labels = service.get("labels") or {}
|
||||
if isinstance(labels, list):
|
||||
mapped = {}
|
||||
for item in labels:
|
||||
if "=" in str(item):
|
||||
k, v = str(item).split("=", 1)
|
||||
mapped[k] = v
|
||||
return mapped
|
||||
if isinstance(labels, dict):
|
||||
return {str(k): str(v) for k, v in labels.items()}
|
||||
return {}
|
||||
|
||||
|
||||
def extract_hosts(rule: str) -> list[str]:
|
||||
hosts: list[str] = []
|
||||
for m in HOST_MATCH_RE.findall(rule or ""):
|
||||
for host in TICKED_HOST_RE.findall(m):
|
||||
hosts.append(host.strip())
|
||||
return hosts
|
||||
|
||||
|
||||
def render_svg(dot_path: Path, svg_path: Path) -> None:
|
||||
subprocess.run(["dot", "-Tsvg", str(dot_path), "-o", str(svg_path)], check=True)
|
||||
|
||||
|
||||
def write_dot(path: Path, lines: list[str]) -> None:
|
||||
path.write_text("\n".join(lines) + "\n")
|
||||
|
||||
|
||||
def infer_host(service_name: str, service: dict) -> str:
|
||||
labels = parse_labels(service)
|
||||
for key in ("com.docker.compose.project", "infra.host", "host"):
|
||||
value = labels.get(key)
|
||||
if value:
|
||||
return value.lower()
|
||||
s = service_name.lower()
|
||||
if "raspi" in s or "pi" in s:
|
||||
return "raspberrypi"
|
||||
if "proxmox" in s:
|
||||
return "proxmox"
|
||||
return "docker"
|
||||
|
||||
|
||||
def generate_physical_topology(compose: dict, out_dot: Path, out_svg: Path) -> None:
|
||||
services = compose.get("services") or {}
|
||||
hosts: dict[str, list[str]] = {}
|
||||
for name, svc in services.items():
|
||||
hosts.setdefault(infer_host(name, svc), []).append(name)
|
||||
|
||||
lines = [
|
||||
"digraph PhysicalTopology {",
|
||||
" graph [rankdir=LR, compound=true, splines=ortho, nodesep=0.45, ranksep=0.75, fontname=\"Helvetica\"];",
|
||||
" node [fontname=\"Helvetica\", fontsize=10, style=\"rounded,filled\", fillcolor=\"#ffffff\"];",
|
||||
" edge [fontname=\"Helvetica\", fontsize=9];",
|
||||
]
|
||||
|
||||
for host, host_services in sorted(hosts.items()):
|
||||
lines.extend(
|
||||
[
|
||||
f' subgraph "cluster_{host}" {{',
|
||||
f' label="{host}";',
|
||||
' style="rounded,filled";',
|
||||
' color="#c7d6f5";',
|
||||
' fillcolor="#eef3ff";',
|
||||
f' "host:{host}" [label="{host}", shape=box3d, fillcolor="#d4e3ff"];',
|
||||
]
|
||||
)
|
||||
for service in sorted(host_services):
|
||||
lines.append(
|
||||
f' "svc:{service}" [label="{service}", shape=box, fillcolor="#dff2e1"];'
|
||||
)
|
||||
lines.append(f' "host:{host}" -> "svc:{service}" [style=dashed, color="#6b7280"];')
|
||||
lines.append(" }")
|
||||
|
||||
lines.append("}")
|
||||
write_dot(out_dot, lines)
|
||||
render_svg(out_dot, out_svg)
|
||||
|
||||
|
||||
def generate_docker_traefik_dynu(compose: dict, out_dot: Path, out_svg: Path) -> None:
|
||||
services = compose.get("services") or {}
|
||||
networks = compose.get("networks") or {}
|
||||
known_domains: dict[str, str] = {}
|
||||
|
||||
lines = [
|
||||
"digraph DockerTraefikDynu {",
|
||||
" graph [rankdir=LR, compound=true, splines=ortho, nodesep=0.5, ranksep=1.0, fontname=\"Helvetica\"];",
|
||||
" node [fontname=\"Helvetica\", fontsize=10, style=\"rounded,filled\"];",
|
||||
" edge [fontname=\"Helvetica\", fontsize=9];",
|
||||
' "ext:dynu" [label="Dynu / Public DNS", shape=oval, fillcolor="#fde68a"];',
|
||||
' "svc:traefik" [label="traefik", shape=box, fillcolor="#bfdbfe"];',
|
||||
]
|
||||
|
||||
for net in sorted(networks.keys()):
|
||||
lines.append(f' "net:{net}" [label="{net}", shape=ellipse, fillcolor="#f3f4f6"];')
|
||||
|
||||
for svc_name, svc in sorted(services.items()):
|
||||
lines.append(f' "svc:{svc_name}" [label="{svc_name}", shape=box, fillcolor="#dcfce7"];')
|
||||
svc_nets = svc.get("networks") or []
|
||||
if isinstance(svc_nets, dict):
|
||||
svc_nets = svc_nets.keys()
|
||||
for net in svc_nets:
|
||||
lines.append(f' "svc:{svc_name}" -> "net:{net}" [color="#6b7280"];')
|
||||
|
||||
labels = parse_labels(svc)
|
||||
router_prefix = "traefik.http.routers."
|
||||
service_prefix = "traefik.http.services."
|
||||
|
||||
routers = sorted({k[len(router_prefix):].split(".", 1)[0] for k in labels if k.startswith(router_prefix)})
|
||||
for router in routers:
|
||||
rule = labels.get(f"{router_prefix}{router}.rule", "")
|
||||
router_service = labels.get(f"{router_prefix}{router}.service", svc_name)
|
||||
middlewares = labels.get(f"{router_prefix}{router}.middlewares", "")
|
||||
entrypoints = labels.get(f"{router_prefix}{router}.entrypoints", "")
|
||||
tls = labels.get(f"{router_prefix}{router}.tls", "false")
|
||||
lines.append(f' "router:{router}" [label="router:{router}\\nentry:{entrypoints} tls:{tls}", shape=diamond, fillcolor="#fbcfe8"];')
|
||||
lines.append(f' "svc:traefik" -> "router:{router}";')
|
||||
lines.append(f' "router:{router}" -> "svc:{svc_name}" [label="service:{router_service}"];')
|
||||
|
||||
for host in extract_hosts(rule):
|
||||
clean = sanitize_domain(host, known_domains)
|
||||
lines.append(f' "dns:{clean}" [label="{clean}", shape=note, fillcolor="#fde68a"];')
|
||||
lines.append(f' "ext:dynu" -> "dns:{clean}";')
|
||||
lines.append(f' "dns:{clean}" -> "router:{router}";')
|
||||
|
||||
for middleware in [m.strip() for m in middlewares.split(",") if m.strip()]:
|
||||
lines.append(f' "mw:{middleware}" [label="{middleware}", shape=hexagon, fillcolor="#ddd6fe"];')
|
||||
lines.append(f' "router:{router}" -> "mw:{middleware}" [style=dashed];')
|
||||
|
||||
lb_services = sorted({k[len(service_prefix):].split(".", 1)[0] for k in labels if k.startswith(service_prefix)})
|
||||
for lb in lb_services:
|
||||
port = labels.get(f"{service_prefix}{lb}.loadbalancer.server.port", "")
|
||||
lines.append(f' "lb:{svc_name}:{lb}" [label="lb:{lb}\\nport:{port}", shape=component, fillcolor="#fecaca"];')
|
||||
lines.append(f' "lb:{svc_name}:{lb}" -> "svc:{svc_name}";')
|
||||
|
||||
lines.append("}")
|
||||
write_dot(out_dot, lines)
|
||||
render_svg(out_dot, out_svg)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("legacy", nargs="*")
|
||||
parser.add_argument("--compose")
|
||||
parser.add_argument("--out-dir")
|
||||
args = parser.parse_args()
|
||||
|
||||
require_dot()
|
||||
|
||||
if args.compose and args.out_dir:
|
||||
compose_path = Path(args.compose)
|
||||
out_dir = Path(args.out_dir)
|
||||
elif len(args.legacy) == 3:
|
||||
compose_path = Path(args.legacy[0])
|
||||
out_dir = Path(args.legacy[1]).parent
|
||||
else:
|
||||
raise SystemExit("Usage: generate-diagrams.py --compose <compose.yml> --out-dir <dir>")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
compose = load_compose(compose_path)
|
||||
|
||||
generate_docker_traefik_dynu(compose, out_dir / "docker-traefik-dynu.dot", out_dir / "docker-traefik-dynu.svg")
|
||||
generate_physical_topology(compose, out_dir / "physical-topology.dot", out_dir / "physical-topology.svg")
|
||||
generate_docker_traefik_dynu(compose, out_dir / "docker-compose.dot", out_dir / "docker-compose.svg")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -20,27 +20,50 @@ def sanitize_text(content: str) -> str:
|
||||
content = re.sub(r'(?m)^([A-Z0-9_]*(?:PASSWORD|TOKEN|API_KEY|SECRET)[A-Z0-9_]*)\s*[:=]\s*.*$', r'\1=<redacted>', content)
|
||||
return content
|
||||
|
||||
|
||||
for name in ['compose-inventory.md', 'traefik-routes.md', 'prometheus-rules.md']:
|
||||
src = src_generated / name
|
||||
if src.exists():
|
||||
(out_dir / name).write_text(sanitize_text(src.read_text(errors='ignore')))
|
||||
|
||||
svg_src = src_diagrams / 'docker-compose.svg'
|
||||
if svg_src.exists():
|
||||
(out_dir / 'docker-compose.svg').write_text(sanitize_text(svg_src.read_text(errors='ignore')))
|
||||
for svg_name in ['docker-compose.svg', 'physical-topology.svg', 'docker-traefik-dynu.svg']:
|
||||
src = src_diagrams / svg_name
|
||||
if src.exists():
|
||||
(out_dir / svg_name).write_text(sanitize_text(src.read_text(errors='ignore')))
|
||||
|
||||
(out_dir / 'index.md').write_text(
|
||||
"""# Public Infrastructure Summary
|
||||
|
||||
This folder contains sanitized documentation generated from the infrastructure repository.
|
||||
This documentation is generated from the infrastructure repository. Sensitive values are redacted.
|
||||
|
||||
Sensitive values such as internal domain names, private IP addresses, tokens, passwords, and secrets are redacted.
|
||||
## Infrastructure diagrams
|
||||
|
||||
### Physical / virtual topology
|
||||
|
||||
|
||||
|
||||
### Docker, Traefik and Dynu routing
|
||||
|
||||
|
||||
|
||||
## Documents
|
||||
|
||||
- [Diagrams](diagrams.md)
|
||||
- [Compose Inventory](compose-inventory.md)
|
||||
- [Traefik Routes](traefik-routes.md)
|
||||
- [Prometheus Rules](prometheus-rules.md)
|
||||
- [Docker Compose Diagram](docker-compose.svg)
|
||||
"""
|
||||
)
|
||||
|
||||
(out_dir / 'diagrams.md').write_text(
|
||||
"""# Infrastructure diagrams
|
||||
|
||||
## Physical / virtual topology
|
||||
|
||||
|
||||
|
||||
## Docker, Traefik and Dynu routing
|
||||
|
||||
|
||||
"""
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user