diff --git a/README.md b/README.md index fe83d78..27791d1 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,21 @@ flowchart TB For request-flow and network detail, see [docs/architecture.md](docs/architecture.md). +### Regenerating architecture docs (Prometheus + Dynu DNS) + +```bash +# Refresh Dynu live inventory and generated resources/import helpers +cd infrastructure/terraform/dynu +terraform apply -refresh-only +python3 scripts/generate-brownfield-records.py --overwrite + +# Regenerate architecture docs from Prometheus + Dynu inventory +cd ../../.. +python3 scripts/render_prometheus_docs.py \ + --inventory-file docs/runtime/prometheus-inventory.json \ + --dynu-dns-inventory-file infrastructure/terraform/dynu/generated/dynu_dns_records_inventory.json +``` + --- ## Codex setup and maintenance scripts diff --git a/scripts/render_prometheus_docs.py b/scripts/render_prometheus_docs.py index 3ce01f9..b158c0d 100755 --- a/scripts/render_prometheus_docs.py +++ b/scripts/render_prometheus_docs.py @@ -26,6 +26,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--diagrams-dir", default="docs/diagrams", help="Diagram output directory.") parser.add_argument("--readme-file", default="README.md", help="README path for regeneration notes.") parser.add_argument("--architecture-file", default="docs/architecture.md", help="Architecture markdown path.") + parser.add_argument( + "--dynu-dns-inventory-file", + default="infrastructure/terraform/dynu/generated/dynu_dns_records_inventory.json", + help="Path to Dynu DNS brownfield inventory JSON.", + ) + parser.add_argument("--skip-dynu-dns", action="store_true", help="Skip Dynu DNS inventory loading/rendering.") parser.add_argument("--network-file", default="docs/network.md", help="Network markdown path.") parser.add_argument("--coverage-file", default="docs/monitoring-coverage.md", help="Coverage markdown path.") parser.add_argument("--dry-run", action="store_true", help="Print changes instead of writing files.") @@ -41,6 +47,109 @@ def load_json(path: Path) -> dict[str, Any]: return data +def load_optional_json(path: Path) -> Any | None: + if not path.exists(): + return None + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def normalize_dynu_dns_records(payload: Any) -> list[dict[str, Any]]: + records_payload = payload + if isinstance(payload, dict): + if isinstance(payload.get("records"), list): + records_payload = payload["records"] + elif isinstance(payload.get("value"), list): + records_payload = payload["value"] + if not isinstance(records_payload, list): + raise ValueError("Dynu DNS inventory must be a list or object with list field `records`/`value`.") + + normalized: list[dict[str, Any]] = [] + for record in records_payload: + if not isinstance(record, dict): + continue + normalized.append( + { + "id": record.get("id"), + "domain_id": record.get("domain_id"), + "domain_name": str(record.get("domain_name") or "unknown"), + "hostname": str(record.get("hostname") or "unknown"), + "node_name": str(record.get("node_name") or "unknown"), + "record_type": str(record.get("record_type") or "unknown"), + "content": record.get("content"), + "state": record.get("state") if "state" in record else record.get("enabled"), + "ttl": record.get("ttl"), + "updated_on": record.get("updated_on"), + } + ) + normalized.sort( + key=lambda r: ( + str(r.get("domain_name") or ""), + str(r.get("hostname") or ""), + str(r.get("record_type") or ""), + str(r.get("id") or ""), + ) + ) + return normalized + + +def render_dynu_dns_architecture_section(records: list[dict[str, Any]], inventory_path: Path | None) -> str: + domains = sorted({r["domain_name"] for r in records if r["domain_name"] != "unknown"}) + dynamic_count = 0 + static_count = 0 + disabled_count = 0 + rows: list[list[str]] = [] + for record in records: + rtype = str(record.get("record_type") or "").upper() + raw_content = record.get("content") + content = "" if raw_content is None else str(raw_content).strip() + is_dynamic = rtype in {"A", "AAAA"} and not content + mode = "dynamic" if is_dynamic else ("static" if content else ("record" if rtype not in {"A", "AAAA"} else "static")) + target = "dynamic" if is_dynamic else (content or ("unknown" if not content else content)) + if is_dynamic: + dynamic_count += 1 + else: + static_count += 1 + + enabled_value = record.get("state") + enabled_text = "unknown" if enabled_value is None else str(bool(enabled_value)).lower() + if enabled_value is False: + disabled_count += 1 + + rows.append( + [ + str(record.get("hostname") or "unknown"), + rtype or "unknown", + target, + mode, + str(record.get("ttl") if record.get("ttl") is not None else "unknown"), + enabled_text, + str(record.get("id") if record.get("id") is not None else "unknown"), + str(record.get("updated_on") or "unknown"), + ] + ) + + domain_text = ", ".join(domains) if domains else "unknown" + lines = [ + "### Dynu DNS brownfield inventory", + "", + "Dynu DNS is managed as a brownfield reconciliation source. Terraform imports the root domain and individual DNS records into state, while generated configuration provides reviewable management intent.", + "", + f"- Inventory source: `{inventory_path}`" if inventory_path else "- Inventory source: `unknown`", + f"- Records observed: `{len(records)}`", + f"- Domains observed: `{domain_text}`", + f"- Dynamic A/AAAA records: `{dynamic_count}`", + f"- Static records: `{static_count}`", + f"- Disabled records: `{disabled_count}`", + "", + "#### DNS records", + "", + markdown_table(["hostname", "type", "target/content", "mode", "ttl", "enabled", "record id", "updated"], rows or [["none", "", "", "", "", "", "", ""]]), + "", + ] + return "\n".join(lines) + + def merged_labels(target: dict[str, Any]) -> dict[str, str]: discovered = target.get("discovered_labels") or {} labels = target.get("labels") or {} @@ -244,12 +353,18 @@ def render_network_doc(inventory: dict[str, Any], targets: list[dict[str, Any]]) return "\n".join(lines) -def render_architecture_section(inventory: dict[str, Any], targets: list[dict[str, Any]]) -> str: +def render_architecture_section( + inventory: dict[str, Any], + targets: list[dict[str, Any]], + dynu_records: list[dict[str, Any]] | None = None, + dynu_inventory_path: Path | None = None, + dynu_inventory_missing: bool = False, +) -> str: summaries = summarize_targets(targets, normalize_targets({"targets": inventory.get("unhealthy_targets") or []})) notes = inventory.get("notes") or [] lines = [ - "## Runtime visibility from Prometheus", + "## Runtime and infrastructure inventory", "", GENERATED_BEGIN, "", @@ -267,12 +382,42 @@ def render_architecture_section(inventory: dict[str, Any], targets: list[dict[st [[job, str(data["active"]), str(data["unhealthy"])] for job, data in summaries["by_job"].items()] or [["none", "0", "0"]], ), "", - "### Data sources", - "", - "- `docs/runtime/prometheus-inventory.json` (normalized runtime export)", - "- Prometheus scrape metadata (`targets` + label sets)", - "- Existing repository architecture docs for declared topology", ] + if dynu_records is not None: + lines.extend(["", render_dynu_dns_architecture_section(dynu_records, dynu_inventory_path).rstrip(), ""]) + elif dynu_inventory_missing: + lines.extend( + [ + "### Dynu DNS brownfield inventory", + "", + f"Dynu DNS inventory was not found at `{dynu_inventory_path}`.", + "", + "Generate it with:", + "", + "```bash", + "cd infrastructure/terraform/dynu", + "python3 scripts/generate-brownfield-records.py --overwrite", + "```", + "", + ] + ) + lines.extend( + [ + "### Data sources", + "", + "- `docs/runtime/prometheus-inventory.json` (normalized runtime export)", + ] + ) + if dynu_records is not None: + lines.append(f"- `{dynu_inventory_path}` (Dynu DNS brownfield inventory)") + elif dynu_inventory_missing: + lines.append("- Dynu DNS inventory not available; run the Dynu brownfield generator.") + lines.extend( + [ + "- Prometheus scrape metadata (`targets` + label sets)", + "- Existing repository architecture docs for declared topology", + ] + ) if notes: lines.extend(["", "### Notes from inventory", ""]) for note in notes: @@ -284,14 +429,26 @@ def render_architecture_section(inventory: dict[str, Any], targets: list[dict[st def upsert_generated_section(path: Path, section_markdown: str, dry_run: bool, verbose: bool) -> None: existing = path.read_text(encoding="utf-8") if path.exists() else "" section_body = section_markdown + legacy_heading = "## Runtime visibility from Prometheus" + new_heading = "## Runtime and infrastructure inventory" if GENERATED_BEGIN in existing and GENERATED_END in existing: + # Migration: replace legacy heading that exists outside the generated markers. + existing = re.sub( + rf"^{re.escape(legacy_heading)}(?=\n)", + new_heading, + existing, + count=1, + flags=re.MULTILINE, + ) pattern = re.compile( rf"{re.escape(GENERATED_BEGIN)}.*?{re.escape(GENERATED_END)}", re.DOTALL, ) replacement = "\n".join( - line for line in section_body.splitlines() if line.strip() not in {"## Runtime visibility from Prometheus"} + line + for line in section_body.splitlines() + if line.strip() not in {legacy_heading, new_heading} ) updated = pattern.sub(replacement.strip(), existing) else: @@ -384,6 +541,7 @@ def main() -> int: inventory_path = Path(args.inventory_file) docs_dir = Path(args.docs_dir) diagrams_dir = Path(args.diagrams_dir) + dynu_inventory_path = Path(args.dynu_dns_inventory_file) inventory = load_json(inventory_path) targets = normalize_targets(inventory) @@ -394,7 +552,25 @@ def main() -> int: coverage_md = render_monitoring_coverage(inventory, targets) network_md = render_network_doc(inventory, targets) - architecture_section = render_architecture_section(inventory, targets) + dynu_records: list[dict[str, Any]] | None = None + dynu_inventory_missing = False + if not args.skip_dynu_dns: + try: + dynu_payload = load_optional_json(dynu_inventory_path) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON in Dynu DNS inventory file {dynu_inventory_path}: {exc}") from exc + if dynu_payload is None: + dynu_inventory_missing = True + else: + dynu_records = normalize_dynu_dns_records(dynu_payload) + + architecture_section = render_architecture_section( + inventory, + targets, + dynu_records=dynu_records, + dynu_inventory_path=dynu_inventory_path, + dynu_inventory_missing=dynu_inventory_missing, + ) monitoring_mmd = render_monitoring_mermaid(targets) architecture_mmd = render_architecture_mermaid(targets)