From 3c655dabcf8a76c5c3b1e5cf1c851c535c4fc976 Mon Sep 17 00:00:00 2001 From: beatz174-bit Date: Wed, 13 May 2026 13:40:50 +1000 Subject: [PATCH] Harden docs generation pipeline with strict source refresh --- .gitignore | 13 +++ scripts/docs/generate-all.sh | 113 ++++++++++++++++++-- scripts/docs/generate_dynu_dns_inventory.sh | 38 +++++++ 3 files changed, 154 insertions(+), 10 deletions(-) create mode 100755 scripts/docs/generate_dynu_dns_inventory.sh diff --git a/.gitignore b/.gitignore index f16243b..32dde1b 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,16 @@ secrets/* core/traefik/certs/* !core/traefik/certs/.gitkeep site/ + +# Docs generation artifacts intentionally tracked +!data/terraform/proxmox-inventory.json +!infrastructure/terraform/dynu/generated/ +!infrastructure/terraform/dynu/generated/dynu_dns_records_inventory.json +!docs/generated/ +!docs/generated/docker-compose.resolved.yml +!docs/generated/host-topology.md +!docs/diagrams/ +!docs/diagrams/*.svg +!docs/public/ +!docs/public/*.md +!docs/public/*.svg diff --git a/scripts/docs/generate-all.sh b/scripts/docs/generate-all.sh index 01e5489..e509c4a 100755 --- a/scripts/docs/generate-all.sh +++ b/scripts/docs/generate-all.sh @@ -1,22 +1,115 @@ #!/usr/bin/env bash set -euo pipefail + ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" cd "$ROOT" -mkdir -p docs/generated docs/diagrams docs/public -scripts/docs/render-compose-config.sh -python3 scripts/docs/generate-compose-inventory.py docs/generated/docker-compose.resolved.yml docs/generated/compose-inventory.md -python3 scripts/docs/generate-traefik-routes.py docs/generated/docker-compose.resolved.yml docs/generated/traefik-routes.md -python3 scripts/docs/generate-docs-index.py docs/generated/index.md + +ALLOW_STALE_TERRAFORM=0 +SKIP_TERRAFORM=0 +SKIP_DNS=0 + +usage() { + cat <<'USAGE' +Usage: scripts/docs/generate-all.sh [--allow-stale-terraform] [--skip-terraform] [--skip-dns] + +Options: + --allow-stale-terraform Use existing Terraform JSON when terraform is unavailable. + --skip-terraform Skip Terraform refresh and require existing JSON artifacts. + --skip-dns Skip Dynu DNS Terraform refresh. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --allow-stale-terraform) ALLOW_STALE_TERRAFORM=1 ;; + --skip-terraform) SKIP_TERRAFORM=1 ;; + --skip-dns) SKIP_DNS=1 ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: Unknown option: $1" >&2; usage; exit 1 ;; + esac + shift +done + +mkdir -p docs/generated docs/diagrams docs/public data/terraform +[[ -d infrastructure/terraform/proxmox ]] && mkdir -p infrastructure/terraform/proxmox/generated +[[ -d infrastructure/terraform/dynu ]] && mkdir -p infrastructure/terraform/dynu/generated + +if ! scripts/docs/render-compose-config.sh; then + echo "ERROR: Docker Compose config render failed." >&2 + echo "Check services-up.sh, compose files, and default-environment.env." >&2 + exit 1 +fi + +if [[ ! -s docs/generated/docker-compose.resolved.yml ]]; then + echo "ERROR: Expected non-empty docs/generated/docker-compose.resolved.yml after compose render." >&2 + exit 1 +fi + +PROXMOX_DIR="infrastructure/terraform/proxmox" +PRIMARY_HOST_INV="data/terraform/proxmox-inventory.json" +FALLBACK_HOST_INV="infrastructure/terraform/proxmox/generated/infrastructure_inventory.json" HOST_INVENTORY="" -for p in data/terraform/proxmox-inventory.json infrastructure/terraform/proxmox/generated/infrastructure_inventory.json; do - if [[ -f "$p" ]]; then + +if [[ -d "$PROXMOX_DIR" ]]; then + if [[ "$SKIP_TERRAFORM" -eq 1 ]]; then + echo "INFO: --skip-terraform set; using pre-generated Terraform JSON artifacts." >&2 + elif command -v terraform >/dev/null 2>&1; then + scripts/docs/build_host_topology.sh + elif [[ "$ALLOW_STALE_TERRAFORM" -eq 1 ]]; then + echo "WARNING: terraform unavailable; using stale Terraform JSON due to --allow-stale-terraform." >&2 + else + echo "ERROR: terraform is not available, but $PROXMOX_DIR exists and must be refreshed." >&2 + echo "Install terraform and rerun scripts/docs/generate-all.sh, or use --allow-stale-terraform/--skip-terraform intentionally." >&2 + exit 1 + fi +else + echo "ERROR: Required Terraform directory missing: $PROXMOX_DIR" >&2 + echo "Physical topology diagrams require host inventory from Terraform." >&2 + exit 1 +fi + +for p in "$PRIMARY_HOST_INV" "$FALLBACK_HOST_INV"; do + if [[ -s "$p" ]]; then HOST_INVENTORY="$p" break fi done +if [[ -z "$HOST_INVENTORY" ]]; then + echo "ERROR: Host topology inventory is missing." >&2 + echo "Expected one of: $PRIMARY_HOST_INV or $FALLBACK_HOST_INV" >&2 + echo "Run scripts/docs/build_host_topology.sh from a machine with Terraform state access, then rerun scripts/docs/generate-all.sh." >&2 + exit 1 +fi + DNS_INVENTORY="infrastructure/terraform/dynu/generated/dynu_dns_records_inventory.json" -GEN_ARGS=(--compose docs/generated/docker-compose.resolved.yml --out-dir docs/diagrams --domain-display redacted-label) -[[ -n "$HOST_INVENTORY" ]] && GEN_ARGS+=(--host-inventory "$HOST_INVENTORY") -[[ -f "$DNS_INVENTORY" ]] && GEN_ARGS+=(--dns-inventory "$DNS_INVENTORY") +if [[ "$SKIP_DNS" -eq 0 && -d infrastructure/terraform/dynu ]]; then + if [[ "$SKIP_TERRAFORM" -eq 1 ]]; then + echo "INFO: Skipping Dynu Terraform refresh due to --skip-terraform." >&2 + elif command -v terraform >/dev/null 2>&1; then + scripts/docs/generate_dynu_dns_inventory.sh + elif [[ "$ALLOW_STALE_TERRAFORM" -eq 1 ]]; then + echo "WARNING: terraform unavailable; using stale Dynu DNS inventory due to --allow-stale-terraform." >&2 + else + echo "ERROR: terraform unavailable; cannot refresh Dynu DNS inventory while infrastructure/terraform/dynu exists." >&2 + echo "Install terraform and rerun, or use --allow-stale-terraform/--skip-terraform intentionally." >&2 + exit 1 + fi +fi + +python3 scripts/docs/generate-compose-inventory.py docs/generated/docker-compose.resolved.yml docs/generated/compose-inventory.md +python3 scripts/docs/generate-traefik-routes.py docs/generated/docker-compose.resolved.yml docs/generated/traefik-routes.md +python3 scripts/docs/generate-docs-index.py docs/generated/index.md + +GEN_ARGS=(--compose docs/generated/docker-compose.resolved.yml --out-dir docs/diagrams --domain-display redacted-label --host-inventory "$HOST_INVENTORY") +[[ -s "$DNS_INVENTORY" && "$SKIP_DNS" -eq 0 ]] && GEN_ARGS+=(--dns-inventory "$DNS_INVENTORY") python3 scripts/docs/generate-diagrams.py "${GEN_ARGS[@]}" python3 scripts/docs/sanitize-public-docs.py docs/generated docs/diagrams docs/public + +[[ -s docs/public/physical-topology.svg ]] || { echo "ERROR: docs/public/physical-topology.svg missing or empty." >&2; exit 1; } +if grep -Fq "Host inventory JSON not found" docs/public/physical-topology.svg || grep -Fq "Generate terraform inventory" docs/public/physical-topology.svg; then + echo "ERROR: docs/public/physical-topology.svg contains placeholder/error text; host inventory refresh failed." >&2 + exit 1 +fi +[[ -s docs/public/docker-traefik-dynu.svg ]] || { echo "ERROR: docs/public/docker-traefik-dynu.svg missing or empty." >&2; exit 1; } +[[ -s docs/generated/docker-compose.resolved.yml ]] || { echo "ERROR: docs/generated/docker-compose.resolved.yml missing or empty." >&2; exit 1; } +[[ -s docs/generated/host-topology.md ]] || { echo "ERROR: docs/generated/host-topology.md missing or empty." >&2; exit 1; } diff --git a/scripts/docs/generate_dynu_dns_inventory.sh b/scripts/docs/generate_dynu_dns_inventory.sh new file mode 100755 index 0000000..7ff5cf5 --- /dev/null +++ b/scripts/docs/generate_dynu_dns_inventory.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +dynu_dir="${repo_root}/infrastructure/terraform/dynu" +out_json="${dynu_dir}/generated/dynu_dns_records_inventory.json" + +if [[ ! -d "${dynu_dir}" ]]; then + echo "ERROR: Dynu Terraform directory not found at ${dynu_dir}." >&2 + exit 1 +fi + +if ! command -v terraform >/dev/null 2>&1; then + echo "ERROR: terraform is required to refresh Dynu DNS inventory." >&2 + exit 1 +fi + +mkdir -p "$(dirname "${out_json}")" + +( + cd "${dynu_dir}" + if terraform output -json dynu_dns_inventory > "${out_json}"; then + : + elif terraform output -json dynu_dns_records_catalog > "${out_json}"; then + : + else + echo "ERROR: Failed to export Dynu DNS inventory from Terraform outputs. Tried: dynu_dns_inventory, dynu_dns_records_catalog." >&2 + echo "Run 'cd infrastructure/terraform/dynu && terraform output' to inspect available outputs." >&2 + exit 1 + fi +) + +if [[ ! -s "${out_json}" ]]; then + echo "ERROR: Dynu DNS inventory output file is empty: ${out_json}" >&2 + exit 1 +fi + +echo "Generated: ${out_json}"