Merge pull request #75 from beatz174-bit/codex/update-docs-generation-workflow-for-reliability

Harden docs generation pipeline to refresh compose and Terraform data first

.gitignore

@@ -30,3 +30,16 @@ secrets/*
 core/traefik/certs/*
 !core/traefik/certs/.gitkeep
 site/
+
+# Docs generation artifacts intentionally tracked
+!data/terraform/proxmox-inventory.json
+!infrastructure/terraform/dynu/generated/
+!infrastructure/terraform/dynu/generated/dynu_dns_records_inventory.json
+!docs/generated/
+!docs/generated/docker-compose.resolved.yml
+!docs/generated/host-topology.md
+!docs/diagrams/
+!docs/diagrams/*.svg
+!docs/public/
+!docs/public/*.md
+!docs/public/*.svg

scripts/docs/generate-all.sh

@@ -1,22 +1,115 @@
 #!/usr/bin/env bash
 set -euo pipefail
+
 ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
 cd "$ROOT"
-mkdir -p docs/generated docs/diagrams docs/public
+
-scripts/docs/render-compose-config.sh
+ALLOW_STALE_TERRAFORM=0
-python3 scripts/docs/generate-compose-inventory.py docs/generated/docker-compose.resolved.yml docs/generated/compose-inventory.md
+SKIP_TERRAFORM=0
-python3 scripts/docs/generate-traefik-routes.py docs/generated/docker-compose.resolved.yml docs/generated/traefik-routes.md
+SKIP_DNS=0
-python3 scripts/docs/generate-docs-index.py docs/generated/index.md
+
+usage() {
+  cat <<'USAGE'
+Usage: scripts/docs/generate-all.sh [--allow-stale-terraform] [--skip-terraform] [--skip-dns]
+
+Options:
+  --allow-stale-terraform  Use existing Terraform JSON when terraform is unavailable.
+  --skip-terraform         Skip Terraform refresh and require existing JSON artifacts.
+  --skip-dns               Skip Dynu DNS Terraform refresh.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --allow-stale-terraform) ALLOW_STALE_TERRAFORM=1 ;;
+    --skip-terraform) SKIP_TERRAFORM=1 ;;
+    --skip-dns) SKIP_DNS=1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "ERROR: Unknown option: $1" >&2; usage; exit 1 ;;
+  esac
+  shift
+done
+
+mkdir -p docs/generated docs/diagrams docs/public data/terraform
+[[ -d infrastructure/terraform/proxmox ]] && mkdir -p infrastructure/terraform/proxmox/generated
+[[ -d infrastructure/terraform/dynu ]] && mkdir -p infrastructure/terraform/dynu/generated
+
+if ! scripts/docs/render-compose-config.sh; then
+  echo "ERROR: Docker Compose config render failed." >&2
+  echo "Check services-up.sh, compose files, and default-environment.env." >&2
+  exit 1
+fi
+
+if [[ ! -s docs/generated/docker-compose.resolved.yml ]]; then
+  echo "ERROR: Expected non-empty docs/generated/docker-compose.resolved.yml after compose render." >&2
+  exit 1
+fi
+
+PROXMOX_DIR="infrastructure/terraform/proxmox"
+PRIMARY_HOST_INV="data/terraform/proxmox-inventory.json"
+FALLBACK_HOST_INV="infrastructure/terraform/proxmox/generated/infrastructure_inventory.json"
 HOST_INVENTORY=""
-for p in data/terraform/proxmox-inventory.json infrastructure/terraform/proxmox/generated/infrastructure_inventory.json; do
+
-  if [[ -f "$p" ]]; then
+if [[ -d "$PROXMOX_DIR" ]]; then
+  if [[ "$SKIP_TERRAFORM" -eq 1 ]]; then
+    echo "INFO: --skip-terraform set; using pre-generated Terraform JSON artifacts." >&2
+  elif command -v terraform >/dev/null 2>&1; then
+    scripts/docs/build_host_topology.sh
+  elif [[ "$ALLOW_STALE_TERRAFORM" -eq 1 ]]; then
+    echo "WARNING: terraform unavailable; using stale Terraform JSON due to --allow-stale-terraform." >&2
+  else
+    echo "ERROR: terraform is not available, but $PROXMOX_DIR exists and must be refreshed." >&2
+    echo "Install terraform and rerun scripts/docs/generate-all.sh, or use --allow-stale-terraform/--skip-terraform intentionally." >&2
+    exit 1
+  fi
+else
+  echo "ERROR: Required Terraform directory missing: $PROXMOX_DIR" >&2
+  echo "Physical topology diagrams require host inventory from Terraform." >&2
+  exit 1
+fi
+
+for p in "$PRIMARY_HOST_INV" "$FALLBACK_HOST_INV"; do
+  if [[ -s "$p" ]]; then
     HOST_INVENTORY="$p"
     break
   fi
 done
+if [[ -z "$HOST_INVENTORY" ]]; then
+  echo "ERROR: Host topology inventory is missing." >&2
+  echo "Expected one of: $PRIMARY_HOST_INV or $FALLBACK_HOST_INV" >&2
+  echo "Run scripts/docs/build_host_topology.sh from a machine with Terraform state access, then rerun scripts/docs/generate-all.sh." >&2
+  exit 1
+fi
+
 DNS_INVENTORY="infrastructure/terraform/dynu/generated/dynu_dns_records_inventory.json"
-GEN_ARGS=(--compose docs/generated/docker-compose.resolved.yml --out-dir docs/diagrams --domain-display redacted-label)
+if [[ "$SKIP_DNS" -eq 0 && -d infrastructure/terraform/dynu ]]; then
-[[ -n "$HOST_INVENTORY" ]] && GEN_ARGS+=(--host-inventory "$HOST_INVENTORY")
+  if [[ "$SKIP_TERRAFORM" -eq 1 ]]; then
-[[ -f "$DNS_INVENTORY" ]] && GEN_ARGS+=(--dns-inventory "$DNS_INVENTORY")
+    echo "INFO: Skipping Dynu Terraform refresh due to --skip-terraform." >&2
+  elif command -v terraform >/dev/null 2>&1; then
+    scripts/docs/generate_dynu_dns_inventory.sh
+  elif [[ "$ALLOW_STALE_TERRAFORM" -eq 1 ]]; then
+    echo "WARNING: terraform unavailable; using stale Dynu DNS inventory due to --allow-stale-terraform." >&2
+  else
+    echo "ERROR: terraform unavailable; cannot refresh Dynu DNS inventory while infrastructure/terraform/dynu exists." >&2
+    echo "Install terraform and rerun, or use --allow-stale-terraform/--skip-terraform intentionally." >&2
+    exit 1
+  fi
+fi
+
+python3 scripts/docs/generate-compose-inventory.py docs/generated/docker-compose.resolved.yml docs/generated/compose-inventory.md
+python3 scripts/docs/generate-traefik-routes.py docs/generated/docker-compose.resolved.yml docs/generated/traefik-routes.md
+python3 scripts/docs/generate-docs-index.py docs/generated/index.md
+
+GEN_ARGS=(--compose docs/generated/docker-compose.resolved.yml --out-dir docs/diagrams --domain-display redacted-label --host-inventory "$HOST_INVENTORY")
+[[ -s "$DNS_INVENTORY" && "$SKIP_DNS" -eq 0 ]] && GEN_ARGS+=(--dns-inventory "$DNS_INVENTORY")
 python3 scripts/docs/generate-diagrams.py "${GEN_ARGS[@]}"
 python3 scripts/docs/sanitize-public-docs.py docs/generated docs/diagrams docs/public
+
+[[ -s docs/public/physical-topology.svg ]] || { echo "ERROR: docs/public/physical-topology.svg missing or empty." >&2; exit 1; }
+if grep -Fq "Host inventory JSON not found" docs/public/physical-topology.svg || grep -Fq "Generate terraform inventory" docs/public/physical-topology.svg; then
+  echo "ERROR: docs/public/physical-topology.svg contains placeholder/error text; host inventory refresh failed." >&2
+  exit 1
+fi
+[[ -s docs/public/docker-traefik-dynu.svg ]] || { echo "ERROR: docs/public/docker-traefik-dynu.svg missing or empty." >&2; exit 1; }
+[[ -s docs/generated/docker-compose.resolved.yml ]] || { echo "ERROR: docs/generated/docker-compose.resolved.yml missing or empty." >&2; exit 1; }
+[[ -s docs/generated/host-topology.md ]] || { echo "ERROR: docs/generated/host-topology.md missing or empty." >&2; exit 1; }

scripts/docs/generate_dynu_dns_inventory.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+dynu_dir="${repo_root}/infrastructure/terraform/dynu"
+out_json="${dynu_dir}/generated/dynu_dns_records_inventory.json"
+
+if [[ ! -d "${dynu_dir}" ]]; then
+  echo "ERROR: Dynu Terraform directory not found at ${dynu_dir}." >&2
+  exit 1
+fi
+
+if ! command -v terraform >/dev/null 2>&1; then
+  echo "ERROR: terraform is required to refresh Dynu DNS inventory." >&2
+  exit 1
+fi
+
+mkdir -p "$(dirname "${out_json}")"
+
+(
+  cd "${dynu_dir}"
+  if terraform output -json dynu_dns_inventory > "${out_json}"; then
+    :
+  elif terraform output -json dynu_dns_records_catalog > "${out_json}"; then
+    :
+  else
+    echo "ERROR: Failed to export Dynu DNS inventory from Terraform outputs. Tried: dynu_dns_inventory, dynu_dns_records_catalog." >&2
+    echo "Run 'cd infrastructure/terraform/dynu && terraform output' to inspect available outputs." >&2
+    exit 1
+  fi
+)
+
+if [[ ! -s "${out_json}" ]]; then
+  echo "ERROR: Dynu DNS inventory output file is empty: ${out_json}" >&2
+  exit 1
+fi
+
+echo "Generated: ${out_json}"