#!/usr/bin/env bash
# deploy.sh — Full lifecycle management for the Docker Swarm HA cluster.
#
# Provisions two NixOS Proxmox VMs (ha-docker-1, ha-docker-2) as dual-manager
# Docker Swarm nodes sharing NFS storage from the existing HA file-server
# cluster.  Both nodes are managers so either can accept Docker API and
# `docker stack` commands.
#
# Usage:
#   scripts/docker-swarm/deploy.sh [options]
#   scripts/docker-swarm/deploy.sh --destroy [options]
#
# Phases (all run by default; skip any with --skip-<phase>):
#   1. ensure-bridge     Create vmbr3 (swarm cluster bridge) on the Proxmox node.
#   2. sync-keys         Generate SSH host keys for both nodes (clan vars).
#   3. ipa-hosts         Create IPA host objects + sops-encrypted keytabs.
#   4. create-vms        Build NixOS disk images and create VMs via create-proxmox-resource.sh.
#   5. add-hardware      Attach vmbr2 (storage) and vmbr3 (swarm) NICs; start VMs.
#   6. boot-wait         Wait for SSH on both LAN IPs.
#   7. refresh-sops-keys Detect disko key drift; re-encrypt secrets; commit.
#   8. init-swarm        docker swarm init on node1; manager join on node2; label nodes.
#   9. dns               Register storage.home and swarm.home A records in FreeIPA.
#  10. verify            docker node ls; NFS mount check; swarm health.
#
# Options:
#   --node <host>            Proxmox host (default: pve1.sweet.home)
#   --vmid1 <n>              VMID for ha-docker-1 (default: 202)
#   --vmid2 <n>              VMID for ha-docker-2 (default: 203)
#   --storage <pool>         Proxmox storage pool (default: local-zfs)
#   --swarm-bridge <br>      Bridge for Docker Swarm cluster network (default: vmbr3)
#   --storage-bridge <br>    Bridge for NFS storage network (default: vmbr2)
#   --memory <MB>            RAM per node (default: 4096)
#   --cores <n>              vCPUs per node (default: 4)
#   --skip-ensure-bridge     Skip vmbr3 creation/check
#   --skip-sync-keys         Skip sync-host-keys.sh (clan vars already exist)
#   --skip-ipa-hosts         Skip IPA host account creation (keytabs already exist)
#   --skip-create-vms        Skip VM creation (VMs already exist)
#   --skip-add-hardware      Skip NIC attachment (already attached)
#   --skip-boot-wait         Skip boot/SSH wait (VMs already running)
#   --skip-refresh-sops-keys Skip sops host-key drift fix
#   --skip-init-swarm        Skip swarm initialisation (already initialised)
#   --skip-dns               Skip FreeIPA DNS record creation
#   --skip-verify            Skip post-deploy health checks
#   --force-rebuild          Pass --force-rebuild to create-proxmox-resource.sh
#   --destroy                Stop and delete both VMs (skip all other phases)
#   --dry-run                Print what would run without executing
#   -h|--help                Show this message
#
# Prerequisites:
#   - SSH access to the Proxmox node as $PROXMOX_SSH_USER (wayne).
#   - sops age key in the standard location (used by sync-host-keys.sh).
#   - SSH access to domain-controller.sweet.home as $PROXMOX_SSH_USER for DNS phase.
#   - For --skip-sync-keys: clan vars already in vars/per-machine/proxmox-ha-docker-{1,2}/.
#   - For --skip-ipa-hosts: secrets/ha-docker-{1,2}.keytab already exist and are committed.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"

# shellcheck source=../env.sh
source "${REPO_ROOT}/scripts/env.sh"

# ── Defaults ──────────────────────────────────────────────────────────────────

NODE="${PVE1_HOST}"   # deploy.sh targets pve1 by default (authorised for this cluster)
VMID1=202
VMID2=203
STORAGE="${PROXMOX_STORAGE:-local-zfs}"
SWARM_BRIDGE="vmbr3"
STORAGE_BRIDGE="vmbr2"
MEMORY_MB=4096
CORES=4

SKIP_ENSURE_BRIDGE=false
SKIP_SYNC_KEYS=false
SKIP_IPA_HOSTS=false
SKIP_CREATE_VMS=false
SKIP_ADD_HARDWARE=false
SKIP_BOOT_WAIT=false
SKIP_REFRESH_SOPS_KEYS=false
SKIP_INIT_SWARM=false
SKIP_DNS=false
SKIP_VERIFY=false
FORCE_REBUILD=false
DESTROY=false
DRY_RUN=false

# ── Variables from repo (mirrors variables.nix) ───────────────────────────────

NODE1_HOST="ha-docker-1"
NODE2_HOST="ha-docker-2"
NODE1_LAN_IP="192.168.2.230"
NODE2_LAN_IP="192.168.2.231"
NODE1_SWARM_IP="192.168.30.230"
NODE2_SWARM_IP="192.168.30.231"
NODE1_STORAGE_IP="192.168.20.230"
NODE2_STORAGE_IP="192.168.20.231"
SWARM_CIDR="192.168.30.0/24"
STORAGE_CIDR="192.168.20.0/24"
STORAGE_ZONE="storage.home"
SWARM_ZONE="swarm.home"
SSH_USER="${PROXMOX_SSH_USER:-wayne}"
DC_HOST="${IPA_SERVER:-domain-controller.sweet.home}"

# ── Argument parsing ──────────────────────────────────────────────────────────

usage() {
  sed -n '/^# Usage:/,/^[^#]/{ /^#/{ s/^# \?//; p } }' "$0"
  exit "${1:-0}"
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --node)               NODE="$2";           shift 2 ;;
    --vmid1)              VMID1="$2";          shift 2 ;;
    --vmid2)              VMID2="$2";          shift 2 ;;
    --storage)            STORAGE="$2";        shift 2 ;;
    --swarm-bridge)       SWARM_BRIDGE="$2";   shift 2 ;;
    --storage-bridge)     STORAGE_BRIDGE="$2"; shift 2 ;;
    --memory)             MEMORY_MB="$2";      shift 2 ;;
    --cores)              CORES="$2";          shift 2 ;;
    --skip-ensure-bridge)     SKIP_ENSURE_BRIDGE=true;     shift ;;
    --skip-sync-keys)         SKIP_SYNC_KEYS=true;         shift ;;
    --skip-ipa-hosts)         SKIP_IPA_HOSTS=true;         shift ;;
    --skip-create-vms)        SKIP_CREATE_VMS=true;        shift ;;
    --skip-add-hardware)      SKIP_ADD_HARDWARE=true;      shift ;;
    --skip-boot-wait)         SKIP_BOOT_WAIT=true;         shift ;;
    --skip-refresh-sops-keys) SKIP_REFRESH_SOPS_KEYS=true; shift ;;
    --skip-init-swarm)        SKIP_INIT_SWARM=true;        shift ;;
    --skip-dns)               SKIP_DNS=true;               shift ;;
    --skip-verify)            SKIP_VERIFY=true;            shift ;;
    --force-rebuild)          FORCE_REBUILD=true;          shift ;;
    --destroy)                DESTROY=true;                shift ;;
    --dry-run)                DRY_RUN=true;                shift ;;
    -h|--help) usage 0 ;;
    *) echo "Unknown option: $1" >&2; usage 1 ;;
  esac
done

# ── Helpers ───────────────────────────────────────────────────────────────────

log()  { echo "==> $*"; }
logn() { echo "    $*"; }
err()  { echo "ERROR: $*" >&2; exit 1; }

run() {
  if $DRY_RUN; then
    echo "[dry-run] $*"
  else
    "$@"
  fi
}

pve() {
  if $DRY_RUN; then
    echo "[dry-run] ssh ${SSH_USER}@${NODE} sudo $*"
  else
    ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
  fi
}

pve_check() {
  # Read-only probe — always executes even in dry-run.
  ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
}

SWARM_USER="nixos"

n1() {
  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
    "${SWARM_USER}@${NODE1_LAN_IP}" sudo "$@" 2>/dev/null
}

n2() {
  ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
    "${SWARM_USER}@${NODE2_LAN_IP}" sudo "$@" 2>/dev/null
}

dc() {
  # Run ipa commands on domain-controller as $SSH_USER.
  if $DRY_RUN; then
    echo "[dry-run] ssh ${SSH_USER}@${DC_HOST} $*"
    return 0
  fi
  ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${DC_HOST}" "$@"
}

wait_for_ssh() {
  local ip="$1" label="$2"
  if $DRY_RUN; then
    logn "[dry-run] Skipping SSH wait for ${label} (${ip})"
    return 0
  fi
  local deadline=$(( $(date +%s) + 300 ))
  log "Waiting for SSH on ${label} (${ip}) — up to 5 min..."
  while [[ $(date +%s) -lt $deadline ]]; do
    if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=3 \
        -o BatchMode=yes "${SWARM_USER}@${ip}" true 2>/dev/null; then
      logn "${label} is up."
      return 0
    fi
    sleep 5
  done
  err "Timed out waiting for SSH on ${label} (${ip})"
}

# ── Destroy mode ──────────────────────────────────────────────────────────────

if $DESTROY; then
  log "Destroying Docker Swarm VMs (${VMID1}=${NODE1_HOST}, ${VMID2}=${NODE2_HOST}) on ${NODE}"
  for vmid in "$VMID1" "$VMID2"; do
    STATUS=$(pve "qm status ${vmid} 2>/dev/null" 2>/dev/null || true)
    if echo "$STATUS" | grep -q "running"; then
      log "Stopping VMID ${vmid}..."
      pve "qm stop ${vmid} --skiplock 1"
      sleep 5
    fi
    if $DRY_RUN || pve "qm config ${vmid} >/dev/null 2>&1"; then
      log "Deleting VMID ${vmid}..."
      run pve "qm destroy ${vmid} --purge 1"
    else
      logn "VMID ${vmid} not found — already gone."
    fi
  done
  log "Done — swarm VMs destroyed."
  exit 0
fi

# ── Phase 1: Ensure swarm bridge ──────────────────────────────────────────────

if ! $SKIP_ENSURE_BRIDGE; then
  log "Phase 1: Ensuring swarm bridge ${SWARM_BRIDGE} on ${NODE}"
  if pve_check "test -d /sys/class/net/${SWARM_BRIDGE}" &>/dev/null; then
    logn "${SWARM_BRIDGE} already exists — skipping."
  else
    logn "Creating isolated internal bridge ${SWARM_BRIDGE} (no upstream port, ${SWARM_CIDR})"
    BRIDGE_CONF="auto ${SWARM_BRIDGE}
iface ${SWARM_BRIDGE} inet manual
    bridge-ports none
    bridge-stp off
    bridge-fd 0"
    if $DRY_RUN; then
      echo "[dry-run] Would write /etc/network/interfaces.d/${SWARM_BRIDGE}.conf and ifup it"
    else
      ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
        "echo '${BRIDGE_CONF}' | sudo tee /etc/network/interfaces.d/${SWARM_BRIDGE}.conf > /dev/null && sudo ifup ${SWARM_BRIDGE}"
      logn "${SWARM_BRIDGE} created and brought up."
    fi
  fi
fi

# ── Phase 2: Sync host keys ───────────────────────────────────────────────────

if ! $SKIP_SYNC_KEYS; then
  log "Phase 2: Syncing SSH host keys for both swarm targets"
  for target in proxmox-ha-docker-1 proxmox-ha-docker-2; do
    CLAN_DIR="${REPO_ROOT}/vars/per-machine/${target}/openssh"
    if [[ -d "$CLAN_DIR" ]]; then
      logn "Clan vars for ${target} already exist — skipping."
    else
      logn "Generating host keys for ${target}..."
      run bash "${REPO_ROOT}/scripts/secrets/sync-host-keys.sh" "$target"
    fi
  done
fi

# ── Phase 3: IPA host accounts ────────────────────────────────────────────────

if ! $SKIP_IPA_HOSTS; then
  log "Phase 3: Creating IPA host accounts and keytabs"
  IPA_SCRIPT="${REPO_ROOT}/scripts/ipa/create-nixos-ipa-host-account.sh"
  for host in "${NODE1_HOST}" "${NODE2_HOST}"; do
    KEYTAB="${REPO_ROOT}/secrets/${host}.keytab"
    if [[ -f "$KEYTAB" ]]; then
      logn "Keytab for ${host} already exists — skipping."
    else
      logn "Creating IPA host account and keytab for ${host}..."
      run bash "$IPA_SCRIPT" "$host"
    fi
  done

  if ! $DRY_RUN; then
    # Keytabs must be committed and pushed before VMs rebuild from Gitea.
    CURRENT_BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD)"
    logn "Committing keytabs and pushing to Gitea (branch: ${CURRENT_BRANCH})..."
    (cd "${REPO_ROOT}" && \
      git add secrets/ha-docker-1.keytab secrets/ha-docker-2.keytab .sops.yaml && \
      git commit -m "secrets(ha-docker): add IPA keytabs for ha-docker-1 and ha-docker-2" || true && \
      git push origin "${CURRENT_BRANCH}")
    logn "Pushed."
  fi
fi

# ── Phase 3.5: Prepare Proxmox node for building ─────────────────────────────

if ! $SKIP_CREATE_VMS && ! $DRY_RUN; then
  CURRENT_BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD)"

  local_ssh() { ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "$*"; }
  if local_ssh "test -d /nix" &>/dev/null && ! local_ssh "test -w /nix" &>/dev/null; then
    logn "/nix exists but not writable by ${SSH_USER} — fixing ownership with sudo..."
    local_ssh "sudo chown -R ${SSH_USER} /nix"
    logn "Done."
  fi
  unset -f local_ssh

  # Use PROXMOX_REMOTE_REPO_DIR (from env.sh) so the build path is consistent
  # with what create-proxmox-resource.sh will use.  The default is
  # /home/<user>/nixos (the standalone nixos repo clone on pve1), but can be
  # overridden to e.g. /home/<user>/infrastructure/nixos when the infrastructure
  # mono-repo is checked out on pve1 instead.
  REMOTE_REPO="${PROXMOX_REMOTE_REPO_DIR:-/home/${SSH_USER}/nixos}"
  if pve_check "test -d ${REMOTE_REPO}/.git" &>/dev/null; then
    REMOTE_BRANCH=$(ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
      "cd ${REMOTE_REPO} && git rev-parse --abbrev-ref HEAD 2>/dev/null")
    if [[ "$REMOTE_BRANCH" != "$CURRENT_BRANCH" ]]; then
      logn "Remote repo (${REMOTE_REPO}) is on '${REMOTE_BRANCH}', switching to '${CURRENT_BRANCH}'..."
      if ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
          "cd ${REMOTE_REPO} && git fetch origin && git checkout '${CURRENT_BRANCH}' && git pull --ff-only" 2>&1; then
        logn "Done."
      else
        logn "WARNING: branch switch failed — proceeding anyway (create-proxmox-resource.sh will retry)"
      fi
    fi
  else
    logn "Remote repo ${REMOTE_REPO} not found on ${NODE} — create-proxmox-resource.sh will clone it."
  fi
fi

# ── Phase 4: Create VMs ───────────────────────────────────────────────────────

if ! $SKIP_CREATE_VMS; then
  log "Phase 4: Building and creating swarm VMs on ${NODE}"
  CREATE="${REPO_ROOT}/scripts/proxmox/create-proxmox-resource.sh"
  REBUILD_FLAG=""
  $FORCE_REBUILD && REBUILD_FLAG="--force-rebuild"

  for spec in "${VMID1}:${NODE1_HOST}:proxmox-ha-docker-1" "${VMID2}:${NODE2_HOST}:proxmox-ha-docker-2"; do
    IFS=: read -r vmid host_name flake_target <<< "$spec"
    log "Creating ${flake_target} (VMID ${vmid}) on ${NODE}..."
    # --force-rebuild is always passed: create-proxmox-resource.sh only calls
    # sync_remote_host_keys (which bakes the clan-var SSH key into the disk
    # image) when it actually builds.  Reusing a cached image skips that step
    # and leaves the VM unable to decrypt sops secrets on first boot.
    run bash "$CREATE" \
      --type vm \
      --host "$host_name" \
      --vmid "$vmid" \
      --node "$NODE" \
      --storage "$STORAGE" \
      --memory "$MEMORY_MB" \
      --cores "$CORES" \
      --force-rebuild
  done
fi

# ── Phase 5: Add NICs and start VMs ──────────────────────────────────────────

if ! $SKIP_ADD_HARDWARE; then
  log "Phase 5: Attaching storage (${STORAGE_BRIDGE}) and swarm (${SWARM_BRIDGE}) NICs"
  for vmid in "$VMID1" "$VMID2"; do
    logn "VMID ${vmid}: stopping to add NICs..."
    pve "qm stop ${vmid} --skiplock 1 2>/dev/null; sleep 3" || true

    logn "Adding net1 (${STORAGE_BRIDGE} — NFS storage)..."
    pve "qm set ${vmid} --net1 virtio,bridge=${STORAGE_BRIDGE},firewall=0"

    logn "Adding net2 (${SWARM_BRIDGE} — Docker Swarm)..."
    pve "qm set ${vmid} --net2 virtio,bridge=${SWARM_BRIDGE},firewall=0"

    logn "Starting VMID ${vmid}..."
    pve "qm start ${vmid}"
  done
fi

# ── Phase 6: Wait for SSH ─────────────────────────────────────────────────────

if ! $SKIP_BOOT_WAIT; then
  log "Phase 6: Waiting for both nodes to come up on LAN IPs"
  wait_for_ssh "$NODE1_LAN_IP" "$NODE1_HOST"
  wait_for_ssh "$NODE2_LAN_IP" "$NODE2_HOST"
  logn "Both nodes are SSHable."
  sleep 10  # let systemd finish activation
fi

# ── Phase 7: Refresh sops host-key registrations ─────────────────────────────
#
# Disko builds raw disk images: each new VM boots with a freshly-generated SSH
# host key, not the one pre-seeded in clan vars.  Scan the running VMs; if
# their ed25519 keys differ from the clan var, update the clan var, rewrite
# the .sops.yaml anchor, and re-encrypt all affected sops files.

if ! $SKIP_REFRESH_SOPS_KEYS; then
  if $DRY_RUN; then
    logn "[dry-run] Would scan VM host keys and refresh .sops.yaml / secrets if needed"
  else
    log "Phase 7: Refreshing sops host-key registrations (disko key drift fix)"
    SOPS_UPDATED=false

    for spec in \
      "${NODE1_LAN_IP}:proxmox-ha-docker-1:${NODE1_HOST}" \
      "${NODE2_LAN_IP}:proxmox-ha-docker-2:${NODE2_HOST}"; do
      IFS=: read -r node_ip flake_target host_name <<< "$spec"
      CLAN_PUB="${REPO_ROOT}/vars/per-machine/${flake_target}/openssh/ssh_host_ed25519_key.pub/value"

      logn "Scanning ed25519 host key from ${host_name} (${node_ip})..."
      RAW=$(ssh-keyscan -t ed25519 "${node_ip}" 2>/dev/null | grep -v "^#") || true
      if [[ -z "$RAW" ]]; then
        logn "WARNING: no ed25519 key returned for ${node_ip} — skipping"
        continue
      fi
      SCANNED_TYPE=$(awk '{print $2}' <<< "$RAW")
      SCANNED_KEY=$(awk  '{print $3}' <<< "$RAW")
      SCANNED_PUBKEY="${SCANNED_TYPE} ${SCANNED_KEY} ${host_name}"

      CURRENT=$(tr -d '\n' < "$CLAN_PUB" 2>/dev/null || true)
      if [[ "$SCANNED_PUBKEY" == "$CURRENT" ]]; then
        logn "${host_name}: clan var matches running key — no update needed"
        continue
      fi

      logn "${host_name}: key drift detected — updating clan var"
      logn "  old: ${CURRENT}"
      logn "  new: ${SCANNED_PUBKEY}"
      echo "$SCANNED_PUBKEY" > "$CLAN_PUB"
      SOPS_UPDATED=true

      ANCHOR="${flake_target}"
      NEW_AGE=$(echo "$SCANNED_PUBKEY" | \
        nix run --quiet --no-warn-dirty nixpkgs#ssh-to-age 2>/dev/null)
      [[ -z "$NEW_AGE" ]] && err "ssh-to-age produced no output for ${host_name}"
      logn "  new age key: ${NEW_AGE}"
      sed -i "/&${ANCHOR} /s| age[a-z0-9]*$| ${NEW_AGE}|" "${REPO_ROOT}/.sops.yaml"
    done

    if $SOPS_UPDATED; then
      logn "Running sops updatekeys on affected secrets..."
      SOPS="nix run --quiet --no-warn-dirty nixpkgs#sops --"
      (cd "${REPO_ROOT}" && \
        $SOPS updatekeys -y secrets/common.yaml && \
        $SOPS updatekeys -y secrets/ha-docker-1.keytab && \
        $SOPS updatekeys -y secrets/ha-docker-2.keytab)

      logn "Committing refreshed host keys and re-encrypted secrets..."
      (cd "${REPO_ROOT}" && \
        git add \
          vars/per-machine/proxmox-ha-docker-1/openssh/ssh_host_ed25519_key.pub/value \
          vars/per-machine/proxmox-ha-docker-2/openssh/ssh_host_ed25519_key.pub/value \
          .sops.yaml \
          secrets/common.yaml \
          secrets/ha-docker-1.keytab \
          secrets/ha-docker-2.keytab && \
        git commit -m "secrets(ha-docker): refresh sops host-key registrations for new VM instances" || true)
      logn "Sops keys refreshed and committed."
    fi
  fi
fi

# ── Phase 8: Initialise Docker Swarm ─────────────────────────────────────────

if ! $SKIP_INIT_SWARM; then
  log "Phase 8: Initialising Docker Swarm"

  if $DRY_RUN; then
    logn "[dry-run] Would run: docker swarm init --advertise-addr ${NODE1_SWARM_IP} --data-path-addr ${NODE1_SWARM_IP} on ${NODE1_HOST}"
    logn "[dry-run] Would join ${NODE2_HOST} as manager"
    logn "[dry-run] Would label both nodes"
  else
    # Check if node1 is already a swarm manager.
    if n1 "docker info --format '{{.Swarm.LocalNodeState}}'" 2>/dev/null | grep -q "active"; then
      logn "${NODE1_HOST} is already in a swarm — skipping init."
    else
      logn "Initialising swarm on ${NODE1_HOST} (advertise: ${NODE1_SWARM_IP})..."
      n1 "docker swarm init \
        --advertise-addr ${NODE1_SWARM_IP} \
        --data-path-addr ${NODE1_SWARM_IP}"
      logn "Swarm initialised on ${NODE1_HOST}."
    fi

    # Check if node2 is already joined.
    if n2 "docker info --format '{{.Swarm.LocalNodeState}}'" 2>/dev/null | grep -q "active"; then
      logn "${NODE2_HOST} is already in the swarm — skipping join."
    else
      logn "Fetching manager join token from ${NODE1_HOST}..."
      JOIN_TOKEN=$(n1 "docker swarm join-token manager -q")
      [[ -z "$JOIN_TOKEN" ]] && err "Failed to get swarm manager join token from ${NODE1_HOST}"

      logn "Joining ${NODE2_HOST} as manager (advertise: ${NODE2_SWARM_IP})..."
      n2 "docker swarm join \
        --token ${JOIN_TOKEN} \
        --advertise-addr ${NODE2_SWARM_IP} \
        --data-path-addr ${NODE2_SWARM_IP} \
        ${NODE1_SWARM_IP}:2377"
      logn "${NODE2_HOST} joined as manager."
    fi

    # Label nodes for service placement constraints.
    logn "Labelling swarm nodes..."
    n1 "docker node update --label-add node=${NODE1_HOST} ${NODE1_HOST}" || true
    n1 "docker node update --label-add node=${NODE2_HOST} ${NODE2_HOST}" || true
    logn "Labels applied."
  fi
fi

# ── Phase 9: DNS registration ─────────────────────────────────────────────────

if ! $SKIP_DNS; then
  log "Phase 9: Registering DNS records in FreeIPA"

  if $DRY_RUN; then
    logn "[dry-run] Would create/verify ${SWARM_ZONE} zone and add A records"
  else
    # Check for and create the swarm.home zone if absent.
    if ! dc "ipa dnszone-show ${SWARM_ZONE}" >/dev/null 2>&1; then
      logn "Creating ${SWARM_ZONE} DNS zone..."
      dc "ipa dnszone-add ${SWARM_ZONE} \
        --name-server=${DC_HOST}. \
        --admin-email=hostmaster@${SWARM_ZONE}"
      # Reverse zone for 192.168.30.x
      dc "ipa dnszone-add 30.168.192.in-addr.arpa \
        --name-server=${DC_HOST}. \
        --admin-email=hostmaster@${SWARM_ZONE}" 2>/dev/null || \
        logn "  (reverse zone 30.168.192.in-addr.arpa already exists or skipped)"
    else
      logn "${SWARM_ZONE} zone already exists."
    fi

    # storage.home A records (zone already exists from HA cluster setup).
    for spec in "${NODE1_HOST}:${NODE1_STORAGE_IP}" "${NODE2_HOST}:${NODE2_STORAGE_IP}"; do
      IFS=: read -r hostname ip <<< "$spec"
      logn "Adding ${hostname}.${STORAGE_ZONE} → ${ip}"
      dc "ipa dnsrecord-add ${STORAGE_ZONE} ${hostname} --a-rec=${ip} --a-create-reverse" 2>/dev/null || \
        logn "  (record already exists or reverse zone missing — continuing)"
    done

    # swarm.home A records.
    for spec in "${NODE1_HOST}:${NODE1_SWARM_IP}" "${NODE2_HOST}:${NODE2_SWARM_IP}"; do
      IFS=: read -r hostname ip <<< "$spec"
      logn "Adding ${hostname}.${SWARM_ZONE} → ${ip}"
      dc "ipa dnsrecord-add ${SWARM_ZONE} ${hostname} --a-rec=${ip} --a-create-reverse" 2>/dev/null || \
        logn "  (record already exists — continuing)"
    done
  fi
fi

# ── Phase 10: Verify ──────────────────────────────────────────────────────────

if ! $SKIP_VERIFY; then
  log "Phase 10: Verifying swarm health"

  if $DRY_RUN; then
    logn "[dry-run] Would verify swarm node list and NFS mounts"
  else
    logn "Swarm node list:"
    n1 "docker node ls" || err "docker node ls failed on ${NODE1_HOST}"

    logn "Checking swarm state on both nodes..."
    for spec in "${NODE1_LAN_IP}:${NODE1_HOST}" "${NODE2_LAN_IP}:${NODE2_HOST}"; do
      IFS=: read -r ip hostname <<< "$spec"
      STATE=$(ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
        "${SWARM_USER}@${ip}" "sudo docker info --format '{{.Swarm.LocalNodeState}}'" 2>/dev/null)
      if [[ "$STATE" != "active" ]]; then
        err "${hostname} swarm state is '${STATE}', expected 'active'"
      fi
      logn "  ${hostname}: swarm=${STATE} ✓"
    done

    logn "Checking NFS mounts on both nodes..."
    for spec in "${NODE1_LAN_IP}:${NODE1_HOST}" "${NODE2_LAN_IP}:${NODE2_HOST}"; do
      IFS=: read -r ip hostname <<< "$spec"
      NFS_OK=$(ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
        "${SWARM_USER}@${ip}" "df -h /mnt/docker/config 2>/dev/null | grep -c nfs || echo 0" 2>/dev/null)
      if [[ "$NFS_OK" -ge 1 ]]; then
        logn "  ${hostname}: /mnt/docker/config NFS mount ✓"
      else
        logn "  WARNING: ${hostname}: /mnt/docker/config does not appear to be NFS-mounted"
        logn "  (automount may still be pending — try: ssh nixos@${ip} 'ls /mnt/docker/config')"
      fi
    done

    logn "Checking overlay network..."
    NETWORKS=$(n1 "docker network ls --filter driver=overlay --format '{{.Name}}'")
    if echo "$NETWORKS" | grep -q "ingress"; then
      logn "  ingress overlay network present ✓"
    else
      logn "  WARNING: ingress overlay network not found — swarm may not be fully initialised"
    fi
  fi
fi

log "Deploy complete. Both nodes are ready for 'docker stack deploy'."
log "Connect to either manager:"
log "  ssh nixos@${NODE1_LAN_IP}   (${NODE1_HOST})"
log "  ssh nixos@${NODE2_LAN_IP}   (${NODE2_HOST})"