Archived
beatzaplentyandClaude Sonnet 4.6
3b6ac30946
The phase 3.5 branch-switch step was hardcoded to /home/user/nixos, which assumes the standalone nixos Gitea repo is checked out there. When the infrastructure mono-repo is used instead, the path differs. Now reads PROXMOX_REMOTE_REPO_DIR (same variable create-proxmox-resource.sh uses) with graceful fallback if the branch switch fails. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DASH15okNvWeY1rVJmyJoJ
596 lines
25 KiB
Bash
Executable File
596 lines
25 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# deploy.sh — Full lifecycle management for the Docker Swarm HA cluster.
|
|
#
|
|
# Provisions two NixOS Proxmox VMs (ha-docker-1, ha-docker-2) as dual-manager
|
|
# Docker Swarm nodes sharing NFS storage from the existing HA file-server
|
|
# cluster. Both nodes are managers so either can accept Docker API and
|
|
# `docker stack` commands.
|
|
#
|
|
# Usage:
|
|
# scripts/docker-swarm/deploy.sh [options]
|
|
# scripts/docker-swarm/deploy.sh --destroy [options]
|
|
#
|
|
# Phases (all run by default; skip any with --skip-<phase>):
|
|
# 1. ensure-bridge Create vmbr3 (swarm cluster bridge) on the Proxmox node.
|
|
# 2. sync-keys Generate SSH host keys for both nodes (clan vars).
|
|
# 3. ipa-hosts Create IPA host objects + sops-encrypted keytabs.
|
|
# 4. create-vms Build NixOS disk images and create VMs via create-proxmox-resource.sh.
|
|
# 5. add-hardware Attach vmbr2 (storage) and vmbr3 (swarm) NICs; start VMs.
|
|
# 6. boot-wait Wait for SSH on both LAN IPs.
|
|
# 7. refresh-sops-keys Detect disko key drift; re-encrypt secrets; commit.
|
|
# 8. init-swarm docker swarm init on node1; manager join on node2; label nodes.
|
|
# 9. dns Register storage.home and swarm.home A records in FreeIPA.
|
|
# 10. verify docker node ls; NFS mount check; swarm health.
|
|
#
|
|
# Options:
|
|
# --node <host> Proxmox host (default: pve1.sweet.home)
|
|
# --vmid1 <n> VMID for ha-docker-1 (default: 202)
|
|
# --vmid2 <n> VMID for ha-docker-2 (default: 203)
|
|
# --storage <pool> Proxmox storage pool (default: local-zfs)
|
|
# --swarm-bridge <br> Bridge for Docker Swarm cluster network (default: vmbr3)
|
|
# --storage-bridge <br> Bridge for NFS storage network (default: vmbr2)
|
|
# --memory <MB> RAM per node (default: 4096)
|
|
# --cores <n> vCPUs per node (default: 4)
|
|
# --skip-ensure-bridge Skip vmbr3 creation/check
|
|
# --skip-sync-keys Skip sync-host-keys.sh (clan vars already exist)
|
|
# --skip-ipa-hosts Skip IPA host account creation (keytabs already exist)
|
|
# --skip-create-vms Skip VM creation (VMs already exist)
|
|
# --skip-add-hardware Skip NIC attachment (already attached)
|
|
# --skip-boot-wait Skip boot/SSH wait (VMs already running)
|
|
# --skip-refresh-sops-keys Skip sops host-key drift fix
|
|
# --skip-init-swarm Skip swarm initialisation (already initialised)
|
|
# --skip-dns Skip FreeIPA DNS record creation
|
|
# --skip-verify Skip post-deploy health checks
|
|
# --force-rebuild Pass --force-rebuild to create-proxmox-resource.sh
|
|
# --destroy Stop and delete both VMs (skip all other phases)
|
|
# --dry-run Print what would run without executing
|
|
# -h|--help Show this message
|
|
#
|
|
# Prerequisites:
|
|
# - SSH access to the Proxmox node as $PROXMOX_SSH_USER (wayne).
|
|
# - sops age key in the standard location (used by sync-host-keys.sh).
|
|
# - SSH access to domain-controller.sweet.home as $PROXMOX_SSH_USER for DNS phase.
|
|
# - For --skip-sync-keys: clan vars already in vars/per-machine/proxmox-ha-docker-{1,2}/.
|
|
# - For --skip-ipa-hosts: secrets/ha-docker-{1,2}.keytab already exist and are committed.
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|
|
|
# shellcheck source=../env.sh
|
|
source "${REPO_ROOT}/scripts/env.sh"
|
|
|
|
# ── Defaults ──────────────────────────────────────────────────────────────────
|
|
|
|
NODE="${PVE1_HOST}" # deploy.sh targets pve1 by default (authorised for this cluster)
|
|
VMID1=202
|
|
VMID2=203
|
|
STORAGE="${PROXMOX_STORAGE:-local-zfs}"
|
|
SWARM_BRIDGE="vmbr3"
|
|
STORAGE_BRIDGE="vmbr2"
|
|
MEMORY_MB=4096
|
|
CORES=4
|
|
|
|
SKIP_ENSURE_BRIDGE=false
|
|
SKIP_SYNC_KEYS=false
|
|
SKIP_IPA_HOSTS=false
|
|
SKIP_CREATE_VMS=false
|
|
SKIP_ADD_HARDWARE=false
|
|
SKIP_BOOT_WAIT=false
|
|
SKIP_REFRESH_SOPS_KEYS=false
|
|
SKIP_INIT_SWARM=false
|
|
SKIP_DNS=false
|
|
SKIP_VERIFY=false
|
|
FORCE_REBUILD=false
|
|
DESTROY=false
|
|
DRY_RUN=false
|
|
|
|
# ── Variables from repo (mirrors variables.nix) ───────────────────────────────
|
|
|
|
NODE1_HOST="ha-docker-1"
|
|
NODE2_HOST="ha-docker-2"
|
|
NODE1_LAN_IP="192.168.2.230"
|
|
NODE2_LAN_IP="192.168.2.231"
|
|
NODE1_SWARM_IP="192.168.30.230"
|
|
NODE2_SWARM_IP="192.168.30.231"
|
|
NODE1_STORAGE_IP="192.168.20.230"
|
|
NODE2_STORAGE_IP="192.168.20.231"
|
|
SWARM_CIDR="192.168.30.0/24"
|
|
STORAGE_CIDR="192.168.20.0/24"
|
|
STORAGE_ZONE="storage.home"
|
|
SWARM_ZONE="swarm.home"
|
|
SSH_USER="${PROXMOX_SSH_USER:-wayne}"
|
|
DC_HOST="${IPA_SERVER:-domain-controller.sweet.home}"
|
|
|
|
# ── Argument parsing ──────────────────────────────────────────────────────────
|
|
|
|
usage() {
|
|
sed -n '/^# Usage:/,/^[^#]/{ /^#/{ s/^# \?//; p } }' "$0"
|
|
exit "${1:-0}"
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--node) NODE="$2"; shift 2 ;;
|
|
--vmid1) VMID1="$2"; shift 2 ;;
|
|
--vmid2) VMID2="$2"; shift 2 ;;
|
|
--storage) STORAGE="$2"; shift 2 ;;
|
|
--swarm-bridge) SWARM_BRIDGE="$2"; shift 2 ;;
|
|
--storage-bridge) STORAGE_BRIDGE="$2"; shift 2 ;;
|
|
--memory) MEMORY_MB="$2"; shift 2 ;;
|
|
--cores) CORES="$2"; shift 2 ;;
|
|
--skip-ensure-bridge) SKIP_ENSURE_BRIDGE=true; shift ;;
|
|
--skip-sync-keys) SKIP_SYNC_KEYS=true; shift ;;
|
|
--skip-ipa-hosts) SKIP_IPA_HOSTS=true; shift ;;
|
|
--skip-create-vms) SKIP_CREATE_VMS=true; shift ;;
|
|
--skip-add-hardware) SKIP_ADD_HARDWARE=true; shift ;;
|
|
--skip-boot-wait) SKIP_BOOT_WAIT=true; shift ;;
|
|
--skip-refresh-sops-keys) SKIP_REFRESH_SOPS_KEYS=true; shift ;;
|
|
--skip-init-swarm) SKIP_INIT_SWARM=true; shift ;;
|
|
--skip-dns) SKIP_DNS=true; shift ;;
|
|
--skip-verify) SKIP_VERIFY=true; shift ;;
|
|
--force-rebuild) FORCE_REBUILD=true; shift ;;
|
|
--destroy) DESTROY=true; shift ;;
|
|
--dry-run) DRY_RUN=true; shift ;;
|
|
-h|--help) usage 0 ;;
|
|
*) echo "Unknown option: $1" >&2; usage 1 ;;
|
|
esac
|
|
done
|
|
|
|
# ── Helpers ───────────────────────────────────────────────────────────────────
|
|
|
|
log() { echo "==> $*"; }
|
|
logn() { echo " $*"; }
|
|
err() { echo "ERROR: $*" >&2; exit 1; }
|
|
|
|
run() {
|
|
if $DRY_RUN; then
|
|
echo "[dry-run] $*"
|
|
else
|
|
"$@"
|
|
fi
|
|
}
|
|
|
|
pve() {
|
|
if $DRY_RUN; then
|
|
echo "[dry-run] ssh ${SSH_USER}@${NODE} sudo $*"
|
|
else
|
|
ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
|
|
fi
|
|
}
|
|
|
|
pve_check() {
|
|
# Read-only probe — always executes even in dry-run.
|
|
ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
|
|
}
|
|
|
|
SWARM_USER="nixos"
|
|
|
|
n1() {
|
|
ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
|
|
"${SWARM_USER}@${NODE1_LAN_IP}" sudo "$@" 2>/dev/null
|
|
}
|
|
|
|
n2() {
|
|
ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
|
|
"${SWARM_USER}@${NODE2_LAN_IP}" sudo "$@" 2>/dev/null
|
|
}
|
|
|
|
dc() {
|
|
# Run ipa commands on domain-controller as $SSH_USER.
|
|
if $DRY_RUN; then
|
|
echo "[dry-run] ssh ${SSH_USER}@${DC_HOST} $*"
|
|
return 0
|
|
fi
|
|
ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${DC_HOST}" "$@"
|
|
}
|
|
|
|
wait_for_ssh() {
|
|
local ip="$1" label="$2"
|
|
if $DRY_RUN; then
|
|
logn "[dry-run] Skipping SSH wait for ${label} (${ip})"
|
|
return 0
|
|
fi
|
|
local deadline=$(( $(date +%s) + 300 ))
|
|
log "Waiting for SSH on ${label} (${ip}) — up to 5 min..."
|
|
while [[ $(date +%s) -lt $deadline ]]; do
|
|
if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=3 \
|
|
-o BatchMode=yes "${SWARM_USER}@${ip}" true 2>/dev/null; then
|
|
logn "${label} is up."
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
err "Timed out waiting for SSH on ${label} (${ip})"
|
|
}
|
|
|
|
# ── Destroy mode ──────────────────────────────────────────────────────────────
|
|
|
|
if $DESTROY; then
|
|
log "Destroying Docker Swarm VMs (${VMID1}=${NODE1_HOST}, ${VMID2}=${NODE2_HOST}) on ${NODE}"
|
|
for vmid in "$VMID1" "$VMID2"; do
|
|
STATUS=$(pve "qm status ${vmid} 2>/dev/null" 2>/dev/null || true)
|
|
if echo "$STATUS" | grep -q "running"; then
|
|
log "Stopping VMID ${vmid}..."
|
|
pve "qm stop ${vmid} --skiplock 1"
|
|
sleep 5
|
|
fi
|
|
if $DRY_RUN || pve "qm config ${vmid} >/dev/null 2>&1"; then
|
|
log "Deleting VMID ${vmid}..."
|
|
run pve "qm destroy ${vmid} --purge 1"
|
|
else
|
|
logn "VMID ${vmid} not found — already gone."
|
|
fi
|
|
done
|
|
log "Done — swarm VMs destroyed."
|
|
exit 0
|
|
fi
|
|
|
|
# ── Phase 1: Ensure swarm bridge ──────────────────────────────────────────────
|
|
|
|
if ! $SKIP_ENSURE_BRIDGE; then
|
|
log "Phase 1: Ensuring swarm bridge ${SWARM_BRIDGE} on ${NODE}"
|
|
if pve_check "test -d /sys/class/net/${SWARM_BRIDGE}" &>/dev/null; then
|
|
logn "${SWARM_BRIDGE} already exists — skipping."
|
|
else
|
|
logn "Creating isolated internal bridge ${SWARM_BRIDGE} (no upstream port, ${SWARM_CIDR})"
|
|
BRIDGE_CONF="auto ${SWARM_BRIDGE}
|
|
iface ${SWARM_BRIDGE} inet manual
|
|
bridge-ports none
|
|
bridge-stp off
|
|
bridge-fd 0"
|
|
if $DRY_RUN; then
|
|
echo "[dry-run] Would write /etc/network/interfaces.d/${SWARM_BRIDGE}.conf and ifup it"
|
|
else
|
|
ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
|
|
"echo '${BRIDGE_CONF}' | sudo tee /etc/network/interfaces.d/${SWARM_BRIDGE}.conf > /dev/null && sudo ifup ${SWARM_BRIDGE}"
|
|
logn "${SWARM_BRIDGE} created and brought up."
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ── Phase 2: Sync host keys ───────────────────────────────────────────────────
|
|
|
|
if ! $SKIP_SYNC_KEYS; then
|
|
log "Phase 2: Syncing SSH host keys for both swarm targets"
|
|
for target in proxmox-ha-docker-1 proxmox-ha-docker-2; do
|
|
CLAN_DIR="${REPO_ROOT}/vars/per-machine/${target}/openssh"
|
|
if [[ -d "$CLAN_DIR" ]]; then
|
|
logn "Clan vars for ${target} already exist — skipping."
|
|
else
|
|
logn "Generating host keys for ${target}..."
|
|
run bash "${REPO_ROOT}/scripts/secrets/sync-host-keys.sh" "$target"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# ── Phase 3: IPA host accounts ────────────────────────────────────────────────
|
|
|
|
if ! $SKIP_IPA_HOSTS; then
|
|
log "Phase 3: Creating IPA host accounts and keytabs"
|
|
IPA_SCRIPT="${REPO_ROOT}/scripts/ipa/create-nixos-ipa-host-account.sh"
|
|
for host in "${NODE1_HOST}" "${NODE2_HOST}"; do
|
|
KEYTAB="${REPO_ROOT}/secrets/${host}.keytab"
|
|
if [[ -f "$KEYTAB" ]]; then
|
|
logn "Keytab for ${host} already exists — skipping."
|
|
else
|
|
logn "Creating IPA host account and keytab for ${host}..."
|
|
run bash "$IPA_SCRIPT" "$host"
|
|
fi
|
|
done
|
|
|
|
if ! $DRY_RUN; then
|
|
# Keytabs must be committed and pushed before VMs rebuild from Gitea.
|
|
CURRENT_BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD)"
|
|
logn "Committing keytabs and pushing to Gitea (branch: ${CURRENT_BRANCH})..."
|
|
(cd "${REPO_ROOT}" && \
|
|
git add secrets/ha-docker-1.keytab secrets/ha-docker-2.keytab .sops.yaml && \
|
|
git commit -m "secrets(ha-docker): add IPA keytabs for ha-docker-1 and ha-docker-2" || true && \
|
|
git push origin "${CURRENT_BRANCH}")
|
|
logn "Pushed."
|
|
fi
|
|
fi
|
|
|
|
# ── Phase 3.5: Prepare Proxmox node for building ─────────────────────────────
|
|
|
|
if ! $SKIP_CREATE_VMS && ! $DRY_RUN; then
|
|
CURRENT_BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD)"
|
|
|
|
local_ssh() { ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "$*"; }
|
|
if local_ssh "test -d /nix" &>/dev/null && ! local_ssh "test -w /nix" &>/dev/null; then
|
|
logn "/nix exists but not writable by ${SSH_USER} — fixing ownership with sudo..."
|
|
local_ssh "sudo chown -R ${SSH_USER} /nix"
|
|
logn "Done."
|
|
fi
|
|
unset -f local_ssh
|
|
|
|
# Use PROXMOX_REMOTE_REPO_DIR (from env.sh) so the build path is consistent
|
|
# with what create-proxmox-resource.sh will use. The default is
|
|
# /home/<user>/nixos (the standalone nixos repo clone on pve1), but can be
|
|
# overridden to e.g. /home/<user>/infrastructure/nixos when the infrastructure
|
|
# mono-repo is checked out on pve1 instead.
|
|
REMOTE_REPO="${PROXMOX_REMOTE_REPO_DIR:-/home/${SSH_USER}/nixos}"
|
|
if pve_check "test -d ${REMOTE_REPO}/.git" &>/dev/null; then
|
|
REMOTE_BRANCH=$(ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
|
|
"cd ${REMOTE_REPO} && git rev-parse --abbrev-ref HEAD 2>/dev/null")
|
|
if [[ "$REMOTE_BRANCH" != "$CURRENT_BRANCH" ]]; then
|
|
logn "Remote repo (${REMOTE_REPO}) is on '${REMOTE_BRANCH}', switching to '${CURRENT_BRANCH}'..."
|
|
if ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
|
|
"cd ${REMOTE_REPO} && git fetch origin && git checkout '${CURRENT_BRANCH}' && git pull --ff-only" 2>&1; then
|
|
logn "Done."
|
|
else
|
|
logn "WARNING: branch switch failed — proceeding anyway (create-proxmox-resource.sh will retry)"
|
|
fi
|
|
fi
|
|
else
|
|
logn "Remote repo ${REMOTE_REPO} not found on ${NODE} — create-proxmox-resource.sh will clone it."
|
|
fi
|
|
fi
|
|
|
|
# ── Phase 4: Create VMs ───────────────────────────────────────────────────────
|
|
|
|
if ! $SKIP_CREATE_VMS; then
|
|
log "Phase 4: Building and creating swarm VMs on ${NODE}"
|
|
CREATE="${REPO_ROOT}/scripts/proxmox/create-proxmox-resource.sh"
|
|
REBUILD_FLAG=""
|
|
$FORCE_REBUILD && REBUILD_FLAG="--force-rebuild"
|
|
|
|
for spec in "${VMID1}:${NODE1_HOST}:proxmox-ha-docker-1" "${VMID2}:${NODE2_HOST}:proxmox-ha-docker-2"; do
|
|
IFS=: read -r vmid host_name flake_target <<< "$spec"
|
|
log "Creating ${flake_target} (VMID ${vmid}) on ${NODE}..."
|
|
# --force-rebuild is always passed: create-proxmox-resource.sh only calls
|
|
# sync_remote_host_keys (which bakes the clan-var SSH key into the disk
|
|
# image) when it actually builds. Reusing a cached image skips that step
|
|
# and leaves the VM unable to decrypt sops secrets on first boot.
|
|
run bash "$CREATE" \
|
|
--type vm \
|
|
--host "$host_name" \
|
|
--vmid "$vmid" \
|
|
--node "$NODE" \
|
|
--storage "$STORAGE" \
|
|
--memory "$MEMORY_MB" \
|
|
--cores "$CORES" \
|
|
--force-rebuild
|
|
done
|
|
fi
|
|
|
|
# ── Phase 5: Add NICs and start VMs ──────────────────────────────────────────
|
|
|
|
if ! $SKIP_ADD_HARDWARE; then
|
|
log "Phase 5: Attaching storage (${STORAGE_BRIDGE}) and swarm (${SWARM_BRIDGE}) NICs"
|
|
for vmid in "$VMID1" "$VMID2"; do
|
|
logn "VMID ${vmid}: stopping to add NICs..."
|
|
pve "qm stop ${vmid} --skiplock 1 2>/dev/null; sleep 3" || true
|
|
|
|
logn "Adding net1 (${STORAGE_BRIDGE} — NFS storage)..."
|
|
pve "qm set ${vmid} --net1 virtio,bridge=${STORAGE_BRIDGE},firewall=0"
|
|
|
|
logn "Adding net2 (${SWARM_BRIDGE} — Docker Swarm)..."
|
|
pve "qm set ${vmid} --net2 virtio,bridge=${SWARM_BRIDGE},firewall=0"
|
|
|
|
logn "Starting VMID ${vmid}..."
|
|
pve "qm start ${vmid}"
|
|
done
|
|
fi
|
|
|
|
# ── Phase 6: Wait for SSH ─────────────────────────────────────────────────────
|
|
|
|
if ! $SKIP_BOOT_WAIT; then
|
|
log "Phase 6: Waiting for both nodes to come up on LAN IPs"
|
|
wait_for_ssh "$NODE1_LAN_IP" "$NODE1_HOST"
|
|
wait_for_ssh "$NODE2_LAN_IP" "$NODE2_HOST"
|
|
logn "Both nodes are SSHable."
|
|
sleep 10 # let systemd finish activation
|
|
fi
|
|
|
|
# ── Phase 7: Refresh sops host-key registrations ─────────────────────────────
|
|
#
|
|
# Disko builds raw disk images: each new VM boots with a freshly-generated SSH
|
|
# host key, not the one pre-seeded in clan vars. Scan the running VMs; if
|
|
# their ed25519 keys differ from the clan var, update the clan var, rewrite
|
|
# the .sops.yaml anchor, and re-encrypt all affected sops files.
|
|
|
|
if ! $SKIP_REFRESH_SOPS_KEYS; then
|
|
if $DRY_RUN; then
|
|
logn "[dry-run] Would scan VM host keys and refresh .sops.yaml / secrets if needed"
|
|
else
|
|
log "Phase 7: Refreshing sops host-key registrations (disko key drift fix)"
|
|
SOPS_UPDATED=false
|
|
|
|
for spec in \
|
|
"${NODE1_LAN_IP}:proxmox-ha-docker-1:${NODE1_HOST}" \
|
|
"${NODE2_LAN_IP}:proxmox-ha-docker-2:${NODE2_HOST}"; do
|
|
IFS=: read -r node_ip flake_target host_name <<< "$spec"
|
|
CLAN_PUB="${REPO_ROOT}/vars/per-machine/${flake_target}/openssh/ssh_host_ed25519_key.pub/value"
|
|
|
|
logn "Scanning ed25519 host key from ${host_name} (${node_ip})..."
|
|
RAW=$(ssh-keyscan -t ed25519 "${node_ip}" 2>/dev/null | grep -v "^#") || true
|
|
if [[ -z "$RAW" ]]; then
|
|
logn "WARNING: no ed25519 key returned for ${node_ip} — skipping"
|
|
continue
|
|
fi
|
|
SCANNED_TYPE=$(awk '{print $2}' <<< "$RAW")
|
|
SCANNED_KEY=$(awk '{print $3}' <<< "$RAW")
|
|
SCANNED_PUBKEY="${SCANNED_TYPE} ${SCANNED_KEY} ${host_name}"
|
|
|
|
CURRENT=$(tr -d '\n' < "$CLAN_PUB" 2>/dev/null || true)
|
|
if [[ "$SCANNED_PUBKEY" == "$CURRENT" ]]; then
|
|
logn "${host_name}: clan var matches running key — no update needed"
|
|
continue
|
|
fi
|
|
|
|
logn "${host_name}: key drift detected — updating clan var"
|
|
logn " old: ${CURRENT}"
|
|
logn " new: ${SCANNED_PUBKEY}"
|
|
echo "$SCANNED_PUBKEY" > "$CLAN_PUB"
|
|
SOPS_UPDATED=true
|
|
|
|
ANCHOR="${flake_target}"
|
|
NEW_AGE=$(echo "$SCANNED_PUBKEY" | \
|
|
nix run --quiet --no-warn-dirty nixpkgs#ssh-to-age 2>/dev/null)
|
|
[[ -z "$NEW_AGE" ]] && err "ssh-to-age produced no output for ${host_name}"
|
|
logn " new age key: ${NEW_AGE}"
|
|
sed -i "/&${ANCHOR} /s| age[a-z0-9]*$| ${NEW_AGE}|" "${REPO_ROOT}/.sops.yaml"
|
|
done
|
|
|
|
if $SOPS_UPDATED; then
|
|
logn "Running sops updatekeys on affected secrets..."
|
|
SOPS="nix run --quiet --no-warn-dirty nixpkgs#sops --"
|
|
(cd "${REPO_ROOT}" && \
|
|
$SOPS updatekeys -y secrets/common.yaml && \
|
|
$SOPS updatekeys -y secrets/ha-docker-1.keytab && \
|
|
$SOPS updatekeys -y secrets/ha-docker-2.keytab)
|
|
|
|
logn "Committing refreshed host keys and re-encrypted secrets..."
|
|
(cd "${REPO_ROOT}" && \
|
|
git add \
|
|
vars/per-machine/proxmox-ha-docker-1/openssh/ssh_host_ed25519_key.pub/value \
|
|
vars/per-machine/proxmox-ha-docker-2/openssh/ssh_host_ed25519_key.pub/value \
|
|
.sops.yaml \
|
|
secrets/common.yaml \
|
|
secrets/ha-docker-1.keytab \
|
|
secrets/ha-docker-2.keytab && \
|
|
git commit -m "secrets(ha-docker): refresh sops host-key registrations for new VM instances" || true)
|
|
logn "Sops keys refreshed and committed."
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ── Phase 8: Initialise Docker Swarm ─────────────────────────────────────────
|
|
|
|
if ! $SKIP_INIT_SWARM; then
|
|
log "Phase 8: Initialising Docker Swarm"
|
|
|
|
if $DRY_RUN; then
|
|
logn "[dry-run] Would run: docker swarm init --advertise-addr ${NODE1_SWARM_IP} --data-path-addr ${NODE1_SWARM_IP} on ${NODE1_HOST}"
|
|
logn "[dry-run] Would join ${NODE2_HOST} as manager"
|
|
logn "[dry-run] Would label both nodes"
|
|
else
|
|
# Check if node1 is already a swarm manager.
|
|
if n1 "docker info --format '{{.Swarm.LocalNodeState}}'" 2>/dev/null | grep -q "active"; then
|
|
logn "${NODE1_HOST} is already in a swarm — skipping init."
|
|
else
|
|
logn "Initialising swarm on ${NODE1_HOST} (advertise: ${NODE1_SWARM_IP})..."
|
|
n1 "docker swarm init \
|
|
--advertise-addr ${NODE1_SWARM_IP} \
|
|
--data-path-addr ${NODE1_SWARM_IP}"
|
|
logn "Swarm initialised on ${NODE1_HOST}."
|
|
fi
|
|
|
|
# Check if node2 is already joined.
|
|
if n2 "docker info --format '{{.Swarm.LocalNodeState}}'" 2>/dev/null | grep -q "active"; then
|
|
logn "${NODE2_HOST} is already in the swarm — skipping join."
|
|
else
|
|
logn "Fetching manager join token from ${NODE1_HOST}..."
|
|
JOIN_TOKEN=$(n1 "docker swarm join-token manager -q")
|
|
[[ -z "$JOIN_TOKEN" ]] && err "Failed to get swarm manager join token from ${NODE1_HOST}"
|
|
|
|
logn "Joining ${NODE2_HOST} as manager (advertise: ${NODE2_SWARM_IP})..."
|
|
n2 "docker swarm join \
|
|
--token ${JOIN_TOKEN} \
|
|
--advertise-addr ${NODE2_SWARM_IP} \
|
|
--data-path-addr ${NODE2_SWARM_IP} \
|
|
${NODE1_SWARM_IP}:2377"
|
|
logn "${NODE2_HOST} joined as manager."
|
|
fi
|
|
|
|
# Label nodes for service placement constraints.
|
|
logn "Labelling swarm nodes..."
|
|
n1 "docker node update --label-add node=${NODE1_HOST} ${NODE1_HOST}" || true
|
|
n1 "docker node update --label-add node=${NODE2_HOST} ${NODE2_HOST}" || true
|
|
logn "Labels applied."
|
|
fi
|
|
fi
|
|
|
|
# ── Phase 9: DNS registration ─────────────────────────────────────────────────
|
|
|
|
if ! $SKIP_DNS; then
|
|
log "Phase 9: Registering DNS records in FreeIPA"
|
|
|
|
if $DRY_RUN; then
|
|
logn "[dry-run] Would create/verify ${SWARM_ZONE} zone and add A records"
|
|
else
|
|
# Check for and create the swarm.home zone if absent.
|
|
if ! dc "ipa dnszone-show ${SWARM_ZONE}" >/dev/null 2>&1; then
|
|
logn "Creating ${SWARM_ZONE} DNS zone..."
|
|
dc "ipa dnszone-add ${SWARM_ZONE} \
|
|
--name-server=${DC_HOST}. \
|
|
--admin-email=hostmaster@${SWARM_ZONE}"
|
|
# Reverse zone for 192.168.30.x
|
|
dc "ipa dnszone-add 30.168.192.in-addr.arpa \
|
|
--name-server=${DC_HOST}. \
|
|
--admin-email=hostmaster@${SWARM_ZONE}" 2>/dev/null || \
|
|
logn " (reverse zone 30.168.192.in-addr.arpa already exists or skipped)"
|
|
else
|
|
logn "${SWARM_ZONE} zone already exists."
|
|
fi
|
|
|
|
# storage.home A records (zone already exists from HA cluster setup).
|
|
for spec in "${NODE1_HOST}:${NODE1_STORAGE_IP}" "${NODE2_HOST}:${NODE2_STORAGE_IP}"; do
|
|
IFS=: read -r hostname ip <<< "$spec"
|
|
logn "Adding ${hostname}.${STORAGE_ZONE} → ${ip}"
|
|
dc "ipa dnsrecord-add ${STORAGE_ZONE} ${hostname} --a-rec=${ip} --a-create-reverse" 2>/dev/null || \
|
|
logn " (record already exists or reverse zone missing — continuing)"
|
|
done
|
|
|
|
# swarm.home A records.
|
|
for spec in "${NODE1_HOST}:${NODE1_SWARM_IP}" "${NODE2_HOST}:${NODE2_SWARM_IP}"; do
|
|
IFS=: read -r hostname ip <<< "$spec"
|
|
logn "Adding ${hostname}.${SWARM_ZONE} → ${ip}"
|
|
dc "ipa dnsrecord-add ${SWARM_ZONE} ${hostname} --a-rec=${ip} --a-create-reverse" 2>/dev/null || \
|
|
logn " (record already exists — continuing)"
|
|
done
|
|
fi
|
|
fi
|
|
|
|
# ── Phase 10: Verify ──────────────────────────────────────────────────────────
|
|
|
|
if ! $SKIP_VERIFY; then
|
|
log "Phase 10: Verifying swarm health"
|
|
|
|
if $DRY_RUN; then
|
|
logn "[dry-run] Would verify swarm node list and NFS mounts"
|
|
else
|
|
logn "Swarm node list:"
|
|
n1 "docker node ls" || err "docker node ls failed on ${NODE1_HOST}"
|
|
|
|
logn "Checking swarm state on both nodes..."
|
|
for spec in "${NODE1_LAN_IP}:${NODE1_HOST}" "${NODE2_LAN_IP}:${NODE2_HOST}"; do
|
|
IFS=: read -r ip hostname <<< "$spec"
|
|
STATE=$(ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
|
|
"${SWARM_USER}@${ip}" "sudo docker info --format '{{.Swarm.LocalNodeState}}'" 2>/dev/null)
|
|
if [[ "$STATE" != "active" ]]; then
|
|
err "${hostname} swarm state is '${STATE}', expected 'active'"
|
|
fi
|
|
logn " ${hostname}: swarm=${STATE} ✓"
|
|
done
|
|
|
|
logn "Checking NFS mounts on both nodes..."
|
|
for spec in "${NODE1_LAN_IP}:${NODE1_HOST}" "${NODE2_LAN_IP}:${NODE2_HOST}"; do
|
|
IFS=: read -r ip hostname <<< "$spec"
|
|
NFS_OK=$(ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
|
|
"${SWARM_USER}@${ip}" "df -h /mnt/docker/config 2>/dev/null | grep -c nfs || echo 0" 2>/dev/null)
|
|
if [[ "$NFS_OK" -ge 1 ]]; then
|
|
logn " ${hostname}: /mnt/docker/config NFS mount ✓"
|
|
else
|
|
logn " WARNING: ${hostname}: /mnt/docker/config does not appear to be NFS-mounted"
|
|
logn " (automount may still be pending — try: ssh nixos@${ip} 'ls /mnt/docker/config')"
|
|
fi
|
|
done
|
|
|
|
logn "Checking overlay network..."
|
|
NETWORKS=$(n1 "docker network ls --filter driver=overlay --format '{{.Name}}'")
|
|
if echo "$NETWORKS" | grep -q "ingress"; then
|
|
logn " ingress overlay network present ✓"
|
|
else
|
|
logn " WARNING: ingress overlay network not found — swarm may not be fully initialised"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
log "Deploy complete. Both nodes are ready for 'docker stack deploy'."
|
|
log "Connect to either manager:"
|
|
log " ssh nixos@${NODE1_LAN_IP} (${NODE1_HOST})"
|
|
log " ssh nixos@${NODE2_LAN_IP} (${NODE2_HOST})"
|