Archived
beatzaplentyandClaude Sonnet 4.6
1263c7540c
When create-proxmox-resource.sh clones the repo to pve1, it stays on whatever branch was checked out. Add a pre-build phase that detects branch mismatch and switches the remote clone to the current local branch before building, so the Proxmox node always builds from the same commits we're deploying. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8
377 lines
15 KiB
Bash
Executable File
377 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# deploy.sh — Full lifecycle management for the HA file-server cluster.
|
||
#
|
||
# Handles everything from zero (no VMs, no secrets) through a running,
|
||
# tested cluster, and optionally tears it back down.
|
||
#
|
||
# Usage:
|
||
# scripts/ha/deploy.sh [options]
|
||
# scripts/ha/deploy.sh --destroy [options]
|
||
#
|
||
# Phases (all run by default; skip any with --skip-*):
|
||
# 1. ensure-bridge Create storage bridge (vmbr1) on the Proxmox node if absent.
|
||
# 2. sync-keys Generate SSH host keys and register age keys for both nodes.
|
||
# 3. create-vms Build disk images and create both VMs via create-proxmox-resource.sh.
|
||
# 4. add-hardware Attach storage NIC (vmbr1) and DRBD data disk to each VM.
|
||
# 5. boot-wait Start VMs, wait for SSH on both nodes.
|
||
# 6. cluster-init Form the cluster: DRBD, corosync, Pacemaker, NFS, VIP.
|
||
# Also encrypts the generated corosync authkey into the repo.
|
||
# 7. run-tests Run acceptance tests (T1–T7).
|
||
#
|
||
# Options:
|
||
# --node <host> Proxmox host to deploy on (default: pve1.sweet.home)
|
||
# --vmid1 <n> VMID for ha-server-1 (default: 200)
|
||
# --vmid2 <n> VMID for ha-server-2 (default: 201)
|
||
# --storage <pool> Proxmox storage pool (default: local-zfs)
|
||
# --storage-bridge <br> Bridge for HA storage network (default: vmbr1)
|
||
# --drbd-disk-gb <n> DRBD data disk size in GB (default: 32)
|
||
# --memory <MB> RAM per node (default: 4096)
|
||
# --cores <n> vCPUs per node (default: 4)
|
||
# --skip-ensure-bridge Skip storage bridge creation/check
|
||
# --skip-sync-keys Skip sync-host-keys.sh (clan vars already exist)
|
||
# --skip-create-vms Skip VM creation (VMs already exist)
|
||
# --skip-add-hardware Skip net1/scsi1 attachment (already attached)
|
||
# --skip-boot-wait Skip boot/SSH wait (VMs already running)
|
||
# --skip-cluster-init Skip cluster formation (cluster already configured)
|
||
# --skip-tests Skip acceptance tests
|
||
# --force-rebuild Pass --force-rebuild to create-proxmox-resource.sh
|
||
# --destroy Stop and delete both VMs (skip all other phases)
|
||
# --dry-run Print what would run without executing
|
||
# -h|--help Show this message
|
||
#
|
||
# Prerequisites:
|
||
# - SSH access to the Proxmox node as $PROXMOX_SSH_USER (wayne).
|
||
# - sops age key in the standard location (used by sync-host-keys.sh).
|
||
# - For --skip-sync-keys: clan vars already in vars/per-machine/proxmox-ha-server-{1,2}/.
|
||
# - For full tests: secrets/common.yaml decryptable on both nodes (run
|
||
# `sops updatekeys secrets/common.yaml` after sync-keys).
|
||
|
||
set -euo pipefail
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||
|
||
# shellcheck source=../env.sh
|
||
source "${REPO_ROOT}/scripts/env.sh"
|
||
|
||
# ── Defaults ──────────────────────────────────────────────────────────────────
|
||
|
||
NODE="${PROXMOX_HOST:-$PVE1_HOST}"
|
||
VMID1=200
|
||
VMID2=201
|
||
STORAGE="${PROXMOX_STORAGE:-local-zfs}"
|
||
STORAGE_BRIDGE="vmbr1"
|
||
DRBD_DISK_GB=32
|
||
MEMORY_MB=4096
|
||
CORES=4
|
||
|
||
SKIP_ENSURE_BRIDGE=false
|
||
SKIP_SYNC_KEYS=false
|
||
SKIP_CREATE_VMS=false
|
||
SKIP_ADD_HARDWARE=false
|
||
SKIP_BOOT_WAIT=false
|
||
SKIP_CLUSTER_INIT=false
|
||
SKIP_TESTS=false
|
||
FORCE_REBUILD=false
|
||
DESTROY=false
|
||
DRY_RUN=false
|
||
|
||
# ── Variables from repo ───────────────────────────────────────────────────────
|
||
|
||
NODE1_HOST="ha-server-1"
|
||
NODE2_HOST="ha-server-2"
|
||
NODE1_IP="192.168.2.228"
|
||
NODE2_IP="192.168.2.227"
|
||
STORAGE_IP1="192.168.4.228"
|
||
STORAGE_IP2="192.168.4.227"
|
||
STORAGE_CIDR="192.168.4.0/29"
|
||
SSH_USER="${PROXMOX_SSH_USER:-wayne}"
|
||
|
||
# ── Argument parsing ──────────────────────────────────────────────────────────
|
||
|
||
usage() {
|
||
sed -n '/^# Usage:/,/^[^#]/{ /^#/{ s/^# \?//; p } }' "$0"
|
||
exit "${1:-0}"
|
||
}
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--node) NODE="$2"; shift 2 ;;
|
||
--vmid1) VMID1="$2"; shift 2 ;;
|
||
--vmid2) VMID2="$2"; shift 2 ;;
|
||
--storage) STORAGE="$2"; shift 2 ;;
|
||
--storage-bridge) STORAGE_BRIDGE="$2"; shift 2 ;;
|
||
--drbd-disk-gb) DRBD_DISK_GB="$2"; shift 2 ;;
|
||
--memory) MEMORY_MB="$2"; shift 2 ;;
|
||
--cores) CORES="$2"; shift 2 ;;
|
||
--skip-ensure-bridge) SKIP_ENSURE_BRIDGE=true; shift ;;
|
||
--skip-sync-keys) SKIP_SYNC_KEYS=true; shift ;;
|
||
--skip-create-vms) SKIP_CREATE_VMS=true; shift ;;
|
||
--skip-add-hardware) SKIP_ADD_HARDWARE=true; shift ;;
|
||
--skip-boot-wait) SKIP_BOOT_WAIT=true; shift ;;
|
||
--skip-cluster-init) SKIP_CLUSTER_INIT=true; shift ;;
|
||
--skip-tests) SKIP_TESTS=true; shift ;;
|
||
--force-rebuild) FORCE_REBUILD=true; shift ;;
|
||
--destroy) DESTROY=true; shift ;;
|
||
--dry-run) DRY_RUN=true; shift ;;
|
||
-h|--help) usage 0 ;;
|
||
*) echo "Unknown option: $1" >&2; usage 1 ;;
|
||
esac
|
||
done
|
||
|
||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||
|
||
log() { echo "==> $*"; }
|
||
logn() { echo " $*"; }
|
||
err() { echo "ERROR: $*" >&2; exit 1; }
|
||
|
||
run() {
|
||
if $DRY_RUN; then
|
||
echo "[dry-run] $*"
|
||
else
|
||
"$@"
|
||
fi
|
||
}
|
||
|
||
pve() {
|
||
# Run a command on the Proxmox node via SSH.
|
||
if $DRY_RUN; then
|
||
echo "[dry-run] ssh ${SSH_USER}@${NODE} sudo $*"
|
||
else
|
||
ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
|
||
fi
|
||
}
|
||
|
||
pve_check() {
|
||
# Run a read-only probe on the Proxmox node — always executes even in dry-run.
|
||
ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo $*"
|
||
}
|
||
|
||
n1() {
|
||
# Run a command on ha-server-1 via SSH.
|
||
ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE1_IP}" "$@" 2>/dev/null
|
||
}
|
||
|
||
n2() {
|
||
# Run a command on ha-server-2 via SSH.
|
||
ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=5 "root@${NODE2_IP}" "$@" 2>/dev/null
|
||
}
|
||
|
||
wait_for_ssh() {
|
||
local ip="$1" label="$2"
|
||
if $DRY_RUN; then
|
||
logn "[dry-run] Skipping SSH wait for ${label} (${ip})"
|
||
return 0
|
||
fi
|
||
local deadline=$(( $(date +%s) + 300 ))
|
||
log "Waiting for SSH on ${label} (${ip}) — up to 5 min..."
|
||
while [[ $(date +%s) -lt $deadline ]]; do
|
||
if ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no -o ConnectTimeout=3 \
|
||
-o BatchMode=yes "root@${ip}" true 2>/dev/null; then
|
||
logn "${label} is up."
|
||
return 0
|
||
fi
|
||
sleep 5
|
||
done
|
||
err "Timed out waiting for SSH on ${label} (${ip})"
|
||
}
|
||
|
||
# ── Destroy mode ─────────────────────────────────────────────────────────────
|
||
|
||
if $DESTROY; then
|
||
log "Destroying HA cluster VMs (${VMID1}=${NODE1_HOST}, ${VMID2}=${NODE2_HOST}) on ${NODE}"
|
||
for vmid in "$VMID1" "$VMID2"; do
|
||
STATUS=$(pve "qm status ${vmid} 2>/dev/null" 2>/dev/null || true)
|
||
if echo "$STATUS" | grep -q "running"; then
|
||
log "Stopping VMID ${vmid}..."
|
||
pve "qm stop ${vmid} --skiplock 1"
|
||
sleep 5
|
||
fi
|
||
if $DRY_RUN || pve "qm config ${vmid} >/dev/null 2>&1"; then
|
||
log "Deleting VMID ${vmid}..."
|
||
run pve "qm destroy ${vmid} --purge 1"
|
||
else
|
||
logn "VMID ${vmid} not found — already gone."
|
||
fi
|
||
done
|
||
log "Done — cluster VMs destroyed."
|
||
exit 0
|
||
fi
|
||
|
||
# ── Phase 1: Storage bridge ───────────────────────────────────────────────────
|
||
|
||
if ! $SKIP_ENSURE_BRIDGE; then
|
||
log "Phase 1: Ensuring storage bridge ${STORAGE_BRIDGE} on ${NODE}"
|
||
if pve_check "test -d /sys/class/net/${STORAGE_BRIDGE}" &>/dev/null; then
|
||
logn "${STORAGE_BRIDGE} already exists — skipping."
|
||
else
|
||
logn "Creating isolated internal bridge ${STORAGE_BRIDGE} (no upstream port, ${STORAGE_CIDR})"
|
||
BRIDGE_CONF="auto ${STORAGE_BRIDGE}
|
||
iface ${STORAGE_BRIDGE} inet manual
|
||
bridge-ports none
|
||
bridge-stp off
|
||
bridge-fd 0"
|
||
if $DRY_RUN; then
|
||
echo "[dry-run] Would write /etc/network/interfaces.d/${STORAGE_BRIDGE}.conf and ifup it"
|
||
else
|
||
ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
|
||
"echo '${BRIDGE_CONF}' | sudo tee /etc/network/interfaces.d/${STORAGE_BRIDGE}.conf > /dev/null && sudo ifup ${STORAGE_BRIDGE}"
|
||
logn "${STORAGE_BRIDGE} created and brought up."
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# ── Phase 2: Sync host keys ───────────────────────────────────────────────────
|
||
|
||
if ! $SKIP_SYNC_KEYS; then
|
||
log "Phase 2: Syncing SSH host keys for both HA targets"
|
||
for target in proxmox-ha-server-1 proxmox-ha-server-2; do
|
||
CLAN_DIR="${REPO_ROOT}/vars/per-machine/${target}/openssh"
|
||
if [[ -d "$CLAN_DIR" ]]; then
|
||
logn "Clan vars for ${target} already exist — skipping."
|
||
else
|
||
logn "Generating host keys for ${target}..."
|
||
run bash "${REPO_ROOT}/scripts/secrets/sync-host-keys.sh" "$target"
|
||
fi
|
||
done
|
||
fi
|
||
|
||
# ── Phase 2.5: Prepare Proxmox node for building ─────────────────────────────
|
||
if ! $SKIP_CREATE_VMS && ! $DRY_RUN; then
|
||
CURRENT_BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD)"
|
||
|
||
# Fix /nix ownership if it exists but belongs to a different UID.
|
||
# pve1's IPA-enrolled wayne (UID 50002) can't write to a store created by
|
||
# another UID — passwordless sudo corrects it once.
|
||
if pve_check "test -d /nix" &>/dev/null && ! pve_check "test -w /nix" &>/dev/null; then
|
||
logn "/nix exists but not writable by ${SSH_USER} — fixing ownership with sudo (one-time)..."
|
||
ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" "sudo chown -R ${SSH_USER} /nix"
|
||
logn "Done."
|
||
fi
|
||
|
||
# Ensure the remote clone is on the correct branch so create-proxmox-resource.sh
|
||
# builds from the same commits we're deploying.
|
||
REMOTE_REPO="/home/${SSH_USER}/nixos"
|
||
if pve_check "test -d ${REMOTE_REPO}/.git" &>/dev/null; then
|
||
REMOTE_BRANCH=$(ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
|
||
"cd ${REMOTE_REPO} && git rev-parse --abbrev-ref HEAD 2>/dev/null")
|
||
if [[ "$REMOTE_BRANCH" != "$CURRENT_BRANCH" ]]; then
|
||
logn "Remote clone is on '${REMOTE_BRANCH}', switching to '${CURRENT_BRANCH}'..."
|
||
ssh -i ~/.ssh/id_ed25519 "${SSH_USER}@${NODE}" \
|
||
"cd ${REMOTE_REPO} && git fetch origin && git checkout '${CURRENT_BRANCH}' && git pull --ff-only"
|
||
logn "Done."
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# ── Phase 3: Create VMs ───────────────────────────────────────────────────────
|
||
|
||
if ! $SKIP_CREATE_VMS; then
|
||
log "Phase 3: Building and creating VMs on ${NODE}"
|
||
|
||
REBUILD_FLAG=""
|
||
$FORCE_REBUILD && REBUILD_FLAG="--force-rebuild"
|
||
|
||
CREATE="${REPO_ROOT}/scripts/proxmox/create-proxmox-resource.sh"
|
||
|
||
for spec in "${VMID1}:ha-server-1:proxmox-ha-server-1" "${VMID2}:ha-server-2:proxmox-ha-server-2"; do
|
||
IFS=: read -r vmid host_name flake_target <<< "$spec"
|
||
log "Creating ${flake_target} (VMID ${vmid}) on ${NODE}..."
|
||
run bash "$CREATE" \
|
||
--type vm \
|
||
--host "$host_name" \
|
||
--vmid "$vmid" \
|
||
--node "$NODE" \
|
||
--storage "$STORAGE" \
|
||
--memory "$MEMORY_MB" \
|
||
--cores "$CORES" \
|
||
${REBUILD_FLAG}
|
||
done
|
||
fi
|
||
|
||
# ── Phase 4: Add storage NIC and DRBD disk ────────────────────────────────────
|
||
|
||
if ! $SKIP_ADD_HARDWARE; then
|
||
log "Phase 4: Attaching storage NIC (${STORAGE_BRIDGE}) and DRBD disk (${DRBD_DISK_GB}G) to each VM"
|
||
for vmid in "$VMID1" "$VMID2"; do
|
||
log " VMID ${vmid}: stopping to add hardware..."
|
||
pve "qm stop ${vmid} --skiplock 1 2>/dev/null; sleep 3" || true
|
||
|
||
logn "Adding net1 (${STORAGE_BRIDGE})..."
|
||
pve "qm set ${vmid} --net1 virtio,bridge=${STORAGE_BRIDGE},firewall=0"
|
||
|
||
logn "Adding scsi1 (${STORAGE}:${DRBD_DISK_GB}G for DRBD)..."
|
||
pve "qm set ${vmid} --scsi1 ${STORAGE}:${DRBD_DISK_GB},format=raw"
|
||
|
||
logn "Starting VMID ${vmid}..."
|
||
pve "qm start ${vmid}"
|
||
done
|
||
fi
|
||
|
||
# ── Phase 5: Wait for SSH ─────────────────────────────────────────────────────
|
||
|
||
if ! $SKIP_BOOT_WAIT; then
|
||
log "Phase 5: Waiting for both nodes to come up"
|
||
wait_for_ssh "$NODE1_IP" "$NODE1_HOST"
|
||
wait_for_ssh "$NODE2_IP" "$NODE2_HOST"
|
||
logn "Both nodes are SSHable."
|
||
# Give systemd a few seconds to settle after activation
|
||
sleep 10
|
||
fi
|
||
|
||
# ── Phase 6: Cluster init ─────────────────────────────────────────────────────
|
||
|
||
if ! $SKIP_CLUSTER_INIT; then
|
||
log "Phase 6: Initialising HA cluster"
|
||
|
||
CLUSTER_INIT="${REPO_ROOT}/scripts/ha/cluster-init.sh"
|
||
[[ -x "$CLUSTER_INIT" ]] || chmod +x "$CLUSTER_INIT"
|
||
|
||
if $DRY_RUN; then
|
||
logn "[dry-run] Would scp cluster-init.sh to root@${NODE1_IP} and run it"
|
||
else
|
||
logn "Uploading cluster-init.sh to ${NODE1_HOST}..."
|
||
scp -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no \
|
||
"$CLUSTER_INIT" "root@${NODE1_IP}:/tmp/cluster-init.sh"
|
||
|
||
logn "Running cluster-init.sh on ${NODE1_HOST}..."
|
||
ssh -i ~/.ssh/id_ed25519 -o StrictHostKeyChecking=no "root@${NODE1_IP}" \
|
||
"NODE1=${NODE1_HOST} NODE2=${NODE2_HOST} \
|
||
NODE1_IP=${NODE1_IP} NODE2_IP=${NODE2_IP} \
|
||
VIP=192.168.2.229 XFS_MOUNT=/srv/ha-data \
|
||
ISCSI_IQN=iqn.2026-01.home.sweet:ha-storage \
|
||
VMID_NODE1=${VMID1} VMID_NODE2=${VMID2} \
|
||
bash /tmp/cluster-init.sh"
|
||
|
||
# Encrypt the corosync authkey generated by cluster-init and commit it.
|
||
log " Encrypting corosync authkey into secrets/ha-corosync-authkey..."
|
||
AUTHKEY_TMP="${REPO_ROOT}/secrets/ha-corosync-authkey.tmp"
|
||
n1 "cat /etc/corosync/authkey" > "$AUTHKEY_TMP"
|
||
if [[ ! -s "$AUTHKEY_TMP" ]]; then
|
||
err "corosync authkey on node1 is empty — cluster-init may have failed."
|
||
fi
|
||
mv "$AUTHKEY_TMP" "${REPO_ROOT}/secrets/ha-corosync-authkey"
|
||
(cd "${REPO_ROOT}" && nix run nixpkgs#sops -- -e --input-type binary -i secrets/ha-corosync-authkey)
|
||
logn "Authkey encrypted. Committing..."
|
||
(cd "${REPO_ROOT}" && git add secrets/ha-corosync-authkey && \
|
||
git commit -m "secrets(ha): encrypt corosync authkey generated by cluster-init")
|
||
logn "Committed."
|
||
fi
|
||
fi
|
||
|
||
# ── Phase 7: Acceptance tests ─────────────────────────────────────────────────
|
||
|
||
if ! $SKIP_TESTS; then
|
||
log "Phase 7: Running acceptance tests (T1–T7)"
|
||
if $DRY_RUN; then
|
||
logn "[dry-run] Would run acceptance-tests.sh against ${NODE1_HOST}/${NODE2_HOST}"
|
||
else
|
||
NODE1="$NODE1_HOST" NODE2="$NODE2_HOST" \
|
||
NODE1_IP="$NODE1_IP" NODE2_IP="$NODE2_IP" \
|
||
VIP="192.168.2.229" \
|
||
bash "${REPO_ROOT}/scripts/ha/acceptance-tests.sh"
|
||
fi
|
||
fi
|
||
|
||
log "Deploy complete."
|