Archived
Compare commits
2
Commits
43b4cc6aa6
...
684351b89b
| Author | SHA1 | Date | |
|---|---|---|---|
  beatzaplentyandClaude Sonnet 4.6
|
684351b89b | ||
|
beatzaplenty
|
327ff0b44d |
@@ -135,14 +135,33 @@ for i in $(seq 1 30); do
|
||||
done
|
||||
|
||||
# ── 2. DRBD initialisation ────────────────────────────────────────────────
|
||||
# Down DRBD first on both nodes before (re-)initialising metadata.
|
||||
# This ensures /dev/sdb is not held open by the kernel module, which would
|
||||
# trigger a drbdmeta TTY-confirmation prompt ("stdin not a TTY, not waiting
|
||||
# for confirmation") during the write-dev-uuid step even when --force is set.
|
||||
log "Detaching DRBD on $NODE1 (idempotent pre-init clean-up)..."
|
||||
# Put both nodes in Pacemaker standby before touching DRBD metadata.
|
||||
# Without this, the OCF DRBD agent races: it sees drbdadm-down as a failure
|
||||
# and immediately calls drbdadm-up again, leaving /dev/sdb busy when
|
||||
# create-md / write-dev-uuid runs. On a fresh cluster with no resources
|
||||
# configured this is a no-op; on a re-run it stops the race.
|
||||
log "Setting both nodes to Pacemaker standby for DRBD metadata init..."
|
||||
crm_standby -N "$NODE1" -v on 2>/dev/null || true
|
||||
crm_standby -N "$NODE2" -v on 2>/dev/null || true
|
||||
|
||||
# Wait for Pacemaker to actually stop DRBD (if it was managing it).
|
||||
log "Waiting for DRBD to stop under Pacemaker control..."
|
||||
for i in $(seq 1 30); do
|
||||
n1_role=$(drbdadm role ha-data 2>/dev/null || echo "Unconfigured")
|
||||
n2_role=$(n2_ssh "drbdadm role ha-data 2>/dev/null" 2>/dev/null || echo "Unconfigured")
|
||||
if [[ "$n1_role" == "Unconfigured" ]] && [[ "$n2_role" == "Unconfigured" ]]; then
|
||||
log "DRBD stopped on both nodes"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 30 ]] && warn "DRBD still active after 60s standby — forcing down anyway"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log "Detaching DRBD on $NODE1 (belt-and-suspenders after standby)..."
|
||||
drbdadm down ha-data 2>/dev/null || true
|
||||
log "Detaching DRBD on $NODE2 (idempotent pre-init clean-up)..."
|
||||
log "Detaching DRBD on $NODE2..."
|
||||
n2_ssh "drbdadm down ha-data 2>/dev/null || true"
|
||||
sleep 2
|
||||
|
||||
log "Initialising DRBD metadata on $NODE1..."
|
||||
if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate\|Inconsistent\|Diskless"; then
|
||||
@@ -157,6 +176,10 @@ log "Bringing up DRBD on both nodes..."
|
||||
drbdadm up ha-data 2>/dev/null || true
|
||||
n2_ssh "drbdadm up ha-data" 2>/dev/null || true
|
||||
|
||||
log "Clearing Pacemaker standby — DRBD is up, letting Pacemaker resume..."
|
||||
crm_standby -N "$NODE1" -v off 2>/dev/null || true
|
||||
crm_standby -N "$NODE2" -v off 2>/dev/null || true
|
||||
|
||||
log "Forcing $NODE1 to DRBD Primary for initial sync..."
|
||||
drbdadm primary ha-data --force
|
||||
|
||||
|
||||
+95
-6
@@ -32,6 +32,7 @@
|
||||
# --skip-create-vms Skip VM creation (VMs already exist)
|
||||
# --skip-add-hardware Skip net1/scsi1 attachment (already attached)
|
||||
# --skip-boot-wait Skip boot/SSH wait (VMs already running)
|
||||
# --skip-refresh-sops-keys Skip scanning running VMs for fresh SSH host keys
|
||||
# --skip-cluster-init Skip cluster formation (cluster already configured)
|
||||
# --skip-tests Skip acceptance tests
|
||||
# --force-rebuild Pass --force-rebuild to create-proxmox-resource.sh
|
||||
@@ -70,6 +71,7 @@ SKIP_SYNC_KEYS=false
|
||||
SKIP_CREATE_VMS=false
|
||||
SKIP_ADD_HARDWARE=false
|
||||
SKIP_BOOT_WAIT=false
|
||||
SKIP_REFRESH_SOPS_KEYS=false
|
||||
SKIP_CLUSTER_INIT=false
|
||||
SKIP_TESTS=false
|
||||
FORCE_REBUILD=false
|
||||
@@ -108,8 +110,9 @@ while [[ $# -gt 0 ]]; do
|
||||
--skip-sync-keys) SKIP_SYNC_KEYS=true; shift ;;
|
||||
--skip-create-vms) SKIP_CREATE_VMS=true; shift ;;
|
||||
--skip-add-hardware) SKIP_ADD_HARDWARE=true; shift ;;
|
||||
--skip-boot-wait) SKIP_BOOT_WAIT=true; shift ;;
|
||||
--skip-cluster-init) SKIP_CLUSTER_INIT=true; shift ;;
|
||||
--skip-boot-wait) SKIP_BOOT_WAIT=true; shift ;;
|
||||
--skip-refresh-sops-keys) SKIP_REFRESH_SOPS_KEYS=true; shift ;;
|
||||
--skip-cluster-init) SKIP_CLUSTER_INIT=true; shift ;;
|
||||
--skip-tests) SKIP_TESTS=true; shift ;;
|
||||
--force-rebuild) FORCE_REBUILD=true; shift ;;
|
||||
--destroy) DESTROY=true; shift ;;
|
||||
@@ -275,14 +278,16 @@ fi
|
||||
if ! $SKIP_CREATE_VMS; then
|
||||
log "Phase 3: Building and creating VMs on ${NODE}"
|
||||
|
||||
REBUILD_FLAG=""
|
||||
$FORCE_REBUILD && REBUILD_FLAG="--force-rebuild"
|
||||
|
||||
CREATE="${REPO_ROOT}/scripts/proxmox/create-proxmox-resource.sh"
|
||||
|
||||
for spec in "${VMID1}:ha-server-1:proxmox-ha-server-1" "${VMID2}:ha-server-2:proxmox-ha-server-2"; do
|
||||
IFS=: read -r vmid host_name flake_target <<< "$spec"
|
||||
log "Creating ${flake_target} (VMID ${vmid}) on ${NODE}..."
|
||||
# Always --force-rebuild: create-proxmox-resource.sh only calls
|
||||
# sync_remote_host_keys (which populates host-keys/ for proxmox.nix to
|
||||
# bake the clan-var SSH key into the disko image) when it actually builds.
|
||||
# Reusing a cached image skips that step, so destroy+recreate would reuse
|
||||
# an image with a stale/random key baked in → sops fails on first boot.
|
||||
run bash "$CREATE" \
|
||||
--type vm \
|
||||
--host "$host_name" \
|
||||
@@ -291,7 +296,7 @@ if ! $SKIP_CREATE_VMS; then
|
||||
--storage "$STORAGE" \
|
||||
--memory "$MEMORY_MB" \
|
||||
--cores "$CORES" \
|
||||
${REBUILD_FLAG}
|
||||
--force-rebuild
|
||||
done
|
||||
fi
|
||||
|
||||
@@ -325,6 +330,90 @@ if ! $SKIP_BOOT_WAIT; then
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
# ── Phase 5.5: Refresh sops host-key registrations ───────────────────────────
|
||||
#
|
||||
# Disko builds raw disk images; each new VM boots with a freshly-generated SSH
|
||||
# host key rather than the one pre-seeded in clan vars. This phase scans the
|
||||
# actual running VMs, and if their ed25519 host keys differ from what clan vars
|
||||
# record: updates the clan var pub-key files, rewrites the .sops.yaml age-key
|
||||
# anchors, and re-encrypts all affected sops files so the nodes can decrypt
|
||||
# secrets on the next nixos-rebuild. Safe no-op when keys haven't changed.
|
||||
|
||||
if ! $SKIP_REFRESH_SOPS_KEYS; then
|
||||
if $DRY_RUN; then
|
||||
logn "[dry-run] Would scan VM host keys and refresh .sops.yaml / secrets if needed"
|
||||
else
|
||||
log "Phase 5.5: Refreshing sops host-key registrations (disko key drift fix)"
|
||||
SOPS_UPDATED=false
|
||||
|
||||
for spec in \
|
||||
"${NODE1_IP}:proxmox-ha-server-1:${NODE1_HOST}" \
|
||||
"${NODE2_IP}:proxmox-ha-server-2:${NODE2_HOST}"; do
|
||||
IFS=: read -r node_ip flake_target host_name <<< "$spec"
|
||||
CLAN_PUB="${REPO_ROOT}/vars/per-machine/${flake_target}/openssh/ssh_host_ed25519_key.pub/value"
|
||||
|
||||
logn "Scanning ed25519 host key from ${host_name} (${node_ip})..."
|
||||
RAW=$(ssh-keyscan -t ed25519 "${node_ip}" 2>/dev/null | grep -v "^#") || true
|
||||
if [[ -z "$RAW" ]]; then
|
||||
logn "WARNING: no ed25519 key returned by ssh-keyscan for ${node_ip} — skipping"
|
||||
continue
|
||||
fi
|
||||
# ssh-keyscan returns: <ip> ssh-ed25519 <b64key>
|
||||
SCANNED_TYPE=$(awk '{print $2}' <<< "$RAW")
|
||||
SCANNED_KEY=$(awk '{print $3}' <<< "$RAW")
|
||||
SCANNED_PUBKEY="${SCANNED_TYPE} ${SCANNED_KEY} ${host_name}"
|
||||
|
||||
CURRENT=$(tr -d '\n' < "$CLAN_PUB" 2>/dev/null || true)
|
||||
if [[ "$SCANNED_PUBKEY" == "$CURRENT" ]]; then
|
||||
logn "${host_name}: clan var matches running key — no update needed"
|
||||
continue
|
||||
fi
|
||||
|
||||
logn "${host_name}: key drift detected — updating clan var"
|
||||
logn " old: ${CURRENT}"
|
||||
logn " new: ${SCANNED_PUBKEY}"
|
||||
echo "$SCANNED_PUBKEY" > "$CLAN_PUB"
|
||||
SOPS_UPDATED=true
|
||||
|
||||
# Rewrite the .sops.yaml anchor for this host with the new age key.
|
||||
ANCHOR="${flake_target}" # e.g. proxmox-ha-server-1
|
||||
NEW_AGE=$(echo "$SCANNED_PUBKEY" | \
|
||||
nix run --quiet --no-warn-dirty nixpkgs#ssh-to-age 2>/dev/null)
|
||||
if [[ -z "$NEW_AGE" ]]; then
|
||||
err "ssh-to-age produced no output for ${host_name} — check nixpkgs#ssh-to-age"
|
||||
fi
|
||||
logn " new age key: ${NEW_AGE}"
|
||||
sed -i "/&${ANCHOR} /s| age[a-z0-9]*$| ${NEW_AGE}|" "${REPO_ROOT}/.sops.yaml"
|
||||
done
|
||||
|
||||
if $SOPS_UPDATED; then
|
||||
logn "Running sops updatekeys on affected secrets..."
|
||||
SOPS="nix run --quiet --no-warn-dirty nixpkgs#sops --"
|
||||
(cd "${REPO_ROOT}" && \
|
||||
$SOPS updatekeys -y secrets/common.yaml && \
|
||||
$SOPS updatekeys -y secrets/ha-server-1.yaml && \
|
||||
$SOPS updatekeys -y secrets/ha-server-2.yaml && \
|
||||
$SOPS updatekeys -y secrets/ha-server-1.keytab && \
|
||||
$SOPS updatekeys -y secrets/ha-server-2.keytab)
|
||||
# Note: ha-corosync-authkey is re-generated and re-encrypted by cluster-init below.
|
||||
|
||||
logn "Committing refreshed host keys and re-encrypted secrets..."
|
||||
(cd "${REPO_ROOT}" && \
|
||||
git add \
|
||||
vars/per-machine/proxmox-ha-server-1/openssh/ssh_host_ed25519_key.pub/value \
|
||||
vars/per-machine/proxmox-ha-server-2/openssh/ssh_host_ed25519_key.pub/value \
|
||||
.sops.yaml \
|
||||
secrets/common.yaml \
|
||||
secrets/ha-server-1.yaml \
|
||||
secrets/ha-server-2.yaml \
|
||||
secrets/ha-server-1.keytab \
|
||||
secrets/ha-server-2.keytab && \
|
||||
git commit -m "secrets(ha): refresh sops host-key registrations for new VM instances" || true)
|
||||
logn "Sops keys refreshed and committed."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Phase 6: Cluster init ─────────────────────────────────────────────────────
|
||||
|
||||
if ! $SKIP_CLUSTER_INIT; then
|
||||
|
||||
@@ -1,23 +1,22 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:hGnZibXsfTgXvl115clWkWduiQQRpA5KhUs+kxO6U03LhE+Yeq0hLRmtICnnMC502YOfprGUC8wSZ6YOJ/7ZIp86Ob+LxahpsN1k/+rJc3/l6dQEZhIr2yyIPOXkSS89OWh4wVdvPOWLwNtg1A7RwIGFBMQZ47Gh15XtADvotI8hAj0VBuN+41y2DwhDXzP+iRZExDIeuOLaezlv/vxENVgweEII9QAoW7LntZjeAv0t2cduYx3X3ASbcFmPIuVilFHZS1nG9djmmZs129hlnYCzDYqhq+Hl8JsvYDV8XWJnWhYEo9OrssBD6Wq8Lx+F7j6NeBHpxg7UjepWkDSzdQ==,iv:3c9qJNIay6x17qrUW39sqLQW3eHn/S3X51CwikNV4rs=,tag:KIL0e55SJVoPHd2B1DklxQ==,type:str]",
|
||||
"data": "ENC[AES256_GCM,data:XlDQdaP04ylg3C+WcGmyWfEndJyHcrhOBtBfe1cdswzrZ71Fd226pBdUNDc/+ZbZDHHRN1BHsBchHH9KFOW9yvtEIIp8MbSD9XFucNg3hXgSdLFOckoVYyBMlKl4FGRguk8RuntmZIq0Qkj4Lzaws2iiVo/IlvQjw2KiGLhTW+8jmuz0h2V7qrDj+f2HPfl6vgNtPWi/C8Xf40xMN54ibmXcVHfx/7Rr+WFdONVG5/l8qTOorOFnwu0wYm8hPaMFidrUZ0pisTfduXCBBPS8N8oizBo+hq88aUE11p9UXuypINmB4zwDZytxAOqiVgvk6B9ZUz/M/SGcsSh81L+8Sg==,iv:5snRRhXUtme9357ODJSWJuZkT/UK3Mx5kE+/sFvToEc=,tag:oF5wXKkP3lcgJ5inAAWogg==,type:str]",
|
||||
"sops": {
|
||||
"age": [
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4NHpHVEVzWWEvdEpRcUFu\nN1RxWDFmRCtnZzByZXE1SzVGeTl0TTRDdmswCkVKakFVUmp1VWFpU0lmVEVWTThW\nZVBIaTF0Tjg2VXJLbXZJL082Y2lRU1kKLS0tIG9pOWppVU1IK0RBS1d5N2RON1Fz\nb2JRZGhtOXBtb0pWNWZjc2xZbXY5Q2cKwod8V72Fyc8glDtxJ/7sDsChm8Nfu/2t\n/DpkajHkF+VfIGPYTDNcR4UfA0oE6WJGF7GIJrvokgT4/smomPiyZQ==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3TDdoZDAwd2tXSWR0S0g2\nL3g0ZWpIY2ZUa2FTc3ZoN25MYkoyNGh5dHc0CjBQV3V5MWtzTkJlNzBlL2tQZ2tX\nVUZSN01YdVowRGQxb1phc1Fvb1M5TkkKLS0tIGpIUW4wdW9nOHpFTlI5U0Z6eFNL\nNHlsNW12YnV3eGczc2VmVlNRWUIwSjgKAez1om042Wv4Yik8sa7TxPQvx2jiZ5Y/\nAt3ScwclG7qaA0CKZ2aVClIhy8QVZQiFLMpB1AkkyH1fRiM2Fao7pQ==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
|
||||
},
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBieXRmSDM1U2tQMEQ3OXUy\nQ016OGRRWGptekZ0SW85bVpYYmpyN2JkejNFCjBka0NwREQ3Yy9IRDJZUmczRkVy\nSjMxck1MZzVHUUpQcUYzNkFPRk1jL0EKLS0tICszSlBEbFFzRXNYM2p1bzhDUkVN\nNWptWU5VYWh4OGdObHpWQ3J4K1k2Zk0Kmnb3y8ZnjTiN+wt79DrxCRYpVjyLttnt\nShqKnhTeoieVs0lEwtprjOv4ZYj8kYfAIh+FLrualJaFeiha3I5nsQ==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQcnk0dWg5Tm9oMWZRY24r\nbkh4QXVZWVRBcmdreEFzZG1YclVJeG54ajFZClhmZk5rUU1qYnZidkpiR1pIRG1D\nd052M2lpbkFGY00wMW95QVM5a2FLZjgKLS0tIG15RUZqN24rQWZBaUxYMzgrZlAr\ndmU2cDNTVXdKV3J0bEd4UnJWY1NmVjgKPxgCeJxPDBbv09KzYiU28XoMsJe5ydkg\nyxHrpM1kJjxVyW1t2wA92NtEEFszXj0ds4+IOYzpyESw8McGwSjiqw==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age1vwk8r03u46rpvmw5fy20t8ud9qdhm0nx5se5ayze98y55e7zg5yqc8862n"
|
||||
},
|
||||
{
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlMm9lY2lkRy9wNFRVSXhw\nMkF5RzZZS1FhWC9kS2RUT2dqUGRxRHFJSERvCjBBSGxBRVpwa3J6ZUlmRDFqOGM0\nYWNBNlFqc2JWZ0JPWmpncHdHNnR1NkkKLS0tIGkrYVlDcUZvNmdsQUc2VndzdlEw\nSzJPaXpNQk1HN2hNbkZEbzcxQ050TUkKSn/llmRaJkdf/5mQfYkycnrPhNKZCgon\n12K9UM87mAnsf8vFR1MzHSRIYuY+CVLHjzu4DfiILNQa9c/JGlVwfg==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5M2xqdWpmVmJOTEMzV2JQ\nS2dmdmwzUUdraFEwU0ZEdHplUUxBdExMU3dNCnVZVG44SytJUHBGV0QrUFB0aGV1\nMmRqUW5XRmhML0J1V1Fwd0hKSGhRcmsKLS0tIHNXWmdKMGhpdWtrOHlFaDN6MGM3\nRUVUOGdBbWl5Z0V1Yk9kRkFwdFFDMEEK9MhgZRB0sCJvFpjEV9S7a+WRP2a2uKlu\n9EReC/dZFfIpDzqqz3hy/IPU+4s3q4vN4TpNhGD/FnqdCd/149tv8Q==\n-----END AGE ENCRYPTED FILE-----\n",
|
||||
"recipient": "age152vrhznh92hspusx9rqxt9c70u8w3uvlys4ymc7zuv3qnxjm5c7qmp72xp"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2026-07-28T11:19:22Z",
|
||||
"mac": "ENC[AES256_GCM,data:2grZR+vFcdTQnVn4PASqgxqi2kHues26IjeCYvVt3ykFPOnAuc2lGnzCzm6AzcrSPeNBTNBGmsSw5FqHO2JWC9KIE6Y9PjzOX87PeSvQqVWfbwbz0c4PZ7myoUkqTF5cv7wvum1JwDrL2p0Sw21cGa0km3y//ovz7OJOsK6CFo8=,iv:BIgrCaJEZCkcFB3xmNVF71FdqfDDP9txLPHgFi5SIy4=,tag:m5dkyNagf47cLq0XEcyS8w==,type:str]",
|
||||
"unencrypted_suffix": "_unencrypted",
|
||||
"lastmodified": "2026-07-28T12:27:29Z",
|
||||
"mac": "ENC[AES256_GCM,data:/DdKK2qi+GgnlVSbygVNcvQP+StFYVtrmAHevNacF58m5v4KSLNfLvTmCwZX1LQWOZHTbOFoEDlTf+Z5b2N1EbrYsQOszg0dZTFv34MbgBpPQmfKUp53LsxQv0w1OKTbZbOIGjCan1adGWQZvmHq9gjvXXHJx7R42k5xMB062a0=,iv:H4NSl16at6ETIa92VI99f3jnqTvA0qD7/g/8zYK6SAo=,tag:uJ/RFHwInHpyYD8A7nOpyA==,type:str]",
|
||||
"version": "3.13.3"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user