fix(ha): per-node DRBD disk assignment (NODE2 data disk is /dev/sda not /dev/sdb)

Proxmox VM disk ordering differs between the two HA nodes:
  ha-server-1: sda=OS (50G), sdb=DRBD data (32G)
  ha-server-2: sda=DRBD data (32G), sdb=OS (50G)

The DRBD resource config was using a shared disk=/dev/sdb which targeted
the OS disk on ha-server-2, causing drbdmeta and drbdadm up to operate
on the mounted root filesystem (hence "Device or resource busy").

Changes:
- variables.nix: add haServer1DrbdDisk/haServer2DrbdDisk
- cluster-config.nix: move volume block inside per-host on{} sections
  so each node uses the correct backing disk
- cluster-init.sh: use NODE1_DRBD_DISK/NODE2_DRBD_DISK variables;
  add runtime check that patches /etc/drbd.d/*.res on the running nodes
  if the deployed config points to the wrong disk (workaround for VMs
  built before this fix; redeploy with --force-rebuild to make permanent)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

modules/ha/cluster-config.nix

@@ -68,18 +68,22 @@
       }
 
       resource ha-data {
-        volume 0 {
-          device    /dev/drbd0;
-          disk      /dev/sdb;
-          meta-disk internal;
-        }
-
         on ${vars.haServer1Host} {
           address ${vars.haServer1StorageIp}:${toString vars.ports.haServerDrbd};
+          volume 0 {
+            device    /dev/drbd0;
+            disk      ${vars.haServer1DrbdDisk};
+            meta-disk internal;
+          }
         }
 
         on ${vars.haServer2Host} {
           address ${vars.haServer2StorageIp}:${toString vars.ports.haServerDrbd};
+          volume 0 {
+            device    /dev/drbd0;
+            disk      ${vars.haServer2DrbdDisk};
+            meta-disk internal;
+          }
         }
       }
     '';

scripts/ha/cluster-init.sh

@@ -32,6 +32,11 @@ ISCSI_IQN="${ISCSI_IQN:-iqn.2026-01.home.sweet:ha-storage}"  # vars.haIscsiIqn
 ISCSI_LUN_FILE="${XFS_MOUNT}/iscsi-lun.img"
 ISCSI_LUN_SIZE="10G"
 DRBD_DEVICE="/dev/drbd0"
+# DRBD backing disk per node — disk ordering can differ between Proxmox VMs.
+# Defaults match haServer{1,2}DrbdDisk in variables.nix.  Override if your
+# VM was created with disks in a different order.
+NODE1_DRBD_DISK="${NODE1_DRBD_DISK:-/dev/sdb}"
+NODE2_DRBD_DISK="${NODE2_DRBD_DISK:-/dev/sda}"
 VMID_NODE1="${VMID_NODE1:-}"            # set by deploy.sh; needed for STONITH
 VMID_NODE2="${VMID_NODE2:-}"
 PVE_HOST="${PVE_HOST:-pve1.sweet.home}"
@@ -149,7 +154,7 @@ done
 # ── 2. DRBD initialisation ────────────────────────────────────────────────
 # Put both nodes in Pacemaker standby before touching DRBD metadata.
 # Without this, the OCF DRBD agent races: it sees drbdadm-down as a failure
-# and immediately calls drbdadm-up again, leaving /dev/sdb busy when
+# and immediately calls drbdadm-up again, leaving the backing disk busy when
 # create-md / write-dev-uuid runs.  On a fresh cluster with no resources
 # configured this is a no-op; on a re-run it stops the race.
 log "Setting both nodes to Pacemaker standby for DRBD metadata init..."
@@ -175,25 +180,42 @@ log "Detaching DRBD on $NODE2..."
 n2_ssh "drbdadm down ha-data 2>/dev/null || true"
 sleep 2
 
+# Verify the DRBD config on each node targets the correct backing disk.
+# Proxmox VM disk ordering is not guaranteed — the data disk may appear as
+# /dev/sda on one node and /dev/sdb on the other.  If the deployed NixOS
+# config was generated before variables.nix had per-node disk assignments, the
+# config may point to the wrong device.  Patch it in-place so drbdadm up
+# attaches to the right disk; rebuild+redeploy to make this permanent.
+n1_conf_disk=$(drbdadm sh-ll-dev ha-data 2>/dev/null || true)
+if [[ -n "$n1_conf_disk" && "$n1_conf_disk" != "$NODE1_DRBD_DISK" ]]; then
+  warn "$NODE1 DRBD config says disk=$n1_conf_disk; patching to $NODE1_DRBD_DISK (redeploy to make permanent)"
+  sed -i "s|${n1_conf_disk}|${NODE1_DRBD_DISK}|g" /etc/drbd.d/*.res 2>/dev/null || true
+fi
+n2_conf_disk=$(n2_ssh "drbdadm sh-ll-dev ha-data 2>/dev/null" 2>/dev/null || true)
+if [[ -n "$n2_conf_disk" && "$n2_conf_disk" != "$NODE2_DRBD_DISK" ]]; then
+  warn "$NODE2 DRBD config says disk=$n2_conf_disk; patching to $NODE2_DRBD_DISK (redeploy to make permanent)"
+  n2_ssh "sed -i 's|${n2_conf_disk}|${NODE2_DRBD_DISK}|g' /etc/drbd.d/*.res" 2>/dev/null || true
+fi
+
 log "Initialising DRBD metadata on $NODE1..."
 # Use drbdmeta --force directly for BOTH create-md and write-dev-uuid.
 # drbdadm create-md --force passes --force to drbdmeta create-md but NOT to
 # the write-dev-uuid sub-call it makes internally, so write-dev-uuid fails when
-# /dev/sdb is still busy (udev auto-attach, stale DRBD state, etc.) and stdin
-# is not a TTY: "stdin not a TTY, not waiting for confirmation" → exit 20.
+# the backing disk is still busy and stdin is not a TTY:
+# "stdin not a TTY, not waiting for confirmation" → exit 20.
 # Calling drbdmeta --force directly bypasses the exclusive-open confirmation on
 # both steps without needing a TTY, regardless of whether the device is busy.
 if ! drbdadm dstate ha-data 2>/dev/null | grep -q "UpToDate"; then
   UUID1=$(_rand_uuid)
-  drbdmeta --force 0 v08 /dev/sdb internal create-md
-  drbdmeta --force 0 v08 /dev/sdb internal write-dev-uuid "$UUID1"
+  drbdmeta --force 0 v08 "${NODE1_DRBD_DISK}" internal create-md
+  drbdmeta --force 0 v08 "${NODE1_DRBD_DISK}" internal write-dev-uuid "$UUID1"
 fi
 
 log "Initialising DRBD metadata on $NODE2..."
 if ! n2_ssh "drbdadm dstate ha-data 2>/dev/null | grep -q UpToDate" 2>/dev/null; then
   UUID2=$(n2_ssh "cat /proc/sys/kernel/random/uuid 2>/dev/null | tr -d '-' | cut -c1-16 | tr '[:lower:]' '[:upper:]'")
-  n2_ssh "drbdmeta --force 0 v08 /dev/sdb internal create-md"
-  n2_ssh "drbdmeta --force 0 v08 /dev/sdb internal write-dev-uuid ${UUID2}"
+  n2_ssh "drbdmeta --force 0 v08 ${NODE2_DRBD_DISK} internal create-md"
+  n2_ssh "drbdmeta --force 0 v08 ${NODE2_DRBD_DISK} internal write-dev-uuid ${UUID2}"
 fi
 
 log "Bringing up DRBD on both nodes..."

secrets/ha-corosync-authkey

@@ -1,22 +1,22 @@
 {
-	"data": "ENC[AES256_GCM,data:slsTAZVNC7yDJDwqcQV7yTXtW5EJuIGFZ4BSI9sWwDYiPVjGR0hXAUdgp5rQ0h6FlWHl0Ir/Ayoy2ONz0BwWsOtjb+u5Z9piEU7MY2TyWUkED804iiAy30Df5VzWc7cqhDjBPFc3iKwvi35u1QASVl/HW5LX99DJctx1CdA159In2YFUjeEt3nFUqR5ietwPcfWwpjnnKbkK1Cx7hEh/TdPjDOvzEbxKjGZzb+mhL1jAI9bg93VYcI2E02hkIAG2/LbaoVTaFbSmjuNbzNSY9yphKGZZGVCrPDTh/kV345OVVOIkZrGuD3P1WAeDE/NFLCPQ6PfUHFiGNS2DnqDQKw==,iv:Uy22v9N8OodIH71s3Rk5ltGBMpTOUC64yXttokH6rJs=,tag:1Y6vybFk8KeqjnEJRNKmgA==,type:str]",
+	"data": "ENC[AES256_GCM,data:Dqb0FfDXcSofPjHC6wuSjjHJ1VLtZ4RMqU71xBgxOuR+mV6fFP0LoNZcZp0an28XHyf3qLDwnGWvPMf8d9kYNpP0WC+nEqCDvtQjkraJ77j5y57+UNOqdM3tkJ85z5Cm0cQd2DoV4es/mtX6jzDotWW06ESCHtpcefz5SZ4PsYREIT09LmnHdJQ1sFYS0SQxtGa8oLSu7fcobgvV043GS35pq9BICHpCcUzr2LgpDFDpNGU1vSWN9QmwnmKpuf3kQKOYmCJLFsOn96cGnMbgsWc6U3zMOTwOHxgsracG169hlSS8fH7xLWWNgDDq+ILmREsWac5K680j98UXrSbn9g==,iv:iPlhl0GYlLFZOxdU+cHzCoVEkKYbg/X1XTP0lf0Mo0M=,tag:64U+dJ/vO3Q2BMkrtNI6Gw==,type:str]",
 	"sops": {
 		"age": [
 			{
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvcEl6cC91TTU1QUliWGwz\nUTEzaWhFZ2FMZmJpUmVOZml4Q2xHektPM0ZVCmdVWnVheS9pSjdDcmhFYnF3Y01o\nbENFUkh0a3RzcThRYnhYUDlmVE1uek0KLS0tIEdFajRtTTNvNytpRzNuZkRzOUov\nN200WFZwV3k4U0laUGoramFzTnRlSEkKbDEy25Zr666CfIYPBCKazwmXAhv3tyr7\nqBMO+hvDEilQUhVaIv//C1NKLrlQNsdEj/OKjOrcWek8linV/Uj/Ag==\n-----END AGE ENCRYPTED FILE-----\n",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpcjBTMVBHZE1MSWhuakVo\nLzJiYm0xYzdzL2RETllTSDBqSU5qZ05rR3g4CjJGV21aU0xSbzlJekg5RDFtUVdX\nSjRWRUdLRnZTVHk5U2pqVytQbXQySUUKLS0tIFVJZXp5Vll0aUczR1pRT3RSeitX\nckYvSFZoTmNrLzgrNTUyYk9vTkJ5UzQK3f5ShqRPTfdonFkHfXc7RWDgBZlhX1+u\npyWitr+t8O26hTPNBDy3jA1mascupr7r8VCVQkgU6rA8ujitAK7x9g==\n-----END AGE ENCRYPTED FILE-----\n",
 				"recipient": "age1njap586hc0q43kr03g6c8eqhdsmk8zcafkl3f83xwlc2gqhlmfgs4tmwad"
 			},
 			{
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSdDE1dUdZUENzbE1INitR\neDU0czFKTy82aUxjL0dURWJtZU9LTkFpSFRnCjhSd29FYWpVaUhUd3JkMGFJMGI3\nYXVMNFZDQjVVc0xpWjhOaXgwRzhMQWMKLS0tIFRGaHl4S3hIY04zZW4vU1VsWVgx\ndkE1aWpuYXFLSzBLeXJFL0gzRHFsY2MKILDSAkt4FV6KyExx420OtJQHYMrQKp5m\nOse14ihVqTUYRFoA1OuL8uIGa7x4bUKEQC0FXPyGY/GIl2hsNixdeA==\n-----END AGE ENCRYPTED FILE-----\n",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxZUZXZkx1bVNvUnZMRFRQ\nd2N4QlBRTnoyaHpYWjZxN3JvUUJybW81Wm1BCm9oanJSM1RJemhtZUpIczIrYzFq\neVVjTnFkeUgvaVhDRGtzczJqdkdzL3MKLS0tIGVBRTZHWmw5aE9kcmpTMkkzbFg0\nT0dRVGRoYk1XWnBOWk84QTZzOFZ6UlUKpJieegDxYg5HmhH7HUptoff0UqYqUsIc\nBq3b5ChaZwSScbQzYz+18kBvdGgBEx4fB1q8vY+8SC+wJ5MVMh8lSQ==\n-----END AGE ENCRYPTED FILE-----\n",
 				"recipient": "age1k73g8x47hs93wcv7qh92n3htz8pl295g49hyvlrf3570mts0hgys5g04d6"
 			},
 			{
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2OGNEYW1oR1NFMWl6UVhQ\naEhueEhmT2VETGwvWGxHOEx0M1l3akR4dWlNClA0M2hQQ2kydlk2cExMQnZsWlVR\nZlhvOE0wOWZVVjVkcS9FbEJOaDIxU1EKLS0tIG5Jb21SUk9hUWNjQ3M4Y25ES0VI\nWDJ5bE4zdnRFeXh2Zm4rNUozZEZNM3MKGLeaMwuFNER/jY13LcLfSlkPJNoIfu+F\nnZKlv1e/mYNNCbZ+pW9ZDTPocHzp2KRm6ji3OO1GZC8re5Q7wChkPQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNWUxqT3BxSXF5bkxsUTRy\nTWZVei9JTHFhcmNNUkRjcklzV0Y1VllXQ0hRCjVTRmQ4WHFmUnM0RzNKYnF1dE4y\nOU1mQ3ZDcjZoVDd4VnlodHR5c1Q2UTgKLS0tIGhhbU1kWmlrWHZPVVZMOVU3RWFV\nM0txajIwK01NNElITXZDWU9KMGJDOVkK8k5hLKWoFEvfNT6TctSHPuIAZu9xj4t8\n5unQOFkAShHx6VcTz0vM8xMGa7bvTfoDHce/HXN42vbuVhvYJUzJRg==\n-----END AGE ENCRYPTED FILE-----\n",
 				"recipient": "age1fefy6dk8zn5c3edwmrs9vwx79quftnt784m628t9e34q3ft3cehqz8u72r"
 			}
 		],
-		"lastmodified": "2026-07-28T13:30:12Z",
-		"mac": "ENC[AES256_GCM,data:ZjVCaXOy3Sn+h7i7kCJpcTccE7v1NLLejnZsU8EAoRH9fvUDL681Z6vzv+WDEVDJ04raLAQLnlPJ7gdvq/HVGSdrv8rvl4KNAk34u+njVg/qpeGa1D9NvwxbZlOZTBgy9Xcwy9bPF8jqfchNlUt5naUkk8TlHbgsA1ageG28BkU=,iv:hT1SR7vjGtPX/ATRS/WJvmS0b2z5T35i+OJ6bvVz6AM=,tag:vxk7RRBXU14VziBmWxBjHg==,type:str]",
+		"lastmodified": "2026-07-28T13:48:23Z",
+		"mac": "ENC[AES256_GCM,data:+2LDHPXVbrjBfIFHjo5Oh+enWy3j0zt85+MMxbMIKOtHOMadXuodlfJN7Igr6GKlMxsP8KvCemfTTL07xuS6T69mrWt8Lpsc/SmA8Xabl99BXtT9oAjD8PlZ85DjBelhuILZ2j1dMW9b1P3b5qfiLvRKnGXw5gqZvuyKmLI9q/k=,iv:FTCyYwi813j3UVMnHPCmMcJwqSkHO4NELsQ7oZlBZk0=,tag:C0luCrScx+23u8RJUeHZYg==,type:str]",
 		"version": "3.13.3"
 	}
 }

variables.nix

@@ -108,6 +108,11 @@
   haStoragePrefixLength = 29; # storage subnet prefix length (/29)
   haStorageRoot = "/srv/ha-data"; # XFS-over-DRBD mount point on the Active node
   haIscsiIqn = "iqn.2026-01.home.sweet:ha-storage";
+  # DRBD backing device on each node. Disk ordering can differ between Proxmox
+  # VMs depending on the order disks were added; these must match the actual
+  # block device that is NOT the OS disk on each node (verify with lsblk).
+  haServer1DrbdDisk = "/dev/sdb"; # data disk on ha-server-1 (OS disk is /dev/sda)
+  haServer2DrbdDisk = "/dev/sda"; # data disk on ha-server-2 (OS disk is /dev/sdb)
 
   # Storage
   storageRoot = "/tank"; # ZFS pool root on `server`