feat(ha): add NixOS modules for DRBD+XFS+LIO+Corosync+Pacemaker HA stack

All 7 acceptance tests pass on live NixOS 25.11 VMs (VMIDs 200/201 on
pve1).  Failover completes in ~5 s with data integrity verified.

modules/ha/pacemaker-stack.nix — fixes four NixOS-specific breakages:
  - systemd StateDirectory resets /var/lib/pacemaker to root:root; removed
    and replaced with ExecStartPre to create/chown dirs as hacluster
  - HA_SBIN_DIR points to a non-existent Nix store path; overridden to
    /run/current-system/sw/bin so crm_master resolves correctly
  - OCF agents need an explicit broad PATH (iproute2, util-linux, xfsprogs,
    drbd, bash, etc.) — NixOS services have no implicit PATH
  - FUSER=true bypasses the psmisc fuser check_binary call in the
    Filesystem OCF agent (psmisc not installed on minimal hosts)

modules/ha/iscsi-target.nix — LIO iSCSI target via targetctl with a
Python/rtslib_fb ExecStop that explicitly clears the kernel LIO state
(not just saves JSON), so the XFS backing store's file descriptor is
released before umount — preventing EBUSY stop timeouts on failover.
Includes an empty-config guard so the secondary node never overwrites
the primary's saveconfig.json with an empty one.

test-lab/ha/common.nix — updated to import both modules, use fencing
dont-care (no STONITH in test lab), omit LVM handlers (non-existent on
NixOS paths), and merge repeated services/networking attr sets to satisfy
statix W20.  test-lab/ha/acceptance-tests.sh — final v4 with crm_standby
fix (pacemaker 3.x API).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HaH1cSGvhogRP5ExoF6nD8

test-lab/ha/common.nix

@@ -11,39 +11,6 @@ let
   # Test-only corosync authkey (128 bytes = 1024 bits minimum for corosync).
   # Not secret — this is a disposable test cluster, not production.
   testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
-
-  # OCF agent path (from pacemaker's --with-ocfdir).
-  # We expose this on PATH so custom scripts can find each other.
-  ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d";
-
-  # Concatenated PATH that includes all binaries OCF agents and pacemaker
-  # lrmd children need.  This is the workaround for nixpkgs#207891 (PATH
-  # not set correctly for OCF agent child processes).
-  ocfBinPath = lib.concatStringsSep ":" [
-    "${pkgs.iproute2}/bin"
-    "${pkgs.iproute2}/sbin"
-    "${pkgs.iputils}/bin"
-    "${pkgs.util-linux}/bin"
-    "${pkgs.util-linux}/sbin"
-    "${pkgs.gawk}/bin"
-    "${pkgs.gnugrep}/bin"
-    "${pkgs.gnused}/bin"
-    "${pkgs.coreutils}/bin"
-    "${pkgs.bash}/bin"
-    "${pkgs.procps}/bin"
-    "${pkgs.xfsprogs}/bin"
-    "${pkgs.drbd}/bin"
-    "${pkgs.targetcli-fb}/bin"
-    "${pkgs.python3}/bin"
-    "/run/current-system/sw/bin"
-    "/run/current-system/sw/sbin"
-    "/usr/local/sbin"
-    "/usr/local/bin"
-    "/usr/sbin"
-    "/usr/bin"
-    "/sbin"
-    "/bin"
-  ];
 in
 {
   system.stateVersion = "26.05";
@@ -52,82 +19,86 @@ in
   imports = [
     ../../modules/hardware-configuration/vm/proxmox.nix
     ../../modules/boot/efi.nix
+    ../../modules/ha/pacemaker-stack.nix
+    ../../modules/ha/iscsi-target.nix
   ];
 
   # ── Nix settings ──────────────────────────────────────────────────────
   nix.settings.experimental-features = [ "nix-command" "flakes" ];
 
-  # ── SSH ───────────────────────────────────────────────────────────────
-  services.openssh = {
-    enable = true;
-    settings.PermitRootLogin = "yes";
-  };
   users.users.root.openssh.authorizedKeys.keys = [
     vars.adminSshKey
     # Claude Code session key (this machine) — test-lab only
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGygkCljN6uKpdJbHTOQtn8ZnH+wKXDLAwrDFbLrE/65 nixos@nixos"
   ];
 
-  # Allow QEMU guest exec for key injection fallback
-  services.qemuGuest.enable = true;
+  # ── Services ──────────────────────────────────────────────────────────
+  services = {
+    openssh = {
+      enable = true;
+      settings.PermitRootLogin = "yes";
+    };
 
-  # ── Networking ────────────────────────────────────────────────────────
-  networking.useDHCP = false;
-  networking.defaultGateway = "192.168.2.1";
-  networking.nameservers = [ "192.168.2.1" "8.8.8.8" ];
+    # Allow QEMU guest exec for key injection fallback
+    qemuGuest.enable = true;
 
-  # ── DRBD ──────────────────────────────────────────────────────────────
-  services.drbd.enable = true;
-  services.drbd.config = ''
-    global {
-      usage-count yes;
-    }
+    drbd = {
+      enable = true;
+      config = ''
+        global {
+          usage-count yes;
+        }
 
-    common {
-      net {
-        protocol C;
-        ping-int 1;
-        verify-alg sha256;
-        after-sb-0pri discard-zero-changes;
-        after-sb-1pri discard-secondary;
-      }
-      disk {
-        # resource-only: DRBD itself won't fence (Pacemaker handles STONITH);
-        # the DRBD resource agent uses fencing to guard primary promotion.
-        fencing resource-only;
-      }
-      handlers {
-        # NOTE: LVM-specific before/after-resync-target handlers omitted
-        # (raw /dev/sdb, no LVM). split-brain handler also omitted —
-        # Pacemaker STONITH manages split-brain fencing for the test cluster.
-      }
-    }
+        common {
+          net {
+            protocol C;
+            ping-int 1;
+            verify-alg sha256;
+            after-sb-0pri discard-zero-changes;
+            after-sb-1pri discard-secondary;
+          }
+          disk {
+            # dont-care: DRBD itself won't fence before promoting.  Production
+            # clusters should use resource-only here and configure a STONITH
+            # fence agent (e.g. fence_pve_ssh) in Pacemaker so DRBD can safely
+            # protect against split-brain without risking dual-Primary.
+            # For this test cluster (no fence device) dont-care lets promotion
+            # proceed; the DRBD kernel module still refuses dual-Primary without
+            # allow-two-primaries in net {}.
+            fencing dont-care;
+            # LVM before/after-resync-target handlers omitted: the LVM snapshot
+            # scripts (/usr/lib/drbd/snapshot-resync-target-lvm.sh) don't exist
+            # on NixOS paths.  If present, DRBD calls them on resync and exits 127,
+            # dropping the peer connection and leaving the secondary Outdated.
+          }
+        }
 
-    resource ha-data {
-      volume 0 {
-        device    /dev/drbd0;
-        disk      /dev/sdb;   # scsi1 in Proxmox VM → sdb
-        meta-disk internal;
-      }
+        resource ha-data {
+          volume 0 {
+            device    /dev/drbd0;
+            disk      /dev/sdb;   # scsi1 in Proxmox VM → sdb
+            meta-disk internal;
+          }
 
-      on ha-test-node1 {
-        address ${node1Ip}:${toString drbdPort};
-      }
+          on ha-test-node1 {
+            address ${node1Ip}:${toString drbdPort};
+          }
 
-      on ha-test-node2 {
-        address ${node2Ip}:${toString drbdPort};
-      }
-    }
-  '';
+          on ha-test-node2 {
+            address ${node2Ip}:${toString drbdPort};
+          }
+        }
+      '';
+    };
 
-  # ── Corosync ──────────────────────────────────────────────────────────
-  services.corosync = {
-    enable = true;
-    clusterName = "ha-test";
-    nodelist = [
-      { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; }
-      { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; }
-    ];
+    # services.corosync.enable = true is set by modules/ha/pacemaker-stack.nix
+    corosync = {
+      clusterName = "ha-test";
+      nodelist = [
+        { nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; }
+        { nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; }
+      ];
+    };
   };
 
   # Corosync authkey (test-only, not secret — generated with
@@ -137,81 +108,13 @@ in
     mode = "0400";
   };
 
-  # ── Pacemaker ─────────────────────────────────────────────────────────
-  services.pacemaker.enable = true;
-
-  # Fix for nixpkgs#207891:
-  # 1. Ensure CIB directories are owned by hacluster before starting.
-  #    The stock module sets StateDirectory=pacemaker (owned by root);
-  #    pacemaker internally drops to hacluster but needs to write there.
-  # 2. Set PATH so OCF agent child processes can find all required binaries.
-  systemd.services.pacemaker.serviceConfig = {
-    ExecStartPre = [
-      "${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'"
-    ];
-  };
-  systemd.services.pacemaker.environment = {
-    PATH = lib.mkForce ocfBinPath;
-    # Expose OCF root so pacemaker and lrmd agree on where agents live.
-    OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
-  };
-
-  # pacemaker-execd is the local resource executor that calls OCF agents.
-  # Give it the same PATH so OCF scripts can find all required binaries.
-  systemd.services.pacemaker-execd.environment = {
-    PATH = lib.mkForce ocfBinPath;
-    OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
-  };
-
-  # ── LIO / iSCSI target ────────────────────────────────────────────────
-  # targetcli-fb is the management tool; actual kernel support is via
-  # the LIO modules.  We add a systemd service that saves/restores the
-  # target configuration so Pacemaker can trigger it via a systemd-class
-  # resource.
-  boot.kernelModules = [
-    "target_core_mod"
-    "iscsi_target_mod"
-    "target_core_file"
-    "target_core_pscsi"
-    "target_core_user"
-    "configfs"
-  ];
-
-  # configfs must be mounted for rtslib/targetcli to work
-  systemd.mounts = [{
-    where = "/sys/kernel/config";
-    what = "configfs";
-    type = "configfs";
-    wantedBy = [ "multi-user.target" ];
-    before = [ "targetctl.service" ];
-  }];
-
-  # targetctl: save/restore LIO configuration (mirrors Debian's package)
-  systemd.services.targetctl = {
-    description = "LIO iSCSI target config save/restore";
-    wantedBy = [ "multi-user.target" ];
-    after = [ "sys-kernel-config.mount" "network.target" ];
-    requires = [ "sys-kernel-config.mount" ];
-    serviceConfig = {
-      Type = "oneshot";
-      RemainAfterExit = true;
-      ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json";
-      ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json";
-    };
-    unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
-  };
-
   # ── Packages ──────────────────────────────────────────────────────────
+  # corosync, pacemaker, ocf-resource-agents, targetcli-fb already added
+  # by the ha/ modules; add the remaining stack-specific tools here.
   environment.systemPackages = with pkgs; [
-    # HA stack
-    corosync # corosync-cfgtool, corosync-quorumtool
-    pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI
-    drbd # drbdadm, drbdsetup, drbdmon
-    ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …)
-
     # Storage
+    drbd # drbdadm, drbdsetup, drbdmon
     xfsprogs # mkfs.xfs, xfs_admin, xfs_info
-    targetcli-fb # targetcli shell + targetctl
 
     # Networking / debug
     iproute2 # ip, ss
@@ -227,33 +130,32 @@ in
     htop
   ];
 
-  # ── Firewall ──────────────────────────────────────────────────────────
-  networking.firewall = {
-    enable = true;
-    allowedTCPPorts = [
-      22 # SSH
-      3260 # iSCSI
-      3121 # pacemaker-remoted
-      2224 # pcsd
-      drbdPort # DRBD replication
-    ];
-    allowedUDPPorts = [
-      5404 # corosync cluster
-      5405 # corosync cluster
-      5407 # corosync crypto
-    ];
-    # Corosync uses ports 5404-5407 UDP; allow them on the cluster net
-    extraCommands = ''
-      iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT
-      iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT
-    '';
-  };
+  # ── Networking ────────────────────────────────────────────────────────
+  networking = {
+    useDHCP = false;
+    defaultGateway = "192.168.2.1";
+    nameservers = [ "192.168.2.1" "8.8.8.8" ];
 
-  # ── tmpfiles: target config dir ────────────────────────────────────────
-  systemd.tmpfiles.rules = [
-    "d /etc/target 0750 root root -"
-    "f /etc/target/saveconfig.json 0640 root root -"
-  ];
+    firewall = {
+      enable = true;
+      allowedTCPPorts = [
+        22 # SSH
+        3260 # iSCSI
+        3121 # pacemaker-remoted
+        2224 # pcsd
+        drbdPort
+      ];
+      allowedUDPPorts = [
+        5404 # corosync
+        5405 # corosync
+        5407 # corosync crypto
+      ];
+      extraCommands = ''
+        iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT
+        iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT
+      '';
+    };
+  };
 
   # ── Locale / time ─────────────────────────────────────────────────────
   time.timeZone = vars.timeZone;