Archived
test-lab: add two-node HA file-server test cluster config
Disposable test VMs (ha-test-node1 / ha-test-node2, VMIDs 200/201 on pve1) to evaluate whether the DRBD + XFS + LIO + Corosync + Pacemaker stack runs correctly on NixOS before deciding NixOS vs Debian for production. Includes: - test-lab/ha/disko.nix: 20G boot disk layout (smaller than production) - test-lab/ha/common.nix: shared HA stack (drbd, corosync, pacemaker, targetcli-fb, xfsprogs), OCF PATH workaround for nixpkgs#207891 - test-lab/ha/node1.nix / node2.nix: per-node hostname + static IP - test-lab/ha/fence-pve-ssh.py: Proxmox SSH fence agent for STONITH - test-lab/ha/cluster-init.sh: one-shot cluster bootstrap script - test-lab/ha/cluster-enable-stonith.sh: enables STONITH post-key-deploy - flake.nix: adds ha-test-node1 / ha-test-node2 nixosConfigurations (bypasses mkTarget / clan-core / sops-nix — test-only) These VMs must be destroyed once acceptance testing is complete. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
beatzaplentyandClaude Sonnet 4.6 committed
1 parent
d3d6382360
commit
750121e9dd
8 files changed
+854
-1
No files matched your search
@@ -0,0 +1,256 @@
|
||||
# Shared HA stack config for both test nodes.
|
||||
# These are throwaway test VMs — not production hosts.
|
||||
# No sops-nix, no clan, no home-manager.
|
||||
{ lib, pkgs, vars, ... }:
|
||||
|
||||
let
|
||||
node1Ip = "192.168.2.200";
|
||||
node2Ip = "192.168.2.201";
|
||||
drbdPort = 7789;
|
||||
|
||||
# Test-only corosync authkey (128 bytes minimum).
|
||||
# Not secret — this is a disposable test cluster, not production.
|
||||
testAuthKey = "ha-test-cluster-auth-key-NOT-FOR-PRODUCTION-use-corosync-keygen-for-real-clusters-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
|
||||
|
||||
# OCF agent path (from pacemaker's --with-ocfdir).
|
||||
# We expose this on PATH so custom scripts can find each other.
|
||||
ocfPath = "${pkgs.ocf-resource-agents}/usr/lib/ocf/resource.d";
|
||||
|
||||
# Concatenated PATH that includes all binaries OCF agents and pacemaker
|
||||
# lrmd children need. This is the workaround for nixpkgs#207891 (PATH
|
||||
# not set correctly for OCF agent child processes).
|
||||
ocfBinPath = lib.concatStringsSep ":" [
|
||||
"${pkgs.iproute2}/bin"
|
||||
"${pkgs.iproute2}/sbin"
|
||||
"${pkgs.iputils}/bin"
|
||||
"${pkgs.util-linux}/bin"
|
||||
"${pkgs.util-linux}/sbin"
|
||||
"${pkgs.gawk}/bin"
|
||||
"${pkgs.gnugrep}/bin"
|
||||
"${pkgs.gnused}/bin"
|
||||
"${pkgs.coreutils}/bin"
|
||||
"${pkgs.bash}/bin"
|
||||
"${pkgs.procps}/bin"
|
||||
"${pkgs.xfsprogs}/bin"
|
||||
"${pkgs.drbd}/bin"
|
||||
"${pkgs.targetcli-fb}/bin"
|
||||
"${pkgs.python3}/bin"
|
||||
"/run/current-system/sw/bin"
|
||||
"/run/current-system/sw/sbin"
|
||||
"/usr/local/sbin"
|
||||
"/usr/local/bin"
|
||||
"/usr/sbin"
|
||||
"/usr/bin"
|
||||
"/sbin"
|
||||
"/bin"
|
||||
];
|
||||
in
|
||||
{
|
||||
system.stateVersion = "26.05";
|
||||
|
||||
# ── Hardware (Proxmox VM) ──────────────────────────────────────────────
|
||||
imports = [
|
||||
../../modules/hardware-configuration/vm/proxmox.nix
|
||||
../../modules/boot/efi.nix
|
||||
];
|
||||
|
||||
# ── Nix settings ──────────────────────────────────────────────────────
|
||||
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
||||
|
||||
# ── SSH ───────────────────────────────────────────────────────────────
|
||||
services.openssh = {
|
||||
enable = true;
|
||||
settings.PermitRootLogin = "yes";
|
||||
};
|
||||
users.users.root.openssh.authorizedKeys.keys = [ vars.adminSshKey ];
|
||||
|
||||
# ── Networking ────────────────────────────────────────────────────────
|
||||
networking.useDHCP = false;
|
||||
networking.defaultGateway = "192.168.2.1";
|
||||
networking.nameservers = [ "192.168.2.1" "8.8.8.8" ];
|
||||
|
||||
# ── DRBD ──────────────────────────────────────────────────────────────
|
||||
services.drbd.enable = true;
|
||||
services.drbd.config = ''
|
||||
global {
|
||||
usage-count yes;
|
||||
}
|
||||
|
||||
common {
|
||||
net {
|
||||
protocol C;
|
||||
ping-int 1;
|
||||
verify-alg sha256;
|
||||
after-sb-0pri discard-zero-changes;
|
||||
after-sb-1pri discard-secondary;
|
||||
}
|
||||
disk {
|
||||
# resource-only: DRBD itself won't fence (Pacemaker handles STONITH);
|
||||
# the DRBD resource agent uses fencing to guard primary promotion.
|
||||
fencing resource-only;
|
||||
}
|
||||
handlers {
|
||||
# Called by DRBD when a split-brain is detected and this node is
|
||||
# in the secondary role — notifies pacemaker to fence the split node.
|
||||
split-brain "/usr/lib/drbd/notify-split-brain.sh root";
|
||||
before-resync-target "/usr/lib/drbd/snapshot-resync-target-lvm.sh -p 15 -- -c 16k";
|
||||
after-resync-target "/usr/lib/drbd/unsnapshot-resync-target-lvm.sh";
|
||||
}
|
||||
}
|
||||
|
||||
resource ha-data {
|
||||
volume 0 {
|
||||
device /dev/drbd0;
|
||||
disk /dev/sdb; # scsi1 in Proxmox VM → sdb
|
||||
meta-disk internal;
|
||||
}
|
||||
|
||||
on ha-test-node1 {
|
||||
address ${node1Ip}:${toString drbdPort};
|
||||
}
|
||||
|
||||
on ha-test-node2 {
|
||||
address ${node2Ip}:${toString drbdPort};
|
||||
}
|
||||
}
|
||||
'';
|
||||
|
||||
# ── Corosync ──────────────────────────────────────────────────────────
|
||||
services.corosync = {
|
||||
enable = true;
|
||||
clusterName = "ha-test";
|
||||
nodelist = [
|
||||
{ nodeid = 1; name = "ha-test-node1"; ring_addrs = [ node1Ip ]; }
|
||||
{ nodeid = 2; name = "ha-test-node2"; ring_addrs = [ node2Ip ]; }
|
||||
];
|
||||
};
|
||||
|
||||
# Corosync authkey (test-only, not secret — generated with
|
||||
# `corosync-keygen` for production).
|
||||
environment.etc."corosync/authkey" = {
|
||||
source = builtins.toFile "authkey" testAuthKey;
|
||||
mode = "0400";
|
||||
};
|
||||
|
||||
# ── Pacemaker ─────────────────────────────────────────────────────────
|
||||
services.pacemaker.enable = true;
|
||||
|
||||
# Fix for nixpkgs#207891:
|
||||
# 1. Ensure CIB directories are owned by hacluster before starting.
|
||||
# The stock module sets StateDirectory=pacemaker (owned by root);
|
||||
# pacemaker internally drops to hacluster but needs to write there.
|
||||
# 2. Set PATH so OCF agent child processes can find all required binaries.
|
||||
systemd.services.pacemaker.serviceConfig = {
|
||||
ExecStartPre = [
|
||||
"${pkgs.bash}/bin/bash -c 'for d in /var/lib/pacemaker /var/lib/pacemaker/cib /var/lib/pacemaker/cores /var/lib/pacemaker/pengine /var/lib/pacemaker/blackbox /var/lib/pacemaker/hostcache; do mkdir -p \"$d\" && chown hacluster:pacemaker \"$d\"; done'"
|
||||
];
|
||||
};
|
||||
systemd.services.pacemaker.environment = {
|
||||
PATH = lib.mkForce ocfBinPath;
|
||||
# Expose OCF root so pacemaker and lrmd agree on where agents live.
|
||||
OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
|
||||
};
|
||||
|
||||
# pacemaker-execd is the local resource executor that calls OCF agents.
|
||||
# Give it the same PATH so OCF scripts can find all required binaries.
|
||||
systemd.services.pacemaker-execd.environment = {
|
||||
PATH = lib.mkForce ocfBinPath;
|
||||
OCF_ROOT = "${pkgs.ocf-resource-agents}/usr/lib/ocf";
|
||||
};
|
||||
|
||||
# ── LIO / iSCSI target ────────────────────────────────────────────────
|
||||
# targetcli-fb is the management tool; actual kernel support is via
|
||||
# the LIO modules. We add a systemd service that saves/restores the
|
||||
# target configuration so Pacemaker can trigger it via a systemd-class
|
||||
# resource.
|
||||
boot.kernelModules = [
|
||||
"target_core_mod"
|
||||
"iscsi_target_mod"
|
||||
"target_core_file"
|
||||
"target_core_pscsi"
|
||||
"target_core_user"
|
||||
"configfs"
|
||||
];
|
||||
|
||||
# configfs must be mounted for rtslib/targetcli to work
|
||||
systemd.mounts = [{
|
||||
where = "/sys/kernel/config";
|
||||
what = "configfs";
|
||||
type = "configfs";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [ "targetctl.service" ];
|
||||
}];
|
||||
|
||||
# targetctl: save/restore LIO configuration (mirrors Debian's package)
|
||||
systemd.services.targetctl = {
|
||||
description = "LIO iSCSI target config save/restore";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "sys-kernel-config.mount" "network.target" ];
|
||||
requires = [ "sys-kernel-config.mount" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = "${pkgs.targetcli-fb}/bin/targetctl restore /etc/target/saveconfig.json";
|
||||
ExecStop = "${pkgs.targetcli-fb}/bin/targetctl save /etc/target/saveconfig.json";
|
||||
};
|
||||
unitConfig.ConditionFileNotEmpty = "/etc/target/saveconfig.json";
|
||||
};
|
||||
|
||||
# ── Packages ──────────────────────────────────────────────────────────
|
||||
environment.systemPackages = with pkgs; [
|
||||
# HA stack
|
||||
corosync # corosync-cfgtool, corosync-quorumtool
|
||||
pacemaker # crm_mon, crm_resource, cibadmin, crm_attribute, pcs CLI
|
||||
drbd # drbdadm, drbdsetup, drbdmon
|
||||
ocf-resource-agents # OCF heartbeat agents (Filesystem, IPaddr2, drbd, …)
|
||||
|
||||
# Storage
|
||||
xfsprogs # mkfs.xfs, xfs_admin, xfs_info
|
||||
targetcli-fb # targetcli shell + targetctl
|
||||
|
||||
# Networking / debug
|
||||
iproute2 # ip, ss
|
||||
iputils # ping
|
||||
tcpdump
|
||||
lsof
|
||||
|
||||
# Scripting / config
|
||||
python3
|
||||
curl
|
||||
jq
|
||||
vim
|
||||
htop
|
||||
];
|
||||
|
||||
# ── Firewall ──────────────────────────────────────────────────────────
|
||||
networking.firewall = {
|
||||
enable = true;
|
||||
allowedTCPPorts = [
|
||||
22 # SSH
|
||||
3260 # iSCSI
|
||||
3121 # pacemaker-remoted
|
||||
2224 # pcsd
|
||||
drbdPort # DRBD replication
|
||||
];
|
||||
allowedUDPPorts = [
|
||||
5404 # corosync cluster
|
||||
5405 # corosync cluster
|
||||
5407 # corosync crypto
|
||||
];
|
||||
# Corosync uses ports 5404-5407 UDP; allow them on the cluster net
|
||||
extraCommands = ''
|
||||
iptables -A INPUT -s ${node1Ip}/32 -j ACCEPT
|
||||
iptables -A INPUT -s ${node2Ip}/32 -j ACCEPT
|
||||
'';
|
||||
};
|
||||
|
||||
# ── tmpfiles: target config dir ────────────────────────────────────────
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /etc/target 0750 root root -"
|
||||
"f /etc/target/saveconfig.json 0640 root root -"
|
||||
];
|
||||
|
||||
# ── Locale / time ─────────────────────────────────────────────────────
|
||||
time.timeZone = vars.timeZone;
|
||||
i18n.defaultLocale = "en_AU.UTF-8";
|
||||
}
|
||||
Reference in new issue
Block a user