nixos/tests/iscsi-multipath-root.nix

   1 import ./make-test-python.nix (
   2   { pkgs, lib, ... }:
   3   let
   4     initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
   5     targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
   6   in
   7   {
   8     name = "iscsi";
   9     meta = {
  10       maintainers = pkgs.lib.teams.deshaw.members;
  11     };
  12
  13     nodes = {
  14       target = { config, pkgs, lib, ... }: {
  15         virtualisation.vlans = [ 1 2 ];
  16         services.target = {
  17           enable = true;
  18           config = {
  19             fabric_modules = [ ];
  20             storage_objects = [
  21               {
  22                 dev = "/dev/vdb";
  23                 name = "test";
  24                 plugin = "block";
  25                 write_back = true;
  26                 wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
  27               }
  28             ];
  29             targets = [
  30               {
  31                 fabric = "iscsi";
  32                 tpgs = [
  33                   {
  34                     enable = true;
  35                     attributes = {
  36                       authentication = 0;
  37                       generate_node_acls = 1;
  38                     };
  39                     luns = [
  40                       {
  41                         alias = "94dfe06967";
  42                         alua_tg_pt_gp_name = "default_tg_pt_gp";
  43                         index = 0;
  44                         storage_object = "/backstores/block/test";
  45                       }
  46                     ];
  47                     node_acls = [
  48                       {
  49                         mapped_luns = [
  50                           {
  51                             alias = "d42f5bdf8a";
  52                             index = 0;
  53                             tpg_lun = 0;
  54                             write_protect = false;
  55                           }
  56                         ];
  57                         node_wwn = initiatorName;
  58                       }
  59                     ];
  60                     portals = [
  61                       {
  62                         ip_address = "0.0.0.0";
  63                         iser = false;
  64                         offload = false;
  65                         port = 3260;
  66                       }
  67                     ];
  68                     tag = 1;
  69                   }
  70                 ];
  71                 wwn = targetName;
  72               }
  73             ];
  74           };
  75         };
  76
  77         networking.firewall.allowedTCPPorts = [ 3260 ];
  78         networking.firewall.allowedUDPPorts = [ 3260 ];
  79
  80         virtualisation.memorySize = 2048;
  81         virtualisation.emptyDiskImages = [ 2048 ];
  82       };
  83
  84       initiatorAuto = { nodes, config, pkgs, ... }: {
  85         virtualisation.vlans = [ 1 2 ];
  86
  87         services.multipath = {
  88           enable = true;
  89           defaults = ''
  90             find_multipaths yes
  91             user_friendly_names yes
  92           '';
  93           pathGroups = [
  94             {
  95               alias = 123456;
  96               wwid = "3600140592b17c3f6b404168b082ceeb7";
  97             }
  98           ];
  99         };
 100
 101         services.openiscsi = {
 102           enable = true;
 103           enableAutoLoginOut = true;
 104           discoverPortal = "target";
 105           name = initiatorName;
 106         };
 107
 108         environment.systemPackages = with pkgs; [
 109           xfsprogs
 110         ];
 111
 112         environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;
 113
 114         nix.settings = {
 115           substituters = lib.mkForce [ ];
 116           hashed-mirrors = null;
 117           connect-timeout = 1;
 118         };
 119       };
 120
 121       initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
 122         boot.initrd.network.enable = true;
 123         boot.loader.grub.enable = false;
 124
 125         boot.kernelParams = lib.mkOverride 5 (
 126           [
 127             "boot.shell_on_fail"
 128             "console=tty1"
 129             "ip=192.168.1.1:::255.255.255.0::ens9:none"
 130             "ip=192.168.2.1:::255.255.255.0::ens10:none"
 131           ]
 132         );
 133
 134         # defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
 135         virtualisation.writableStore = false;
 136         virtualisation.vlans = [ 1 2 ];
 137
 138         services.multipath = {
 139           enable = true;
 140           defaults = ''
 141             find_multipaths yes
 142             user_friendly_names yes
 143           '';
 144           pathGroups = [
 145             {
 146               alias = 123456;
 147               wwid = "3600140592b17c3f6b404168b082ceeb7";
 148             }
 149           ];
 150         };
 151
 152         fileSystems = lib.mkOverride 5 {
 153           "/" = {
 154             fsType = "xfs";
 155             device = "/dev/mapper/123456";
 156             options = [ "_netdev" ];
 157           };
 158         };
 159
 160         boot.initrd.extraFiles."etc/multipath/wwids".source = pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
 161
 162         boot.iscsi-initiator = {
 163           discoverPortal = "target";
 164           name = initiatorName;
 165           target = targetName;
 166           extraIscsiCommands = ''
 167             iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
 168           '';
 169         };
 170       };
 171
 172     };
 173
 174     testScript = { nodes, ... }: ''
 175       target.start()
 176       target.wait_for_unit("iscsi-target.service")
 177
 178       initiatorAuto.start()
 179
 180       initiatorAuto.wait_for_unit("iscsid.service")
 181       initiatorAuto.wait_for_unit("iscsi.service")
 182       initiatorAuto.get_unit_info("iscsi")
 183
 184       # Expecting this to fail since we should already know about 192.168.1.3
 185       initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
 186       # Expecting this to succeed since we don't yet know about 192.168.2.3
 187       initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
 188
 189       # /dev/sda is provided by iscsi on target
 190       initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
 191
 192       initiatorAuto.succeed("mkfs.xfs /dev/sda")
 193       initiatorAuto.succeed("mkdir /mnt")
 194
 195       # Start by verifying /dev/sda and /dev/sdb are both the same disk
 196       initiatorAuto.succeed("mount /dev/sda /mnt")
 197       initiatorAuto.succeed("touch /mnt/hi")
 198       initiatorAuto.succeed("umount /mnt")
 199
 200       initiatorAuto.succeed("mount /dev/sdb /mnt")
 201       initiatorAuto.succeed("test -e /mnt/hi")
 202       initiatorAuto.succeed("umount /mnt")
 203
 204       initiatorAuto.succeed("systemctl restart multipathd")
 205       initiatorAuto.succeed("multipath -ll | systemd-cat")
 206
 207       # Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
 208       initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
 209       initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
 210       initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
 211       initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
 212       initiatorAuto.succeed(
 213         "nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
 214       )
 215       initiatorAuto.succeed("umount /mnt")
 216       initiatorAuto.shutdown()
 217
 218       initiatorRootDisk.start()
 219       initiatorRootDisk.wait_for_unit("multi-user.target")
 220       initiatorRootDisk.wait_for_unit("iscsid")
 221
 222       # Log in over both nodes
 223       initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
 224       initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
 225       initiatorRootDisk.succeed("systemctl restart multipathd")
 226       initiatorRootDisk.succeed("multipath -ll | systemd-cat")
 227
 228       # Verify we can write and sync the root disk
 229       initiatorRootDisk.succeed("mkdir /scratch")
 230       initiatorRootDisk.succeed("touch /scratch/both-up")
 231       initiatorRootDisk.succeed("sync /scratch")
 232
 233       # Verify we can write to the root with ens9 (sda, 192.168.1.3) down
 234       initiatorRootDisk.succeed("ip link set ens9 down")
 235       initiatorRootDisk.succeed("touch /scratch/ens9-down")
 236       initiatorRootDisk.succeed("sync /scratch")
 237       initiatorRootDisk.succeed("ip link set ens9 up")
 238
 239       # todo: better way to wait until multipath notices the link is back
 240       initiatorRootDisk.succeed("sleep 5")
 241       initiatorRootDisk.succeed("touch /scratch/both-down")
 242       initiatorRootDisk.succeed("sync /scratch")
 243
 244       # Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
 245       initiatorRootDisk.succeed("ip link set ens10 down")
 246       initiatorRootDisk.succeed("touch /scratch/ens10-down")
 247       initiatorRootDisk.succeed("sync /scratch")
 248       initiatorRootDisk.succeed("ip link set ens10 up")
 249       initiatorRootDisk.succeed("touch /scratch/ens10-down")
 250       initiatorRootDisk.succeed("sync /scratch")
 251
 252       initiatorRootDisk.succeed("ip link set ens9 up")
 253       initiatorRootDisk.succeed("ip link set ens10 up")
 254       initiatorRootDisk.shutdown()
 255
 256       # Verify we can boot with the target's eth1 down, forcing
 257       # it to multipath via the second link
 258       target.succeed("ip link set eth1 down")
 259       initiatorRootDisk.start()
 260       initiatorRootDisk.wait_for_unit("multi-user.target")
 261       initiatorRootDisk.wait_for_unit("iscsid")
 262       initiatorRootDisk.succeed("test -e /scratch/both-up")
 263     '';
 264   }
 265 )
 266
 267