nixos/tests/cassandra.nix

   1 import ./make-test-python.nix ({ pkgs, lib, testPackage ? pkgs.cassandra, ... }:
   2 let
   3   clusterName = "NixOS Automated-Test Cluster";
   4
   5   testRemoteAuth = lib.versionAtLeast testPackage.version "3.11";
   6   jmxRoles = [{ username = "me"; password = "password"; }];
   7   jmxRolesFile = ./cassandra-jmx-roles;
   8   jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}";
   9   jmxPort = 7200;  # Non-standard port so it doesn't accidentally work
  10   jmxPortStr = toString jmxPort;
  11
  12   # Would usually be assigned to 512M.
  13   # Set it to a different value, so that we can check whether our config
  14   # actually changes it.
  15   numMaxHeapSize = "400";
  16   getHeapLimitCommand = ''
  17     nodetool info -p ${jmxPortStr} | grep "^Heap Memory" | awk '{print $NF}'
  18   '';
  19   checkHeapLimitCommand = pkgs.writeShellScript "check-heap-limit.sh" ''
  20     [ 1 -eq "$(echo "$(${getHeapLimitCommand}) < ${numMaxHeapSize}" | ${pkgs.bc}/bin/bc)" ]
  21   '';
  22
  23   cassandraCfg = ipAddress:
  24     { enable = true;
  25       inherit clusterName;
  26       listenAddress = ipAddress;
  27       rpcAddress = ipAddress;
  28       seedAddresses = [ "192.168.1.1" ];
  29       package = testPackage;
  30       maxHeapSize = "${numMaxHeapSize}M";
  31       heapNewSize = "100M";
  32       inherit jmxPort;
  33     };
  34   nodeCfg = ipAddress: extra: {pkgs, config, ...}: rec {
  35     environment.systemPackages = [ testPackage ];
  36     networking = {
  37       firewall.allowedTCPPorts = [ 7000 9042 services.cassandra.jmxPort ];
  38       useDHCP = false;
  39       interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [
  40         { address = ipAddress; prefixLength = 24; }
  41       ];
  42     };
  43     services.cassandra = cassandraCfg ipAddress // extra;
  44   };
  45 in
  46 {
  47   name = "cassandra-${testPackage.version}";
  48   meta = {
  49     maintainers = with lib.maintainers; [ johnazoidberg ];
  50   };
  51
  52   nodes = {
  53     cass0 = nodeCfg "192.168.1.1" {};
  54     cass1 = nodeCfg "192.168.1.2" (lib.optionalAttrs testRemoteAuth { inherit jmxRoles; remoteJmx = true; });
  55     cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; };
  56   };
  57
  58   testScript = ''
  59     # Check configuration
  60     with subtest("Timers exist"):
  61         cass0.succeed("systemctl list-timers | grep cassandra-full-repair.timer")
  62         cass0.succeed("systemctl list-timers | grep cassandra-incremental-repair.timer")
  63
  64     with subtest("Can connect via cqlsh"):
  65         cass0.wait_for_unit("cassandra.service")
  66         cass0.wait_until_succeeds("nc -z cass0 9042")
  67         cass0.succeed("echo 'show version;' | cqlsh cass0")
  68
  69     with subtest("Nodetool is operational"):
  70         cass0.wait_for_unit("cassandra.service")
  71         cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
  72         cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass0'")
  73
  74     with subtest("Cluster name was set"):
  75         cass0.wait_for_unit("cassandra.service")
  76         cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
  77         cass0.wait_until_succeeds(
  78             "nodetool describecluster -p ${jmxPortStr} | grep 'Name: ${clusterName}'"
  79         )
  80
  81     with subtest("Heap limit set correctly"):
  82         # Nodetool takes a while until it can display info
  83         cass0.wait_until_succeeds("nodetool info -p ${jmxPortStr}")
  84         cass0.succeed("${checkHeapLimitCommand}")
  85
  86     # Check cluster interaction
  87     with subtest("Bring up cluster"):
  88         cass1.wait_for_unit("cassandra.service")
  89         cass1.wait_until_succeeds(
  90             "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"
  91         )
  92         cass0.succeed("nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'")
  93   '' + lib.optionalString testRemoteAuth ''
  94     with subtest("Remote authenticated jmx"):
  95         # Doesn't work if not enabled
  96         cass0.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
  97         cass1.fail("nc -z 192.168.1.1 ${jmxPortStr}")
  98         cass1.fail("nodetool -p ${jmxPortStr} -h 192.168.1.1 status")
  99
 100         # Works if enabled
 101         cass1.wait_until_succeeds("nc -z localhost ${jmxPortStr}")
 102         cass0.succeed("nodetool -p ${jmxPortStr} -h 192.168.1.2 ${jmxAuthArgs} status")
 103   '' + ''
 104     with subtest("Break and fix node"):
 105         cass1.block()
 106         cass0.wait_until_succeeds(
 107             "nodetool status -p ${jmxPortStr} --resolve-ip | egrep -c '^DN[[:space:]]+cass1'"
 108         )
 109         cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN'  | grep 1")
 110         cass1.unblock()
 111         cass1.wait_until_succeeds(
 112             "nodetool -p ${jmxPortStr} ${jmxAuthArgs} status | egrep -c '^UN'  | grep 2"
 113         )
 114         cass0.succeed("nodetool status -p ${jmxPortStr} | egrep -c '^UN'  | grep 2")
 115
 116     with subtest("Replace crashed node"):
 117         cass1.block()  # .crash() waits until it's fully shutdown
 118         cass2.start()
 119         cass0.wait_until_fails(
 120             "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass1'"
 121         )
 122
 123         cass2.wait_for_unit("cassandra.service")
 124         cass0.wait_until_succeeds(
 125             "nodetool status -p ${jmxPortStr} --resolve-ip | egrep '^UN[[:space:]]+cass2'"
 126         )
 127   '';
 128
 129   passthru = {
 130     inherit testPackage;
 131   };
 132 })