nixos/tests/etcd/etcd-cluster.nix

   1 # This test runs simple etcd cluster
   2
   3 import ../make-test-python.nix ({ pkgs, ... } : let
   4
   5   runWithOpenSSL = file: cmd: pkgs.runCommand file {
   6     buildInputs = [ pkgs.openssl ];
   7   } cmd;
   8
   9   ca_key = runWithOpenSSL "ca-key.pem" "openssl genrsa -out $out 2048";
  10   ca_pem = runWithOpenSSL "ca.pem" ''
  11     openssl req \
  12       -x509 -new -nodes -key ${ca_key} \
  13       -days 10000 -out $out -subj "/CN=etcd-ca"
  14   '';
  15   etcd_key = runWithOpenSSL "etcd-key.pem" "openssl genrsa -out $out 2048";
  16   etcd_csr = runWithOpenSSL "etcd.csr" ''
  17     openssl req \
  18        -new -key ${etcd_key} \
  19        -out $out -subj "/CN=etcd" \
  20        -config ${openssl_cnf}
  21   '';
  22   etcd_cert = runWithOpenSSL "etcd.pem" ''
  23     openssl x509 \
  24       -req -in ${etcd_csr} \
  25       -CA ${ca_pem} -CAkey ${ca_key} \
  26       -CAcreateserial -out $out \
  27       -days 365 -extensions v3_req \
  28       -extfile ${openssl_cnf}
  29   '';
  30
  31   etcd_client_key = runWithOpenSSL "etcd-client-key.pem"
  32     "openssl genrsa -out $out 2048";
  33
  34   etcd_client_csr = runWithOpenSSL "etcd-client-key.pem" ''
  35     openssl req \
  36       -new -key ${etcd_client_key} \
  37       -out $out -subj "/CN=etcd-client" \
  38       -config ${client_openssl_cnf}
  39   '';
  40
  41   etcd_client_cert = runWithOpenSSL "etcd-client.crt" ''
  42     openssl x509 \
  43       -req -in ${etcd_client_csr} \
  44       -CA ${ca_pem} -CAkey ${ca_key} -CAcreateserial \
  45       -out $out -days 365 -extensions v3_req \
  46       -extfile ${client_openssl_cnf}
  47   '';
  48
  49   openssl_cnf = pkgs.writeText "openssl.cnf" ''
  50     ions = v3_req
  51     distinguished_name = req_distinguished_name
  52     [req_distinguished_name]
  53     [ v3_req ]
  54     basicConstraints = CA:FALSE
  55     keyUsage = digitalSignature, keyEncipherment
  56     extendedKeyUsage = serverAuth, clientAuth
  57     subjectAltName = @alt_names
  58     [alt_names]
  59     DNS.1 = node1
  60     DNS.2 = node2
  61     DNS.3 = node3
  62     IP.1 = 127.0.0.1
  63   '';
  64
  65   client_openssl_cnf = pkgs.writeText "client-openssl.cnf" ''
  66     ions = v3_req
  67     distinguished_name = req_distinguished_name
  68     [req_distinguished_name]
  69     [ v3_req ]
  70     basicConstraints = CA:FALSE
  71     keyUsage = digitalSignature, keyEncipherment
  72     extendedKeyUsage = clientAuth
  73   '';
  74
  75   nodeConfig = {
  76     services = {
  77       etcd = {
  78         enable = true;
  79         keyFile = etcd_key;
  80         certFile = etcd_cert;
  81         trustedCaFile = ca_pem;
  82         clientCertAuth = true;
  83         listenClientUrls = ["https://127.0.0.1:2379"];
  84         listenPeerUrls = ["https://0.0.0.0:2380"];
  85       };
  86     };
  87
  88     environment.variables = {
  89       ETCD_CERT_FILE = "${etcd_client_cert}";
  90       ETCD_KEY_FILE = "${etcd_client_key}";
  91       ETCD_CA_FILE = "${ca_pem}";
  92       ETCDCTL_ENDPOINTS = "https://127.0.0.1:2379";
  93       ETCDCTL_CACERT = "${ca_pem}";
  94       ETCDCTL_CERT = "${etcd_cert}";
  95       ETCDCTL_KEY = "${etcd_key}";
  96     };
  97
  98     networking.firewall.allowedTCPPorts = [ 2380 ];
  99   };
 100 in {
 101   name = "etcd-cluster";
 102
 103   meta = with pkgs.lib.maintainers; {
 104     maintainers = [ offline ];
 105   };
 106
 107   nodes = {
 108     node1 = { ... }: {
 109       require = [nodeConfig];
 110       services.etcd = {
 111         initialCluster = ["node1=https://node1:2380" "node2=https://node2:2380"];
 112         initialAdvertisePeerUrls = ["https://node1:2380"];
 113       };
 114     };
 115
 116     node2 = { ... }: {
 117       require = [nodeConfig];
 118       services.etcd = {
 119         initialCluster = ["node1=https://node1:2380" "node2=https://node2:2380"];
 120         initialAdvertisePeerUrls = ["https://node2:2380"];
 121       };
 122     };
 123
 124     node3 = { ... }: {
 125       require = [nodeConfig];
 126       services.etcd = {
 127         initialCluster = ["node1=https://node1:2380" "node2=https://node2:2380" "node3=https://node3:2380"];
 128         initialAdvertisePeerUrls = ["https://node3:2380"];
 129         initialClusterState = "existing";
 130       };
 131     };
 132   };
 133
 134   testScript = ''
 135     with subtest("should start etcd cluster"):
 136         node1.start()
 137         node2.start()
 138         node1.wait_for_unit("etcd.service")
 139         node2.wait_for_unit("etcd.service")
 140         node2.wait_until_succeeds("etcdctl endpoint status")
 141         node1.succeed("etcdctl put /foo/bar 'Hello world'")
 142         node2.succeed("etcdctl get /foo/bar | grep 'Hello world'")
 143
 144     with subtest("should add another member"):
 145         node1.wait_until_succeeds("etcdctl member add node3 --peer-urls=https://node3:2380")
 146         node3.start()
 147         node3.wait_for_unit("etcd.service")
 148         node3.wait_until_succeeds("etcdctl member list | grep 'node3'")
 149         node3.succeed("etcdctl endpoint status")
 150
 151     with subtest("should survive member crash"):
 152         node3.crash()
 153         node1.succeed("etcdctl endpoint status")
 154         node1.succeed("etcdctl put /foo/bar 'Hello degraded world'")
 155         node1.succeed("etcdctl get /foo/bar | grep 'Hello degraded world'")
 156   '';
 157 })