nixos/tests/k3s/multi-node.nix

   1 # A test that runs a multi-node k3s cluster and verify pod networking works across nodes
   2 import ../make-test-python.nix (
   3   {
   4     pkgs,
   5     lib,
   6     k3s,
   7     ...
   8   }:
   9   let
  10     imageEnv = pkgs.buildEnv {
  11       name = "k3s-pause-image-env";
  12       paths = with pkgs; [
  13         tini
  14         bashInteractive
  15         coreutils
  16         socat
  17       ];
  18     };
  19     pauseImage = pkgs.dockerTools.streamLayeredImage {
  20       name = "test.local/pause";
  21       tag = "local";
  22       contents = imageEnv;
  23       config.Entrypoint = [
  24         "/bin/tini"
  25         "--"
  26         "/bin/sleep"
  27         "inf"
  28       ];
  29     };
  30     # A daemonset that responds 'server' on port 8000
  31     networkTestDaemonset = pkgs.writeText "test.yml" ''
  32       apiVersion: apps/v1
  33       kind: DaemonSet
  34       metadata:
  35         name: test
  36         labels:
  37           name: test
  38       spec:
  39         selector:
  40           matchLabels:
  41             name: test
  42         template:
  43           metadata:
  44             labels:
  45               name: test
  46           spec:
  47             containers:
  48             - name: test
  49               image: test.local/pause:local
  50               imagePullPolicy: Never
  51               resources:
  52                 limits:
  53                   memory: 20Mi
  54               command: ["socat", "TCP4-LISTEN:8000,fork", "EXEC:echo server"]
  55     '';
  56     tokenFile = pkgs.writeText "token" "p@s$w0rd";
  57   in
  58   {
  59     name = "${k3s.name}-multi-node";
  60
  61     nodes = {
  62       server =
  63         { pkgs, ... }:
  64         {
  65           environment.systemPackages = with pkgs; [
  66             gzip
  67             jq
  68           ];
  69           # k3s uses enough resources the default vm fails.
  70           virtualisation.memorySize = 1536;
  71           virtualisation.diskSize = 4096;
  72
  73           services.k3s = {
  74             inherit tokenFile;
  75             enable = true;
  76             role = "server";
  77             package = k3s;
  78             clusterInit = true;
  79             extraFlags = [
  80               "--disable coredns"
  81               "--disable local-storage"
  82               "--disable metrics-server"
  83               "--disable servicelb"
  84               "--disable traefik"
  85               "--node-ip 192.168.1.1"
  86               "--pause-image test.local/pause:local"
  87             ];
  88           };
  89           networking.firewall.allowedTCPPorts = [
  90             2379
  91             2380
  92             6443
  93           ];
  94           networking.firewall.allowedUDPPorts = [ 8472 ];
  95           networking.firewall.trustedInterfaces = [ "flannel.1" ];
  96           networking.useDHCP = false;
  97           networking.defaultGateway = "192.168.1.1";
  98           networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
  99             {
 100               address = "192.168.1.1";
 101               prefixLength = 24;
 102             }
 103           ];
 104         };
 105
 106       server2 =
 107         { pkgs, ... }:
 108         {
 109           environment.systemPackages = with pkgs; [
 110             gzip
 111             jq
 112           ];
 113           virtualisation.memorySize = 1536;
 114           virtualisation.diskSize = 4096;
 115
 116           services.k3s = {
 117             inherit tokenFile;
 118             enable = true;
 119             serverAddr = "https://192.168.1.1:6443";
 120             clusterInit = false;
 121             extraFlags = builtins.toString [
 122               "--disable"
 123               "coredns"
 124               "--disable"
 125               "local-storage"
 126               "--disable"
 127               "metrics-server"
 128               "--disable"
 129               "servicelb"
 130               "--disable"
 131               "traefik"
 132               "--node-ip"
 133               "192.168.1.3"
 134               "--pause-image"
 135               "test.local/pause:local"
 136             ];
 137           };
 138           networking.firewall.allowedTCPPorts = [
 139             2379
 140             2380
 141             6443
 142           ];
 143           networking.firewall.allowedUDPPorts = [ 8472 ];
 144           networking.firewall.trustedInterfaces = [ "flannel.1" ];
 145           networking.useDHCP = false;
 146           networking.defaultGateway = "192.168.1.3";
 147           networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
 148             {
 149               address = "192.168.1.3";
 150               prefixLength = 24;
 151             }
 152           ];
 153         };
 154
 155       agent =
 156         { pkgs, ... }:
 157         {
 158           virtualisation.memorySize = 1024;
 159           virtualisation.diskSize = 2048;
 160           services.k3s = {
 161             inherit tokenFile;
 162             enable = true;
 163             role = "agent";
 164             serverAddr = "https://192.168.1.3:6443";
 165             extraFlags = lib.concatStringsSep " " [
 166               "--pause-image"
 167               "test.local/pause:local"
 168               "--node-ip"
 169               "192.168.1.2"
 170             ];
 171           };
 172           networking.firewall.allowedTCPPorts = [ 6443 ];
 173           networking.firewall.allowedUDPPorts = [ 8472 ];
 174           networking.firewall.trustedInterfaces = [ "flannel.1" ];
 175           networking.useDHCP = false;
 176           networking.defaultGateway = "192.168.1.2";
 177           networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
 178             {
 179               address = "192.168.1.2";
 180               prefixLength = 24;
 181             }
 182           ];
 183         };
 184     };
 185
 186     testScript = ''
 187       machines = [server, server2, agent]
 188       for m in machines:
 189           m.start()
 190           m.wait_for_unit("k3s")
 191
 192       is_aarch64 = "${toString pkgs.stdenv.hostPlatform.isAarch64}" == "1"
 193
 194       # wait for the agent to show up
 195       server.wait_until_succeeds("k3s kubectl get node agent")
 196
 197       for m in machines:
 198           m.succeed("k3s check-config")
 199           m.succeed(
 200               "${pauseImage} | k3s ctr image import -"
 201           )
 202
 203       server.succeed("k3s kubectl cluster-info")
 204       # Also wait for our service account to show up; it takes a sec
 205       server.wait_until_succeeds("k3s kubectl get serviceaccount default")
 206
 207       # Now create a pod on each node via a daemonset and verify they can talk to each other.
 208       server.succeed("k3s kubectl apply -f ${networkTestDaemonset}")
 209       server.wait_until_succeeds(f'[ "$(k3s kubectl get ds test -o json | jq .status.numberReady)" -eq {len(machines)} ]')
 210
 211       # Get pod IPs
 212       pods = server.succeed("k3s kubectl get po -o json | jq '.items[].metadata.name' -r").splitlines()
 213       pod_ips = [server.succeed(f"k3s kubectl get po {name} -o json | jq '.status.podIP' -cr").strip() for name in pods]
 214
 215       # Verify each server can ping each pod ip
 216       for pod_ip in pod_ips:
 217           server.succeed(f"ping -c 1 {pod_ip}")
 218           agent.succeed(f"ping -c 1 {pod_ip}")
 219
 220       # Verify the pods can talk to each other
 221       resp = server.wait_until_succeeds(f"k3s kubectl exec {pods[0]} -- socat TCP:{pod_ips[1]}:8000 -")
 222       assert resp.strip() == "server"
 223       resp = server.wait_until_succeeds(f"k3s kubectl exec {pods[1]} -- socat TCP:{pod_ips[0]}:8000 -")
 224       assert resp.strip() == "server"
 225
 226       # Cleanup
 227       server.succeed("k3s kubectl delete -f ${networkTestDaemonset}")
 228
 229       for m in machines:
 230           m.shutdown()
 231     '';
 232
 233     meta.maintainers = lib.teams.k3s.members;
 234   }
 235 )