nixos/tests/k3s/kubelet-config.nix

   1 # A test that sets extra kubelet configuration and enables graceful node shutdown
   2 import ../make-test-python.nix (
   3   {
   4     pkgs,
   5     lib,
   6     k3s,
   7     ...
   8   }:
   9   let
  10     nodeName = "test";
  11     shutdownGracePeriod = "1m13s";
  12     shutdownGracePeriodCriticalPods = "13s";
  13     podsPerCore = 3;
  14     memoryThrottlingFactor = 0.69;
  15     containerLogMaxSize = "5Mi";
  16   in
  17   {
  18     name = "${k3s.name}-kubelet-config";
  19     nodes.machine =
  20       { pkgs, ... }:
  21       {
  22         environment.systemPackages = [ pkgs.jq ];
  23
  24         # k3s uses enough resources the default vm fails.
  25         virtualisation.memorySize = 1536;
  26         virtualisation.diskSize = 4096;
  27
  28         services.k3s = {
  29           enable = true;
  30           package = k3s;
  31           # Slightly reduce resource usage
  32           extraFlags = [
  33             "--disable coredns"
  34             "--disable local-storage"
  35             "--disable metrics-server"
  36             "--disable servicelb"
  37             "--disable traefik"
  38             "--node-name ${nodeName}"
  39           ];
  40           gracefulNodeShutdown = {
  41             enable = true;
  42             inherit shutdownGracePeriod shutdownGracePeriodCriticalPods;
  43           };
  44           extraKubeletConfig = {
  45             inherit podsPerCore memoryThrottlingFactor containerLogMaxSize;
  46           };
  47         };
  48       };
  49
  50     testScript = ''
  51       import json
  52
  53       start_all()
  54       machine.wait_for_unit("k3s")
  55       # wait until the node is ready
  56       machine.wait_until_succeeds(r"""kubectl get node ${nodeName} -ojson | jq -e '.status.conditions[] | select(.type == "Ready") | .status == "True"'""")
  57       # test whether the kubelet registered an inhibitor lock
  58       machine.succeed("systemd-inhibit --list --no-legend | grep \"kubelet.*k3s-server.*shutdown\"")
  59       # run kubectl proxy in the background, close stdout through redirection to not wait for the command to finish
  60       machine.execute("kubectl proxy --address 127.0.0.1 --port=8001 >&2 &")
  61       machine.wait_until_succeeds("nc -z 127.0.0.1 8001")
  62       # get the kubeletconfig
  63       kubelet_config=json.loads(machine.succeed("curl http://127.0.0.1:8001/api/v1/nodes/${nodeName}/proxy/configz | jq '.kubeletconfig'"))
  64
  65       with subtest("Kubelet config values are set correctly"):
  66         assert kubelet_config["shutdownGracePeriod"] == "${shutdownGracePeriod}", \
  67           f"unexpected value for shutdownGracePeriod: {kubelet_config["shutdownGracePeriod"]}"
  68         assert kubelet_config["shutdownGracePeriodCriticalPods"] == "${shutdownGracePeriodCriticalPods}", \
  69           f"unexpected value for shutdownGracePeriodCriticalPods: {kubelet_config["shutdownGracePeriodCriticalPods"]}"
  70         assert kubelet_config["podsPerCore"] == ${toString podsPerCore}, \
  71           f"unexpected value for podsPerCore: {kubelet_config["podsPerCore"]}"
  72         assert kubelet_config["memoryThrottlingFactor"] == ${toString memoryThrottlingFactor}, \
  73           f"unexpected value for memoryThrottlingFactor: {kubelet_config["memoryThrottlingFactor"]}"
  74         assert kubelet_config["containerLogMaxSize"] == "${containerLogMaxSize}", \
  75           f"unexpected value for containerLogMaxSize: {kubelet_config["containerLogMaxSize"]}"
  76     '';
  77
  78     meta.maintainers = lib.teams.k3s.members;
  79   }
  80 )