nixos/modules/services/cluster/kubernetes/addons/dns.nix

   1 { config, options, pkgs, lib, ... }:
   2
   3 with lib;
   4
   5 let
   6   version = "1.7.1";
   7   cfg = config.services.kubernetes.addons.dns;
   8   ports = {
   9     dns = 10053;
  10     health = 10054;
  11     metrics = 10055;
  12   };
  13 in {
  14   options.services.kubernetes.addons.dns = {
  15     enable = mkEnableOption (lib.mdDoc "kubernetes dns addon");
  16
  17     clusterIp = mkOption {
  18       description = lib.mdDoc "Dns addon clusterIP";
  19
  20       # this default is also what kubernetes users
  21       default = (
  22         concatStringsSep "." (
  23           take 3 (splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange
  24         ))
  25       ) + ".254";
  26       defaultText = literalMD ''
  27         The `x.y.z.254` IP of
  28         `config.${options.services.kubernetes.apiserver.serviceClusterIpRange}`.
  29       '';
  30       type = types.str;
  31     };
  32
  33     clusterDomain = mkOption {
  34       description = lib.mdDoc "Dns cluster domain";
  35       default = "cluster.local";
  36       type = types.str;
  37     };
  38
  39     replicas = mkOption {
  40       description = lib.mdDoc "Number of DNS pod replicas to deploy in the cluster.";
  41       default = 2;
  42       type = types.int;
  43     };
  44
  45     reconcileMode = mkOption {
  46       description = lib.mdDoc ''
  47         Controls the addon manager reconciliation mode for the DNS addon.
  48
  49         Setting reconcile mode to EnsureExists makes it possible to tailor DNS behavior by editing the coredns ConfigMap.
  50
  51         See: <https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/README.md>.
  52       '';
  53       default = "Reconcile";
  54       type = types.enum [ "Reconcile" "EnsureExists" ];
  55     };
  56
  57     coredns = mkOption {
  58       description = lib.mdDoc "Docker image to seed for the CoreDNS container.";
  59       type = types.attrs;
  60       default = {
  61         imageName = "coredns/coredns";
  62         imageDigest = "sha256:4a6e0769130686518325b21b0c1d0688b54e7c79244d48e1b15634e98e40c6ef";
  63         finalImageTag = version;
  64         sha256 = "02r440xcdsgi137k5lmmvp0z5w5fmk8g9mysq5pnysq1wl8sj6mw";
  65       };
  66     };
  67
  68     corefile = mkOption {
  69       description = lib.mdDoc ''
  70         Custom coredns corefile configuration.
  71
  72         See: <https://coredns.io/manual/toc/#configuration>.
  73       '';
  74       type = types.str;
  75       default = ''
  76         .:${toString ports.dns} {
  77           errors
  78           health :${toString ports.health}
  79           kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
  80             pods insecure
  81             fallthrough in-addr.arpa ip6.arpa
  82           }
  83           prometheus :${toString ports.metrics}
  84           forward . /etc/resolv.conf
  85           cache 30
  86           loop
  87           reload
  88           loadbalance
  89         }'';
  90       defaultText = literalExpression ''
  91         '''
  92           .:${toString ports.dns} {
  93             errors
  94             health :${toString ports.health}
  95             kubernetes ''${config.services.kubernetes.addons.dns.clusterDomain} in-addr.arpa ip6.arpa {
  96               pods insecure
  97               fallthrough in-addr.arpa ip6.arpa
  98             }
  99             prometheus :${toString ports.metrics}
 100             forward . /etc/resolv.conf
 101             cache 30
 102             loop
 103             reload
 104             loadbalance
 105           }
 106         '''
 107       '';
 108     };
 109   };
 110
 111   config = mkIf cfg.enable {
 112     services.kubernetes.kubelet.seedDockerImages =
 113       singleton (pkgs.dockerTools.pullImage cfg.coredns);
 114
 115     services.kubernetes.addonManager.bootstrapAddons = {
 116       coredns-cr = {
 117         apiVersion = "rbac.authorization.k8s.io/v1";
 118         kind = "ClusterRole";
 119         metadata = {
 120           labels = {
 121             "addonmanager.kubernetes.io/mode" = "Reconcile";
 122             k8s-app = "kube-dns";
 123             "kubernetes.io/cluster-service" = "true";
 124             "kubernetes.io/bootstrapping" = "rbac-defaults";
 125           };
 126           name = "system:coredns";
 127         };
 128         rules = [
 129           {
 130             apiGroups = [ "" ];
 131             resources = [ "endpoints" "services" "pods" "namespaces" ];
 132             verbs = [ "list" "watch" ];
 133           }
 134           {
 135             apiGroups = [ "" ];
 136             resources = [ "nodes" ];
 137             verbs = [ "get" ];
 138           }
 139         ];
 140       };
 141
 142       coredns-crb = {
 143         apiVersion = "rbac.authorization.k8s.io/v1";
 144         kind = "ClusterRoleBinding";
 145         metadata = {
 146           annotations = {
 147             "rbac.authorization.kubernetes.io/autoupdate" = "true";
 148           };
 149           labels = {
 150             "addonmanager.kubernetes.io/mode" = "Reconcile";
 151             k8s-app = "kube-dns";
 152             "kubernetes.io/cluster-service" = "true";
 153             "kubernetes.io/bootstrapping" = "rbac-defaults";
 154           };
 155           name = "system:coredns";
 156         };
 157         roleRef = {
 158           apiGroup = "rbac.authorization.k8s.io";
 159           kind = "ClusterRole";
 160           name = "system:coredns";
 161         };
 162         subjects = [
 163           {
 164             kind = "ServiceAccount";
 165             name = "coredns";
 166             namespace = "kube-system";
 167           }
 168         ];
 169       };
 170     };
 171
 172     services.kubernetes.addonManager.addons = {
 173       coredns-sa = {
 174         apiVersion = "v1";
 175         kind = "ServiceAccount";
 176         metadata = {
 177           labels = {
 178             "addonmanager.kubernetes.io/mode" = "Reconcile";
 179             k8s-app = "kube-dns";
 180             "kubernetes.io/cluster-service" = "true";
 181           };
 182           name = "coredns";
 183           namespace = "kube-system";
 184         };
 185       };
 186
 187       coredns-cm = {
 188         apiVersion = "v1";
 189         kind = "ConfigMap";
 190         metadata = {
 191           labels = {
 192             "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
 193             k8s-app = "kube-dns";
 194             "kubernetes.io/cluster-service" = "true";
 195           };
 196           name = "coredns";
 197           namespace = "kube-system";
 198         };
 199         data = {
 200           Corefile = cfg.corefile;
 201         };
 202       };
 203
 204       coredns-deploy = {
 205         apiVersion = "apps/v1";
 206         kind = "Deployment";
 207         metadata = {
 208           labels = {
 209             "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
 210             k8s-app = "kube-dns";
 211             "kubernetes.io/cluster-service" = "true";
 212             "kubernetes.io/name" = "CoreDNS";
 213           };
 214           name = "coredns";
 215           namespace = "kube-system";
 216         };
 217         spec = {
 218           replicas = cfg.replicas;
 219           selector = {
 220             matchLabels = { k8s-app = "kube-dns"; };
 221           };
 222           strategy = {
 223             rollingUpdate = { maxUnavailable = 1; };
 224             type = "RollingUpdate";
 225           };
 226           template = {
 227             metadata = {
 228               labels = {
 229                 k8s-app = "kube-dns";
 230               };
 231             };
 232             spec = {
 233               containers = [
 234                 {
 235                   args = [ "-conf" "/etc/coredns/Corefile" ];
 236                   image = with cfg.coredns; "${imageName}:${finalImageTag}";
 237                   imagePullPolicy = "Never";
 238                   livenessProbe = {
 239                     failureThreshold = 5;
 240                     httpGet = {
 241                       path = "/health";
 242                       port = ports.health;
 243                       scheme = "HTTP";
 244                     };
 245                     initialDelaySeconds = 60;
 246                     successThreshold = 1;
 247                     timeoutSeconds = 5;
 248                   };
 249                   name = "coredns";
 250                   ports = [
 251                     {
 252                       containerPort = ports.dns;
 253                       name = "dns";
 254                       protocol = "UDP";
 255                     }
 256                     {
 257                       containerPort = ports.dns;
 258                       name = "dns-tcp";
 259                       protocol = "TCP";
 260                     }
 261                     {
 262                       containerPort = ports.metrics;
 263                       name = "metrics";
 264                       protocol = "TCP";
 265                     }
 266                   ];
 267                   resources = {
 268                     limits = {
 269                       memory = "170Mi";
 270                     };
 271                     requests = {
 272                       cpu = "100m";
 273                       memory = "70Mi";
 274                     };
 275                   };
 276                   securityContext = {
 277                     allowPrivilegeEscalation = false;
 278                     capabilities = {
 279                       drop = [ "all" ];
 280                     };
 281                     readOnlyRootFilesystem = true;
 282                   };
 283                   volumeMounts = [
 284                     {
 285                       mountPath = "/etc/coredns";
 286                       name = "config-volume";
 287                       readOnly = true;
 288                     }
 289                   ];
 290                 }
 291               ];
 292               dnsPolicy = "Default";
 293               nodeSelector = {
 294                 "beta.kubernetes.io/os" = "linux";
 295               };
 296               serviceAccountName = "coredns";
 297               tolerations = [
 298                 {
 299                   effect = "NoSchedule";
 300                   key = "node-role.kubernetes.io/master";
 301                 }
 302                 {
 303                   key = "CriticalAddonsOnly";
 304                   operator = "Exists";
 305                 }
 306               ];
 307               volumes = [
 308                 {
 309                   configMap = {
 310                     items = [
 311                       {
 312                         key = "Corefile";
 313                         path = "Corefile";
 314                       }
 315                     ];
 316                     name = "coredns";
 317                   };
 318                   name = "config-volume";
 319                 }
 320               ];
 321             };
 322           };
 323         };
 324       };
 325
 326       coredns-svc = {
 327         apiVersion = "v1";
 328         kind = "Service";
 329         metadata = {
 330           annotations = {
 331             "prometheus.io/port" = toString ports.metrics;
 332             "prometheus.io/scrape" = "true";
 333           };
 334           labels = {
 335             "addonmanager.kubernetes.io/mode" = "Reconcile";
 336             k8s-app = "kube-dns";
 337             "kubernetes.io/cluster-service" = "true";
 338             "kubernetes.io/name" = "CoreDNS";
 339           };
 340           name = "kube-dns";
 341           namespace = "kube-system";
 342         };
 343         spec = {
 344           clusterIP = cfg.clusterIp;
 345           ports = [
 346             {
 347               name = "dns";
 348               port = 53;
 349               targetPort = ports.dns;
 350               protocol = "UDP";
 351             }
 352             {
 353               name = "dns-tcp";
 354               port = 53;
 355               targetPort = ports.dns;
 356               protocol = "TCP";
 357             }
 358           ];
 359           selector = { k8s-app = "kube-dns"; };
 360         };
 361       };
 362     };
 363
 364     services.kubernetes.kubelet.clusterDns = mkDefault cfg.clusterIp;
 365   };
 366
 367   meta.buildDocsInSandbox = false;
 368 }