nixos/modules/services/cluster/kubernetes/addons/dns.nix

   1 { config, options, pkgs, lib, ... }:
   2 let
   3   version = "1.10.1";
   4   cfg = config.services.kubernetes.addons.dns;
   5   ports = {
   6     dns = 10053;
   7     health = 10054;
   8     metrics = 10055;
   9   };
  10 in {
  11   options.services.kubernetes.addons.dns = {
  12     enable = lib.mkEnableOption "kubernetes dns addon";
  13
  14     clusterIp = lib.mkOption {
  15       description = "Dns addon clusterIP";
  16
  17       # this default is also what kubernetes users
  18       default = (
  19         lib.concatStringsSep "." (
  20           lib.take 3 (lib.splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange
  21         ))
  22       ) + ".254";
  23       defaultText = lib.literalMD ''
  24         The `x.y.z.254` IP of
  25         `config.${options.services.kubernetes.apiserver.serviceClusterIpRange}`.
  26       '';
  27       type = lib.types.str;
  28     };
  29
  30     clusterDomain = lib.mkOption {
  31       description = "Dns cluster domain";
  32       default = "cluster.local";
  33       type = lib.types.str;
  34     };
  35
  36     replicas = lib.mkOption {
  37       description = "Number of DNS pod replicas to deploy in the cluster.";
  38       default = 2;
  39       type = lib.types.int;
  40     };
  41
  42     reconcileMode = lib.mkOption {
  43       description = ''
  44         Controls the addon manager reconciliation mode for the DNS addon.
  45
  46         Setting reconcile mode to EnsureExists makes it possible to tailor DNS behavior by editing the coredns ConfigMap.
  47
  48         See: <https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/README.md>.
  49       '';
  50       default = "Reconcile";
  51       type = lib.types.enum [ "Reconcile" "EnsureExists" ];
  52     };
  53
  54     coredns = lib.mkOption {
  55       description = "Docker image to seed for the CoreDNS container.";
  56       type = lib.types.attrs;
  57       default = {
  58         imageName = "coredns/coredns";
  59         imageDigest = "sha256:a0ead06651cf580044aeb0a0feba63591858fb2e43ade8c9dea45a6a89ae7e5e";
  60         finalImageTag = version;
  61         sha256 = "0wg696920smmal7552a2zdhfncndn5kfammfa8bk8l7dz9bhk0y1";
  62       };
  63     };
  64
  65     corefile = lib.mkOption {
  66       description = ''
  67         Custom coredns corefile configuration.
  68
  69         See: <https://coredns.io/manual/toc/#configuration>.
  70       '';
  71       type = lib.types.str;
  72       default = ''
  73         .:${toString ports.dns} {
  74           errors
  75           health :${toString ports.health}
  76           kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
  77             pods insecure
  78             fallthrough in-addr.arpa ip6.arpa
  79           }
  80           prometheus :${toString ports.metrics}
  81           forward . /etc/resolv.conf
  82           cache 30
  83           loop
  84           reload
  85           loadbalance
  86         }'';
  87       defaultText = lib.literalExpression ''
  88         '''
  89           .:${toString ports.dns} {
  90             errors
  91             health :${toString ports.health}
  92             kubernetes ''${config.services.kubernetes.addons.dns.clusterDomain} in-addr.arpa ip6.arpa {
  93               pods insecure
  94               fallthrough in-addr.arpa ip6.arpa
  95             }
  96             prometheus :${toString ports.metrics}
  97             forward . /etc/resolv.conf
  98             cache 30
  99             loop
 100             reload
 101             loadbalance
 102           }
 103         '''
 104       '';
 105     };
 106   };
 107
 108   config = lib.mkIf cfg.enable {
 109     services.kubernetes.kubelet.seedDockerImages =
 110       lib.singleton (pkgs.dockerTools.pullImage cfg.coredns);
 111
 112     services.kubernetes.addonManager.bootstrapAddons = {
 113       coredns-cr = {
 114         apiVersion = "rbac.authorization.k8s.io/v1";
 115         kind = "ClusterRole";
 116         metadata = {
 117           labels = {
 118             "addonmanager.kubernetes.io/mode" = "Reconcile";
 119             k8s-app = "kube-dns";
 120             "kubernetes.io/cluster-service" = "true";
 121             "kubernetes.io/bootstrapping" = "rbac-defaults";
 122           };
 123           name = "system:coredns";
 124         };
 125         rules = [
 126           {
 127             apiGroups = [ "" ];
 128             resources = [ "endpoints" "services" "pods" "namespaces" ];
 129             verbs = [ "list" "watch" ];
 130           }
 131           {
 132             apiGroups = [ "" ];
 133             resources = [ "nodes" ];
 134             verbs = [ "get" ];
 135           }
 136           {
 137             apiGroups = [ "discovery.k8s.io" ];
 138             resources = [ "endpointslices" ];
 139             verbs = [ "list" "watch" ];
 140           }
 141         ];
 142       };
 143
 144       coredns-crb = {
 145         apiVersion = "rbac.authorization.k8s.io/v1";
 146         kind = "ClusterRoleBinding";
 147         metadata = {
 148           annotations = {
 149             "rbac.authorization.kubernetes.io/autoupdate" = "true";
 150           };
 151           labels = {
 152             "addonmanager.kubernetes.io/mode" = "Reconcile";
 153             k8s-app = "kube-dns";
 154             "kubernetes.io/cluster-service" = "true";
 155             "kubernetes.io/bootstrapping" = "rbac-defaults";
 156           };
 157           name = "system:coredns";
 158         };
 159         roleRef = {
 160           apiGroup = "rbac.authorization.k8s.io";
 161           kind = "ClusterRole";
 162           name = "system:coredns";
 163         };
 164         subjects = [
 165           {
 166             kind = "ServiceAccount";
 167             name = "coredns";
 168             namespace = "kube-system";
 169           }
 170         ];
 171       };
 172     };
 173
 174     services.kubernetes.addonManager.addons = {
 175       coredns-sa = {
 176         apiVersion = "v1";
 177         kind = "ServiceAccount";
 178         metadata = {
 179           labels = {
 180             "addonmanager.kubernetes.io/mode" = "Reconcile";
 181             k8s-app = "kube-dns";
 182             "kubernetes.io/cluster-service" = "true";
 183           };
 184           name = "coredns";
 185           namespace = "kube-system";
 186         };
 187       };
 188
 189       coredns-cm = {
 190         apiVersion = "v1";
 191         kind = "ConfigMap";
 192         metadata = {
 193           labels = {
 194             "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
 195             k8s-app = "kube-dns";
 196             "kubernetes.io/cluster-service" = "true";
 197           };
 198           name = "coredns";
 199           namespace = "kube-system";
 200         };
 201         data = {
 202           Corefile = cfg.corefile;
 203         };
 204       };
 205
 206       coredns-deploy = {
 207         apiVersion = "apps/v1";
 208         kind = "Deployment";
 209         metadata = {
 210           labels = {
 211             "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
 212             k8s-app = "kube-dns";
 213             "kubernetes.io/cluster-service" = "true";
 214             "kubernetes.io/name" = "CoreDNS";
 215           };
 216           name = "coredns";
 217           namespace = "kube-system";
 218         };
 219         spec = {
 220           replicas = cfg.replicas;
 221           selector = {
 222             matchLabels = { k8s-app = "kube-dns"; };
 223           };
 224           strategy = {
 225             rollingUpdate = { maxUnavailable = 1; };
 226             type = "RollingUpdate";
 227           };
 228           template = {
 229             metadata = {
 230               labels = {
 231                 k8s-app = "kube-dns";
 232               };
 233             };
 234             spec = {
 235               containers = [
 236                 {
 237                   args = [ "-conf" "/etc/coredns/Corefile" ];
 238                   image = with cfg.coredns; "${imageName}:${finalImageTag}";
 239                   imagePullPolicy = "Never";
 240                   livenessProbe = {
 241                     failureThreshold = 5;
 242                     httpGet = {
 243                       path = "/health";
 244                       port = ports.health;
 245                       scheme = "HTTP";
 246                     };
 247                     initialDelaySeconds = 60;
 248                     successThreshold = 1;
 249                     timeoutSeconds = 5;
 250                   };
 251                   name = "coredns";
 252                   ports = [
 253                     {
 254                       containerPort = ports.dns;
 255                       name = "dns";
 256                       protocol = "UDP";
 257                     }
 258                     {
 259                       containerPort = ports.dns;
 260                       name = "dns-tcp";
 261                       protocol = "TCP";
 262                     }
 263                     {
 264                       containerPort = ports.metrics;
 265                       name = "metrics";
 266                       protocol = "TCP";
 267                     }
 268                   ];
 269                   resources = {
 270                     limits = {
 271                       memory = "170Mi";
 272                     };
 273                     requests = {
 274                       cpu = "100m";
 275                       memory = "70Mi";
 276                     };
 277                   };
 278                   securityContext = {
 279                     allowPrivilegeEscalation = false;
 280                     capabilities = {
 281                       drop = [ "all" ];
 282                     };
 283                     readOnlyRootFilesystem = true;
 284                   };
 285                   volumeMounts = [
 286                     {
 287                       mountPath = "/etc/coredns";
 288                       name = "config-volume";
 289                       readOnly = true;
 290                     }
 291                   ];
 292                 }
 293               ];
 294               dnsPolicy = "Default";
 295               nodeSelector = {
 296                 "beta.kubernetes.io/os" = "linux";
 297               };
 298               serviceAccountName = "coredns";
 299               tolerations = [
 300                 {
 301                   effect = "NoSchedule";
 302                   key = "node-role.kubernetes.io/master";
 303                 }
 304                 {
 305                   key = "CriticalAddonsOnly";
 306                   operator = "Exists";
 307                 }
 308               ];
 309               volumes = [
 310                 {
 311                   configMap = {
 312                     items = [
 313                       {
 314                         key = "Corefile";
 315                         path = "Corefile";
 316                       }
 317                     ];
 318                     name = "coredns";
 319                   };
 320                   name = "config-volume";
 321                 }
 322               ];
 323             };
 324           };
 325         };
 326       };
 327
 328       coredns-svc = {
 329         apiVersion = "v1";
 330         kind = "Service";
 331         metadata = {
 332           annotations = {
 333             "prometheus.io/port" = toString ports.metrics;
 334             "prometheus.io/scrape" = "true";
 335           };
 336           labels = {
 337             "addonmanager.kubernetes.io/mode" = "Reconcile";
 338             k8s-app = "kube-dns";
 339             "kubernetes.io/cluster-service" = "true";
 340             "kubernetes.io/name" = "CoreDNS";
 341           };
 342           name = "kube-dns";
 343           namespace = "kube-system";
 344         };
 345         spec = {
 346           clusterIP = cfg.clusterIp;
 347           ports = [
 348             {
 349               name = "dns";
 350               port = 53;
 351               targetPort = ports.dns;
 352               protocol = "UDP";
 353             }
 354             {
 355               name = "dns-tcp";
 356               port = 53;
 357               targetPort = ports.dns;
 358               protocol = "TCP";
 359             }
 360           ];
 361           selector = { k8s-app = "kube-dns"; };
 362         };
 363       };
 364     };
 365
 366     services.kubernetes.kubelet.clusterDns = lib.mkDefault [ cfg.clusterIp ];
 367   };
 368
 369   meta.buildDocsInSandbox = false;
 370 }