cluster: deploy coredns

2019-01-14 00:02:59 +01:00 · 2019-01-14 00:02:59 +01:00 · 49b9a13d28
commit 49b9a13d28
parent 5bebbebe3e
5 changed files with 253 additions and 52 deletions
--- a/cluster/certs/ca.srl
+++ b/cluster/certs/ca.srl
@ -1 +1 @@
-80F13FCE5DBBF730
+80F13FCE5DBBF736
--- a/cluster/certs/kube-apiserver.crt
+++ b/cluster/certs/kube-apiserver.crt
@ -1,10 +1,10 @@
 -----BEGIN CERTIFICATE-----
-MIIF0DCCA7igAwIBAgIJAIDxP85du/cnMA0GCSqGSIb3DQEBCwUAMIG3MQswCQYD
+MIIGCzCCA/OgAwIBAgIJAIDxP85du/c2MA0GCSqGSIb3DQEBCwUAMIG3MQswCQYD
 VQQGEwJQTDEUMBIGA1UECAwLTWF6b3dpZWNraWUxETAPBgNVBAcMCFdhcnN6YXdh
 MS4wLAYDVQQKDCVTdG93YXJ6eXN6ZW5pZSBXYXJzemF3c2tpIEhhY2tlcnNwYWNl
 MRAwDgYDVQQLDAdoc2Nsb3VkMRowGAYDVQQDDBFCb290c3RyYXAgTm9kZSBDQTEh
-MB8GCSqGSIb3DQEJARYScTNrQGhhY2tlcnNwYWNlLnBsMB4XDTE5MDExMzE5NDU0
-NVoXDTIwMDExMzE5NDU0NVowgZYxCzAJBgNVBAYTAlBMMRQwEgYDVQQIDAtNYXpv
+MB8GCSqGSIb3DQEJARYScTNrQGhhY2tlcnNwYWNlLnBsMB4XDTE5MDExMzIyMzgy
+NVoXDTIwMDExMzIyMzgyNVowgZYxCzAJBgNVBAYTAlBMMRQwEgYDVQQIDAtNYXpv
 d2llY2tpZTERMA8GA1UEBwwIV2Fyc3phd2ExLjAsBgNVBAoMJVN0b3dhcnp5c3pl
 bmllIFdhcnN6YXdza2kgSGFja2Vyc3BhY2UxFzAVBgNVBAsMDkt1YmVybmV0ZXMg
 QVBJMRUwEwYDVQQDDAxrMC5oc3dhdy5uZXQwggIiMA0GCSqGSIb3DQEBAQUAA4IC
@ -19,16 +19,17 @@ OLzA2QfiWV9maF/Ue/GUcurSwnsPNtsDy0sqffYjNpsdxHB25OH7abqDmbayNx/x
 wWs87E5VPZ7LuM5QJNg6ZBLJ7B81rvw3BYTar0H2YfLGeTjhktJ9fJVjx7gvAagB
 RnipgSOLN4fiB68wTe8lyLLH+7+ZtfZl8myRzkoDvHc0iBeZa0Pr2iGCLfR5Fkqo
 hU7nVRremTfIodygtTMdSozpOWRMaLJV1WJfMiB91rs+mwMBhncqa3Hp6QIDAQAB
-MA0GCSqGSIb3DQEBCwUAA4ICAQBMkv4dG3gybWdggc5aCZqyanp+CU506ejVpAd2
-oPgJnvcAR1DVnHer2hMFlRk4lt1rSPsRv1bqOQLgBkOEbUJhknaSD6CknmfriX1/
-ZdBwB9JHy7E/S4QDrm/8s6HiWcKYW6eK35aP4bF8ebDp+PBmYOrHRl85vNqtjeMJ
-iyXznQFL1kiuT2hBcMiQeVbEz4o0u/yAlNIxL3PXKXn0AVyW0LjLI+EAd8lCfGKy
-SkJf2gw/UWx3s+rEctA6qrB29PBR03PTHvXfb53ILh8KuIh3hU3+EED7puNhNvrS
-qWthIe5hAVOEaE9GfHCqdelQELrrYhAVMuO+PqtsGwZruEY6dpI493Aq+lfd+2TT
-pRG/isoGvGh+Lg+pwV3DLuGnnMH47iUHnPPXbYRBvSpnhC80vx8Bnbn4l7TpuZYo
-KLo5heP+Mb4sueG7KjuoOHRXcI3vHgKD2XjXFokdLBAy+75Ik0YNUWbvK76PiajE
-znic15ws1lTJiY16z+JPdjpLh+ddXf2DDnFhkWNy/Fxt+dIm0Bhdgin0rPKpE8Sv
-BIKiagL0VpDlVW5DUQe9ZVNW3zvyb3fvis+4+SmPcHDEq6ULgQQD6NjuPTKa7Suo
-pm7SxeMmP1c0B28S1wqZAh0mxrD/yUhKA/ZagRktAhCr+CXAgePEqeCZCx0XGqUw
-d/ZySw==
+ozkwNzAdBgNVHREEFjAUhwQKCgwBggxrMC5oc3dhdy5uZXQwCQYDVR0TBAIwADAL
+BgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQELBQADggIBAAOx8hojlmZWH7cfLaqiSrHJ
+nzcUARJYeQzTqHYggOR0TBkI/kSOkgXeDky8mya/9zcn+/GdDvxfgivZ6J7ZLAtF
+JU6BXnBaVqG/N2oBM6qzXedU4SBDCtS+/3vhAdrk8e1gml5kAYEaw0IDeHqMtW63
+k8QtWeg3+I5/u28X46+xEhkWhk6ofXoDOs3ytSU0lUqi4FFI4JCeCM/fgHWDJUNh
+Z+5O3ovQmn0d2N/O7sK4toBPAPv7TcvaaBvM4OYnLPoZ4PSfbgK0ZFfhEaJ0jVNH
+5AApv/0eT1utOxuZX9xdCkrVrGHPmS+HJM/Q0n19Iqzxa6ZBCrqBlUjuo51DUisd
+knwTczSke7d2AEjgbKk00eGd10aSjXk/wFQwTxK6jbdmKztRvP88zq8FilRdYcqN
+4TX48bjFo6NjSmyo0kXgZJeIcHM4pKrNyOAz6WRt39da+KUZKXy1oy/3PXUhiOaX
+5rGt2H8FVxTg4XBsRBvwvKFxyse7mKFYyWuQa97ugfg66PxNLdmOn5WJgy/CcHYy
+89np4pgkNjQ6K/OE0OgMN5WbrVU231b+hzNkYl6XT3iB7vZOChLkl5afdiFXzFg1
+kbnQTWu3BwZhCg2mNGs9pGqJ0FvL1KHfgsLt3VBWjepFkOOWYvrmjqwxCxZaRIrn
+g0RSHtqS2xS9EAjqVG0/
 -----END CERTIFICATE-----
--- a/cluster/kube/cluster.jsonnet
+++ b/cluster/kube/cluster.jsonnet
@ -1,6 +1,7 @@
 # Top level cluster configuration.

 local kube = import "../../kube/kube.libsonnet";
+local coredns = import "lib/coredns.libsonnet";

 local Cluster(fqdn) = {
    local cluster = self,
@ -37,9 +38,14 @@ local Cluster(fqdn) = {
                name: fqdn,
            },
        ],
-    }
+    },
+
+    // CoreDNS for this cluster.
+    dns: coredns.Environment {
+    },
 };

+
 {
    k0: Cluster("k0.hswaw.net"),
 }
--- a/cluster/kube/lib/coredns.libsonnet
+++ b/cluster/kube/lib/coredns.libsonnet
@ -0,0 +1,196 @@
+# Deploy a per-cluster CoreDNS
+
+local kube = import "../../../kube/kube.libsonnet";
+
+{
+    Environment: {
+        local env = self,
+        local cfg = env.cfg,
+        cfg:: {
+            image: "coredns/coredns:1.3.0",
+            namespace: "kube-system",
+            upstream_server: "185.236.240.1",
+            cluster_domain: "cluster.local",
+            reverse_cidrs: ["in-addr.arpa", "ip6.arpa"],
+            clusterIP: "10.10.12.254",
+        },
+
+        sa: kube.ServiceAccount("coredns") {
+            metadata+: {
+                namespace: cfg.namespace,
+            },
+        },
+
+        cr: kube.ClusterRole("system:coredns") {
+            metadata+: {
+                labels: {
+                    "kubernetes.io/bootstrapping": "rbac-defaults",
+                },
+            },
+            rules: [
+                {
+                    apiGroups: [""],
+                    resources: ["endpoints", "services", "pods", "namespaces"],
+                    verbs: ["list", "watch"],
+                },
+                {
+                    apiGroups: [""],
+                    resources: ["nodes"],
+                    verbs: ["get"],
+                },
+            ],
+        },
+
+        crb: kube.ClusterRoleBinding("system:coredns") {
+            metadata+: {
+                labels: {
+                    "kubernetes.io/bootstrapping": "rbac-defaults",
+                },
+                annotations+: {
+                    "rbac.authorization.kubernetes.io/autoupdate": "true",
+                },
+            },
+            roleRef: {
+                apiGroup: "rbac.authorization.k8s.io",
+                kind: "ClusterRole",
+                name: env.cr.metadata.name,
+            },
+            subjects: [
+                {
+                    kind: "ServiceAccount",
+                    name: env.sa.metadata.name,
+                    namespace: env.sa.metadata.namespace,
+                },
+            ],
+        },
+
+        cm: kube.ConfigMap("coredns") {
+            local map = self,
+
+            upstream_server:: cfg.upstream_server,
+            cluster_domain:: cfg.cluster_domain,
+            reverse_cidrs:: std.join(" ", cfg.reverse_cidrs),
+
+            metadata+: {
+                namespace: cfg.namespace,
+            },
+            data: {
+                Corefile: |||
+                     .:53 {
+                        log
+                        errors
+                        health
+                        kubernetes %s %s {
+                            pods insecure
+                            upstream
+                            fallthrough in-addr.arpa ip6.arpa
+                        }
+                        prometheus :9153
+                        proxy . %s
+                        cache 30
+                        loop
+                        reload
+                        loadbalance
+                     }
+                ||| % [map.cluster_domain, map.reverse_cidrs, map.upstream_server]
+            },
+        },
+
+        deployment: kube.Deployment("coredns") {
+            metadata+: {
+                namespace: cfg.namespace,
+                labels+: {
+                    "k8s-app": "coredns",
+                },
+            },
+            spec+: {
+                replicas: 2,
+                strategy: {
+                    type: "RollingUpdate",
+                    rollingUpdate: { maxUnavailable: 1 },
+                },
+                template+: {
+                    spec+: {
+                        serviceAccountName: env.sa.metadata.name,
+                        tolerations: [
+                            { key: "CriticalAddonsOnly", operator: "Exists" },
+                        ],
+                        dnsPolicy: "Default",
+                        volumes_: {
+                            config: {
+                                configMap: {
+                                    name: env.cm.metadata.name,
+                                    items: [ { key: "Corefile", path: "Corefile" } ],
+                                },
+                            },
+                        },
+                        containers_: {
+                            coredns: kube.Container("coredns") {
+                                local container = self,
+
+                                image: cfg.image,
+                                args: [
+                                    "-conf", "%s/Corefile" % container.volumeMounts[0].mountPath,
+                                ],
+                                imagePullPolicy: "IfNotPresent",
+                                resources: {
+                                    limits: { memory: "170Mi" },
+                                    requests: { memory: "70Mi", cpu: "100m" },
+                                },
+                                volumeMounts_: {
+                                    config: {
+                                        mountPath: "/etc/coredns",
+                                    },
+                                },
+                                ports_: {
+                                    dns: {
+                                        containerPort: 53,
+                                        protocol: "UDP",
+                                    },
+                                    "dns-tcp": {
+                                        containerPort: 53,
+                                        protocol: "TCP",
+                                    },
+                                    metrics: {
+                                        containerPort: 9153,
+                                        protocol: "TCP",
+                                    },
+                                },
+                                securityContext: {
+                                    allowPrivilegeEscalation: false,
+                                    capabilities: {
+                                        add: ["NET_BIND_SERVICE"],
+                                        drop: ["all"],
+                                    },
+                                    readOnlyRootFilesystem: true,
+                                },
+                                livenessProbe: {
+                                    httpGet: {
+                                        path: "/health",
+                                        port: 8080,
+                                        scheme: "HTTP",
+                                    },
+                                    initialDelaySeconds: 60,
+                                    timeoutSeconds: 5,
+                                    successThreshold: 1,
+                                    failureThreshold: 5,
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+        svc: kube.Service("coredns") {
+            local svc = self,
+            metadata+: {
+                namespace: cfg.namespace,
+            },
+            target_pod: env.deployment.spec.template,
+            spec+: {
+                ports: [ { name: p.name, port: p.containerPort, protocol: p.protocol } for p in svc.target_pod.spec.containers[0].ports ],
+                clusterIP: cfg.clusterIP,
+            },
+        },
+    },
+}
--- a/tools/clustercfg.py
+++ b/tools/clustercfg.py
@ -96,8 +96,20 @@ def openssl_config(san):
    if san:
        config.seek(0, 2)
        config.write(b'\n[SAN]\n')
+        config.write(b'subjectAltName = @alt_names\n')
+        config.write(b'basicConstraints = CA:FALSE\nkeyUsage = nonRepudiation, digitalSignature, keyEncipherment\n')
+        config.write(b'[alt_names]\n')
+
+        ipcnt = 1
+        dnscnt = 1
        for s in san:
-            config.write('subjectAltName=DNS:{}\n'.format(s).encode())
+            parts = s.split(':')
+            if s.startswith('DNS'):
+                config.write('DNS.{} = {}\n'.format(dnscnt, parts[1]).encode())
+                dnscnt += 1
+            elif s.startswith('IP'):
+                config.write('IP.{} = {}\n'.format(ipcnt, parts[1]).encode())
+                ipcnt += 1

    f = tempfile.NamedTemporaryFile(delete=False)
    path = f.name
@ -192,32 +204,30 @@ def shared_cert(pki, c, fqdn, cert_name, subj, san=[], days=365):
    else:
        generate_cert = True

-    if not generate_cert:
-        return False
+    if generate_cert:
+        local_csr_f = tempfile.NamedTemporaryFile(delete=False)
+        local_csr = local_csr_f.name
+        local_csr_f.close()

-    local_csr_f = tempfile.NamedTemporaryFile(delete=False)
-    local_csr = local_csr_f.name
-    local_csr_f.close()
+        local_config = openssl_config(san)

-    local_config = openssl_config(san)
+        subprocess.check_call([
+            'openssl', 'req', '-new',
+            '-key', local_key,
+            '-out', local_csr,
+            '-subj', str(subj),
+            '-config', local_config,
+        ] + ([
+            '-reqexts', 'SAN',
+        ] if san else []))

-    subprocess.check_call([
-        'openssl', 'req', '-new',
-        '-key', local_key,
-        '-out', local_csr,
-        '-subj', str(subj),
-        '-config', local_config,
-    ] + ([
-        '-reqexts', 'SAN',
-    ] if san else []))
-
-    pki.sign(local_csr, local_cert, local_config, days)
+        pki.sign(local_csr, local_cert, local_config, days)
+        os.remove(local_csr)
+        os.remove(local_config)

    c.put(local=local_key, remote=remote_key)
    c.put(local=local_cert, remote=remote_cert)

-    os.remove(local_csr)
-    os.remove(local_config)
    return True


@ -312,27 +322,15 @@ def nodestrap(args):

    modified = False
    modified |= remote_cert(p, c, fqdn, "node", Subject(Subject.hswaw, 'Node Certificate', fqdn))
-    modified |= remote_cert(p, c, fqdn, "kube-node", Subject('system:nodes', 'Kubelet Certificate', 'system:node:' + fqdn), san=[fqdn,])
+    modified |= remote_cert(p, c, fqdn, "kube-node", Subject('system:nodes', 'Kubelet Certificate', 'system:node:' + fqdn), san=["DNS:"+fqdn,])
    for component in ['controller-manager', 'proxy', 'scheduler']:
        o = 'system:kube-{}'.format(component)
        ou = 'Kuberneter Component {}'.format(component)
        modified |= shared_cert(p, c, fqdn, 'kube-{}'.format(component), Subject(o, ou, o))
-    modified |= shared_cert(p, c, fqdn, 'kube-apiserver', Subject(Subject.hswaw, 'Kubernetes API', cluster))
+    modified |= shared_cert(p, c, fqdn, 'kube-apiserver', Subject(Subject.hswaw, 'Kubernetes API', cluster), san=['IP:10.10.12.1', 'DNS:' + cluster])
    modified |= shared_cert(p, c, fqdn, 'kube-serviceaccounts', Subject(Subject.hswaw, 'Kubernetes Service Account Signer', 'service-accounts'))

-    if modified:
-        logger.info('{}: cert(s) modified, restarting services...'.format(fqdn))
-
-        services = [
-            'kubelet', 'kube-proxy',
-            'kube-apiserver', 'kube-controller-manager', 'kube-scheduler',
-            'etcd'
-        ]
-
-        for s in services:
-            c.run('systemctl stop {}'.format(s))
-        for s in services[::-1]:
-            c.run('systemctl start {}'.format(s))
+    c.run('nixos-rebuild switch')

 def usage():
    sys.stderr.write("Usage: {} <nodestrap|admincreds>\n".format(sys.argv[0]))