hscloud/cluster/kube/cluster.libsonnet

# Common cluster configuration.
# This defines what Kubernetes resources are required to turn a bare k8s
# deployment into a fully working cluster.
# These assume that you're running on bare metal, and using the corresponding
# NixOS deployment that we do.

local kube = import "../../kube/kube.libsonnet";
local policies = import "../../kube/policies.libsonnet";

local calico = import "lib/calico.libsonnet";
local certmanager = import "lib/cert-manager.libsonnet";
local coredns = import "lib/coredns.libsonnet";
local identd = import "lib/identd.libsonnet";
local metallb = import "lib/metallb.libsonnet";
local metrics = import "lib/metrics.libsonnet";
local nginx = import "lib/nginx.libsonnet";
local prodvider = import "lib/prodvider.libsonnet";
local rook = import "lib/rook.libsonnet";
local pki = import "lib/pki.libsonnet";

{
    Cluster(short, realm):: {
        local cluster = self,
        local cfg = cluster.cfg,

        short:: short,
        realm:: realm,
        fqdn:: "%s.%s" % [cluster.short, cluster.realm],

        cfg:: {
            // Storage class used for internal services (like registry). This must
            // be set to a valid storage class. This can either be a cloud provider class
            // (when running on GKE &co) or a storage class created using rook.
            storageClassNameRedundant: error "storageClassNameRedundant must be set",

            // CRDB cluster name (if present), passed to prodvider.
            crdbCluster: "",
        },

        // These are required to let the API Server contact kubelets.
        crAPIServerToKubelet: kube.ClusterRole("system:kube-apiserver-to-kubelet") {
            metadata+: {
                annotations+: {
                    "rbac.authorization.kubernetes.io/autoupdate": "true",
                },
                labels+: {
                    "kubernetes.io/bootstrapping": "rbac-defaults",
                },
            },
            rules: [
                {
                    apiGroups: [""],
                    resources: ["nodes/%s" % r for r in [ "proxy", "stats", "log", "spec", "metrics" ]],
                    verbs: ["*"],
                },
            ],
        },
        crbAPIServer: kube.ClusterRoleBinding("system:kube-apiserver") {
            roleRef: {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "ClusterRole",
                name: cluster.crAPIServerToKubelet.metadata.name,
            },
            subjects: [
                {
                    apiGroup: "rbac.authorization.k8s.io",
                    kind: "User",
                    # A cluster API Server authenticates with a certificate whose CN is == to the FQDN of the cluster.
                    name: cluster.fqdn,
                },
            ],
        },

        // This ClusterRole is bound to all humans that log in via prodaccess/prodvider/SSO.
        // It should allow viewing of non-sensitive data for debugability and openness.
        crViewer: kube.ClusterRole("system:viewer") {
            rules: [
                {
                    apiGroups: [""],
                    resources: [
                        "nodes",
                        "namespaces",
                        "pods",
                        "configmaps",
                        "services",
                    ],
                    verbs: ["list"],
                },
                {
                    apiGroups: ["metrics.k8s.io"],
                    resources: [
                        "nodes",
                        "pods",
                    ],
                    verbs: ["list"],
                },
                {
                    apiGroups: ["apps"],
                    resources: [
                        "statefulsets",
                    ],
                    verbs: ["list"],
                },
                {
                    apiGroups: ["extensions"],
                    resources: [
                        "deployments",
                        "ingresses",
                    ],
                    verbs: ["list"],
                },
                {
                    apiGroups: ["rbac.authorization.k8s.io"],
                    resources: [
                        "clusterroles",
                        "roles",
                        "clusterrolebindings",
                        "rolebindings",
                    ],
                    verbs: ["list", "get"],
                },
            ],
        },
        // This ClusterRole is applied (scoped to personal namespace) to all humans.
        crFullInNamespace: kube.ClusterRole("system:admin-namespace") {
            rules: [
                {
                    apiGroups: ["", "extensions", "apps"],
                    resources: ["*"],
                    verbs: ["*"],
                },
                {
                    apiGroups: ["batch"],
                    resources: ["jobs", "cronjobs"],
                    verbs: ["*"],
                },
                {
                    apiGroups: ["networking.k8s.io"],
                    resources: ["ingresses"],
                    verbs: ["*"],
                },
                {
                    apiGroups: ["cert-manager.io/v1"],
                    resources: ["certificates"],
                    verbs: ["*"],
                },
                {
                    apiGroups: ["networking.k8s.io"],
                    resources: ["networkpolicies"],
                    verbs: ["list", "get"],
                },
            ],
        },
        // This ClusterRoleBindings allows root access to cluster admins.
        crbAdmins: kube.ClusterRoleBinding("system:admins") {
            roleRef: {
                apiGroup: "rbac.authorization.k8s.io",
                kind: "ClusterRole",
                name: "cluster-admin",
            },
            subjects: [
                {
                    apiGroup: "rbac.authorization.k8s.io",
                    kind: "User",
                    name: user + "@hackerspace.pl",
                } for user in [
                    "q3k",
                    "implr",
                    "informatic",
                    "radex",
                ]
            ],
        },

        podSecurityPolicies: policies.Cluster {},

        allowInsecureNamespaces: [
            policies.AllowNamespaceInsecure("kube-system"),
            policies.AllowNamespaceInsecure("metallb-system"),
        ],

        // Allow all service accounts (thus all controllers) to create secure pods.
        crbAllowServiceAccountsSecure: kube.ClusterRoleBinding("policy:allow-all-secure") {
            roleRef_: cluster.podSecurityPolicies.secureRole,
            subjects: [
                {
                    kind: "Group",
                    apiGroup: "rbac.authorization.k8s.io",
                    name: "system:serviceaccounts",
                }
            ],
        },

        // Calico network fabric
        calico: calico.Environment {},

        // CoreDNS for this cluster.
        dns: coredns.Environment {
            cfg+: {
                cluster_domains: [
                    "cluster.local",
                    cluster.fqdn,
                ],
            },
        },

        // Metrics Server
        metrics: metrics.Environment {},

        // Metal Load Balancer
        metallb: metallb.Environment {},

        // Main nginx Ingress Controller
        nginx: nginx.Environment {},

        // Cert-manager (Let's Encrypt, CA, ...)
        certmanager: certmanager.Environment {},

        issuer: kube.ClusterIssuer("letsencrypt-prod") {
            spec: {
                acme: {
                    server: "https://acme-v02.api.letsencrypt.org/directory",
                    email: "bofh@hackerspace.pl",
                    privateKeySecretRef: {
                        name: "letsencrypt-prod"
                    },
                    solvers: [
                        { http01: { ingress: {} } },
                    ]
                },
            },
        },

        // Ident service
        identd: identd.Environment {},

        // Rook Ceph storage operator.
        rook: rook.Operator {
            operator+: {
                spec+: {
                    // Downscaled because of b.hswaw.net/6.
                    replicas: 0,
                },
            },
        },

        // TLS PKI machinery (compatibility with mirko)
        pki: pki.Environment(cluster.short, cluster.realm),

        // Prodvider
        prodvider: prodvider.Environment {
            cfg+: {
                kubeFqdn: cluster.fqdn,
                kubeApiserverAddress: "kubernetes.default.svc.%s" % [cluster.fqdn],
                prodviderCN: "prodvider.%s" % [cluster.fqdn],
                crdbCluster: cluster.cfg.crdbCluster,

                pki: {
                    intermediate: {
                        cert: importstr "../certs/ca-kube-prodvider.cert",
                        key: importstr "../secrets/plain/ca-kube-prodvider.key",
                    },
                    kube: {
                        cert: importstr "../certs/ca-kube.crt",
                    },
                }
            },
        },
    },
}