# Top level cluster configuration. local kube = import "../../kube/kube.libsonnet"; local coredns = import "lib/coredns.libsonnet"; local metrics = import "lib/metrics.libsonnet"; local calico = import "lib/calico.libsonnet"; local metallb = import "lib/metallb.libsonnet"; local nginx = import "lib/nginx.libsonnet"; local rook = import "lib/rook.libsonnet"; local certmanager = import "lib/cert-manager.libsonnet"; local Cluster(fqdn) = { local cluster = self, // These are required to let the API Server contact kubelets. crAPIServerToKubelet: kube.ClusterRole("system:kube-apiserver-to-kubelet") { metadata+: { annotations+: { "rbac.authorization.kubernetes.io/autoupdate": "true", }, labels+: { "kubernets.io/bootstrapping": "rbac-defaults", }, }, rules: [ { apiGroups: [""], resources: ["nodes/%s" % r for r in [ "proxy", "stats", "log", "spec", "metrics" ]], verbs: ["*"], }, ], }, crbAPIServer: kube.ClusterRoleBinding("system:kube-apiserver") { roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: cluster.crAPIServerToKubelet.metadata.name, }, subjects: [ { apiGroup: "rbac.authorization.k8s.io", kind: "User", # A cluster API Server authenticates with a certificate whose CN is == to the FQDN of the cluster. name: fqdn, }, ], }, // Calico network fabric calico: calico.Environment {}, // CoreDNS for this cluster. dns: coredns.Environment {}, // Metrics Server metrics: metrics.Environment {}, // Metal Load Balancer metallb: metallb.Environment { cfg+: { addressPools: [ { name: "public-v4-1", protocol: "layer2", addresses: ["185.236.240.50-185.236.240.63"] }, ], }, }, // Main nginx Ingress Controller nginx: nginx.Environment {}, certmanager: certmanager.Environment {}, issuer: certmanager.ClusterIssuer("letsencrypt-prod") { spec: { acme: { server: "https://acme-v02.api.letsencrypt.org/directory", email: "bofh@hackerspace.pl", privateKeySecretRef: { name: "letsencrypt-prod" }, http01: {}, }, }, }, // Rook Ceph storage rook: rook.Operator {}, // waw1 ceph cluster cephWaw1: rook.Cluster(cluster.rook, "ceph-waw1") { spec: { mon: { count: 3, allowMultiplePerNode: false, }, storage: { useAllNodes: false, useAllDevices: false, config: { databaseSizeMB: "1024", journalSizeMB: "1024", }, nodes: [ { name: "bc01n01.hswaw.net", location: "rack=dcr01 chassis=bc01 host=bc01n01", devices: [ { name: "sda" } ], }, { name: "bc01n02.hswaw.net", location: "rack=dcr01 chassis=bc01 host=bc01n02", devices: [ { name: "sda" } ], }, { name: "bc01n03.hswaw.net", location: "rack=dcr01 chassis=bc01 host=bc01n03", devices: [ { name: "sda" } ], }, ], }, }, }, // redundant block storage cephWaw1Redundant: rook.ECBlockPool(cluster.cephWaw1, "waw-hdd-redundant-1") { spec: { failureDomain: "host", erasureCoded: { dataChunks: 2, codingChunks: 1, }, }, }, cephWaw1Object: rook.S3ObjectStore(cluster.cephWaw1, "waw-hdd-redundant-1-object") { spec: { metadataPool: { failureDomain: "host", replicated: { size: 3 }, }, dataPool: { failureDomain: "host", erasureCoded: { dataChunks: 2, codingChunks: 1, }, }, }, }, }; { k0: Cluster("k0.hswaw.net"), }