From 662a3cdcca01f22540a16ca9f778e59a383e5213 Mon Sep 17 00:00:00 2001 From: Sergiusz Bazanski Date: Thu, 20 Jun 2019 19:45:03 +0200 Subject: [PATCH] cluster/kube/lib/cockroachdb: refactor We refactor this library to: - support multiple databases, but with a strong suggestion of having one per k8s cluster - drop the database creation logic - redo naming (allowing for two options: multiple clusters per namespace or an exclusive namespace for the cluster) - unhardcode dns names --- cluster/kube/lib/cockroachdb.libsonnet | 741 ++++++++++++------------- 1 file changed, 368 insertions(+), 373 deletions(-) diff --git a/cluster/kube/lib/cockroachdb.libsonnet b/cluster/kube/lib/cockroachdb.libsonnet index b3aed8a8..def9bcba 100644 --- a/cluster/kube/lib/cockroachdb.libsonnet +++ b/cluster/kube/lib/cockroachdb.libsonnet @@ -1,419 +1,414 @@ # Deploy a 3-node CockroachDB cluster in secure mode. +# Can be used either in own namespace or in an existing one: +# crdb: cockroachdb.Cluster("q3kdb") { +# cfg+: { +# namespace: "q3k", // if not given, will create 'q3kdb' namespace +# }, +#}, +# +# After the cluster is up, you can get to an administrateive SQL shell: +# $ kubectl -n q3k exec -it q3kdb-client /cockroach/cockroach sql +# root@q3kdb-cockroachdb-0.q3kdb-internal.q3k.svc.cluster.local:26257/defaultdb> +# +# Then, you can create some users and databases for applications: +# defaultdb> CREATE DATABASE wykop; +# defaultdb> CREATE USER bialkov PASSWORD hackme; +# defaultdb> GRANT ALL ON DATABASE wykop to bialkov; +# +# You are then ready to access the database via the public service from your application. +# +# PGCLIENTENCODING=utf8 psql -h q3kdb-public -p 26257 -U bialkov wykop +# Password for user bialkov: +# psql (10.9 (Ubuntu 10.9-0ubuntu0.18.04.1), server 9.5.0) +# SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES128-GCM-SHA256, bits: 128, compression: off) +# Type "help" for help. +# +# wykop=> + + local kube = import "../../../kube/kube.libsonnet"; local cm = import "cert-manager.libsonnet"; { - local cockroachdb = self, - local crdb = cockroachdb, - local cfg = crdb.cfg, + Cluster(name): { + local cluster = self, - cfg:: { - namespace: error "namespace must be set", - appName: error "app name must be set", - prefix: "", # if set, should be 'foo-', - - image: "cockroachdb/cockroach:v19.1.0", - database: error "database name must be set", - username: error "username must be set", - password: error "password must be set", - }, - - makeName(suffix):: cfg.prefix + suffix, - - metadata:: { - namespace: cfg.namespace, - labels: { - "app.kubernetes.io/name": cfg.appName, - "app.kubernetes.io/managed-by": "kubecfg", - "app.kubernetes.io/component": "cockroachdb", + cfg:: { + image: "cockroachdb/cockroach:v19.1.0", + namespace: null, + ownNamespace: cluster.cfg.namespace == null, }, - }, - pki: { - selfSignedIssuer: cm.Issuer("cockroachdb-selfsigned-issuer") { - metadata+: crdb.metadata, - spec: { - selfSigned: {}, + namespaceName:: if cluster.cfg.namespace != null then cluster.cfg.namespace else name, + + metadata:: { + namespace: cluster.namespaceName, + labels: { + "app.kubernetes.io/name": "cockroachdb", + "app.kubernetes.io/managed-by": "kubecfg", + "app.kubernetes.io/component": "cockroachdb", }, }, - selfSignedKeypair: cm.Certificate("cockroachdb-cluster-ca-keypair") { - metadata+: crdb.metadata, - spec: { - secretName: "cockroachdb-cluster-ca-keypair", - duration: "43800h0m0s", // 5 years - isCA: true, - issuerRef: { - name: crdb.pki.selfSignedIssuer.metadata.name, + namespace: { + [if cluster.cfg.ownNamespace then "ns"]: kube.Namespace(cluster.namespaceName), + }, + + name(suffix):: if cluster.cfg.ownNamespace then suffix else name + "-" + suffix, + + hosts:: ["%s-%d.%s.cluster.local" % [cluster.statefulSet.metadata.name, n, cluster.internalService.host] for n in std.range(0, cluster.statefulSet.spec.replicas)], + + pki: { + selfSignedIssuer: cm.Issuer(cluster.name("selfsigned")) { + metadata+: cluster.metadata, + spec: { + selfSigned: {}, }, - commonName: "cockroachdb-cluster-ca", }, - }, - clusterIssuer: cm.Issuer("cockroachdb-cluster-ca") { - metadata+: crdb.metadata, - spec: { - ca: { - secretName: crdb.pki.selfSignedKeypair.metadata.name, + selfSignedKeypair: cm.Certificate(cluster.name("cluster-ca")) { + metadata+: cluster.metadata, + spec: { + secretName: cluster.name("cluster-ca"), + duration: "43800h0m0s", // 5 years + isCA: true, + issuerRef: { + name: cluster.pki.selfSignedIssuer.metadata.name, + }, + commonName: "cockroachdb-cluster-ca", + }, + }, + + clusterIssuer: cm.Issuer(cluster.name("cluster-ca")) { + metadata+: cluster.metadata, + spec: { + ca: { + secretName: cluster.pki.selfSignedKeypair.metadata.name, + }, + }, + }, + + nodeCertificate: cm.Certificate(cluster.name("node")) { + metadata+: cluster.metadata, + spec: { + secretName: "cockroachdb-node-cert", + duration: "43800h0m0s", // 5 years + issuerRef: { + name: cluster.pki.clusterIssuer.metadata.name, + }, + commonName: "node", + dnsNames: [ + "localhost", + "127.0.0.1", + cluster.publicService.metadata.name, + std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]), + std.join(".", [cluster.publicService.host, "cluster.local" ]), + std.join(".", [ "*", cluster.internalService.metadata.name ]), + std.join(".", [ "*", cluster.internalService.metadata.name, cluster.metadata.namespace ]), + std.join(".", [ "*", cluster.internalService.host, "cluster.local" ]), + ], + }, + }, + + clientCertificate: cm.Certificate(cluster.name("client")) { + metadata+: cluster.metadata, + spec: { + secretName: cluster.name("client-certificate"), + duration: "43800h0m0s", // 5 years + issuerRef: { + name: cluster.pki.clusterIssuer.metadata.name, + }, + commonName: "root", }, }, }, - nodeCertificate: cm.Certificate("cockroachdb-node-cert") { - metadata+: crdb.metadata, - spec: { - secretName: "cockroachdb-node-cert", - duration: "43800h0m0s", // 5 years - issuerRef: { - name: crdb.pki.clusterIssuer.metadata.name, + serviceAccount: kube.ServiceAccount(cluster.name("cockroachdb")) { + metadata+: cluster.metadata, + }, + + role: kube.Role(cluster.name("cockroachdb")) { + metadata+: cluster.metadata, + rules: [ + { + apiGroups: [ "" ], + resources: [ "secrets" ], + verbs: [ "get" ], }, - commonName: "node", - dnsNames: [ - "localhost", - "127.0.0.1", - crdb.publicService.metadata.name, - std.join(".", [crdb.publicService.metadata.name, cfg.namespace ]), - std.join(".", [crdb.publicService.host, "cluster.local" ]), - std.join(".", [ "*", crdb.internalService.metadata.name ]), - std.join(".", [ "*", crdb.internalService.metadata.name, cfg.namespace ]), - std.join(".", [ "*", crdb.internalService.host, "cluster.local" ]), + ], + }, + + roleBinding: kube.RoleBinding(cluster.name("cockroachdb")) { + metadata+: cluster.metadata, + roleRef_: cluster.role, + subjects_: [cluster.serviceAccount], + }, + + publicService: kube.Service(cluster.name("public")) { + metadata+: cluster.metadata, + target_pod:: cluster.statefulSet.spec.template, + spec+: { + ports: [ + { name: "grpc", port: 26257, targetPort: 26257 }, + { name: "http", port: 8080, targetPort: 8080 }, ], }, }, - clientCertificate: cm.Certificate("cockroachdb-client-cert") { - metadata+: crdb.metadata, + internalService: kube.Service(cluster.name("internal")) { + metadata+: cluster.metadata + { + annotations+: { + "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", + "prometheus.io/scrape": "true", + "prometheus.io/path": "_status/vars", + "prometheus.io/port": "8080", + }, + }, + target_pod:: cluster.statefulSet.spec.template, + spec+: { + ports: [ + { name: "grpc", port: 26257, targetPort: 26257 }, + { name: "http", port: 8080, targetPort: 8080 }, + ], + publishNotReadyAddresses: true, + clusterIP: "None", + }, + }, + + podDisruptionBudget: kube.PodDisruptionBudget(cluster.name("pod")) { + metadata+: cluster.metadata, spec: { - secretName: "cockroachdb-client-cert", - duration: "43800h0m0s", // 5 years - issuerRef: { - name: crdb.pki.clusterIssuer.metadata.name, + selector: { + matchLabels: { + "app.kubernetes.io/component": "cockroachdb", + }, }, - commonName: "root", + maxUnavailable: 1, }, }, - }, - serviceAccount: kube.ServiceAccount("cockroachdb") { - metadata+: crdb.metadata, - }, - - role: kube.Role("cockroachdb") { - metadata+: crdb.metadata, - rules: [ - { - apiGroups: [ "" ], - resources: [ "secrets" ], - verbs: [ "get" ], - }, - ], - }, - - roleBinding: kube.RoleBinding("cockroachdb") { - metadata+: crdb.metadata, - roleRef: { - apiGroup: "rbac.authorization.k8s.io", - kind: "Role", - name: "cockroachdb", - }, - subjects: [ - { - kind: "ServiceAccount", - name: crdb.serviceAccount.metadata.name, - namespace: cfg.namespace, - }, - ], - }, - - publicService: kube.Service(crdb.makeName("cockroachdb-public")) { - metadata+: crdb.metadata, - target_pod:: crdb.statefulSet.spec.template, - spec+: { - ports: [ - { name: "grpc", port: 26257, targetPort: 26257 }, - { name: "http", port: 8080, targetPort: 8080 }, - ], - }, - }, - - internalService: kube.Service(crdb.makeName("cockroachdb")) { - metadata+: crdb.metadata + { - annotations+: { - "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", - "prometheus.io/scrape": "true", - "prometheus.io/path": "_status/vars", - "prometheus.io/port": "8080", - }, - }, - target_pod:: crdb.statefulSet.spec.template, - spec+: { - ports: [ - { name: "grpc", port: 26257, targetPort: 26257 }, - { name: "http", port: 8080, targetPort: 8080 }, - ], - publishNotReadyAddresses: true, - clusterIP: "None", - }, - }, - - podDisruptionBudget: kube.PodDisruptionBudget(crdb.makeName("cockroachdb-budget")) { - metadata+: crdb.metadata, - spec: { - selector: { - matchLabels: { - "app.kubernetes.io/component": "cockroachdb", + statefulSet: kube.StatefulSet(cluster.name("cockroachdb")) { + metadata+: cluster.metadata { + labels+: { + "app.kubernetes.io/component": "server", }, }, - maxUnavailable: 1, - }, - }, - - statefulSet: kube.StatefulSet(crdb.makeName("cockroachdb")) { - metadata+: crdb.metadata, - spec+: { - serviceName: crdb.internalService.metadata.name, - replicas: 3, - template: { - metadata+: crdb.metadata, - spec+: { - dnsPolicy: "ClusterFirst", - serviceAccountName: crdb.serviceAccount.metadata.name, - affinity: { - podAntiAffinity: { - preferredDuringSchedulingIgnoredDuringExecution: [ - { - weight: 100, - podAffinityTerm: { - labelSelector: { - matchExpressions: [ - { - key: "app.kubernetes.io/component", - operator: "In", - values: [ "cockroachdb" ], - }, - ], + spec+: { + serviceName: cluster.internalService.metadata.name, + replicas: 3, + template: { + metadata: cluster.statefulSet.metadata, + spec+: { + dnsPolicy: "ClusterFirst", + serviceAccountName: cluster.serviceAccount.metadata.name, + affinity: { + podAntiAffinity: { + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: 100, + podAffinityTerm: { + labelSelector: { + matchExpressions: [ + { + key: "app.kubernetes.io/component", + operator: "In", + values: [ "cockroachdb" ], + }, + ], + }, + topologyKey: "kubernetes.io/hostname", }, - topologyKey: "kubernetes.io/hostname", + }, + ], + }, + }, + containers: [ + kube.Container("cockroachdb") { + image: cluster.cfg.image, + imagePullPolicy: "IfNotPresent", + resources: { + requests: { + cpu: "2", + memory: "6Gi", + }, + limits: { + memory: "6Gi", }, }, - ], - }, + ports_: { + "grpc": { containerPort: 26257 }, + "http": { containerPort: 8080 }, + }, + livenessProbe: { + httpGet: { + path: "/health", + port: "http", + }, + initialDelaySeconds: 30, + periodSeconds: 5, + }, + readinessProbe: { + httpGet: { + path: "/health?ready=1", + port: "http", + }, + initialDelaySeconds: 10, + periodSeconds: 5, + failureThreshold: 2, + }, + volumeMounts: [ + { + name: "datadir", + mountPath: "/cockroach/cockroach-data", + }, + { + name: "certs", + mountPath: "/cockroach/cockroach-certs/node.crt", + subPath: "tls.crt", + }, + { + name: "certs", + mountPath: "/cockroach/cockroach-certs/node.key", + subPath: "tls.key", + }, + { + name: "certs", + mountPath: "/cockroach/cockroach-certs/ca.crt", + subPath: "ca.crt", + }, + ], + env_: { + "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", + }, + command: [ + "/bin/bash", + "-ecx", + "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --cache 25% --max-sql-memory 25% --join " + std.join(",", cluster.hosts), + ], + }, + ], + terminationGracePeriodSeconds: 60, + volumes: [ + { + name: "datadir", + emptyDir: {}, + }, + { + name: "certs", + secret: { + secretName: cluster.pki.nodeCertificate.spec.secretName, + defaultMode: kube.parseOctal("400"), + }, + }, + ], }, - containers: [ - kube.Container("cockroachdb") { - image: cfg.image, - imagePullPolicy: "IfNotPresent", - resources: { - requests: { - cpu: "2", - memory: "6Gi", - }, - limits: { - memory: "6Gi", - }, - }, - ports_: { - "grpc": { containerPort: 26257 }, - "http": { containerPort: 8080 }, - }, - livenessProbe: { - httpGet: { - path: "/health", - port: "http", - }, - initialDelaySeconds: 30, - periodSeconds: 5, - }, - readinessProbe: { - httpGet: { - path: "/health?ready=1", - port: "http", - }, - initialDelaySeconds: 10, - periodSeconds: 5, - failureThreshold: 2, - }, - volumeMounts: [ - { - name: "datadir", - mountPath: "/cockroach/cockroach-data", - }, - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/node.crt", - subPath: "tls.crt", - }, - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/node.key", - subPath: "tls.key", - }, - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/ca.crt", - subPath: "ca.crt", - }, - ], - env_: { - "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", - }, - command: [ - "/bin/bash", - "-ecx", - "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%", - ], - }, - ], - terminationGracePeriodSeconds: 60, - volumes: [ - { - name: "datadir", - emptyDir: {}, - }, - { - name: "certs", - secret: { - secretName: crdb.pki.nodeCertificate.spec.secretName, - defaultMode: kube.parseOctal("400"), - }, - }, - ], }, - }, - podManagementPolicy: "Parallel", - updateStrategy: { - type: "RollingUpdate", - }, - }, - }, - - initJob: kube.Job(crdb.makeName("cockroachdb-init")) { - metadata+: crdb.metadata, - spec: { - template: { - metadata+: crdb.metadata, - spec+: { - serviceAccountName: crdb.serviceAccount.metadata.name, - initContainers: [ - kube.Container("cluster-init") { - image: cfg.image, - imagePullPolicy: "IfNotPresent", - env_: { - "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", - }, - command: [ - "/bin/bash", - "-ecx", - "/cockroach/cockroach init --host=cockroachdb-0.cockroachdb", - ], - volumeMounts: [ - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/ca.crt", - subPath: "ca.crt", - }, - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/client.root.crt", - subPath: "tls.crt", - }, - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/client.root.key", - subPath: "tls.key", - }, - ], - }, - ], - containers: [ - kube.Container("db-init") { - image: cfg.image, - imagePullPolicy: "IfNotPresent", - env_: { - "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", - "DB_NAME": cfg.database, - "DB_USERNAME": cfg.username, - "DB_PASSWORD": cfg.password, - }, - command: [ - "/bin/bash", - "-ec", - "/cockroach/cockroach sql -e \"CREATE DATABASE ${DB_NAME}; CREATE USER ${DB_USERNAME} PASSWORD '${DB_PASSWORD}'; GRANT ALL ON DATABASE ${DB_NAME} TO ${DB_USERNAME};\" --host=cockroachdb-0.cockroachdb", - ], - volumeMounts: [ - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/ca.crt", - subPath: "ca.crt", - }, - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/client.root.crt", - subPath: "tls.crt", - }, - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/client.root.key", - subPath: "tls.key", - }, - ], - }, - ], - restartPolicy: "OnFailure", - volumes: [ - { - name: "certs", - secret: { - secretName: crdb.pki.clientCertificate.spec.secretName, - defaultMode: kube.parseOctal("400") - } - }, - ], + podManagementPolicy: "Parallel", + updateStrategy: { + type: "RollingUpdate", }, }, }, - }, - clientPod: kube.Pod(crdb.makeName("cockroachdb-client")) { - metadata+: crdb.metadata, - spec: { - terminationGracePeriodSeconds: 5, - containers: [ - kube.Container("cockroachdb-client") { - image: cfg.image, - env_: { - "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", + initJob: kube.Job(cluster.name("init")) { + metadata+: cluster.metadata, + spec: { + template: { + metadata+: cluster.metadata, + spec+: { + serviceAccountName: cluster.serviceAccount.metadata.name, + containers: [ + kube.Container("cluster-init") { + image: cluster.cfg.image, + imagePullPolicy: "IfNotPresent", + env_: { + "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", + }, + command: [ + "/bin/bash", + "-ecx", + "/cockroach/cockroach init --host=" + cluster.hosts[0], + ], + volumeMounts: [ + { + name: "certs", + mountPath: "/cockroach/cockroach-certs/ca.crt", + subPath: "ca.crt", + }, + { + name: "certs", + mountPath: "/cockroach/cockroach-certs/client.root.crt", + subPath: "tls.crt", + }, + { + name: "certs", + mountPath: "/cockroach/cockroach-certs/client.root.key", + subPath: "tls.key", + }, + ], + }, + ], + restartPolicy: "OnFailure", + volumes: [ + { + name: "certs", + secret: { + secretName: cluster.pki.clientCertificate.spec.secretName, + defaultMode: kube.parseOctal("400") + } + }, + ], }, - command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely - volumeMounts: [ - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/ca.crt", - subPath: "ca.crt", - }, - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/client.root.crt", - subPath: "tls.crt", - }, - { - name: "certs", - mountPath: "/cockroach/cockroach-certs/client.root.key", - subPath: "tls.key", - }, - ], }, - ], - volumes: [ - { - name: "certs", - secret: { - secretName: crdb.pki.clientCertificate.spec.secretName, - defaultMode: kube.parseOctal("400") - } + }, + }, + + clientPod: kube.Pod(cluster.name("client")) { + metadata+: cluster.metadata { + labels+: { + "app.kubernetes.io/component": "client", }, - ], + }, + spec: { + terminationGracePeriodSeconds: 5, + containers: [ + kube.Container("cockroachdb-client") { + image: cluster.cfg.image, + env_: { + "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", + "COCKROACH_HOST": cluster.hosts[0], + }, + command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely + volumeMounts: [ + { + name: "certs", + mountPath: "/cockroach/cockroach-certs/ca.crt", + subPath: "ca.crt", + }, + { + name: "certs", + mountPath: "/cockroach/cockroach-certs/client.root.crt", + subPath: "tls.crt", + }, + { + name: "certs", + mountPath: "/cockroach/cockroach-certs/client.root.key", + subPath: "tls.key", + }, + ], + }, + ], + volumes: [ + { + name: "certs", + secret: { + secretName: cluster.pki.clientCertificate.spec.secretName, + defaultMode: kube.parseOctal("400") + } + }, + ], + }, }, }, }