cheshire
/
hscloud
forked from hswaw/hscloud
1
0
Fork 0
Code Pull Requests Activity

cluster/kube/lib/cockroachdb: refactor 

We refactor this library to:

 - support multiple databases, but with a strong suggestion of having
   one per k8s cluster
 - drop the database creation logic
 - redo naming (allowing for two options: multiple clusters per
   namespace or an exclusive namespace for the cluster)
 - unhardcode dns names
master
q3k 2019-06-20 19:45:03 +02:00
parent 224a50bbfe
commit 662a3cdcca
1 changed files with 368 additions and 373 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

225
cluster/kube/lib/cockroachdb.libsonnet
View File

@ -1,106 +1,135 @@
# Deploy a 3-node CockroachDB cluster in secure mode. # Deploy a 3-node CockroachDB cluster in secure mode.
# Can be used either in own namespace or in an existing one:
# crdb: cockroachdb.Cluster("q3kdb") {
# cfg+: {
# namespace: "q3k", // if not given, will create 'q3kdb' namespace
# },
#},
#
# After the cluster is up, you can get to an administrateive SQL shell:
# $ kubectl -n q3k exec -it q3kdb-client /cockroach/cockroach sql
# root@q3kdb-cockroachdb-0.q3kdb-internal.q3k.svc.cluster.local:26257/defaultdb>
#
# Then, you can create some users and databases for applications:
# defaultdb> CREATE DATABASE wykop;
# defaultdb> CREATE USER bialkov PASSWORD hackme;
# defaultdb> GRANT ALL ON DATABASE wykop to bialkov;
#
# You are then ready to access the database via the public service from your application.
#
# PGCLIENTENCODING=utf8 psql -h q3kdb-public -p 26257 -U bialkov wykop
# Password for user bialkov:
# psql (10.9 (Ubuntu 10.9-0ubuntu0.18.04.1), server 9.5.0)
# SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES128-GCM-SHA256, bits: 128, compression: off)
# Type "help" for help.
#
# wykop=>
local kube = import "../../../kube/kube.libsonnet"; local kube = import "../../../kube/kube.libsonnet";
local cm = import "cert-manager.libsonnet"; local cm = import "cert-manager.libsonnet";
{ {
local cockroachdb = self, Cluster(name): {
local crdb = cockroachdb, local cluster = self,
local cfg = crdb.cfg,
cfg:: { cfg:: {
namespace: error "namespace must be set",
appName: error "app name must be set",
prefix: "", # if set, should be 'foo-',
image: "cockroachdb/cockroach:v19.1.0", image: "cockroachdb/cockroach:v19.1.0",
database: error "database name must be set", namespace: null,
username: error "username must be set", ownNamespace: cluster.cfg.namespace == null,
password: error "password must be set",
}, },
makeName(suffix):: cfg.prefix + suffix, namespaceName:: if cluster.cfg.namespace != null then cluster.cfg.namespace else name,
metadata:: { metadata:: {
namespace: cfg.namespace, namespace: cluster.namespaceName,
labels: { labels: {
"app.kubernetes.io/name": cfg.appName, "app.kubernetes.io/name": "cockroachdb",
"app.kubernetes.io/managed-by": "kubecfg", "app.kubernetes.io/managed-by": "kubecfg",
"app.kubernetes.io/component": "cockroachdb", "app.kubernetes.io/component": "cockroachdb",
}, },
}, },
namespace: {
[if cluster.cfg.ownNamespace then "ns"]: kube.Namespace(cluster.namespaceName),
},
name(suffix):: if cluster.cfg.ownNamespace then suffix else name + "-" + suffix,
hosts:: ["%s-%d.%s.cluster.local" % [cluster.statefulSet.metadata.name, n, cluster.internalService.host] for n in std.range(0, cluster.statefulSet.spec.replicas)],
pki: { pki: {
selfSignedIssuer: cm.Issuer("cockroachdb-selfsigned-issuer") { selfSignedIssuer: cm.Issuer(cluster.name("selfsigned")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
spec: { spec: {
selfSigned: {}, selfSigned: {},
}, },
}, },
selfSignedKeypair: cm.Certificate("cockroachdb-cluster-ca-keypair") { selfSignedKeypair: cm.Certificate(cluster.name("cluster-ca")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
spec: { spec: {
secretName: "cockroachdb-cluster-ca-keypair", secretName: cluster.name("cluster-ca"),
duration: "43800h0m0s", // 5 years duration: "43800h0m0s", // 5 years
isCA: true, isCA: true,
issuerRef: { issuerRef: {
name: crdb.pki.selfSignedIssuer.metadata.name, name: cluster.pki.selfSignedIssuer.metadata.name,
}, },
commonName: "cockroachdb-cluster-ca", commonName: "cockroachdb-cluster-ca",
}, },
}, },
clusterIssuer: cm.Issuer("cockroachdb-cluster-ca") { clusterIssuer: cm.Issuer(cluster.name("cluster-ca")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
spec: { spec: {
ca: { ca: {
secretName: crdb.pki.selfSignedKeypair.metadata.name, secretName: cluster.pki.selfSignedKeypair.metadata.name,
}, },
}, },
}, },
nodeCertificate: cm.Certificate("cockroachdb-node-cert") { nodeCertificate: cm.Certificate(cluster.name("node")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
spec: { spec: {
secretName: "cockroachdb-node-cert", secretName: "cockroachdb-node-cert",
duration: "43800h0m0s", // 5 years duration: "43800h0m0s", // 5 years
issuerRef: { issuerRef: {
name: crdb.pki.clusterIssuer.metadata.name, name: cluster.pki.clusterIssuer.metadata.name,
}, },
commonName: "node", commonName: "node",
dnsNames: [ dnsNames: [
"localhost", "localhost",
"127.0.0.1", "127.0.0.1",
crdb.publicService.metadata.name, cluster.publicService.metadata.name,
std.join(".", [crdb.publicService.metadata.name, cfg.namespace ]), std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]),
std.join(".", [crdb.publicService.host, "cluster.local" ]), std.join(".", [cluster.publicService.host, "cluster.local" ]),
std.join(".", [ "*", crdb.internalService.metadata.name ]), std.join(".", [ "*", cluster.internalService.metadata.name ]),
std.join(".", [ "*", crdb.internalService.metadata.name, cfg.namespace ]), std.join(".", [ "*", cluster.internalService.metadata.name, cluster.metadata.namespace ]),
std.join(".", [ "*", crdb.internalService.host, "cluster.local" ]), std.join(".", [ "*", cluster.internalService.host, "cluster.local" ]),
], ],
}, },
}, },
clientCertificate: cm.Certificate("cockroachdb-client-cert") { clientCertificate: cm.Certificate(cluster.name("client")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
spec: { spec: {
secretName: "cockroachdb-client-cert", secretName: cluster.name("client-certificate"),
duration: "43800h0m0s", // 5 years duration: "43800h0m0s", // 5 years
issuerRef: { issuerRef: {
name: crdb.pki.clusterIssuer.metadata.name, name: cluster.pki.clusterIssuer.metadata.name,
}, },
commonName: "root", commonName: "root",
}, },
}, },
}, },
serviceAccount: kube.ServiceAccount("cockroachdb") { serviceAccount: kube.ServiceAccount(cluster.name("cockroachdb")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
}, },
role: kube.Role("cockroachdb") { role: kube.Role(cluster.name("cockroachdb")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
rules: [ rules: [
{ {
apiGroups: [ "" ], apiGroups: [ "" ],
@ -110,25 +139,15 @@ local cm = import "cert-manager.libsonnet";
], ],
}, },
roleBinding: kube.RoleBinding("cockroachdb") { roleBinding: kube.RoleBinding(cluster.name("cockroachdb")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
roleRef: { roleRef_: cluster.role,
apiGroup: "rbac.authorization.k8s.io", subjects_: [cluster.serviceAccount],
kind: "Role",
name: "cockroachdb",
},
subjects: [
{
kind: "ServiceAccount",
name: crdb.serviceAccount.metadata.name,
namespace: cfg.namespace,
},
],
}, },
publicService: kube.Service(crdb.makeName("cockroachdb-public")) { publicService: kube.Service(cluster.name("public")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
target_pod:: crdb.statefulSet.spec.template, target_pod:: cluster.statefulSet.spec.template,
spec+: { spec+: {
ports: [ ports: [
{ name: "grpc", port: 26257, targetPort: 26257 }, { name: "grpc", port: 26257, targetPort: 26257 },
@ -137,8 +156,8 @@ local cm = import "cert-manager.libsonnet";
}, },
}, },
internalService: kube.Service(crdb.makeName("cockroachdb")) { internalService: kube.Service(cluster.name("internal")) {
metadata+: crdb.metadata + { metadata+: cluster.metadata + {
annotations+: { annotations+: {
"service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
"prometheus.io/scrape": "true", "prometheus.io/scrape": "true",
@ -146,7 +165,7 @@ local cm = import "cert-manager.libsonnet";
"prometheus.io/port": "8080", "prometheus.io/port": "8080",
}, },
}, },
target_pod:: crdb.statefulSet.spec.template, target_pod:: cluster.statefulSet.spec.template,
spec+: { spec+: {
ports: [ ports: [
{ name: "grpc", port: 26257, targetPort: 26257 }, { name: "grpc", port: 26257, targetPort: 26257 },
@ -157,8 +176,8 @@ local cm = import "cert-manager.libsonnet";
}, },
}, },
podDisruptionBudget: kube.PodDisruptionBudget(crdb.makeName("cockroachdb-budget")) { podDisruptionBudget: kube.PodDisruptionBudget(cluster.name("pod")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
spec: { spec: {
selector: { selector: {
matchLabels: { matchLabels: {
@ -169,16 +188,20 @@ local cm = import "cert-manager.libsonnet";
}, },
}, },
statefulSet: kube.StatefulSet(crdb.makeName("cockroachdb")) { statefulSet: kube.StatefulSet(cluster.name("cockroachdb")) {
metadata+: crdb.metadata, metadata+: cluster.metadata {
labels+: {
"app.kubernetes.io/component": "server",
},
},
spec+: { spec+: {
serviceName: crdb.internalService.metadata.name, serviceName: cluster.internalService.metadata.name,
replicas: 3, replicas: 3,
template: { template: {
metadata+: crdb.metadata, metadata: cluster.statefulSet.metadata,
spec+: { spec+: {
dnsPolicy: "ClusterFirst", dnsPolicy: "ClusterFirst",
serviceAccountName: crdb.serviceAccount.metadata.name, serviceAccountName: cluster.serviceAccount.metadata.name,
affinity: { affinity: {
podAntiAffinity: { podAntiAffinity: {
preferredDuringSchedulingIgnoredDuringExecution: [ preferredDuringSchedulingIgnoredDuringExecution: [
@ -202,7 +225,7 @@ local cm = import "cert-manager.libsonnet";
}, },
containers: [ containers: [
kube.Container("cockroachdb") { kube.Container("cockroachdb") {
image: cfg.image, image: cluster.cfg.image,
imagePullPolicy: "IfNotPresent", imagePullPolicy: "IfNotPresent",
resources: { resources: {
requests: { requests: {
@ -261,7 +284,7 @@ local cm = import "cert-manager.libsonnet";
command: [ command: [
"/bin/bash", "/bin/bash",
"-ecx", "-ecx",
"exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%", "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --cache 25% --max-sql-memory 25% --join " + std.join(",", cluster.hosts),
], ],
}, },
], ],
@ -274,7 +297,7 @@ local cm = import "cert-manager.libsonnet";
{ {
name: "certs", name: "certs",
secret: { secret: {
secretName: crdb.pki.nodeCertificate.spec.secretName, secretName: cluster.pki.nodeCertificate.spec.secretName,
defaultMode: kube.parseOctal("400"), defaultMode: kube.parseOctal("400"),
}, },
}, },
@ -288,16 +311,16 @@ local cm = import "cert-manager.libsonnet";
}, },
}, },
initJob: kube.Job(crdb.makeName("cockroachdb-init")) { initJob: kube.Job(cluster.name("init")) {
metadata+: crdb.metadata, metadata+: cluster.metadata,
spec: { spec: {
template: { template: {
metadata+: crdb.metadata, metadata+: cluster.metadata,
spec+: { spec+: {
serviceAccountName: crdb.serviceAccount.metadata.name, serviceAccountName: cluster.serviceAccount.metadata.name,
initContainers: [ containers: [
kube.Container("cluster-init") { kube.Container("cluster-init") {
image: cfg.image, image: cluster.cfg.image,
imagePullPolicy: "IfNotPresent", imagePullPolicy: "IfNotPresent",
env_: { env_: {
"COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
@ -305,41 +328,7 @@ local cm = import "cert-manager.libsonnet";
command: [ command: [
"/bin/bash", "/bin/bash",
"-ecx", "-ecx",
"/cockroach/cockroach init --host=cockroachdb-0.cockroachdb", "/cockroach/cockroach init --host=" + cluster.hosts[0],
],
volumeMounts: [
{
name: "certs",
mountPath: "/cockroach/cockroach-certs/ca.crt",
subPath: "ca.crt",
},
{
name: "certs",
mountPath: "/cockroach/cockroach-certs/client.root.crt",
subPath: "tls.crt",
},
{
name: "certs",
mountPath: "/cockroach/cockroach-certs/client.root.key",
subPath: "tls.key",
},
],
},
],
containers: [
kube.Container("db-init") {
image: cfg.image,
imagePullPolicy: "IfNotPresent",
env_: {
"COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
"DB_NAME": cfg.database,
"DB_USERNAME": cfg.username,
"DB_PASSWORD": cfg.password,
},
command: [
"/bin/bash",
"-ec",
"/cockroach/cockroach sql -e \"CREATE DATABASE ${DB_NAME}; CREATE USER ${DB_USERNAME} PASSWORD '${DB_PASSWORD}'; GRANT ALL ON DATABASE ${DB_NAME} TO ${DB_USERNAME};\" --host=cockroachdb-0.cockroachdb",
], ],
volumeMounts: [ volumeMounts: [
{ {
@ -365,7 +354,7 @@ local cm = import "cert-manager.libsonnet";
{ {
name: "certs", name: "certs",
secret: { secret: {
secretName: crdb.pki.clientCertificate.spec.secretName, secretName: cluster.pki.clientCertificate.spec.secretName,
defaultMode: kube.parseOctal("400") defaultMode: kube.parseOctal("400")
} }
}, },
@ -375,15 +364,20 @@ local cm = import "cert-manager.libsonnet";
}, },
}, },
clientPod: kube.Pod(crdb.makeName("cockroachdb-client")) { clientPod: kube.Pod(cluster.name("client")) {
metadata+: crdb.metadata, metadata+: cluster.metadata {
labels+: {
"app.kubernetes.io/component": "client",
},
},
spec: { spec: {
terminationGracePeriodSeconds: 5, terminationGracePeriodSeconds: 5,
containers: [ containers: [
kube.Container("cockroachdb-client") { kube.Container("cockroachdb-client") {
image: cfg.image, image: cluster.cfg.image,
env_: { env_: {
"COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs", "COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
"COCKROACH_HOST": cluster.hosts[0],
}, },
command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely
volumeMounts: [ volumeMounts: [
@ -409,11 +403,12 @@ local cm = import "cert-manager.libsonnet";
{ {
name: "certs", name: "certs",
secret: { secret: {
secretName: crdb.pki.clientCertificate.spec.secretName, secretName: cluster.pki.clientCertificate.spec.secretName,
defaultMode: kube.parseOctal("400") defaultMode: kube.parseOctal("400")
} }
}, },
], ],
}, },
}, },
},
} }