forked from hswaw/hscloud
441 lines
19 KiB
Plaintext
441 lines
19 KiB
Plaintext
# Deploy a CockroachDB cluster in secure mode.
|
|
# This creates an N-node cluster based on a given static topology.
|
|
|
|
# Can be used either in own namespace or in an existing one:
|
|
# crdb: cockroachdb.Cluster("q3kdb") {
|
|
# cfg+: {
|
|
# namespace: "q3k", // if not given, will create 'q3kdb' namespace
|
|
# topology: [
|
|
# { name: "a", node: "bc01n01.hswaw.net" },
|
|
# { name: "b", node: "bc01n02.hswaw.net" },
|
|
# { name: "c", node: "bc01n03.hswaw.net" },
|
|
# ],
|
|
# hostPath: "/var/db/cockroach-q3k",
|
|
# },
|
|
#},
|
|
#
|
|
# After the cluster is up, you can get to an administrateive SQL shell:
|
|
# $ NS=q3k kubectl -n $NS exec -it $(kubectl -n $NS get pods -o name | grep client- | cut -d/ -f 2) /cockroach/cockroach sql
|
|
# root@q3kdb-cockroachdb-0.q3kdb-internal.q3k.svc.cluster.local:26257/defaultdb>
|
|
#
|
|
# Then, you can create some users and databases for applications:
|
|
# defaultdb> CREATE DATABASE wykop;
|
|
# defaultdb> CREATE USER bialkov PASSWORD hackme;
|
|
# defaultdb> GRANT ALL ON DATABASE wykop TO bialkov;
|
|
#
|
|
# You are then ready to access the database via the public service from your application.
|
|
#
|
|
# PGCLIENTENCODING=utf8 psql -h q3kdb-public -p 26257 -U bialkov wykop
|
|
# Password for user bialkov:
|
|
# psql (10.9 (Ubuntu 10.9-0ubuntu0.18.04.1), server 9.5.0)
|
|
# SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES128-GCM-SHA256, bits: 128, compression: off)
|
|
# Type "help" for help.
|
|
#
|
|
# wykop=>
|
|
|
|
|
|
local kube = import "../../../kube/kube.libsonnet";
|
|
local policies = import "../../../kube/policies.libsonnet";
|
|
|
|
{
|
|
Cluster(name): {
|
|
local cluster = self,
|
|
|
|
cfg:: {
|
|
image: "cockroachdb/cockroach:v20.2.4",
|
|
|
|
# Must be unique per cluster.
|
|
portServe: 26257,
|
|
portHttp: 8080,
|
|
hostPath: error "hostPath must be defined",
|
|
topology: error "topology must be defined",
|
|
clients: [],
|
|
|
|
namespace: null,
|
|
ownNamespace: cluster.cfg.namespace == null,
|
|
extraDNS: [],
|
|
},
|
|
|
|
namespaceName:: if cluster.cfg.namespace != null then cluster.cfg.namespace else name,
|
|
|
|
metadata:: {
|
|
namespace: cluster.namespaceName,
|
|
labels: {
|
|
"app.kubernetes.io/name": "cockroachdb",
|
|
"app.kubernetes.io/managed-by": "kubecfg",
|
|
"app.kubernetes.io/component": "cockroachdb",
|
|
},
|
|
},
|
|
|
|
namespace: {
|
|
[if cluster.cfg.ownNamespace then "ns"]: kube.Namespace(cluster.namespaceName),
|
|
},
|
|
|
|
insecurePolicy: policies.AllowNamespaceInsecure(cluster.namespaceName),
|
|
|
|
name(suffix):: if cluster.cfg.ownNamespace then suffix else name + "-" + suffix,
|
|
|
|
pki: {
|
|
selfSignedIssuer: kube.Issuer(cluster.name("selfsigned")) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
selfSigned: {},
|
|
},
|
|
},
|
|
|
|
selfSignedKeypair: kube.Certificate(cluster.name("cluster-ca")) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
secretName: cluster.name("cluster-ca"),
|
|
duration: "43800h0m0s", // 5 years
|
|
isCA: true,
|
|
issuerRef: {
|
|
name: cluster.pki.selfSignedIssuer.metadata.name,
|
|
},
|
|
commonName: "cockroachdb-cluster-ca",
|
|
},
|
|
},
|
|
|
|
clusterIssuer: kube.Issuer(cluster.name("cluster-ca")) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
ca: {
|
|
secretName: cluster.pki.selfSignedKeypair.metadata.name,
|
|
},
|
|
},
|
|
},
|
|
|
|
nodeCertificate: kube.Certificate(cluster.name("node")) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
secretName: "cockroachdb-node-cert",
|
|
duration: "43800h0m0s", // 5 years
|
|
issuerRef: {
|
|
name: cluster.pki.clusterIssuer.metadata.name,
|
|
},
|
|
commonName: "node",
|
|
dnsNames: [
|
|
cluster.publicService.metadata.name,
|
|
std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]),
|
|
cluster.publicService.host,
|
|
std.join(".", [cluster.publicService.host, "cluster.local" ]),
|
|
std.join(".", [cluster.publicService.metadata.name, cluster.metadata.namespace ]),
|
|
] + [
|
|
"%s.cluster.local" % s.service.host
|
|
for s in cluster.servers
|
|
] + cluster.cfg.extraDNS,
|
|
},
|
|
},
|
|
|
|
clientCertificate: kube.Certificate(cluster.name("client")) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
secretName: cluster.name("client-certificate"),
|
|
duration: "43800h0m0s", // 5 years
|
|
issuerRef: {
|
|
name: cluster.pki.clusterIssuer.metadata.name,
|
|
},
|
|
commonName: "root",
|
|
},
|
|
},
|
|
},
|
|
|
|
serviceAccount: kube.ServiceAccount(cluster.name("cockroachdb")) {
|
|
metadata+: cluster.metadata,
|
|
},
|
|
|
|
role: kube.Role(cluster.name("cockroachdb")) {
|
|
metadata+: cluster.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [ "" ],
|
|
resources: [ "secrets" ],
|
|
verbs: [ "get" ],
|
|
},
|
|
],
|
|
},
|
|
|
|
roleBinding: kube.RoleBinding(cluster.name("cockroachdb")) {
|
|
metadata+: cluster.metadata,
|
|
roleRef_: cluster.role,
|
|
subjects_: [cluster.serviceAccount],
|
|
},
|
|
|
|
publicService: kube.Service(cluster.name("public")) {
|
|
metadata+: cluster.metadata,
|
|
target_pod:: cluster.servers[0].deploy.spec.template,
|
|
spec+: {
|
|
ports: [
|
|
{ name: "grpc", port: cluster.cfg.portServe, targetPort: cluster.cfg.portServe },
|
|
{ name: "http", port: cluster.cfg.portHttp, targetPort: cluster.cfg.portHttp },
|
|
],
|
|
type: "LoadBalancer",
|
|
},
|
|
},
|
|
|
|
podDisruptionBudget: kube.PodDisruptionBudget(cluster.name("pod")) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
selector: {
|
|
matchLabels: {
|
|
"app.kubernetes.io/component": "cockroachdb",
|
|
},
|
|
},
|
|
maxUnavailable: 1,
|
|
},
|
|
},
|
|
|
|
servers: [
|
|
{
|
|
local server = self,
|
|
service: kube.Service(cluster.name("server-" + el.name)) {
|
|
metadata+: cluster.metadata + {
|
|
annotations+: {
|
|
"service.alpha.kubernetes.io/tolerate-unready-endpoints": "true",
|
|
"prometheus.io/scrape": "true",
|
|
"prometheus.io/path": "_status/vars",
|
|
"prometheus.io/port": std.toString(cluster.cfg.portHttp),
|
|
},
|
|
},
|
|
target_pod:: server.deploy.spec.template,
|
|
spec+: {
|
|
ports: [
|
|
{ name: "grpc", port: cluster.cfg.portServe, targetPort: cluster.cfg.portServe },
|
|
{ name: "http", port: cluster.cfg.portHttp, targetPort: cluster.cfg.portHttp },
|
|
],
|
|
publishNotReadyAddresses: true,
|
|
clusterIP: "None",
|
|
},
|
|
},
|
|
deploy: kube.Deployment(cluster.name("server-" + el.name)) {
|
|
metadata+: cluster.metadata {
|
|
labels+: {
|
|
"app.kubernetes.io/component": "server",
|
|
"kubernetes.hackerspace.pl/cockroachdb-server": el.name,
|
|
},
|
|
},
|
|
spec+: {
|
|
template+: {
|
|
metadata: server.deploy.metadata,
|
|
spec+: {
|
|
dnsPolicy: "ClusterFirst",
|
|
serviceAccountName: cluster.serviceAccount.metadata.name,
|
|
nodeSelector: {
|
|
"kubernetes.io/hostname": el.node,
|
|
},
|
|
containers: [
|
|
kube.Container("cockroachdb") {
|
|
image: cluster.cfg.image,
|
|
imagePullPolicy: "IfNotPresent",
|
|
resources: {
|
|
requests: {
|
|
cpu: "2",
|
|
memory: "6Gi",
|
|
},
|
|
limits: {
|
|
memory: "6Gi",
|
|
},
|
|
},
|
|
ports_: {
|
|
"grpc": { containerPort: cluster.cfg.portServe },
|
|
"http": { containerPort: cluster.cfg.portHttp },
|
|
},
|
|
livenessProbe: {
|
|
httpGet: {
|
|
path: "/health",
|
|
port: "http",
|
|
},
|
|
initialDelaySeconds: 30,
|
|
periodSeconds: 5,
|
|
},
|
|
readinessProbe: {
|
|
httpGet: {
|
|
path: "/health?ready=1",
|
|
port: "http",
|
|
},
|
|
initialDelaySeconds: 10,
|
|
periodSeconds: 5,
|
|
failureThreshold: 2,
|
|
},
|
|
volumeMounts: [
|
|
{
|
|
name: "datadir",
|
|
mountPath: "/cockroach/cockroach-data",
|
|
},
|
|
{
|
|
name: "certs",
|
|
mountPath: "/cockroach/cockroach-certs/node.crt",
|
|
subPath: "tls.crt",
|
|
},
|
|
{
|
|
name: "certs",
|
|
mountPath: "/cockroach/cockroach-certs/node.key",
|
|
subPath: "tls.key",
|
|
},
|
|
{
|
|
name: "certs",
|
|
mountPath: "/cockroach/cockroach-certs/ca.crt",
|
|
subPath: "ca.crt",
|
|
},
|
|
],
|
|
env_: {
|
|
"COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
|
|
},
|
|
command: [
|
|
"/cockroach/cockroach", "start",
|
|
"--logtostderr",
|
|
"--certs-dir", "/cockroach/cockroach-certs",
|
|
"--advertise-host", "%s.cluster.local" % server.service.host,
|
|
"--cache", "25%", "--max-sql-memory", "25%",
|
|
"--join", std.join(",", ["%s.cluster.local:%d" % [s.service.host, cluster.cfg.portServe] for s in cluster.servers]),
|
|
"--listen-addr=0.0.0.0:%d" % cluster.cfg.portServe,
|
|
"--http-addr=0.0.0.0:%d" % cluster.cfg.portHttp,
|
|
],
|
|
},
|
|
],
|
|
terminationGracePeriodSeconds: 60,
|
|
volumes: [
|
|
{
|
|
name: "datadir",
|
|
hostPath: {
|
|
path: cluster.cfg.hostPath,
|
|
},
|
|
},
|
|
{
|
|
name: "certs",
|
|
secret: {
|
|
secretName: cluster.pki.nodeCertificate.spec.secretName,
|
|
defaultMode: kube.parseOctal("400"),
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
for el in cluster.cfg.topology
|
|
],
|
|
|
|
initJob: kube.Job(cluster.name("init")) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
template: {
|
|
metadata+: cluster.metadata,
|
|
spec+: {
|
|
serviceAccountName: cluster.serviceAccount.metadata.name,
|
|
containers: [
|
|
kube.Container("cluster-init") {
|
|
image: cluster.cfg.image,
|
|
imagePullPolicy: "IfNotPresent",
|
|
env_: {
|
|
"COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
|
|
},
|
|
command: [
|
|
"/bin/bash",
|
|
"-ecx",
|
|
"/cockroach/cockroach init --host=%s.cluster.local:%d || true" % [cluster.servers[0].service.host, cluster.cfg.portServe],
|
|
],
|
|
volumeMounts: [
|
|
{
|
|
name: "certs",
|
|
mountPath: "/cockroach/cockroach-certs/ca.crt",
|
|
subPath: "ca.crt",
|
|
},
|
|
{
|
|
name: "certs",
|
|
mountPath: "/cockroach/cockroach-certs/client.root.crt",
|
|
subPath: "tls.crt",
|
|
},
|
|
{
|
|
name: "certs",
|
|
mountPath: "/cockroach/cockroach-certs/client.root.key",
|
|
subPath: "tls.key",
|
|
},
|
|
],
|
|
},
|
|
],
|
|
restartPolicy: "OnFailure",
|
|
volumes: [
|
|
{
|
|
name: "certs",
|
|
secret: {
|
|
secretName: cluster.pki.clientCertificate.spec.secretName,
|
|
defaultMode: kube.parseOctal("400")
|
|
}
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
Client(name):: {
|
|
certificate: kube.Certificate(cluster.name("client-%s" % name)) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
secretName: cluster.name("client-%s-certificate" % name),
|
|
duration: "43800h0m0s", // 5 years
|
|
issuerRef: {
|
|
name: cluster.pki.clusterIssuer.metadata.name,
|
|
},
|
|
commonName: name,
|
|
},
|
|
},
|
|
},
|
|
|
|
client: kube.Deployment(cluster.name("client")) {
|
|
metadata+: cluster.metadata {
|
|
labels+: {
|
|
"app.kubernetes.io/component": "client",
|
|
},
|
|
},
|
|
spec+: {
|
|
template: {
|
|
metadata: cluster.client.metadata,
|
|
spec+: {
|
|
terminationGracePeriodSeconds: 5,
|
|
containers: [
|
|
kube.Container("cockroachdb-client") {
|
|
image: cluster.cfg.image,
|
|
env_: {
|
|
"COCKROACH_CERTS_DIR": "/cockroach/cockroach-certs",
|
|
"COCKROACH_HOST": cluster.publicService.host,
|
|
"COCKROACH_PORT": std.toString(cluster.cfg.portServe),
|
|
},
|
|
command: ["sleep", "2147483648"], //(FIXME) keep the client pod running indefinitely
|
|
volumeMounts: [
|
|
{
|
|
name: "certs",
|
|
mountPath: "/cockroach/cockroach-certs/ca.crt",
|
|
subPath: "ca.crt",
|
|
},
|
|
{
|
|
name: "certs",
|
|
mountPath: "/cockroach/cockroach-certs/client.root.crt",
|
|
subPath: "tls.crt",
|
|
},
|
|
{
|
|
name: "certs",
|
|
mountPath: "/cockroach/cockroach-certs/client.root.key",
|
|
subPath: "tls.key",
|
|
},
|
|
],
|
|
},
|
|
],
|
|
volumes: [
|
|
{
|
|
name: "certs",
|
|
secret: {
|
|
secretName: cluster.pki.clientCertificate.spec.secretName,
|
|
defaultMode: kube.parseOctal("400")
|
|
}
|
|
},
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|