From a7e26ccfe10b430a6cf8b58c5b3537bfe1e8b57d Mon Sep 17 00:00:00 2001 From: Sergiusz Bazanski Date: Fri, 21 Jun 2019 20:38:35 +0200 Subject: [PATCH] app/gerrit/kube: implement This change impelements the k8s machinery for Gerrit. This might look somewhat complex at first, but the gist of it is: - k8s mounts etc, git, cache, db, index as RW PVs - k8s mounts a configmap containing gerrit.conf into an external directory - k8s mounts a secret containing secure.conf into an external directory - on startup, gerrit's entrypoint will copy over {gerrit,secure}.conf and start a small updater script that copies over gerrit.conf if there's any change. This should, in theory, make gerrit reload its config. This is already running on production. You're probably looking at this change through the instance deployed by itself :) Change-Id: Ida9dff721c17cf4da7fb6ccbb54d2c4024672572 --- .bazelrc | 3 + WORKSPACE | 9 ++ app/gerrit/BUILD | 33 +++++ app/gerrit/entrypoint.sh | 43 +++++++ app/gerrit/kube/gerrit.libsonnet | 209 +++++++++++++++++++++++++++++++ app/gerrit/kube/prod.jsonnet | 19 +++ 6 files changed, 316 insertions(+) create mode 100644 .bazelrc create mode 100644 app/gerrit/BUILD create mode 100755 app/gerrit/entrypoint.sh create mode 100644 app/gerrit/kube/gerrit.libsonnet create mode 100644 app/gerrit/kube/prod.jsonnet diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 00000000..17fa9d43 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,3 @@ +# Required for app/gerrit/gerrit-oauth-provider +build --workspace_status_command=./tools/workspace-status.sh +test --build_tests_only diff --git a/WORKSPACE b/WORKSPACE index d2d7cf6d..96648b6a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -60,6 +60,7 @@ container_repositories() # Docker base images load("@io_bazel_rules_docker//container:container.bzl", "container_pull") + container_pull( name = "prodimage-bionic", registry = "index.docker.io", @@ -68,6 +69,14 @@ container_pull( digest = "sha256:b36667c98cf8f68d4b7f1fb8e01f742c2ed26b5f0c965a788e98dfe589a4b3e4", ) +container_pull( + name = "gerrit-3.0.0", + registry = "index.docker.io", + repository = "gerritcodereview/gerrit", + tag = "3.0.0-ubuntu18", + digest = "sha256:f107729011d8b81611e35a0ad452f21a424c1820664e9f95d135ad411e87b9bb", +) + # HTTP stuff from the Internet load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") http_file( diff --git a/app/gerrit/BUILD b/app/gerrit/BUILD new file mode 100644 index 00000000..aaff8a05 --- /dev/null +++ b/app/gerrit/BUILD @@ -0,0 +1,33 @@ +load("@io_bazel_rules_docker//container:container.bzl", "container_image") + +container_image( + name="with_plugins", + base="@gerrit-3.0.0//image", + files = [ + "//app/gerrit/gerrit-oauth-provider:gerrit-oauth-provider", + ], + # we cannot drop it directly in /var/gerrit/plugins as that changes the + # directory owner to 0:0 and then breaks the gerrit installer that wants + # to overwrite plugins. + directory = "/var/gerrit-plugins", +) +container_image( + name="3.0.0-r7", + base=":with_plugins", + files = [":entrypoint.sh"], + directory = "/", + entrypoint = ["/entrypoint.sh"], +) + +genrule( + name = "push_latest", + srcs = [":3.0.0-r7"], + outs = ["version.sh"], + executable = True, + cmd = """ + tag=3.0.0-r7 + docker tag bazel/app/gerrit:$$tag registry.k0.hswaw.net/app/gerrit:$$tag + docker push registry.k0.hswaw.net/app/gerrit:$$tag + echo -ne "#!/bin/sh\necho Pushed $$tag.\n" > $(OUTS) + """, +) diff --git a/app/gerrit/entrypoint.sh b/app/gerrit/entrypoint.sh new file mode 100755 index 00000000..ffea5f30 --- /dev/null +++ b/app/gerrit/entrypoint.sh @@ -0,0 +1,43 @@ +#!/bin/bash -e + +ls -la /var/gerrit/* + +if [ ! -d /var/gerrit/git/All-Projects.git ] || [ "$1" == "init" ] +then + echo "Initializing Gerrit site ..." + java -jar /var/gerrit/bin/gerrit.war init --batch --install-all-plugins -d /var/gerrit + java -jar /var/gerrit/bin/gerrit.war reindex -d /var/gerrit +fi + +echo "Running hscloud init setup..." + +rm -f /var/gerrit/etc/gerrit.config +cp /var/gerrit-config/gerrit.config /var/gerrit/etc/gerrit.config + +rm -f /var/gerrit/etc/secure.config +cp /var/gerrit-secure/secure.config /var/gerrit/etc/secure.config + +cp /var/gerrit-plugins/* /var/gerrit/plugins/ + +echo "Starting config updater..." +# Keep copying config over in background. We cannot run directly from +# the configmap filesystem as gerrit really wants a read-write FS. +( + src=/var/gerrit-config/gerrit.config + dst=/var/gerrit/etc/gerrit.config + while true; do + sleep 60 + if ! cmp -s $src $dst; then + echo "HSCLOUD: bumping config" + cp $src $dst + fi + done +) & + +ls -la /var/gerrit/* + +if [ "$1" != "init" ] +then + echo "Running Gerrit ..." + exec /var/gerrit/bin/gerrit.sh run +fi diff --git a/app/gerrit/kube/gerrit.libsonnet b/app/gerrit/kube/gerrit.libsonnet new file mode 100644 index 00000000..2cd6f594 --- /dev/null +++ b/app/gerrit/kube/gerrit.libsonnet @@ -0,0 +1,209 @@ +local kube = import "../../../kube/kube.libsonnet"; + +{ + local gerrit = self, + local cfg = gerrit.cfg, + + cfg:: { + namespace: error "namespace must be set", + appName: "gerrit", + prefix: "", # if set, should be 'foo-' + domain: error "domain must be set", + identity: error "identity (UUID) must be set", + + // The secret must contain a key named 'secure.config' containing (at least): + // [auth] + // registerEmailPrivateKey = + // [plugin "gerrit-oauth-provider-warsawhackerspace-oauth"] + // client-id = foo + // client-secret = bar + // [sendemail] + // smtpPass = foo + // [receiveemail] + // password = bar + secureSecret: error "secure secret name must be set", + + storageClass: error "storage class must be set", + storageSize: { + git: "50Gi", // Main storage for repositories and NoteDB. + index: "10Gi", // Secondary Lucene index + cache: "10Gi", // H2 cache databases + db: "1Gi", // NoteDB is used, so database is basically empty (H2 accountPatchReviewDatabase) + etc: "1Gi", // Random site stuff. + }, + + email: { + server: "mail.hackerspace.pl", + username: "gerrit", + address: "gerrit@hackerspace.pl", + }, + + tag: "3.0.0-r7", + image: "registry.k0.hswaw.net/app/gerrit:" + cfg.tag, + resources: { + requests: { + cpu: "100m", + memory: "500Mi", + }, + limits: { + cpu: "1", + memory: "2Gi", + }, + }, + }, + + name(suffix):: cfg.prefix + suffix, + + metadata(component):: { + namespace: cfg.namespace, + labels: { + "app.kubernetes.io/name": cfg.appName, + "app.kubernetes.io/managed-by": "kubecfg", + "app.kubernetes.io/component": "component", + }, + }, + + configmap: kube.ConfigMap(gerrit.name("gerrit")) { + metadata+: gerrit.metadata("configmap"), + data: { + "gerrit.config": ||| + [gerrit] + basePath = git + canonicalWebUrl = https://%(domain)s/ + serverId = %(identity)s + + [container] + javaOptions = -Djava.security.edg=file:/dev/./urandom + + [auth] + type = OAUTH + gitBasicAuthPolicy = HTTP + + [httpd] + listenUrl = proxy-http://*:8080 + + [sshd] + advertisedAddress = %(domain)s + + [user] + email = %(emailAddress)s + + [sendemail] + enable = true + from = MIXED + smtpServer = %(emailServer)s + smtpServerPort = 465 + smtpEncryption = ssl + smtpUser = %(emailUser)s + + [receiveemail] + protocol = IMAP + host = %(emailServer)s + username = %(emailUser)s + encryption = TLS + enableImapIdle = true + + ||| % { + domain: cfg.domain, + identity: cfg.identity, + emailAddress: cfg.email.address, + emailServer: cfg.email.server, + emailUser: cfg.email.username, + }, + }, + }, + + volumes: { + [name]: kube.PersistentVolumeClaim(gerrit.name(name)) { + metadata+: gerrit.metadata("storage"), + spec+: { + storageClassName: cfg.storageClassName, + accessModes: ["ReadWriteOnce"], + resources: { + requests: { + storage: cfg.storageSize[name], + }, + }, + }, + } + for name in ["etc", "git", "index", "cache", "db"] + }, + + local volumeMounts = { + [name]: { mountPath: "/var/gerrit/%s" % name } + for name in ["etc", "git", "index", "cache", "db"] + } { + // ConfigMap gets mounted here + config: { mountPath: "/var/gerrit-config" }, + // SecureSecret gets mounted here + secure: { mountPath: "/var/gerrit-secure" }, + }, + deployment: kube.Deployment(gerrit.name("gerrit")) { + metadata+: gerrit.metadata("deployment"), + spec+: { + replicas: 1, + template+: { + spec+: { + securityContext: { + fsGroup: 1000, # gerrit uid + }, + volumes_: { + config: kube.ConfigMapVolume(gerrit.configmap), + secure: { secret: { secretName: cfg.secureSecret} }, + } { + [name]: kube.PersistentVolumeClaimVolume(gerrit.volumes[name]) + for name in ["etc", "git", "index", "cache", "db"] + }, + containers_: { + gerrit: kube.Container(gerrit.name("gerrit")) { + image: cfg.image, + ports_: { + http: { containerPort: 8080 }, + ssh: { containerPort: 29418 }, + }, + resources: cfg.resources, + volumeMounts_: volumeMounts, + }, + }, + }, + }, + }, + }, + + svc: kube.Service(gerrit.name("gerrit")) { + metadata+: gerrit.metadata("service"), + target_pod:: gerrit.deployment.spec.template, + spec+: { + ports: [ + { name: "http", port: 80, targetPort: 8080, protocol: "TCP" }, + { name: "ssh", port: 22, targetPort: 29418, protocol: "TCP" }, + ], + type: "ClusterIP", + }, + }, + + ingress: kube.Ingress(gerrit.name("gerrit")) { + metadata+: gerrit.metadata("ingress") { + annotations+: { + "kubernetes.io/tls-acme": "true", + "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod", + "nginx.ingress.kubernetes.io/proxy-body-size": "0", + }, + }, + spec+: { + tls: [ + { hosts: [cfg.domain], secretName: gerrit.name("acme") }, + ], + rules: [ + { + host: cfg.domain, + http: { + paths: [ + { path: "/", backend: gerrit.svc.name_port }, + ], + }, + } + ], + }, + }, +} diff --git a/app/gerrit/kube/prod.jsonnet b/app/gerrit/kube/prod.jsonnet new file mode 100644 index 00000000..565772f3 --- /dev/null +++ b/app/gerrit/kube/prod.jsonnet @@ -0,0 +1,19 @@ +local kube = import "../../../kube/kube.libsonnet"; +local gerrit = import "gerrit.libsonnet"; +{ + namespace: kube.Namespace("gerrit"), + + gerrit: gerrit { + cfg+: { + namespace: "gerrit", + prefix: "", + + domain: "gerrit.hackerspace.pl", + identity: "7b6244cf-e30b-42c5-ba91-c329ef4e6cf1", + + storageClassName: "waw-hdd-redundant-1", + + secureSecret: "gerrit", + }, + }, +}