From 4d61d20aec3564207ebc23ef601e9de3187d2353 Mon Sep 17 00:00:00 2001 From: Sergiusz Bazanski Date: Sun, 21 Jul 2019 16:56:41 +0200 Subject: [PATCH] app/registry: integrate into cluster/kube This makes a registry be automatically part of the cluster infrastructure. Tested by running kubecfg diff, no diffs (apart from out-of-date ACLs) found. Change-Id: Ic0635e789cf3fb851f410bcf2865326f1fa87545 --- app/registry/prod.jsonnet | 318 --------------------------- cluster/kube/cluster.jsonnet | 23 +- cluster/kube/lib/registry.libsonnet | 321 ++++++++++++++++++++++++++++ cluster/kube/lib/rook.libsonnet | 2 + 4 files changed, 345 insertions(+), 319 deletions(-) delete mode 100644 app/registry/prod.jsonnet create mode 100644 cluster/kube/lib/registry.libsonnet diff --git a/app/registry/prod.jsonnet b/app/registry/prod.jsonnet deleted file mode 100644 index 8e3b5627..00000000 --- a/app/registry/prod.jsonnet +++ /dev/null @@ -1,318 +0,0 @@ -# registry.k0.hswaw.net, a private docker registry -# This needs an oauth2 secret provisioned, create with: -# kubectl -n registry create secret generic auth --from-literal=oauth2_secret=... -# kubectl get secrets rook-ceph-object-user-waw-hdd-redundant-1-object-registry -n ceph-waw1 -o yaml --export | kubectl replace -f - -n registry - -local kube = import "../../kube/kube.libsonnet"; -local cm = import "../../cluster/kube/lib/cert-manager.libsonnet"; - -{ - local app = self, - local cfg = app.cfg, - cfg:: { - namespace: "registry", - domain: "k0.hswaw.net", - storageClassName: "waw-hdd-redundant-1", - }, - - metadata(component):: { - namespace: cfg.namespace, - labels: { - "app.kubernetes.io/name": "registry", - "app.kubernetes.io/managed-by": "kubecfg", - "app.kubernetes.io/component": component, - }, - }, - - namespace: kube.Namespace(cfg.namespace), - - registryIssuer: cm.Issuer("registry-issuer") { - metadata+: app.metadata("registry-issuer"), - spec: { - selfSigned: {}, - }, - }, - authCertificate: cm.Certificate("auth") { - metadata+: app.metadata("auth"), - spec: { - secretName: "auth-internal", - duration: "43800h0m0s", // 5 years - issuerRef: { - name: app.registryIssuer.metadata.name, - }, - commonName: "auth.registry", - }, - }, - registryCertificate: cm.Certificate("registry") { - metadata+: app.metadata("registry"), - spec: { - secretName: "registry-internal", - duration: "43800h0m0s", // 5 years - issuerRef: { - name: app.registryIssuer.metadata.name, - }, - commonName: "registry.registry", - }, - }, - - registryConfig: kube.ConfigMap("registry-config") { - metadata+: app.metadata("registry-config"), - data: { - "config.yml": std.manifestYamlDoc({ - version: "0.1", - log: { - fields: { - service: "registry", - }, - }, - storage: { - cache: { - blobdescriptor: "inmemory", - }, - s3: { - regionendpoint: "https://object.ceph-waw1.hswaw.net", - bucket: "registry", - region: "waw-hdd-redunant-1-object:default-placement", - }, - }, - http: { - addr: ":5000", - headers: { - "X-Content-Type-Options": ["nosniff"], - }, - tls: { - certificate: "/certs/tls.crt", - key: "/certs/tls.key", - }, - debug: { - addr: "localhost:5001", - }, - }, - health: { - storagedriver: { - enabled: true, - interval: "10s", - threshold: 3, - }, - }, - auth: { - token: { - realm: "https://registry.%s/auth" % [cfg.domain], - service: "my.docker.registry", - issuer: "registry.%s auth server" % [cfg.domain], - rootcertbundle: "/authcerts/tls.crt", - }, - }, - }), - }, - }, - - authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") { - metadata+: app.metadata("auth-token-storage"), - spec+: { - storageClassName: cfg.storageClassName, - accessModes: [ "ReadWriteOnce" ], - resources: { - requests: { - storage: "1Gi", - }, - }, - }, - }, - - authConfig: kube.ConfigMap("auth-config") { - metadata+: app.metadata("auth-config"), - data: { - "auth_config.yml": std.manifestYamlDoc({ - server: { - addr: ":5001", - certificate: "/certs/tls.crt", - key: "/certs/tls.key", - }, - token: { - issuer: "registry.%s auth server" % [cfg.domain], - expiration: 900, - }, - oauth2: { - client_id: "registry", - client_secret_file: "/secrets/oauth2_secret", - authorize_url: "https://sso.hackerspace.pl/oauth/authorize", - access_token_url: "https://sso.hackerspace.pl/oauth/token", - profile_url: "https://sso.hackerspace.pl/api/1/profile", - redirect_url: "https://registry.k0.hswaw.net/oauth2", - username_key: "username", - token_db: "/data/oauth2_tokens.ldb", - registry_url: "https://registry.k0.hswaw.net", - }, - users: { - [""]: {}, // '' user are anonymous users. - }, - local data = self, - pushers:: [ - { who: ["q3k", "inf"], what: "vms/*" }, - { who: ["q3k", "inf"], what: "app/*" }, - { who: ["q3k", "inf"], what: "go/svc/*" }, - ], - acl: [ - { - match: {account: "/.+/", name: "${account}/*"}, - actions: ["*"], - comment: "Logged in users have full access to images that are in their 'namespace'", - }, - { - match: {account: "/.+/", type: "registry", name: "catalog"}, - actions: ["*"], - comment: "Logged in users can query the catalog.", - }, - { - match: {account: ""}, - actions: ["pull"], - comment: "Anyone can pull all images.", - }, - ] + [ - { - match: { - account: "/(%s)/" % std.join("|", p.who), - name: p.what, - }, - actions: ["*"], - comment: "%s can push to %s" % [std.join(", ", p.who), p.what], - } - for p in data.pushers - ], - }), - } - }, - - authDeployment: kube.Deployment("auth") { - metadata+: app.metadata("auth"), - spec+: { - replicas: 1, - template+: { - spec+: { - volumes_: { - data: kube.PersistentVolumeClaimVolume(app.authVolumeClaim), - config: kube.ConfigMapVolume(app.authConfig), - certs: { - secret: { secretName: app.authCertificate.spec.secretName }, - }, - secrets: { - secret: { secretName: "auth" }, - }, - }, - containers_: { - auth: kube.Container("auth") { - image: "informatic/docker_auth:2019040307", - volumeMounts_: { - config: { mountPath: "/config" }, - certs: { mountPath: "/certs" }, - secrets: { mountPath: "/secrets" }, - data: { mountPath: "/data" }, - }, - }, - }, - }, - }, - }, - }, - authService: kube.Service("auth") { - metadata+: app.metadata("auth"), - target_pod:: app.authDeployment.spec.template, - spec+: { - type: "ClusterIP", - ports: [ - { name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" }, - ], - } - }, - registryDeployment: kube.Deployment("docker-registry") { - metadata+: app.metadata("docker-registry"), - spec+: { - replicas: 1, - template+: { - spec+: { - volumes_: { - config: kube.ConfigMapVolume(app.registryConfig), - certs: { - secret: { secretName: app.registryCertificate.spec.secretName }, - }, - authcerts: { - secret: { secretName: app.authCertificate.spec.secretName }, - }, - }, - containers_: { - registry: kube.Container("docker-registry") { - image: "registry:2", - args: ["/config/config.yml"], - volumeMounts_: { - config: { mountPath: "/config" }, - certs: { mountPath: "/certs" }, - authcerts: { mountPath: "/authcerts" }, - }, - env_: { - REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: { - name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry", - key: "AccessKey" - }}, - REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: { - name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry", - key: "SecretKey", - }}, - }, - }, - }, - }, - }, - }, - }, - registryService: kube.Service("docker-registry") { - metadata+: app.metadata("docker-registry"), - target_pod:: app.registryDeployment.spec.template, - spec+: { - type: "ClusterIP", - ports: [ - { name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" }, - ], - } - }, - registryIngress: kube.Ingress("registry") { - metadata+: app.metadata("registry") { - annotations+: { - "kubernetes.io/tls-acme": "true", - "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod", - "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", - "nginx.ingress.kubernetes.io/proxy-body-size": "0", - }, - }, - spec+: { - tls: [ - { - hosts: ["registry.%s" % [cfg.domain]], - secretName: "registry-tls", - }, - ], - rules: [ - { - host: "registry.%s" % [cfg.domain], - http: { - paths: [ - { path: "/auth", backend: app.authService.name_port }, - { path: "/", backend: app.authService.name_port }, - { path: "/v2/", backend: app.registryService.name_port }, - ] - }, - } - ], - }, - }, - - registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") { - metadata+: { - namespace: "ceph-waw1", - }, - spec: { - store: "waw-hdd-redundant-1-object", - displayName: "docker-registry user", - }, - }, -} diff --git a/cluster/kube/cluster.jsonnet b/cluster/kube/cluster.jsonnet index fc0db19a..d0b77bdc 100644 --- a/cluster/kube/cluster.jsonnet +++ b/cluster/kube/cluster.jsonnet @@ -9,10 +9,19 @@ local coredns = import "lib/coredns.libsonnet"; local metallb = import "lib/metallb.libsonnet"; local metrics = import "lib/metrics.libsonnet"; local nginx = import "lib/nginx.libsonnet"; +local registry = import "lib/registry.libsonnet"; local rook = import "lib/rook.libsonnet"; local Cluster(fqdn) = { local cluster = self, + local cfg = cluster.cfg, + + cfg:: { + // Storage class used for internal services (like registry). This must + // be set to a valid storage class. This can either be a cloud provider class + // (when running on GKE &co) or a storage class created using rook. + storageClassNameRedundant: error "storageClassNameRedundant must be set", + }, // These are required to let the API Server contact kubelets. crAPIServerToKubelet: kube.ClusterRole("system:kube-apiserver-to-kubelet") { @@ -88,13 +97,25 @@ local Cluster(fqdn) = { }, }, }, + + // Docker registry + registry: registry.Environment { + cfg+: { + domain: "registry.%s" % [fqdn], + storageClassName: cfg.storageClassNameRedundant, + }, + }, }; { k0: { local k0 = self, - cluster: Cluster("k0.hswaw.net"), + cluster: Cluster("k0.hswaw.net") { + cfg+: { + storageClassNameRedundant: k0.ceph.blockRedundant.name, + }, + }, cockroach: { waw1: cockroachdb.Cluster("crdb-waw1") { cfg+: { diff --git a/cluster/kube/lib/registry.libsonnet b/cluster/kube/lib/registry.libsonnet new file mode 100644 index 00000000..8b57dd7e --- /dev/null +++ b/cluster/kube/lib/registry.libsonnet @@ -0,0 +1,321 @@ +# Deploy a Docker Registry in a cluster. + +# This needs an oauth2 secret provisioned, create with: +# kubectl -n registry create secret generic auth --from-literal=oauth2_secret=... +# kubectl get secrets rook-ceph-object-user--object-registry -n -o yaml --export | kubectl replace -f - -n registry + +local kube = import "../../../kube/kube.libsonnet"; +local cm = import "cert-manager.libsonnet"; + +{ + Environment: { + local env = self, + local cfg = env.cfg, + cfg:: { + namespace: "registry", + domain: error "domain must be set", + storageClassName: error "storageClassName must be set", + }, + + metadata(component):: { + namespace: cfg.namespace, + labels: { + "app.kubernetes.io/name": "registry", + "app.kubernetes.io/managed-by": "kubecfg", + "app.kubernetes.io/component": component, + }, + }, + + namespace: kube.Namespace(cfg.namespace), + + registryIssuer: cm.Issuer("registry-issuer") { + metadata+: env.metadata("registry-issuer"), + spec: { + selfSigned: {}, + }, + }, + authCertificate: cm.Certificate("auth") { + metadata+: env.metadata("auth"), + spec: { + secretName: "auth-internal", + duration: "43800h0m0s", // 5 years + issuerRef: { + name: env.registryIssuer.metadata.name, + }, + commonName: "auth.registry", + }, + }, + registryCertificate: cm.Certificate("registry") { + metadata+: env.metadata("registry"), + spec: { + secretName: "registry-internal", + duration: "43800h0m0s", // 5 years + issuerRef: { + name: env.registryIssuer.metadata.name, + }, + commonName: "registry.registry", + }, + }, + + registryConfig: kube.ConfigMap("registry-config") { + metadata+: env.metadata("registry-config"), + data: { + "config.yml": std.manifestYamlDoc({ + version: "0.1", + log: { + fields: { + service: "registry", + }, + }, + storage: { + cache: { + blobdescriptor: "inmemory", + }, + s3: { + regionendpoint: "https://object.ceph-waw1.hswaw.net", + bucket: "registry", + region: "waw-hdd-redunant-1-object:default-placement", + }, + }, + http: { + addr: ":5000", + headers: { + "X-Content-Type-Options": ["nosniff"], + }, + tls: { + certificate: "/certs/tls.crt", + key: "/certs/tls.key", + }, + debug: { + addr: "localhost:5001", + }, + }, + health: { + storagedriver: { + enabled: true, + interval: "10s", + threshold: 3, + }, + }, + auth: { + token: { + realm: "https://%s/auth" % [cfg.domain], + service: "my.docker.registry", + issuer: "%s auth server" % [cfg.domain], + rootcertbundle: "/authcerts/tls.crt", + }, + }, + }), + }, + }, + + authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") { + metadata+: env.metadata("auth-token-storage"), + spec+: { + storageClassName: cfg.storageClassName, + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: "1Gi", + }, + }, + }, + }, + + authConfig: kube.ConfigMap("auth-config") { + metadata+: env.metadata("auth-config"), + data: { + "auth_config.yml": std.manifestYamlDoc({ + server: { + addr: ":5001", + certificate: "/certs/tls.crt", + key: "/certs/tls.key", + }, + token: { + issuer: "%s auth server" % [cfg.domain], + expiration: 900, + }, + oauth2: { + client_id: "registry", + client_secret_file: "/secrets/oauth2_secret", + authorize_url: "https://sso.hackerspace.pl/oauth/authorize", + access_token_url: "https://sso.hackerspace.pl/oauth/token", + profile_url: "https://sso.hackerspace.pl/api/1/profile", + redirect_url: "https://registry.k0.hswaw.net/oauth2", + username_key: "username", + token_db: "/data/oauth2_tokens.ldb", + registry_url: "https://registry.k0.hswaw.net", + }, + users: { + [""]: {}, // '' user are anonymous users. + }, + local data = self, + pushers:: [ + { who: ["q3k", "inf"], what: "vms/*" }, + { who: ["q3k", "inf"], what: "app/*" }, + { who: ["q3k", "inf"], what: "go/svc/*" }, + ], + acl: [ + { + match: {account: "/.+/", name: "${account}/*"}, + actions: ["*"], + comment: "Logged in users have full access to images that are in their 'namespace'", + }, + { + match: {account: "/.+/", type: "registry", name: "catalog"}, + actions: ["*"], + comment: "Logged in users can query the catalog.", + }, + { + match: {account: ""}, + actions: ["pull"], + comment: "Anyone can pull all images.", + }, + ] + [ + { + match: { + account: "/(%s)/" % std.join("|", p.who), + name: p.what, + }, + actions: ["*"], + comment: "%s can push to %s" % [std.join(", ", p.who), p.what], + } + for p in data.pushers + ], + }), + } + }, + + authDeployment: kube.Deployment("auth") { + metadata+: env.metadata("auth"), + spec+: { + replicas: 1, + template+: { + spec+: { + volumes_: { + data: kube.PersistentVolumeClaimVolume(env.authVolumeClaim), + config: kube.ConfigMapVolume(env.authConfig), + certs: { + secret: { secretName: env.authCertificate.spec.secretName }, + }, + secrets: { + secret: { secretName: "auth" }, + }, + }, + containers_: { + auth: kube.Container("auth") { + image: "informatic/docker_auth:2019040307", + volumeMounts_: { + config: { mountPath: "/config" }, + certs: { mountPath: "/certs" }, + secrets: { mountPath: "/secrets" }, + data: { mountPath: "/data" }, + }, + }, + }, + }, + }, + }, + }, + authService: kube.Service("auth") { + metadata+: env.metadata("auth"), + target_pod:: env.authDeployment.spec.template, + spec+: { + type: "ClusterIP", + ports: [ + { name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" }, + ], + } + }, + registryDeployment: kube.Deployment("docker-registry") { + metadata+: env.metadata("docker-registry"), + spec+: { + replicas: 1, + template+: { + spec+: { + volumes_: { + config: kube.ConfigMapVolume(env.registryConfig), + certs: { + secret: { secretName: env.registryCertificate.spec.secretName }, + }, + authcerts: { + secret: { secretName: env.authCertificate.spec.secretName }, + }, + }, + containers_: { + registry: kube.Container("docker-registry") { + image: "registry:2", + args: ["/config/config.yml"], + volumeMounts_: { + config: { mountPath: "/config" }, + certs: { mountPath: "/certs" }, + authcerts: { mountPath: "/authcerts" }, + }, + env_: { + REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: { + name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry", + key: "AccessKey" + }}, + REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: { + name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry", + key: "SecretKey", + }}, + }, + }, + }, + }, + }, + }, + }, + registryService: kube.Service("docker-registry") { + metadata+: env.metadata("docker-registry"), + target_pod:: env.registryDeployment.spec.template, + spec+: { + type: "ClusterIP", + ports: [ + { name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" }, + ], + } + }, + registryIngress: kube.Ingress("registry") { + metadata+: env.metadata("registry") { + annotations+: { + "kubernetes.io/tls-acme": "true", + "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod", + "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", + "nginx.ingress.kubernetes.io/proxy-body-size": "0", + }, + }, + spec+: { + tls: [ + { + hosts: [cfg.domain], + secretName: "registry-tls", + }, + ], + rules: [ + { + host: cfg.domain, + http: { + paths: [ + { path: "/auth", backend: env.authService.name_port }, + { path: "/", backend: env.authService.name_port }, + { path: "/v2/", backend: env.registryService.name_port }, + ] + }, + } + ], + }, + }, + + registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") { + metadata+: { + namespace: "ceph-waw1", + }, + spec: { + store: "waw-hdd-redundant-1-object", + displayName: "docker-registry user", + }, + }, + } +} diff --git a/cluster/kube/lib/rook.libsonnet b/cluster/kube/lib/rook.libsonnet index 52236540..512c6a09 100644 --- a/cluster/kube/lib/rook.libsonnet +++ b/cluster/kube/lib/rook.libsonnet @@ -491,6 +491,8 @@ local kube = import "../../../kube/kube.libsonnet"; ECBlockPool(cluster, name):: { local pool = self, + name:: name, + spec:: error "spec must be specified", pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {