1
0
Fork 0

Merge "app/registry: integrate into cluster/kube"

master
q3k 2019-08-02 00:28:10 +00:00 committed by Gerrit Code Review
commit f774f2f31d
4 changed files with 345 additions and 319 deletions

View File

@ -1,318 +0,0 @@
# registry.k0.hswaw.net, a private docker registry
# This needs an oauth2 secret provisioned, create with:
# kubectl -n registry create secret generic auth --from-literal=oauth2_secret=...
# kubectl get secrets rook-ceph-object-user-waw-hdd-redundant-1-object-registry -n ceph-waw1 -o yaml --export | kubectl replace -f - -n registry
local kube = import "../../kube/kube.libsonnet";
local cm = import "../../cluster/kube/lib/cert-manager.libsonnet";
{
local app = self,
local cfg = app.cfg,
cfg:: {
namespace: "registry",
domain: "k0.hswaw.net",
storageClassName: "waw-hdd-redundant-1",
},
metadata(component):: {
namespace: cfg.namespace,
labels: {
"app.kubernetes.io/name": "registry",
"app.kubernetes.io/managed-by": "kubecfg",
"app.kubernetes.io/component": component,
},
},
namespace: kube.Namespace(cfg.namespace),
registryIssuer: cm.Issuer("registry-issuer") {
metadata+: app.metadata("registry-issuer"),
spec: {
selfSigned: {},
},
},
authCertificate: cm.Certificate("auth") {
metadata+: app.metadata("auth"),
spec: {
secretName: "auth-internal",
duration: "43800h0m0s", // 5 years
issuerRef: {
name: app.registryIssuer.metadata.name,
},
commonName: "auth.registry",
},
},
registryCertificate: cm.Certificate("registry") {
metadata+: app.metadata("registry"),
spec: {
secretName: "registry-internal",
duration: "43800h0m0s", // 5 years
issuerRef: {
name: app.registryIssuer.metadata.name,
},
commonName: "registry.registry",
},
},
registryConfig: kube.ConfigMap("registry-config") {
metadata+: app.metadata("registry-config"),
data: {
"config.yml": std.manifestYamlDoc({
version: "0.1",
log: {
fields: {
service: "registry",
},
},
storage: {
cache: {
blobdescriptor: "inmemory",
},
s3: {
regionendpoint: "https://object.ceph-waw1.hswaw.net",
bucket: "registry",
region: "waw-hdd-redunant-1-object:default-placement",
},
},
http: {
addr: ":5000",
headers: {
"X-Content-Type-Options": ["nosniff"],
},
tls: {
certificate: "/certs/tls.crt",
key: "/certs/tls.key",
},
debug: {
addr: "localhost:5001",
},
},
health: {
storagedriver: {
enabled: true,
interval: "10s",
threshold: 3,
},
},
auth: {
token: {
realm: "https://registry.%s/auth" % [cfg.domain],
service: "my.docker.registry",
issuer: "registry.%s auth server" % [cfg.domain],
rootcertbundle: "/authcerts/tls.crt",
},
},
}),
},
},
authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") {
metadata+: app.metadata("auth-token-storage"),
spec+: {
storageClassName: cfg.storageClassName,
accessModes: [ "ReadWriteOnce" ],
resources: {
requests: {
storage: "1Gi",
},
},
},
},
authConfig: kube.ConfigMap("auth-config") {
metadata+: app.metadata("auth-config"),
data: {
"auth_config.yml": std.manifestYamlDoc({
server: {
addr: ":5001",
certificate: "/certs/tls.crt",
key: "/certs/tls.key",
},
token: {
issuer: "registry.%s auth server" % [cfg.domain],
expiration: 900,
},
oauth2: {
client_id: "registry",
client_secret_file: "/secrets/oauth2_secret",
authorize_url: "https://sso.hackerspace.pl/oauth/authorize",
access_token_url: "https://sso.hackerspace.pl/oauth/token",
profile_url: "https://sso.hackerspace.pl/api/1/profile",
redirect_url: "https://registry.k0.hswaw.net/oauth2",
username_key: "username",
token_db: "/data/oauth2_tokens.ldb",
registry_url: "https://registry.k0.hswaw.net",
},
users: {
[""]: {}, // '' user are anonymous users.
},
local data = self,
pushers:: [
{ who: ["q3k", "inf"], what: "vms/*" },
{ who: ["q3k", "inf"], what: "app/*" },
{ who: ["q3k", "inf"], what: "go/svc/*" },
],
acl: [
{
match: {account: "/.+/", name: "${account}/*"},
actions: ["*"],
comment: "Logged in users have full access to images that are in their 'namespace'",
},
{
match: {account: "/.+/", type: "registry", name: "catalog"},
actions: ["*"],
comment: "Logged in users can query the catalog.",
},
{
match: {account: ""},
actions: ["pull"],
comment: "Anyone can pull all images.",
},
] + [
{
match: {
account: "/(%s)/" % std.join("|", p.who),
name: p.what,
},
actions: ["*"],
comment: "%s can push to %s" % [std.join(", ", p.who), p.what],
}
for p in data.pushers
],
}),
}
},
authDeployment: kube.Deployment("auth") {
metadata+: app.metadata("auth"),
spec+: {
replicas: 1,
template+: {
spec+: {
volumes_: {
data: kube.PersistentVolumeClaimVolume(app.authVolumeClaim),
config: kube.ConfigMapVolume(app.authConfig),
certs: {
secret: { secretName: app.authCertificate.spec.secretName },
},
secrets: {
secret: { secretName: "auth" },
},
},
containers_: {
auth: kube.Container("auth") {
image: "informatic/docker_auth:2019040307",
volumeMounts_: {
config: { mountPath: "/config" },
certs: { mountPath: "/certs" },
secrets: { mountPath: "/secrets" },
data: { mountPath: "/data" },
},
},
},
},
},
},
},
authService: kube.Service("auth") {
metadata+: app.metadata("auth"),
target_pod:: app.authDeployment.spec.template,
spec+: {
type: "ClusterIP",
ports: [
{ name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" },
],
}
},
registryDeployment: kube.Deployment("docker-registry") {
metadata+: app.metadata("docker-registry"),
spec+: {
replicas: 1,
template+: {
spec+: {
volumes_: {
config: kube.ConfigMapVolume(app.registryConfig),
certs: {
secret: { secretName: app.registryCertificate.spec.secretName },
},
authcerts: {
secret: { secretName: app.authCertificate.spec.secretName },
},
},
containers_: {
registry: kube.Container("docker-registry") {
image: "registry:2",
args: ["/config/config.yml"],
volumeMounts_: {
config: { mountPath: "/config" },
certs: { mountPath: "/certs" },
authcerts: { mountPath: "/authcerts" },
},
env_: {
REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
key: "AccessKey"
}},
REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
key: "SecretKey",
}},
},
},
},
},
},
},
},
registryService: kube.Service("docker-registry") {
metadata+: app.metadata("docker-registry"),
target_pod:: app.registryDeployment.spec.template,
spec+: {
type: "ClusterIP",
ports: [
{ name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" },
],
}
},
registryIngress: kube.Ingress("registry") {
metadata+: app.metadata("registry") {
annotations+: {
"kubernetes.io/tls-acme": "true",
"certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
"nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
"nginx.ingress.kubernetes.io/proxy-body-size": "0",
},
},
spec+: {
tls: [
{
hosts: ["registry.%s" % [cfg.domain]],
secretName: "registry-tls",
},
],
rules: [
{
host: "registry.%s" % [cfg.domain],
http: {
paths: [
{ path: "/auth", backend: app.authService.name_port },
{ path: "/", backend: app.authService.name_port },
{ path: "/v2/", backend: app.registryService.name_port },
]
},
}
],
},
},
registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") {
metadata+: {
namespace: "ceph-waw1",
},
spec: {
store: "waw-hdd-redundant-1-object",
displayName: "docker-registry user",
},
},
}

View File

@ -9,10 +9,19 @@ local coredns = import "lib/coredns.libsonnet";
local metallb = import "lib/metallb.libsonnet";
local metrics = import "lib/metrics.libsonnet";
local nginx = import "lib/nginx.libsonnet";
local registry = import "lib/registry.libsonnet";
local rook = import "lib/rook.libsonnet";
local Cluster(fqdn) = {
local cluster = self,
local cfg = cluster.cfg,
cfg:: {
// Storage class used for internal services (like registry). This must
// be set to a valid storage class. This can either be a cloud provider class
// (when running on GKE &co) or a storage class created using rook.
storageClassNameRedundant: error "storageClassNameRedundant must be set",
},
// These are required to let the API Server contact kubelets.
crAPIServerToKubelet: kube.ClusterRole("system:kube-apiserver-to-kubelet") {
@ -88,13 +97,25 @@ local Cluster(fqdn) = {
},
},
},
// Docker registry
registry: registry.Environment {
cfg+: {
domain: "registry.%s" % [fqdn],
storageClassName: cfg.storageClassNameRedundant,
},
},
};
{
k0: {
local k0 = self,
cluster: Cluster("k0.hswaw.net"),
cluster: Cluster("k0.hswaw.net") {
cfg+: {
storageClassNameRedundant: k0.ceph.blockRedundant.name,
},
},
cockroach: {
waw1: cockroachdb.Cluster("crdb-waw1") {
cfg+: {

View File

@ -0,0 +1,321 @@
# Deploy a Docker Registry in a cluster.
# This needs an oauth2 secret provisioned, create with:
# kubectl -n registry create secret generic auth --from-literal=oauth2_secret=...
# kubectl get secrets rook-ceph-object-user-<ceph-pool>-object-registry -n <ceph-namespace> -o yaml --export | kubectl replace -f - -n registry
local kube = import "../../../kube/kube.libsonnet";
local cm = import "cert-manager.libsonnet";
{
Environment: {
local env = self,
local cfg = env.cfg,
cfg:: {
namespace: "registry",
domain: error "domain must be set",
storageClassName: error "storageClassName must be set",
},
metadata(component):: {
namespace: cfg.namespace,
labels: {
"app.kubernetes.io/name": "registry",
"app.kubernetes.io/managed-by": "kubecfg",
"app.kubernetes.io/component": component,
},
},
namespace: kube.Namespace(cfg.namespace),
registryIssuer: cm.Issuer("registry-issuer") {
metadata+: env.metadata("registry-issuer"),
spec: {
selfSigned: {},
},
},
authCertificate: cm.Certificate("auth") {
metadata+: env.metadata("auth"),
spec: {
secretName: "auth-internal",
duration: "43800h0m0s", // 5 years
issuerRef: {
name: env.registryIssuer.metadata.name,
},
commonName: "auth.registry",
},
},
registryCertificate: cm.Certificate("registry") {
metadata+: env.metadata("registry"),
spec: {
secretName: "registry-internal",
duration: "43800h0m0s", // 5 years
issuerRef: {
name: env.registryIssuer.metadata.name,
},
commonName: "registry.registry",
},
},
registryConfig: kube.ConfigMap("registry-config") {
metadata+: env.metadata("registry-config"),
data: {
"config.yml": std.manifestYamlDoc({
version: "0.1",
log: {
fields: {
service: "registry",
},
},
storage: {
cache: {
blobdescriptor: "inmemory",
},
s3: {
regionendpoint: "https://object.ceph-waw1.hswaw.net",
bucket: "registry",
region: "waw-hdd-redunant-1-object:default-placement",
},
},
http: {
addr: ":5000",
headers: {
"X-Content-Type-Options": ["nosniff"],
},
tls: {
certificate: "/certs/tls.crt",
key: "/certs/tls.key",
},
debug: {
addr: "localhost:5001",
},
},
health: {
storagedriver: {
enabled: true,
interval: "10s",
threshold: 3,
},
},
auth: {
token: {
realm: "https://%s/auth" % [cfg.domain],
service: "my.docker.registry",
issuer: "%s auth server" % [cfg.domain],
rootcertbundle: "/authcerts/tls.crt",
},
},
}),
},
},
authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage") {
metadata+: env.metadata("auth-token-storage"),
spec+: {
storageClassName: cfg.storageClassName,
accessModes: [ "ReadWriteOnce" ],
resources: {
requests: {
storage: "1Gi",
},
},
},
},
authConfig: kube.ConfigMap("auth-config") {
metadata+: env.metadata("auth-config"),
data: {
"auth_config.yml": std.manifestYamlDoc({
server: {
addr: ":5001",
certificate: "/certs/tls.crt",
key: "/certs/tls.key",
},
token: {
issuer: "%s auth server" % [cfg.domain],
expiration: 900,
},
oauth2: {
client_id: "registry",
client_secret_file: "/secrets/oauth2_secret",
authorize_url: "https://sso.hackerspace.pl/oauth/authorize",
access_token_url: "https://sso.hackerspace.pl/oauth/token",
profile_url: "https://sso.hackerspace.pl/api/1/profile",
redirect_url: "https://registry.k0.hswaw.net/oauth2",
username_key: "username",
token_db: "/data/oauth2_tokens.ldb",
registry_url: "https://registry.k0.hswaw.net",
},
users: {
[""]: {}, // '' user are anonymous users.
},
local data = self,
pushers:: [
{ who: ["q3k", "inf"], what: "vms/*" },
{ who: ["q3k", "inf"], what: "app/*" },
{ who: ["q3k", "inf"], what: "go/svc/*" },
],
acl: [
{
match: {account: "/.+/", name: "${account}/*"},
actions: ["*"],
comment: "Logged in users have full access to images that are in their 'namespace'",
},
{
match: {account: "/.+/", type: "registry", name: "catalog"},
actions: ["*"],
comment: "Logged in users can query the catalog.",
},
{
match: {account: ""},
actions: ["pull"],
comment: "Anyone can pull all images.",
},
] + [
{
match: {
account: "/(%s)/" % std.join("|", p.who),
name: p.what,
},
actions: ["*"],
comment: "%s can push to %s" % [std.join(", ", p.who), p.what],
}
for p in data.pushers
],
}),
}
},
authDeployment: kube.Deployment("auth") {
metadata+: env.metadata("auth"),
spec+: {
replicas: 1,
template+: {
spec+: {
volumes_: {
data: kube.PersistentVolumeClaimVolume(env.authVolumeClaim),
config: kube.ConfigMapVolume(env.authConfig),
certs: {
secret: { secretName: env.authCertificate.spec.secretName },
},
secrets: {
secret: { secretName: "auth" },
},
},
containers_: {
auth: kube.Container("auth") {
image: "informatic/docker_auth:2019040307",
volumeMounts_: {
config: { mountPath: "/config" },
certs: { mountPath: "/certs" },
secrets: { mountPath: "/secrets" },
data: { mountPath: "/data" },
},
},
},
},
},
},
},
authService: kube.Service("auth") {
metadata+: env.metadata("auth"),
target_pod:: env.authDeployment.spec.template,
spec+: {
type: "ClusterIP",
ports: [
{ name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" },
],
}
},
registryDeployment: kube.Deployment("docker-registry") {
metadata+: env.metadata("docker-registry"),
spec+: {
replicas: 1,
template+: {
spec+: {
volumes_: {
config: kube.ConfigMapVolume(env.registryConfig),
certs: {
secret: { secretName: env.registryCertificate.spec.secretName },
},
authcerts: {
secret: { secretName: env.authCertificate.spec.secretName },
},
},
containers_: {
registry: kube.Container("docker-registry") {
image: "registry:2",
args: ["/config/config.yml"],
volumeMounts_: {
config: { mountPath: "/config" },
certs: { mountPath: "/certs" },
authcerts: { mountPath: "/authcerts" },
},
env_: {
REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
key: "AccessKey"
}},
REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
key: "SecretKey",
}},
},
},
},
},
},
},
},
registryService: kube.Service("docker-registry") {
metadata+: env.metadata("docker-registry"),
target_pod:: env.registryDeployment.spec.template,
spec+: {
type: "ClusterIP",
ports: [
{ name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" },
],
}
},
registryIngress: kube.Ingress("registry") {
metadata+: env.metadata("registry") {
annotations+: {
"kubernetes.io/tls-acme": "true",
"certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
"nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
"nginx.ingress.kubernetes.io/proxy-body-size": "0",
},
},
spec+: {
tls: [
{
hosts: [cfg.domain],
secretName: "registry-tls",
},
],
rules: [
{
host: cfg.domain,
http: {
paths: [
{ path: "/auth", backend: env.authService.name_port },
{ path: "/", backend: env.authService.name_port },
{ path: "/v2/", backend: env.registryService.name_port },
]
},
}
],
},
},
registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") {
metadata+: {
namespace: "ceph-waw1",
},
spec: {
store: "waw-hdd-redundant-1-object",
displayName: "docker-registry user",
},
},
}
}

View File

@ -491,6 +491,8 @@ local kube = import "../../../kube/kube.libsonnet";
ECBlockPool(cluster, name):: {
local pool = self,
name:: name,
spec:: error "spec must be specified",
pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {