mirror of https://gerrit.hackerspace.pl/hscloud
320 lines
13 KiB
Plaintext
320 lines
13 KiB
Plaintext
# Deploy a Docker Registry in a cluster.
|
|
|
|
# This needs an oauth2 secret provisioned, create with:
|
|
# kubectl -n registry create secret generic auth --from-literal=oauth2_secret=...
|
|
# kubectl get secrets rook-ceph-object-user-<ceph-pool>-object-registry -n <ceph-namespace> -o yaml --export | kubectl replace -f - -n registry
|
|
|
|
local kube = import "../../../kube/kube.libsonnet";
|
|
|
|
{
|
|
Environment: {
|
|
local env = self,
|
|
local cfg = env.cfg,
|
|
cfg:: {
|
|
namespace: "registry",
|
|
domain: error "domain must be set",
|
|
storageClassName: error "storageClassName must be set",
|
|
objectStoreName: error "objectStoreName must be set",
|
|
},
|
|
|
|
metadata(component):: {
|
|
namespace: cfg.namespace,
|
|
labels: {
|
|
"app.kubernetes.io/name": "registry",
|
|
"app.kubernetes.io/managed-by": "kubecfg",
|
|
"app.kubernetes.io/component": component,
|
|
},
|
|
},
|
|
|
|
namespace: kube.Namespace(cfg.namespace),
|
|
|
|
registryIssuer: kube.Issuer("registry-issuer") {
|
|
metadata+: env.metadata("registry-issuer"),
|
|
spec: {
|
|
selfSigned: {},
|
|
},
|
|
},
|
|
authCertificate: kube.Certificate("auth") {
|
|
metadata+: env.metadata("auth"),
|
|
spec: {
|
|
secretName: "auth-internal",
|
|
duration: "43800h0m0s", // 5 years
|
|
issuerRef: {
|
|
name: env.registryIssuer.metadata.name,
|
|
},
|
|
commonName: "auth.registry",
|
|
},
|
|
},
|
|
registryCertificate: kube.Certificate("registry") {
|
|
metadata+: env.metadata("registry"),
|
|
spec: {
|
|
secretName: "registry-internal",
|
|
duration: "43800h0m0s", // 5 years
|
|
issuerRef: {
|
|
name: env.registryIssuer.metadata.name,
|
|
},
|
|
commonName: "registry.registry",
|
|
},
|
|
},
|
|
|
|
registryConfig: kube.ConfigMap("registry-config") {
|
|
metadata+: env.metadata("registry-config"),
|
|
data: {
|
|
"config.yml": std.manifestYamlDoc({
|
|
version: "0.1",
|
|
log: {
|
|
level: "debug",
|
|
fields: {
|
|
service: "registry",
|
|
},
|
|
},
|
|
storage: {
|
|
cache: {
|
|
blobdescriptor: "inmemory",
|
|
},
|
|
s3: {
|
|
regionendpoint: "https://object.ceph-waw3.hswaw.net",
|
|
bucket: "registry",
|
|
region: "waw-hdd-redunant-3-object:default-placement",
|
|
},
|
|
},
|
|
http: {
|
|
addr: ":5000",
|
|
headers: {
|
|
"X-Content-Type-Options": ["nosniff"],
|
|
},
|
|
tls: {
|
|
certificate: "/certs/tls.crt",
|
|
key: "/certs/tls.key",
|
|
},
|
|
debug: {
|
|
addr: "localhost:5001",
|
|
},
|
|
},
|
|
health: {
|
|
storagedriver: {
|
|
enabled: true,
|
|
interval: "10s",
|
|
threshold: 3,
|
|
},
|
|
},
|
|
auth: {
|
|
token: {
|
|
realm: "https://%s/auth" % [cfg.domain],
|
|
service: "my.docker.registry",
|
|
issuer: "%s auth server" % [cfg.domain],
|
|
rootcertbundle: "/authcerts/tls.crt",
|
|
},
|
|
},
|
|
}),
|
|
},
|
|
},
|
|
|
|
authVolumeClaim: kube.PersistentVolumeClaim("auth-token-storage-3") {
|
|
metadata+: env.metadata("auth-token-storage-3"),
|
|
storage:: "1Gi",
|
|
storageClass:: cfg.storageClassName,
|
|
},
|
|
|
|
authConfig: kube.ConfigMap("auth-config") {
|
|
metadata+: env.metadata("auth-config"),
|
|
data: {
|
|
"auth_config.yml": std.manifestYamlDoc({
|
|
server: {
|
|
addr: ":5001",
|
|
certificate: "/certs/tls.crt",
|
|
key: "/certs/tls.key",
|
|
},
|
|
token: {
|
|
issuer: "%s auth server" % [cfg.domain],
|
|
expiration: 900,
|
|
},
|
|
oauth2: {
|
|
client_id: "registry",
|
|
client_secret_file: "/secrets/oauth2_secret",
|
|
authorize_url: "https://sso.hackerspace.pl/oauth/authorize",
|
|
access_token_url: "https://sso.hackerspace.pl/oauth/token",
|
|
profile_url: "https://sso.hackerspace.pl/api/1/profile",
|
|
redirect_url: "https://registry.k0.hswaw.net/oauth2",
|
|
username_key: "username",
|
|
token_db: "/data/oauth2_tokens.ldb",
|
|
registry_url: "https://registry.k0.hswaw.net",
|
|
},
|
|
users: {
|
|
[""]: {}, // '' user are anonymous users.
|
|
},
|
|
local data = self,
|
|
pushers:: [
|
|
{ who: ["q3k", "informatic"], what: "vms/*" },
|
|
{ who: ["q3k", "informatic"], what: "app/*" },
|
|
{ who: ["q3k", "informatic"], what: "go/svc/*" },
|
|
{ who: ["q3k"], what: "bgpwtf/*" },
|
|
{ who: ["q3k"], what: "devtools/*" },
|
|
{ who: ["q3k"], what: "games/factorio/*" },
|
|
{ who: ["q3k", "informatic"], what: "cluster/*" },
|
|
],
|
|
acl: [
|
|
{
|
|
match: {
|
|
account: "/(%s)/" % std.join("|", p.who),
|
|
name: p.what,
|
|
},
|
|
actions: ["*"],
|
|
comment: "%s can push to %s" % [std.join(", ", p.who), p.what],
|
|
}
|
|
for p in data.pushers
|
|
] + [
|
|
{
|
|
match: {account: "/.+/", name: "${account}/*"},
|
|
actions: ["*"],
|
|
comment: "Logged in users have full access to images that are in their 'namespace'",
|
|
},
|
|
{
|
|
match: {account: "/.+/", type: "registry", name: "catalog"},
|
|
actions: ["*"],
|
|
comment: "Logged in users can query the catalog.",
|
|
},
|
|
{
|
|
match: {account: "/.*/"},
|
|
actions: ["pull"],
|
|
comment: "Anyone can pull all images.",
|
|
},
|
|
],
|
|
}),
|
|
}
|
|
},
|
|
|
|
authDeployment: kube.Deployment("auth") {
|
|
metadata+: env.metadata("auth"),
|
|
spec+: {
|
|
replicas: 1,
|
|
template+: {
|
|
spec+: {
|
|
volumes_: {
|
|
data: kube.PersistentVolumeClaimVolume(env.authVolumeClaim),
|
|
config: kube.ConfigMapVolume(env.authConfig),
|
|
certs: {
|
|
secret: { secretName: env.authCertificate.spec.secretName },
|
|
},
|
|
secrets: {
|
|
secret: { secretName: "auth" },
|
|
},
|
|
},
|
|
containers_: {
|
|
auth: kube.Container("auth") {
|
|
image: "informatic/docker_auth:2019040307",
|
|
volumeMounts_: {
|
|
config: { mountPath: "/config" },
|
|
certs: { mountPath: "/certs" },
|
|
secrets: { mountPath: "/secrets" },
|
|
data: { mountPath: "/data" },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
authService: kube.Service("auth") {
|
|
metadata+: env.metadata("auth"),
|
|
target:: env.authDeployment,
|
|
spec+: {
|
|
type: "ClusterIP",
|
|
ports: [
|
|
{ name: "auth", port: 5001, targetPort: 5001, protocol: "TCP" },
|
|
],
|
|
}
|
|
},
|
|
registryDeployment: kube.Deployment("docker-registry") {
|
|
metadata+: env.metadata("docker-registry"),
|
|
spec+: {
|
|
replicas: 1,
|
|
template+: {
|
|
spec+: {
|
|
volumes_: {
|
|
config: kube.ConfigMapVolume(env.registryConfig),
|
|
certs: {
|
|
secret: { secretName: env.registryCertificate.spec.secretName },
|
|
},
|
|
authcerts: {
|
|
secret: { secretName: env.authCertificate.spec.secretName },
|
|
},
|
|
},
|
|
containers_: {
|
|
registry: kube.Container("docker-registry") {
|
|
image: "registry:2.7.1",
|
|
args: ["/config/config.yml"],
|
|
volumeMounts_: {
|
|
config: { mountPath: "/config" },
|
|
certs: { mountPath: "/certs" },
|
|
authcerts: { mountPath: "/authcerts" },
|
|
},
|
|
env_: {
|
|
REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
|
|
name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName},
|
|
key: "AccessKey"
|
|
}},
|
|
REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
|
|
name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName},
|
|
key: "SecretKey",
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
registryService: kube.Service("docker-registry") {
|
|
metadata+: env.metadata("docker-registry"),
|
|
target:: env.registryDeployment,
|
|
spec+: {
|
|
type: "ClusterIP",
|
|
ports: [
|
|
{ name: "registry", port: 5000, targetPort: 5000, protocol: "TCP" },
|
|
],
|
|
}
|
|
},
|
|
registryIngress: kube.Ingress("registry") {
|
|
metadata+: env.metadata("registry") {
|
|
annotations+: {
|
|
"kubernetes.io/tls-acme": "true",
|
|
"cert-manager.io/cluster-issuer": "letsencrypt-prod",
|
|
"nginx.ingress.kubernetes.io/backend-protocol": "HTTPS",
|
|
"nginx.ingress.kubernetes.io/proxy-body-size": "0",
|
|
},
|
|
},
|
|
spec+: {
|
|
tls: [
|
|
{
|
|
hosts: [cfg.domain],
|
|
secretName: "registry-tls",
|
|
},
|
|
],
|
|
rules: [
|
|
{
|
|
host: cfg.domain,
|
|
http: {
|
|
paths: [
|
|
{ path: "/auth", backend: env.authService.name_port },
|
|
{ path: "/", backend: env.authService.name_port },
|
|
{ path: "/v2/", backend: env.registryService.name_port },
|
|
]
|
|
},
|
|
}
|
|
],
|
|
},
|
|
},
|
|
|
|
registryStorageUser: kube.CephObjectStoreUser("registry") {
|
|
metadata+: {
|
|
namespace: "ceph-waw3",
|
|
},
|
|
spec: {
|
|
store: cfg.objectStorageName,
|
|
displayName: "docker-registry user",
|
|
},
|
|
},
|
|
}
|
|
}
|