app/mastodon: deploy

Change-Id: I88c104d1a8d5627355b01a8c48dc235635fca5ed Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1421 Reviewed-by: implr <implr@hackerspace.pl>
2022-11-17 19:30:05 +00:00 · 2022-11-17 19:30:05 +00:00 · 16842119d1
parent 08c32996e5
commit 16842119d1
7 changed files with 502 additions and 1 deletions
--- a/app/mastodon/README.md
+++ b/app/mastodon/README.md
@ -0,0 +1,14 @@
+Hackerspace Mastodon
+===
+
+Updating
+---
+
+1. Bump cfg.image
+2. `kubecfg update`
+3. Exec into a web container and `bundle exec rails db:migrate` to execute post-deployment migrations.
+
+Prod notes
+---
+
+Webfinger for hackerspace.pl is configured on boston-packets nginx, as that still fronts `hackerspace.pl`.
--- a/app/mastodon/kube/mastodon.libsonnet
+++ b/app/mastodon/kube/mastodon.libsonnet
@ -0,0 +1,326 @@
+local kube = import "../../../kube/kube.libsonnet";
+local postgres = import "../../../kube/postgres.libsonnet";
+local redis = import "../../../kube/redis.libsonnet";
+
+{
+    local app = self,
+    local cfg = app.cfg,
+
+    cfg:: {
+        namespace: error "cfg.namespace must be set",
+        # Domain as seen in the fediverse.
+        localDomain: error "cfg.localDomain must be set",
+        # Domain where the web interface is running. If different,
+        # localDomain's real server must be configured to forward
+        # /.well-known/webfinger to webDomain.
+        webDomain: cfg.localDomain,
+        images: {
+            mastodon: "tootsuite/mastodon:v4.0.2@sha256:21c20181a5d44ff553e9e8f7d8d2e53b2551cc8c7ac900760e056445b88e7438",
+        },
+        passwords: {
+            # generate however you like
+            postgres: error "cfg.secrets.postgres must be set",
+            # generate however you like
+            redis: error "cfg.secrets.redis must be set",
+        },
+        smtp: {
+            user: "mastodon",
+            from: "mastodon-noreply@hackerspace.pl",
+            # from mail server
+            password: error "cfg.smtp.password must be set",
+        },
+        secrets: {
+            # generate with podman run --rm -it tootsuite/mastodon:v4.0.2 bundle exec rake secret
+            keyBase: error "cfg.secrets.keyBase must be set",
+            # generate with podman run --rm -it tootsuite/mastodon:v4.0.2 bundle exec rake secret
+            otp: error "cfg.secrets.otp must be set",
+            # generate with podman run --rm -it tootsuite/mastodon:v4.0.2 bundle exec rake mastodon:webpush:generate_vapid_key
+            vapid: {
+                private: error "cfg.secrets.vapid.private must be set",
+                public: error "cfg.secrets.vapid.public must be set",
+            }
+        },
+        oidc: {
+            clientID: error "cfg.oidc.clientID must be set",
+            clientSecret: error "cfg.oidc.clientSecret must be set",
+        },
+        objectStorage: {
+            bucket: error "cfg.objectStorage.bucket must be set",
+            accessKeyId: error "cfg.objectStorage.accessKeyId must be set",
+            secretAccessKey: error "cfg.objectStorage.secretAccessKey must be set",
+        },
+
+        scaling: {
+            web: 1,
+            sidekiq: 1,
+        },
+    },
+
+    // Unified env var based config used for {web, streaming, sidekiq}.
+    // Sample available in https://github.com/mastodon/mastodon/blob/main/.env.production.sample
+    env:: {
+        LOCAL_DOMAIN: cfg.localDomain,
+        WEB_DOMAIN: cfg.webDomain,
+
+        // REDIS_PASS is not used directly by the apps, it's just used to embed
+        // a secret fragment into REDIS_URL.
+        REDIS_PASS: kube.SecretKeyRef(app.config, "redis-pass"),
+        REDIS_URL: "redis://:$(REDIS_PASS)@%s" % [app.redis.svc.host_colon_port],
+
+        DB_HOST: app.postgres.svc.host,
+        DB_USER: "mastodon",
+        DB_NAME: "mastodon",
+        DB_PASS: kube.SecretKeyRef(app.config, "postgres-pass"),
+        DB_PORT: "5432",
+
+        ES_ENABLED: "false",
+
+        SECRET_KEY_BASE: kube.SecretKeyRef(app.config, "secret-key-base"),
+        OTP_SECRET: kube.SecretKeyRef(app.config, "otp-secret"),
+
+        VAPID_PRIVATE_KEY: kube.SecretKeyRef(app.config, "vapid-private"),
+        VAPID_PUBLIC_KEY: kube.SecretKeyRef(app.config, "vapid-public"),
+
+        SMTP_SERVER: "mail.hackerspace.pl",
+        SMTP_PORT: "587",
+        SMTP_LOGIN: "mastodon",
+        SMTP_PASSWORD: kube.SecretKeyRef(app.config, "smtp-password"),
+        SMTP_FROM_ADDRESS: "mastodon-noreply@hackerspace.pl",
+
+        S3_ENABLED: "true",
+        S3_BUCKET: cfg.objectStorage.bucket,
+        AWS_ACCESS_KEY_ID: kube.SecretKeyRef(app.config, "object-access-key-id"),
+        AWS_SECRET_ACCESS_KEY: kube.SecretKeyRef(app.config, "object-secret-access-key"),
+        S3_HOSTNAME: "object.ceph-waw3.hswaw.net",
+        S3_ENDPOINT: "https://object.ceph-waw3.hswaw.net",
+
+        IP_RETENTION_PERIOD: "31556952",
+        SESSION_RETENTION_PERIOD: "31556952",
+
+        OIDC_ENABLED: "true",
+        OIDC_DISPLAY_NAME: "Use Warsaw Hackerspace SSO",
+        OIDC_ISSUER: "https://sso.hackerspace.pl",
+        OIDC_DISCOVERY: "false",
+        OIDC_SCOPE: "openid,profile:read",
+        OIDC_UID_FIELD: "uid",
+        OIDC_CLIENT_ID: cfg.oidc.clientId,
+        OIDC_REDIRECT_URI: "https://%s/auth/auth/openid_connect/callback" % [cfg.webDomain],
+        OIDC_SECURITY_ASSUME_EMAIL_IS_VERIFIED: "true",
+        OIDC_CLIENT_SECRET: kube.SecretKeyRef(app.config, "oidc-client-secret"),
+        OIDC_AUTH_ENDPOINT: "https://sso.hackerspace.pl/oauth/authorize",
+        OIDC_TOKEN_ENDPOINT: "https://sso.hackerspace.pl/oauth/token",
+        OIDC_USER_INFO_ENDPOINT: "https://sso.hackerspace.pl/api/1/userinfo",
+        OIDC_JWKS_URI: "https://sso.hackerspace.pl/.well-known/jwks.json",
+    },
+
+    namespace: kube.Namespace(cfg.namespace),
+    local ns = self.namespace,
+
+    postgres: postgres {
+        cfg+: {
+            namespace: cfg.namespace,
+            appName: "mastodon",
+            database: "mastodon",
+            username: "mastodon",
+            prefix: "waw3-",
+            password: kube.SecretKeyRef(app.config, "postgres-pass"),
+            storageClassName: "waw-hdd-redundant-3",
+            storageSize: "100Gi",
+        },
+    },
+
+    redis: redis {
+        cfg+: {
+            namespace: cfg.namespace,
+            appName: "mastodon",
+            storageClassName: "waw-hdd-redundant-3",
+            prefix: "waw3-",
+            password: kube.SecretKeyRef(app.config, "redis-pass"),
+        },
+    },
+
+    web: ns.Contain(kube.Deployment("web")) {
+        spec+: {
+            minReadySeconds: 10,
+            replicas: cfg.scaling.web,
+            template+: {
+                spec+: {
+                    initContainers_: {
+                        migrate: kube.Container("migrate") {
+                            image: cfg.images.mastodon,
+                            env_: app.env {
+                                SKIP_POST_DEPLOYMENT_MIGRATIONS: "true",
+                            },
+                            command: [
+                                "bundle", "exec",
+                                "rails", "db:migrate",
+                            ],
+                        },
+                    },
+                    containers_: {
+                        default: kube.Container("default") {
+                            image: cfg.images.mastodon,
+                            env_: app.env,
+                            command: [
+                                "bundle", "exec",
+                                "rails", "s", "-p", "3000",
+                            ],
+                            ports_: {
+                                web: { containerPort: 3000 },
+                            },
+                            readinessProbe: {
+                                httpGet: {
+                                    path: "/health",
+                                    port: "web",
+                                },
+                                failureThreshold: 10,
+                                periodSeconds: 5,
+                            },
+                            resources: {
+                                requests: {
+                                    cpu: "250m",
+                                    memory: "1024M",
+                                },
+                                limits: {
+                                    cpu: "1",
+                                    memory: "1024M",
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    },
+
+    sidekiq: ns.Contain(kube.Deployment("sidekiq")) {
+        spec+: {
+            replicas: cfg.scaling.sidekiq,
+            minReadySeconds: 10,
+            template+: {
+                spec+: {
+                    containers_: {
+                        default: kube.Container("default") {
+                            image: cfg.images.mastodon,
+                            env_: app.env,
+                            command: [
+                                "bundle", "exec",
+                                "sidekiq",
+                            ],
+                            resources: {
+                                requests: {
+                                    cpu: "250m",
+                                    memory: "1024M",
+                                },
+                                limits: {
+                                    cpu: "1",
+                                    memory: "1024M",
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    },
+
+    streaming: ns.Contain(kube.Deployment("streaming")) {
+        spec+: {
+            minReadySeconds: 10,
+            template+: {
+                spec+: {
+                    containers_: {
+                        default: kube.Container("default") {
+                            image: cfg.images.mastodon,
+                            env_: app.env {
+                                "STREAMING_CLUSTER_NUM": "1",
+                            },
+                            command: [
+                                "node", "./streaming",
+                            ],
+                            ports_: {
+                                web: { containerPort: 4000 },
+                            },
+                            readinessProbe: {
+                                httpGet: {
+                                    path: "/api/v1/streaming/health",
+                                    port: "web",
+                                },
+                                failureThreshold: 1,
+                                periodSeconds: 5,
+                            },
+                            resources: {
+                                requests: {
+                                    cpu: "250m",
+                                    memory: "1024M",
+                                },
+                                limits: {
+                                    cpu: "1",
+                                    memory: "1024M",
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    },
+
+    svcWeb: ns.Contain(kube.Service("web")) {
+        target_pod: app.web.spec.template,
+    },
+
+    svcStreaming: ns.Contain(kube.Service("streaming")) {
+        target_pod: app.streaming.spec.template,
+    },
+
+
+    ingress: ns.Contain(kube.Ingress("mastodon")) {
+        metadata+: {
+            annotations+: {
+                "kubernetes.io/tls-acme": "true",
+                "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
+                "nginx.ingress.kubernetes.io/proxy-body-size": "0",
+            },
+        },
+        spec+: {
+            tls: [
+                {
+                    hosts: [cfg.webDomain],
+                    secretName: "mastodon-ingress-tls",
+                },
+            ],
+            rules: [
+                {
+                    host: cfg.webDomain,
+                    http: {
+                        paths: [
+                            { path: "/", backend: app.svcWeb.name_port },
+                            { path: "/api/v1/streaming", backend: app.svcStreaming.name_port },
+                        ],
+                    },
+                },
+            ],
+        },
+    },
+
+    config: ns.Contain(kube.Secret("config")) {
+        data_: {
+            "postgres-pass": cfg.passwords.postgres,
+            "redis-pass": cfg.passwords.redis,
+
+            "secret-key-base": cfg.secrets.keyBase,
+            "otp-secret": cfg.secrets.otp,
+
+            "vapid-private": cfg.secrets.vapid.private,
+            "vapid-public": cfg.secrets.vapid.public,
+
+            "smtp-password": cfg.smtp.password,
+
+            "object-access-key-id": cfg.objectStorage.accessKeyId,
+            "object-secret-access-key": cfg.objectStorage.secretAccessKey,
+
+            "oidc-client-secret": cfg.oidc.clientSecret,
+        },
+    },
+}
--- a/app/mastodon/kube/prod.jsonnet
+++ b/app/mastodon/kube/prod.jsonnet
@ -0,0 +1,19 @@
+local mastodon = import "mastodon.libsonnet";
+local secrets = import "secrets/plain/prod.libsonnet";
+
+mastodon {
+    cfg+: secrets {
+        namespace: "mastodon-hackerspace-prod",
+        localDomain: "hackerspace.pl",
+        webDomain: "social.hackerspace.pl",
+
+        objectStorage+: {
+            bucket: "mastodon-prod",
+        },
+
+        scaling: {
+            web: 5,
+            sidekiq: 5,
+        },
+    },
+}
--- a/app/mastodon/kube/qa.jsonnet
+++ b/app/mastodon/kube/qa.jsonnet
@ -0,0 +1,13 @@
+local mastodon = import "mastodon.libsonnet";
+local secrets = import "secrets/plain/qa.libsonnet";
+
+mastodon {
+    cfg+: secrets {
+        namespace: "mastodon-hackerspace-qa",
+        localDomain: "social-qa-2.hackerspace.pl",
+
+        objectStorage+: {
+            bucket: "mastodon-qa",
+        },
+    },
+}
--- a/app/mastodon/kube/secrets/cipher/prod.libsonnet
+++ b/app/mastodon/kube/secrets/cipher/prod.libsonnet
@ -0,0 +1,53 @@
+-----BEGIN PGP MESSAGE-----
+
+hQEMAzhuiT4RC8VbAQf/b2CAX8GX4w3ZPQksmjZVJqm8HkCNoljF88uFMHfwu7CN
+aQWjXkIGGxQ3A2kei/2H0AVF5GTvLN+Od6OJmxYnYuuGwAJE93L58Qc6MhF+At0Z
+9lqrH13AWZbg2QhkklmdkwNdb5rrLaV9PpXFRRJvj5Wf8cQwl1EDVQ/owXVebHuR
+UokPCRzsJwu8BD8861bMLbSPF9RNI7g4arFjtEo9rF/rlxW39BgWyB0MhkFNr6en
+AaaC0LdAXbsUVzur9WDRtIW8N2JokLR3CKd6odw65ilnE6rkMmBRGu6mJeSsaEOw
+UHiLOAI7g4x15WRAdaCsiSGRA+FF+RdQzeBpvaNL+IUBDANcG2tp6fXqvgEH/33D
+OubfxaMrrVZuO0xv7EQS3hWjBHgpWKhg6k8dSeEpWJ2vnwl0W/XNGmL0CaptIusL
+cTbAv12iYocyCk9cvcFqxPnBAvlkP/+R/7c5bLrUnyTBKL8D1lq2UnIfvGkt7+XT
+2lKB5yhbBJtrkLmaHDLeLhHkUIY0gaRZPuHM1ndEC+rwWlSD7gdHRU4pShqsZKiK
+rUrT3GFatf1T+4YNfuYoJFaOwGzzC9pfouOAOVp8RjMfVZlU3e+jGNd5VLrFyimj
+uG7tkHDgqZij8Wypggi7x3uSBrCO4ugmOsSJ0Nj3HODbJtbXe9uwGBhlG49LmyMZ
+H0XaFXb+R7qiIwe2z6eFAgwDodoT8VqRl4UBD/9taTxGIsQHMH1zsfZZiosqfYWc
+2TGif9ikX1+KqvXO775S5slrJNvirEDNI/IfHtLYBl/MqOZWK6M/ucNQ7p6EUnjE
+j1Eo/7x/Fd7U5/Y/1/hPCbiSZTt2QWfoCZj2fwjC0UenE2EpDWafZXh9zA4enfL+
+GBbsxCAiVvEl5IiiEr9bSUEG4d2tCUhF3wNVKKpjW4N/060ASCTW6BKgay/1wfHl
+DltNG2sPeeIo63TyEBfvzu8MfdUay4feWj1B1giWkTLqor40YStCXha6i9sowY5T
+j+m1jWp9ovXbYTjxcUrGVZg71Qb35Biz8bDds35aqG9ueNAJx4NXOXqoxtCRLUOs
+80Px86ylzJ6Ho3qwVdCocqLKs5aBs99/Ak6+uv2JISy4VGfjZV1iBC0sduRZqTqc
+PTkiSVVmvfcIwB8msD/nQuN6Jwi7kSqbeG6R4migxxLUgwvLlIB/mops8EaKDptU
+gwlIaICTPMXHs/hH8L+GGvh90ECl2G0gg8Q1BQVDCwOuH/0+jticiKDqQEwbxYrT
+L0ncr8IpD0mNF1bnp2Qf8z2m68t5gbTCyRyHIr9nOOFhnj9zdNPssEvYZKY7ABxJ
+OiabkOTX5CGiaLdHaTyvOb/fRQexAAz8bK/wn2k7AKKtOfPGosuRWtkH3xeuKSyK
+5tjBRzVul8VFcSft5oUCDAPiA8lOXOuz7wEP/1glyDoyapw82lgCpNTtj5cGvh2z
+bN4NWeGVCEJo9ZvQo4w5cvXJ6PLQnzkgQ3+i6fvIP8FPEMDSTLFU7PazNTe1lVn5
+ZeASPYDoOzYBrHD+jLCQm94BRR+M9RklThsrRJ04y3jXaybLZYn3GjgPbnmMm0F7
+I1fniMrx5AgQwjAH6KKelzkkeZI+5poM4dhUH6LFj7KSARCHIKbql6LRTvkQy7fv
+BDLjpCV2rsWA3tB42FF7V6vvKRVp4NLQwxuw5vnUAkkDyIjVPzevYgVgA0pZpQu0
+gaFPaB4hfsPvsH+ur5mBrCNJ2i0M66jtj+dAnc5eaxnJRpsHbngyY09zWo4r75d0
+whUW1V91y7fBZq2aVVWnS+9KjsMl6bmvdHtq92JVnkd5411dbu3cDvrzATrg6GcF
+A7UNd1a7AIl6oLYbxox7ZUP57kAHdcHwL1d5Ge6hlGUWrsTIU4ETlx2y/jm2cPh6
+j6pUxcQ6Q19lZDJSndd7UJMNc60svtma96v5WsuxF2nLrobT5pco6otwS5Znt/ON
+c7C2tFuYwKy+KAKUoAgvqtEcyDTlJrvQjdUrlvVhRqSnU7Nn0I//cA+00Uvgtl8b
+/TVp4gbwvAKleB6ecqWv8kr77OBAnmC0yPnirZToJmM69ZbN2REvKNTylXUmbUIT
+OPV9B/YPRwMxkKB70ukBdWQd+XjKUzLaigHcEuR5yE598ocAA0t9su9uzli3lutI
+woK2/FjxDottjQSMrAT1OXo5A5fEeEBd5l77Sqrk4pXM/w7+mDuDd/uTD0gJjGeD
+10X6znukIRTRlge94UgDlgIGgPrHYgvwplgoLKh9w4h9pPPky8uUiC84XwYXhwxQ
+fJqyUl0XwaPvR9R7taTxxAhVq91IIWUB0gMRHFX5UqlltoSAt3Y8CemcLJ5ThI7C
+OSJjQiFbtworRRWrRNGsNA7apgUvPGA1wCUm+nNrbmPENGvqwaRic0OPvBFRomiA
+E/bOmabksF5AWvFWqaVLWH5Th1MVQYJ12ml3rUhRGBhi1q3zlbXi3AyHTwzlKJRk
+WJYhavJk1viwbRG1sGoakJch3d6OUGrk3MJqH0kTz7mSM0q+OePgcxMKMVJ+/m1H
+SMIfHfglMzDsMKPrmRhIj7pm99Znfp0tALeRpwNtiXM+MtDHg6VBW5cAqvPl10Bg
+8VetxFwkiuhSuTsCsxuKQ5TMgf7OewkO8RKeBfZrTsTw+8JCGgV3xxfEHni9d/jE
+5cGGP/mdjntLx8WHujfxUVJXXVwoSh8xj/zHGfnGpmq3gOufesLvqFydIDimPJNH
+3riLIY8c9KiX64VOon2yHsq5EimlOQ7RIgd3pSa3mPHXdLlq2VsLECVZ6Y23OcAN
+0oA20HSBmhw1sRQ7l00zEcSpMcm4db3+dc1rW2cP846RqjpflO+Jm3GEZBoDvNYQ
+0iyItQ6ovf+zM9i/Zbr/sMsjXitsXKTGavZQriZQYYBCvbify0c6pC3tnOLNxcci
+o7lE01uDDzA3U2OS1Ag3JjtZFhes6cH/mNhiA7lLCb+0nVHwaWgDOtNmRGRdmOyG
+zfEWfgqk4kyfinH/ngmVEypgtNZaDD8fMehfEogGBXaZGRA+/7N25PqelIpi0W1A
+7vgdC75KuC00VB5Liw==
+=I1JW
+-----END PGP MESSAGE-----
--- a/app/mastodon/kube/secrets/cipher/qa.libsonnet
+++ b/app/mastodon/kube/secrets/cipher/qa.libsonnet
@ -0,0 +1,53 @@
+-----BEGIN PGP MESSAGE-----
+
+hQEMAzhuiT4RC8VbAQf+OM4E3rq/8zzhNF7ai2lVF1Qv39T8mvayka1ZKJa9Nsjp
+aeTb6t6ZnZZpZjjTft7q7h1cFgvzysZhjAi+nTqL/dqh2+1HYS8BqIjTlsKODK3h
+xKrwgMdVjNkjhNZoaa9Or4waD3ROVgyGQKvrBtgqxnZTR69w0beq5kFh9bLmzt7+
+XW3PjNBHRLyFJ+59kBbjscTx5AIZBeBwk2pknDYKodWAC4njHs0WZqLpNHn98aZo
+LXuTXYo3ufwCxaMo+rNrRlz/jK5yujbJ0fPYlBRyBLxUERVcFg9OjIAj2GywlctJ
+EG8KuC+juTl5Gna+fxFOJ1wNunPHBiy4NAg6h3DRfIUBDANcG2tp6fXqvgEH/30F
+4WAl4g1W4F1//HqjWw6wXg6gSHr5LnHeL7cMWYgqQPMxFRtXssjqZfFXdto/F7zg
+G5EtxUo1dXst008KPl1FyTCibQdGEGcv9dNBhv086r7u3sZw+4ty426pVV0Fy4ea
+MvciP6psMAKGbmLZxirGLTUCX2qswG+qKFAlEc1k69DNYO5yquxgzXCZxaObrQ5k
+CkRHWsUU5foWsJ05e2zEN8TjAUJYf/Af13JOiEdGaKkrWtSsnQ0yA941G+eVpj83
+lLlk64Lw4uPXRuszdo4hMqy/NT4oSrXpVpDV6/ZyXfi/xLHLDUL1fLJNCKYFPMN
+gGSvFgUkBhooUtF619WFAgwDodoT8VqRl4UBEACVfftCs45cd8sLYGW3Wvuqp6J2
+aJCTt2Q5t60ZU+MMabO+0sE5l/tdQDvE+cDzZU/g+OFvj6emwbEM1pH7lEbgUoga
+8rb6ULXTHhIraYQC7Dzsz8jwbMJYve+ho0Gxu0cEeuiPYU5EH5/t1X8m3QEHKyrn
+J9mJkCkb7sy4UAowTIJISQ2HG5c9DA/NgVQiBDb4UMU7EAAPyRkf0RZN3R40XpGF
+3rmT/uY5BqgZPHplx/l1Q6Beit22B38M/9fjd7FR6u5NNKTmyMYndInq5iQkDE3E
+SZe5l3ja20CgTAk6C7N7EYqyrLNjo5tqc783I1SgREXhB02rXlYYN5VQy/0BCBBP
+/Rh0l3LCb27uYjeS6OlGa6VZJ0EOmJYYyVx8baPogHh2oHxE2Pm59PNG2s8XbaTk
+KB9j2wxbR2PZ5yuWV0Nbs4eMgBlNSE9qvThs5RX+dujeKsicZQK8znqm9lMHfZy0
+soQloej36RB9zB3U1e1l5rV+cW4w1aH4WFKPX/cfy4u7bfzMRDAK0aBBUBbgb7L0
+zxjUZ9aUvyrCkT9MtLS3FFjOwXoFhV8nnNBsnLhnF00f3675IGSxqW8SiTaP8l0Y
+HmGwBpgx5yffyb5mJOJBY2Asqy5AvhBn7MoAKuajdYIZZR4QRqUcAKd2uEUy8x4R
+udPrrNQ8kRYdZQ7M04UCDAPiA8lOXOuz7wEP/Rsq9eSc29wNOQuVzGQXdpCSr/hF
+ocWL3x0C+21/F1S7p9rqWjfjbYbMUkSHI5I4K2UP+KLudWpJgSziOxRMK8A9Tbz1
+CpqozkWx7ZTaU4lDUpDiM5yBCL/YuZyVid/I1mawIjbvG7wlFAYTYWDgNMvKscjo
+QphDK/ugGqcdO9DaWE22J4KzR/wMSxCJzoAmLe7WOWN836sUYuB8McUEzN+J+QI3
+WtZQ7Rjdo59mt1/fYewrmqTdBkwBobfCKYEk1hBfnKxjBkYxNf9MfibkOvL1ISHL
+5OK4KUZtVQeKh4gvlKwT0t9K3I6khA5A8ZxsM+mgmuzZrFFfyJlR35pXepFajwJU
+FbREiTi7wEOG3Y/Z2wmSEWhgZmX1EbnerCTzLyKbpYgEveYGEBj5Y8CoieJpGEJa
+SajFs69NqdhzU4Af0zvqqLxgDEaJO5syR3Ndfl6rDU1IfvSIGacMJxQRrZvl2eoO
+18/UnYsRda7WrLoSV7DgyXzr34OmfbBh3eE0zlSegWR/rErhyH7j1jbR/+DhmKr9
+jg1uY6BKcT7+/N42u444Fg+nD4Rr+UYsovgp1aM6mwG5yFFyM3jsZQyWQu/VRB/T
+m89sl+u6kr8o1soRei51pw2Ai706Pdj8eOvqLMh1UTwAafJIiq8oNJy3kbZKJSGH
+DDOaiwI7rGmLUWQH0ukBavUSpESe6bdK1D8OJe+o+GJ/tHYERWvAIDWLlpeVJYbK
+iBxLcIkSkb4pNDrXyJviWPD70Iq9dz+9+lX2OB0gJiK0wyqF7BPtrAhlkMQdCkbV
+dxsCLvWMzFSf2bsShO5USKyhQDANbWOAxqqstAjlwH+8nwuSxx91wUzRVc72SXoe
+pmyaUh32LSU/Va40UIilrBYynnHye9BO4hHsUqS3HOWzd+em26fz8zwlzLqzZku5
+6Zz1qvxHEFxVafsb9GO+/a4XLlenOsGXX0wsuoFXvo8V4nWvnqaE9G0wRKlg/r88
+DdoxbjaJeJdr2xgLFzmRJ3m/OP18Y8wFm6+QJEvEZfjUIAHjDh9I9E2aB0+XuLRQ
+HKKuVvAQ5tsGI51saUxB6fQD+Lhs3KUCZbb0deFhNQnZL0d6luld476dlR8bOjjY
+We+95uaE0TixMgH66Ja6tgpdvqREMIx6UjlDxPuCR+Ww590JDGiPb9/pieC5nfmy
+Ct7rhqZBO9FB8UohATRkeqKoj1VRVxJU6ePJtR8YnZA5k/C4H9v7FswRGgqEasuw
+4zU2FVG5qrQK76q0okYOB4PPEFl0bU26q0ILz1EoieXkfR8xmgXB0JS5Uxd4FwFU
+mmq0wLL5bYo4FPKKVN85/KmziMlheRxbWiJqMb3o7zX1Zx82t3q7AIbd+ULOWsAP
+Yg+0+skKOUE5+379nnKOAzLQ1v03tvxclbbyAhpXHcZOBXWlNP4u0F6A01u/TBDd
+7t2RUCYH+w8fa/riOsl+TEg6D66X3ignZrIN4Hml6bPNBg3KJlqT3sdv97TrIeLZ
+IVuE0gY/kJZ0H4NO4PkIOLwFG7MvXh7t+1k5sfkho8Oles8ki2bqJKx8Knpp/DhT
+zIyo9Y+EMYfvnA0JH69Z5JjUDekCEgMx/Riv/9nvI/Vd/3Ql8g0m6aObouwWeW7H
+itB4qx0pTjKb+BHZQfUo
+=pemS
+-----END PGP MESSAGE-----
--- a/cluster/kube/k0.libsonnet
+++ b/cluster/kube/k0.libsonnet
@ -109,7 +109,7 @@ local rook = import "lib/rook.libsonnet";
            waw3: rook.Cluster(k0.cluster.rook, "ceph-waw3") {
                spec: {
                    mon: {
-                        count: 1,
+                        count: 3,
                        allowMultiplePerNode: false,
                    },
                    resources: {
@ -350,6 +350,27 @@ local rook = import "lib/rook.libsonnet";
                        displayName: "informatic",
                    },
                },
+                # mastodon qa and prod
+                mastodonWaw3: {
+                    qa: kube.CephObjectStoreUser("mastodon-qa") {
+                        metadata+: {
+                            namespace: "ceph-waw3",
+                        },
+                        spec: {
+                            store: "waw-hdd-redundant-3-object",
+                            displayName: "mastodon-qa",
+                        },
+                    },
+                    prod: kube.CephObjectStoreUser("mastodon-prod") {
+                        metadata+: {
+                            namespace: "ceph-waw3",
+                        },
+                        spec: {
+                            store: "waw-hdd-redundant-3-object",
+                            displayName: "mastodon-prod",
+                        },
+                    },
+                },
            },
        },

@ -413,6 +434,8 @@ local rook = import "lib/rook.libsonnet";
                        { namespace: "redmine", dns: "xn--137h.hswaw.net" },
                        { namespace: "speedtest", dns: "speedtest.hackerspace.pl" },
                        { namespace: "sso", dns: "sso.hackerspace.pl" },
+                        { namespace: "mastodon-hackerspace-qa", dns: "social-qa-2.hackerspace.pl" },
+                        { namespace: "mastodon-hackerspace-prod", dns: "social.hackerspace.pl" },

                        { namespace: "ceph-waw3", dns: "ceph-waw3.hswaw.net" },
                        { namespace: "ceph-waw3", dns: "object.ceph-waw3.hswaw.net" },