app/matrix: appservice workers

This change extracts appservice workers (deployed and tested) and prepares for
federation sender workers extraction (still partially broken)

Change-Id: I2d63fe44538ea2a7c5fd492f6ce119bc35a9eb03
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1101
Reviewed-by: informatic <informatic@hackerspace.pl>
Reviewed-by: q3k <q3k@hackerspace.pl>

app/matrix/lib/matrix-ng.libsonnet

@@ -219,6 +219,11 @@ local mediaRepo = import "./media-repo.libsonnet";
             storageClassName: cfg.storageClassName,
             storageSize: "100Gi",
             initdbArgs: "--encoding='UTF8' --lc-collate='C' --lc-ctype='C'",
+
+            opts: {
+                max_connections: "300",
+                shared_buffers: "80MB",
+            },
         },
     },
 
@@ -228,6 +233,7 @@ local mediaRepo = import "./media-repo.libsonnet";
             appName: "synapse",
             storageClassName: cfg.storageClassName,
             password: { secretKeyRef: { name: "synapse", key: "redis_password" } },
+            persistence: false,
         },
     },
 

app/matrix/lib/synapse.libsonnet

@@ -15,6 +15,9 @@ local kube = import "../../../kube/kube.libsonnet";
         cas: { enable: false },
         oidc: { enable: false },
 
+        appserviceWorker: false,
+        federationWorker: false,
+
         macaroonSecretKey: error "cfg.macaroonSecretKey needs to be set",
         registrationSharedSecret: error "cfg.registationSharedSecret needs to be set",
         workerReplicationSecret: error "cfg.workerReplicationSecret needs to be set",
@@ -50,6 +53,16 @@ local kube = import "../../../kube/kube.libsonnet";
             "/appservices/%s/registration.yaml" % [k]
             for k in std.objectFields(app.appservices)
         ],
+
+        notify_appservices: cfg.appserviceWorker == false,
+
+        # FIXME(informatic) Rolling out with federationWorkers = true breaks
+        # *some* federation, needs investigation...
+        #send_federation: cfg.federationWorker == false,
+        #federation_sender_instances: if cfg.federationWorker then [
+        #    "%s-%s" % [app.federationSenderWorker.deployment.metadata.name, idx]
+        #    for idx in std.range(0, app.federationSenderWorker.deployment.spec.replicas)
+        #] else [],
     } + (if cfg.cas.enable then {
         cas_config: {
             enabled: true,
@@ -119,6 +132,10 @@ local kube = import "../../../kube/kube.libsonnet";
                                     exec python -m ${SYNAPSE_WORKER} --config-path /conf/homeserver.yaml --config-path /tmp/secrets.yaml --config-path /tmp/local.yaml
                                 |||
                             ],
+                            resources: {
+                                requests: { cpu: "300m", memory: "1Gi" },
+                                limits: { cpu: "1500m", memory: "2Gi" },
+                            },
                             ports_: {
                                 http: { containerPort: 8008 },
                                 metrics: { containerPort: 9092 },
@@ -179,8 +196,21 @@ local kube = import "../../../kube/kube.libsonnet";
     main: {
         deployment: app.SynapseWorker("synapse", "synapse.app.homeserver", kube.Deployment) {
             cfg+: {
-                # Main process doesn't need any configuration customization
+                localConfig: {
-                localConfig: {}
+                    # Following configuration values need to cause master
+                    # process restart.
+                    notify_appservices: app.config.notify_appservices,
+                    # send_federation: app.config.send_federation,
+                    # federation_sender_instances: app.config.federation_sender_instances,
+                }
+            },
+            spec+: {
+                strategy+: {
+                    rollingUpdate: {
+                        maxSurge: 0,
+                        maxUnavailable: 1,
+                    },
+                },
             },
         },
         svc: app.ns.Contain(kube.Service("synapse")) {
@@ -207,6 +237,10 @@ local kube = import "../../../kube/kube.libsonnet";
                         x_forwarded: true,
                         bind_addresses: ["::"],
                         resources: [{ names: ["client", "federation"]}],
+                    }, {
+                        port: 9092,
+                        type: "metrics",
+                        bind_address: "0.0.0.0",
                     }],
                 },
             },
@@ -281,6 +315,10 @@ local kube = import "../../../kube/kube.libsonnet";
                         x_forwarded: true,
                         bind_addresses: ["::"],
                         resources: [{ names: ["media"]}],
+                    }, {
+                        port: 9092,
+                        type: "metrics",
+                        bind_address: "0.0.0.0",
                     }],
                 },
             },
@@ -289,4 +327,48 @@ local kube = import "../../../kube/kube.libsonnet";
             target_pod:: app.mediaWorker.deployment.spec.template,
         },
     },
+
+    appserviceWorker: if cfg.appserviceWorker then {
+        # Worker responsible for sending traffic to registered appservices
+        deployment: app.SynapseWorker("synapse-appservice", "synapse.app.appservice", kube.StatefulSet) {
+            cfg+: {
+                localConfig+: {
+                    worker_listeners: [{
+                        type: "http",
+                        port: 8008,
+                        x_forwarded: true,
+                        bind_addresses: ["::"],
+                        resources: [{ names: [] }],
+                    }, {
+                        port: 9092,
+                        type: "metrics",
+                        bind_address: "0.0.0.0",
+                    }],
+                },
+            },
+        },
+    } else null,
+
+    federationSenderWorker: if cfg.federationWorker then {
+        deployment: app.SynapseWorker("synapse-federation-sender", "synapse.app.federation_sender", kube.StatefulSet) {
+            cfg+: {
+                localConfig+: {
+                    worker_listeners: [{
+                        type: "http",
+                        port: 8008,
+                        x_forwarded: true,
+                        bind_addresses: ["::"],
+                        resources: [{ names: [] }],
+                    }, {
+                        port: 9092,
+                        type: "metrics",
+                        bind_address: "0.0.0.0",
+                    }],
+                },
+            },
+            spec+: {
+                replicas: 2,
+            },
+        },
+    } else null,
 }

app/matrix/matrix.hackerspace.pl.jsonnet

@@ -43,6 +43,15 @@ matrix {
     },
 
     synapse+: {
+        cfg+: {
+            appserviceWorker: true,
+            federationWorker: false,
+        },
+
+        config+: {
+            federation_metrics_domains: ["matrix.org", "evolved.systems", "narupo.pl", "staging-matrix.inf.re"]
+        },
+
         genericWorker+: {
             deployment+: {
                 spec+: {

kube/redis.libsonnet

@@ -14,6 +14,9 @@ local kube = import "kube.libsonnet";
         prefix: "", # if set, should be 'foo-'
         password: null,
 
+        # Persist data to disk
+        persistence: true,
+
         image: "redis:5.0.4-alpine",
         resources: {
             requests: {
@@ -65,8 +68,9 @@ local kube = import "kube.libsonnet";
                             image: cfg.image,
                             args: [
                                 "redis-server",
-                                "--appendonly", "yes",
+                            ]
-                            ] + (if cfg.password != null then ["--requirepass", "$(REDIS_PASSWORD)"] else []),
+                            + (if cfg.persistence then ["--appendonly", "yes"] else [])
+                            + (if cfg.password != null then ["--requirepass", "$(REDIS_PASSWORD)"] else []),
                             ports_: {
                                 client: { containerPort: 6379 },
                             },