From 529e181497476aaa0e1df222e2e07040da9a5418 Mon Sep 17 00:00:00 2001 From: Piotr Dobrowolski Date: Sat, 13 Feb 2021 19:44:37 +0100 Subject: [PATCH] app/matrix: appservice workers This change extracts appservice workers (deployed and tested) and prepares for federation sender workers extraction (still partially broken) Change-Id: I2d63fe44538ea2a7c5fd492f6ce119bc35a9eb03 Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1101 Reviewed-by: informatic Reviewed-by: q3k --- app/matrix/lib/matrix-ng.libsonnet | 6 ++ app/matrix/lib/synapse.libsonnet | 86 +++++++++++++++++++++++- app/matrix/matrix.hackerspace.pl.jsonnet | 9 +++ kube/redis.libsonnet | 8 ++- 4 files changed, 105 insertions(+), 4 deletions(-) diff --git a/app/matrix/lib/matrix-ng.libsonnet b/app/matrix/lib/matrix-ng.libsonnet index 1fc07e72..20c33e8a 100644 --- a/app/matrix/lib/matrix-ng.libsonnet +++ b/app/matrix/lib/matrix-ng.libsonnet @@ -219,6 +219,11 @@ local mediaRepo = import "./media-repo.libsonnet"; storageClassName: cfg.storageClassName, storageSize: "100Gi", initdbArgs: "--encoding='UTF8' --lc-collate='C' --lc-ctype='C'", + + opts: { + max_connections: "300", + shared_buffers: "80MB", + }, }, }, @@ -228,6 +233,7 @@ local mediaRepo = import "./media-repo.libsonnet"; appName: "synapse", storageClassName: cfg.storageClassName, password: { secretKeyRef: { name: "synapse", key: "redis_password" } }, + persistence: false, }, }, diff --git a/app/matrix/lib/synapse.libsonnet b/app/matrix/lib/synapse.libsonnet index 80ab8f0e..0bd4b74c 100644 --- a/app/matrix/lib/synapse.libsonnet +++ b/app/matrix/lib/synapse.libsonnet @@ -15,6 +15,9 @@ local kube = import "../../../kube/kube.libsonnet"; cas: { enable: false }, oidc: { enable: false }, + appserviceWorker: false, + federationWorker: false, + macaroonSecretKey: error "cfg.macaroonSecretKey needs to be set", registrationSharedSecret: error "cfg.registationSharedSecret needs to be set", workerReplicationSecret: error "cfg.workerReplicationSecret needs to be set", @@ -50,6 +53,16 @@ local kube = import "../../../kube/kube.libsonnet"; "/appservices/%s/registration.yaml" % [k] for k in std.objectFields(app.appservices) ], + + notify_appservices: cfg.appserviceWorker == false, + + # FIXME(informatic) Rolling out with federationWorkers = true breaks + # *some* federation, needs investigation... + #send_federation: cfg.federationWorker == false, + #federation_sender_instances: if cfg.federationWorker then [ + # "%s-%s" % [app.federationSenderWorker.deployment.metadata.name, idx] + # for idx in std.range(0, app.federationSenderWorker.deployment.spec.replicas) + #] else [], } + (if cfg.cas.enable then { cas_config: { enabled: true, @@ -119,6 +132,10 @@ local kube = import "../../../kube/kube.libsonnet"; exec python -m ${SYNAPSE_WORKER} --config-path /conf/homeserver.yaml --config-path /tmp/secrets.yaml --config-path /tmp/local.yaml ||| ], + resources: { + requests: { cpu: "300m", memory: "1Gi" }, + limits: { cpu: "1500m", memory: "2Gi" }, + }, ports_: { http: { containerPort: 8008 }, metrics: { containerPort: 9092 }, @@ -179,8 +196,21 @@ local kube = import "../../../kube/kube.libsonnet"; main: { deployment: app.SynapseWorker("synapse", "synapse.app.homeserver", kube.Deployment) { cfg+: { - # Main process doesn't need any configuration customization - localConfig: {} + localConfig: { + # Following configuration values need to cause master + # process restart. + notify_appservices: app.config.notify_appservices, + # send_federation: app.config.send_federation, + # federation_sender_instances: app.config.federation_sender_instances, + } + }, + spec+: { + strategy+: { + rollingUpdate: { + maxSurge: 0, + maxUnavailable: 1, + }, + }, }, }, svc: app.ns.Contain(kube.Service("synapse")) { @@ -207,6 +237,10 @@ local kube = import "../../../kube/kube.libsonnet"; x_forwarded: true, bind_addresses: ["::"], resources: [{ names: ["client", "federation"]}], + }, { + port: 9092, + type: "metrics", + bind_address: "0.0.0.0", }], }, }, @@ -281,6 +315,10 @@ local kube = import "../../../kube/kube.libsonnet"; x_forwarded: true, bind_addresses: ["::"], resources: [{ names: ["media"]}], + }, { + port: 9092, + type: "metrics", + bind_address: "0.0.0.0", }], }, }, @@ -289,4 +327,48 @@ local kube = import "../../../kube/kube.libsonnet"; target_pod:: app.mediaWorker.deployment.spec.template, }, }, + + appserviceWorker: if cfg.appserviceWorker then { + # Worker responsible for sending traffic to registered appservices + deployment: app.SynapseWorker("synapse-appservice", "synapse.app.appservice", kube.StatefulSet) { + cfg+: { + localConfig+: { + worker_listeners: [{ + type: "http", + port: 8008, + x_forwarded: true, + bind_addresses: ["::"], + resources: [{ names: [] }], + }, { + port: 9092, + type: "metrics", + bind_address: "0.0.0.0", + }], + }, + }, + }, + } else null, + + federationSenderWorker: if cfg.federationWorker then { + deployment: app.SynapseWorker("synapse-federation-sender", "synapse.app.federation_sender", kube.StatefulSet) { + cfg+: { + localConfig+: { + worker_listeners: [{ + type: "http", + port: 8008, + x_forwarded: true, + bind_addresses: ["::"], + resources: [{ names: [] }], + }, { + port: 9092, + type: "metrics", + bind_address: "0.0.0.0", + }], + }, + }, + spec+: { + replicas: 2, + }, + }, + } else null, } diff --git a/app/matrix/matrix.hackerspace.pl.jsonnet b/app/matrix/matrix.hackerspace.pl.jsonnet index 8d341c43..3af255cb 100644 --- a/app/matrix/matrix.hackerspace.pl.jsonnet +++ b/app/matrix/matrix.hackerspace.pl.jsonnet @@ -43,6 +43,15 @@ matrix { }, synapse+: { + cfg+: { + appserviceWorker: true, + federationWorker: false, + }, + + config+: { + federation_metrics_domains: ["matrix.org", "evolved.systems", "narupo.pl", "staging-matrix.inf.re"] + }, + genericWorker+: { deployment+: { spec+: { diff --git a/kube/redis.libsonnet b/kube/redis.libsonnet index 6c69d2b2..6cfe13b4 100644 --- a/kube/redis.libsonnet +++ b/kube/redis.libsonnet @@ -14,6 +14,9 @@ local kube = import "kube.libsonnet"; prefix: "", # if set, should be 'foo-' password: null, + # Persist data to disk + persistence: true, + image: "redis:5.0.4-alpine", resources: { requests: { @@ -65,8 +68,9 @@ local kube = import "kube.libsonnet"; image: cfg.image, args: [ "redis-server", - "--appendonly", "yes", - ] + (if cfg.password != null then ["--requirepass", "$(REDIS_PASSWORD)"] else []), + ] + + (if cfg.persistence then ["--appendonly", "yes"] else []) + + (if cfg.password != null then ["--requirepass", "$(REDIS_PASSWORD)"] else []), ports_: { client: { containerPort: 6379 }, },