diff --git a/app/matrix/lib/matrix-ng.libsonnet b/app/matrix/lib/matrix-ng.libsonnet index 941c46a4..3455b117 100644 --- a/app/matrix/lib/matrix-ng.libsonnet +++ b/app/matrix/lib/matrix-ng.libsonnet @@ -28,6 +28,51 @@ # * https://{homeserver}/_synapse/oidc/callback is added to allowed callback URLs list # * openid scope is enabled for configured client # +# In order to deploy matrix-media-repo as a replacement for synapse built-in +# media workers the following steps need to be carried out: +# +# 1. Generate password and bootstrap extra postgres user +# pwgen 32 1 > secrets/plain/media-repo-$ns-postgres +# echo "create database mediarepo; create user mediarepo with password '$(cat secrets/plain/media-repo-$ns-postgres)'; grant all privileges on database mediarepo to mediarepo;" | kubectl -n $ns exec -it deploy/waw3-postgres psql +# secretstore sync secrets +# +# 2. Fetch Ceph RGW credentials +# kubectl get secrets -n ceph-waw3 rook-ceph-object-user-waw-hdd-redundant-3-object-$ns -o json | jq '.data|map_values(@base64d)' > secrets/plain/media-repo-$ns-ceph.json +# secretstore sync secrets +# +# 3. Create an apropriate bucket using s3cmd +# s3cmd --access_key="$(jq -r '.AccessKey' secrets/plain/media-repo-$ns-ceph.json)" --secret_key="$(jq -r '.SecretKey' secrets/plain/media-repo-$ns-ceph.json)" --host=object.ceph-waw3.hswaw.net --host-bucket=object.ceph-waw3.hswaw.net mb s3://media-repo-$ns +# +# 4. Add relevant configuration overrides in cfg.mediaRepo key for your +# deployment configuration file: +# +# mediaRepo+: { +# enable: true, +# route: false, +# s3+: { +# endpoint: std.strReplace((import "secrets/plain/media-repo-$ns-ceph.json").Endpoint, "http://", ""), +# accessKey: (import "secrets/plain/media-repo-$ns-ceph.json").AccessKey, +# secretKey: (import "secrets/plain/media-repo-$ns-ceph.json").SecretKey, +# bucketName: "media-repo-$ns", +# region: "eu", +# }, +# db+: { +# password: std.strReplace(importstr "secrets/plain/media-repo-$ns-postgres", "\n", ""), +# }, +# }, +# +# 5. Additionally, when migrating from already deployed synapse media worker the +# following command needs to be run in order to import existing media files: +# kubectl -n $ns exec deploy/media-repo -- import_synapse -baseUrl http://synapse-media:8008 -dbHost waw3-postgres -dbPassword "$(kubectl -n $ns get secret synapse -o json | jq -r '.data.postgres_password | @base64d')" -config /config/config.yaml -serverName 'SERVER_NAME' +# +# 6. After migrating data over from native synapse media worker storage traffic +# can be rerouted to matrix-media-repo by switching cfg.mediaRepo.route flag +# to true +# +# 7. Run import step #5 again to make sure no media were left missing in old +# media worker deployment - import operation is indempotent and can be ran +# against a synapse media worker that's not handling user traffic anymore. +# # Sequencing appservices is fun. The appservice needs to run first (for # instance, via a bootstrap job), and on startup it will spit out a # registration file. This registration file then needs to be fed to synapse - @@ -49,6 +94,7 @@ local riot = import "./riot.libsonnet"; local cas = import "./cas.libsonnet"; local wellKnown = import "./wellknown.libsonnet"; local synapse = import "./synapse.libsonnet"; +local mediaRepo = import "./media-repo.libsonnet"; { local app = self, @@ -68,6 +114,7 @@ local synapse = import "./synapse.libsonnet"; appserviceIRC: "matrixdotorg/matrix-appservice-irc:release-v0.29.0", appserviceTelegram: "dock.mau.dev/tulir/mautrix-telegram@sha256:c6e25cb57e1b67027069e8dc2627338df35d156315c004a6f2b34b6aeaa79f77", wellKnown: "registry.k0.hswaw.net/q3k/wellknown:1611960794-adbf560851a46ad0e58b42f0daad7ef19535687c", + mediaRepo: "turt2live/matrix-media-repo:v1.2.8", }, # OpenID Connect provider configuration. @@ -119,6 +166,33 @@ local synapse = import "./synapse.libsonnet"; # Serve /.well-known/matrix configuration endpoints required when using # cfg.webDomain directly as mxid. wellKnown: false, + + # matrix-media-repo S3-based media storage container + mediaRepo: { + enable: false, + + # Route /_matrix/media/ endpoints to matrix-media-repo. Set this + # to true after migrating media files to matrix-media-repo. + route: false, + + s3: { + endpoint: error "mediaRepo.s3.endpoint needs to be set", + accessKey: error "mediaRepo.s3.accessKey needs to be set", + secretKey: error "mediaRepo.s3.secretKey needs to be set", + bucketName: error "mediaRepo.s3.bucketName needs to be set", + region: error "mediaRepo.s3.region needs to be set", + }, + + db: { + username: "mediarepo", + password: error "mediaRepo.db.password needs to be set", + database: "mediarepo", + host: "waw3-postgres", + }, + }, + + # List of administrative users MXIDs (used in matrix-media-repo only) + admins: [], }, # DEPRECATED: this needs to be removed in favor of namespace.Contain() in @@ -184,6 +258,21 @@ local synapse = import "./synapse.libsonnet"; }, } else {}, + mediaRepo: if cfg.mediaRepo.enable then mediaRepo { + ns: app.namespace, + cfg+: { + image: cfg.images.mediaRepo, + + homeservers: [ + {name: cfg.serverName, csApi: "https://" + cfg.webDomain} + ], + admins: cfg.admins, + + s3: cfg.mediaRepo.s3, + db: cfg.mediaRepo.db, + }, + } else {}, + synapse: synapse { ns: app.namespace, postgres: app.postgres3, @@ -231,7 +320,7 @@ local synapse = import "./synapse.libsonnet"; for path in app.synapse.genericWorker.paths ] + [ { path: "/", backend: app.riot.svc.name_port }, - { path: "/_matrix/media/", backend: app.synapse.mediaWorker.svc.name_port }, + { path: "/_matrix/media/", backend: if cfg.mediaRepo.route then app.mediaRepo.svc.name_port else app.synapse.mediaWorker.svc.name_port }, { path: "/_matrix/", backend: app.synapse.main.svc.name_port }, # Used by OpenID Connect login flow diff --git a/app/matrix/lib/media-repo.libsonnet b/app/matrix/lib/media-repo.libsonnet new file mode 100644 index 00000000..338dc781 --- /dev/null +++ b/app/matrix/lib/media-repo.libsonnet @@ -0,0 +1,97 @@ +local kube = import "../../../kube/kube.libsonnet"; + +{ + local app = self, + local cfg = app.cfg, + cfg:: { + image: error "cfg.image needs to be set", + + homeservers: [], + admins: [], + + s3: { + endpoint: error "cfg.s3.endpoint needs to be set", + accessKey: error "cfg.s3.accessKey needs to be set", + secretKey: error "cfg.s3.secretKey needs to be set", + bucketName: error "cfg.s3.bucketName needs to be set", + region: error "cfg.s3.region needs to be set", + }, + + db: { + username: error "cfg.db.username needs to be set", + password: error "cfg.db.password needs to be set", + database: error "cfg.db.database needs to be set", + host: error "cfg.db.host needs to be set", + }, + }, + + ns:: error "ns needs to be a kube.Namespace object", + + config:: { + repo: { + bindAddress: "0.0.0.0", + port: 8000, + }, + database: { + postgres: "postgres://%s:%s@%s/%s?sslmode=disable" % [cfg.db.username, cfg.db.password, cfg.db.host, cfg.db.database], + }, + homeservers: cfg.homeservers, + admins: cfg.admins, + datastores: [ + { + type: "s3", + enabled: true, + forKinds: ["all"], + opts: { + tempPath: "/tmp/mediarepo_s3_upload", + endpoint: cfg.s3.endpoint, + accessKeyId: cfg.s3.accessKey, + accessSecret: cfg.s3.secretKey, + ssl: false, + bucketName: cfg.s3.bucketName, + region: cfg.s3.region, + }, + } + ], + }, + + configSecret: app.ns.Contain(kube.Secret("media-repo-config")) { + data_: { + "config.yaml": std.manifestJsonEx(app.config, ""), + }, + }, + + deployment: app.ns.Contain(kube.Deployment("media-repo")) { + spec+: { + replicas: 1, + template+: { + spec+: { + volumes_: { + config: kube.SecretVolume(app.configSecret), + tempdir: kube.EmptyDirVolume(), + }, + containers_: { + repo: kube.Container("media-repo") { + image: cfg.image, + command: ["/usr/local/bin/media_repo"], + ports_: { + http: { containerPort: 8000 }, + }, + env_: { + REPO_CONFIG: "/config", + }, + volumeMounts_: { + config: { mountPath: "/config" }, + tempdir: { mountPath: "/tmp/mediarepo_s3_upload" }, + }, + }, + }, + }, + }, + }, + }, + + svc: app.ns.Contain(kube.Service("media-repo")) { + target_pod:: app.deployment.spec.template, + }, +}