cheshire
/
hscloud
forked from hswaw/hscloud
1
0
Fork 0
Code Pull Requests Activity

app/matrix: matrix-media-repo RGW-based media storage 

Change-Id: I459bd78eee52fd349a16f31a48346d3258ef50a4
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1081
Reviewed-by: q3k <q3k@hackerspace.pl>
master
informatic 2021-09-10 23:54:34 +02:00 committed by informatic
parent 0e6c6720d9
commit 122d5e5864
2 changed files with 187 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
app/matrix/lib/matrix-ng.libsonnet
View File

@ -28,6 +28,51 @@
# * https://{homeserver}/_synapse/oidc/callback is added to allowed callback URLs list
# * openid scope is enabled for configured client
#
# In order to deploy matrix-media-repo as a replacement for synapse built-in
# media workers the following steps need to be carried out:
#
# 1. Generate password and bootstrap extra postgres user
# pwgen 32 1 > secrets/plain/media-repo-$ns-postgres
# echo "create database mediarepo; create user mediarepo with password '$(cat secrets/plain/media-repo-$ns-postgres)'; grant all privileges on database mediarepo to mediarepo;" | kubectl -n $ns exec -it deploy/waw3-postgres psql
# secretstore sync secrets
#
# 2. Fetch Ceph RGW credentials
# kubectl get secrets -n ceph-waw3 rook-ceph-object-user-waw-hdd-redundant-3-object-$ns -o json | jq '.data|map_values(@base64d)' > secrets/plain/media-repo-$ns-ceph.json
# secretstore sync secrets
#
# 3. Create an apropriate bucket using s3cmd
# s3cmd --access_key="$(jq -r '.AccessKey' secrets/plain/media-repo-$ns-ceph.json)" --secret_key="$(jq -r '.SecretKey' secrets/plain/media-repo-$ns-ceph.json)" --host=object.ceph-waw3.hswaw.net --host-bucket=object.ceph-waw3.hswaw.net mb s3://media-repo-$ns
#
# 4. Add relevant configuration overrides in cfg.mediaRepo key for your
# deployment configuration file:
#
# mediaRepo+: {
# enable: true,
# route: false,
# s3+: {
# endpoint: std.strReplace((import "secrets/plain/media-repo-$ns-ceph.json").Endpoint, "http://", ""),
# accessKey: (import "secrets/plain/media-repo-$ns-ceph.json").AccessKey,
# secretKey: (import "secrets/plain/media-repo-$ns-ceph.json").SecretKey,
# bucketName: "media-repo-$ns",
# region: "eu",
# },
# db+: {
# password: std.strReplace(importstr "secrets/plain/media-repo-$ns-postgres", "\n", ""),
# },
# },
#
# 5. Additionally, when migrating from already deployed synapse media worker the
# following command needs to be run in order to import existing media files:
# kubectl -n $ns exec deploy/media-repo -- import_synapse -baseUrl http://synapse-media:8008 -dbHost waw3-postgres -dbPassword "$(kubectl -n $ns get secret synapse -o json | jq -r '.data.postgres_password | @base64d')" -config /config/config.yaml -serverName 'SERVER_NAME'
#
# 6. After migrating data over from native synapse media worker storage traffic
# can be rerouted to matrix-media-repo by switching cfg.mediaRepo.route flag
# to true
#
# 7. Run import step #5 again to make sure no media were left missing in old
# media worker deployment - import operation is indempotent and can be ran
# against a synapse media worker that's not handling user traffic anymore.
#
# Sequencing appservices is fun. The appservice needs to run first (for
# instance, via a bootstrap job), and on startup it will spit out a
# registration file. This registration file then needs to be fed to synapse -
@ -49,6 +94,7 @@ local riot = import "./riot.libsonnet";
local cas = import "./cas.libsonnet";
local wellKnown = import "./wellknown.libsonnet";
local synapse = import "./synapse.libsonnet";
local mediaRepo = import "./media-repo.libsonnet";
{
local app = self,
@ -68,6 +114,7 @@ local synapse = import "./synapse.libsonnet";
appserviceIRC: "matrixdotorg/matrix-appservice-irc:release-v0.29.0",
appserviceTelegram: "dock.mau.dev/tulir/mautrix-telegram@sha256:c6e25cb57e1b67027069e8dc2627338df35d156315c004a6f2b34b6aeaa79f77",
wellKnown: "registry.k0.hswaw.net/q3k/wellknown:1611960794-adbf560851a46ad0e58b42f0daad7ef19535687c",
mediaRepo: "turt2live/matrix-media-repo:v1.2.8",
},
# OpenID Connect provider configuration.
@ -119,6 +166,33 @@ local synapse = import "./synapse.libsonnet";
# Serve /.well-known/matrix configuration endpoints required when using
# cfg.webDomain directly as mxid.
wellKnown: false,
# matrix-media-repo S3-based media storage container
mediaRepo: {
enable: false,
# Route /_matrix/media/ endpoints to matrix-media-repo. Set this
# to true after migrating media files to matrix-media-repo.
route: false,
s3: {
endpoint: error "mediaRepo.s3.endpoint needs to be set",
accessKey: error "mediaRepo.s3.accessKey needs to be set",
secretKey: error "mediaRepo.s3.secretKey needs to be set",
bucketName: error "mediaRepo.s3.bucketName needs to be set",
region: error "mediaRepo.s3.region needs to be set",
},
db: {
username: "mediarepo",
password: error "mediaRepo.db.password needs to be set",
database: "mediarepo",
host: "waw3-postgres",
},
},
# List of administrative users MXIDs (used in matrix-media-repo only)
admins: [],
},
# DEPRECATED: this needs to be removed in favor of namespace.Contain() in
@ -184,6 +258,21 @@ local synapse = import "./synapse.libsonnet";
},
} else {},
mediaRepo: if cfg.mediaRepo.enable then mediaRepo {
ns: app.namespace,
cfg+: {
image: cfg.images.mediaRepo,
homeservers: [
{name: cfg.serverName, csApi: "https://" + cfg.webDomain}
],
admins: cfg.admins,
s3: cfg.mediaRepo.s3,
db: cfg.mediaRepo.db,
},
} else {},
synapse: synapse {
ns: app.namespace,
postgres: app.postgres3,
@ -231,7 +320,7 @@ local synapse = import "./synapse.libsonnet";
for path in app.synapse.genericWorker.paths
] + [
{ path: "/", backend: app.riot.svc.name_port },
{ path: "/_matrix/media/", backend: app.synapse.mediaWorker.svc.name_port },
{ path: "/_matrix/media/", backend: if cfg.mediaRepo.route then app.mediaRepo.svc.name_port else app.synapse.mediaWorker.svc.name_port },
{ path: "/_matrix/", backend: app.synapse.main.svc.name_port },
# Used by OpenID Connect login flow

97
app/matrix/lib/media-repo.libsonnet Normal file
View File

@ -0,0 +1,97 @@
local kube = import "../../../kube/kube.libsonnet";
{
local app = self,
local cfg = app.cfg,
cfg:: {
image: error "cfg.image needs to be set",
homeservers: [],
admins: [],
s3: {
endpoint: error "cfg.s3.endpoint needs to be set",
accessKey: error "cfg.s3.accessKey needs to be set",
secretKey: error "cfg.s3.secretKey needs to be set",
bucketName: error "cfg.s3.bucketName needs to be set",
region: error "cfg.s3.region needs to be set",
},
db: {
username: error "cfg.db.username needs to be set",
password: error "cfg.db.password needs to be set",
database: error "cfg.db.database needs to be set",
host: error "cfg.db.host needs to be set",
},
},
ns:: error "ns needs to be a kube.Namespace object",
config:: {
repo: {
bindAddress: "0.0.0.0",
port: 8000,
},
database: {
postgres: "postgres://%s:%s@%s/%s?sslmode=disable" % [cfg.db.username, cfg.db.password, cfg.db.host, cfg.db.database],
},
homeservers: cfg.homeservers,
admins: cfg.admins,
datastores: [
{
type: "s3",
enabled: true,
forKinds: ["all"],
opts: {
tempPath: "/tmp/mediarepo_s3_upload",
endpoint: cfg.s3.endpoint,
accessKeyId: cfg.s3.accessKey,
accessSecret: cfg.s3.secretKey,
ssl: false,
bucketName: cfg.s3.bucketName,
region: cfg.s3.region,
},
}
],
},
configSecret: app.ns.Contain(kube.Secret("media-repo-config")) {
data_: {
"config.yaml": std.manifestJsonEx(app.config, ""),
},
},
deployment: app.ns.Contain(kube.Deployment("media-repo")) {
spec+: {
replicas: 1,
template+: {
spec+: {
volumes_: {
config: kube.SecretVolume(app.configSecret),
tempdir: kube.EmptyDirVolume(),
},
containers_: {
repo: kube.Container("media-repo") {
image: cfg.image,
command: ["/usr/local/bin/media_repo"],
ports_: {
http: { containerPort: 8000 },
},
env_: {
REPO_CONFIG: "/config",
},
volumeMounts_: {
config: { mountPath: "/config" },
tempdir: { mountPath: "/tmp/mediarepo_s3_upload" },
},
},
},
},
},
},
},
svc: app.ns.Contain(kube.Service("media-repo")) {
target_pod:: app.deployment.spec.template,
},
}