From 4f7caf8d868deecbd1387aa803f65d43032b3f5a Mon Sep 17 00:00:00 2001 From: Serge Bazanski Date: Sat, 10 Oct 2020 17:58:09 +0200 Subject: [PATCH] ops/monitoring: deploy grafana This is a basic grafana running on: https://monitoring-global-dashboard.k0.hswaw.net/ It contains a data source pointing at the corresponding global victoria metrics. There's no dashboards, these will be provisioned soon via jsonnet/grafonnet. Change-Id: I84873bc323d1727096e3ce818fae122a9af3e191 --- ops/monitoring/k0.jsonnet | 12 +- ops/monitoring/lib/global.libsonnet | 162 +++++++++++++++++- .../secrets/cipher/global-oauth-client-secret | 40 +++++ 3 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 ops/monitoring/secrets/cipher/global-oauth-client-secret diff --git a/ops/monitoring/k0.jsonnet b/ops/monitoring/k0.jsonnet index 62810c5d..30dd687c 100644 --- a/ops/monitoring/k0.jsonnet +++ b/ops/monitoring/k0.jsonnet @@ -9,6 +9,7 @@ local global = import "lib/global.libsonnet"; storageClasses+: { prometheus: "waw-hdd-redundant-3", victoria: "waw-hdd-redundant-3", + grafana: "waw-hdd-redundant-3", }, }, @@ -25,15 +26,22 @@ local global = import "lib/global.libsonnet"; // Global tier - victoria metrics. global: global.Global("k0") { cfg+: cfg { + oauth: { + clientId: "22659ba3-c8b2-4855-9553-f78884e0d743", + clientSecret: std.split(importstr "secrets/plain/global-oauth-client-secret", "\n")[0], + }, hosts: { globalAPI: "monitoring-global-api.k0.hswaw.net", + globalDashboard: "monitoring-global-dashboard.k0.hswaw.net", }, agents: [ // Ingestion from k0 cluster tier. { username: k0.cluster.cfg.username, password: std.split(importstr "secrets/plain/global-agent-cluster-k0", "\n")[0], }, - // Access from q3k's test Grafana. - { username: "grafana", password: std.split(importstr "secrets/plain/global-agent-grafana", "\n")[0], }, ], + loopbackGrafanaUser: { + username: "grafana", + password: std.split(importstr "secrets/plain/global-agent-grafana", "\n")[0], + }, }, }, } diff --git a/ops/monitoring/lib/global.libsonnet b/ops/monitoring/lib/global.libsonnet index dbdbebb1..f001c99b 100644 --- a/ops/monitoring/lib/global.libsonnet +++ b/ops/monitoring/lib/global.libsonnet @@ -18,11 +18,13 @@ local kube = import "../../../kube/kube.libsonnet"; images: { victoria: "victoriametrics/victoria-metrics:v1.40.0", vmauth: "victoriametrics/vmauth:v1.40.0", + grafana: "grafana/grafana:7.2.1", }, hosts: { // DNS hostname that this global tier will use. Ingress will run under it. globalAPI: error "hosts.globalAPI must be set", + globalDashboard: error "hosts.globalDashboard must be set", }, storageClasses: { @@ -30,6 +32,11 @@ local kube = import "../../../kube/kube.libsonnet"; victoria: error "storageClasses.victoria must be set", }, + oauth: { + clientId: error "oauth.clientId must be set", + clientSecret: error "oauth.clientSecret must be set", + }, + // A list of agents that will push metrics to this instance. // List of: // { @@ -41,10 +48,14 @@ local kube = import "../../../kube/kube.libsonnet"; // Generated URLs that agents should use to ship metrics over. Both require HTTP basic // auth, configured via cfg.agents. - // The internal URL should be used for agents colocated in the same Kubernetes cluster. + // The internal URL that should be used for agents colocated in the same Kubernetes cluster. internalIngestURL:: "http://%s/api/v1/write" % [global.victoria.serviceAPI.host_colon_port], - // The glboal URL should be used for agents sending data over the internet. + // The internal URL that should be used for readers colocated in the same Kubernetes cluster. + internalReadURL:: "http://%s/" % [global.victoria.serviceAPI.host_colon_port], + // The global URL that should be used for agents sending data over the internet. globalIngestURL:: "https://%s/api/v1/write" % [cfg.hosts.globalAPI], + // The global URL that should be used for readers over the internet. + globalReadURL:: "https://%s" % [cfg.hosts.globalAPI], namespace: kube.Namespace(cfg.namespace), local ns = global.namespace, @@ -73,7 +84,7 @@ local kube = import "../../../kube/kube.libsonnet"; password: a.password, url_prefix: "http://localhost:8428", } - for a in cfg.agents + for a in (cfg.agents + [cfg.loopbackGrafanaUser]) ], }) + "\n") }, @@ -145,5 +156,150 @@ local kube = import "../../../kube/kube.libsonnet"; }, }, }, + + grafana: { + local grafana = self, + + // grafana.ini, serialized to secret. + ini:: { + sections: { + "auth": { + "disable_login_form": true, + "oauth_auto_login": true, + }, + "security": { + # We do not disable basic auth, as we want to use builtin + # users as API users (eg for config reload), but we want + # to disable the default admin:admin user. + "disable_initial_admin_creation": true, + }, + "auth.generic_oauth": { + enabled: true, + client_id: cfg.oauth.clientId, + client_secret: cfg.oauth.clientSecret, + auth_url: "https://sso-v2.hackerspace.pl/oauth/authorize", + token_url: "https://sso-v2.hackerspace.pl/oauth/token", + api_url: "https://sso-v2.hackerspace.pl/api/1/userinfo", + scopes: "openid", + email_attribute_path: "email", + allow_sign_up: true, + role_attribute_path: "contains(groups, 'grafana-admin')", + }, + "server": { + domain: cfg.hosts.globalDashboard, + root_url: "https://%s/" % [ cfg.hosts.globalDashboard ], + }, + }, + }, + + datasources:: { + apiVersion: 1, + datasources: [ + { + name: "victoria-global", + type: "prometheus", + uid: "victoria-global", + isDefault: true, + url: global.internalReadURL, + basicAuth: true, + basicAuthUser: cfg.loopbackGrafanaUser.username, + secureJsonData: { + basicAuthPassword: cfg.loopbackGrafanaUser.password, + }, + }, + ], + }, + + config: ns.Contain(kube.Secret("grafana-config")) { + data+: { + "grafana.ini": std.base64(std.manifestIni(grafana.ini)), + "datasources.yaml": std.base64(std.manifestYamlDoc(grafana.datasources)), + }, + }, + + pvc: ns.Contain(kube.PersistentVolumeClaim("grafana-data")) { + spec+: { + storageClassName: cfg.storageClasses.grafana, + accessModes: ["ReadWriteOnce"], + resources: { + requests: { + storage: "8Gi", + }, + }, + }, + }, + + deploy: ns.Contain(kube.Deployment("grafana")) { + spec+: { + template+: { + spec+: { + containers_: { + default: kube.Container("default") { + image: cfg.images.grafana, + ports_: { + public: { containerPort: 3000 }, + }, + env_: { + GF_PATHS_CONFIG: "/etc/hscloud-config/grafana.ini", + GF_PATHS_PROVISIONING: "/etc/hscloud-config/provisioning", + GF_PATHS_DATA: "/var/lib/grafana", + }, + volumeMounts_: { + config: { mountPath: "/etc/hscloud-config", }, + data: { mountPath: "/var/lib/grafana", }, + }, + resources: { + requests: { cpu: "100m", memory: "256M", }, + limits: { cpu: "200m", memory: "512M", }, + }, + }, + }, + volumes_: { + data: kube.PersistentVolumeClaimVolume(grafana.pvc), + config: kube.SecretVolume(grafana.config) { + secret+: { + items: [ + { key: "grafana.ini", path: "grafana.ini", }, + { key: "datasources.yaml", path: "provisioning/datasources/datasources.yaml", }, + ], + }, + }, + }, + }, + }, + }, + }, + + service: ns.Contain(kube.Service("grafana-public")) { + target_pod: grafana.deploy.spec.template, + spec+: { + ports: [ + { name: "public", port: 3000, targetPort: 3000, protocol: "TCP" }, + ], + }, + }, + + ingress: ns.Contain(kube.Ingress("grafana-public")) { + metadata+: { + annotations+: { + "kubernetes.io/tls-acme": "true", + "certmanager.k8s.io/cluster-issuer": "letsencrypt-prod", + }, + }, + spec+: { + tls: [ + { hosts: [cfg.hosts.globalDashboard], secretName: "ingress-grafana-tls" }, + ], + rules: [ + { + host: cfg.hosts.globalDashboard, + http: { + paths: [ { path: "/", backend: { serviceName: grafana.service.metadata.name, servicePort: 3000 } }, ], + }, + } + ], + }, + }, + }, } } diff --git a/ops/monitoring/secrets/cipher/global-oauth-client-secret b/ops/monitoring/secrets/cipher/global-oauth-client-secret new file mode 100644 index 00000000..77b4e89d --- /dev/null +++ b/ops/monitoring/secrets/cipher/global-oauth-client-secret @@ -0,0 +1,40 @@ +-----BEGIN PGP MESSAGE----- + +hQEMAzhuiT4RC8VbAQf9Ei3B4VGp5X1sBBvdpD0P1gbZcOMuQrChLKf4WFTkJ31V +7iK88YzJXM1VN0/GdTS4xk30D9Bh6nkbyWqSQ6e5mI6rU06DHjEF4nH/rCVNNImx +2lsAfHkvyBYV2rzMD+v7o/WWcR0RzemtopJvJXahM39Dd4WKQEqilcvwFM3p/zAG +p9svNEpangRCw4viNeP8RzBIHl6d73gcLwYtlmmj/URR4hVh0QByvJE+8tZJaelg +D2ILnnv30If51H6iRjUSdQYiScPyAc0Ooe7nLNyiZJHe2unv1wpFK/ppW5nTLc6J +Jl3ku5k5Fza5GLImxT+r3LFaGCUZwI2Ilh+aixOd8YUBDANcG2tp6fXqvgEIAM4s +Vty4caVhY8wIK4shv+2N8VXxaa8AHBMycfsAdrMG7ohrVLBJcNCs2CfYDRcLLxXq +y/PU53hffCgg19g1np+8rsYis5JXS8Uqri/54T/S4cMid1UaCq2BIs+1A/9j780G +4GGArAFDS451t5QjWzXl2W0ZVTeTSVC3s93psht10cZt8APAxlefkoPwSbb2kYz5 +CCOmUGGLwHB87xBl0jRZ55A2Qe77637YEvbRBr79OhztSIJ1WJjkNFLqOVbCDcR0 +IH9kVES2fN/4KCI772P+Rmh330B13UHk9xnu1xEJsi57HjCof+zwGvmEfNrKtS9d +knHAlDPycEVnQMDVNUOFAgwDodoT8VqRl4UBD/902MbY7Psg+wm7s1ybsclWRA1q +lJToPhB1NeDhdh/9l51kWT5JvUjS6jCvoGHyJvnxXR6Ot3i+8mjEiHZf6amu5gvq +skvzQwt+XwtIOaUxJChfRhk+GoyT6EpSHXYDNWKfWPG4gUaM42o8S7BObyjGjwXE +kTf3bvw50YNqJo7DmSJ1yS/sY4/J9wWT0jz0jSc9PjpAI9qw8vbWSrfbMa7EWos3 +ENyIDl0GlF5S13J5GtyOCQLh9TsHi+zCe/jhmu4uhSeHxyuGru+UvNE1ME0XIUAS +fUJ5dLIfdLH+ILBRBZ+G0XRT/3XkWlyhuRZf7ALU3tG1wXRV1evc0zv7kEcz2hQm +gUPXkZzcFIG1cO3r9FhBvAM86p+UHSdsXdRXSVWsH12QFDjv8ZollPzO3ZztQI6a +R6E3WQ1nyiFjVTHKrCus89UDqBtAiYujfuwLcDYP9wMBW7JpETd1qurccSnL3duh +3jkKGHeskQPkB9UrT1P66zUjT/gAFDy5/sfVxoO5y+jPAJS9owYrONAoQtTL0HcA +4ixmaDb3ZzBt1LAfDDlGSjt4agQVfVLeGPF/zrFS4GrqzPDREyfTAsYdokA+y0LM +XI6mSsHd01HPGpRbsE5ABOO88sqRnuD8KBxWpgaG+Z8zn1uuf7n1L2JRWpFcd8h/ +C09qbhK0+9C80HBZqoUCDAPiA8lOXOuz7wEP/R81sepe2UgcwMuBQmrn30y+kN0i +93zhYDVJFYUF07b7ociu2OnGFCnFF3ZQNao3ZvSuKoCKkQvcf7mxHA9xkFjiwGAi +elhHDQcUt8IriosGNhSArujEZ1kc1Nk9MWQKRSLhVXNtdTrn4e15OPXO+AR7CszW +Kz9Mwo9BNPzu7Zwq1JfUOExpDPT6fPVHZNnzg3KU4s2HRcrLD9JEE2i2/VxbmszH +aTy+/1kF8hHSfRV0Q7NcjRAbztWrd47HqsWmmWzjcjnSKNV1n7P5AcB06Yjdf0+0 +xEuehwseJs6OhL3MxCsQoFuM9xhm7W/rfGQe+JvJc9Hxb60AgoMGJ1GSHz8xhjyx +EOujnIabcUeOm0h0twEi98+OJTlKss1YPdcKMPCit7SJZX8k6t2deOp8t0x9R5hH +v30DRSVgNeqDkBK0dEouR3xLzNz8yardFqVpM88w4D/npUQ5RB6+1af5LFYrm4zG +kEit4bYdJVpfgt0ZRFoyWaAiAt07ARFmoWeQRRDrpbX+ddKAFmvHl0oyRy/QF2xx +P6YT8UyEDNraXchAf4cBjuCuiRyqVqaPAOLp3rKmEBBiXddRX9fsq24/9X5QY4o8 +Kemf0fbH9ndsL4vPrJI/j7nvbgq2dpFuHlnFgE5EUEFoPcDI1GI6hUr5UUffnjzM +aOPp1vxrxhwQy0IG0nQBTdekgVaPiqP+AxVfQbSjz6zNotSJMPAvbx1aNWmxesXO +eZYeMaSVRnSHub97eb6hn167olrcrAzPxFssb7iTEQh2Xs6PeWbe0FsTz0Fim/yY +iIw5GlFw15/afo86hbDgrK0j2ZiafKvZYC2EtKoYGzAoxA== +=8iTI +-----END PGP MESSAGE-----