forked from hswaw/hscloud
306 lines
13 KiB
Plaintext
306 lines
13 KiB
Plaintext
local kube = import "../../../kube/kube.libsonnet";
|
|
|
|
{
|
|
// Global sets up a global tier instance of the hscloud monitoring infrastructure.
|
|
//
|
|
// This currently consists of Victoria Metrics, to which the agent tier sends metrics data via
|
|
// the prometheus remote_write protocol.
|
|
// Victoria Metrics is here used as a long-term storage solution. However, right now, it
|
|
// just keeps data locally on disk. In the future, S3 snapshots/backups should be introduced.
|
|
Global(name):: {
|
|
local global = self,
|
|
local cfg = global.cfg,
|
|
|
|
cfg:: {
|
|
name: name,
|
|
namespace: "monitoring-global-%s" % [cfg.name],
|
|
|
|
images: {
|
|
victoria: "victoriametrics/victoria-metrics:v1.40.0",
|
|
vmauth: "victoriametrics/vmauth:v1.40.0",
|
|
grafana: "grafana/grafana:7.2.1",
|
|
},
|
|
|
|
hosts: {
|
|
// DNS hostname that this global tier will use. Ingress will run under it.
|
|
globalAPI: error "hosts.globalAPI must be set",
|
|
globalDashboard: error "hosts.globalDashboard must be set",
|
|
},
|
|
|
|
storageClasses: {
|
|
// Storage class used for main data retention.
|
|
victoria: error "storageClasses.victoria must be set",
|
|
},
|
|
|
|
oauth: {
|
|
clientId: error "oauth.clientId must be set",
|
|
clientSecret: error "oauth.clientSecret must be set",
|
|
},
|
|
|
|
// A list of agents that will push metrics to this instance.
|
|
// List of:
|
|
// {
|
|
// username: the username that the agent will authenticate with
|
|
// password: the password that the agent will authenticate with
|
|
// }
|
|
agents: [],
|
|
},
|
|
|
|
// Generated URLs that agents should use to ship metrics over. Both require HTTP basic
|
|
// auth, configured via cfg.agents.
|
|
// The internal URL that should be used for agents colocated in the same Kubernetes cluster.
|
|
internalIngestURL:: "http://%s/api/v1/write" % [global.victoria.serviceAPI.host_colon_port],
|
|
// The internal URL that should be used for readers colocated in the same Kubernetes cluster.
|
|
internalReadURL:: "http://%s/" % [global.victoria.serviceAPI.host_colon_port],
|
|
// The global URL that should be used for agents sending data over the internet.
|
|
globalIngestURL:: "https://%s/api/v1/write" % [cfg.hosts.globalAPI],
|
|
// The global URL that should be used for readers over the internet.
|
|
globalReadURL:: "https://%s" % [cfg.hosts.globalAPI],
|
|
|
|
namespace: kube.Namespace(cfg.namespace),
|
|
local ns = global.namespace,
|
|
|
|
victoria: {
|
|
local victoria = self,
|
|
|
|
pvc: ns.Contain(kube.PersistentVolumeClaim("victoria-data")) {
|
|
spec+: {
|
|
storageClassName: cfg.storageClasses.victoria,
|
|
accessModes: ["ReadWriteOnce"],
|
|
resources: {
|
|
requests: {
|
|
storage: "64Gi",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
authSecret: ns.Contain(kube.Secret("vmauth")) {
|
|
data+: {
|
|
"config.yaml": std.base64(std.manifestJson({
|
|
users: [
|
|
{
|
|
username: a.username,
|
|
password: a.password,
|
|
url_prefix: "http://localhost:8428",
|
|
}
|
|
for a in (cfg.agents + [cfg.loopbackGrafanaUser])
|
|
],
|
|
}) + "\n")
|
|
},
|
|
},
|
|
|
|
deploy: ns.Contain(kube.Deployment("victoria")) {
|
|
spec+: {
|
|
template+: {
|
|
spec+: {
|
|
containers_: {
|
|
default: kube.Container("default") {
|
|
image: cfg.images.victoria,
|
|
volumeMounts_: {
|
|
data: { mountPath: "/victoria-metrics-data", },
|
|
},
|
|
},
|
|
vmauth: kube.Container("vmauth") {
|
|
image: cfg.images.vmauth,
|
|
command: [
|
|
"/vmauth-prod",
|
|
"-auth.config", "/mnt/secret/config.yaml",
|
|
],
|
|
volumeMounts_: {
|
|
secret: { mountPath: "/mnt/secret", },
|
|
},
|
|
ports_: {
|
|
api: { containerPort: 8427 }
|
|
},
|
|
}
|
|
},
|
|
volumes_: {
|
|
data: kube.PersistentVolumeClaimVolume(victoria.pvc),
|
|
secret: kube.SecretVolume(victoria.authSecret),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
serviceAPI: ns.Contain(kube.Service("victoria-api")) {
|
|
target_pod: victoria.deploy.spec.template,
|
|
spec+: {
|
|
ports: [
|
|
{ name: "api", port: 8427, targetPort: 8427, protocol: "TCP" },
|
|
],
|
|
type: "ClusterIP",
|
|
},
|
|
},
|
|
|
|
ingressAPI: ns.Contain(kube.Ingress("victoria-api")) {
|
|
metadata+: {
|
|
annotations+: {
|
|
"kubernetes.io/tls-acme": "true",
|
|
"certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
|
|
},
|
|
},
|
|
spec+: {
|
|
tls: [
|
|
{ hosts: [cfg.hosts.globalAPI], secretName: "ingress-tls" },
|
|
],
|
|
rules: [
|
|
{
|
|
host: cfg.hosts.globalAPI,
|
|
http: {
|
|
paths: [ { path: "/", backend: { serviceName: victoria.serviceAPI.metadata.name, servicePort: 8427 } }, ],
|
|
},
|
|
}
|
|
],
|
|
},
|
|
},
|
|
},
|
|
|
|
grafana: {
|
|
local grafana = self,
|
|
|
|
// grafana.ini, serialized to secret.
|
|
ini:: {
|
|
sections: {
|
|
"auth": {
|
|
"disable_login_form": true,
|
|
"oauth_auto_login": true,
|
|
},
|
|
"security": {
|
|
# We do not disable basic auth, as we want to use builtin
|
|
# users as API users (eg for config reload), but we want
|
|
# to disable the default admin:admin user.
|
|
"disable_initial_admin_creation": true,
|
|
},
|
|
"auth.generic_oauth": {
|
|
enabled: true,
|
|
client_id: cfg.oauth.clientId,
|
|
client_secret: cfg.oauth.clientSecret,
|
|
auth_url: "https://sso.hackerspace.pl/oauth/authorize",
|
|
token_url: "https://sso.hackerspace.pl/oauth/token",
|
|
api_url: "https://sso.hackerspace.pl/api/1/userinfo",
|
|
scopes: "openid",
|
|
email_attribute_path: "email",
|
|
allow_sign_up: true,
|
|
role_attribute_path: "contains(groups, 'grafana-admin')",
|
|
},
|
|
"server": {
|
|
domain: cfg.hosts.globalDashboard,
|
|
root_url: "https://%s/" % [ cfg.hosts.globalDashboard ],
|
|
},
|
|
},
|
|
},
|
|
|
|
datasources:: {
|
|
apiVersion: 1,
|
|
datasources: [
|
|
{
|
|
name: "victoria-global",
|
|
type: "prometheus",
|
|
uid: "victoria-global",
|
|
isDefault: true,
|
|
url: global.internalReadURL,
|
|
basicAuth: true,
|
|
basicAuthUser: cfg.loopbackGrafanaUser.username,
|
|
secureJsonData: {
|
|
basicAuthPassword: cfg.loopbackGrafanaUser.password,
|
|
},
|
|
},
|
|
],
|
|
},
|
|
|
|
config: ns.Contain(kube.Secret("grafana-config")) {
|
|
data+: {
|
|
"grafana.ini": std.base64(std.manifestIni(grafana.ini)),
|
|
"datasources.yaml": std.base64(std.manifestYamlDoc(grafana.datasources)),
|
|
},
|
|
},
|
|
|
|
pvc: ns.Contain(kube.PersistentVolumeClaim("grafana-data")) {
|
|
spec+: {
|
|
storageClassName: cfg.storageClasses.grafana,
|
|
accessModes: ["ReadWriteOnce"],
|
|
resources: {
|
|
requests: {
|
|
storage: "8Gi",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
deploy: ns.Contain(kube.Deployment("grafana")) {
|
|
spec+: {
|
|
template+: {
|
|
spec+: {
|
|
containers_: {
|
|
default: kube.Container("default") {
|
|
image: cfg.images.grafana,
|
|
ports_: {
|
|
public: { containerPort: 3000 },
|
|
},
|
|
env_: {
|
|
GF_PATHS_CONFIG: "/etc/hscloud-config/grafana.ini",
|
|
GF_PATHS_PROVISIONING: "/etc/hscloud-config/provisioning",
|
|
GF_PATHS_DATA: "/var/lib/grafana",
|
|
},
|
|
volumeMounts_: {
|
|
config: { mountPath: "/etc/hscloud-config", },
|
|
data: { mountPath: "/var/lib/grafana", },
|
|
},
|
|
resources: {
|
|
requests: { cpu: "100m", memory: "256M", },
|
|
limits: { cpu: "200m", memory: "512M", },
|
|
},
|
|
},
|
|
},
|
|
volumes_: {
|
|
data: kube.PersistentVolumeClaimVolume(grafana.pvc),
|
|
config: kube.SecretVolume(grafana.config) {
|
|
secret+: {
|
|
items: [
|
|
{ key: "grafana.ini", path: "grafana.ini", },
|
|
{ key: "datasources.yaml", path: "provisioning/datasources/datasources.yaml", },
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
service: ns.Contain(kube.Service("grafana-public")) {
|
|
target_pod: grafana.deploy.spec.template,
|
|
spec+: {
|
|
ports: [
|
|
{ name: "public", port: 3000, targetPort: 3000, protocol: "TCP" },
|
|
],
|
|
},
|
|
},
|
|
|
|
ingress: ns.Contain(kube.Ingress("grafana-public")) {
|
|
metadata+: {
|
|
annotations+: {
|
|
"kubernetes.io/tls-acme": "true",
|
|
"certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
|
|
},
|
|
},
|
|
spec+: {
|
|
tls: [
|
|
{ hosts: [cfg.hosts.globalDashboard], secretName: "ingress-grafana-tls" },
|
|
],
|
|
rules: [
|
|
{
|
|
host: cfg.hosts.globalDashboard,
|
|
http: {
|
|
paths: [ { path: "/", backend: { serviceName: grafana.service.metadata.name, servicePort: 3000 } }, ],
|
|
},
|
|
}
|
|
],
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|