hswaw
/
hscloud
mirror of https://gerrit.hackerspace.pl/hscloud
4
0
Fork 2
Code Actions

ops/monitoring/lib/cluster.libsonnet: scrape based on annotations 

This adds automatic scraping of pods and services based on presence of
annotations:
- prometheus.io/scrape
- prometheus.io/port
- prometheus.io/path

Change-Id: I1c1afecc75c30278889de1f6ca0b17da69997295
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1850
Reviewed-by: implr <implr@hackerspace.pl>
changes/50/1850/4
viq 2024-01-15 23:53:17 +01:00
parent 4e46d5017a
commit 30a563c49f
1 changed files with 120 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
ops/monitoring/lib/cluster.libsonnet
View File

@ -169,25 +169,124 @@ local kube = import "../../../kube/kube.libsonnet";
static_configs: [
{ targets: ["rook-ceph-mgr.ceph-waw3.svc.cluster.local:9283"], labels: { core_service: "ceph-mgr", ceph_cluster: "waw3" } }
],
}
],
},
remote_write: [
// Srape pods having "prometheus.io/scrape: true" annotation
// https://github.com/prometheus/prometheus/blob/release-2.48/documentation/examples/prometheus-kubernetes.yml#L262
{
url: u.remote,
basic_auth: {
username: cluster.cfg.username,
password: u.password,
job_name: "kubernetes-pods",
scheme: "http",
scrape_interval: "30s",
kubernetes_sd_configs: [ {"role": "pod"} ],
relabel_configs: [
// "prometheus.io/scrape = true"
{
source_labels: ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"],
action: "keep",
regex: "true",
},
// "prometheus.io/path = something"
{
source_labels: ["__meta_kubernetes_pod_annotation_prometheus_io_path"],
action: "replace",
target_label: "__metrics_path__",
regex: "(.+)",
},
// "prometheus.io/port = 12345"
{
source_labels: ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"],
action: "replace",
regex: "([^:]+)(?::\\d+)?;(\\d+)",
replacement: "$1:$2",
target_label: "__address__",
},
{
action: "labelmap",
regex: "__meta_kubernetes_pod_label_(.+)",
},
{
source_labels: ["__meta_kubernetes_namespace"],
action: "replace",
target_label: "namespace",
},
{
source_labels: ["__meta_kubernetes_pod_name"],
action: "replace",
target_label: "pod"
},
],
},
{
job_name: "kubernetes-service-endpoints",
kubernetes_sd_configs: [ {"role": "endpoints"} ],
scheme: "http",
scrape_interval: "30s",
tls_config: {
insecure_skip_verify: true,
},
}
for u in cluster.cfg.upstreams
],
},
relabel_configs: [
// "prometheus.io/scrape = true"
{
source_labels: ["__meta_kubernetes_service_annotation_prometheus_io_scrape"],
action: "keep",
regex: "true",
},
// "prometheus.io/path = something"
{
source_labels: ["__meta_kubernetes_service_annotation_prometheus_io_path"],
action: "replace",
target_label: "__metrics_path__",
regex: "(.+)",
},
// "prometheus.io/port = 12345"
{
source_labels: ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"],
action: "replace",
regex: "([^:]+)(?::\\d+)?;(\\d+)",
replacement: "$1:$2",
target_label: "__address__",
},
// "prometheus.io/scheme = https"
{
source_labels: ["__meta_kubernetes_service_annotation_prometheus_io_scheme"],
action: "replace",
target_label: "__scheme__",
regex: "(https?)",
},
{
action: "labelmap",
regex: "__meta_kubernetes_service_label_(.+)",
},
{
source_labels: ["__meta_kubernetes_namespace"],
action: "replace",
target_label: "namespace",
},
{
source_labels: ["__meta_kubernetes_service_name"],
action: "replace",
target_label: "service",
},
],
},
],
configmap: kube.ConfigMap("prometheus-cluster") {
metadata+: {
namespace: cfg.namespace,
},
remote_write: [
{
url: u.remote,
basic_auth: {
username: cluster.cfg.username,
password: u.password,
},
}
for u in cluster.cfg.upstreams
],
},
configmap: kube.ConfigMap("prometheus-cluster") {
metadata+: {
namespace: cfg.namespace,
},
data: {
"prometheus.yml": std.manifestYamlDoc(prometheus.configuration),
},
@ -222,6 +321,12 @@ local kube = import "../../../kube/kube.libsonnet";
},
spec+: {
template+: {
metadata+: {
annotations+: {
"prometheus.io/scrape": "true",
"prometheus.io/port": "9090",
},
},
spec+: {
containers_: {
default: kube.Container("default") {