From 1257389d3dba07fedfb43ce4b6a5862142d4b832 Mon Sep 17 00:00:00 2001
From: Serge Bazanski <q3k@hackerspace.pl>
Date: Sat, 10 Oct 2020 14:55:08 +0200
Subject: [PATCH] k0: expose controller-manager and scheduler metrics

We want to be able to scrape controller-manager and scheduler metrics
into Prometheus. For that, each of them needs to:

 1) listen on a secure port
 2) have authn enabled

With this, any k8s user with the right permissions (and a bearer token
or TLS certificate) can come in and access metrics over a node's public
IP address. Access without a certificate/token gets thrown into the
system:anonymous user, which as no access to any API.

Change-Id: I267680f92f748ba63b6762e6aaba3c417446e50b
---
 cluster/nix/defs-cluster-k0.nix    |  4 +++-
 cluster/nix/modules/kubernetes.nix | 19 ++++++++++++++++---
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/cluster/nix/defs-cluster-k0.nix b/cluster/nix/defs-cluster-k0.nix
index 832c741a..c3519cc1 100644
--- a/cluster/nix/defs-cluster-k0.nix
+++ b/cluster/nix/defs-cluster-k0.nix
@@ -70,7 +70,9 @@ in rec {
   ports = {
     k8sAPIServerPlain = 4000;
     k8sAPIServerSecure = 4001;
-    k8sControllerManagerPlain = 0; # 4002; do not serve plain http
+    k8sControllerManagerPlain = 0; # would be 4002; do not serve plain http
     k8sControllerManagerSecure = 4003;
+    k8sSchedulerPlain = 0; # would be 4004; do not serve plain http
+    k8sSchedulerSecure = 4005;
   };
 }
diff --git a/cluster/nix/modules/kubernetes.nix b/cluster/nix/modules/kubernetes.nix
index 10560cd9..b36e806b 100644
--- a/cluster/nix/modules/kubernetes.nix
+++ b/cluster/nix/modules/kubernetes.nix
@@ -154,7 +154,10 @@ in rec {
       '';
     };
 
-    controllerManager = {
+    controllerManager = let
+      top = config.services.kubernetes;
+      kubeconfig = top.lib.mkKubeConfig "controller-manager" pki.kube.controllermanager.config;
+    in {
       enable = true;
       bindAddress = "0.0.0.0";
       insecurePort = ports.k8sControllerManagerPlain;
@@ -165,16 +168,26 @@ in rec {
         --service-cluster-ip-range=10.10.12.0/24 \
         --use-service-account-credentials=true \
         --secure-port=${toString ports.k8sControllerManagerSecure}\
+        --authentication-kubeconfig=${kubeconfig}\
+        --authorization-kubeconfig=${kubeconfig}\
       '';
       kubeconfig = pki.kube.controllermanager.config;
     };
 
-    scheduler = {
+    scheduler = let
+      top = config.services.kubernetes;
+      kubeconfig = top.lib.mkKubeConfig "scheduler" pki.kube.controllermanager.config;
+    in {
       enable = true;
       address = "0.0.0.0";
-      port = 0;
+      port = ports.k8sSchedulerPlain;
       leaderElect = true;
       kubeconfig = pki.kube.scheduler.config;
+      extraOpts = ''
+        --secure-port=${toString ports.k8sSchedulerSecure}\
+        --authentication-kubeconfig=${kubeconfig}\
+        --authorization-kubeconfig=${kubeconfig}\
+      '';
     };
 
     proxy = {