k0: expose controller-manager and scheduler metrics

We want to be able to scrape controller-manager and scheduler metrics
into Prometheus. For that, each of them needs to:

 1) listen on a secure port
 2) have authn enabled

With this, any k8s user with the right permissions (and a bearer token
or TLS certificate) can come in and access metrics over a node's public
IP address. Access without a certificate/token gets thrown into the
system:anonymous user, which as no access to any API.

Change-Id: I267680f92f748ba63b6762e6aaba3c417446e50b

cluster/nix/defs-cluster-k0.nix

@@ -70,7 +70,9 @@ in rec {
   ports = {
     k8sAPIServerPlain = 4000;
     k8sAPIServerSecure = 4001;
-    k8sControllerManagerPlain = 0; # 4002; do not serve plain http
+    k8sControllerManagerPlain = 0; # would be 4002; do not serve plain http
     k8sControllerManagerSecure = 4003;
+    k8sSchedulerPlain = 0; # would be 4004; do not serve plain http
+    k8sSchedulerSecure = 4005;
   };
 }

cluster/nix/modules/kubernetes.nix

@@ -154,7 +154,10 @@ in rec {
       '';
     };
 
-    controllerManager = {
+    controllerManager = let
+      top = config.services.kubernetes;
+      kubeconfig = top.lib.mkKubeConfig "controller-manager" pki.kube.controllermanager.config;
+    in {
       enable = true;
       bindAddress = "0.0.0.0";
       insecurePort = ports.k8sControllerManagerPlain;
@@ -165,16 +168,26 @@ in rec {
         --service-cluster-ip-range=10.10.12.0/24 \
         --use-service-account-credentials=true \
         --secure-port=${toString ports.k8sControllerManagerSecure}\
+        --authentication-kubeconfig=${kubeconfig}\
+        --authorization-kubeconfig=${kubeconfig}\
       '';
       kubeconfig = pki.kube.controllermanager.config;
     };
 
-    scheduler = {
+    scheduler = let
+      top = config.services.kubernetes;
+      kubeconfig = top.lib.mkKubeConfig "scheduler" pki.kube.controllermanager.config;
+    in {
       enable = true;
       address = "0.0.0.0";
-      port = 0;
+      port = ports.k8sSchedulerPlain;
       leaderElect = true;
       kubeconfig = pki.kube.scheduler.config;
+      extraOpts = ''
+        --secure-port=${toString ports.k8sSchedulerSecure}\
+        --authentication-kubeconfig=${kubeconfig}\
+        --authorization-kubeconfig=${kubeconfig}\
+      '';
     };
 
     proxy = {