From bbc5a43d778ca1079c08c5a095fc0d8f7327a901 Mon Sep 17 00:00:00 2001
From: Serge Bazanski <q3k@hackerspace.pl>
Date: Fri, 31 Mar 2023 22:39:26 +0000
Subject: [PATCH] cluster: move kubernetes services to temporary CA bundle

This is already deployed, and it allows Kubernetes components
(temporary) freedom to use the old or new CA cert.

Change-Id: I8ac7f773a333c30fa22902b8edc327c0c700a482
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1490
Reviewed-by: q3k <q3k@hackerspace.pl>
---
 cluster/machines/modules/kube-common.nix      |  4 +++-
 .../machines/modules/kube-controlplane.nix    | 20 +++++++++++++------
 cluster/machines/modules/kube-dataplane.nix   |  4 +++-
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/cluster/machines/modules/kube-common.nix b/cluster/machines/modules/kube-common.nix
index 6707efaf..f4c6066c 100644
--- a/cluster/machines/modules/kube-common.nix
+++ b/cluster/machines/modules/kube-common.nix
@@ -86,7 +86,9 @@ in {
       # We do not use any nixpkgs predefined roles for k8s. Instead, we enable
       # k8s components manually.
       roles = [];
-      caFile = cfg.pki.kube.apiserver.ca;
+      # TODO(q3k): undo after CA migration done
+      #caFile = cfg.pki.kube.apiserver.ca;
+      caFile = ../../certs/ca-kube-new-and-old.crt;
       clusterCidr = "10.10.16.0/20";
       addons.dns.enable = false;
     };
diff --git a/cluster/machines/modules/kube-controlplane.nix b/cluster/machines/modules/kube-controlplane.nix
index 8efda584..d38b91fc 100644
--- a/cluster/machines/modules/kube-controlplane.nix
+++ b/cluster/machines/modules/kube-controlplane.nix
@@ -82,7 +82,8 @@ in {
       # k8s components manually.
       roles = [];
       addons.dns.enable = false;
-      caFile = pki.kube.apiserver.ca;
+      # TODO(q3k): undo after CA migration done
+      #caFile = pki.kube.apiserver.ca;
       clusterCidr = "10.10.16.0/20";
 
       apiserver = rec {
@@ -102,11 +103,15 @@ in {
 
         tlsCertFile = pki.kube.apiserver.cert;
         tlsKeyFile = pki.kube.apiserver.key;
-        clientCaFile = pki.kube.apiserver.ca;
+        # TODO(q3k): undo after CA migration done
+        #clientCaFile = pki.kube.apiserver.ca;
+        clientCaFile = ../../certs/ca-kube-new-and-old.crt;
 
         kubeletHttps = true;
         # Same CA as main APIServer CA.
-        kubeletClientCaFile = pki.kube.apiserver.ca;
+        # TODO(q3k): undo after CA migration done
+        #kubeletClientCaFile = pki.kube.apiserver.ca;
+        kubeletClientCaFile = ../../certs/ca-kube-new-and-old.crt;
         kubeletClientCertFile = pki.kube.apiserver.cert;
         kubeletClientKeyFile = pki.kube.apiserver.key;
 
@@ -145,21 +150,24 @@ in {
         leaderElect = true;
         serviceAccountKeyFile = pki.kube.serviceaccounts.key;
         rootCaFile = pki.kube.ca;
+        # TODO(q3k): undo after CA migration done 
         extraOpts = ''
           --service-cluster-ip-range=10.10.12.0/24 \
           --use-service-account-credentials=true \
           --secure-port=${toString cfg.portControllerManagerSecure}\
           --authentication-kubeconfig=${kubeconfig}\
           --authorization-kubeconfig=${kubeconfig}\
+          --root-ca-file=${../../certs/ca-kube-new-and-old.crt}\
         '';
         kubeconfig = pki.kube.controllermanager.config;
       };
 
       scheduler = let
         top = config.services.kubernetes;
-        # BUG: this should be scheduler
-        # TODO(q3k): change after big nix change
-        kubeconfig = top.lib.mkKubeConfig "scheduler" pki.kube.controllermanager.config;
+        # TODO(q3k): undo after CA migration done 
+        kubeconfig = top.lib.mkKubeConfig "scheduler" (pki.kube.scheduler.config //  {
+          ca = ../../certs/ca-kube-new-and-old.crt;
+        });
       in {
         enable = true;
         address = "0.0.0.0";
diff --git a/cluster/machines/modules/kube-dataplane.nix b/cluster/machines/modules/kube-dataplane.nix
index 45efcd27..fd87dbc6 100644
--- a/cluster/machines/modules/kube-dataplane.nix
+++ b/cluster/machines/modules/kube-dataplane.nix
@@ -72,7 +72,9 @@ in {
         hostname = fqdn;
         tlsCertFile = pki.kube.kubelet.cert;
         tlsKeyFile = pki.kube.kubelet.key;
-        clientCaFile = pki.kube.kubelet.ca;
+        # TODO(q3k): undo after CA migration done
+        #clientCaFile = pki.kube.kubelet.ca;
+        clientCaFile = ../../certs/ca-kube-new-and-old.crt;
         nodeIp = config.hscloud.base.ipAddr;
         networkPlugin = "cni";
         clusterDns = "10.10.12.254";