nix/cluster-configuration: pin nixpkgs for k8s

We pin nixpkgs for k8s and also bypass some issues with the new k8s
startup sequencing.

We also pin the kernel to 5.1.

Next step is to also pin nixpkgs for the rest of the system, I think we
can do it from within cluster-configuration.nix.

nix/cluster-configuration.nix

@@ -6,6 +6,14 @@ let
   node = (builtins.head (builtins.filter (n: n.fqdn == fqdn) nodes));
   otherNodes = (builtins.filter (n: n.fqdn != fqdn) nodes);
 
+  # Pin for k8s packages. This is so that upagrading the system will not upgrade the k8s control or data planes.
+  k8spkgs = import (fetchGit {
+    name = "nixos-unstable-2019-04-12";
+    url = https://github.com/nixos/nixpkgs/;
+    rev = "1fc591f9a5bd1b016b5d66dfab29560073955a14";
+  }) {};
+
+
 in rec {
   imports =
     [ # Include the results of the hardware scan.
@@ -17,6 +25,7 @@ in rec {
   boot.loader.grub.version = 2;
   boot.loader.grub.device = node.diskBoot;
 
+  boot.kernelPackages = pkgs.linuxPackages_5_1;
   boot.kernelParams = [ "boot.shell_on_fail" ];
 
   time.timeZone = "Europe/Warsaw";
@@ -49,6 +58,8 @@ in rec {
     '';
     serviceConfig.Type = "oneshot";
   };
+  # Otherwise fetchGit nixpkgs pin fails.
+  systemd.services.nixos-upgrade.path = [ pkgs.git ];
 
   # Enable the OpenSSH daemon.
   services.openssh.enable = true;
@@ -107,6 +118,8 @@ in rec {
   };
 
   services.kubernetes = {
+    # Pin to specific k8s package.
+    package = k8spkgs.kubernetes;
     roles = []; # We do not use any nixpkgs predefined roles for k8s. Instead,
                 # we enable k8s components manually.
 
@@ -209,5 +222,16 @@ in rec {
         --cni-bin-dir=/opt/cni/bin
       '';
     };
+
+  };
+
+  # https://github.com/NixOS/nixpkgs/issues/60687
+  systemd.services.kube-control-plane-online = {
+    preStart = pkgs.lib.mkForce "";
+  };
+  # this seems to depend on flannel
+  # TODO(q3k): file issue
+  systemd.services.kubelet-online = {
+    script = pkgs.lib.mkForce "sleep 1";
   };
 }