diff --git a/cluster/nix/module-cluster.nix b/cluster/nix/module-cluster.nix deleted file mode 100644 index cc505032..00000000 --- a/cluster/nix/module-cluster.nix +++ /dev/null @@ -1,214 +0,0 @@ -{ config, pkgs, lib, ... }: - -with (( import ./defs-cluster-k0.nix ) config.networking.hostName); -let - # Pin for k8s packages. This is so that upagrading the system will not upgrade the k8s control or data planes. - k8spkgs = import (fetchGit { - # Now at 1.14.3 - name = "nixos-unstable-2019-06-17"; - url = https://github.com/nixos/nixpkgs-channels/; - rev = "415e8e5820b7825fb74a6c7986bf6af725227eaa"; - }) {}; - # Pin for kubelet - k8spkgsKubelet = import (fetchGit { - # Now at 1.13.5 - name = "nixos-unstable-2019-04-12"; - url = https://github.com/nixos/nixpkgs-channels/; - rev = "1fc591f9a5bd1b016b5d66dfab29560073955a14"; - }) {}; - -in rec { - # Disable kubelet service and bring in our own override. - # Also nuke flannel from the orbit. - disabledModules = [ - "services/cluster/kubernetes/kubelet.nix" - "services/cluster/kubernetes/flannel.nix" - ]; - - imports = [ - ./module-kubelet.nix - ]; - - virtualisation.docker.enable = true; - virtualisation.docker.extraOptions = "--iptables=false --ip-masq=false --ip-forward=true"; - - # Docker 1.13 sets iptables FORWARD to DROP. Unfuck this. - systemd.services."docker-iptables-unfuck" = { - enable = true; - wantedBy = [ "kubernetes.target" ]; - description = "Docker iptable Unfuck"; - after = [ "docker.service" ]; - requires = [ "docker.service" ]; - path = [ pkgs.iptables ]; - script = '' - iptables -P FORWARD ACCEPT - ''; - serviceConfig.Type = "oneshot"; - }; - - networking.firewall.enable = false; - - # Point k8s apiserver address at ourselves, as every machine runs an apiserver with this cert name. - networking.extraHosts = '' - 127.0.0.1 ${k8sapi} - ''; - - security.acme.certs = { - host = { - email = acmeEmail; - domain = fqdn; - webroot = services.nginx.virtualHosts.host.root; - }; - }; - - services.nginx = { - enable = true; - virtualHosts.host = { - serverName = fqdn; - root = "/var/www/${fqdn}"; - }; - }; - - services.etcd = rec { - enable = true; - name = fqdn; - listenClientUrls = ["https://0.0.0.0:2379"]; - advertiseClientUrls = ["https://${fqdn}:2379"]; - listenPeerUrls = ["https://0.0.0.0:2380"]; - initialAdvertisePeerUrls = ["https://${fqdn}:2380"]; - initialCluster = (map (n: "${n.fqdn}=https://${n.fqdn}:2380") machines); - initialClusterState = "existing"; - - clientCertAuth = true; - trustedCaFile = pki.etcd.server.ca; - certFile = pki.etcd.server.cert; - keyFile = pki.etcd.server.key; - - peerClientCertAuth = true; - peerTrustedCaFile = pki.etcdPeer.ca; - peerCertFile = pki.etcdPeer.cert; - peerKeyFile = pki.etcdPeer.key; - - extraConf = { - PEER_CLIENT_CERT_AUTH = "true"; - }; - }; - - services.kubernetes = { - # Pin to specific k8s package. - package = k8spkgs.kubernetes; - roles = []; # We do not use any nixpkgs predefined roles for k8s. Instead, - # we enable k8s components manually. - - caFile = pki.kube.apiserver.ca; - clusterCidr = "10.10.16.0/20"; - - path = [ pkgs.e2fsprogs ]; # kubelet wants to mkfs.ext4 when mounting pvcs - - addons.dns.enable = false; - - apiserver = rec { - enable = true; - insecurePort = ports.k8sAPIServerPlain; - securePort = ports.k8sAPIServerSecure; - advertiseAddress = "${machine.ipAddr}"; - - etcd = { - # https://github.com/kubernetes/kubernetes/issues/72102 - servers = (map (n: "https://${n.fqdn}:2379") ( [ machine ] )); - caFile = pki.etcd.kube.ca; - keyFile = pki.etcd.kube.key; - certFile = pki.etcd.kube.cert; - }; - - tlsCertFile = pki.kube.apiserver.cert; - tlsKeyFile = pki.kube.apiserver.key; - - clientCaFile = pki.kube.apiserver.ca; - - kubeletHttps = true; - kubeletClientCaFile = pki.kube.apiserver.ca; - kubeletClientCertFile = pki.kube.apiserver.cert; - kubeletClientKeyFile = pki.kube.apiserver.key; - - serviceAccountKeyFile = pki.kube.serviceaccounts.key; - - allowPrivileged = true; - serviceClusterIpRange = "10.10.12.0/24"; - runtimeConfig = "api/all,authentication.k8s.io/v1beta1"; - authorizationMode = ["Node" "RBAC"]; - enableAdmissionPlugins = ["NamespaceLifecycle" "NodeRestriction" "LimitRanger" "ServiceAccount" "DefaultStorageClass" "ResourceQuota" "PodSecurityPolicy"]; - extraOpts = '' - --apiserver-count=5 \ - --proxy-client-cert-file=${pki.kubeFront.apiserver.cert} \ - --proxy-client-key-file=${pki.kubeFront.apiserver.key} \ - --requestheader-allowed-names= \ - --requestheader-client-ca-file=${pki.kubeFront.apiserver.ca} \ - --requestheader-extra-headers-prefix=X-Remote-Extra- \ - --requestheader-group-headers=X-Remote-Group \ - --requestheader-username-headers=X-Remote-User \ - -v=5 - ''; - }; - - controllerManager = { - enable = true; - bindAddress = "0.0.0.0"; - insecurePort = ports.k8sControllerManagerPlain; - leaderElect = true; - serviceAccountKeyFile = pki.kube.serviceaccounts.key; - rootCaFile = pki.kube.ca; - extraOpts = '' - --service-cluster-ip-range=10.10.12.0/24 \ - --use-service-account-credentials=true \ - --secure-port=${toString ports.k8sControllerManagerSecure}\ - ''; - kubeconfig = pki.kube.controllermanager.config; - }; - - scheduler = { - enable = true; - address = "0.0.0.0"; - port = 0; - leaderElect = true; - kubeconfig = pki.kube.scheduler.config; - }; - - proxy = { - enable = true; - kubeconfig = pki.kube.proxy.config; - extraOpts = '' - --hostname-override=${fqdn}\ - --proxy-mode=iptables - ''; - }; - - kubelet = { - enable = true; - unschedulable = false; - hostname = fqdn; - tlsCertFile = pki.kube.kubelet.cert; - tlsKeyFile = pki.kube.kubelet.key; - clientCaFile = pki.kube.kubelet.ca; - nodeIp = machine.ipAddr; - networkPlugin = "cni"; - clusterDns = "10.10.12.254"; - kubeconfig = pki.kube.kubelet.config; - extraOpts = '' - --read-only-port=0 - ''; - package = k8spkgsKubelet.kubernetes; - }; - - }; - - # https://github.com/NixOS/nixpkgs/issues/60687 - systemd.services.kube-control-plane-online = { - preStart = pkgs.lib.mkForce ""; - }; - # this seems to depend on flannel - # TODO(q3k): file issue - systemd.services.kubelet-online = { - script = pkgs.lib.mkForce "sleep 1"; - }; -}