1
0
Fork 0

k0: host os bump wip

This bumps it on bc01n01, but nowhere else yet.

We have to vendor some more kubelet bits unfortunately.

Change-Id: Ifb169dd9c2c19d60f88d946d065d4446141601b1
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1465
Reviewed-by: implr <implr@hackerspace.pl>
master
q3k 2022-11-18 14:39:45 +00:00 committed by q3k
parent 45394bf3b0
commit ef3aab6a14
10 changed files with 1575 additions and 11 deletions

View File

@ -29,6 +29,30 @@ in {
description = "IPv4 address of gateway.";
};
};
# Override current nixos kubernetes with our vendored fork.
# Also nuke flannel from orbit.
disabledModules = [
"services/cluster/kubernetes/apiserver.nix"
"services/cluster/kubernetes/controller-manager.nix"
"services/cluster/kubernetes/default.nix"
"services/cluster/kubernetes/kubelet.nix"
"services/cluster/kubernetes/pki.nix"
"services/cluster/kubernetes/proxy.nix"
"services/cluster/kubernetes/scheduler.nix"
"services/cluster/kubernetes/flannel.nix"
];
imports = [
./vendor/apiserver.nix
./vendor/controller-manager.nix
./vendor/default.nix
./vendor/kubelet.nix
./vendor/pki.nix
./vendor/proxy.nix
./vendor/scheduler.nix
];
config = rec {
boot.loader.grub.enable = true;
boot.loader.grub.version = 2;

View File

@ -33,19 +33,10 @@ in {
};
};
# Disable kubelet service and bring in our own override.
# Also nuke flannel from the orbit.
disabledModules = [
"services/cluster/kubernetes/kubelet.nix"
"services/cluster/kubernetes/flannel.nix"
];
imports = [
./kubelet.nix
./kube-common.nix
];
config = mkIf cfg.enable {
# If we're not running the control plane, render a hostsfile that points at
# all other control plane nodes. Otherwise, the control plane module will

View File

@ -0,0 +1,473 @@
# Vendored from nixpkgs git 44ad80ab1036c5cc83ada4bfa451dac9939f2a10
# Copyright (c) 2003-2023 Eelco Dolstra and the Nixpkgs/NixOS contributors
# SPDX-License-Identifier: MIT
{ config, lib, pkgs, ... }:
with lib;
let
top = config.services.kubernetes;
cfg = top.apiserver;
isRBACEnabled = elem "RBAC" cfg.authorizationMode;
apiserverServiceIP = (concatStringsSep "." (
take 3 (splitString "." cfg.serviceClusterIpRange
)) + ".1");
in
{
imports = [
(mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "admissionControl" ] [ "services" "kubernetes" "apiserver" "enableAdmissionPlugins" ])
(mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "address" ] ["services" "kubernetes" "apiserver" "bindAddress"])
(mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "port" ] ["services" "kubernetes" "apiserver" "insecurePort"])
(mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "publicAddress" ] "")
(mkRenamedOptionModule [ "services" "kubernetes" "etcd" "servers" ] [ "services" "kubernetes" "apiserver" "etcd" "servers" ])
(mkRenamedOptionModule [ "services" "kubernetes" "etcd" "keyFile" ] [ "services" "kubernetes" "apiserver" "etcd" "keyFile" ])
(mkRenamedOptionModule [ "services" "kubernetes" "etcd" "certFile" ] [ "services" "kubernetes" "apiserver" "etcd" "certFile" ])
(mkRenamedOptionModule [ "services" "kubernetes" "etcd" "caFile" ] [ "services" "kubernetes" "apiserver" "etcd" "caFile" ])
];
###### interface
options.services.kubernetes.apiserver = with lib.types; {
advertiseAddress = mkOption {
description = ''
Kubernetes apiserver IP address on which to advertise the apiserver
to members of the cluster. This address must be reachable by the rest
of the cluster.
'';
default = null;
type = nullOr str;
};
allowPrivileged = mkOption {
description = "Whether to allow privileged containers on Kubernetes.";
default = false;
type = bool;
};
authorizationMode = mkOption {
description = ''
Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/>
'';
default = ["RBAC" "Node"]; # Enabling RBAC by default, although kubernetes default is AllowAllow
type = listOf (enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "Webhook" "RBAC" "Node"]);
};
authorizationPolicy = mkOption {
description = ''
Kubernetes apiserver authorization policy file. See
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authorization/"/>
'';
default = [];
type = listOf attrs;
};
basicAuthFile = mkOption {
description = ''
Kubernetes apiserver basic authentication file. See
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/>
'';
default = null;
type = nullOr path;
};
bindAddress = mkOption {
description = ''
The IP address on which to listen for the --secure-port port.
The associated interface(s) must be reachable by the rest
of the cluster, and by CLI/web clients.
'';
default = "0.0.0.0";
type = str;
};
clientCaFile = mkOption {
description = "Kubernetes apiserver CA file for client auth.";
default = top.caFile;
type = nullOr path;
};
disableAdmissionPlugins = mkOption {
description = ''
Kubernetes admission control plugins to disable. See
<link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/>
'';
default = [];
type = listOf str;
};
enable = mkEnableOption "Kubernetes apiserver";
enableAdmissionPlugins = mkOption {
description = ''
Kubernetes admission control plugins to enable. See
<link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/>
'';
default = [
"NamespaceLifecycle" "LimitRanger" "ServiceAccount"
"ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds"
"NodeRestriction"
];
example = [
"NamespaceLifecycle" "NamespaceExists" "LimitRanger"
"SecurityContextDeny" "ServiceAccount" "ResourceQuota"
"PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass"
];
type = listOf str;
};
etcd = {
servers = mkOption {
description = "List of etcd servers.";
default = ["http://127.0.0.1:2379"];
type = types.listOf types.str;
};
keyFile = mkOption {
description = "Etcd key file.";
default = null;
type = types.nullOr types.path;
};
certFile = mkOption {
description = "Etcd cert file.";
default = null;
type = types.nullOr types.path;
};
caFile = mkOption {
description = "Etcd ca file.";
default = top.caFile;
type = types.nullOr types.path;
};
};
extraOpts = mkOption {
description = "Kubernetes apiserver extra command line options.";
default = "";
type = str;
};
extraSANs = mkOption {
description = "Extra x509 Subject Alternative Names to be added to the kubernetes apiserver tls cert.";
default = [];
type = listOf str;
};
featureGates = mkOption {
description = "List set of feature gates";
default = top.featureGates;
type = listOf str;
};
insecureBindAddress = mkOption {
description = "The IP address on which to serve the --insecure-port.";
default = "127.0.0.1";
type = str;
};
insecurePort = mkOption {
description = "Kubernetes apiserver insecure listening port. (0 = disabled)";
default = 0;
type = int;
};
kubeletClientCaFile = mkOption {
description = "Path to a cert file for connecting to kubelet.";
default = top.caFile;
type = nullOr path;
};
kubeletClientCertFile = mkOption {
description = "Client certificate to use for connections to kubelet.";
default = null;
type = nullOr path;
};
kubeletClientKeyFile = mkOption {
description = "Key to use for connections to kubelet.";
default = null;
type = nullOr path;
};
kubeletHttps = mkOption {
description = "Whether to use https for connections to kubelet.";
default = true;
type = bool;
};
preferredAddressTypes = mkOption {
description = "List of the preferred NodeAddressTypes to use for kubelet connections.";
type = nullOr str;
default = null;
};
proxyClientCertFile = mkOption {
description = "Client certificate to use for connections to proxy.";
default = null;
type = nullOr path;
};
proxyClientKeyFile = mkOption {
description = "Key to use for connections to proxy.";
default = null;
type = nullOr path;
};
runtimeConfig = mkOption {
description = ''
Api runtime configuration. See
<link xlink:href="https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/"/>
'';
default = "authentication.k8s.io/v1beta1=true";
example = "api/all=false,api/v1=true";
type = str;
};
storageBackend = mkOption {
description = ''
Kubernetes apiserver storage backend.
'';
default = "etcd3";
type = enum ["etcd2" "etcd3"];
};
securePort = mkOption {
description = "Kubernetes apiserver secure port.";
default = 6443;
type = int;
};
serviceAccountKeyFile = mkOption {
description = ''
Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
used to verify ServiceAccount tokens. By default tls private key file
is used.
'';
default = null;
type = nullOr path;
};
serviceClusterIpRange = mkOption {
description = ''
A CIDR notation IP range from which to assign service cluster IPs.
This must not overlap with any IP ranges assigned to nodes for pods.
'';
default = "10.0.0.0/24";
type = str;
};
tlsCertFile = mkOption {
description = "Kubernetes apiserver certificate file.";
default = null;
type = nullOr path;
};
tlsKeyFile = mkOption {
description = "Kubernetes apiserver private key file.";
default = null;
type = nullOr path;
};
tokenAuthFile = mkOption {
description = ''
Kubernetes apiserver token authentication file. See
<link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/authentication"/>
'';
default = null;
type = nullOr path;
};
verbosity = mkOption {
description = ''
Optional glog verbosity level for logging statements. See
<link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/>
'';
default = null;
type = nullOr int;
};
webhookConfig = mkOption {
description = ''
Kubernetes apiserver Webhook config file. It uses the kubeconfig file format.
See <link xlink:href="https://kubernetes.io/docs/reference/access-authn-authz/webhook/"/>
'';
default = null;
type = nullOr path;
};
};
###### implementation
config = mkMerge [
(mkIf cfg.enable {
systemd.services.kube-apiserver = {
description = "Kubernetes APIServer Service";
wantedBy = [ "kubernetes.target" ];
after = [ "network.target" ];
serviceConfig = {
Slice = "kubernetes.slice";
ExecStart = ''${top.package}/bin/kube-apiserver \
--allow-privileged=${boolToString cfg.allowPrivileged} \
--authorization-mode=${concatStringsSep "," cfg.authorizationMode} \
${optionalString (elem "ABAC" cfg.authorizationMode)
"--authorization-policy-file=${
pkgs.writeText "kube-auth-policy.jsonl"
(concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.authorizationPolicy)
}"
} \
${optionalString (elem "Webhook" cfg.authorizationMode)
"--authorization-webhook-config-file=${cfg.webhookConfig}"
} \
--bind-address=${cfg.bindAddress} \
${optionalString (cfg.advertiseAddress != null)
"--advertise-address=${cfg.advertiseAddress}"} \
${optionalString (cfg.clientCaFile != null)
"--client-ca-file=${cfg.clientCaFile}"} \
--disable-admission-plugins=${concatStringsSep "," cfg.disableAdmissionPlugins} \
--enable-admission-plugins=${concatStringsSep "," cfg.enableAdmissionPlugins} \
--etcd-servers=${concatStringsSep "," cfg.etcd.servers} \
${optionalString (cfg.etcd.caFile != null)
"--etcd-cafile=${cfg.etcd.caFile}"} \
${optionalString (cfg.etcd.certFile != null)
"--etcd-certfile=${cfg.etcd.certFile}"} \
${optionalString (cfg.etcd.keyFile != null)
"--etcd-keyfile=${cfg.etcd.keyFile}"} \
${optionalString (cfg.featureGates != [])
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
${optionalString (cfg.basicAuthFile != null)
"--basic-auth-file=${cfg.basicAuthFile}"} \
--kubelet-https=${boolToString cfg.kubeletHttps} \
${optionalString (cfg.kubeletClientCaFile != null)
"--kubelet-certificate-authority=${cfg.kubeletClientCaFile}"} \
${optionalString (cfg.kubeletClientCertFile != null)
"--kubelet-client-certificate=${cfg.kubeletClientCertFile}"} \
${optionalString (cfg.kubeletClientKeyFile != null)
"--kubelet-client-key=${cfg.kubeletClientKeyFile}"} \
${optionalString (cfg.preferredAddressTypes != null)
"--kubelet-preferred-address-types=${cfg.preferredAddressTypes}"} \
${optionalString (cfg.proxyClientCertFile != null)
"--proxy-client-cert-file=${cfg.proxyClientCertFile}"} \
${optionalString (cfg.proxyClientKeyFile != null)
"--proxy-client-key-file=${cfg.proxyClientKeyFile}"} \
--insecure-bind-address=${cfg.insecureBindAddress} \
--insecure-port=${toString cfg.insecurePort} \
${optionalString (cfg.runtimeConfig != "")
"--runtime-config=${cfg.runtimeConfig}"} \
--secure-port=${toString cfg.securePort} \
${optionalString (cfg.serviceAccountKeyFile!=null)
"--service-account-key-file=${cfg.serviceAccountKeyFile}"} \
--service-cluster-ip-range=${cfg.serviceClusterIpRange} \
--storage-backend=${cfg.storageBackend} \
${optionalString (cfg.tlsCertFile != null)
"--tls-cert-file=${cfg.tlsCertFile}"} \
${optionalString (cfg.tlsKeyFile != null)
"--tls-private-key-file=${cfg.tlsKeyFile}"} \
${optionalString (cfg.tokenAuthFile != null)
"--token-auth-file=${cfg.tokenAuthFile}"} \
${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
${cfg.extraOpts}
'';
WorkingDirectory = top.dataDir;
User = "kubernetes";
Group = "kubernetes";
AmbientCapabilities = "cap_net_bind_service";
Restart = "on-failure";
RestartSec = 5;
};
};
services.etcd = {
clientCertAuth = mkDefault true;
peerClientCertAuth = mkDefault true;
listenClientUrls = mkDefault ["https://0.0.0.0:2379"];
listenPeerUrls = mkDefault ["https://0.0.0.0:2380"];
advertiseClientUrls = mkDefault ["https://${top.masterAddress}:2379"];
initialCluster = mkDefault ["${top.masterAddress}=https://${top.masterAddress}:2380"];
name = mkDefault top.masterAddress;
initialAdvertisePeerUrls = mkDefault ["https://${top.masterAddress}:2380"];
};
services.kubernetes.addonManager.bootstrapAddons = mkIf isRBACEnabled {
apiserver-kubelet-api-admin-crb = {
apiVersion = "rbac.authorization.k8s.io/v1";
kind = "ClusterRoleBinding";
metadata = {
name = "system:kube-apiserver:kubelet-api-admin";
};
roleRef = {
apiGroup = "rbac.authorization.k8s.io";
kind = "ClusterRole";
name = "system:kubelet-api-admin";
};
subjects = [{
kind = "User";
name = "system:kube-apiserver";
}];
};
};
services.kubernetes.pki.certs = with top.lib; {
apiServer = mkCert {
name = "kube-apiserver";
CN = "kubernetes";
hosts = [
"kubernetes.default.svc"
"kubernetes.default.svc.${top.addons.dns.clusterDomain}"
cfg.advertiseAddress
top.masterAddress
apiserverServiceIP
"127.0.0.1"
] ++ cfg.extraSANs;
action = "systemctl restart kube-apiserver.service";
};
apiserverProxyClient = mkCert {
name = "kube-apiserver-proxy-client";
CN = "front-proxy-client";
action = "systemctl restart kube-apiserver.service";
};
apiserverKubeletClient = mkCert {
name = "kube-apiserver-kubelet-client";
CN = "system:kube-apiserver";
action = "systemctl restart kube-apiserver.service";
};
apiserverEtcdClient = mkCert {
name = "kube-apiserver-etcd-client";
CN = "etcd-client";
action = "systemctl restart kube-apiserver.service";
};
clusterAdmin = mkCert {
name = "cluster-admin";
CN = "cluster-admin";
fields = {
O = "system:masters";
};
privateKeyOwner = "root";
};
etcd = mkCert {
name = "etcd";
CN = top.masterAddress;
hosts = [
"etcd.local"
"etcd.${top.addons.dns.clusterDomain}"
top.masterAddress
cfg.advertiseAddress
];
privateKeyOwner = "etcd";
action = "systemctl restart etcd.service";
};
};
})
];
}

View File

@ -0,0 +1,171 @@
# Vendored from nixpkgs git 44ad80ab1036c5cc83ada4bfa451dac9939f2a10
# Copyright (c) 2003-2023 Eelco Dolstra and the Nixpkgs/NixOS contributors
# SPDX-License-Identifier: MIT
{ config, lib, pkgs, ... }:
with lib;
let
top = config.services.kubernetes;
cfg = top.controllerManager;
in
{
imports = [
(mkRenamedOptionModule [ "services" "kubernetes" "controllerManager" "address" ] ["services" "kubernetes" "controllerManager" "bindAddress"])
(mkRenamedOptionModule [ "services" "kubernetes" "controllerManager" "port" ] ["services" "kubernetes" "controllerManager" "insecurePort"])
];
###### interface
options.services.kubernetes.controllerManager = with lib.types; {
allocateNodeCIDRs = mkOption {
description = "Whether to automatically allocate CIDR ranges for cluster nodes.";
default = true;
type = bool;
};
bindAddress = mkOption {
description = "Kubernetes controller manager listening address.";
default = "127.0.0.1";
type = str;
};
clusterCidr = mkOption {
description = "Kubernetes CIDR Range for Pods in cluster.";
default = top.clusterCidr;
type = str;
};
enable = mkEnableOption "Kubernetes controller manager";
extraOpts = mkOption {
description = "Kubernetes controller manager extra command line options.";
default = "";
type = str;
};
featureGates = mkOption {
description = "List set of feature gates";
default = top.featureGates;
type = listOf str;
};
insecurePort = mkOption {
description = "Kubernetes controller manager insecure listening port.";
default = 0;
type = int;
};
kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes controller manager";
leaderElect = mkOption {
description = "Whether to start leader election before executing main loop.";
type = bool;
default = true;
};
rootCaFile = mkOption {
description = ''
Kubernetes controller manager certificate authority file included in
service account's token secret.
'';
default = top.caFile;
type = nullOr path;
};
securePort = mkOption {
description = "Kubernetes controller manager secure listening port.";
default = 10252;
type = int;
};
serviceAccountKeyFile = mkOption {
description = ''
Kubernetes controller manager PEM-encoded private RSA key file used to
sign service account tokens
'';
default = null;
type = nullOr path;
};
tlsCertFile = mkOption {
description = "Kubernetes controller-manager certificate file.";
default = null;
type = nullOr path;
};
tlsKeyFile = mkOption {
description = "Kubernetes controller-manager private key file.";
default = null;
type = nullOr path;
};
verbosity = mkOption {
description = ''
Optional glog verbosity level for logging statements. See
<link xlink:href="https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md"/>
'';
default = null;
type = nullOr int;
};
};
###### implementation
config = mkIf cfg.enable {
systemd.services.kube-controller-manager = {
description = "Kubernetes Controller Manager Service";
wantedBy = [ "kubernetes.target" ];
after = [ "kube-apiserver.service" ];
serviceConfig = {
RestartSec = "30s";
Restart = "on-failure";
Slice = "kubernetes.slice";
ExecStart = ''${top.package}/bin/kube-controller-manager \
--allocate-node-cidrs=${boolToString cfg.allocateNodeCIDRs} \
--bind-address=${cfg.bindAddress} \
${optionalString (cfg.clusterCidr!=null)
"--cluster-cidr=${cfg.clusterCidr}"} \
${optionalString (cfg.featureGates != [])
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \
--kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \
--leader-elect=${boolToString cfg.leaderElect} \
${optionalString (cfg.rootCaFile!=null)
"--root-ca-file=${cfg.rootCaFile}"} \
--port=${toString cfg.insecurePort} \
--secure-port=${toString cfg.securePort} \
${optionalString (cfg.serviceAccountKeyFile!=null)
"--service-account-private-key-file=${cfg.serviceAccountKeyFile}"} \
${optionalString (cfg.tlsCertFile!=null)
"--tls-cert-file=${cfg.tlsCertFile}"} \
${optionalString (cfg.tlsKeyFile!=null)
"--tls-private-key-file=${cfg.tlsKeyFile}"} \
${optionalString (elem "RBAC" top.apiserver.authorizationMode)
"--use-service-account-credentials"} \
${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
${cfg.extraOpts}
'';
WorkingDirectory = top.dataDir;
User = "kubernetes";
Group = "kubernetes";
};
path = top.path;
};
services.kubernetes.pki.certs = with top.lib; {
controllerManager = mkCert {
name = "kube-controller-manager";
CN = "kube-controller-manager";
action = "systemctl restart kube-controller-manager.service";
};
controllerManagerClient = mkCert {
name = "kube-controller-manager-client";
CN = "system:kube-controller-manager";
action = "systemctl restart kube-controller-manager.service";
};
};
services.kubernetes.controllerManager.kubeconfig.server = mkDefault top.apiserverAddress;
};
}

View File

@ -0,0 +1,291 @@
# Vendored from nixpkgs git 44ad80ab1036c5cc83ada4bfa451dac9939f2a10
# Copyright (c) 2003-2023 Eelco Dolstra and the Nixpkgs/NixOS contributors
# SPDX-License-Identifier: MIT
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.kubernetes;
mkKubeConfig = name: conf: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON {
apiVersion = "v1";
kind = "Config";
clusters = [{
name = "local";
cluster.certificate-authority = conf.caFile or cfg.caFile;
cluster.server = conf.server;
}];
users = [{
inherit name;
user = {
client-certificate = conf.certFile;
client-key = conf.keyFile;
};
}];
contexts = [{
context = {
cluster = "local";
user = name;
};
current-context = "local";
}];
});
caCert = secret "ca";
etcdEndpoints = ["https://${cfg.masterAddress}:2379"];
mkCert = { name, CN, hosts ? [], fields ? {}, action ? "",
privateKeyOwner ? "kubernetes" }: rec {
inherit name caCert CN hosts fields action;
cert = secret name;
key = secret "${name}-key";
privateKeyOptions = {
owner = privateKeyOwner;
group = "nogroup";
mode = "0600";
path = key;
};
};
secret = name: "${cfg.secretsPath}/${name}.pem";
mkKubeConfigOptions = prefix: {
server = mkOption {
description = "${prefix} kube-apiserver server address.";
type = types.str;
};
caFile = mkOption {
description = "${prefix} certificate authority file used to connect to kube-apiserver.";
type = types.nullOr types.path;
default = cfg.caFile;
};
certFile = mkOption {
description = "${prefix} client certificate file used to connect to kube-apiserver.";
type = types.nullOr types.path;
default = null;
};
keyFile = mkOption {
description = "${prefix} client key file used to connect to kube-apiserver.";
type = types.nullOr types.path;
default = null;
};
};
in {
imports = [
(mkRemovedOptionModule [ "services" "kubernetes" "verbose" ] "")
];
###### interface
options.services.kubernetes = {
roles = mkOption {
description = ''
Kubernetes role that this machine should take.
Master role will enable etcd, apiserver, scheduler, controller manager
addon manager, flannel and proxy services.
Node role will enable flannel, docker, kubelet and proxy services.
'';
default = [];
type = types.listOf (types.enum ["master" "node"]);
};
package = mkOption {
description = "Kubernetes package to use.";
type = types.package;
default = pkgs.kubernetes;
defaultText = "pkgs.kubernetes";
};
kubeconfig = mkKubeConfigOptions "Default kubeconfig";
apiserverAddress = mkOption {
description = ''
Clusterwide accessible address for the kubernetes apiserver,
including protocol and optional port.
'';
example = "https://kubernetes-apiserver.example.com:6443";
type = types.str;
};
caFile = mkOption {
description = "Default kubernetes certificate authority";
type = types.nullOr types.path;
default = null;
};
dataDir = mkOption {
description = "Kubernetes root directory for managing kubelet files.";
default = "/var/lib/kubernetes";
type = types.path;
};
easyCerts = mkOption {
description = "Automatically setup x509 certificates and keys for the entire cluster.";
default = false;
type = types.bool;
};
featureGates = mkOption {
description = "List set of feature gates.";
default = [];
type = types.listOf types.str;
};
masterAddress = mkOption {
description = "Clusterwide available network address or hostname for the kubernetes master server.";
example = "master.example.com";
type = types.str;
};
path = mkOption {
description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
type = types.listOf types.package;
default = [];
};
clusterCidr = mkOption {
description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
default = "10.1.0.0/16";
type = types.nullOr types.str;
};
lib = mkOption {
description = "Common functions for the kubernetes modules.";
default = {
inherit mkCert;
inherit mkKubeConfig;
inherit mkKubeConfigOptions;
};
type = types.attrs;
};
secretsPath = mkOption {
description = "Default location for kubernetes secrets. Not a store location.";
type = types.path;
default = cfg.dataDir + "/secrets";
};
};
###### implementation
config = mkMerge [
(mkIf cfg.easyCerts {
services.kubernetes.pki.enable = mkDefault true;
services.kubernetes.caFile = caCert;
})
(mkIf (elem "master" cfg.roles) {
services.kubernetes.apiserver.enable = mkDefault true;
services.kubernetes.scheduler.enable = mkDefault true;
services.kubernetes.controllerManager.enable = mkDefault true;
services.kubernetes.addonManager.enable = mkDefault true;
services.kubernetes.proxy.enable = mkDefault true;
services.etcd.enable = true; # Cannot mkDefault because of flannel default options
services.kubernetes.kubelet = {
enable = mkDefault true;
taints = mkIf (!(elem "node" cfg.roles)) {
master = {
key = "node-role.kubernetes.io/master";
value = "true";
effect = "NoSchedule";
};
};
};
})
(mkIf (all (el: el == "master") cfg.roles) {
# if this node is only a master make it unschedulable by default
services.kubernetes.kubelet.unschedulable = mkDefault true;
})
(mkIf (elem "node" cfg.roles) {
services.kubernetes.kubelet.enable = mkDefault true;
services.kubernetes.proxy.enable = mkDefault true;
})
# Using "services.kubernetes.roles" will automatically enable easyCerts and flannel
(mkIf (cfg.roles != []) {
services.kubernetes.flannel.enable = mkDefault true;
services.flannel.etcd.endpoints = mkDefault etcdEndpoints;
services.kubernetes.easyCerts = mkDefault true;
})
(mkIf cfg.apiserver.enable {
services.kubernetes.pki.etcClusterAdminKubeconfig = mkDefault "kubernetes/cluster-admin.kubeconfig";
services.kubernetes.apiserver.etcd.servers = mkDefault etcdEndpoints;
})
(mkIf cfg.kubelet.enable {
virtualisation.docker = {
enable = mkDefault true;
# kubernetes needs access to logs
logDriver = mkDefault "json-file";
# iptables must be disabled for kubernetes
extraOptions = "--iptables=false --ip-masq=false";
};
})
(mkIf (cfg.apiserver.enable || cfg.controllerManager.enable) {
services.kubernetes.pki.certs = {
serviceAccount = mkCert {
name = "service-account";
CN = "system:service-account-signer";
action = ''
systemctl reload \
kube-apiserver.service \
kube-controller-manager.service
'';
};
};
})
(mkIf (
cfg.apiserver.enable ||
cfg.scheduler.enable ||
cfg.controllerManager.enable ||
cfg.kubelet.enable ||
cfg.proxy.enable ||
cfg.addonManager.enable
) {
systemd.targets.kubernetes = {
description = "Kubernetes";
wantedBy = [ "multi-user.target" ];
};
systemd.tmpfiles.rules = [
"d /opt/cni/bin 0755 root root -"
"d /run/kubernetes 0755 kubernetes kubernetes -"
"d /var/lib/kubernetes 0755 kubernetes kubernetes -"
];
users.users.kubernetes = {
uid = config.ids.uids.kubernetes;
description = "Kubernetes user";
extraGroups = [ "docker" ];
group = "kubernetes";
home = cfg.dataDir;
createHome = true;
};
users.groups.kubernetes.gid = config.ids.gids.kubernetes;
# dns addon is enabled by default
services.kubernetes.addons.dns.enable = mkDefault true;
services.kubernetes.apiserverAddress = mkDefault ("https://${if cfg.apiserver.advertiseAddress != null
then cfg.apiserver.advertiseAddress
else "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"}");
})
];
}

View File

@ -1,3 +1,7 @@
# Vendored from nixpkgs git 44ad80ab1036c5cc83ada4bfa451dac9939f2a10
# Copyright (c) 2003-2023 Eelco Dolstra and the Nixpkgs/NixOS contributors
# SPDX-License-Identifier: MIT
#
# Same as upstream kubelet.nix module from nixpkgs, but with the following
# changes:
# - cni tunables nuked and replaced with static host dirs, so that calico
@ -226,7 +230,7 @@ in
TasksMax = "infinity";
OOMScoreAdjust = "-999";
ExecStart = "${pkgs.containerd}/bin/containerd -c ${./containerd.toml}";
ExecStart = "${pkgs.containerd}/bin/containerd -c ${../containerd.toml}";
};
};

405
cluster/machines/modules/vendor/pki.nix vendored Normal file
View File

@ -0,0 +1,405 @@
# Vendored from nixpkgs git 44ad80ab1036c5cc83ada4bfa451dac9939f2a10
# Copyright (c) 2003-2023 Eelco Dolstra and the Nixpkgs/NixOS contributors
# SPDX-License-Identifier: MIT
{ config, lib, pkgs, ... }:
with lib;
let
top = config.services.kubernetes;
cfg = top.pki;
csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (builtins.toJSON {
key = {
algo = "rsa";
size = 2048;
};
names = singleton cfg.caSpec;
});
csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (builtins.toJSON {
key = {
algo = "rsa";
size = 2048;
};
CN = top.masterAddress;
hosts = [top.masterAddress] ++ cfg.cfsslAPIExtraSANs;
});
cfsslAPITokenBaseName = "apitoken.secret";
cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}";
certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
cfsslAPITokenLength = 32;
clusterAdminKubeconfig = with cfg.certs.clusterAdmin;
top.lib.mkKubeConfig "cluster-admin" {
server = top.apiserverAddress;
certFile = cert;
keyFile = key;
};
remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
in
{
###### interface
options.services.kubernetes.pki = with lib.types; {
enable = mkEnableOption "easyCert issuer service";
certs = mkOption {
description = "List of certificate specs to feed to cert generator.";
default = {};
type = attrs;
};
genCfsslCACert = mkOption {
description = ''
Whether to automatically generate cfssl CA certificate and key,
if they don't exist.
'';
default = true;
type = bool;
};
genCfsslAPICerts = mkOption {
description = ''
Whether to automatically generate cfssl API webserver TLS cert and key,
if they don't exist.
'';
default = true;
type = bool;
};
cfsslAPIExtraSANs = mkOption {
description = ''
Extra x509 Subject Alternative Names to be added to the cfssl API webserver TLS cert.
'';
default = [];
example = [ "subdomain.example.com" ];
type = listOf str;
};
genCfsslAPIToken = mkOption {
description = ''
Whether to automatically generate cfssl API-token secret,
if they doesn't exist.
'';
default = true;
type = bool;
};
pkiTrustOnBootstrap = mkOption {
description = "Whether to always trust remote cfssl server upon initial PKI bootstrap.";
default = true;
type = bool;
};
caCertPathPrefix = mkOption {
description = ''
Path-prefrix for the CA-certificate to be used for cfssl signing.
Suffixes ".pem" and "-key.pem" will be automatically appended for
the public and private keys respectively.
'';
default = "${config.services.cfssl.dataDir}/ca";
type = str;
};
caSpec = mkOption {
description = "Certificate specification for the auto-generated CAcert.";
default = {
CN = "kubernetes-cluster-ca";
O = "NixOS";
OU = "services.kubernetes.pki.caSpec";
L = "auto-generated";
};
type = attrs;
};
etcClusterAdminKubeconfig = mkOption {
description = ''
Symlink a kubeconfig with cluster-admin privileges to environment path
(/etc/&lt;path&gt;).
'';
default = null;
type = nullOr str;
};
};
###### implementation
config = mkIf cfg.enable
(let
cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
cfsslCert = "${cfsslCertPathPrefix}.pem";
cfsslKey = "${cfsslCertPathPrefix}-key.pem";
in
{
services.cfssl = mkIf (top.apiserver.enable) {
enable = true;
address = "0.0.0.0";
tlsCert = cfsslCert;
tlsKey = cfsslKey;
configFile = toString (pkgs.writeText "cfssl-config.json" (builtins.toJSON {
signing = {
profiles = {
default = {
usages = ["digital signature"];
auth_key = "default";
expiry = "720h";
};
};
};
auth_keys = {
default = {
type = "standard";
key = "file:${cfsslAPITokenPath}";
};
};
}));
};
systemd.services.cfssl.preStart = with pkgs; with config.services.cfssl; mkIf (top.apiserver.enable)
(concatStringsSep "\n" [
"set -e"
(optionalString cfg.genCfsslCACert ''
if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then
${cfssl}/bin/cfssl genkey -initca ${csrCA} | \
${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix}
fi
'')
(optionalString cfg.genCfsslAPICerts ''
if [ ! -f "${dataDir}/cfssl.pem" ]; then
${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \
${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix}
fi
'')
(optionalString cfg.genCfsslAPIToken ''
if [ ! -f "${cfsslAPITokenPath}" ]; then
head -c ${toString (cfsslAPITokenLength / 2)} /dev/urandom | od -An -t x | tr -d ' ' >"${cfsslAPITokenPath}"
fi
chown cfssl "${cfsslAPITokenPath}" && chmod 400 "${cfsslAPITokenPath}"
'')]);
systemd.services.kube-certmgr-bootstrap = {
description = "Kubernetes certmgr bootstrapper";
wantedBy = [ "certmgr.service" ];
after = [ "cfssl.target" ];
script = concatStringsSep "\n" [''
set -e
# If there's a cfssl (cert issuer) running locally, then don't rely on user to
# manually paste it in place. Just symlink.
# otherwise, create the target file, ready for users to insert the token
if [ -f "${cfsslAPITokenPath}" ]; then
ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}"
else
touch "${certmgrAPITokenPath}" && chmod 600 "${certmgrAPITokenPath}"
fi
''
(optionalString (cfg.pkiTrustOnBootstrap) ''
if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
fi
'')
];
serviceConfig = {
RestartSec = "10s";
Restart = "on-failure";
};
};
services.certmgr = {
enable = true;
package = pkgs.certmgr-selfsigned;
svcManager = "command";
specs =
let
mkSpec = _: cert: {
inherit (cert) action;
authority = {
inherit remote;
file.path = cert.caCert;
root_ca = cert.caCert;
profile = "default";
auth_key_file = certmgrAPITokenPath;
};
certificate = {
path = cert.cert;
};
private_key = cert.privateKeyOptions;
request = {
hosts = [cert.CN] ++ cert.hosts;
<