diff --git a/cluster/kube/cluster.libsonnet b/cluster/kube/cluster.libsonnet index 1826b0c5..6e9da288 100644 --- a/cluster/kube/cluster.libsonnet +++ b/cluster/kube/cluster.libsonnet @@ -10,6 +10,7 @@ local policies = import "../../kube/policies.libsonnet"; local calico = import "lib/calico.libsonnet"; local certmanager = import "lib/cert-manager.libsonnet"; local coredns = import "lib/coredns.libsonnet"; +local identd = import "lib/identd.libsonnet"; local metallb = import "lib/metallb.libsonnet"; local metrics = import "lib/metrics.libsonnet"; local nginx = import "lib/nginx.libsonnet"; @@ -204,6 +205,9 @@ local pki = import "lib/pki.libsonnet"; }, }, + // Ident service + identd: identd.Environment {}, + // Rook Ceph storage operator. rook: rook.Operator { operator+: { diff --git a/cluster/kube/k0-identd.jsonnet b/cluster/kube/k0-identd.jsonnet new file mode 100644 index 00000000..8cb0202f --- /dev/null +++ b/cluster/kube/k0-identd.jsonnet @@ -0,0 +1,7 @@ +// Only the identd instance in k0. + +local k0 = (import "k0.libsonnet").k0; + +{ + identd: k0.cluster.identd, +} diff --git a/cluster/kube/lib/identd.libsonnet b/cluster/kube/lib/identd.libsonnet new file mode 100644 index 00000000..dcc181b2 --- /dev/null +++ b/cluster/kube/lib/identd.libsonnet @@ -0,0 +1,112 @@ +// Deploys identd, an ident protocol (RFC1413) service on all cluster nodes. +// +// See //cluster/identd for more information about the service and how it +// works. +// +// Deployment notes: +// - We run the service within the host network namespace, as that's the only +// way to reliably route traffic destined for a given node into a service +// running on that node. We could use NodePort services, but low-numbered +// ports are denied by kubelet with out current configuration. +// - We pass the host containerd socket to the service, and the service runs +// as root. This means that the service basically has root on the machine. +// The fact that it's directly exposed to the Internet isn't great, but the +// only alternative seems to to split this up into two services - one +// responsible for maintaining the state of translations/pods (running as +// root), another responding to the actual queries. Considering this is Go +// and we run a bare minimum of code (no mirko default service), this is +// probably good enough and not worth the extra complexity. +// - The service has to be able to resolve the node IP on which it's running, +// to know what address to look up in the conntrack table. Currently it does +// so by connection to the k8s API and getting information about its own pod +// and then getting the node's IP address from there. +// This might be overly complicated, as perhaps we can +// 1. figure out the server IP from the incoming ident connections, or +// 2. find a better way to retrieve the nodeIP via the 'downstream API' +// TODO(q3k): figure this out. + +local kube = import "../../../kube/kube.libsonnet"; +local policies = import "../../../kube/policies.libsonnet"; + +{ + Environment: { + local env = self, + local cfg = env.cfg, + cfg:: { + namespace: "identd", + image: "registry.k0.hswaw.net/q3k/identd:315532800-9f29c14dad77036d0820948139b5aee3fac25d59", + }, + + namespace: kube.Namespace(cfg.namespace), + local ns = self.namespace, + + allowInsecure: policies.AllowNamespaceInsecure(cfg.namespace), + + sa: ns.Contain(kube.ServiceAccount("identd")), + role: ns.Contain(kube.Role("access-pod-info")) { + rules: [ + { + apiGroups: [""], + resources: ["pods"], + verbs: ["get"], + }, + ], + }, + rb: ns.Contain(kube.RoleBinding("identd")) { + roleRef_: env.role, + subjects_: [env.sa], + }, + + daemonset: ns.Contain(kube.DaemonSet("identd")) { + spec+: { + template+: { + spec+: { + serviceAccountName: env.sa.metadata.name, + hostNetwork: true, + containers_: { + default: kube.Container("default") { + image: cfg.image, + env_: { + POD_NAME: kube.FieldRef("metadata.name"), + POD_NAMESPACE: kube.FieldRef("metadata.namespace"), + }, + command: [ + "/cluster/identd/identd", + "-identd_listen", "0.0.0.0:113", + "-identd_conntrack_proc", "/host/conntrack", + "-identd_containerd_socket", "/host/containerd.sock", + // Used by the service to figure out which + // node it's running on. + "-identd_pod_name", "$(POD_NAME)", + "-identd_pod_namespace", "$(POD_NAMESPACE)", + "-logtostderr", + ], + volumeMounts_: { + conntrack: { mountPath: "/host/conntrack", }, + containerd: { mountPath: "/host/containerd.sock", }, + }, + resources: { + requests: { + cpu: "0.1", + memory: "64M", + }, + // Allow identd to spike to 1 CPU. This + // makes it faster when fetching + // information from containerd. + limits: { + cpu: "1", + memory: "256M", + }, + }, + }, + }, + volumes_: { + conntrack: kube.HostPathVolume("/proc/net/nf_conntrack", "File"), + containerd: kube.HostPathVolume("/var/run/containerd/containerd.sock", "Socket"), + }, + }, + }, + }, + }, + } +}