diff --git a/cluster/kube/cluster.jsonnet b/cluster/kube/cluster.jsonnet index c79a8279..800e47a1 100644 --- a/cluster/kube/cluster.jsonnet +++ b/cluster/kube/cluster.jsonnet @@ -5,6 +5,7 @@ local coredns = import "lib/coredns.libsonnet"; local metrics = import "lib/metrics.libsonnet"; local calico = import "lib/calico.libsonnet"; local metallb = import "lib/metallb.libsonnet"; +local nginx = import "lib/nginx.libsonnet"; local Cluster(fqdn) = { local cluster = self, @@ -51,6 +52,8 @@ local Cluster(fqdn) = { metrics: metrics.Environment {}, // Metal Load Balancer metallb: metallb.Environment {}, + // Main nginx Ingress Controller + nginx: nginx.Environment {}, }; diff --git a/cluster/kube/lib/metallb.libsonnet b/cluster/kube/lib/metallb.libsonnet index a00163b9..fd682b7b 100644 --- a/cluster/kube/lib/metallb.libsonnet +++ b/cluster/kube/lib/metallb.libsonnet @@ -24,6 +24,9 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat cfg:: { namespace: "metallb-system", namespaceCreate: true, + version:: "master", + imageController: "metallb/controller:" + cfg.version, + imageSpeaker: "metallb/speaker:" + cfg.version, }, ns: if cfg.namespaceCreate then kube.Namespace(cfg.namespace), @@ -106,5 +109,93 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat name: env.roleWatcher.metadata.name, }, }, + + deployController: kube.Deployment("controller") { + metadata+: { + namespace: cfg.namespace, + }, + spec+: { + revisionHistoryLimit: 3, + template+: { + spec+: { + serviceAccountName: env.saController.metadata.name, + terminationGracePeriodSeconds: 0, + securityContext: { + runAsNonRoot: true, + runAsUser: 65534, # nobody + }, + containers_: { + controller: kube.Container("controller") { + image: cfg.imageController, + args: [ "--port=7472", "--config=config" ], + ports: [ + { name: "monitoring", containerPort: 7472 }, + ], + resources: { + limits: { cpu: "100m", memory: "100Mi" }, + }, + securityContext: { + allowPrivilegeEscalation: false, + capabilities: { drop: [ "all" ] }, + readOnlyRootFilesystem: true, + }, + }, + }, + }, + }, + }, + }, + + daemonsetSpeaker: kube.DaemonSet("speaker") { + metadata+: { + namespace: cfg.namespace, + }, + spec+: { + template+: { + spec+: { + serviceAccountName: env.saSpeaker.metadata.name, + hostNetwork: true, + containers_: { + speaker: kube.Container("speaker") { + image: cfg.imageSpeaker, + args: [ "--port=7472", "--config=config" ], + env_: { + METALLB_NODE_NAME: kube.FieldRef("spec.nodeName"), + }, + ports: [ + { name: "monitoring", containerPort: 7472 }, + ], + resources: { + limits: { cpu: "100m", memory: "100Mi" }, + }, + securityContext: { + allowPrivilegeEscalation: false, + capabilities: { drop: [ "all" ], add: [ "net_raw" ] }, + readOnlyRootFilesystem: true, + }, + }, + }, + }, + }, + }, + }, + + configMap: kube.ConfigMap("config") { + local cm = self, + metadata+: { + namespace: cfg.namespace, + }, + data: { + config: std.manifestYamlDoc({ + "address-pools": [ + { + name: "public-v4-1", + protocol: "layer2", + addresses: ["185.236.240.50-185.236.240.63",], + } + ], + }), + }, + }, }, } diff --git a/cluster/kube/lib/nginx.libsonnet b/cluster/kube/lib/nginx.libsonnet new file mode 100644 index 00000000..a6d10f18 --- /dev/null +++ b/cluster/kube/lib/nginx.libsonnet @@ -0,0 +1,212 @@ +# Deploy a per-cluster Nginx Ingress Controller + +local kube = import "../../../kube/kube.libsonnet"; + +{ + Environment: { + local env = self, + local cfg = env.cfg, + cfg:: { + image: "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.23.0", + namespace: "nginx-system", + }, + + metadata:: { + namespace: cfg.namespace, + labels: { + "app.kubernetes.io/name": "ingress-nginx", + "app.kubernetes.io/part-of": "ingress-nginx", + }, + }, + + namespace: kube.Namespace(cfg.namespace), + + maps: { + make(name):: kube.ConfigMap(name) { + metadata+: env.metadata, + }, + configuration: env.maps.make("nginx-configuration"), + tcp: env.maps.make("tcp-services"), + udp: env.maps.make("udp-services"), + }, + + sa: kube.ServiceAccount("nginx-ingress-serviceaccount") { + metadata+: env.metadata, + }, + + cr: kube.ClusterRole("nginx-ingress-clusterrole") { + metadata+: env.metadata { + namespace:: null, + }, + rules: [ + { + apiGroups: [""], + resources: ["configmaps", "endpoints", "nodes", "pods", "secrets"], + verbs: ["list", "watch"], + }, + { + apiGroups: [""], + resources: ["nodes"], + verbs: ["get"], + }, + { + apiGroups: [""], + resources: ["services"], + verbs: ["get", "list", "watch"], + }, + { + apiGroups: ["extensions"], + resources: ["ingresses"], + verbs: ["get", "list", "watch"], + }, + { + apiGroups: [""], + resources: ["events"], + verbs: ["create", "patch"], + }, + { + apiGroups: ["extensions"], + resources: ["ingresses/status"], + verbs: ["update"], + }, + ], + }, + + crb: kube.ClusterRoleBinding("nginx-ingress-clusterrole-nisa-binding") { + metadata+: env.metadata { + namespace:: null, + }, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "ClusterRole", + name: env.cr.metadata.name, + }, + subjects: [ + { + kind: "ServiceAccount", + name: env.sa.metadata.name, + namespace: env.sa.metadata.namespace, + }, + ], + }, + + role: kube.Role("nginx-ingress-role") { + metadata+: env.metadata, + rules : [ + { + apiGroups: [""], + resources: ["configmaps", "pods", "secrets", "namespaces"], + verbs: ["get"], + }, + { + apiGroups: [""], + resources: ["configmaps"], + resourceNames: ["ingress-controller-leader-nginx"], + verbs: ["get", "update"], + }, + { + apiGroups: [""], + resources: ["configmaps"], + verbs: ["create"], + }, + { + apiGroups: [""], + resources: ["endpoints"], + verbs: ["get"], + }, + ], + }, + + roleb: kube.RoleBinding("nginx-ingress-role-nisa-binding") { + metadata+: env.metadata, + roleRef: { + apiGroup: "rbac.authorization.k8s.io", + kind: "Role", + name: env.role.metadata.name, + }, + subjects: [ + { + kind: "ServiceAccount", + name: env.sa.metadata.name, + namespace: env.sa.metadata.namespace, + }, + ], + }, + + service: kube.Service("ingress-nginx") { + metadata+: env.metadata, + target_pod:: env.deployment.spec.template, + spec+: { + type: "LoadBalancer", + ports: [ + { name: "http", port: 80, targetPort: 80, protocol: "TCP" }, + { name: "https", port: 443, targetPort: 443, protocol: "TCP" }, + ], + }, + }, + + deployment: kube.Deployment("nginx-ingress-controller") { + metadata+: env.metadata, + spec+: { + replicas: 1, + template+: { + spec+: { + serviceAccountName: env.sa.metadata.name, + containers_: { + controller: kube.Container("nginx-ingress-controller") { + image: cfg.image, + args: [ + "/nginx-ingress-controller", + "--configmap=%s/%s" % [cfg.namespace, env.maps.configuration.metadata.name], + "--tcp-services-configmap=%s/%s" % [cfg.namespace, env.maps.tcp.metadata.name], + "--udp-services-configmap=%s/%s" % [cfg.namespace, env.maps.udp.metadata.name], + "--publish-service=%s/%s" % [cfg.namespace, env.service.metadata.name], + "--annotations-prefix=nginx.ingress.kubernetes.io", + ], + env_: { + POD_NAME: kube.FieldRef("metadata.name"), + POD_NAMESPACE: kube.FieldRef("metadata.namespace"), + }, + ports_: { + http: { containerPort: 80 }, + https: { containerPort: 443 }, + }, + livenessProbe: { + failureThreshold: 3, + httpGet: { + path: "/healthz", + port: 10254, + scheme: "HTTP", + }, + initialDelaySeconds: 10, + periodSeconds: 10, + successThreshold: 1, + timeoutSeconds: 10, + }, + readinessProbe: { + failureThreshold: 3, + httpGet: { + path: "/healthz", + port: 10254, + scheme: "HTTP", + }, + periodSeconds: 10, + successThreshold: 1, + timeoutSeconds: 10, + }, + securityContext: { + allowPrivilegeEscalation: true, + capabilities: { + drop: ["ALL"], + add: ["NET_BIND_SERVICE"], + }, + runAsUser: 33, + }, + }, + }, + }, + }, + }, + }, + }, +}