forked from hswaw/hscloud
331 lines
13 KiB
Plaintext
331 lines
13 KiB
Plaintext
# Deploy a per-cluster Nginx Ingress Controller
|
|
|
|
local kube = import "../../../kube/kube.libsonnet";
|
|
local policies = import "../../../kube/policies.libsonnet";
|
|
|
|
{
|
|
Environment: {
|
|
local env = self,
|
|
local cfg = env.cfg,
|
|
cfg:: {
|
|
# Built from nginx-ingress-controller/Dockerfile:
|
|
#
|
|
# $ cd cluster/kube/lib/nginx-ingress-controller
|
|
# $ docker build -t eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1 .
|
|
# [..]
|
|
# (2/8) Upgrading libcrypto1.1 (1.1.1i-r0 -> 1.1.1k-r0)
|
|
# (3/8) Upgrading libssl1.1 (1.1.1i-r0 -> 1.1.1k-r0
|
|
# [...]
|
|
# (8/8) Upgrading openssl (1.1.1i-r0 -> 1.1.1k-r0)
|
|
# $ docker push eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1
|
|
#
|
|
# TODO(q3k): unfork this once openssl 1.1.1k lands in upstream
|
|
# nginx-ingress-controller.
|
|
image: "eu.gcr.io/bgpwtf/nginx-ingress-controller:v0.44.0-r1",
|
|
namespace: "nginx-system",
|
|
},
|
|
|
|
metadata:: {
|
|
namespace: cfg.namespace,
|
|
labels: {
|
|
"app.kubernetes.io/name": "ingress-nginx",
|
|
"app.kubernetes.io/part-of": "ingress-nginx",
|
|
},
|
|
},
|
|
|
|
namespace: kube.Namespace(cfg.namespace),
|
|
|
|
allowInsecure: policies.AllowNamespaceInsecure(cfg.namespace),
|
|
|
|
maps: {
|
|
make(name):: kube.ConfigMap(name) {
|
|
metadata+: env.metadata,
|
|
},
|
|
configuration: env.maps.make("nginx-configuration") {
|
|
data: {
|
|
"proxy-set-headers": "%s/nginx-custom-headers" % [cfg.namespace],
|
|
},
|
|
},
|
|
customHeaders: env.maps.make("nginx-custom-headers") {
|
|
data: {
|
|
# RFC6648 deprecates X-prefixed headers as a convention in
|
|
# multiple application protocols, including HTTP. It
|
|
# recommends that any new headers should just start off
|
|
# with a final standardized name, ie. suggests to use
|
|
# Toaster-ID instead of X-Toaster-ID.
|
|
#
|
|
# However, it also acknowledges that headers likely to
|
|
# never be standardized can still be prefixed with OrgName-
|
|
# or other constructs. And since we're not even attempting
|
|
# to standardize anything here, this is what we use to
|
|
# prefix hscloud-specific headers.
|
|
#
|
|
# Hscloud == hscloud, this repository.
|
|
# Nic == nginx-ingress-controller, this ingress controller.
|
|
|
|
# Set source port/addr. Source-IP duplicates
|
|
# X-Forwarded-For, but is added for consistency with
|
|
# Source-Port.
|
|
#
|
|
# Source-IP is an IP address in two possible formats:
|
|
# IPv4: "1.2.3.4"
|
|
# IPv6: "2a0d:1234::42"
|
|
# Any other format received by services should be
|
|
# considered invalid, and the service should assume a
|
|
# misconfiguration of the N-I-C.
|
|
"Hscloud-Nic-Source-IP": "${remote_addr}",
|
|
# Source-Port is a stringified TCP port, encoding a port
|
|
# number from 1 to 65535. Any other value received by
|
|
# services should be considered invalid, and the service
|
|
# should assume a misconfiguration of the N-I-C.
|
|
"Hscloud-Nic-Source-Port": "${remote_port}",
|
|
},
|
|
},
|
|
tcp: env.maps.make("tcp-services") {
|
|
data: {
|
|
"22": "gerrit/gerrit:22",
|
|
}
|
|
},
|
|
udp: env.maps.make("udp-services"),
|
|
},
|
|
|
|
sa: kube.ServiceAccount("nginx-ingress-serviceaccount") {
|
|
metadata+: env.metadata,
|
|
},
|
|
|
|
cr: kube.ClusterRole("nginx-ingress-clusterrole") {
|
|
metadata+: env.metadata {
|
|
namespace:: null,
|
|
},
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps", "endpoints", "nodes", "pods", "secrets"],
|
|
verbs: ["list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["services"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["extensions", "networking.k8s.io"],
|
|
resources: ["ingresses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["events"],
|
|
verbs: ["create", "patch"],
|
|
},
|
|
{
|
|
apiGroups: ["extensions", "networking.k8s.io"],
|
|
resources: ["ingresses/status"],
|
|
verbs: ["update"],
|
|
},
|
|
{
|
|
apiGroups: ["extensions", "networking.k8s.io"],
|
|
resources: ["ingressclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
],
|
|
},
|
|
|
|
crb: kube.ClusterRoleBinding("nginx-ingress-clusterrole-nisa-binding") {
|
|
metadata+: env.metadata {
|
|
namespace:: null,
|
|
},
|
|
roleRef: {
|
|
apiGroup: "rbac.authorization.k8s.io",
|
|
kind: "ClusterRole",
|
|
name: env.cr.metadata.name,
|
|
},
|
|
subjects: [
|
|
{
|
|
kind: "ServiceAccount",
|
|
name: env.sa.metadata.name,
|
|
namespace: env.sa.metadata.namespace,
|
|
},
|
|
],
|
|
},
|
|
|
|
role: kube.Role("nginx-ingress-role") {
|
|
metadata+: env.metadata,
|
|
rules : [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["namespaces"],
|
|
verbs: ["get"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps", "pods", "secrets", "endpoints"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["services"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["extensions", "networking.k8s.io"],
|
|
resources: ["ingresses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["extensions", "networking.k8s.io"],
|
|
resources: ["ingresses/status"],
|
|
verbs: ["update"],
|
|
},
|
|
{
|
|
apiGroups: ["extensions", "networking.k8s.io"],
|
|
resources: ["ingressclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
resourceNames: ["ingress-controller-leader-nginx"],
|
|
verbs: ["get", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["create"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["events"],
|
|
verbs: ["create", "patch"],
|
|
},
|
|
],
|
|
},
|
|
|
|
roleb: kube.RoleBinding("nginx-ingress-role-nisa-binding") {
|
|
metadata+: env.metadata,
|
|
roleRef: {
|
|
apiGroup: "rbac.authorization.k8s.io",
|
|
kind: "Role",
|
|
name: env.role.metadata.name,
|
|
},
|
|
subjects: [
|
|
{
|
|
kind: "ServiceAccount",
|
|
name: env.sa.metadata.name,
|
|
namespace: env.sa.metadata.namespace,
|
|
},
|
|
],
|
|
},
|
|
|
|
service: kube.Service("ingress-nginx") {
|
|
metadata+: env.metadata,
|
|
target_pod:: env.deployment.spec.template,
|
|
spec+: {
|
|
type: "LoadBalancer",
|
|
ports: [
|
|
{ name: "ssh", port: 22, targetPort: 22, protocol: "TCP" },
|
|
{ name: "http", port: 80, targetPort: 80, protocol: "TCP" },
|
|
{ name: "https", port: 443, targetPort: 443, protocol: "TCP" },
|
|
],
|
|
},
|
|
},
|
|
|
|
serviceGitea: kube.Service("ingress-nginx-gitea") {
|
|
metadata+: env.metadata,
|
|
target_pod:: env.deployment.spec.template,
|
|
spec+: {
|
|
type: "LoadBalancer",
|
|
loadBalancerIP: "185.236.240.60",
|
|
ports: [
|
|
{ name: "ssh", port: 22, targetPort: 222, protocol: "TCP" },
|
|
{ name: "http", port: 80, targetPort: 80, protocol: "TCP" },
|
|
{ name: "https", port: 443, targetPort: 443, protocol: "TCP" },
|
|
],
|
|
},
|
|
},
|
|
|
|
deployment: kube.Deployment("nginx-ingress-controller") {
|
|
metadata+: env.metadata,
|
|
spec+: {
|
|
replicas: 5,
|
|
template+: {
|
|
spec+: {
|
|
serviceAccountName: env.sa.metadata.name,
|
|
containers_: {
|
|
controller: kube.Container("nginx-ingress-controller") {
|
|
image: cfg.image,
|
|
imagePullPolicy: "IfNotPresent",
|
|
lifecycle: {
|
|
preStop: {
|
|
exec: {
|
|
command: [ "/wait-shutdown" ],
|
|
},
|
|
},
|
|
},
|
|
args: [
|
|
"/nginx-ingress-controller",
|
|
"--election-id=ingress-controller-leader",
|
|
"--ingress-class=nginx",
|
|
"--configmap=%s/%s" % [cfg.namespace, env.maps.configuration.metadata.name],
|
|
"--tcp-services-configmap=%s/%s" % [cfg.namespace, env.maps.tcp.metadata.name],
|
|
"--udp-services-configmap=%s/%s" % [cfg.namespace, env.maps.udp.metadata.name],
|
|
"--publish-service=%s/%s" % [cfg.namespace, env.service.metadata.name],
|
|
"--annotations-prefix=nginx.ingress.kubernetes.io",
|
|
],
|
|
env_: {
|
|
POD_NAME: kube.FieldRef("metadata.name"),
|
|
POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
|
|
},
|
|
ports_: {
|
|
http: { containerPort: 80 },
|
|
https: { containerPort: 443 },
|
|
},
|
|
livenessProbe: {
|
|
failureThreshold: 3,
|
|
httpGet: {
|
|
path: "/healthz",
|
|
port: 10254,
|
|
scheme: "HTTP",
|
|
},
|
|
initialDelaySeconds: 10,
|
|
periodSeconds: 10,
|
|
successThreshold: 1,
|
|
timeoutSeconds: 10,
|
|
},
|
|
readinessProbe: {
|
|
failureThreshold: 3,
|
|
httpGet: {
|
|
path: "/healthz",
|
|
port: 10254,
|
|
scheme: "HTTP",
|
|
},
|
|
periodSeconds: 10,
|
|
successThreshold: 1,
|
|
timeoutSeconds: 10,
|
|
},
|
|
securityContext: {
|
|
allowPrivilegeEscalation: true,
|
|
capabilities: {
|
|
drop: ["ALL"],
|
|
add: ["NET_BIND_SERVICE"],
|
|
},
|
|
runAsUser: 101,
|
|
},
|
|
resources: {
|
|
limits: { cpu: "2", memory: "4G" },
|
|
requests: { cpu: "1", memory: "1G" },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|