# Deploy hosted calico with its own etcd. local kube = import "../../../kube/kube.libsonnet"; local crdYaml = (std.native("parseYaml"))(importstr "./calico_crd.yml"); local crdMap = { [x.metadata.name]: x for x in crdYaml if x != null }; local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadata.name) { roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "ClusterRole", name: cr.metadata.name, }, subjects: [ { kind: "ServiceAccount", name: sa.metadata.name, namespace: sa.metadata.namespace, }, ], }; { Environment: { local env = self, local cfg = env.cfg, cfg:: { namespace: "kube-system", version: "v3.15.5", imageController: "calico/kube-controllers:" + cfg.version, imageCNI: "calico/cni:" + cfg.version, imageNode: "calico/node:" + cfg.version, }, crds: crdMap, cm: kube.ConfigMap("calico-config") { local cm = self, secretPrefix:: "/calico-secrets/", metadata+: { namespace: cfg.namespace, }, data: { calico_backend: "bird", veth_mtu: "1440", typha_service_name: "none", cni_network_config: ||| { "name": "k8s-pod-network", "cniVersion": "0.3.1", "plugins": [ { "type": "calico", "log_level": "info", "datastore_type": "kubernetes", "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, "ipam": { "type": "calico-ipam" }, "policy": { "type": "k8s" }, "kubernetes": { "kubeconfig": "__KUBECONFIG_FILEPATH__" } }, { "type": "portmap", "snat": true, "capabilities": {"portMappings": true} }, { "type": "bandwidth", "capabilities": {"bandwidth": true} } ] } ||| }, }, secrets: kube.Secret("calico-secrets") { metadata+: { namespace: cfg.namespace, }, data_: { }, }, saNode: kube.ServiceAccount("calico-node") { metadata+: { namespace: cfg.namespace, }, }, crNode: kube.ClusterRole("calico-node") { rules: [ { apiGroups: [""], resources: ["pods", "nodes", "namespaces"], verbs: ["get"], }, { apiGroups: [""], resources: ["endpoints", "services"], verbs: ["watch", "list", "get"], }, { apiGroups: [""], resources: ["configmaps"], verbs: ["get"], }, { apiGroups: [""], resources: ["nodes/status"], verbs: ["patch", "update"], }, { apiGroups: ["networking.k8s.io"], resources: ["networkpolicies"], verbs: ["watch", "list"], }, { apiGroups: [""], resources: ["pods", "namespaces", "serviceaccounts"], verbs: ["list", "watch"], }, { apiGroups: [""], resources: ["pods/status"], verbs: ["patch"], }, { apiGroups: ["crd.projectcalico.org"], resources: ["globalfelixconfigs", "felixconfigurations", "bgppeers", "globalbgpconfigs", "bgpconfigurations", "ippools", "ipamblocks", "globalnetworkpolicies", "globalnetworksets", "networkpolicies", "networksets", "clusterinformations", "hostendpoints", "blockaffinities"], verbs: ["get", "list", "watch"], }, { apiGroups: ["crd.projectcalico.org"], resources: ["ippools", "felixconfigurations", "clusterinformations"], verbs: ["create", "update"], }, { apiGroups: [""], resources: ["nodes"], verbs: ["get", "list", "watch"], }, { apiGroups: ["crd.projectcalico.org"], resources: ["blockaffinities", "ipamblocks", "ipamhandles"], verbs: ["get", "list", "create", "update", "delete"], }, { apiGroups: ["crd.projectcalico.org"], resources: ["ipamconfigs"], verbs: ["get"], }, { apiGroups: ["crd.projectcalico.org"], resources: ["blockaffinities"], verbs: ["watch"], }, { apiGroups: ["apps"], resources: ["daemonsets"], verbs: ["get"], }, ], }, crbNode: bindServiceAccountClusterRole(env.saNode, env.crNode), saController: kube.ServiceAccount("calico-kube-controllers") { metadata+: { namespace: cfg.namespace, }, }, crController: kube.ClusterRole("calico-kube-controllers") { rules: [ { apiGroups: [""], resources: ["nodes"], verbs: ["watch", "list", "get"], }, { apiGroups: [""], resources: ["pods"], verbs: ["get"], }, { apiGroups: ["crd.projectcalico.org"], resources: ["ippools"], verbs: ["list"], }, { apiGroups: ["crd.projectcalico.org"], resources: ["blockaffinities", "ipamblocks", "ipamhandles"], verbs: ["get", "list", "create", "update", "delete"], }, { apiGroups: ["crd.projectcalico.org"], resources: ["hostendpoints"], verbs: ["get", "list", "create", "update", "delete"], }, { apiGroups: ["crd.projectcalico.org"], resources: ["clusterinformations"], verbs: ["get", "create", "update"], }, { apiGroups: ["crd.projectcalico.org"], resources: ["kubecontrollersconfigurations"], verbs: ["get", "create", "update", "watch"], }, ], }, crbController: bindServiceAccountClusterRole(env.saController, env.crController), controller: kube.Deployment("calico-kube-controllers") { metadata+: { namespace: cfg.namespace, labels+: { "k8s-app": "calico-kube-controllers", }, }, spec+: { replicas: 1, strategy: { type: "Recreate" }, template+: { spec+: { nodeSelector: { "kubernetes.io/os": "linux" }, tolerations: [ { key: "CriticalAddonsOnly", operator: "Exists" }, { key: "node-role.kubernetes.io/master", effect: "NoSchedule" }, ], serviceAccountName: env.saController.metadata.name, priorityClassName: "system-cluster-critical", containers_: { "calico-kube-controllers": kube.Container("calico-kube-controllers") { image: cfg.imageController, env_: { DATASTORE_TYPE: "kubernetes", ENABLED_CONTROLLERS: "node", }, volumeMounts_: { secrets: { mountPath: env.cm.secretPrefix, }, }, readinessProbe: { exec: { command: [ "/usr/bin/check-status", "-r" ], }, }, }, }, volumes_: { secrets: kube.SecretVolume(env.secrets), }, }, }, }, }, # ConfigMap that holds overriden bird.cfg.template and bird_ipam.cfg.template. calicoMetallbBird: kube.ConfigMap("calico-metallb-bird") { metadata+: { namespace: cfg.namespace, }, data: { "bird.cfg.template": (importstr "calico-bird.cfg.template"), "bird_ipam.cfg.template": (importstr "calico-bird-ipam.cfg.template"), }, }, nodeDaemon: kube.DaemonSet("calico-node") { metadata+: { namespace: cfg.namespace, labels+: { "k8s-app": "calico-node", }, }, spec+: { template+: { spec+: { nodeSelector: { "kubernetes.io/os": "linux" }, hostNetwork: true, tolerations: [ { effect: "NoSchedule", operator: "Exists" }, { key: "CriticalAddonsOnly", operator: "Exists" }, { effect: "NoExecute", operator: "Exists" }, ], serviceAccountName: env.saNode.metadata.name, terminationGracePeriodSeconds: 0, priorityClassName: "system-cluster-critical", volumes_: { lib_modules: kube.HostPathVolume("/run/current-system/kernel-modules/lib/modules"), var_run_calico: kube.HostPathVolume("/var/run/calico"), var_lib_calico: kube.HostPathVolume("/var/lib/calico"), xtables_lock: kube.HostPathVolume("/run/xtables.lock"), cni_bin: kube.HostPathVolume("/opt/cni/bin"), cni_config: kube.HostPathVolume("/opt/cni/conf"), secrets: kube.SecretVolume(env.secrets), bird_cfg_template: kube.ConfigMapVolume(env.calicoMetallbBird), # TODO flexvol-driver-host, policysync }, initContainers_: { installCNI: kube.Container("install-cni") { image: cfg.imageCNI, command: ["/install-cni.sh"], env_: { CNI_CONF_NAME: "10-calico.conflist", CNI_NETWORK_CONFIG: kube.ConfigMapRef(env.cm, "cni_network_config"), CNI_MTU: kube.ConfigMapRef(env.cm, "veth_mtu"), # Important: our directory is changed from the default (/etc/cni/net.d) # to inside /opt/ above in the cni_config HostPathVolume. # See projectcalico/cni-plugin//k8s-install/scripts/install-cni.sh:24 for reference. CNI_NET_DIR: "/opt/cni/conf", SLEEP: "false", KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } }, }, volumeMounts_: { cni_bin: { mountPath: "/host/opt/cni/bin" }, cni_config: { mountPath: "/host/etc/cni/net.d" }, secrets: { mountPath: env.cm.secretPrefix }, }, securityContext: { privileged: true, }, }, }, containers_: { calicoNode: kube.Container("calico-node") { image: cfg.imageNode, env_: { WAIT_FOR_DATASTORE: "true", NODENAME: kube.FieldRef("spec.nodeName"), DATASTORE_TYPE: "kubernetes", CALICO_NETWORKING_BACKEND: kube.ConfigMapRef(env.cm, "calico_backend"), CLUSTER_TYPE: "k8s,bgp", IP: "autodetect", IP_AUTODETECTION_METHOD: "can-reach=185.236.240.1", CALICO_IPV4POOL_IPIP: "Always", FELIX_IPINIPMTU: kube.ConfigMapRef(env.cm, "veth_mtu"), FELIX_WIREGUARDMTU: kube.ConfigMapRef(env.cm, "veth_mtu"), CALICO_IPV4POOL_CIDR: "10.10.24.0/21", CALICO_DISABLE_FILE_LOGGING: "true", FELIX_DEFAULTENDPOINTTOHOSTACTION: "ACCEPT", FELIX_LOGSEVERITYSCREEN: "info", FELIX_IPV6SUPPORT: "false", FELIX_HEALTHENABLED: "true", FELIX_HEALTHHOST: "127.0.0.1", CALICO_ADVERTISE_CLUSTER_IPS: "10.10.12.0/24", KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } }, }, securityContext: { privileged: true, }, resources: { requests: { cpu: "250m" }, }, livenessProbe: { exec: { command: ["/bin/calico-node", "-bird-live", "-felix-live"], }, periodSeconds: 10, initialDelaySeconds: 10, failureThreshold: 6, }, readinessProbe: { exec: { command: ["/bin/calico-node", "-bird-ready", "-felix-ready"], }, periodSeconds: 10, }, volumeMounts_: { lib_modules: { mountPath: "/lib/modules" }, xtables_lock: { mountPath: "/run/xtables.lock" }, var_run_calico: { mountPath: "/var/run/calico" }, var_lib_calico: { mountPath: "/var/lib/calico" }, secrets: { mountPath: env.cm.secretPrefix }, }, volumeMounts+: [ { name: "bird-cfg-template", mountPath: "/etc/calico/confd/templates/bird.cfg.template", subPath: "bird.cfg.template" }, { name: "bird-cfg-template", mountPath: "/etc/calico/confd/templates/bird_ipam.cfg.template", subPath: "bird_ipam.cfg.template" }, ], }, }, }, }, }, }, }, }