hswaw/hscloud
4
0
Fork 2
mirror of https://gerrit.hackerspace.pl/hscloud synced 2025-01-20 16:33:54 +00:00
Code Activity Actions

calico: move from etcd to crd

Browse source 
Leaving the CRD definitions as YAML, extracted without modifications
from the original install file - this should make upgrades simpler.

Change-Id: I7211d2711e2af014b36dd887a951abb9e1032eb9
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1179
Reviewed-by: q3k <q3k@hackerspace.pl>
This commit is contained in:
implr 2021-12-01 00:36:02 +01:00 committed by implr
parent e3dd96adb0
commit 4d98cf5ca8
3 changed files with 3211 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

130
cluster/kube/lib/calico.libsonnet
View file

@ -1,6 +1,11 @@
# Deploy hosted calico with its own etcd.
local kube = import "../../../kube/kube.libsonnet";
local crdYaml = (std.native("parseYaml"))(importstr "./calico_crd.yml");
local crdMap = {
[x.metadata.name]: x
for x in crdYaml if x != null
};
local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadata.name) {
roleRef: {
@ -27,20 +32,10 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat
imageController: "calico/kube-controllers:" + cfg.version,
imageCNI: "calico/cni:" + cfg.version,
imageNode: "calico/node:" + cfg.version,
// TODO(implr): migrate calico from etcd to apiserver
etcd: {
endpoints: [
"https://bc01n01.hswaw.net:2379",
"https://bc01n02.hswaw.net:2379",
"https://dcr01s22.hswaw.net:2379",
"https://dcr01s24.hswaw.net:2379",
],
ca: importstr "../../certs/ca-etcd.crt",
cert: importstr "../../certs/etcd-calico.cert",
key: importstr "../../secrets/plain/etcd-calico.key",
},
},
crds: crdMap,
cm: kube.ConfigMap("calico-config") {
local cm = self,
secretPrefix:: "/calico-secrets/",
@ -50,12 +45,6 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat
},
data: {
etcd_endpoints: std.join(",", cfg.etcd.endpoints),
etcd_ca: cm.secretPrefix + "etcd-ca",
etcd_cert: cm.secretPrefix + "etcd-cert",
etcd_key: cm.secretPrefix + "etcd-key",
calico_backend: "bird",
veth_mtu: "1440",
@ -69,11 +58,8 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat
{
"type": "calico",
"log_level": "info",
"etcd_endpoints": "__ETCD_ENDPOINTS__",
"etcd_key_file": "__ETCD_KEY_FILE__",
"etcd_cert_file": "__ETCD_CERT_FILE__",
"etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
"datastore_type": "etcdv3",
"datastore_type": "kubernetes",
"nodename": "__KUBERNETES_NODE_NAME__",
"mtu": __CNI_MTU__,
"ipam": {
"type": "calico-ipam"
@ -106,9 +92,6 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat
},
data_: {
"etcd-ca": cfg.etcd.ca,
"etcd-cert": cfg.etcd.cert,
"etcd-key": cfg.etcd.key,
},
},
@ -140,16 +123,56 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat
resources: ["nodes/status"],
verbs: ["patch", "update"],
},
{
apiGroups: ["networking.k8s.io"],
resources: ["networkpolicies"],
verbs: ["watch", "list"],
},
{
apiGroups: [""],
resources: ["pods", "namespaces", "serviceaccounts"],
verbs: ["list", "watch"],
},
{
apiGroups: [""],
resources: ["pods/status"],
verbs: ["patch"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["globalfelixconfigs", "felixconfigurations", "bgppeers", "globalbgpconfigs", "bgpconfigurations", "ippools", "ipamblocks", "globalnetworkpolicies", "globalnetworksets", "networkpolicies", "networksets", "clusterinformations", "hostendpoints", "blockaffinities"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["ippools", "felixconfigurations", "clusterinformations"],
verbs: ["create", "update"],
},
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["blockaffinities", "ipamblocks", "ipamhandles"],
verbs: ["get", "list", "create", "update", "delete"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["ipamconfigs"],
verbs: ["get"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["blockaffinities"],
verbs: ["watch"],
},
{
apiGroups: ["apps"],
resources: ["daemonsets"],
verbs: ["get"],
},
],
},
@ -165,13 +188,38 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat
rules: [
{
apiGroups: [""],
resources: ["nodes", "pods", "namespaces", "serviceaccounts"],
resources: ["nodes"],
verbs: ["watch", "list", "get"],
},
{
apiGroups: ["networking.k8s.io"],
resources: ["networkpolicies"],
verbs: ["watch", "list"],
apiGroups: [""],
resources: ["pods"],
verbs: ["get"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["ippools"],
verbs: ["list"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["blockaffinities", "ipamblocks", "ipamhandles"],
verbs: ["get", "list", "create", "update", "delete"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["hostendpoints"],
verbs: ["get", "list", "create", "update", "delete"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["clusterinformations"],
verbs: ["get", "create", "update"],
},
{
apiGroups: ["crd.projectcalico.org"],
resources: ["kubecontrollersconfigurations"],
verbs: ["get", "create", "update", "watch"],
},
],
},
@ -199,16 +247,12 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat
],
serviceAccountName: env.saController.metadata.name,
priorityClassName: "system-cluster-critical",
hostNetwork: true,
containers_: {
"calico-kube-controllers": kube.Container("calico-kube-controllers") {
image: cfg.imageController,
env_: {
ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"),
ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"),
ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"),
ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"),
ENABLED_CONTROLLERS: "policy,namespace,serviceaccount,workloadendpoint,node",
DATASTORE_TYPE: "kubernetes",
ENABLED_CONTROLLERS: "node",
},
volumeMounts_: {
secrets: {
@ -281,16 +325,11 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat
env_: {
CNI_CONF_NAME: "10-calico.conflist",
CNI_NETWORK_CONFIG: kube.ConfigMapRef(env.cm, "cni_network_config"),
ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"),
CNI_MTU: kube.ConfigMapRef(env.cm, "veth_mtu"),
# Important: our directory is changed from the default (/etc/cni/net.d)
# to inside /opt/ above in the cni_config HostPathVolume.
# See projectcalico/cni-plugin//k8s-install/scripts/install-cni.sh:24 for reference.
CNI_NET_DIR: "/opt/cni/conf",
# TODO(implr) needed?
CNI_CONF_ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"),
CNI_CONF_ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"),
CNI_CONF_ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"),
SLEEP: "false",
KUBERNETES_NODE_NAME: { fieldRef: { fieldPath: "spec.nodeName" } },
},
@ -308,12 +347,9 @@ local bindServiceAccountClusterRole(sa, cr) = kube.ClusterRoleBinding(cr.metadat
calicoNode: kube.Container("calico-node") {
image: cfg.imageNode,
env_: {
DATASTORE_TYPE: "etcdv3",
ETCD_ENDPOINTS: kube.ConfigMapRef(env.cm, "etcd_endpoints"),
ETCD_CA_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_ca"),
ETCD_KEY_FILE: kube.ConfigMapRef(env.cm, "etcd_key"),
ETCD_CERT_FILE: kube.ConfigMapRef(env.cm, "etcd_cert"),
CALICO_K8S_NODE_REF: kube.FieldRef("spec.nodeName"),
WAIT_FOR_DATASTORE: "true",
NODENAME: kube.FieldRef("spec.nodeName"),
DATASTORE_TYPE: "kubernetes",
CALICO_NETWORKING_BACKEND: kube.ConfigMapRef(env.cm, "calico_backend"),
CLUSTER_TYPE: "k8s,bgp",
IP: "autodetect",

3126
cluster/kube/lib/calico_crd.yml Normal file
View file

File diff suppressed because it is too large Load diff

12
cluster/tools/calicoctl.sh
View file

@ -6,16 +6,8 @@ source tools/hscloud/lib.sh || exit 1
function main() {
local ws=$(hscloud::workspace_location)
export ETCD_ENDPOINTS="https://bc01n01.hswaw.net:2379,https://bc01n01.hswaw.net:2379,https://bc01n01.hswaw.net:2379"
export ETCD_KEY_FILE="$ws/cluster/secrets/plain/etcd-calico.key"
export ETCD_CERT_FILE="$ws/cluster/certs/etcd-calico.cert"
export ETCD_CA_CERT_FILE="$ws/cluster/certs/ca-etcd.crt"
if [ ! -f "$ETCD_KEY_FILE" ] ; then
$(hscloud::must_rlocation hscloud/tools/secretstore) decrypt "$ws/cluster/secrets/cipher/etcd-calico.key" "$ETCD_KEY_FILE"
fi
export DATASTORE_TYPE=kubernetes
export KUBECONFIG=~/.kube/config
"$(hscloud::must_rlocation calicoctl_3_15/file/calicoctl)" "$@"
}