forked from hswaw/hscloud
1211 lines
49 KiB
Plaintext
1211 lines
49 KiB
Plaintext
# Deploy Rook/Ceph Operator
|
|
|
|
local kube = import "../../../kube/hscloud.libsonnet";
|
|
local policies = import "../../../kube/policies.libsonnet";
|
|
|
|
local oa = kube.OpenAPI;
|
|
|
|
{
|
|
Operator: {
|
|
local env = self,
|
|
local cfg = env.cfg,
|
|
cfg:: {
|
|
image: "rook/ceph:v1.6.9",
|
|
namespace: "rook-ceph-system",
|
|
},
|
|
|
|
metadata:: {
|
|
namespace: cfg.namespace,
|
|
labels: {
|
|
"operator": "rook",
|
|
"storage-backend": "ceph",
|
|
},
|
|
},
|
|
|
|
namespace: kube.Namespace(cfg.namespace),
|
|
|
|
policyInsecure: policies.AllowNamespaceInsecure(cfg.namespace),
|
|
|
|
// Grab CRDs from upstream YAML.
|
|
//
|
|
// We use a bit of jsonnet to remove some fields that kubebuilder (used
|
|
// upstream) added and to override preserveUnknownFIelds (as some older
|
|
// deployment apparently set it to true, which doesn't work for new
|
|
// CRDs that have default values).
|
|
crds: [
|
|
(el {
|
|
metadata+: {
|
|
annotations:: null,
|
|
creationTimestamp:: null,
|
|
},
|
|
status:: null,
|
|
spec+: {
|
|
preserveUnknownFields: false,
|
|
},
|
|
})
|
|
for el in (std.native("parseYaml")(importstr "rook-crds.yaml")) if el != null
|
|
],
|
|
|
|
sa: {
|
|
system: kube.ServiceAccount("rook-ceph-system") {
|
|
metadata+: env.metadata,
|
|
},
|
|
csiCephfsPlugin: kube.ServiceAccount("rook-csi-cephfs-plugin-sa") {
|
|
metadata+: env.metadata,
|
|
},
|
|
csiCephfsProvisioner: kube.ServiceAccount("rook-csi-cephfs-provisioner-sa") {
|
|
metadata+: env.metadata,
|
|
},
|
|
csiRbdPlugin: kube.ServiceAccount("rook-csi-rbd-plugin-sa") {
|
|
metadata+: env.metadata,
|
|
},
|
|
csiRbdProvisioner: kube.ServiceAccount("rook-csi-rbd-provisioner-sa") {
|
|
metadata+: env.metadata,
|
|
},
|
|
},
|
|
|
|
crs: {
|
|
clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: ["", "apps", "extensions"],
|
|
resources: ["secrets", "pods", "pods/log", "services", "configmaps", "deployments", "daemonsets"],
|
|
verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
|
|
},
|
|
],
|
|
},
|
|
global: kube.ClusterRole("rook-ceph-global") {
|
|
metadata+: env.metadata {
|
|
namespace:: null,
|
|
labels+: {
|
|
"rbac.ceph.rook.io/aggregate-to-rook-ceph-global": "true",
|
|
},
|
|
},
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["pods", "nodes", "nodes/proxy", "services"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"],
|
|
verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["storageclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["batch"],
|
|
resources: ["jobs", "cronjobs"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["ceph.rook.io"],
|
|
resources: ["*"],
|
|
verbs: ["*"],
|
|
},
|
|
{
|
|
apiGroups: ["rook.io"],
|
|
resources: ["*"],
|
|
verbs: ["*"],
|
|
},
|
|
{
|
|
apiGroups: ["policy", "apps", "extensions"],
|
|
resources: ["poddisruptionbudgets", "deployments", "replicasets"],
|
|
verbs: ["*"],
|
|
},
|
|
{
|
|
apiGroups: ["healthchecking.openshift.io"],
|
|
resources: ["machinedisruptionbudgets"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["machine.openshift.io"],
|
|
resources: ["machines"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["csidrivers"],
|
|
verbs: ["create", "delete", "get", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["k8s.cni.cncf.io"],
|
|
resources: ["network-attachment-definitions"],
|
|
verbs: ["get"],
|
|
},
|
|
],
|
|
},
|
|
|
|
mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps", "nodes", "nodes/proxy"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["events"],
|
|
verbs: ["create", "patch", "list", "get", "watch"],
|
|
},
|
|
]
|
|
},
|
|
objectBucket: kube.ClusterRole("rook-ceph-object-bucket") {
|
|
metadata+: env.metadata {
|
|
namespace:: null,
|
|
},
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["secrets", "configmaps"],
|
|
verbs: ["*"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["storageclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["objectbucket.io"],
|
|
resources: ["*"],
|
|
verbs: ["*"],
|
|
},
|
|
],
|
|
},
|
|
|
|
cephfsCSINodeplugin: kube.ClusterRole("cephfs-csi-nodeplugin") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get", "list", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["namespaces"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumes"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["volumeattachments"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
],
|
|
},
|
|
|
|
cephfsExternalProvisionerRunner: kube.ClusterRole("cephfs-external-provisioner-runner") {
|
|
metadata+: env.metadata {
|
|
namespace:: null,
|
|
labels+: {
|
|
"rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner": "true",
|
|
},
|
|
},
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["secrets"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumes"],
|
|
verbs: ["get", "list", "watch", "create", "delete", "update", "patch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumeclaims"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["storageclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["events"],
|
|
verbs: ["list", "watch", "create", "update", "patch"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshots"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshotcontents"],
|
|
verbs: ["create", "get", "list", "watch", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshotclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshotcontents/status"],
|
|
verbs: ["update"],
|
|
},
|
|
{
|
|
apiGroups: ["apiextensions.k8s.io"],
|
|
resources: ["customresourcedefinitions"],
|
|
verbs: ["create", "list", "watch", "delete", "get", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshots/status"],
|
|
verbs: ["update"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["volumeattachments"],
|
|
verbs: ["get", "list", "watch", "update", "patch"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["volumeattachments/status"],
|
|
verbs: ["patch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumeclaims/status"],
|
|
verbs: ["update", "patch"],
|
|
},
|
|
],
|
|
},
|
|
|
|
rbdCSINodeplugin: kube.ClusterRole("rbd-csi-nodeplugin") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["secrets"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get", "list", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["namespaces"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumes"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["volumeattachments"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
],
|
|
},
|
|
|
|
rbdExternalProvisionerRunner: kube.ClusterRole("rbd-external-provisioner-runner") {
|
|
metadata+: env.metadata {
|
|
namespace:: null,
|
|
labels+: {
|
|
"rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner": "true",
|
|
},
|
|
},
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["secrets"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumes"],
|
|
verbs: ["get", "list", "watch", "create", "delete", "update", "patch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumeclaims"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["volumeattachments"],
|
|
verbs: ["get", "list", "watch", "update", "patch"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["volumeattachments/status"],
|
|
verbs: ["patch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["storageclasses"],
|
|
verbs: ["get", "list", "watch"]
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["events"],
|
|
verbs: ["list", "watch", "create", "update", "patch"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshots"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshotcontents"],
|
|
verbs: ["create", "get", "list", "watch", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshotclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshotcontents/status"],
|
|
verbs: ["update"],
|
|
},
|
|
{
|
|
apiGroups: ["apiextensions.k8s.io"],
|
|
resources: ["customresourcedefinitions"],
|
|
verbs: ["create", "list", "watch", "delete", "get", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshots/status"],
|
|
verbs: ["update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumeclaims/status"],
|
|
verbs: ["update", "patch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get"],
|
|
},
|
|
{
|
|
apiGroups: ["replication.storage.openshift.io"],
|
|
resources: ["volumereplications", "volumereplicationclasses"],
|
|
verbs: ["create", "delete", "get", "list", "patch", "update", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["replication.storage.openshift.io"],
|
|
resources: ["volumereplications/finalizers"],
|
|
verbs: ["update"],
|
|
},
|
|
{
|
|
apiGroups: ["replication.storage.openshift.io"],
|
|
resources: ["volumereplications/status"],
|
|
verbs: ["get", "patch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["replication.storage.openshift.io"],
|
|
resources: ["volumereplicationclasses/status"],
|
|
verbs: ["get"],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
|
|
crbs: {
|
|
global: kube.ClusterRoleBinding("ceph-rook-global") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.global,
|
|
subjects_: [env.sa.system],
|
|
},
|
|
objectBucket: kube.ClusterRoleBinding("rook-ceph-object-bucket") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.objectBucket,
|
|
subjects_: [env.sa.system],
|
|
},
|
|
cephfsCSINodeplugin: kube.ClusterRoleBinding("cephfs-csi-nodeplugin") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.cephfsCSINodeplugin,
|
|
subjects_: [env.sa.csiCephfsPlugin],
|
|
},
|
|
cephfsCSIProvisioner: kube.ClusterRoleBinding("cephfs-csi-provisioner") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.cephfsExternalProvisionerRunner,
|
|
subjects_: [env.sa.csiCephfsProvisioner],
|
|
},
|
|
rbdCSINodeplugin: kube.ClusterRoleBinding("rbd-csi-nodeplugin") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.rbdCSINodeplugin,
|
|
subjects_: [env.sa.csiRbdPlugin],
|
|
},
|
|
rbdCSIProvisioner: kube.ClusterRoleBinding("rbd-csi-provisioner") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.rbdExternalProvisionerRunner,
|
|
subjects_: [env.sa.csiRbdProvisioner],
|
|
},
|
|
},
|
|
|
|
roles: {
|
|
system: kube.Role("ceph-rook-system") {
|
|
metadata+: env.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["pods", "configmaps", "services"],
|
|
verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["apps"],
|
|
resources: ["daemonsets", "statefulsets", "deployments"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["k8s.cni.cncf.io"],
|
|
resources: ["network-attachment-definitions"],
|
|
verbs: ["get"],
|
|
},
|
|
],
|
|
},
|
|
cephfsExternalProvisioner: kube.Role("cephfs-external-provisioner-cfg") {
|
|
metadata+: env.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["endpoints"],
|
|
verbs: ["get", "watch", "list", "delete", "update", "create"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list", "create", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["coordination.k8s.io"],
|
|
resources: ["leases"],
|
|
verbs: ["get" ,"watch", "list", "delete", "update", "create"],
|
|
},
|
|
],
|
|
},
|
|
rbdExternalProvisioner: kube.Role("rbd-external-provisioner-cfg") {
|
|
metadata+: env.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["endpoints"],
|
|
verbs: ["get", "watch", "list", "delete", "update", "create"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list", "watch", "create", "delete", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["coordination.k8s.io"],
|
|
resources: ["leases"],
|
|
verbs: ["get" ,"watch", "list", "delete", "update", "create"],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
|
|
rbs: {
|
|
system: kube.RoleBinding("ceph-rook-system") {
|
|
metadata+: env.metadata,
|
|
roleRef_: env.roles.system,
|
|
subjects_: [env.sa.system],
|
|
},
|
|
cephfsCSIProvisioner: kube.RoleBinding("cephfs-csi-provisioner-role-cfg") {
|
|
metadata+: env.metadata,
|
|
roleRef_: env.roles.cephfsExternalProvisioner,
|
|
subjects_: [env.sa.csiCephfsProvisioner],
|
|
},
|
|
rbdCSIProvisioner: kube.RoleBinding("rbd-csi-provisioner-role-cfg") {
|
|
metadata+: env.metadata,
|
|
roleRef_: env.roles.rbdExternalProvisioner,
|
|
subjects_: [env.sa.csiRbdProvisioner],
|
|
},
|
|
},
|
|
|
|
operator: kube.Deployment("rook-ceph-operator") {
|
|
metadata+: env.metadata,
|
|
spec+: {
|
|
template+: {
|
|
spec+: {
|
|
serviceAccountName: env.sa.system.metadata.name,
|
|
containers_: {
|
|
operator: kube.Container("rook-ceph-operator") {
|
|
image: cfg.image,
|
|
args: ["ceph", "operator"],
|
|
volumeMounts_: {
|
|
"rook-config": { mountPath: "/var/lib/rook" },
|
|
"default-config-dir": { mountPath: "/etc/ceph" },
|
|
},
|
|
env_: {
|
|
LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/",
|
|
ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false",
|
|
ROOK_LOG_LEVEL: "INFO",
|
|
ROOK_MON_HEALTHCHECK_INTERVAL: "45s",
|
|
ROOK_MON_OUT_TIMEOUT: "600s",
|
|
ROOK_DISCOVER_DEVICES_INTERVAL: "60m",
|
|
ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false",
|
|
ROOK_ENABLE_SELINUX_RELABELING: "true",
|
|
ROOK_ENABLE_FSGROUP: "true",
|
|
NODE_NAME: kube.FieldRef("spec.nodeName"),
|
|
POD_NAME: kube.FieldRef("metadata.name"),
|
|
POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
|
|
ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubernetes",
|
|
ROOK_ENABLE_FLEX_DRIVER: "true",
|
|
},
|
|
},
|
|
},
|
|
volumes_: {
|
|
"rook-config": { emptyDir: {} },
|
|
"default-config-dir": { emptyDir: {} },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
// Create a new Ceph cluster in a new namespace.
|
|
Cluster(operator, name):: {
|
|
local cluster = self,
|
|
spec:: error "please define cluster spec",
|
|
|
|
|
|
metadata:: {
|
|
namespace: name,
|
|
},
|
|
|
|
name(suffix):: cluster.metadata.namespace + "-" + suffix,
|
|
|
|
namespace: kube.Namespace(cluster.metadata.namespace),
|
|
|
|
sa: {
|
|
// service accounts need to be hardcoded, see operator source.
|
|
osd: kube.ServiceAccount("rook-ceph-osd") {
|
|
metadata+: cluster.metadata,
|
|
},
|
|
mgr: kube.ServiceAccount("rook-ceph-mgr") {
|
|
metadata+: cluster.metadata,
|
|
},
|
|
cmdReporter: kube.ServiceAccount("rook-ceph-cmd-reporter") {
|
|
metadata+: cluster.metadata,
|
|
},
|
|
},
|
|
|
|
roles: {
|
|
osd: kube.Role(cluster.name("osd")) {
|
|
metadata+: cluster.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
],
|
|
},
|
|
osdCluster: kube.ClusterRole(cluster.name("osd-cluster")) {
|
|
metadata+: cluster.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
],
|
|
},
|
|
mgr: kube.Role(cluster.name("mgr")) {
|
|
metadata+: cluster.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["pods", "services"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["batch"],
|
|
resources: ["jobs"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["ceph.rook.io"],
|
|
resources: ["*"],
|
|
verbs: ["*"],
|
|
},
|
|
],
|
|
},
|
|
cmdReporter: kube.Role(cluster.name("cmd-reporter")) {
|
|
metadata+: cluster.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["pods", "configmaps"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
],
|
|
},
|
|
mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) {
|
|
metadata+: cluster.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list", "watch"],
|
|
}
|
|
],
|
|
},
|
|
},
|
|
|
|
rbs: [
|
|
kube.RoleBinding(cluster.name(el.name)) {
|
|
metadata+: cluster.metadata,
|
|
roleRef_: el.role,
|
|
subjects_: [el.sa],
|
|
},
|
|
for el in [
|
|
// Allow Operator SA to perform Cluster Mgmt in this namespace.
|
|
{ name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa.system },
|
|
{ name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd },
|
|
{ name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr },
|
|
{ name: "cmd-reporter", role: cluster.roles.cmdReporter, sa: cluster.sa.cmdReporter },
|
|
{ name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr },
|
|
]
|
|
],
|
|
|
|
mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) {
|
|
metadata+: {
|
|
namespace: operator.cfg.namespace,
|
|
},
|
|
roleRef_: cluster.roles.mgrSystem,
|
|
subjects_: [cluster.sa.mgr],
|
|
},
|
|
|
|
osdClusterRB: kube.ClusterRoleBinding(cluster.name("osd-cluster")) {
|
|
metadata+: {
|
|
namespace:: null,
|
|
},
|
|
roleRef_: cluster.roles.osdCluster,
|
|
subjects_: [cluster.sa.osd],
|
|
},
|
|
|
|
|
|
cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
cephVersion: {
|
|
image: "quay.io/ceph/ceph:v16.2.5",
|
|
allowUnsupported: true,
|
|
},
|
|
dataDirHostPath: if name == "ceph-waw2" then "/var/lib/rook" else "/var/lib/rook-%s" % [name],
|
|
dashboard: {
|
|
ssl: false,
|
|
enabled: true,
|
|
port: 8080,
|
|
},
|
|
} + cluster.spec,
|
|
},
|
|
|
|
dashboardService: kube.Service(cluster.name("dashboard")) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
ports: [
|
|
{ name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" },
|
|
],
|
|
selector: {
|
|
app: "rook-ceph-mgr",
|
|
rook_cluster: name,
|
|
},
|
|
type: "ClusterIP",
|
|
},
|
|
},
|
|
|
|
dashboardIngress: kube.SimpleIngress(cluster.name("dashboard")) {
|
|
hosts:: ["%s.hswaw.net" % name],
|
|
target_service:: cluster.dashboardService,
|
|
},
|
|
|
|
# Benji is a backup tool, external to rook, that we use for backing up
|
|
# RBDs.
|
|
benji: {
|
|
sa: kube.ServiceAccount(cluster.name("benji")) {
|
|
metadata+: cluster.metadata,
|
|
},
|
|
|
|
cr: kube.ClusterRole(cluster.name("benji")) {
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: [
|
|
"persistentvolumes",
|
|
"persistentvolumeclaims"
|
|
],
|
|
verbs: ["list", "get"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: [
|
|
"events",
|
|
],
|
|
verbs: ["create", "update"],
|
|
},
|
|
],
|
|
},
|
|
|
|
crb: kube.ClusterRoleBinding(cluster.name("benji")) {
|
|
roleRef_: cluster.benji.cr,
|
|
subjects_: [cluster.benji.sa],
|
|
},
|
|
|
|
config: kube.Secret(cluster.name("benji-config")) {
|
|
metadata+: cluster.metadata,
|
|
data_: {
|
|
"benji.yaml": std.manifestJson({
|
|
configurationVersion: '1',
|
|
databaseEngine: 'sqlite:////data/benji.sqlite',
|
|
defaultStorage: 'wasabi',
|
|
storages: [
|
|
{
|
|
name: "wasabi",
|
|
storageId: 1,
|
|
module: "s3",
|
|
configuration: cluster.spec.benji.s3Configuration {
|
|
activeTransforms: ["encrypt"],
|
|
},
|
|
},
|
|
],
|
|
transforms: [
|
|
{
|
|
name: "encrypt",
|
|
module: "aes_256_gcm",
|
|
configuration: {
|
|
# not secret.
|
|
kdfSalt: "T2huZzZpcGhhaWM3QWVwaDhybzRhaDNhbzFpc2VpOWFobDNSZWVQaGVvTWV1bmVaYWVsNHRoYWg5QWVENHNoYWg0ZGFoN3Rlb3NvcHVuZzNpZXZpMm9vTG9vbmc1YWlmb0RlZXAwYmFobDlab294b2hjaG9odjRhbzFsYWkwYWk=",
|
|
kdfIterations: 2137,
|
|
password: cluster.spec.benji.encryptionPassword,
|
|
},
|
|
},
|
|
],
|
|
ios: [
|
|
{ name: pool, module: "rbd" }
|
|
for pool in cluster.spec.benji.pools
|
|
],
|
|
}),
|
|
},
|
|
},
|
|
|
|
# Yes, Benji keeps data (backup metadata) on the ceph cluster that
|
|
# it backs up. However:
|
|
# - we add a command to benji-k8s to also copy over the sqlite
|
|
# database over to s3
|
|
# - benji can, in a pinch, restore without a database if a version
|
|
# is known: https://benji-backup.me/restore.html#restoring-without-a-database
|
|
data: kube.PersistentVolumeClaim(cluster.name("benji-data")) {
|
|
metadata+: cluster.metadata,
|
|
spec+: {
|
|
storageClassName: cluster.spec.benji.metadataStorageClass,
|
|
accessModes: [ "ReadWriteOnce" ],
|
|
resources: {
|
|
requests: {
|
|
storage: "1Gi",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
# Extra scripts.
|
|
extrabins: kube.ConfigMap(cluster.name("benji-extrabins")) {
|
|
metadata+: cluster.metadata,
|
|
data: {
|
|
"metabackup.sh" : |||
|
|
# Make backups of sqlite3 metadata used by Benji.
|
|
# The backups live in the same bucket as backups, and the metabackups
|
|
# are named `metabackup-0..10`, where 0 is the newest backup. Any time
|
|
# this script is called, backups get shifted one way to the left (9 to 10,
|
|
# 8 to 9, etc). This ensures we have at least 10 backup replicas.
|
|
|
|
set -e
|
|
|
|
which s3cmd || pip install --upgrade s3cmd
|
|
|
|
AWS_ACCESS_KEY_ID=$(jq -r .storages[0].configuration.awsAccessKeyId < /etc/benji/benji.yaml)
|
|
AWS_SECRET_ACCESS_KEY=$(jq -r .storages[0].configuration.awsSecretAccessKey < /etc/benji/benji.yaml)
|
|
BUCKET=$(jq -r .storages[0].configuration.bucketName < /etc/benji/benji.yaml)
|
|
|
|
s3() {
|
|
s3cmd --host=s3.wasabisys.com \
|
|
"--host-bucket=%(bucket)s.s3.wasabisys.com" \
|
|
--region=eu-central-1 \
|
|
--access_key=$AWS_ACCESS_KEY_ID \
|
|
--secret_key=$AWS_SECRET_ACCESS_KEY \
|
|
"$@"
|
|
}
|
|
|
|
# Copy over old backups, if they exist.
|
|
for i in `seq 9 -1 0`; do
|
|
from="s3://$BUCKET/metabackup-$i.sqlite3"
|
|
to="s3://$BUCKET/metabackup-$((i+1)).sqlite3"
|
|
|
|
if [[ $(s3 ls $from | wc -l) -eq 0 ]]; then
|
|
echo "$from does not exist, skipping shift."
|
|
continue
|
|
fi
|
|
echo "Moving $from to $to..."
|
|
s3 mv $from $to
|
|
done
|
|
|
|
# Make new metabackup.
|
|
s3 put /data/benji.sqlite s3://$BUCKET/metabackup-0.sqlite3
|
|
|
|
|||,
|
|
"get-rook-creds.sh": |||
|
|
# Based on the Rook Toolbox /usr/local/bin/toolbox.sh script.
|
|
# Copyright 2016 The Rook Authors. All rights reserved.
|
|
|
|
CEPH_CONFIG="/etc/ceph/ceph.conf"
|
|
MON_CONFIG="/etc/rook/mon-endpoints"
|
|
KEYRING_FILE="/etc/ceph/keyring"
|
|
|
|
# create a ceph config file in its default location so ceph/rados tools can be used
|
|
# without specifying any arguments
|
|
write_endpoints() {
|
|
endpoints=$(cat ${MON_CONFIG})
|
|
|
|
# filter out the mon names
|
|
mon_endpoints=$(echo ${endpoints} | sed 's/[a-z]\+=//g')
|
|
|
|
# filter out the legacy mon names
|
|
mon_endpoints=$(echo ${mon_endpoints} | sed 's/rook-ceph-mon[0-9]\+=//g')
|
|
|
|
DATE=$(date)
|
|
echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
|
|
cat <<EOF > ${CEPH_CONFIG}
|
|
[global]
|
|
mon_host = ${mon_endpoints}
|
|
|
|
[client.admin]
|
|
keyring = ${KEYRING_FILE}
|
|
EOF
|
|
}
|
|
|
|
# watch the endpoints config file and update if the mon endpoints ever change
|
|
watch_endpoints() {
|
|
# get the timestamp for the target of the soft link
|
|
real_path=$(realpath ${MON_CONFIG})
|
|
initial_time=$(stat -c %Z ${real_path})
|
|
while true; do
|
|
real_path=$(realpath ${MON_CONFIG})
|
|
latest_time=$(stat -c %Z ${real_path})
|
|
|
|
if [[ "${latest_time}" != "${initial_time}" ]]; then
|
|
write_endpoints
|
|
initial_time=${latest_time}
|
|
fi
|
|
sleep 10
|
|
done
|
|
}
|
|
|
|
# create the keyring file
|
|
cat <<EOF > ${KEYRING_FILE}
|
|
[client.admin]
|
|
key = ${ROOK_ADMIN_SECRET}
|
|
EOF
|
|
|
|
# write the initial config file
|
|
write_endpoints
|
|
|
|
# continuously update the mon endpoints if they fail over
|
|
watch_endpoints &
|
|
|||
|
|
},
|
|
},
|
|
|
|
cronjob: kube.CronJob(cluster.name("benji")) {
|
|
metadata+: cluster.metadata,
|
|
spec+: { # CronJob Spec
|
|
schedule: "42 0 * * *", # Daily at 42 minute past midnight.
|
|
jobTemplate+: {
|
|
spec+: { # Job Spec
|
|
selector:: null,
|
|
template+: {
|
|
spec+: { # PodSpec
|
|
serviceAccountName: cluster.benji.sa.metadata.name,
|
|
containers_: {
|
|
benji: kube.Container(cluster.name("benji")) {
|
|
image: "ghcr.io/elemental-lf/benji-k8s@sha256:8ab74b210bfdf6ebbf0017144b34fb840ad9ad225edf27dd3394fbd8332e649d",
|
|
volumeMounts_: {
|
|
extrabins: { mountPath: "/usr/local/extrabins" },
|
|
monendpoints: { mountPath: "/etc/rook" },
|
|
benjiconfig: { mountPath: "/etc/benji" },
|
|
data: { mountPath: "/data" },
|
|
},
|
|
env_: {
|
|
ROOK_ADMIN_SECRET: { secretKeyRef: { name: "rook-ceph-mon", key: "admin-secret" }},
|
|
},
|
|
command: [
|
|
"bash", "-c", |||
|
|
bash /usr/local/extrabins/get-rook-creds.sh
|
|
benji-backup-pvc
|
|
benji-command enforce latest3,hours48,days7,months12
|
|
benji-command cleanup
|
|
bash /usr/local/extrabins/metabackup.sh
|
|
|||
|
|
],
|
|
},
|
|
},
|
|
volumes_: {
|
|
data: kube.PersistentVolumeClaimVolume(cluster.benji.data),
|
|
benjiconfig: kube.SecretVolume(cluster.benji.config),
|
|
extrabins: kube.ConfigMapVolume(cluster.benji.extrabins),
|
|
monendpoints: {
|
|
configMap: {
|
|
name: "rook-ceph-mon-endpoints",
|
|
items: [
|
|
{ key: "data", path: "mon-endpoints" },
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
ReplicatedBlockPool(cluster, name):: {
|
|
local pool = self,
|
|
name:: name,
|
|
|
|
spec:: error "spec must be specified",
|
|
|
|
pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
|
|
metadata+: cluster.metadata,
|
|
spec: pool.spec,
|
|
},
|
|
|
|
storageClass: kube.StorageClass(name) {
|
|
provisioner: "ceph.rook.io/block",
|
|
parameters: {
|
|
blockPool: pool.pool.metadata.name,
|
|
clusterNamespace: pool.pool.metadata.namespace,
|
|
fstype: "ext4",
|
|
},
|
|
reclaimPolicy: "Retain",
|
|
},
|
|
},
|
|
|
|
ECBlockPool(cluster, name):: {
|
|
local pool = self,
|
|
name:: name,
|
|
metadataReplicas:: 3,
|
|
|
|
spec:: error "spec must be specified",
|
|
|
|
pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
|
|
metadata+: cluster.metadata,
|
|
spec: pool.spec,
|
|
},
|
|
metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
failureDomain: "host",
|
|
replicated: {
|
|
size: pool.metadataReplicas,
|
|
},
|
|
},
|
|
},
|
|
|
|
storageClass: kube.StorageClass(name) {
|
|
provisioner: "ceph.rook.io/block",
|
|
parameters: {
|
|
blockPool: pool.metapool.metadata.name,
|
|
dataBlockPool: pool.pool.metadata.name,
|
|
clusterNamespace: pool.pool.metadata.namespace,
|
|
fstype: "ext4",
|
|
},
|
|
reclaimPolicy: "Retain",
|
|
},
|
|
},
|
|
|
|
// This is a rook CephObjectRealm which corresponds to a radosgw realm.
|
|
//
|
|
// A realm is a 'world' of radosgw user-facing metadata, like credentials,
|
|
// buckets, and underlying structures like zones and zonegroups. A realm
|
|
// contains zonegroups and zones, but a single Ceph cluster can actually
|
|
// serve multiple realms, by running multiple radosgw instances.
|
|
S3ObjectRealm(cluster, name):: {
|
|
cluster:: cluster,
|
|
realm: kube._Object("ceph.rook.io/v1", "CephObjectRealm", name) {
|
|
metadata+: cluster.metadata,
|
|
},
|
|
},
|
|
|
|
// This is a rook CephObjectZoneGroup which corresponds to a radosgw
|
|
// zonegroup.
|
|
//
|
|
// A zonegroup contains zones, and zones within a zonegroup will serve a
|
|
// concise view of objects in buckets, and will sync between eachother to
|
|
// eventually contain the same data.
|
|
//
|
|
// A single zonegroup within a realm must be a 'master' zonegroup, and will
|
|
// then hold and replicate the metadata of this realm. All realm operations
|
|
// via radosgw-admin must be performed within the master zonegroup.
|
|
S3ObjectZoneGroup(realm, name):: {
|
|
realm:: realm,
|
|
zonegroup: kube._Object("ceph.rook.io/v1", "CephObjectZoneGroup", name) {
|
|
metadata+: realm.cluster.metadata,
|
|
spec+: {
|
|
realm: realm.realm.metadata.name,
|
|
},
|
|
},
|
|
},
|
|
|
|
// This is a CephObjectZone but also a CephObjectStore.
|
|
//
|
|
// Rook attempts to hide away Ceph's radosgw multisite structures
|
|
// (realm/zonegroup/zone) by presenting a single CRD named
|
|
// 'CephObjectStore'. When such a resource is created, Rook will create a
|
|
// realm, zonegroup and zone under the hood, as a radosgw zone is required
|
|
// to serve data, and a radosgw zone cannot exist without a zonegroup, and
|
|
// a radosgw zonegroup cannot exist without a realm.
|
|
//
|
|
// However, rook also exposes the lower-level API by letting the user
|
|
// specify 'zone' in the ObjectStore's spec, which should point to a
|
|
// CephObjectZone. Then, an entirely different reconciliation codepath is
|
|
// taken and instead users are expected to manage
|
|
// CephObject{Realm,ZoneGroup,Zone} manually at Ceph's native abstraction
|
|
// level.
|
|
//
|
|
// CephObjectStore not only represents a Ceph zone (and possibly
|
|
// zonegroup/realm), but also pods and services that are required to servev
|
|
// radosgw data publicly. That's why S3ObjectStore takes parameters like
|
|
// 'public port' and 'instance number'.
|
|
//
|
|
// To add to the confusion, our S3ObjectStore wrapper also sprinkles in an
|
|
// Ingress with TLS to terminate the above service, and automatically
|
|
// creates a CephObjectZone.
|
|
//
|
|
// This whole jsonent abstraction basically forces users to manually create
|
|
// realms and zonegroups, but makes it very easy to do so. By forcing these
|
|
// to be explicitly created by rook objects, only the 'multi-site'
|
|
// reconciliation codepath is taken in rook, making things a bit more
|
|
// predictable.
|
|
S3ObjectStore(zonegroup, name):: {
|
|
local store = self,
|
|
spec:: {
|
|
dataPool: error "spec.dataPool must be specified",
|
|
metadataPool: error "spec.metadataPool must be specified",
|
|
},
|
|
|
|
cfg:: {
|
|
// We want to have each rgw run under a domain corresponding to the
|
|
// zone it's running in, but also to the zonegroup it's running in.
|
|
// This will allow us to DNS loadbalance a zonegroup to be backed
|
|
// by multiple zone ingresses.
|
|
domainParts: [
|
|
zonegroup.zone.metadata.name,
|
|
zonegroup.zonegroup.metadata.name,
|
|
],
|
|
domains: [
|
|
"object.ceph-%s.hswaw.net" % [part]
|
|
for part in cfg.domainParts
|
|
],
|
|
},
|
|
local cfg = self.cfg,
|
|
|
|
zone: kube._Object("ceph.rook.io/v1", "CephObjectZone", name) {
|
|
metadata+: zonegroup.realm.cluster.metadata,
|
|
spec: store.spec {
|
|
zoneGroup: zonegroup.zonegroup.metadata.name,
|
|
},
|
|
},
|
|
|
|
objectStore: kube._Object("ceph.rook.io/v1", "CephObjectStore", name) {
|
|
metadata+: zonegroup.realm.cluster.metadata,
|
|
spec: {
|
|
gateway: {
|
|
port: 80,
|
|
instances: 1,
|
|
allNodes: false,
|
|
},
|
|
zone: {
|
|
name: name,
|
|
},
|
|
preservePoolsOnDelete: true,
|
|
},
|
|
},
|
|
|
|
objectIngress: kube.Ingress(name) {
|
|
metadata+: zonegroup.realm.cluster.metadata {
|
|
annotations+: {
|
|
"kubernetes.io/tls-acme": "true",
|
|
"cert-manager.io/cluster-issuer": "letsencrypt-prod",
|
|
"nginx.ingress.kubernetes.io/proxy-body-size": "0",
|
|
},
|
|
},
|
|
|
|
spec+: {
|
|
tls: [
|
|
{
|
|
hosts: cfg.domains,
|
|
secretName: "%s-tls" % [name],
|
|
},
|
|
],
|
|
rules: [
|
|
{
|
|
host: domain,
|
|
http: {
|
|
paths: [
|
|
{
|
|
path: "/",
|
|
backend: {
|
|
serviceName: "rook-ceph-rgw-%s" % [name],
|
|
servicePort: 80,
|
|
},
|
|
},
|
|
]
|
|
},
|
|
}
|
|
for domain in cfg.domains
|
|
],
|
|
},
|
|
},
|
|
},
|
|
}
|