forked from hswaw/hscloud
1265 lines
54 KiB
Plaintext
1265 lines
54 KiB
Plaintext
# Deploy Rook/Ceph Operator
|
|
|
|
local kube = import "../../../kube/kube.libsonnet";
|
|
local policies = import "../../../kube/policies.libsonnet";
|
|
|
|
local oa = kube.OpenAPI;
|
|
|
|
{
|
|
Operator: {
|
|
local env = self,
|
|
local cfg = env.cfg,
|
|
cfg:: {
|
|
image: "rook/ceph:v1.2.7",
|
|
namespace: "rook-ceph-system",
|
|
},
|
|
|
|
metadata:: {
|
|
namespace: cfg.namespace,
|
|
labels: {
|
|
"operator": "rook",
|
|
"storage-backend": "ceph",
|
|
},
|
|
},
|
|
|
|
namespace: kube.Namespace(cfg.namespace),
|
|
|
|
policyInsecure: policies.AllowNamespaceInsecure(cfg.namespace),
|
|
|
|
crds: {
|
|
# BUG: cannot control this because of:
|
|
# ERROR Error updating customresourcedefinitions cephclusters.ceph.rook.io: expected kind, but got map
|
|
# TODO(q3k): debug and fix kubecfg (it's _not_ just https://github.com/bitnami/kubecfg/issues/259 )
|
|
cephclusters:: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
|
|
spec+: {
|
|
additionalPrinterColumns: [
|
|
{ name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" },
|
|
{ name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" },
|
|
{ name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
|
|
{ name: "State", type: "string", description: "Current State", JSONPath: ".status.state" },
|
|
{ name: "Health", type: "string", description: "Ceaph Health", JSONPath: ".status.ceph.health" },
|
|
],
|
|
validation: oa.Validation(oa.Dict {
|
|
spec: oa.Dict {
|
|
annotations: oa.Any,
|
|
cephVersion: oa.Dict {
|
|
allowUnsupported: oa.Boolean,
|
|
image: oa.String,
|
|
},
|
|
dashboard: oa.Dict {
|
|
enabled: oa.Boolean,
|
|
urlPrefix: oa.String,
|
|
port: oa.Integer { minimum: 0, maximum: 65535 },
|
|
ssl: oa.Boolean,
|
|
},
|
|
dataDirHostPath: oa.String { pattern: "^/(\\S+)" },
|
|
skipUpgradeChecks: oa.Boolean,
|
|
continueUpgradeAfterChecksEvenIfNotHealthy: oa.Boolean,
|
|
mon: oa.Dict {
|
|
allowMultiplePerNode: oa.Boolean,
|
|
count: oa.Integer { minimum: 0, maximum: 9 },
|
|
preferredCount: oa.Integer { minimum: 0, maximum: 9 },
|
|
},
|
|
mgr: oa.Dict {
|
|
modules: oa.Array(oa.Dict {
|
|
name: oa.String,
|
|
enabled: oa.Boolean,
|
|
}),
|
|
},
|
|
network: oa.Dict {
|
|
hostNetwork: oa.Boolean,
|
|
},
|
|
storage: oa.Dict {
|
|
disruptionManagement: oa.Dict {
|
|
managePodBudgets: oa.Boolean,
|
|
osdMaintenanceTimeout: oa.Integer,
|
|
manageMachineDisruptionBudgets: oa.Boolean,
|
|
},
|
|
useAllNodes: oa.Boolean,
|
|
nodes: oa.Array(oa.Dict {
|
|
name: oa.String,
|
|
config: oa.Dict {
|
|
metadataDevice: oa.String,
|
|
storeType: oa.String { pattern: "^(filestore|bluestore)$" },
|
|
databaseSizeMB: oa.String,
|
|
walSizeMB: oa.String,
|
|
journalSizeMB: oa.String,
|
|
osdsPerDevice: oa.String,
|
|
encryptedDevice: oa.String { pattern: "^(true|false)$" },
|
|
},
|
|
useAllDevices: oa.Boolean,
|
|
deviceFilter: oa.Any,
|
|
directories: oa.Array(oa.Dict {
|
|
path: oa.String,
|
|
}),
|
|
devices: oa.Array(oa.Dict {
|
|
name: oa.String,
|
|
}),
|
|
location: oa.Any,
|
|
resources: oa.Any,
|
|
}),
|
|
useAllDevices: oa.Boolean,
|
|
deviceFilter: oa.Any,
|
|
location: oa.Any,
|
|
directories: oa.Array(oa.Dict {
|
|
path: oa.String,
|
|
}),
|
|
config: oa.Any,
|
|
topologyAware: oa.Boolean,
|
|
},
|
|
monitoring: oa.Dict {
|
|
enabled: oa.Boolean,
|
|
rulesNamespace: oa.String,
|
|
},
|
|
rbdMirroring: oa.Dict {
|
|
workers: oa.Integer,
|
|
},
|
|
placement: oa.Any,
|
|
resources: oa.Any,
|
|
},
|
|
}),
|
|
},
|
|
},
|
|
cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") {
|
|
spec+: {
|
|
additionalPrinterColumns: [
|
|
{ name: "ActiveMDS", type: "string", description: "Number of desired active MDS daemons", JSONPath: ".spec.metadataServer.activeCount" },
|
|
{ name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
|
|
],
|
|
validation: oa.Validation(oa.Dict {
|
|
spec: oa.Dict {
|
|
metadataServer: oa.Dict {
|
|
activeCount: oa.Integer,
|
|
activeStandby: oa.Boolean,
|
|
annotations: oa.Any,
|
|
placement: oa.Any,
|
|
resources: oa.Any,
|
|
},
|
|
metadataPool: oa.Dict {
|
|
failureDomain: oa.String,
|
|
replicated: oa.Dict {
|
|
size: oa.Integer,
|
|
},
|
|
erasureCoded: oa.Dict {
|
|
dataChunks: oa.Integer,
|
|
codingChunks: oa.Integer,
|
|
},
|
|
},
|
|
dataPools: oa.Array(oa.Dict {
|
|
failureDomain: oa.String,
|
|
replicated: oa.Dict {
|
|
site: oa.Integer,
|
|
erasureCoded: oa.Dict {
|
|
dataChunks: oa.Integer,
|
|
codingChunks: oa.Integer,
|
|
},
|
|
},
|
|
})
|
|
},
|
|
}),
|
|
},
|
|
},
|
|
cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") {
|
|
spec+: {
|
|
names+: {
|
|
plural: "cephnfses",
|
|
shortNames: ["nfs"],
|
|
},
|
|
validation: oa.Validation(oa.Dict {
|
|
spec: oa.Dict {
|
|
rados: oa.Dict {
|
|
pool: oa.String,
|
|
namespace: oa.String,
|
|
},
|
|
server: oa.Dict {
|
|
active: oa.Integer,
|
|
annotations: oa.Any,
|
|
placement: oa.Any,
|
|
resources: oa.Any,
|
|
},
|
|
},
|
|
}),
|
|
},
|
|
},
|
|
cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore") {
|
|
spec+: {
|
|
validation: oa.Validation(oa.Dict {
|
|
spec: oa.Dict {
|
|
gateway: oa.Dict {
|
|
type: oa.String,
|
|
sslCertificateRef: oa.Any,
|
|
port: oa.Integer,
|
|
securePort: oa.Any,
|
|
instances: oa.Integer,
|
|
annotations: oa.Any,
|
|
placement: oa.Any,
|
|
resources: oa.Any,
|
|
},
|
|
local poolDef = oa.Dict {
|
|
failureDomain: oa.String,
|
|
replicated: oa.Dict {
|
|
size: oa.Integer,
|
|
},
|
|
erasureCoded: oa.Dict {
|
|
dataChunks: oa.Integer,
|
|
codingChunks: oa.Integer,
|
|
},
|
|
},
|
|
metadataPool: poolDef,
|
|
dataPool: poolDef,
|
|
},
|
|
}),
|
|
},
|
|
},
|
|
cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"),
|
|
cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"),
|
|
volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") {
|
|
spec+: {
|
|
names+: {
|
|
shortNames: ["rv"],
|
|
},
|
|
},
|
|
},
|
|
objectbuckets: kube.CustomResourceDefinition("objectbucket.io", "v1alpha1", "ObjectBucket") {
|
|
spec+: {
|
|
names+: {
|
|
shortNames: ["ob", "obs"],
|
|
},
|
|
scope: "Cluster",
|
|
subresources: { status: {} },
|
|
},
|
|
},
|
|
objectbucketclaims: kube.CustomResourceDefinition("objectbucket.io", "v1alpha1", "ObjectBucketClaim") {
|
|
spec+: {
|
|
names+: {
|
|
shortNames: ["obc", "obcs"],
|
|
},
|
|
subresources: { status: {} },
|
|
},
|
|
},
|
|
cephclients: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephClient") {
|
|
spec+: {
|
|
validation: oa.Validation(oa.Dict {
|
|
spec: oa.Dict {
|
|
caps: oa.Any,
|
|
},
|
|
}),
|
|
},
|
|
},
|
|
},
|
|
|
|
sa: {
|
|
system: kube.ServiceAccount("rook-ceph-system") {
|
|
metadata+: env.metadata,
|
|
},
|
|
csiCephfsPlugin: kube.ServiceAccount("rook-csi-cephfs-plugin-sa") {
|
|
metadata+: env.metadata,
|
|
},
|
|
csiCephfsProvisioner: kube.ServiceAccount("rook-csi-cephfs-provisioner-sa") {
|
|
metadata+: env.metadata,
|
|
},
|
|
csiRbdPlugin: kube.ServiceAccount("rook-csi-rbd-plugin-sa") {
|
|
metadata+: env.metadata,
|
|
},
|
|
csiRbdProvisioner: kube.ServiceAccount("rook-csi-rbd-provisioner-sa") {
|
|
metadata+: env.metadata,
|
|
},
|
|
},
|
|
|
|
crs: {
|
|
clusterMgmt: kube.ClusterRole("rook-ceph-cluster-mgmt") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["secrets", "pods", "pods/log", "services", "configmaps"],
|
|
verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["apps"],
|
|
resources: ["deployments", "daemonsets", "replicasets"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
],
|
|
},
|
|
global: kube.ClusterRole("rook-ceph-global") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["pods", "nodes", "nodes/proxy"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["events", "persistentvolumes", "persistentvolumeclaims", "endpoints"],
|
|
verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["storageclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["batch"],
|
|
resources: ["jobs"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["ceph.rook.io"],
|
|
resources: ["*"],
|
|
verbs: ["*"],
|
|
},
|
|
{
|
|
apiGroups: ["rook.io"],
|
|
resources: ["*"],
|
|
verbs: ["*"],
|
|
},
|
|
{
|
|
apiGroups: ["policy", "apps"],
|
|
resources: ["poddisruptionbudgets", "deployments", "replicasets"],
|
|
verbs: ["*"],
|
|
},
|
|
{
|
|
apiGroups: ["healthchecking.openshift.io"],
|
|
resources: ["machinedisruptionbudgets"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["machine.openshift.io"],
|
|
resources: ["machines"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["csidrivers"],
|
|
verbs: ["create"],
|
|
},
|
|
],
|
|
},
|
|
|
|
// Upstream rook uses split ClusterRoles, with the 'main' role (eg rook-ceph-cluster-mgmt)
|
|
// using aggregationRules to point to a '-rules' role (eg rook-ceph-cluster-mgmt-rules) which
|
|
// contains the actual role rules. This was done to permit for a bettr upgrade experience on
|
|
// systems that only allow for a recreation of a clusterroles (see https://github.com/rook/rook/issues/2634
|
|
// for more background information).
|
|
// We do not use this split because our update mechanism is not broken. However, it seems
|
|
// that Rook started to use these split rules for other reasons, too. For instance, the
|
|
// mgr-cluster role in upstream not only aggregates its equivalent -rules role, but also
|
|
// the rook-ceph-object-bucket role. As such, we split mgr-cluster as they do in upstream.
|
|
// In the future, we may split the rest of the roles in order to stay consisdent with upsteam.
|
|
|
|
mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
aggregationRule: {
|
|
clusterRoleSelectors: [
|
|
{ matchLabels: { "rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster": "true" }},
|
|
],
|
|
},
|
|
},
|
|
mgrClusterRules: kube.ClusterRole("rook-ceph-mgr-cluster-rules") {
|
|
metadata+: env.metadata {
|
|
namespace:: null,
|
|
labels+: {
|
|
"rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster": "true",
|
|
},
|
|
},
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps", "nodes", "nodes/proxy"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["events"],
|
|
verbs: ["create", "patch", "list", "get", "watch"],
|
|
},
|
|
]
|
|
},
|
|
objectBucket: kube.ClusterRole("rook-ceph-object-bucket") {
|
|
metadata+: env.metadata {
|
|
namespace:: null,
|
|
labels+: {
|
|
"rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster": "true",
|
|
},
|
|
},
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["secrets", "configmaps"],
|
|
verbs: ["*"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["storageclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["objectbucket.io"],
|
|
resources: ["*"],
|
|
verbs: ["*"],
|
|
},
|
|
],
|
|
},
|
|
|
|
cephfsCSINodeplugin: kube.ClusterRole("cephfs-csi-nodeplugin") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get", "list", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["namespaces"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumes"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["volumeattachments"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
],
|
|
},
|
|
|
|
cephfsExternalProvisionerRunner: kube.ClusterRole("cephfs-external-provisioner-runner") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["secrets"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumes"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumeclaims"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["storageclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["events"],
|
|
verbs: ["list", "watch", "create", "update", "patch"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["volumeattachments"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
],
|
|
},
|
|
|
|
rbdCSINodeplugin: kube.ClusterRole("rbd-csi-nodeplugin") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["secrets"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get", "list", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["namespaces"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumes"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["volumeattachments"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
],
|
|
},
|
|
|
|
rbdExternalProvisionerRunner: kube.ClusterRole("rbd-external-provisioner-runner") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["secrets"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumes"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["persistentvolumeclaims"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["volumeattachments"],
|
|
verbs: ["get", "list", "watch", "update"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["storage.k8s.io"],
|
|
resources: ["storageclasses"],
|
|
verbs: ["get", "list", "watch"]
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["events"],
|
|
verbs: ["list", "watch", "create", "update", "patch"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshotcontents"],
|
|
verbs: ["create", "get", "list", "watch", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshotclasses"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["apiextensions.k8s.io"],
|
|
resources: ["customresourcedefinitions"],
|
|
verbs: ["create", "list", "watch", "delete", "get", "update"],
|
|
},
|
|
{
|
|
apiGroups: ["snapshot.storage.k8s.io"],
|
|
resources: ["volumesnapshots/status"],
|
|
verbs: ["update"],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
|
|
crbs: {
|
|
global: kube.ClusterRoleBinding("ceph-rook-global") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.global,
|
|
subjects_: [env.sa.system],
|
|
},
|
|
objectBucket: kube.ClusterRoleBinding("rook-ceph-object-bucket") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.objectBucket,
|
|
subjects_: [env.sa.system],
|
|
},
|
|
cephfsCSINodeplugin: kube.ClusterRoleBinding("cepfs-csi-nodeplugin") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.cephfsCSINodeplugin,
|
|
subjects_: [env.sa.csiCephfsPlugin],
|
|
},
|
|
cephfsCSIProvisioner: kube.ClusterRoleBinding("cephfs-csi-provisioner") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.cephfsExternalProvisionerRunner,
|
|
subjects_: [env.sa.csiCephfsProvisioner],
|
|
},
|
|
rbdCSINodeplugin: kube.ClusterRoleBinding("rbd-csi-nodeplugin") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.rbdCSINodeplugin,
|
|
subjects_: [env.sa.csiRbdPlugin],
|
|
},
|
|
rbdCSIProvisioner: kube.ClusterRoleBinding("rbd-csi-provisioner") {
|
|
metadata+: env.metadata { namespace:: null },
|
|
roleRef_: env.crs.rbdExternalProvisionerRunner,
|
|
subjects_: [env.sa.csiRbdProvisioner],
|
|
},
|
|
},
|
|
|
|
roles: {
|
|
system: kube.Role("ceph-rook-system") {
|
|
metadata+: env.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["pods", "configmaps", "services"],
|
|
verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["apps"],
|
|
resources: ["deployments", "statefulsets", "daemonsets"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
],
|
|
},
|
|
cephfsExternalProvisioner: kube.Role("cephfs-external-provisioner-cfg") {
|
|
metadata+: env.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["endpoints"],
|
|
verbs: ["get", "watch", "list", "delete", "update", "create"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list", "create", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["coordination.k8s.io"],
|
|
resources: ["leases"],
|
|
verbs: ["get" ,"watch", "list", "delete", "update", "create"],
|
|
},
|
|
],
|
|
},
|
|
rbdExternalProvisioner: kube.Role("rbd-external-provisioner-cfg") {
|
|
metadata+: env.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["endpoints"],
|
|
verbs: ["get", "watch", "list", "delete", "update", "create"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list", "watch", "create", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["coordination.k8s.io"],
|
|
resources: ["leases"],
|
|
verbs: ["get" ,"watch", "list", "delete", "update", "create"],
|
|
},
|
|
],
|
|
},
|
|
},
|
|
|
|
rbs: {
|
|
system: kube.RoleBinding("ceph-rook-system") {
|
|
metadata+: env.metadata,
|
|
roleRef_: env.roles.system,
|
|
subjects_: [env.sa.system],
|
|
},
|
|
cephfsCSIProvisioner: kube.RoleBinding("cephfs-csi-provisioner-role-cfg") {
|
|
metadata+: env.metadata,
|
|
roleRef_: env.roles.cephfsExternalProvisioner,
|
|
subjects_: [env.sa.csiCephfsProvisioner],
|
|
},
|
|
rbdCSIProvisioner: kube.RoleBinding("rbd-csi-provisioner-role-cfg") {
|
|
metadata+: env.metadata,
|
|
roleRef_: env.roles.rbdExternalProvisioner,
|
|
subjects_: [env.sa.csiRbdProvisioner],
|
|
},
|
|
},
|
|
|
|
operator: kube.Deployment("rook-ceph-operator") {
|
|
metadata+: env.metadata,
|
|
spec+: {
|
|
template+: {
|
|
spec+: {
|
|
serviceAccountName: env.sa.system.metadata.name,
|
|
containers_: {
|
|
operator: kube.Container("rook-ceph-operator") {
|
|
image: cfg.image,
|
|
args: ["ceph", "operator"],
|
|
volumeMounts_: {
|
|
"rook-config": { mountPath: "/var/lib/rook" },
|
|
"default-config-dir": { mountPath: "/etc/ceph" },
|
|
},
|
|
env_: {
|
|
LIB_MODULES_DIR_PATH: "/run/current-system/kernel-modules/lib/modules/",
|
|
ROOK_ALLOW_MULTIPLE_FILESYSTEMS: "false",
|
|
ROOK_LOG_LEVEL: "INFO",
|
|
ROOK_MON_HEALTHCHECK_INTERVAL: "45s",
|
|
ROOK_MON_OUT_TIMEOUT: "600s",
|
|
ROOK_DISCOVER_DEVICES_INTERVAL: "60m",
|
|
ROOK_HOSTPATH_REQUIRES_PRIVILEGED: "false",
|
|
ROOK_ENABLE_SELINUX_RELABELING: "true",
|
|
ROOK_ENABLE_FSGROUP: "true",
|
|
NODE_NAME: kube.FieldRef("spec.nodeName"),
|
|
POD_NAME: kube.FieldRef("metadata.name"),
|
|
POD_NAMESPACE: kube.FieldRef("metadata.namespace"),
|
|
ROOK_CSI_KUBELET_DIR_PATH: "/var/lib/kubernetes"
|
|
},
|
|
},
|
|
},
|
|
volumes_: {
|
|
"rook-config": { emptyDir: {} },
|
|
"default-config-dir": { emptyDir: {} },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
// Create a new Ceph cluster in a new namespace.
|
|
Cluster(operator, name):: {
|
|
local cluster = self,
|
|
spec:: error "please define cluster spec",
|
|
|
|
|
|
metadata:: {
|
|
namespace: name,
|
|
},
|
|
|
|
name(suffix):: cluster.metadata.namespace + "-" + suffix,
|
|
|
|
namespace: kube.Namespace(cluster.metadata.namespace),
|
|
|
|
sa: {
|
|
// service accounts need to be hardcoded, see operator source.
|
|
osd: kube.ServiceAccount("rook-ceph-osd") {
|
|
metadata+: cluster.metadata,
|
|
},
|
|
mgr: kube.ServiceAccount("rook-ceph-mgr") {
|
|
metadata+: cluster.metadata,
|
|
},
|
|
cmdReporter: kube.ServiceAccount("rook-ceph-cmd-reporter") {
|
|
metadata+: cluster.metadata,
|
|
},
|
|
},
|
|
|
|
roles: {
|
|
osd: kube.Role(cluster.name("osd")) {
|
|
metadata+: cluster.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
],
|
|
},
|
|
osdCluster: kube.ClusterRole(cluster.name("osd-cluster")) {
|
|
metadata+: cluster.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["nodes"],
|
|
verbs: ["get", "list"],
|
|
},
|
|
],
|
|
},
|
|
mgr: kube.Role(cluster.name("mgr")) {
|
|
metadata+: cluster.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["pods", "services"],
|
|
verbs: ["get", "list", "watch"],
|
|
},
|
|
{
|
|
apiGroups: ["batch"],
|
|
resources: ["jobs"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
{
|
|
apiGroups: ["ceph.rook.io"],
|
|
resources: ["*"],
|
|
verbs: ["*"],
|
|
},
|
|
],
|
|
},
|
|
cmdReporter: kube.Role(cluster.name("cmd-reporter")) {
|
|
metadata+: cluster.metadata,
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["pods", "configmaps"],
|
|
verbs: ["get", "list", "watch", "create", "update", "delete"],
|
|
},
|
|
],
|
|
},
|
|
mgrSystem: kube.ClusterRole(cluster.name("mgr-system")) {
|
|
metadata+: cluster.metadata { namespace:: null },
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: ["configmaps"],
|
|
verbs: ["get", "list", "watch"],
|
|
}
|
|
],
|
|
},
|
|
},
|
|
|
|
rbs: [
|
|
kube.RoleBinding(cluster.name(el.name)) {
|
|
metadata+: cluster.metadata,
|
|
roleRef_: el.role,
|
|
subjects_: [el.sa],
|
|
},
|
|
for el in [
|
|
// Allow Operator SA to perform Cluster Mgmt in this namespace.
|
|
{ name: "cluster-mgmt", role: operator.crs.clusterMgmt, sa: operator.sa.system },
|
|
{ name: "osd", role: cluster.roles.osd, sa: cluster.sa.osd },
|
|
{ name: "mgr", role: cluster.roles.mgr, sa: cluster.sa.mgr },
|
|
{ name: "cmd-reporter", role: cluster.roles.cmdReporter, sa: cluster.sa.cmdReporter },
|
|
{ name: "mgr-cluster", role: operator.crs.mgrCluster, sa: cluster.sa.mgr },
|
|
]
|
|
],
|
|
|
|
mgrSystemRB: kube.RoleBinding(cluster.name("mgr-system")) {
|
|
metadata+: {
|
|
namespace: operator.cfg.namespace,
|
|
},
|
|
roleRef_: cluster.roles.mgrSystem,
|
|
subjects_: [cluster.sa.mgr],
|
|
},
|
|
|
|
osdClusterRB: kube.ClusterRoleBinding(cluster.name("osd-cluster")) {
|
|
metadata+: {
|
|
namespace:: null,
|
|
},
|
|
roleRef_: cluster.roles.osdCluster,
|
|
subjects_: [cluster.sa.osd],
|
|
},
|
|
|
|
|
|
cluster: kube._Object("ceph.rook.io/v1", "CephCluster", name) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
cephVersion: {
|
|
# https://github.com/rook/rook/issues/2945#issuecomment-483964014
|
|
image: "ceph/ceph:v14.2.16",
|
|
allowUnsupported: true,
|
|
},
|
|
dataDirHostPath: if name == "ceph-waw2" then "/var/lib/rook" else "/var/lib/rook-%s" % [name],
|
|
dashboard: {
|
|
ssl: false,
|
|
enabled: true,
|
|
port: 8080,
|
|
},
|
|
} + cluster.spec,
|
|
},
|
|
|
|
dashboardService: kube.Service(cluster.name("dashboard")) {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
ports: [
|
|
{ name: "dashboard", port: 80, targetPort: 8080, protocol: "TCP" },
|
|
],
|
|
selector: {
|
|
app: "rook-ceph-mgr",
|
|
rook_cluster: name,
|
|
},
|
|
type: "ClusterIP",
|
|
},
|
|
},
|
|
|
|
dashboardIngress: kube.Ingress(cluster.name("dashboard")) {
|
|
metadata+: cluster.metadata {
|
|
annotations+: {
|
|
"kubernetes.io/tls-acme": "true",
|
|
"certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
|
|
},
|
|
},
|
|
spec+: {
|
|
tls: [
|
|
{
|
|
hosts: ["%s.hswaw.net" % name],
|
|
secretName: cluster.name("dashboard"),
|
|
},
|
|
],
|
|
rules: [
|
|
{
|
|
host: "%s.hswaw.net" % name,
|
|
http: {
|
|
paths: [
|
|
{ path: "/", backend: cluster.dashboardService.name_port },
|
|
]
|
|
},
|
|
}
|
|
],
|
|
},
|
|
},
|
|
|
|
# Benji is a backup tool, external to rook, that we use for backing up
|
|
# RBDs.
|
|
benji: {
|
|
sa: kube.ServiceAccount(cluster.name("benji")) {
|
|
metadata+: cluster.metadata,
|
|
},
|
|
|
|
cr: kube.ClusterRole(cluster.name("benji")) {
|
|
rules: [
|
|
{
|
|
apiGroups: [""],
|
|
resources: [
|
|
"persistentvolumes",
|
|
"persistentvolumeclaims"
|
|
],
|
|
verbs: ["list", "get"],
|
|
},
|
|
{
|
|
apiGroups: [""],
|
|
resources: [
|
|
"events",
|
|
],
|
|
verbs: ["create", "update"],
|
|
},
|
|
],
|
|
},
|
|
|
|
crb: kube.ClusterRoleBinding(cluster.name("benji")) {
|
|
roleRef_: cluster.benji.cr,
|
|
subjects_: [cluster.benji.sa],
|
|
},
|
|
|
|
config: kube.Secret(cluster.name("benji-config")) {
|
|
metadata+: cluster.metadata,
|
|
data_: {
|
|
"benji.yaml": std.manifestJson({
|
|
configurationVersion: '1',
|
|
databaseEngine: 'sqlite:////data/benji.sqlite',
|
|
defaultStorage: 'wasabi',
|
|
storages: [
|
|
{
|
|
name: "wasabi",
|
|
storageId: 1,
|
|
module: "s3",
|
|
configuration: cluster.spec.benji.s3Configuration {
|
|
activeTransforms: ["encrypt"],
|
|
},
|
|
},
|
|
],
|
|
transforms: [
|
|
{
|
|
name: "encrypt",
|
|
module: "aes_256_gcm",
|
|
configuration: {
|
|
# not secret.
|
|
kdfSalt: "T2huZzZpcGhhaWM3QWVwaDhybzRhaDNhbzFpc2VpOWFobDNSZWVQaGVvTWV1bmVaYWVsNHRoYWg5QWVENHNoYWg0ZGFoN3Rlb3NvcHVuZzNpZXZpMm9vTG9vbmc1YWlmb0RlZXAwYmFobDlab294b2hjaG9odjRhbzFsYWkwYWk=",
|
|
kdfIterations: 2137,
|
|
password: cluster.spec.benji.encryptionPassword,
|
|
},
|
|
},
|
|
],
|
|
ios: [
|
|
{ name: pool, module: "rbd" }
|
|
for pool in cluster.spec.benji.pools
|
|
],
|
|
}),
|
|
},
|
|
},
|
|
|
|
# Yes, Benji keeps data (backup metadata) on the ceph cluster that
|
|
# it backs up. However:
|
|
# - we add a command to benji-k8s to also copy over the sqlite
|
|
# database over to s3
|
|
# - benji can, in a pinch, restore without a database if a version
|
|
# is known: https://benji-backup.me/restore.html#restoring-without-a-database
|
|
data: kube.PersistentVolumeClaim(cluster.name("benji-data")) {
|
|
metadata+: cluster.metadata,
|
|
spec+: {
|
|
storageClassName: cluster.spec.benji.metadataStorageClass,
|
|
accessModes: [ "ReadWriteOnce" ],
|
|
resources: {
|
|
requests: {
|
|
storage: "1Gi",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
# Extra scripts.
|
|
extrabins: kube.ConfigMap(cluster.name("benji-extrabins")) {
|
|
metadata+: cluster.metadata,
|
|
data: {
|
|
"metabackup.sh" : |||
|
|
# Make backups of sqlite3 metadata used by Benji.
|
|
# The backups live in the same bucket as backups, and the metabackups
|
|
# are named `metabackup-0..10`, where 0 is the newest backup. Any time
|
|
# this script is called, backups get shifted one way to the left (9 to 10,
|
|
# 8 to 9, etc). This ensures we have at least 10 backup replicas.
|
|
|
|
set -e
|
|
|
|
which s3cmd || pip install --upgrade s3cmd
|
|
|
|
AWS_ACCESS_KEY_ID=$(jq -r .storages[0].configuration.awsAccessKeyId < /etc/benji/benji.yaml)
|
|
AWS_SECRET_ACCESS_KEY=$(jq -r .storages[0].configuration.awsSecretAccessKey < /etc/benji/benji.yaml)
|
|
BUCKET=$(jq -r .storages[0].configuration.bucketName < /etc/benji/benji.yaml)
|
|
|
|
s3() {
|
|
s3cmd --host=s3.wasabisys.com \
|
|
"--host-bucket=%(bucket)s.s3.wasabisys.com" \
|
|
--region=eu-central-1 \
|
|
--access_key=$AWS_ACCESS_KEY_ID \
|
|
--secret_key=$AWS_SECRET_ACCESS_KEY \
|
|
"$@"
|
|
}
|
|
|
|
# Copy over old backups, if they exist.
|
|
for i in `seq 9 -1 0`; do
|
|
from="s3://$BUCKET/metabackup-$i.sqlite3"
|
|
to="s3://$BUCKET/metabackup-$((i+1)).sqlite3"
|
|
|
|
if [[ $(s3 ls $from | wc -l) -eq 0 ]]; then
|
|
echo "$from does not exist, skipping shift."
|
|
continue
|
|
fi
|
|
echo "Moving $from to $to..."
|
|
s3 mv $from $to
|
|
done
|
|
|
|
# Make new metabackup.
|
|
s3 put /data/benji.sqlite s3://$BUCKET/metabackup-0.sqlite3
|
|
|
|
|||,
|
|
"get-rook-creds.sh": |||
|
|
# Based on the Rook Toolbox /usr/local/bin/toolbox.sh script.
|
|
# Copyright 2016 The Rook Authors. All rights reserved.
|
|
|
|
CEPH_CONFIG="/etc/ceph/ceph.conf"
|
|
MON_CONFIG="/etc/rook/mon-endpoints"
|
|
KEYRING_FILE="/etc/ceph/keyring"
|
|
|
|
# create a ceph config file in its default location so ceph/rados tools can be used
|
|
# without specifying any arguments
|
|
write_endpoints() {
|
|
endpoints=$(cat ${MON_CONFIG})
|
|
|
|
# filter out the mon names
|
|
mon_endpoints=$(echo ${endpoints} | sed 's/[a-z]\+=//g')
|
|
|
|
# filter out the legacy mon names
|
|
mon_endpoints=$(echo ${mon_endpoints} | sed 's/rook-ceph-mon[0-9]\+=//g')
|
|
|
|
DATE=$(date)
|
|
echo "$DATE writing mon endpoints to ${CEPH_CONFIG}: ${endpoints}"
|
|
cat <<EOF > ${CEPH_CONFIG}
|
|
[global]
|
|
mon_host = ${mon_endpoints}
|
|
|
|
[client.admin]
|
|
keyring = ${KEYRING_FILE}
|
|
EOF
|
|
}
|
|
|
|
# watch the endpoints config file and update if the mon endpoints ever change
|
|
watch_endpoints() {
|
|
# get the timestamp for the target of the soft link
|
|
real_path=$(realpath ${MON_CONFIG})
|
|
initial_time=$(stat -c %Z ${real_path})
|
|
while true; do
|
|
real_path=$(realpath ${MON_CONFIG})
|
|
latest_time=$(stat -c %Z ${real_path})
|
|
|
|
if [[ "${latest_time}" != "${initial_time}" ]]; then
|
|
write_endpoints
|
|
initial_time=${latest_time}
|
|
fi
|
|
sleep 10
|
|
done
|
|
}
|
|
|
|
# create the keyring file
|
|
cat <<EOF > ${KEYRING_FILE}
|
|
[client.admin]
|
|
key = ${ROOK_ADMIN_SECRET}
|
|
EOF
|
|
|
|
# write the initial config file
|
|
write_endpoints
|
|
|
|
# continuously update the mon endpoints if they fail over
|
|
watch_endpoints &
|
|
|||
|
|
},
|
|
},
|
|
|
|
cronjob: kube.CronJob(cluster.name("benji")) {
|
|
metadata+: cluster.metadata,
|
|
spec+: { # CronJob Spec
|
|
schedule: "42 0 * * *", # Daily at 42 minute past midnight.
|
|
jobTemplate+: {
|
|
spec+: { # Job Spec
|
|
selector:: null,
|
|
template+: {
|
|
spec+: { # PodSpec
|
|
serviceAccountName: cluster.benji.sa.metadata.name,
|
|
containers_: {
|
|
benji: kube.Container(cluster.name("benji")) {
|
|
# TODO(q3k): switch back to upstream after pull/65 goes in.
|
|
# Currently this is being built from github.com/q3k/benji.
|
|
# https://github.com/elemental-lf/benji/pull/65
|
|
image: "registry.k0.hswaw.net/q3k/benji-k8s:20191221-2336",
|
|
volumeMounts_: {
|
|
extrabins: { mountPath: "/usr/local/extrabins" },
|
|
monendpoints: { mountPath: "/etc/rook" },
|
|
benjiconfig: { mountPath: "/etc/benji" },
|
|
data: { mountPath: "/data" },
|
|
},
|
|
env_: {
|
|
ROOK_ADMIN_SECRET: { secretKeyRef: { name: "rook-ceph-mon", key: "admin-secret" }},
|
|
},
|
|
command: [
|
|
"bash", "-c", |||
|
|
bash /usr/local/extrabins/get-rook-creds.sh
|
|
benji-backup-pvc %s
|
|
benji-command enforce latest3,hours48,days7,months12
|
|
benji-command cleanup
|
|
bash /usr/local/extrabins/metabackup.sh
|
|
||| % [std.join(" ", ["--pool-filter=%s" % [p] for p in cluster.spec.benji.pools])],
|
|
],
|
|
},
|
|
},
|
|
volumes_: {
|
|
data: kube.PersistentVolumeClaimVolume(cluster.benji.data),
|
|
benjiconfig: kube.SecretVolume(cluster.benji.config),
|
|
extrabins: kube.ConfigMapVolume(cluster.benji.extrabins),
|
|
monendpoints: {
|
|
configMap: {
|
|
name: "rook-ceph-mon-endpoints",
|
|
items: [
|
|
{ key: "data", path: "mon-endpoints" },
|
|
],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
ReplicatedBlockPool(cluster, name):: {
|
|
local pool = self,
|
|
name:: name,
|
|
|
|
spec:: error "spec must be specified",
|
|
|
|
pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
|
|
metadata+: cluster.metadata,
|
|
spec: pool.spec,
|
|
},
|
|
|
|
storageClass: kube.StorageClass(name) {
|
|
provisioner: "ceph.rook.io/block",
|
|
parameters: {
|
|
blockPool: pool.pool.metadata.name,
|
|
clusterNamespace: pool.pool.metadata.namespace,
|
|
fstype: "ext4",
|
|
},
|
|
reclaimPolicy: "Retain",
|
|
},
|
|
},
|
|
|
|
ECBlockPool(cluster, name):: {
|
|
local pool = self,
|
|
name:: name,
|
|
metadataReplicas:: 3,
|
|
|
|
spec:: error "spec must be specified",
|
|
|
|
pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {
|
|
metadata+: cluster.metadata,
|
|
spec: pool.spec,
|
|
},
|
|
metapool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name + "-metadata") {
|
|
metadata+: cluster.metadata,
|
|
spec: {
|
|
failureDomain: "host",
|
|
replicated: {
|
|
size: pool.metadataReplicas,
|
|
},
|
|
},
|
|
},
|
|
|
|
storageClass: kube.StorageClass(name) {
|
|
provisioner: "ceph.rook.io/block",
|
|
parameters: {
|
|
blockPool: pool.metapool.metadata.name,
|
|
dataBlockPool: pool.pool.metadata.name,
|
|
clusterNamespace: pool.pool.metadata.namespace,
|
|
fstype: "ext4",
|
|
},
|
|
reclaimPolicy: "Retain",
|
|
},
|
|
},
|
|
|
|
S3ObjectStore(cluster, name):: {
|
|
local store = self,
|
|
spec:: error "spec must be specified",
|
|
objectStore: kube._Object("ceph.rook.io/v1", "CephObjectStore", name) {
|
|
metadata+: cluster.metadata,
|
|
spec: store.spec {
|
|
gateway: {
|
|
type: "s3",
|
|
port: 80,
|
|
instances: 1,
|
|
allNodes: false,
|
|
},
|
|
},
|
|
},
|
|
|
|
objectIngress: kube.Ingress(name) {
|
|
metadata+: cluster.metadata {
|
|
annotations+: {
|
|
"kubernetes.io/tls-acme": "true",
|
|
"certmanager.k8s.io/cluster-issuer": "letsencrypt-prod",
|
|
"nginx.ingress.kubernetes.io/proxy-body-size": "0",
|
|
},
|
|
},
|
|
spec+: {
|
|
tls: [
|
|
{
|
|
hosts: ["object.%s.hswaw.net" % [cluster.metadata.namespace]],
|
|
secretName: "%s-tls" % [name],
|
|
},
|
|
],
|
|
rules: [
|
|
{
|
|
host: "object.%s.hswaw.net" % [cluster.metadata.namespace],
|
|
http: {
|
|
paths: [
|
|
{
|
|
path: "/",
|
|
backend: {
|
|
serviceName: "rook-ceph-rgw-%s" % [name],
|
|
servicePort: 80,
|
|
},
|
|
},
|
|
]
|
|
},
|
|
}
|
|
],
|
|
},
|
|
},
|
|
},
|
|
}
|