1
0
Fork 0

cluster: k0: bump rook to 1.6

This is needed to get Rook to talk to an external Ceph 16/Pacific
cluster.

This is mostly a bunch of CRD/RBAC changes. Most notably, we yeet our
own CRD rewrite and just slurp in upstream CRD defs.

Change-Id: I08e7042585722ae4440f97019a5212d6cf733fcc
master
q3k 2021-09-11 20:24:27 +00:00
parent 92c8dc6532
commit 464fb04f39
4 changed files with 8904 additions and 290 deletions

View File

@ -212,6 +212,7 @@ local pki = import "lib/pki.libsonnet";
rook: rook.Operator {
operator+: {
spec+: {
// Downscaled because of b.hswaw.net/6.
replicas: 0,
},
},

View File

@ -91,7 +91,20 @@ local rook = import "lib/rook.libsonnet";
// waw1 cluster - dead as of 2019/08/06, data corruption
// waw2 cluster - dead as of 2021/01/22, torn down (horrible M610 RAID controllers are horrible)
// waw3: 6TB SAS 3.5" HDDs
// waw3: 6TB SAS 3.5" HDDs, internal Rook cluster.
//
// Suffers from rook going apeshit and nuking all mons if enough of
// a control plane is up for rook to run but if nodes are
// unavailable to the point of it deciding that no mon exists and
// it should create some new ones, fully nuking the monmap and
// making recovery a pain.
//
// Supposedly new versions of Rook slowly fix these issues, but q3k
// doesn't personally trust this codebase anymore. He'd rather
// manage the actual Ceph cluster myself, we don't need all of this
// magic.
//
// See: b.hswaw.net/6
waw3: rook.Cluster(k0.cluster.rook, "ceph-waw3") {
spec: {
mon: {
@ -118,39 +131,30 @@ local rook = import "lib/rook.libsonnet";
databaseSizeMB: "1024",
journalSizeMB: "1024",
},
nodes: [
{
name: "dcr01s22.hswaw.net",
devices: [
// https://github.com/rook/rook/issues/1228
//{ name: "disk/by-id/wwan-0x" + wwan }
//for wwan in [
// "5000c5008508c433",
// "5000c500850989cf",
// "5000c5008508f843",
// "5000c5008508baf7",
//]
{ name: "sdn" },
{ name: "sda" },
{ name: "sdb" },
{ name: "sdc" },
{ name: "/dev/disk/by-id/wwn-0x" + id }
for id in [
"5000c5008508c433",
"5000c500850989cf",
"5000c5008508f843",
"5000c5008508baf7",
]
],
},
{
name: "dcr01s24.hswaw.net",
devices: [
// https://github.com/rook/rook/issues/1228
//{ name: "disk/by-id/wwan-0x" + wwan }
//for wwan in [
// "5000c5008508ee03",
// "5000c5008508c9ef",
// "5000c5008508df33",
// "5000c5008508dd3b",
//]
{ name: "sdm" },
{ name: "sda" },
{ name: "sdb" },
{ name: "sdc" },
{ name: "/dev/disk/by-id/wwn-0x" + id }
for id in [
"5000c5008508ee03",
"5000c5008508c9ef",
"5000c5008508df33",
"5000c5008508dd3b",
]
],
},
],

File diff suppressed because it is too large Load Diff

View File

@ -10,7 +10,7 @@ local oa = kube.OpenAPI;
local env = self,
local cfg = env.cfg,
cfg:: {
image: "rook/ceph:v1.2.7",
image: "rook/ceph:v1.6.9",
namespace: "rook-ceph-system",
},
@ -26,224 +26,25 @@ local oa = kube.OpenAPI;
policyInsecure: policies.AllowNamespaceInsecure(cfg.namespace),
crds: {
cephclusters: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephCluster") {
spec+: {
additionalPrinterColumns: [
{ name: "DataDirHostPath", type: "string", description: "Directory used on the K8s nodes", JSONPath: ".spec.dataDirHostPath" },
{ name: "MonCount", type: "string", description: "Number of MONs", JSONPath: ".spec.mon.count" },
{ name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
{ name: "State", type: "string", description: "Current State", JSONPath: ".status.state" },
{ name: "Health", type: "string", description: "Ceaph Health", JSONPath: ".status.ceph.health" },
],
validation: oa.Validation(oa.Dict {
spec: oa.Dict {
annotations: oa.Any,
cephVersion: oa.Dict {
allowUnsupported: oa.Boolean,
image: oa.String,
},
dashboard: oa.Dict {
enabled: oa.Boolean,
urlPrefix: oa.String,
port: oa.Integer { minimum: 0, maximum: 65535 },
ssl: oa.Boolean,
},
dataDirHostPath: oa.String { pattern: "^/(\\S+)" },
skipUpgradeChecks: oa.Boolean,
continueUpgradeAfterChecksEvenIfNotHealthy: oa.Boolean,
mon: oa.Dict {
allowMultiplePerNode: oa.Boolean,
count: oa.Integer { minimum: 0, maximum: 9 },
preferredCount: oa.Integer { minimum: 0, maximum: 9 },
},
mgr: oa.Dict {
modules: oa.Array(oa.Dict {
name: oa.String,
enabled: oa.Boolean,
}),
},
network: oa.Dict {
hostNetwork: oa.Boolean,
},
storage: oa.Dict {
disruptionManagement: oa.Dict {
managePodBudgets: oa.Boolean,
osdMaintenanceTimeout: oa.Integer,
manageMachineDisruptionBudgets: oa.Boolean,
},
useAllNodes: oa.Boolean,
nodes: oa.Array(oa.Dict {
name: oa.String,
config: oa.Dict {
metadataDevice: oa.String,
storeType: oa.String { pattern: "^(filestore|bluestore)$" },
databaseSizeMB: oa.String,
walSizeMB: oa.String,
journalSizeMB: oa.String,
osdsPerDevice: oa.String,
encryptedDevice: oa.String { pattern: "^(true|false)$" },
},
useAllDevices: oa.Boolean,
deviceFilter: oa.Any,
directories: oa.Array(oa.Dict {
path: oa.String,
}),
devices: oa.Array(oa.Dict {
name: oa.String,
}),
location: oa.Any,
resources: oa.Any,
}),
useAllDevices: oa.Boolean,
deviceFilter: oa.Any,
location: oa.Any,
directories: oa.Array(oa.Dict {
path: oa.String,
}),
config: oa.Any,
topologyAware: oa.Boolean,
},
monitoring: oa.Dict {
enabled: oa.Boolean,
rulesNamespace: oa.String,
},
rbdMirroring: oa.Dict {
workers: oa.Integer,
},
placement: oa.Any,
resources: oa.Any,
},
}),
// Grab CRDs from upstream YAML.
//
// We use a bit of jsonnet to remove some fields that kubebuilder (used
// upstream) added and to override preserveUnknownFIelds (as some older
// deployment apparently set it to true, which doesn't work for new
// CRDs that have default values).
crds: [
(el {
metadata+: {
annotations:: null,
creationTimestamp:: null,
},
},
cephfilesystems: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephFilesystem") {
status:: null,
spec+: {
additionalPrinterColumns: [
{ name: "ActiveMDS", type: "string", description: "Number of desired active MDS daemons", JSONPath: ".spec.metadataServer.activeCount" },
{ name: "Age", type: "date", JSONPath: ".metadata.creationTimestamp" },
],
validation: oa.Validation(oa.Dict {
spec: oa.Dict {
metadataServer: oa.Dict {
activeCount: oa.Integer,
activeStandby: oa.Boolean,
annotations: oa.Any,
placement: oa.Any,
resources: oa.Any,
},
metadataPool: oa.Dict {
failureDomain: oa.String,
replicated: oa.Dict {
size: oa.Integer,
},
erasureCoded: oa.Dict {
dataChunks: oa.Integer,
codingChunks: oa.Integer,
},
},
dataPools: oa.Array(oa.Dict {
failureDomain: oa.String,
replicated: oa.Dict {
site: oa.Integer,
erasureCoded: oa.Dict {
dataChunks: oa.Integer,
codingChunks: oa.Integer,
},
},
})
},
}),
preserveUnknownFields: false,
},
},
cephnfses: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephNFS") {
spec+: {
names+: {
plural: "cephnfses",
shortNames: ["nfs"],
},
validation: oa.Validation(oa.Dict {
spec: oa.Dict {
rados: oa.Dict {
pool: oa.String,
namespace: oa.String,
},
server: oa.Dict {
active: oa.Integer,
annotations: oa.Any,
placement: oa.Any,
resources: oa.Any,
},
},
}),
},
},
cephobjectstores: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStore") {
spec+: {
validation: oa.Validation(oa.Dict {
spec: oa.Dict {
gateway: oa.Dict {
type: oa.String,
sslCertificateRef: oa.Any,
port: oa.Integer,
securePort: oa.Any,
instances: oa.Integer,
annotations: oa.Any,
placement: oa.Any,
resources: oa.Any,
},
local poolDef = oa.Dict {
failureDomain: oa.String,
replicated: oa.Dict {
size: oa.Integer,
},
erasureCoded: oa.Dict {
dataChunks: oa.Integer,
codingChunks: oa.Integer,
},
},
metadataPool: poolDef,
dataPool: poolDef,
},
}),
},
},
cephobjectstoreusers: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephObjectStoreUser"),
cephblockpools: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephBlockPool"),
volumes: kube.CustomResourceDefinition("rook.io", "v1alpha2", "Volume") {
spec+: {
names+: {
shortNames: ["rv"],
},
},
},
objectbuckets: kube.CustomResourceDefinition("objectbucket.io", "v1alpha1", "ObjectBucket") {
spec+: {
names+: {
shortNames: ["ob", "obs"],
},
scope: "Cluster",
subresources: { status: {} },
},
},
objectbucketclaims: kube.CustomResourceDefinition("objectbucket.io", "v1alpha1", "ObjectBucketClaim") {
spec+: {
names+: {
shortNames: ["obc", "obcs"],
},
subresources: { status: {} },
},
},
cephclients: kube.CustomResourceDefinition("ceph.rook.io", "v1", "CephClient") {
spec+: {
validation: oa.Validation(oa.Dict {
spec: oa.Dict {
caps: oa.Any,
},
}),
},
},
},
})
for el in (std.native("parseYaml")(importstr "rook-crds.yaml")) if el != null
],
sa: {
system: kube.ServiceAccount("rook-ceph-system") {
@ -268,23 +69,23 @@ local oa = kube.OpenAPI;
metadata+: env.metadata { namespace:: null },
rules: [
{
apiGroups: [""],
resources: ["secrets", "pods", "pods/log", "services", "configmaps"],
apiGroups: ["", "apps", "extensions"],
resources: ["secrets", "pods", "pods/log", "services", "configmaps", "deployments", "daemonsets"],
verbs: ["get", "list", "watch", "patch", "create", "update", "delete"],
},
{
apiGroups: ["apps"],
resources: ["deployments", "daemonsets", "replicasets"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
},
],
},
global: kube.ClusterRole("rook-ceph-global") {
metadata+: env.metadata { namespace:: null },
metadata+: env.metadata {
namespace:: null,
labels+: {
"rbac.ceph.rook.io/aggregate-to-rook-ceph-global": "true",
},
},
rules: [
{
apiGroups: [""],
resources: ["pods", "nodes", "nodes/proxy"],
resources: ["pods", "nodes", "nodes/proxy", "services"],
verbs: ["get", "list", "watch"],
},
{
@ -299,7 +100,7 @@ local oa = kube.OpenAPI;
},
{
apiGroups: ["batch"],
resources: ["jobs"],
resources: ["jobs", "cronjobs"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
},
{
@ -313,7 +114,7 @@ local oa = kube.OpenAPI;
verbs: ["*"],
},
{
apiGroups: ["policy", "apps"],
apiGroups: ["policy", "apps", "extensions"],
resources: ["poddisruptionbudgets", "deployments", "replicasets"],
verbs: ["*"],
},
@ -330,37 +131,18 @@ local oa = kube.OpenAPI;
{
apiGroups: ["storage.k8s.io"],
resources: ["csidrivers"],
verbs: ["create"],
verbs: ["create", "delete", "get", "update"],
},
{
apiGroups: ["k8s.cni.cncf.io"],
resources: ["network-attachment-definitions"],
verbs: ["get"],
},
],
},
// Upstream rook uses split ClusterRoles, with the 'main' role (eg rook-ceph-cluster-mgmt)
// using aggregationRules to point to a '-rules' role (eg rook-ceph-cluster-mgmt-rules) which
// contains the actual role rules. This was done to permit for a bettr upgrade experience on
// systems that only allow for a recreation of a clusterroles (see https://github.com/rook/rook/issues/2634
// for more background information).
// We do not use this split because our update mechanism is not broken. However, it seems
// that Rook started to use these split rules for other reasons, too. For instance, the
// mgr-cluster role in upstream not only aggregates its equivalent -rules role, but also
// the rook-ceph-object-bucket role. As such, we split mgr-cluster as they do in upstream.
// In the future, we may split the rest of the roles in order to stay consisdent with upsteam.
mgrCluster: kube.ClusterRole("rook-ceph-mgr-cluster") {
metadata+: env.metadata { namespace:: null },
aggregationRule: {
clusterRoleSelectors: [
{ matchLabels: { "rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster": "true" }},
],
},
},
mgrClusterRules: kube.ClusterRole("rook-ceph-mgr-cluster-rules") {
metadata+: env.metadata {
namespace:: null,
labels+: {
"rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster": "true",
},
},
rules: [
{
apiGroups: [""],
@ -377,9 +159,6 @@ local oa = kube.OpenAPI;
objectBucket: kube.ClusterRole("rook-ceph-object-bucket") {
metadata+: env.metadata {
namespace:: null,
labels+: {
"rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster": "true",
},
},
rules: [
{
@ -432,7 +211,12 @@ local oa = kube.OpenAPI;
},
cephfsExternalProvisionerRunner: kube.ClusterRole("cephfs-external-provisioner-runner") {
metadata+: env.metadata { namespace:: null },
metadata+: env.metadata {
namespace:: null,
labels+: {
"rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner": "true",
},
},
rules: [
{
apiGroups: [""],
@ -442,7 +226,7 @@ local oa = kube.OpenAPI;
{
apiGroups: [""],
resources: ["persistentvolumes"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
verbs: ["get", "list", "watch", "create", "delete", "update", "patch"],
},
{
apiGroups: [""],
@ -459,16 +243,56 @@ local oa = kube.OpenAPI;
resources: ["events"],
verbs: ["list", "watch", "create", "update", "patch"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshots"],
verbs: ["get", "list", "watch", "update"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotcontents"],
verbs: ["create", "get", "list", "watch", "update", "delete"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotclasses"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotcontents/status"],
verbs: ["update"],
},
{
apiGroups: ["apiextensions.k8s.io"],
resources: ["customresourcedefinitions"],
verbs: ["create", "list", "watch", "delete", "get", "update"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshots/status"],
verbs: ["update"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["volumeattachments"],
verbs: ["get", "list", "watch", "update"],
verbs: ["get", "list", "watch", "update", "patch"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["volumeattachments/status"],
verbs: ["patch"],
},
{
apiGroups: [""],
resources: ["nodes"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: [""],
resources: ["persistentvolumeclaims/status"],
verbs: ["update", "patch"],
},
],
},
@ -509,17 +333,22 @@ local oa = kube.OpenAPI;
},
rbdExternalProvisionerRunner: kube.ClusterRole("rbd-external-provisioner-runner") {
metadata+: env.metadata { namespace:: null },
metadata+: env.metadata {
namespace:: null,
labels+: {
"rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner": "true",
},
},
rules: [
{
apiGroups: [""],
resources: ["secrets"],
verbs: ["get", "list"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: [""],
resources: ["persistentvolumes"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
verbs: ["get", "list", "watch", "create", "delete", "update", "patch"],
},
{
apiGroups: [""],
@ -529,7 +358,12 @@ local oa = kube.OpenAPI;
{
apiGroups: ["storage.k8s.io"],
resources: ["volumeattachments"],
verbs: ["get", "list", "watch", "update"],
verbs: ["get", "list", "watch", "update", "patch"],
},
{
apiGroups: ["storage.k8s.io"],
resources: ["volumeattachments/status"],
verbs: ["patch"],
},
{
apiGroups: [""],
@ -546,6 +380,11 @@ local oa = kube.OpenAPI;
resources: ["events"],
verbs: ["list", "watch", "create", "update", "patch"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshots"],
verbs: ["get", "list", "watch", "update"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotcontents"],
@ -556,6 +395,11 @@ local oa = kube.OpenAPI;
resources: ["volumesnapshotclasses"],
verbs: ["get", "list", "watch"],
},
{
apiGroups: ["snapshot.storage.k8s.io"],
resources: ["volumesnapshotcontents/status"],
verbs: ["update"],
},
{
apiGroups: ["apiextensions.k8s.io"],
resources: ["customresourcedefinitions"],
@ -566,6 +410,36 @@ local oa = kube.OpenAPI;
resources: ["volumesnapshots/status"],
verbs: ["update"],
},
{
apiGroups: [""],
resources: ["persistentvolumeclaims/status"],
verbs: ["update", "patch"],
},
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["get"],
},
{
apiGroups: ["replication.storage.openshift.io"],
resources: ["volumereplications", "volumereplicationclasses"],
verbs: ["create", "delete", "get", "list", "patch", "update", "watch"],
},
{
apiGroups: ["replication.storage.openshift.io"],
resources: ["volumereplications/finalizers"],
verbs: ["update"],
},
{
apiGroups: ["replication.storage.openshift.io"],
resources: ["volumereplications/status"],
verbs: ["get", "patch", "update"],
},
{
apiGroups: ["replication.storage.openshift.io"],
resources: ["volumereplicationclasses/status"],
verbs: ["get"],
},
],
},
},
@ -581,7 +455,7 @@ local oa = kube.OpenAPI;
roleRef_: env.crs.objectBucket,
subjects_: [env.sa.system],
},
cephfsCSINodeplugin: kube.ClusterRoleBinding("cepfs-csi-nodeplugin") {
cephfsCSINodeplugin: kube.ClusterRoleBinding("cephfs-csi-nodeplugin") {
metadata+: env.metadata { namespace:: null },
roleRef_: env.crs.cephfsCSINodeplugin,
subjects_: [env.sa.csiCephfsPlugin],
@ -614,9 +488,14 @@ local oa = kube.OpenAPI;
},
{
apiGroups: ["apps"],
resources: ["deployments", "statefulsets", "daemonsets"],
resources: ["daemonsets", "statefulsets", "deployments"],
verbs: ["get", "list", "watch", "create", "update", "delete"],
},
{
apiGroups: ["k8s.cni.cncf.io"],
resources: ["network-attachment-definitions"],
verbs: ["get"],
},
],
},
cephfsExternalProvisioner: kube.Role("cephfs-external-provisioner-cfg") {
@ -650,7 +529,7 @@ local oa = kube.OpenAPI;
{
apiGroups: [""],
resources: ["configmaps"],
verbs: ["get", "list", "watch", "create", "delete"],
verbs: ["get", "list", "watch", "create", "delete", "update"],
},
{
apiGroups: ["coordination.k8s.io"],