cheshire
/
hscloud
forked from hswaw/hscloud
1
0
Fork 0
Code Pull Requests Activity

ceph-waw1 -> ceph-waw2 

Change-Id: I03d6244b9697a9efc06492114ef90cdb01e17601
master
q3k 2019-08-08 17:48:25 +02:00
parent 30317b4278
commit d07861b7df
7 changed files with 38 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/matrix/prod.jsonnet
View File

@ -17,7 +17,7 @@ local postgres = import "../../kube/postgres.libsonnet";
namespace: "matrix",
domain: "matrix.hackerspace.pl",
serverName: "hackerspace.pl",
storageClassName: "waw-hdd-redundant-1",
storageClassName: "waw-hdd-paranoid-2",
synapseImage: "matrixdotorg/synapse:v0.99.4",
riotImage: "bubuntux/riot-web:v1.1.2",
@ -267,7 +267,7 @@ local postgres = import "../../kube/postgres.libsonnet";
image: "registry.k0.hswaw.net/informatic/matrix-appservice-irc:0.11.2",
metadata: {},
config: std.native("parseYaml")(importstr "appservice-irc.yaml")[0],
storageClassName: "waw-hdd-redundant-1",
storageClassName: "waw-hdd-paranoid-2",
},
config: kube.ConfigMap("appservice-irc-%s" % [name]) {

31
cluster/kube/cluster.jsonnet
View File

@ -93,7 +93,7 @@ local Cluster(fqdn) = {
spec+: {
// TODO(q3k): Bring up the operator again when stability gets fixed
// See: https://github.com/rook/rook/issues/3059#issuecomment-492378873
replicas: 0,
replicas: 1,
},
},
},
@ -102,7 +102,8 @@ local Cluster(fqdn) = {
registry: registry.Environment {
cfg+: {
domain: "registry.%s" % [fqdn],
storageClassName: cfg.storageClassNameRedundant,
storageClassName: cfg.storageClassNameParanoid,
objectStorageName: "waw-hdd-redundant-2-object",
},
},
};
@ -113,24 +114,25 @@ local Cluster(fqdn) = {
local k0 = self,
cluster: Cluster("k0.hswaw.net") {
cfg+: {
storageClassNameRedundant: k0.ceph.blockRedundant.name,
storageClassNameParanoid: k0.ceph.blockParanoid.name,
},
},
cockroach: {
waw1: cockroachdb.Cluster("crdb-waw1") {
waw2: cockroachdb.Cluster("crdb-waw2") {
cfg+: {
topology: [
{ name: "bc01n01", node: "bc01n01.hswaw.net" },
{ name: "bc01n02", node: "bc01n02.hswaw.net" },
{ name: "bc01n03", node: "bc01n03.hswaw.net" },
],
hostPath: "/var/db/crdb-waw1",
hostPath: "/var/db/crdb-waw2",
},
},
},
ceph: {
// waw1 cluster
waw1: rook.Cluster(k0.cluster.rook, "ceph-waw1") {
// waw1 cluster - dead as of 2019/08/06, data corruption
// waw2 cluster
waw2: rook.Cluster(k0.cluster.rook, "ceph-waw2") {
spec: {
mon: {
count: 3,
@ -164,7 +166,7 @@ local Cluster(fqdn) = {
},
},
// redundant block storage
blockRedundant: rook.ECBlockPool(k0.ceph.waw1, "waw-hdd-redundant-1") {
blockRedundant: rook.ECBlockPool(k0.ceph.waw2, "waw-hdd-redundant-2") {
spec: {
failureDomain: "host",
erasureCoded: {
@ -173,8 +175,17 @@ local Cluster(fqdn) = {
},
},
},
// paranoid block storage (3 replicas)
blockParanoid: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-paranoid-2") {
spec: {
failureDomain: "host",
replicated: {
size: 3,
},
},
},
// yolo block storage (no replicas!)
blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw1, "waw-hdd-yolo-1") {
blockYolo: rook.ReplicatedBlockPool(k0.ceph.waw2, "waw-hdd-yolo-2") {
spec: {
failureDomain: "host",
replicated: {
@ -182,7 +193,7 @@ local Cluster(fqdn) = {
},
},
},
objectRedundant: rook.S3ObjectStore(k0.ceph.waw1, "waw-hdd-redundant-1-object") {
objectRedundant: rook.S3ObjectStore(k0.ceph.waw2, "waw-hdd-redundant-2-object") {
spec: {
metadataPool: {
failureDomain: "host",

15
cluster/kube/lib/registry.libsonnet
View File

@ -15,6 +15,7 @@ local cm = import "cert-manager.libsonnet";
namespace: "registry",
domain: error "domain must be set",
storageClassName: error "storageClassName must be set",
objectStoreName: error "objectStoreName must be set",
},
metadata(component):: {
@ -72,9 +73,9 @@ local cm = import "cert-manager.libsonnet";
blobdescriptor: "inmemory",
},
s3: {
regionendpoint: "https://object.ceph-waw1.hswaw.net",
regionendpoint: "https://object.ceph-waw2.hswaw.net",
bucket: "registry",
region: "waw-hdd-redunant-1-object:default-placement",
region: "waw-hdd-redunant-2-object:default-placement",
},
},
http: {
@ -154,6 +155,8 @@ local cm = import "cert-manager.libsonnet";
{ who: ["q3k", "inf"], what: "vms/*" },
{ who: ["q3k", "inf"], what: "app/*" },
{ who: ["q3k", "inf"], what: "go/svc/*" },
{ who: ["q3k"], what: "bgpwtf/*" },
{ who: ["q3k"], what: "devtools/*" },
],
acl: [
{
@ -253,11 +256,11 @@ local cm = import "cert-manager.libsonnet";
},
env_: {
REGISTRY_STORAGE_S3_ACCESSKEY: { secretKeyRef: {
name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName},
key: "AccessKey"
}},
REGISTRY_STORAGE_S3_SECRETKEY: { secretKeyRef: {
name: "rook-ceph-object-user-waw-hdd-redundant-1-object-registry",
name: "rook-ceph-object-user-%(objectStorageName)s-registry" % {objectStorageName: cfg.objectStorageName},
key: "SecretKey",
}},
},
@ -310,10 +313,10 @@ local cm = import "cert-manager.libsonnet";
registryStorageUser: kube._Object("ceph.rook.io/v1", "CephObjectStoreUser", "registry") {
metadata+: {
namespace: "ceph-waw1",
namespace: "ceph-waw2",
},
spec: {
store: "waw-hdd-redundant-1-object",
store: cfg.objectStorageName,
displayName: "docker-registry user",
},
},

2
cluster/kube/lib/rook.libsonnet
View File

@ -471,6 +471,8 @@ local kube = import "../../../kube/kube.libsonnet";
ReplicatedBlockPool(cluster, name):: {
local pool = self,
name:: name,
spec:: error "spec must be specified",
pool: kube._Object("ceph.rook.io/v1", "CephBlockPool", name) {

3
cluster/nix/cluster-configuration.nix
View File

@ -138,7 +138,8 @@ in rec {
advertiseAddress = "${node.ipAddr}";
etcd = {
servers = (map (n: "https://${n.fqdn}:2379") nodes);
# https://github.com/kubernetes/kubernetes/issues/72102
servers = (map (n: "https://${n.fqdn}:2379") ( [ node ] ));
caFile = pki.etcd.kube.ca;
keyFile = pki.etcd.kube.key;
certFile = pki.etcd.kube.cert;

2
devtools/gerrit/kube/prod.jsonnet
View File

@ -11,7 +11,7 @@ local gerrit = import "gerrit.libsonnet";
domain: "gerrit.hackerspace.pl",
identity: "7b6244cf-e30b-42c5-ba91-c329ef4e6cf1",
storageClassName: "waw-hdd-redundant-1",
storageClassName: "waw-hdd-paranoid-2",
secureSecret: "gerrit",
},

2
kube/postgres.libsonnet
View File

@ -8,7 +8,7 @@ local kube = import "kube.libsonnet";
cfg:: {
namespace: error "namespace must be set",
appName: error "app name must be set",
storageClassName: "waw-hdd-redundant-1",
storageClassName: "waw-hdd-paranoid-2",
prefix: "", # if set, should be 'foo-'
image: "postgres:10.4",