forked from hswaw/hscloud
This unifies nixpkgs with the one defined in //default.nix and makes it possible to use readTree to build the provisioners: nix-build -A cluster.nix.provision result/bin/provision Change-Id: I68dd70b9c8869c7c0b59f5007981eac03667b862
239 lines
7.7 KiB
Python
239 lines
7.7 KiB
Python
#!/usr/bin/env python
|
|
|
|
from builtins import object
|
|
|
|
import datetime
|
|
from io import BytesIO
|
|
import json
|
|
import logging
|
|
import os
|
|
import tempfile
|
|
import subprocess
|
|
import sys
|
|
|
|
from cryptography import x509
|
|
from cryptography.hazmat.backends import default_backend
|
|
import fabric
|
|
|
|
from tools import secretstore
|
|
|
|
import ca
|
|
|
|
|
|
local_root = os.getenv('hscloud_root')
|
|
if local_root is None:
|
|
raise Exception("Please source env.sh")
|
|
|
|
|
|
cluster = 'k0.hswaw.net'
|
|
ss = secretstore.SecretStore(
|
|
plain_root=os.path.join(local_root, 'cluster/secrets/plain'),
|
|
cipher_root=os.path.join(local_root, 'cluster/secrets/cipher'))
|
|
|
|
|
|
logger = logging.getLogger()
|
|
logger.setLevel(logging.INFO)
|
|
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
|
sh = logging.StreamHandler()
|
|
sh.setFormatter(formatter)
|
|
logger.addHandler(sh)
|
|
|
|
|
|
|
|
def pki_config(key, fqdn):
|
|
machine_name = fqdn.split('.')[0]
|
|
raw = subprocess.check_output([
|
|
'nix', 'eval', '--raw',
|
|
'( ((import ' + local_root + '/cluster/nix/defs-cluster-k0.nix ) "' + machine_name + '").pki.' + key + '.json )',
|
|
])
|
|
return json.loads(raw)
|
|
|
|
|
|
def _file_exists(c, filename):
|
|
res = c.run('stat "{}"'.format(filename), warn=True, hide=True)
|
|
return res.exited == 0
|
|
|
|
|
|
def configure_k8s(username, ca, cert, key):
|
|
subprocess.check_call([
|
|
'kubectl', 'config',
|
|
'set-cluster', 'admin.' + cluster,
|
|
'--certificate-authority=' + ca,
|
|
'--embed-certs=true',
|
|
'--server=https://' + cluster + ':4001',
|
|
])
|
|
subprocess.check_call([
|
|
'kubectl', 'config',
|
|
'set-credentials', username,
|
|
'--client-certificate=' + cert,
|
|
'--client-key=' + key,
|
|
'--embed-certs=true',
|
|
])
|
|
subprocess.check_call([
|
|
'kubectl', 'config',
|
|
'set-context', 'admin.' + cluster,
|
|
'--cluster=' + 'admin.' + cluster,
|
|
'--user=' + username,
|
|
])
|
|
subprocess.check_call([
|
|
'kubectl', 'config',
|
|
'use-context', 'admin.' + cluster,
|
|
])
|
|
|
|
|
|
def admincreds(args):
|
|
if len(args) != 1:
|
|
sys.stderr.write("Usage: admincreds q3k\n")
|
|
return 1
|
|
username = args[0]
|
|
print("")
|
|
print("WARNING WARNING WARNING WARNING WARNING WARNING")
|
|
print("===============================================")
|
|
print("")
|
|
print("You are requesting ADMIN credentials.")
|
|
print("")
|
|
print("You likely shouldn't be doing this, and")
|
|
print("instead should be using `prodaccess`.")
|
|
print("")
|
|
print("===============================================")
|
|
print("WARNING WARNING WARNING WARNING WARNING WARNING")
|
|
print("")
|
|
|
|
## Make kube certificates.
|
|
certs_root = os.path.join(local_root, 'cluster/certs')
|
|
ca_kube = ca.CA(ss, certs_root, 'kube', 'kubernetes main CA')
|
|
|
|
local_key = os.path.join(local_root, '.kubectl/admin.key')
|
|
local_crt = os.path.join(local_root, '.kubectl/admin.crt')
|
|
|
|
kubectl = os.path.join(local_root, '.kubectl')
|
|
if not os.path.exists(kubectl):
|
|
os.mkdir(kubectl)
|
|
|
|
generate_cert = False
|
|
if not os.path.exists(local_key):
|
|
generate_cert = True
|
|
|
|
if os.path.exists(local_crt):
|
|
with open(local_crt, 'rb') as f:
|
|
b = f.read()
|
|
cert = x509.load_pem_x509_certificate(b, default_backend())
|
|
delta = cert.not_valid_after - datetime.datetime.now()
|
|
logger.info("admin: existing cert expiry: {}".format(delta))
|
|
if delta.total_seconds() < 3600 * 24:
|
|
logger.info("admin: expires soon, regenerating")
|
|
generate_cert = True
|
|
else:
|
|
generate_cert = True
|
|
|
|
if not generate_cert:
|
|
return configure_k8s(username, ca_kube._cert, local_crt, local_key)
|
|
|
|
key, csr = ca_kube.gen_key(hosts=['admin', username], o='system:masters', ou='Kube Admin Account')
|
|
crt = ca_kube.sign(csr)
|
|
|
|
with open(local_key, 'w') as f:
|
|
f.write(key)
|
|
|
|
with open(local_crt, 'w') as f:
|
|
f.write(crt)
|
|
|
|
configure_k8s(username, ca_kube._cert, local_crt, local_key)
|
|
|
|
|
|
def nodestrap(args, nocerts=False):
|
|
if len(args) != 1:
|
|
sys.stderr.write("Usage: nodestrap bc01n01.hswaw.net\n")
|
|
return 1
|
|
fqdn = args[0]
|
|
|
|
logger.info("Nodestrapping {}...".format(fqdn))
|
|
r = fabric.Connection('root@{}'.format(fqdn))
|
|
|
|
if not nocerts:
|
|
cfg = dict((k, pki_config(k, fqdn)) for k in [
|
|
'etcdPeer', 'etcd.server', 'etcd.kube'
|
|
])
|
|
certs_root = os.path.join(local_root, 'cluster/certs')
|
|
|
|
# Make etcd peer certificate for node.
|
|
ca_etcd_peer = ca.CA(ss, certs_root, 'etcdpeer', 'etcd peer ca')
|
|
ca_etcd_peer.make_cert('etcdpeer-{}'.format(fqdn), hosts=[fqdn], ou='node etcd peer certificate')
|
|
|
|
# Make etcd server certificate for node and client certificate for kube.
|
|
ca_etcd = ca.CA(ss, certs_root, 'etcd', 'etcd ca')
|
|
|
|
ca_etcd.make_cert('etcd-{}'.format(fqdn), hosts=[fqdn], ou='node etcd server certificate')
|
|
|
|
ca_etcd.make_cert('etcd-kube', hosts=['kube'], ou='kube etcd client certificate')
|
|
|
|
ca_etcd.make_cert('etcd-root', hosts=['root'], ou='root etcd client certificate')
|
|
|
|
ca_etcd.make_cert('etcd-calico', hosts=['calico'], ou='root etcd client certificate')
|
|
|
|
## Make kube certificates.
|
|
ca_kube = ca.CA(ss, certs_root, 'kube', 'kubernetes main CA')
|
|
|
|
# Make prodvider intermediate CA.
|
|
ca_kube.make_cert('ca-kube-prodvider', o='Warsaw Hackerspace', ou='kubernetes prodvider intermediate', hosts=['kubernetes prodvider intermediate CA'], profile='intermediate').ensure()
|
|
|
|
# Make kubelet certificate (per node).
|
|
ca_kube.make_cert('kube-kubelet-'+fqdn, o='system:nodes', ou='Kubelet', hosts=['system:node:'+fqdn, fqdn])
|
|
|
|
# Make apiserver certificate.
|
|
ca_kube.make_cert('kube-apiserver', ou='Kubernetes API', hosts=[cluster, 'kubernetes.default.svc.'+cluster, '10.10.12.1'])
|
|
|
|
# Make service accounts decryption key (as cert for consistency).
|
|
ca_kube.make_cert('kube-serviceaccounts', ou='Kubernetes Service Accounts Signer', hosts=['serviceaccounts'])
|
|
|
|
# Make kube component certificates.
|
|
kube_components = ['controllermanager', 'scheduler', 'proxy']
|
|
cfg = dict((k, pki_config('kube.' + k, fqdn)) for k in kube_components)
|
|
for k in kube_components:
|
|
# meh
|
|
if k == 'controllermanager':
|
|
o = 'system:kube-controller-manager'
|
|
else:
|
|
o = 'system:kube-'+k
|
|
ou = 'Kubernetes Component '+k
|
|
c = ca_kube.make_cert('kube-'+k, ou=ou, o=o, hosts=[o,])
|
|
|
|
## Make kubefront certificates.
|
|
ca_kubefront = ca.CA(ss, certs_root, 'kubefront', 'kubernetes frontend CA')
|
|
ca_kubefront.make_cert('kubefront-apiserver', ou='Kubernetes Frontend', hosts=['apiserver'])
|
|
|
|
## Make admitomatic (admission controller) certificates.
|
|
ca_admitomatic = ca.CA(ss, certs_root, 'admitomatic', 'admitomatic webhook CA')
|
|
ca_admitomatic.make_cert('admitomatic-webhook', ou='Admitomatic Webhook', hosts=['admitomatic.admitomatic.svc'])
|
|
|
|
subprocess.check_call(["nix", "run",
|
|
"-f", local_root,
|
|
"cluster.nix.provision",
|
|
"-c", "provision-{}".format(fqdn.split('.')[0])])
|
|
|
|
|
|
def usage():
|
|
sys.stderr.write("Usage: clustercfg <nodestrap|admincreds>\n")
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 2:
|
|
usage()
|
|
return 1
|
|
|
|
mode = sys.argv[1]
|
|
if mode == "nodestrap":
|
|
return nodestrap(sys.argv[2:])
|
|
elif mode == "nodestrap-nocerts":
|
|
return nodestrap(sys.argv[2:], nocerts=True)
|
|
elif mode == "admincreds":
|
|
return admincreds(sys.argv[2:])
|
|
elif mode == "smoketest":
|
|
sys.stdout.write("Smoke test passed.")
|
|
return 0
|
|
else:
|
|
usage()
|
|
return 1
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main() or 0)
|