From ff8a50cb02c261f12e020e5995d51a5dff7e41ed Mon Sep 17 00:00:00 2001 From: Piotr Dobrowolski Date: Thu, 1 Feb 2024 23:28:14 +0100 Subject: [PATCH] ops: colmena integration Change-Id: I18b9218f2c29a84f7fa769e1a9f561a4385578ca Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1757 Reviewed-by: q3k --- default.nix | 1 + .../hardware-configuration.nix | 4 +- ops/README.md | 20 +-- ops/hive.nix | 88 +++++++++++ ops/machines.nix | 148 ------------------ ops/provision.nix | 74 --------- shell.nix | 2 + 7 files changed, 98 insertions(+), 239 deletions(-) create mode 100644 ops/hive.nix delete mode 100644 ops/machines.nix delete mode 100644 ops/provision.nix diff --git a/default.nix b/default.nix index 787bdf96..662f132a 100644 --- a/default.nix +++ b/default.nix @@ -36,4 +36,5 @@ let in (resForPkgs nixpkgs) // { root = ./.; pkgs = nixpkgs; + hscloudForPkgs = pkgs: resForPkgs pkgs; } diff --git a/hswaw/machines/customs.hackerspace.pl/hardware-configuration.nix b/hswaw/machines/customs.hackerspace.pl/hardware-configuration.nix index 1ba16f7a..84b5d599 100644 --- a/hswaw/machines/customs.hackerspace.pl/hardware-configuration.nix +++ b/hswaw/machines/customs.hackerspace.pl/hardware-configuration.nix @@ -1,11 +1,11 @@ # Do not modify this file! It was generated by ‘nixos-generate-config’ # and may be overwritten by future invocations. Please make changes # to /etc/nixos/configuration.nix instead. -{ config, lib, pkgs, ... }: +{ config, lib, pkgs, modulesPath, ... }: { imports = - [ + [ "${modulesPath}/installer/scan/not-detected.nix" ]; boot.initrd.availableKernelModules = [ "uhci_hcd" "ehci_pci" "ata_piix" "ahci" "usb_storage" "sd_mod" ]; diff --git a/ops/README.md b/ops/README.md index 28dcb144..a8ea64d1 100644 --- a/ops/README.md +++ b/ops/README.md @@ -4,23 +4,13 @@ Operations Deploying NixOS machines --- -Machine configurations are in `ops/machines.nix`. +Machine configurations are in `ops/hive.nix` and are managed with [colmena](https://colmena.cli.rs/unstable/introduction.html). -Wrapper script to show all available machines and provision a single machine: + $ colmena -f ops/hive.nix eval -E '{ nodes, lib, ... }: lib.attrNames nodes' + [INFO ] Using configuration: .../hscloud/ops/hive.nix + ["arcade.waw.hackerspace.pl","bc01n01.hswaw.net","bc01n02.hswaw.net","bc01n05.hswaw.net","customs.hackerspace.pl","dcr01s22.hswaw.net","dcr01s24.hswaw.net","dcr03s16.hswaw.net","edge01.waw.bgp.wtf","larrythebuilder.q3k.org","sound.waw.hackerspace.pl","tv1.waw.hackerspace.pl","tv2.waw.hackerspace.pl"] - $ $(nix-build -A ops.provision) - Available machines: - - bc01n01.hswaw.net - - bc01n02.hswaw.net - - dcr01s22.hswaw.net - - dcr01s24.hswaw.net - - edge01.waw.bgp.wtf - - $ $(nix-build -A ops.provision) edge01.waw.bgp.wtf - -This can be slow, as it evaluates/builds all machines' configs. If you just want to deploy one machine and possible iterate faster: - - $ $(nix-build -A 'ops.machines."edge01.waw.bgp.wtf".config.passthru.hscloud.provision') + $ colmena -f ops/hive.nix apply --on edge01.waw.bgp.wtf Remote Builders (cross-compiling) --- diff --git a/ops/hive.nix b/ops/hive.nix new file mode 100644 index 00000000..3d7eb70d --- /dev/null +++ b/ops/hive.nix @@ -0,0 +1,88 @@ +# Top-level file aggregating all machines managed from hscloud. +# +# This file is meant to be used with colmena. For information about +# building/deploying machines see //ops/README.md. + +let + hscloud = import ../default.nix { }; + pkgs = hscloud.pkgs; + + # TODO(patryk): unpin and upgrade + nixpkgsMachines = import + (pkgs.fetchFromGitHub { + owner = "nixos"; + repo = "nixpkgs-channels"; + rev = "e26c0ffdb013cd378fc2528a44689a8bf35d2a6c"; + sha256 = "1b33hw35fqb9rzszdg5jpiyfvhx2cxpv0qrkyr19zkdpdahzdbss"; + }) + { }; + + mkClusterMachine = path: { + deployment.tags = [ "k8s" ]; + + imports = [ + ../cluster/machines/modules/base.nix + ../cluster/machines/modules/kube-controlplane.nix + ../cluster/machines/modules/kube-dataplane.nix + path + ]; + }; +in +{ + meta = { + nixpkgs = pkgs; + + nodeNixpkgs = { + "bc01n01.hswaw.net" = nixpkgsMachines; + "bc01n05.hswaw.net" = nixpkgsMachines; + "dcr01s22.hswaw.net" = nixpkgsMachines; + "dcr01s24.hswaw.net" = nixpkgsMachines; + "dcr03s16.hswaw.net" = nixpkgsMachines; + + "edge01.waw.bgp.wtf" = nixpkgsMachines; + + "larrythebuilder.q3k.org" = import pkgs.path { system = "aarch64-linux"; }; + "tv1.waw.hackerspace.pl" = import pkgs.path { system = "aarch64-linux"; }; + "tv2.waw.hackerspace.pl" = import pkgs.path { system = "aarch64-linux"; }; + + # TODO update global pkgs to >= 22.05 and remove this override + # building on current pkgs gives error: + # error: The option `services.home-assistant.extraComponents' does not exist. + "sound.waw.hackerspace.pl" = import + (fetchTarball { + # NixOS/nixpkgs/nixos-unstable 2022-09-10 + url = "https://api.github.com/repos/NixOS/nixpkgs/tarball/2da64a81275b68fdad38af669afeda43d401e94b"; + sha256 = "1k71lmzdaa48yqkmsnd22n177qmxxi4gj2qcmdbv0mc6l4f27wd0"; + }) + { }; + }; + + allowApplyAll = false; + }; + + defaults = { nodes, pkgs, ... }: { + _module.args.workspace = hscloud.hscloudForPkgs pkgs; + _module.args.machines = nodes; + }; + + "bc01n01.hswaw.net" = mkClusterMachine ../cluster/machines/bc01n01.hswaw.net.nix; + "bc01n05.hswaw.net" = mkClusterMachine ../cluster/machines/bc01n05.hswaw.net.nix; + "dcr01s22.hswaw.net" = mkClusterMachine ../cluster/machines/dcr01s22.hswaw.net.nix; + "dcr01s24.hswaw.net" = mkClusterMachine ../cluster/machines/dcr01s24.hswaw.net.nix; + "dcr03s16.hswaw.net" = mkClusterMachine ../cluster/machines/dcr03s16.hswaw.net.nix; + + "edge01.waw.bgp.wtf" = { ... }: { + imports = [ + ../bgpwtf/machines/edge01.waw.bgp.wtf.nix + ../bgpwtf/machines/edge01.waw.bgp.wtf-hardware.nix + ]; + }; + + "larrythebuilder.q3k.org" = import ../hswaw/machines/larrythebuilder.q3k.org/configuration.nix; + + "customs.hackerspace.pl" = import ../hswaw/machines/customs.hackerspace.pl/configuration.nix; + "tv1.waw.hackerspace.pl" = import ../hswaw/machines/tv/tv1.nix; + "tv2.waw.hackerspace.pl" = import ../hswaw/machines/tv/tv2.nix; + "sound.waw.hackerspace.pl" = import ../hswaw/machines/sound.waw.hackerspace.pl/configuration.nix; + "arcade.waw.hackerspace.pl" = import ../hswaw/machines/arcade.waw.hackerspace.pl/configuration.nix; +} diff --git a/ops/machines.nix b/ops/machines.nix deleted file mode 100644 index 6e8260ec..00000000 --- a/ops/machines.nix +++ /dev/null @@ -1,148 +0,0 @@ -# Top-level file aggregating all machines managed from hscloud. -# -# This allows to have a common attrset of machines that can be deployed -# in the same way. -# -# For information about building/deploying machines see //ops/README.md. - -{ hscloud, pkgs, hscloudForPkgs, ... }: - -let - # nixpkgs for cluster machines (.hswaw.net). Pinned to an old nixpkgs - # for stability and controlled k8s upgrades. - - nixpkgsMachines = import (pkgs.fetchFromGitHub { - owner = "nixos"; - repo = "nixpkgs"; - rev = "e26c0ffdb013cd378fc2528a44689a8bf35d2a6c"; # 21.11 - sha256 = "1b33hw35fqb9rzszdg5jpiyfvhx2cxpv0qrkyr19zkdpdahzdbss"; - }) { }; - # fixture for convenient upgrades (just import different nixpkgs here) - nixpkgsMachinesNew = nixpkgsMachines; - - - # mkMachine builds NixOS modules into a NixOS derivation. - # It: - # 1) injects passthru.hscloud.provision which deploys that configuration - # over SSH to a production machine. - # 2) injects 'workspace' as a nixos module argument which points to the root - # of the hscloud readTree object. It will contain whatever nixpkgs - # checkout this file has been invoked with, ie. will not be 'mixed in' - # with the pkgs argument. - mkMachine = machines: pkgs: paths: pkgs.nixos ({ config, pkgs, ... }: { - imports = paths; - - config = let - name = config.networking.hostName; - domain = if (config.networking ? domain) && config.networking.domain != null then config.networking.domain else "hswaw.net"; - fqdn = name + "." + domain; - toplevel = config.system.build.toplevel; - - runProvision = '' - #!/bin/sh - set -eu - remote=root@${fqdn} - echo "Configuration for ${fqdn} is ${toplevel}" - nix copy -s --to ssh://$remote ${toplevel} - - running="$(ssh $remote readlink -f /nix/var/nix/profiles/system)" - if [ "$running" == "${toplevel}" ]; then - echo "${fqdn} already running ${toplevel}." - else - echo "/etc/systemd/system diff:" - ssh $remote diff -ur /var/run/current-system/etc/systemd/system ${toplevel}/etc/systemd/system || true - echo "" - echo "" - echo "dry-activate diff:" - ssh $remote ${toplevel}/bin/switch-to-configuration dry-activate - read -p "Do you want to switch to this configuration? " -n 1 -r - echo - if ! [[ $REPLY =~ ^[Yy]$ ]]; then - exit 1 - fi - - echo -ne "\n\nswitch-to-configuration test...\n" - ssh $remote ${toplevel}/bin/switch-to-configuration test - fi - - echo -ne "\n\n" - read -p "Do you want to set this configuration as boot? " -n 1 -r - echo - if ! [[ $REPLY =~ ^[Yy]$ ]]; then - exit 1 - fi - - echo -ne "\n\nsetting system profile...\n" - ssh $remote nix-env -p /nix/var/nix/profiles/system --set ${toplevel} - - echo -ne "\n\nswitch-to-configuration boot...\n" - ssh $remote ${toplevel}/bin/switch-to-configuration boot - ''; - in { - passthru.hscloud.provision = pkgs.writeScript "provision-${fqdn}" runProvision; - - # TODO(q3k): this should be named hscloud, but that seems to not work. Debug and rename. - _module.args.workspace = hscloudForPkgs pkgs; - _module.args.machines = machines; - }; - }); - - mkClusterMachine = machines: path: mkMachine machines nixpkgsMachines [ - ../cluster/machines/modules/base.nix - ../cluster/machines/modules/kube-controlplane.nix - ../cluster/machines/modules/kube-dataplane.nix - path - ]; - - mkClusterMachineNew = machines: path: mkMachine machines nixpkgsMachinesNew [ - ../cluster/machines/modules/base.nix - ../cluster/machines/modules/kube-controlplane.nix - ../cluster/machines/modules/kube-dataplane.nix - path - ]; - - - pkgsArm = import pkgs.path { - system = "aarch64-linux"; - }; - - machines = self: { - "bc01n01.hswaw.net" = mkClusterMachine self ../cluster/machines/bc01n01.hswaw.net.nix; - "bc01n05.hswaw.net" = mkClusterMachine self ../cluster/machines/bc01n05.hswaw.net.nix; - "dcr01s22.hswaw.net" = mkClusterMachine self ../cluster/machines/dcr01s22.hswaw.net.nix; - "dcr01s24.hswaw.net" = mkClusterMachine self ../cluster/machines/dcr01s24.hswaw.net.nix; - "dcr03s16.hswaw.net" = mkClusterMachine self ../cluster/machines/dcr03s16.hswaw.net.nix; - - "edge01.waw.bgp.wtf" = mkMachine self nixpkgsMachines [ - ../bgpwtf/machines/edge01.waw.bgp.wtf.nix - ../bgpwtf/machines/edge01.waw.bgp.wtf-hardware.nix - ]; - - "larrythebuilder.q3k.org" = mkMachine self pkgsArm [ - ../hswaw/machines/larrythebuilder.q3k.org/configuration.nix - ]; - - "customs.hackerspace.pl" = mkMachine self pkgs [ - ../hswaw/machines/customs.hackerspace.pl/configuration.nix - ]; - "tv1.waw.hackerspace.pl" = mkMachine self pkgsArm [ - ../hswaw/machines/tv/tv1.nix - ]; - "tv2.waw.hackerspace.pl" = mkMachine self pkgsArm [ - ../hswaw/machines/tv/tv2.nix - ]; - "sound.waw.hackerspace.pl" = let - # TODO update global pkgs to >= 22.05 and remove this override - # building on current pkgs gives error: - # error: The option `services.home-assistant.extraComponents' does not exist. - pkgs = import (fetchTarball { - # NixOS/nixpkgs/nixos-unstable 2022-09-10 - url = "https://api.github.com/repos/NixOS/nixpkgs/tarball/2da64a81275b68fdad38af669afeda43d401e94b"; - sha256 = "1k71lmzdaa48yqkmsnd22n177qmxxi4gj2qcmdbv0mc6l4f27wd0"; - }) {}; - in mkMachine self pkgs [ - ../hswaw/machines/sound.waw.hackerspace.pl/configuration.nix - ]; - }; - -in pkgs.lib.fix machines diff --git a/ops/provision.nix b/ops/provision.nix deleted file mode 100644 index 76054c42..00000000 --- a/ops/provision.nix +++ /dev/null @@ -1,74 +0,0 @@ -# Top-level wrapper script for calling per-machine provisioners. -# -# Given ops.machines."edge01.waw.bgp.wtf".config.passthru.hscloud.provision, -# this script allows to run it by doing: -# $ $(nix-build -A ops.provision) edge01.waw.bgp.wtf -# Or, to first list all available machines by doing: -# $ $(nix-build -A ops.provision) -# -# The main logic of the provisioner script is in machines.nix. - -{ hscloud, pkgs, lib, ... }: - -with lib; with builtins; - -let - - # All machines from ops.machines, keyed by FQDN. - machines = filterAttrs (n: _: n != "__readTree") hscloud.ops.machines; - # Machines' provisioner scripts, keyed by machine FQDN. - machineProvisioners = mapAttrs (_: v: v.config.passthru.hscloud.provision) machines; - # List of machine FQDNs. - machineNames = attrNames machines; - - # User-friendly list of machines by FQDN. - machineList = concatStringsSep "\n" - (map - (name: " - ${name}") - machineNames); - - # Derivation containing bin/provision-FQDN symlinks to machines' provisioners. - forest = pkgs.linkFarm "provision-forest" - (mapAttrsToList - (fqdn: p: { name = "bin/provision-${fqdn}"; path = p; }) - machineProvisioners); -in - -pkgs.writeScript "provision" '' - #!/bin/sh - name="$1" - - usage() { - echo >&2 "Usage: $0 machine|machine.hswaw.net" - echo >&2 "Available machines:" - echo >&2 "${machineList}" - } - - if [ -z "$name" ]; then - usage - exit 1 - fi - - provisioner="${forest}/bin/provision-$name" - if [ ! -e "$provisioner" ]; then - name="$name.hswaw.net" - provisioner="${forest}/bin/provision-$name" - fi - if [ ! -e "$provisioner" ]; then - usage - exit 1 - fi - # :^) - echo -ne "\e[34mh \e[31ms \e[33mc l \e[34mo \e[32mu \e[31md \e[0m" - echo "" - echo "Starting provisioner for $name..." - echo "" - echo "Too slow to evaluate? Equivalent faster command line that rebuilds just one node:" - echo " \$(nix-build -A 'ops.machines.\"$name\".config.passthru.hscloud.provision')" - echo "" - echo "Or, if you want to deploy the same configuration on different machines, just run" - echo "this script again without re-evaluating nix:" - echo " $0 $name" - echo "" - exec "$provisioner" -'' diff --git a/shell.nix b/shell.nix index 1ef93010..5d155c6c 100644 --- a/shell.nix +++ b/shell.nix @@ -67,6 +67,8 @@ in (pkgs.buildFHSUserEnv { openssh libxcrypt zlib + + colmena ]; multiPkgs = pkgs: [ (pkgs.runCommand "protocols" {}