From 56ff18c486afbd56ebbfe337f8613932b0aefbc9 Mon Sep 17 00:00:00 2001 From: Bartosz Stebel Date: Mon, 30 Aug 2021 23:28:45 +0200 Subject: [PATCH] nix: upgrade readTree Change-Id: I460800dc3d8095e2ae89b8bd6ed7c5f0c90b6ccf --- default.nix | 25 +++------ nix/readtree.nix | 96 -------------------------------- nix/readtree/LICENSE | 22 ++++++++ nix/readtree/README.md | 84 ++++++++++++++++++++++++++++ nix/readtree/default.nix | 116 +++++++++++++++++++++++++++++++++++++++ shell.nix | 2 +- 6 files changed, 231 insertions(+), 114 deletions(-) delete mode 100644 nix/readtree.nix create mode 100644 nix/readtree/LICENSE create mode 100644 nix/readtree/README.md create mode 100644 nix/readtree/default.nix diff --git a/default.nix b/default.nix index 6d9c0c4f..f06a4d3b 100644 --- a/default.nix +++ b/default.nix @@ -5,7 +5,7 @@ with builtins; let fix = f: let x = f x; in x; - readTree = import ./nix/readtree.nix {}; + readTree = import ./nix/readtree {}; # Tracking nixos-unstable as of 2021-08-11. nixpkgsCommit = "e26c0ffdb013cd378fc2528a44689a8bf35d2a6c"; @@ -18,21 +18,12 @@ let config.allowBroken = true; }; -in fix (self: rec { - config = { - hscloud = self // { - root = ./.; - }; - pkgs = nixpkgs; - pkgsSrc = nixpkgsSrc; - - inherit (nixpkgs) lib stdenv; - }; - - bgpwtf = readTree config ./bgpwtf; - cluster = readTree config ./cluster; - hswaw = readTree config ./hswaw; - ops = readTree config ./ops; - +in fix (self: (readTree rec { + hscloud = self; + pkgs = nixpkgs; + pkgsSrc = nixpkgsSrc; + inherit (nixpkgs) lib stdenv; +} ./.) // { + root = ./.; pkgs = nixpkgs; }) diff --git a/nix/readtree.nix b/nix/readtree.nix deleted file mode 100644 index 066d3268..00000000 --- a/nix/readtree.nix +++ /dev/null @@ -1,96 +0,0 @@ -# The MIT License (MIT) -# -# Copyright (c) 2019 Vincent Ambo -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -{ ... }: - -args: initPath: - -let - inherit (builtins) - attrNames - baseNameOf - filter - hasAttr - head - isAttrs - length - listToAttrs - map - match - readDir - substring; - - argsWithPath = parts: - let meta.locatedAt = parts; - in meta // (if isAttrs args then args else args meta); - - readDirVisible = path: - let - children = readDir path; - isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != "."; - names = filter isVisible (attrNames children); - in listToAttrs (map (name: { - inherit name; - value = children.${name}; - }) names); - - # The marker is added to every set that was imported directly by - # readTree. - importWithMark = path: parts: - let imported = import path (argsWithPath parts); - in if (isAttrs imported) - then imported // { __readTree = true; } - else imported; - - nixFileName = file: - let res = match "(.*)\.nix" file; - in if res == null then null else head res; - - readTree = path: parts: - let - dir = readDirVisible path; - self = importWithMark path parts; - joinChild = c: path + ("/" + c); - - # Import subdirectories of the current one, unless the special - # `.skip-subtree` file exists which makes readTree ignore the - # children. - # - # This file can optionally contain information on why the tree - # should be ignored, but its content is not inspected by - # readTree - filterDir = f: dir."${f}" == "directory"; - children = if hasAttr ".skip-subtree" dir then [] else map (c: { - name = c; - value = readTree (joinChild c) (parts ++ [ c ]); - }) (filter filterDir (attrNames dir)); - - # Import Nix files - nixFiles = filter (f: f != null) (map nixFileName (attrNames dir)); - nixChildren = map (c: let p = joinChild (c + ".nix"); in { - name = c; - value = importWithMark p (parts ++ [ c ]); - }) nixFiles; - in if dir ? "default.nix" - then (if isAttrs self then self // (listToAttrs children) else self) - else listToAttrs (nixChildren ++ children); -in readTree initPath [ (baseNameOf initPath) ] diff --git a/nix/readtree/LICENSE b/nix/readtree/LICENSE new file mode 100644 index 00000000..bdc72a2e --- /dev/null +++ b/nix/readtree/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 Vincent Ambo +Copyright (c) 2020-2021 The TVL Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/nix/readtree/README.md b/nix/readtree/README.md new file mode 100644 index 00000000..138abbe3 --- /dev/null +++ b/nix/readtree/README.md @@ -0,0 +1,84 @@ +readTree +======== + +This is a Nix program that builds up an attribute set tree for a large +repository based on the filesystem layout. + +It is in fact the tool that lays out the attribute set of this repository. + +As an example, consider a root (`.`) of a repository and a layout such as: + +``` +. +├── third_party +│   ├── default.nix +│   └── rustpkgs +│   ├── aho-corasick.nix +│   └── serde.nix +└── tools + ├── cheddar + │   └── default.nix + └── roquefort.nix +``` + +When `readTree` is called on that tree, it will construct an attribute set with +this shape: + +```nix +{ + tools = { + cheddar = ...; + roquefort = ...; + }; + + third_party = { + # the `default.nix` of this folder might have had arbitrary other + # attributes here, such as this: + favouriteColour = "orange"; + + rustpkgs = { + aho-corasick = ...; + serde = ...; + }; + }; +} +``` + +Every imported Nix file that yields an attribute set will have a `__readTree = +true;` attribute merged into it. + +## Traversal logic + +`readTree` will follow any subdirectories of a tree and import all Nix files, +with some exceptions: + +* A folder can declare that its children are off-limit by containing a + `.skip-subtree` file. Since the content of the file is not checked, it can be + useful to leave a note for a human in the file. +* If a folder contains a `default.nix` file, no *sibling* Nix files will be + imported - however children are traversed as normal. +* If a folder contains a `default.nix` it is loaded and, if it evaluates to a + set, *merged* with the children. If it evaluates to anything else the children + are *not traversed*. +* The `default.nix` of the top-level folder on which readTree is + called is **not** read to avoid infinite recursion (as, presumably, + this file is where readTree itself is called). + +Traversal is lazy, `readTree` will only build up the tree as requested. This +currently has the downside that directories with no importable files end up in +the tree as empty nodes (`{}`). + +## Import structure + +`readTree` is called with two parameters: The arguments to pass to all imports, +and the initial path at which to start the traversal. + +The package headers in this repository follow the form `{ pkgs, ... }:` where +`pkgs` is a fixed-point of the entire package tree (see the `default.nix` at the +root of the depot). + +In theory `readTree` can pass arguments of different shapes, but I have found +this to be a good solution for the most part. + +Note that `readTree` does not currently make functions overridable, though it is +feasible that it could do that in the future. diff --git a/nix/readtree/default.nix b/nix/readtree/default.nix new file mode 100644 index 00000000..633915f7 --- /dev/null +++ b/nix/readtree/default.nix @@ -0,0 +1,116 @@ +# Copyright (c) 2019 Vincent Ambo +# Copyright (c) 2020-2021 The TVL Authors +# SPDX-License-Identifier: MIT +# +# Provides a function to automatically read a a filesystem structure +# into a Nix attribute set. +# +# Optionally accepts an argument `argsFilter` on import, which is a +# function that receives the current tree location (as a list of +# strings) and the argument set and can arbitrarily modify it. +{ argsFilter ? (x: _parts: x) +, ... }: + +let + inherit (builtins) + attrNames + baseNameOf + concatStringsSep + filter + hasAttr + head + isAttrs + length + listToAttrs + map + match + readDir + substring; + + assertMsg = pred: msg: + if pred + then true + else builtins.trace msg false; + + argsWithPath = args: parts: + let meta.locatedAt = parts; + in meta // (if isAttrs args then args else args meta); + + readDirVisible = path: + let + children = readDir path; + isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != "."; + names = filter isVisible (attrNames children); + in listToAttrs (map (name: { + inherit name; + value = children.${name}; + }) names); + + # Create a mark containing the location of this attribute. + marker = parts: { + __readTree = parts; + }; + + # The marker is added to every set that was imported directly by + # readTree. + importWithMark = args: path: parts: + let + importedFile = import path; + pathType = builtins.typeOf importedFile; + imported = + assert assertMsg + (pathType == "lambda") + "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }"; + importedFile (argsFilter (argsWithPath args parts) parts); + in if (isAttrs imported) + then imported // (marker parts) + else imported; + + nixFileName = file: + let res = match "(.*)\\.nix" file; + in if res == null then null else head res; + + readTree = { args, initPath, rootDir, parts }: + let + dir = readDirVisible initPath; + joinChild = c: initPath + ("/" + c); + + self = if rootDir + then { __readTree = []; } + else importWithMark args initPath parts; + + # Import subdirectories of the current one, unless the special + # `.skip-subtree` file exists which makes readTree ignore the + # children. + # + # This file can optionally contain information on why the tree + # should be ignored, but its content is not inspected by + # readTree + filterDir = f: dir."${f}" == "directory"; + children = if hasAttr ".skip-subtree" dir then [] else map (c: { + name = c; + value = readTree { + args = args; + initPath = (joinChild c); + rootDir = false; + parts = (parts ++ [ c ]); + }; + }) (filter filterDir (attrNames dir)); + + # Import Nix files + nixFiles = filter (f: f != null) (map nixFileName (attrNames dir)); + nixChildren = map (c: let p = joinChild (c + ".nix"); in { + name = c; + value = importWithMark args p (parts ++ [ c ]); + }) nixFiles; + in if dir ? "default.nix" + then (if isAttrs self then self // (listToAttrs children) else self) + else (listToAttrs (nixChildren ++ children) // (marker parts)); + +in { + __functor = _: args: initPath: readTree { + inherit args initPath; + rootDir = true; + parts = []; + }; +} diff --git a/shell.nix b/shell.nix index de3db3c5..04b85012 100644 --- a/shell.nix +++ b/shell.nix @@ -4,7 +4,7 @@ let hscloud = import ./default.nix {}; -in with hscloud.config.pkgs; let +in with hscloud.pkgs; let wrapper = pkgs.writeScript "wrapper.sh" ''