forked from hswaw/hscloud
nix: upgrade readTree
Change-Id: I460800dc3d8095e2ae89b8bd6ed7c5f0c90b6ccf
This commit is contained in:
parent
eed9afe210
commit
56ff18c486
6 changed files with 231 additions and 114 deletions
25
default.nix
25
default.nix
|
@ -5,7 +5,7 @@ with builtins;
|
||||||
let
|
let
|
||||||
fix = f: let x = f x; in x;
|
fix = f: let x = f x; in x;
|
||||||
|
|
||||||
readTree = import ./nix/readtree.nix {};
|
readTree = import ./nix/readtree {};
|
||||||
|
|
||||||
# Tracking nixos-unstable as of 2021-08-11.
|
# Tracking nixos-unstable as of 2021-08-11.
|
||||||
nixpkgsCommit = "e26c0ffdb013cd378fc2528a44689a8bf35d2a6c";
|
nixpkgsCommit = "e26c0ffdb013cd378fc2528a44689a8bf35d2a6c";
|
||||||
|
@ -18,21 +18,12 @@ let
|
||||||
config.allowBroken = true;
|
config.allowBroken = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
in fix (self: rec {
|
in fix (self: (readTree rec {
|
||||||
config = {
|
hscloud = self;
|
||||||
hscloud = self // {
|
pkgs = nixpkgs;
|
||||||
root = ./.;
|
pkgsSrc = nixpkgsSrc;
|
||||||
};
|
inherit (nixpkgs) lib stdenv;
|
||||||
pkgs = nixpkgs;
|
} ./.) // {
|
||||||
pkgsSrc = nixpkgsSrc;
|
root = ./.;
|
||||||
|
|
||||||
inherit (nixpkgs) lib stdenv;
|
|
||||||
};
|
|
||||||
|
|
||||||
bgpwtf = readTree config ./bgpwtf;
|
|
||||||
cluster = readTree config ./cluster;
|
|
||||||
hswaw = readTree config ./hswaw;
|
|
||||||
ops = readTree config ./ops;
|
|
||||||
|
|
||||||
pkgs = nixpkgs;
|
pkgs = nixpkgs;
|
||||||
})
|
})
|
||||||
|
|
|
@ -1,96 +0,0 @@
|
||||||
# The MIT License (MIT)
|
|
||||||
#
|
|
||||||
# Copyright (c) 2019 Vincent Ambo
|
|
||||||
#
|
|
||||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
# of this software and associated documentation files (the "Software"), to deal
|
|
||||||
# in the Software without restriction, including without limitation the rights
|
|
||||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
# copies of the Software, and to permit persons to whom the Software is
|
|
||||||
# furnished to do so, subject to the following conditions:
|
|
||||||
#
|
|
||||||
# The above copyright notice and this permission notice shall be included in all
|
|
||||||
# copies or substantial portions of the Software.
|
|
||||||
#
|
|
||||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
# SOFTWARE.
|
|
||||||
|
|
||||||
{ ... }:
|
|
||||||
|
|
||||||
args: initPath:
|
|
||||||
|
|
||||||
let
|
|
||||||
inherit (builtins)
|
|
||||||
attrNames
|
|
||||||
baseNameOf
|
|
||||||
filter
|
|
||||||
hasAttr
|
|
||||||
head
|
|
||||||
isAttrs
|
|
||||||
length
|
|
||||||
listToAttrs
|
|
||||||
map
|
|
||||||
match
|
|
||||||
readDir
|
|
||||||
substring;
|
|
||||||
|
|
||||||
argsWithPath = parts:
|
|
||||||
let meta.locatedAt = parts;
|
|
||||||
in meta // (if isAttrs args then args else args meta);
|
|
||||||
|
|
||||||
readDirVisible = path:
|
|
||||||
let
|
|
||||||
children = readDir path;
|
|
||||||
isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
|
|
||||||
names = filter isVisible (attrNames children);
|
|
||||||
in listToAttrs (map (name: {
|
|
||||||
inherit name;
|
|
||||||
value = children.${name};
|
|
||||||
}) names);
|
|
||||||
|
|
||||||
# The marker is added to every set that was imported directly by
|
|
||||||
# readTree.
|
|
||||||
importWithMark = path: parts:
|
|
||||||
let imported = import path (argsWithPath parts);
|
|
||||||
in if (isAttrs imported)
|
|
||||||
then imported // { __readTree = true; }
|
|
||||||
else imported;
|
|
||||||
|
|
||||||
nixFileName = file:
|
|
||||||
let res = match "(.*)\.nix" file;
|
|
||||||
in if res == null then null else head res;
|
|
||||||
|
|
||||||
readTree = path: parts:
|
|
||||||
let
|
|
||||||
dir = readDirVisible path;
|
|
||||||
self = importWithMark path parts;
|
|
||||||
joinChild = c: path + ("/" + c);
|
|
||||||
|
|
||||||
# Import subdirectories of the current one, unless the special
|
|
||||||
# `.skip-subtree` file exists which makes readTree ignore the
|
|
||||||
# children.
|
|
||||||
#
|
|
||||||
# This file can optionally contain information on why the tree
|
|
||||||
# should be ignored, but its content is not inspected by
|
|
||||||
# readTree
|
|
||||||
filterDir = f: dir."${f}" == "directory";
|
|
||||||
children = if hasAttr ".skip-subtree" dir then [] else map (c: {
|
|
||||||
name = c;
|
|
||||||
value = readTree (joinChild c) (parts ++ [ c ]);
|
|
||||||
}) (filter filterDir (attrNames dir));
|
|
||||||
|
|
||||||
# Import Nix files
|
|
||||||
nixFiles = filter (f: f != null) (map nixFileName (attrNames dir));
|
|
||||||
nixChildren = map (c: let p = joinChild (c + ".nix"); in {
|
|
||||||
name = c;
|
|
||||||
value = importWithMark p (parts ++ [ c ]);
|
|
||||||
}) nixFiles;
|
|
||||||
in if dir ? "default.nix"
|
|
||||||
then (if isAttrs self then self // (listToAttrs children) else self)
|
|
||||||
else listToAttrs (nixChildren ++ children);
|
|
||||||
in readTree initPath [ (baseNameOf initPath) ]
|
|
22
nix/readtree/LICENSE
Normal file
22
nix/readtree/LICENSE
Normal file
|
@ -0,0 +1,22 @@
|
||||||
|
The MIT License (MIT)
|
||||||
|
|
||||||
|
Copyright (c) 2019 Vincent Ambo
|
||||||
|
Copyright (c) 2020-2021 The TVL Authors
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
84
nix/readtree/README.md
Normal file
84
nix/readtree/README.md
Normal file
|
@ -0,0 +1,84 @@
|
||||||
|
readTree
|
||||||
|
========
|
||||||
|
|
||||||
|
This is a Nix program that builds up an attribute set tree for a large
|
||||||
|
repository based on the filesystem layout.
|
||||||
|
|
||||||
|
It is in fact the tool that lays out the attribute set of this repository.
|
||||||
|
|
||||||
|
As an example, consider a root (`.`) of a repository and a layout such as:
|
||||||
|
|
||||||
|
```
|
||||||
|
.
|
||||||
|
├── third_party
|
||||||
|
│ ├── default.nix
|
||||||
|
│ └── rustpkgs
|
||||||
|
│ ├── aho-corasick.nix
|
||||||
|
│ └── serde.nix
|
||||||
|
└── tools
|
||||||
|
├── cheddar
|
||||||
|
│ └── default.nix
|
||||||
|
└── roquefort.nix
|
||||||
|
```
|
||||||
|
|
||||||
|
When `readTree` is called on that tree, it will construct an attribute set with
|
||||||
|
this shape:
|
||||||
|
|
||||||
|
```nix
|
||||||
|
{
|
||||||
|
tools = {
|
||||||
|
cheddar = ...;
|
||||||
|
roquefort = ...;
|
||||||
|
};
|
||||||
|
|
||||||
|
third_party = {
|
||||||
|
# the `default.nix` of this folder might have had arbitrary other
|
||||||
|
# attributes here, such as this:
|
||||||
|
favouriteColour = "orange";
|
||||||
|
|
||||||
|
rustpkgs = {
|
||||||
|
aho-corasick = ...;
|
||||||
|
serde = ...;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Every imported Nix file that yields an attribute set will have a `__readTree =
|
||||||
|
true;` attribute merged into it.
|
||||||
|
|
||||||
|
## Traversal logic
|
||||||
|
|
||||||
|
`readTree` will follow any subdirectories of a tree and import all Nix files,
|
||||||
|
with some exceptions:
|
||||||
|
|
||||||
|
* A folder can declare that its children are off-limit by containing a
|
||||||
|
`.skip-subtree` file. Since the content of the file is not checked, it can be
|
||||||
|
useful to leave a note for a human in the file.
|
||||||
|
* If a folder contains a `default.nix` file, no *sibling* Nix files will be
|
||||||
|
imported - however children are traversed as normal.
|
||||||
|
* If a folder contains a `default.nix` it is loaded and, if it evaluates to a
|
||||||
|
set, *merged* with the children. If it evaluates to anything else the children
|
||||||
|
are *not traversed*.
|
||||||
|
* The `default.nix` of the top-level folder on which readTree is
|
||||||
|
called is **not** read to avoid infinite recursion (as, presumably,
|
||||||
|
this file is where readTree itself is called).
|
||||||
|
|
||||||
|
Traversal is lazy, `readTree` will only build up the tree as requested. This
|
||||||
|
currently has the downside that directories with no importable files end up in
|
||||||
|
the tree as empty nodes (`{}`).
|
||||||
|
|
||||||
|
## Import structure
|
||||||
|
|
||||||
|
`readTree` is called with two parameters: The arguments to pass to all imports,
|
||||||
|
and the initial path at which to start the traversal.
|
||||||
|
|
||||||
|
The package headers in this repository follow the form `{ pkgs, ... }:` where
|
||||||
|
`pkgs` is a fixed-point of the entire package tree (see the `default.nix` at the
|
||||||
|
root of the depot).
|
||||||
|
|
||||||
|
In theory `readTree` can pass arguments of different shapes, but I have found
|
||||||
|
this to be a good solution for the most part.
|
||||||
|
|
||||||
|
Note that `readTree` does not currently make functions overridable, though it is
|
||||||
|
feasible that it could do that in the future.
|
116
nix/readtree/default.nix
Normal file
116
nix/readtree/default.nix
Normal file
|
@ -0,0 +1,116 @@
|
||||||
|
# Copyright (c) 2019 Vincent Ambo
|
||||||
|
# Copyright (c) 2020-2021 The TVL Authors
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
#
|
||||||
|
# Provides a function to automatically read a a filesystem structure
|
||||||
|
# into a Nix attribute set.
|
||||||
|
#
|
||||||
|
# Optionally accepts an argument `argsFilter` on import, which is a
|
||||||
|
# function that receives the current tree location (as a list of
|
||||||
|
# strings) and the argument set and can arbitrarily modify it.
|
||||||
|
{ argsFilter ? (x: _parts: x)
|
||||||
|
, ... }:
|
||||||
|
|
||||||
|
let
|
||||||
|
inherit (builtins)
|
||||||
|
attrNames
|
||||||
|
baseNameOf
|
||||||
|
concatStringsSep
|
||||||
|
filter
|
||||||
|
hasAttr
|
||||||
|
head
|
||||||
|
isAttrs
|
||||||
|
length
|
||||||
|
listToAttrs
|
||||||
|
map
|
||||||
|
match
|
||||||
|
readDir
|
||||||
|
substring;
|
||||||
|
|
||||||
|
assertMsg = pred: msg:
|
||||||
|
if pred
|
||||||
|
then true
|
||||||
|
else builtins.trace msg false;
|
||||||
|
|
||||||
|
argsWithPath = args: parts:
|
||||||
|
let meta.locatedAt = parts;
|
||||||
|
in meta // (if isAttrs args then args else args meta);
|
||||||
|
|
||||||
|
readDirVisible = path:
|
||||||
|
let
|
||||||
|
children = readDir path;
|
||||||
|
isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
|
||||||
|
names = filter isVisible (attrNames children);
|
||||||
|
in listToAttrs (map (name: {
|
||||||
|
inherit name;
|
||||||
|
value = children.${name};
|
||||||
|
}) names);
|
||||||
|
|
||||||
|
# Create a mark containing the location of this attribute.
|
||||||
|
marker = parts: {
|
||||||
|
__readTree = parts;
|
||||||
|
};
|
||||||
|
|
||||||
|
# The marker is added to every set that was imported directly by
|
||||||
|
# readTree.
|
||||||
|
importWithMark = args: path: parts:
|
||||||
|
let
|
||||||
|
importedFile = import path;
|
||||||
|
pathType = builtins.typeOf importedFile;
|
||||||
|
imported =
|
||||||
|
assert assertMsg
|
||||||
|
(pathType == "lambda")
|
||||||
|
"readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }";
|
||||||
|
importedFile (argsFilter (argsWithPath args parts) parts);
|
||||||
|
in if (isAttrs imported)
|
||||||
|
then imported // (marker parts)
|
||||||
|
else imported;
|
||||||
|
|
||||||
|
nixFileName = file:
|
||||||
|
let res = match "(.*)\\.nix" file;
|
||||||
|
in if res == null then null else head res;
|
||||||
|
|
||||||
|
readTree = { args, initPath, rootDir, parts }:
|
||||||
|
let
|
||||||
|
dir = readDirVisible initPath;
|
||||||
|
joinChild = c: initPath + ("/" + c);
|
||||||
|
|
||||||
|
self = if rootDir
|
||||||
|
then { __readTree = []; }
|
||||||
|
else importWithMark args initPath parts;
|
||||||
|
|
||||||
|
# Import subdirectories of the current one, unless the special
|
||||||
|
# `.skip-subtree` file exists which makes readTree ignore the
|
||||||
|
# children.
|
||||||
|
#
|
||||||
|
# This file can optionally contain information on why the tree
|
||||||
|
# should be ignored, but its content is not inspected by
|
||||||
|
# readTree
|
||||||
|
filterDir = f: dir."${f}" == "directory";
|
||||||
|
children = if hasAttr ".skip-subtree" dir then [] else map (c: {
|
||||||
|
name = c;
|
||||||
|
value = readTree {
|
||||||
|
args = args;
|
||||||
|
initPath = (joinChild c);
|
||||||
|
rootDir = false;
|
||||||
|
parts = (parts ++ [ c ]);
|
||||||
|
};
|
||||||
|
}) (filter filterDir (attrNames dir));
|
||||||
|
|
||||||
|
# Import Nix files
|
||||||
|
nixFiles = filter (f: f != null) (map nixFileName (attrNames dir));
|
||||||
|
nixChildren = map (c: let p = joinChild (c + ".nix"); in {
|
||||||
|
name = c;
|
||||||
|
value = importWithMark args p (parts ++ [ c ]);
|
||||||
|
}) nixFiles;
|
||||||
|
in if dir ? "default.nix"
|
||||||
|
then (if isAttrs self then self // (listToAttrs children) else self)
|
||||||
|
else (listToAttrs (nixChildren ++ children) // (marker parts));
|
||||||
|
|
||||||
|
in {
|
||||||
|
__functor = _: args: initPath: readTree {
|
||||||
|
inherit args initPath;
|
||||||
|
rootDir = true;
|
||||||
|
parts = [];
|
||||||
|
};
|
||||||
|
}
|
|
@ -4,7 +4,7 @@ let
|
||||||
|
|
||||||
hscloud = import ./default.nix {};
|
hscloud = import ./default.nix {};
|
||||||
|
|
||||||
in with hscloud.config.pkgs; let
|
in with hscloud.pkgs; let
|
||||||
|
|
||||||
wrapper = pkgs.writeScript "wrapper.sh"
|
wrapper = pkgs.writeScript "wrapper.sh"
|
||||||
''
|
''
|
||||||
|
|
Loading…
Reference in a new issue