cheshire
/
hscloud
forked from hswaw/hscloud
1
0
Fork 0
Code Pull Requests Activity

nix: upgrade readTree 

Change-Id: I460800dc3d8095e2ae89b8bd6ed7c5f0c90b6ccf
master
implr 2021-08-30 23:28:45 +02:00
parent eed9afe210
commit 56ff18c486
6 changed files with 231 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
default.nix
View File

@ -5,7 +5,7 @@ with builtins;
let
fix = f: let x = f x; in x;
readTree = import ./nix/readtree.nix {};
readTree = import ./nix/readtree {};
# Tracking nixos-unstable as of 2021-08-11.
nixpkgsCommit = "e26c0ffdb013cd378fc2528a44689a8bf35d2a6c";
@ -18,21 +18,12 @@ let
config.allowBroken = true;
};
in fix (self: rec {
config = {
hscloud = self // {
root = ./.;
};
pkgs = nixpkgs;
pkgsSrc = nixpkgsSrc;
inherit (nixpkgs) lib stdenv;
};
bgpwtf = readTree config ./bgpwtf;
cluster = readTree config ./cluster;
hswaw = readTree config ./hswaw;
ops = readTree config ./ops;
in fix (self: (readTree rec {
hscloud = self;
pkgs = nixpkgs;
pkgsSrc = nixpkgsSrc;
inherit (nixpkgs) lib stdenv;
} ./.) // {
root = ./.;
pkgs = nixpkgs;
})

96
nix/readtree.nix
View File

@ -1,96 +0,0 @@
# The MIT License (MIT)
#
# Copyright (c) 2019 Vincent Ambo
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
{ ... }:
args: initPath:
let
inherit (builtins)
attrNames
baseNameOf
filter
hasAttr
head
isAttrs
length
listToAttrs
map
match
readDir
substring;
argsWithPath = parts:
let meta.locatedAt = parts;
in meta // (if isAttrs args then args else args meta);
readDirVisible = path:
let
children = readDir path;
isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
names = filter isVisible (attrNames children);
in listToAttrs (map (name: {
inherit name;
value = children.${name};
}) names);
# The marker is added to every set that was imported directly by
# readTree.
importWithMark = path: parts:
let imported = import path (argsWithPath parts);
in if (isAttrs imported)
then imported // { __readTree = true; }
else imported;
nixFileName = file:
let res = match "(.*)\.nix" file;
in if res == null then null else head res;
readTree = path: parts:
let
dir = readDirVisible path;
self = importWithMark path parts;
joinChild = c: path + ("/" + c);
# Import subdirectories of the current one, unless the special
# `.skip-subtree` file exists which makes readTree ignore the
# children.
#
# This file can optionally contain information on why the tree
# should be ignored, but its content is not inspected by
# readTree
filterDir = f: dir."${f}" == "directory";
children = if hasAttr ".skip-subtree" dir then [] else map (c: {
name = c;
value = readTree (joinChild c) (parts ++ [ c ]);
}) (filter filterDir (attrNames dir));
# Import Nix files
nixFiles = filter (f: f != null) (map nixFileName (attrNames dir));
nixChildren = map (c: let p = joinChild (c + ".nix"); in {
name = c;
value = importWithMark p (parts ++ [ c ]);
}) nixFiles;
in if dir ? "default.nix"
then (if isAttrs self then self // (listToAttrs children) else self)
else listToAttrs (nixChildren ++ children);
in readTree initPath [ (baseNameOf initPath) ]

22
nix/readtree/LICENSE Normal file
View File

@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2019 Vincent Ambo
Copyright (c) 2020-2021 The TVL Authors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

84
nix/readtree/README.md Normal file
View File

@ -0,0 +1,84 @@
readTree
========
This is a Nix program that builds up an attribute set tree for a large
repository based on the filesystem layout.
It is in fact the tool that lays out the attribute set of this repository.
As an example, consider a root (`.`) of a repository and a layout such as:
```
.
├── third_party
│   ├── default.nix
│   └── rustpkgs
│   ├── aho-corasick.nix
│   └── serde.nix
└── tools
├── cheddar
│   └── default.nix
└── roquefort.nix
```
When `readTree` is called on that tree, it will construct an attribute set with
this shape:
```nix
{
tools = {
cheddar = ...;
roquefort = ...;
};
third_party = {
# the `default.nix` of this folder might have had arbitrary other
# attributes here, such as this:
favouriteColour = "orange";
rustpkgs = {
aho-corasick = ...;
serde = ...;
};
};
}
```
Every imported Nix file that yields an attribute set will have a `__readTree =
true;` attribute merged into it.
## Traversal logic
`readTree` will follow any subdirectories of a tree and import all Nix files,
with some exceptions:
* A folder can declare that its children are off-limit by containing a
`.skip-subtree` file. Since the content of the file is not checked, it can be
useful to leave a note for a human in the file.
* If a folder contains a `default.nix` file, no *sibling* Nix files will be
imported - however children are traversed as normal.
* If a folder contains a `default.nix` it is loaded and, if it evaluates to a
set, *merged* with the children. If it evaluates to anything else the children
are *not traversed*.
* The `default.nix` of the top-level folder on which readTree is
called is **not** read to avoid infinite recursion (as, presumably,
this file is where readTree itself is called).
Traversal is lazy, `readTree` will only build up the tree as requested. This
currently has the downside that directories with no importable files end up in
the tree as empty nodes (`{}`).
## Import structure
`readTree` is called with two parameters: The arguments to pass to all imports,
and the initial path at which to start the traversal.
The package headers in this repository follow the form `{ pkgs, ... }:` where
`pkgs` is a fixed-point of the entire package tree (see the `default.nix` at the
root of the depot).
In theory `readTree` can pass arguments of different shapes, but I have found
this to be a good solution for the most part.
Note that `readTree` does not currently make functions overridable, though it is
feasible that it could do that in the future.

116
nix/readtree/default.nix Normal file
View File

@ -0,0 +1,116 @@
# Copyright (c) 2019 Vincent Ambo
# Copyright (c) 2020-2021 The TVL Authors
# SPDX-License-Identifier: MIT
#
# Provides a function to automatically read a a filesystem structure
# into a Nix attribute set.
#
# Optionally accepts an argument `argsFilter` on import, which is a
# function that receives the current tree location (as a list of
# strings) and the argument set and can arbitrarily modify it.
{ argsFilter ? (x: _parts: x)
, ... }:
let
inherit (builtins)
attrNames
baseNameOf
concatStringsSep
filter
hasAttr
head
isAttrs
length
listToAttrs
map
match
readDir
substring;
assertMsg = pred: msg:
if pred
then true
else builtins.trace msg false;
argsWithPath = args: parts:
let meta.locatedAt = parts;
in meta // (if isAttrs args then args else args meta);
readDirVisible = path:
let
children = readDir path;
isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != ".";
names = filter isVisible (attrNames children);
in listToAttrs (map (name: {
inherit name;
value = children.${name};
}) names);
# Create a mark containing the location of this attribute.
marker = parts: {
__readTree = parts;
};
# The marker is added to every set that was imported directly by
# readTree.
importWithMark = args: path: parts:
let
importedFile = import path;
pathType = builtins.typeOf importedFile;
imported =
assert assertMsg
(pathType == "lambda")
"readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }";
importedFile (argsFilter (argsWithPath args parts) parts);
in if (isAttrs imported)
then imported // (marker parts)
else imported;
nixFileName = file:
let res = match "(.*)\\.nix" file;
in if res == null then null else head res;
readTree = { args, initPath, rootDir, parts }:
let
dir = readDirVisible initPath;
joinChild = c: initPath + ("/" + c);
self = if rootDir
then { __readTree = []; }
else importWithMark args initPath parts;
# Import subdirectories of the current one, unless the special
# `.skip-subtree` file exists which makes readTree ignore the
# children.
#
# This file can optionally contain information on why the tree
# should be ignored, but its content is not inspected by
# readTree
filterDir = f: dir."${f}" == "directory";
children = if hasAttr ".skip-subtree" dir then [] else map (c: {
name = c;
value = readTree {
args = args;
initPath = (joinChild c);
rootDir = false;
parts = (parts ++ [ c ]);
};
}) (filter filterDir (attrNames dir));
# Import Nix files
nixFiles = filter (f: f != null) (map nixFileName (attrNames dir));
nixChildren = map (c: let p = joinChild (c + ".nix"); in {
name = c;
value = importWithMark args p (parts ++ [ c ]);
}) nixFiles;
in if dir ? "default.nix"
then (if isAttrs self then self // (listToAttrs children) else self)
else (listToAttrs (nixChildren ++ children) // (marker parts));
in {
__functor = _: args: initPath: readTree {
inherit args initPath;
rootDir = true;
parts = [];
};
}

2
shell.nix
View File

@ -4,7 +4,7 @@ let
hscloud = import ./default.nix {};
in with hscloud.config.pkgs; let
in with hscloud.pkgs; let
wrapper = pkgs.writeScript "wrapper.sh"
''