From 491542589b6b330abfb611a5b49ff0e0fdabdccc Mon Sep 17 00:00:00 2001 From: Serge Bazanski Date: Sat, 24 Oct 2020 20:22:08 +0200 Subject: [PATCH] tools/gostatic: init This adds Bazel/hscloud integration to gostatic, via gostatic_tarball. A sample is provided in //tools/gostatic/example, it can be built using: bazel build //tools/gostatic/example The resulting tarball can then be extracted and viewed in a web browser. Change-Id: Idf8d4a8e0ee3a5ae07f7449a25909478c2d8b105 --- tools/gostatic/BUILD | 8 ++ tools/gostatic/README.md | 43 +++++++ tools/gostatic/example/BUILD | 45 ++++++++ tools/gostatic/example/site.tmpl | 50 ++++++++ tools/gostatic/example/src/blog.atom | 32 ++++++ tools/gostatic/example/src/blog/first.md | 5 + tools/gostatic/example/src/index.html | 9 ++ tools/gostatic/example/src/static/style.css | 1 + tools/gostatic/rules.bzl | 84 ++++++++++++++ tools/gostatic/tarify/BUILD.bazel | 15 +++ tools/gostatic/tarify/main.go | 119 ++++++++++++++++++++ 11 files changed, 411 insertions(+) create mode 100644 tools/gostatic/BUILD create mode 100644 tools/gostatic/README.md create mode 100644 tools/gostatic/example/BUILD create mode 100644 tools/gostatic/example/site.tmpl create mode 100644 tools/gostatic/example/src/blog.atom create mode 100644 tools/gostatic/example/src/blog/first.md create mode 100644 tools/gostatic/example/src/index.html create mode 100644 tools/gostatic/example/src/static/style.css create mode 100644 tools/gostatic/rules.bzl create mode 100644 tools/gostatic/tarify/BUILD.bazel create mode 100644 tools/gostatic/tarify/main.go diff --git a/tools/gostatic/BUILD b/tools/gostatic/BUILD new file mode 100644 index 00000000..13a250dd --- /dev/null +++ b/tools/gostatic/BUILD @@ -0,0 +1,8 @@ +load("//bzl:rules.bzl", "copy_go_binary") + +copy_go_binary( + name = "gostatic", + src = "@com_github_piranha_gostatic//:gostatic", + visibility = ["//visibility:public"], +) + diff --git a/tools/gostatic/README.md b/tools/gostatic/README.md new file mode 100644 index 00000000..6c0f17c6 --- /dev/null +++ b/tools/gostatic/README.md @@ -0,0 +1,43 @@ +gostatic site generator +======================= + +This implements support for [gostatic](https://github.com/piranha/gostatic), a static site generator, inside hscloud. + +Creating a gostatic site +------------------------ + +To get started, copy over the skeleton from //tools/gostatic/example into a new directory. + + mkdir -p personal/foo + cp -rv tools/gostatic/example personal/foo/mysite + +You can also build your own `gostatic_tarball` from scratch if you are familiar enough with gostatic. + +You can then then build a tarball of your site by running: + + bazel build //personal/foo/mysite + +Your site will be built and tarred up into `bazel-bin/personal/foo/mysite/mysite.tar`. You can then use this to populate a container in `docker_rules`. + +TODO(q3k): add a target that starts up a simple web server for testing the rendered site. + +Configuring a gostatic site +--------------------------- + +Configuration is done via the `gostatic_tarball` rule. This mostly generates an upstream gostatic [configuration file](https://github.com/piranha/gostatic#configuration) - please refer to that file for more information. + +| Field | Description | Example | +|-------|-------------|---------| +| `templates` | List of template sources/targets. This is used to populate the TEMPLATES config option. | `[ ":site.tmpl" ]` | +| `source_dir` | BUILDfile-relative source directory containing site sources. This is used to populate the SOURCE config option. All files given in `srcs` must be contained within this directory. | `"src"` | +| `srcs` | List of template sources/targets. This is what will be available to gostatic during compilation. | `[ "src/blog/first.md" ]` | +| `extra_config` | Rest of the gostatic config, ie. rules. | | + +Running gostatic-the-tool +------------------------- + +If you want to run plain gostatic for some odd reason, it's available under: + + bazel run //tools/gostatic + +TODO(q3k): allow running this against a `gostatic_tarball`'s config. diff --git a/tools/gostatic/example/BUILD b/tools/gostatic/example/BUILD new file mode 100644 index 00000000..cb54fec6 --- /dev/null +++ b/tools/gostatic/example/BUILD @@ -0,0 +1,45 @@ +load("//tools/gostatic:rules.bzl", "gostatic_tarball") + +gostatic_tarball( + name = "example", + templates = [ + "site.tmpl", + ], + source_dir = "src", + extra_config = """ +TITLE = Example Site +URL = https://example.com +AUTHOR = Your Name + +blog/*.md: + config + ext .html + directorify + tags tags/*.tag + markdown + template post + template page + +*.tag: blog/*.md + ext .html + directorify + template tag + markdown + template page + +blog.atom: blog/*.md + inner-template + +index.html: blog/*.md + config + inner-template + template page + + """, + srcs = [ + "src/blog/first.md", + "src/static/style.css", + "src/blog.atom", + "src/index.html", + ], +) diff --git a/tools/gostatic/example/site.tmpl b/tools/gostatic/example/site.tmpl new file mode 100644 index 00000000..8878ab2c --- /dev/null +++ b/tools/gostatic/example/site.tmpl @@ -0,0 +1,50 @@ +{{ define "header" }} + + + + + + {{ .Site.Other.Title }}{{ if .Title }}: {{ .Title }}{{ end }} + + + + +{{ end }} + +{{ define "footer" }} + + +{{ end }} + +{{define "date"}} + +{{end}} + +{{ define "page" }}{{ template "header" . }} + {{ .Content }} +{{ template "footer" . }}{{ end }} + +{{ define "post" }} +
+
+

{{ .Title }}

+
+ {{ template "date" .Date }} — + {{ range $i, $t := .Tags }}{{if $i}},{{end}} + {{ $t }}{{ end }} +
+
+
+ {{ .Content }} +
+
+{{ end }} + +{{define "tag"}} +# Pages tagged with {{ .Title }} +{{ range .Site.Pages.WithTag .Title }} +- [{{ .Title }}](../../{{ .Url }}) +{{ end }} +{{ end }} diff --git a/tools/gostatic/example/src/blog.atom b/tools/gostatic/example/src/blog.atom new file mode 100644 index 00000000..3af3d96c --- /dev/null +++ b/tools/gostatic/example/src/blog.atom @@ -0,0 +1,32 @@ + + + {{ .Site.Other.Url }} + {{ .Site.Other.Title }} + {{ with .Site.Pages.Children "blog/" }} + {{ .First.Date.Format "2006-01-02T15:04:05Z07:00" }} + {{ end }} + {{ .Site.Other.Author }} + + gostatic + +{{ with .Site.Pages.Children "blog/" }} +{{ range .Slice 0 5 }} + + {{ .Url }} + {{ or .Other.Author .Site.Other.Author }} + {{ html .Title }} + {{ .Date.Format "2006-01-02T15:04:05Z07:00" }} + {{ range .Tags }} + + {{ end }} + + + {{/* .Process runs here in case only feed changed */}} + {{ with cut "
" "
" .Process.Content }} + {{ html . }} + {{ end }} + + +{{ end }} +{{ end }} + diff --git a/tools/gostatic/example/src/blog/first.md b/tools/gostatic/example/src/blog/first.md new file mode 100644 index 00000000..64812c6d --- /dev/null +++ b/tools/gostatic/example/src/blog/first.md @@ -0,0 +1,5 @@ +title: First Post +date: 2012-12-12 +tags: blog +---- +My first post with [gostatic](https://github.com/piranha/gostatic). diff --git a/tools/gostatic/example/src/index.html b/tools/gostatic/example/src/index.html new file mode 100644 index 00000000..56853ddf --- /dev/null +++ b/tools/gostatic/example/src/index.html @@ -0,0 +1,9 @@ +title: Main Page +---- + diff --git a/tools/gostatic/example/src/static/style.css b/tools/gostatic/example/src/static/style.css new file mode 100644 index 00000000..9f89f00c --- /dev/null +++ b/tools/gostatic/example/src/static/style.css @@ -0,0 +1 @@ +/* put your style rules here */ diff --git a/tools/gostatic/rules.bzl b/tools/gostatic/rules.bzl new file mode 100644 index 00000000..e0326f92 --- /dev/null +++ b/tools/gostatic/rules.bzl @@ -0,0 +1,84 @@ +def _gostatic_tarball_impl(ctx): + out = ctx.actions.declare_directory("out") + tarball_name = ctx.attr.name + ".tar" + tarball = ctx.actions.declare_file(tarball_name) + + config_name = ctx.attr.name + ".config" + config = ctx.actions.declare_file(config_name) + + # Build path to root of sources, based on source_dir + # and location of the instantiating BUILDfile + # (source_dir is defined as relative to the BUILD file). + source_dir = '/'.join(ctx.build_file_path.split('/')[:-1]) + '/' + ctx.attr.source_dir + + # Relative path to go up from generated config to build + # root. This is because gostatic is magical and really + # wants the config to be alongside the source. + up = "../" * (config.path.count("/")) + templates = " ".join([up + f.path for f in ctx.files.templates]) + config_lines = [ + "OUTPUT = {}".format(up + out.path), + "TEMPLATES = {}".format(templates), + "SOURCE = {}".format(up + source_dir), + ] + config_content = "\n".join(config_lines) + config_content += ctx.attr.extra_config + + ctx.actions.write(config, config_content) + + ctx.actions.run( + outputs = [out], + inputs = [config] + ctx.files.templates + ctx.files.srcs, + executable = ctx.file._gostatic, + arguments = [config.path], + ) + ctx.actions.run( + outputs = [tarball], + inputs = [out], + executable = ctx.file._tarify, + arguments = [ + "-site", out.path, + "-tarball", tarball.path, + ], + ) + + return [DefaultInfo(files=depset([tarball]))] + +gostatic_tarball = rule( + implementation = _gostatic_tarball_impl, + attrs = { + "extra_config": attr.string( + mandatory = True, + doc = """ + Gostatic configuration (rules, etc). Do not specify OUTPUT, TEMPLATES + or SOURCES - these are automatically generated. + """, + ), + "source_dir": attr.string( + mandatory = True, + doc = "Root of site sources. Relative to BUILDfile.", + ), + "srcs": attr.label_list( + allow_files = True, + doc = "Site sources, all must be contained within source_dir" + ), + "templates": attr.label_list( + allow_files = True, + doc = "Templates to use (passed to TEMPLATES in gostatic config).", + ), + "_gostatic": attr.label( + default = Label("//tools/gostatic"), + allow_single_file = True, + executable = True, + cfg = "exec", + doc = "Path to gostatic binary.", + ), + "_tarify": attr.label( + default = Label("//tools/gostatic/tarify"), + allow_single_file = True, + executable = True, + cfg = "exec", + doc = "Path to tarify binary.", + ), + }, +) diff --git a/tools/gostatic/tarify/BUILD.bazel b/tools/gostatic/tarify/BUILD.bazel new file mode 100644 index 00000000..7bc841b2 --- /dev/null +++ b/tools/gostatic/tarify/BUILD.bazel @@ -0,0 +1,15 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "code.hackerspace.pl/hscloud/tools/gostatic/tarify", + visibility = ["//visibility:private"], + deps = ["@com_github_golang_glog//:go_default_library"], +) + +go_binary( + name = "tarify", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/tools/gostatic/tarify/main.go b/tools/gostatic/tarify/main.go new file mode 100644 index 00000000..f2922895 --- /dev/null +++ b/tools/gostatic/tarify/main.go @@ -0,0 +1,119 @@ +package main + +// tarify implements a minimal, self-contained, hermetic tarball builder. +// It is currently used with gostatic to take a non-hermetic directory and +// turn it into a hermetic tarball via a glob. +// +// For more information about tree artifacts and hermeticity, see: +// https://jmmv.dev/2019/12/bazel-dynamic-execution-tree-artifacts.html + +import ( + "archive/tar" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/golang/glog" +) + +var ( + flagSite string + flagTarball string +) + +func init() { + flag.Set("logtostderr", "true") +} + +func main() { + flag.StringVar(&flagSite, "site", "", "Site sources") + flag.StringVar(&flagTarball, "tarball", "", "Output tarball") + flag.Parse() + + if flagSite == "" { + glog.Exitf("-site must be set") + } + if flagTarball == "" { + glog.Exitf("-tarball must be set") + } + + f, err := os.Create(flagTarball) + if err != nil { + glog.Exitf("Create(%q): %v", flagTarball, err) + } + defer f.Close() + w := tar.NewWriter(f) + defer w.Close() + + flagSite = strings.TrimSuffix(flagSite, "/") + + // First retrieve all files and sort. This is required for idempotency. + elems := []struct { + path string + info os.FileInfo + }{} + err = filepath.Walk(flagSite, func(inPath string, _ os.FileInfo, err error) error { + // We don't use the given fileinfo, as we want to deref symlinks. + info, err := os.Stat(inPath) + if err != nil { + return fmt.Errorf("Stat: %w", err) + } + elems = append(elems, struct { + path string + info os.FileInfo + }{inPath, info}) + return nil + }) + if err != nil { + glog.Exitf("Walk(%q, _): %v", flagSite, err) + } + sort.Slice(elems, func(i, j int) bool { return elems[i].path < elems[j].path }) + + // Now that we have a sorted list, tar 'em up. + for _, elem := range elems { + inPath := elem.path + info := elem.info + + outPath := strings.TrimPrefix(strings.TrimPrefix(inPath, flagSite), "/") + if outPath == "" { + continue + } + if info.IsDir() { + glog.Infof("D %s", outPath) + if err := w.WriteHeader(&tar.Header{ + Typeflag: tar.TypeDir, + Name: outPath, + Mode: 0755, + }); err != nil { + glog.Exitf("Writing directory header for %q failed: %v", inPath, err) + } + } else { + glog.Infof("F %s", outPath) + if err := w.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: outPath, + Mode: 0644, + // TODO(q3k): this can race (TOCTOU Stat/Open, resulting in "archive/tar: write Too long") + // No idea, how to handle this better though without reading the entire file into memory, + // or trying to do filesystem locks? Besides, in practical use with Bazel this will never + // happen. + Size: info.Size(), + }); err != nil { + glog.Exitf("Writing file header for %q failed: %v", inPath, err) + } + r, err := os.Open(inPath) + if err != nil { + glog.Exitf("Open(%q): %v", inPath, err) + } + defer r.Close() + if _, err := io.Copy(w, r); err != nil { + glog.Exitf("Copy(%q): %v", inPath, err) + } + + } + } +}