cheshire
/
hscloud
forked from hswaw/hscloud
1
0
Fork 0
Code Pull Requests Activity

devtools/{depotview,hackdoc}: tie both together 

Change-Id: I0a1ca3b4fa0e0a074eccbe0f8748839b926db9c1
master
q3k 2020-04-10 17:39:43 +02:00
parent 4c0e9b52c0
commit f157b4d632
16 changed files with 522 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -9,7 +9,7 @@ Any time you see a `//path/like/this`, it refers to the root of hscloud, ie. the
Viewing this documentation
--------------------------
For a please web viewing experience, [see this documentation in hackdoc](https://hackdoc.hackerspace.pl/). This will allow you to read this markdown file (and others) in a pretty, linkable view.
For a pleaseant web viewing experience, [see this documentation in hackdoc](https://hackdoc.hackerspace.pl/). This will allow you to read this markdown file (and others) in a pretty, linkable view.
Getting started
---------------

30
devtools/depotview/proto/depotview.proto
View File

@ -2,6 +2,18 @@ syntax = "proto3";
package depotview;
option go_package = "code.hackerspace.pl/hscloud/devtools/depotview/proto";
service DepotView {
// Resolve a git branch/tag/ref... into a commit hash.
rpc Resolve(ResolveRequest) returns (ResolveResponse);
// Resolve a gerrit change number into a git commit hash.
rpc ResolveGerritChange(ResolveGerritChangeRequest) returns (ResolveGerritChangeResponse);
// Minimal file access API. It kinda stinks.
rpc Stat(StatRequest) returns (StatResponse);
rpc Read(ReadRequest) returns (stream ReadResponse);
}
message ResolveRequest {
string ref = 1;
}
@ -11,6 +23,15 @@ message ResolveResponse {
int64 last_checked = 2;
}
message ResolveGerritChangeRequest {
int64 change = 1;
}
message ResolveGerritChangeResponse {
string hash = 1;
int64 last_checked = 2;
}
message StatRequest {
string hash = 1;
string path = 2;
@ -35,12 +56,3 @@ message ReadResponse {
// Chunk of data. Empty once everything has been sent over.
bytes data = 1;
}
service DepotView {
// Resolve a git branch/tag/ref... into a commit hash.
rpc Resolve(ResolveRequest) returns (ResolveResponse);
// Minimal file access API. It kinda stinks.
rpc Stat(StatRequest) returns (StatResponse);
rpc Read(ReadRequest) returns (stream ReadResponse);
}

6
devtools/depotview/service/BUILD.bazel
View File

@ -2,12 +2,16 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = ["service.go"],
srcs = [
"gerrit.go",
"service.go",
],
importpath = "code.hackerspace.pl/hscloud/devtools/depotview/service",
visibility = ["//visibility:public"],
deps = [
"//devtools/depotview/proto:go_default_library",
"@com_github_go_git_go_git_v5//:go_default_library",
"@com_github_go_git_go_git_v5//config:go_default_library",
"@com_github_go_git_go_git_v5//plumbing:go_default_library",
"@com_github_go_git_go_git_v5//plumbing/filemode:go_default_library",
"@com_github_go_git_go_git_v5//plumbing/object:go_default_library",

56
devtools/depotview/service/gerrit.go Normal file
View File

@ -0,0 +1,56 @@
package service
import (
"strconv"
"strings"
)
type gerritMeta struct {
patchSet int64
changeId string
commit string
}
// parseGerritMetadata takes a NoteDB metadata entry and extracts info from it.
func parseGerritMetadata(messages []string) *gerritMeta {
meta := &gerritMeta{}
for _, message := range messages {
for _, line := range strings.Split(message, "\n") {
line = strings.TrimSpace(line)
if len(line) == 0 {
continue
}
parts := strings.SplitN(line, ":", 2)
if len(parts) < 2 {
continue
}
k, v := parts[0], strings.TrimSpace(parts[1])
switch k {
case "Patch-set":
n, err := strconv.ParseInt(v, 10, 64)
if err != nil {
continue
}
meta.patchSet = n
case "Change-id":
meta.changeId = v
case "Commit":
meta.commit = v
}
}
}
if meta.patchSet == 0 {
return nil
}
if meta.changeId == "" {
return nil
}
if meta.commit == "" {
return nil
}
return meta
}

91
devtools/depotview/service/service.go
View File

@ -2,6 +2,7 @@ package service
import (
"context"
"fmt"
"io"
"regexp"
"strings"
@ -13,6 +14,7 @@ import (
"google.golang.org/grpc/status"
git "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
@ -30,9 +32,9 @@ type Service struct {
remote string
storer storage.Storer
mu sync.Mutex
repo *git.Repository
lastPull time.Time
mu sync.Mutex
repo *git.Repository
lastFetch time.Time
}
func New(remote string) *Service {
@ -42,10 +44,10 @@ func New(remote string) *Service {
}
}
func (s *Service) ensureRepo() error {
func (s *Service) ensureRepo(ctx context.Context) error {
// Clone repository if necessary.
if s.repo == nil {
repo, err := git.Clone(s.storer, nil, &git.CloneOptions{
repo, err := git.CloneContext(ctx, s.storer, nil, &git.CloneOptions{
URL: s.remote,
})
if err != nil {
@ -53,17 +55,24 @@ func (s *Service) ensureRepo() error {
return status.Error(codes.Unavailable, "could not clone repository")
}
s.repo = repo
s.lastPull = time.Now()
}
// Fetch if necessary.
if time.Since(s.lastPull) > time.Minute {
err := s.repo.Fetch(&git.FetchOptions{})
if time.Since(s.lastFetch) > 10*time.Second {
glog.Infof("Fetching...")
err := s.repo.FetchContext(ctx, &git.FetchOptions{
RefSpecs: []config.RefSpec{
config.RefSpec("+refs/heads/*:refs/remotes/origin/*"),
config.RefSpec("+refs/changes/*:refs/changes/*"),
},
Force: true,
})
if err != nil && err != git.NoErrAlreadyUpToDate {
glog.Errorf("Fetch(): %v", err)
} else {
s.lastPull = time.Now()
s.lastFetch = time.Now()
}
}
return nil
@ -77,22 +86,69 @@ func (s *Service) Resolve(ctx context.Context, req *pb.ResolveRequest) (*pb.Reso
return nil, status.Error(codes.InvalidArgument, "ref must be set")
}
if err := s.ensureRepo(); err != nil {
if err := s.ensureRepo(ctx); err != nil {
return nil, err
}
h, err := s.repo.ResolveRevision(plumbing.Revision(req.Ref))
switch {
case err == plumbing.ErrReferenceNotFound:
return &pb.ResolveResponse{Hash: "", LastChecked: s.lastPull.UnixNano()}, nil
return &pb.ResolveResponse{Hash: "", LastChecked: s.lastFetch.UnixNano()}, nil
case err != nil:
return nil, status.Errorf(codes.Unavailable, "git resolve error: %v", err)
default:
return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastPull.UnixNano()}, nil
return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastFetch.UnixNano()}, nil
}
}
func (s *Service) getFile(hash, path string, notFoundOkay bool) (*object.File, error) {
func (s *Service) ResolveGerritChange(ctx context.Context, req *pb.ResolveGerritChangeRequest) (*pb.ResolveGerritChangeResponse, error) {
if err := s.ensureRepo(ctx); err != nil {
return nil, err
}
// I'm totally guessing this, from these examples:
// refs/changes/03/3/meta
// refs/changes/77/77/meta
// refs/changes/47/247/meta
// etc...
shard := fmt.Sprintf("%02d", req.Change%100)
metaRef := fmt.Sprintf("refs/changes/%s/%d/meta", shard, req.Change)
h, err := s.repo.ResolveRevision(plumbing.Revision(metaRef))
switch {
case err == plumbing.ErrReferenceNotFound:
return &pb.ResolveGerritChangeResponse{Hash: "", LastChecked: s.lastFetch.UnixNano()}, nil
case err != nil:
return nil, status.Errorf(codes.Unavailable, "git metadata resolve error: %v", err)
}
c, err := s.repo.CommitObject(*h)
if err != nil {
return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
}
var messages []string
for {
messages = append([]string{c.Message}, messages...)
if len(c.ParentHashes) != 1 {
break
}
c, err = s.repo.CommitObject(c.ParentHashes[0])
if err != nil {
return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
}
}
meta := parseGerritMetadata(messages)
if meta == nil {
return nil, status.Errorf(codes.Internal, "could not parse gerrit metadata for ref %q", metaRef)
}
return &pb.ResolveGerritChangeResponse{Hash: meta.commit, LastChecked: s.lastFetch.UnixNano()}, nil
}
func (s *Service) getFile(ctx context.Context, hash, path string, notFoundOkay bool) (*object.File, error) {
if !reHash.MatchString(hash) {
return nil, status.Error(codes.InvalidArgument, "hash must be valid full git hash string")
}
@ -105,7 +161,7 @@ func (s *Service) getFile(hash, path string, notFoundOkay bool) (*object.File, e
return nil, status.Error(codes.InvalidArgument, "path must be a valid unix or depot-style path")
}
if err := s.ensureRepo(); err != nil {
if err := s.ensureRepo(ctx); err != nil {
return nil, err
}
@ -134,7 +190,7 @@ func (s *Service) Stat(ctx context.Context, req *pb.StatRequest) (*pb.StatRespon
s.mu.Lock()
defer s.mu.Unlock()
file, err := s.getFile(req.Hash, req.Path, true)
file, err := s.getFile(ctx, req.Hash, req.Path, true)
if err != nil {
return nil, err
}
@ -157,7 +213,9 @@ func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error {
s.mu.Lock()
defer s.mu.Unlock()
file, err := s.getFile(req.Hash, req.Path, false)
ctx := srv.Context()
file, err := s.getFile(ctx, req.Hash, req.Path, false)
if err != nil {
return err
}
@ -168,7 +226,6 @@ func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error {
}
defer reader.Close()
ctx := srv.Context()
for {
if ctx.Err() != nil {
return ctx.Err()

4
devtools/hackdoc/BUILD.bazel
View File

@ -10,10 +10,14 @@ go_library(
importpath = "code.hackerspace.pl/hscloud/devtools/hackdoc",
visibility = ["//visibility:private"],
deps = [
"//devtools/depotview/proto:go_default_library",
"//devtools/hackdoc/config:go_default_library",
"//devtools/hackdoc/source:go_default_library",
"//go/mirko:go_default_library",
"//go/pki:go_default_library",
"@com_github_golang_glog//:go_default_library",
"@in_gopkg_russross_blackfriday_v2//:go_default_library",
"@org_golang_google_grpc//:go_default_library",
],
)

5
devtools/hackdoc/README.md
View File

@ -8,6 +8,11 @@ Usage
Any Markdown submitted to hscloud is visible via hackdoc. Simply go to https://hackdoc.hackerspace.pl/path/to/markdown.md to see it rendered.
You can pass a `?ref=foo` URL parameter to a hackdoc URL to get it to render a particular vesrion of the hscloud monorepo. For example:
- https://hackdoc.hackerspace.pl/?ref=master for the `master` branch
- https://hackdoc.hackerspace.pl/?ref=change/249 for the the source code at change '249'
Local Rendering
---------------

27
devtools/hackdoc/config/config.go
View File

@ -1,12 +1,12 @@
package config
import (
"context"
"fmt"
"html/template"
"strings"
"github.com/BurntSushi/toml"
"github.com/golang/glog"
"code.hackerspace.pl/hscloud/devtools/hackdoc/source"
)
@ -73,24 +73,11 @@ func configFileLocations(path string) []string {
return locations
}
func ForPath(s source.Source, path string) (*Config, error) {
func ForPath(ctx context.Context, s source.Source, path string) (*Config, error) {
if path != "//" {
path = strings.TrimRight(path, "/")
}
// Try cache.
cacheKey := fmt.Sprintf("config:%s", path)
if v := s.CacheGet(cacheKey); v != nil {
cfg, ok := v.(*Config)
if !ok {
glog.Errorf("Cache key %q corrupted, deleting", cacheKey)
s.CacheSet([]string{}, cacheKey, nil)
} else {
return cfg, nil
}
}
// Feed cache.
cfg := &Config{
Templates: make(map[string]*template.Template),
Errors: make(map[string]error),
@ -98,14 +85,14 @@ func ForPath(s source.Source, path string) (*Config, error) {
tomlPaths := configFileLocations(path)
for _, p := range tomlPaths {
file, err := s.IsFile(p)
file, err := s.IsFile(ctx, p)
if err != nil {
return nil, fmt.Errorf("IsFile(%q): %w", path, err)
}
if !file {
continue
}
data, err := s.ReadFile(p)
data, err := s.ReadFile(ctx, p)
if err != nil {
return nil, fmt.Errorf("ReadFile(%q): %w", path, err)
}
@ -116,7 +103,7 @@ func ForPath(s source.Source, path string) (*Config, error) {
continue
}
err = cfg.updateFromToml(p, s, c)
err = cfg.updateFromToml(ctx, p, s, c)
if err != nil {
return nil, fmt.Errorf("updating from %q: %w", p, err)
}
@ -125,7 +112,7 @@ func ForPath(s source.Source, path string) (*Config, error) {
return cfg, nil
}
func (c *Config) updateFromToml(p string, s source.Source, t *configToml) error {
func (c *Config) updateFromToml(ctx context.Context, p string, s source.Source, t *configToml) error {
if t.DefaultIndex != nil {
c.DefaultIndex = t.DefaultIndex
}
@ -134,7 +121,7 @@ func (c *Config) updateFromToml(p string, s source.Source, t *configToml) error
tmpl := template.New(k)
for _, source := range v.Sources {
data, err := s.ReadFile(source)
data, err := s.ReadFile(ctx, source)
if err != nil {
c.Errors[p] = fmt.Errorf("reading template file %q: %w", source, err)
return nil

7
devtools/hackdoc/helpers.go
View File

@ -3,6 +3,8 @@ package main
import (
"fmt"
"net/http"
"github.com/golang/glog"
)
func handle404(w http.ResponseWriter, r *http.Request) {
@ -16,3 +18,8 @@ func handle500(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
fmt.Fprintf(w, "500 :(\n")
}
func logRequest(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) {
result := fmt.Sprintf(format, args...)
glog.Infof("result: %s, remote: %q, ua: %q, referrer: %q, host: %q path: %q", result, r.RemoteAddr, r.Header.Get("User-Agent"), r.Header.Get("Referrer"), r.Host, r.URL.Path)
}

175
devtools/hackdoc/main.go
View File

@ -1,24 +1,30 @@
package main
import (
"context"
"flag"
"fmt"
"net/http"
"path/filepath"
"regexp"
"strings"
"time"
"code.hackerspace.pl/hscloud/go/mirko"
"code.hackerspace.pl/hscloud/go/pki"
"github.com/golang/glog"
"google.golang.org/grpc"
dvpb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
"code.hackerspace.pl/hscloud/devtools/hackdoc/config"
"code.hackerspace.pl/hscloud/devtools/hackdoc/source"
)
var (
flagListen = "127.0.0.1:8080"
flagDocRoot = "./docroot"
flagDocRoot = ""
flagDepotViewAddress = ""
flagHackdocURL = ""
flagGitwebURLPattern = "https://gerrit.hackerspace.pl/plugins/gitiles/hscloud/+/%s/%s"
flagGitwebDefaultBranch = "master"
rePagePath = regexp.MustCompile(`^/([A-Za-z0-9_\-/\. ]*)$`)
@ -29,10 +35,11 @@ func init() {
}
func main() {
flag.StringVar(&flagListen, "listen", flagListen, "Address to listen on for HTTP traffic")
flag.StringVar(&flagDocRoot, "docroot", flagDocRoot, "Path from which to serve documents")
flag.StringVar(&flagListen, "pub_listen", flagListen, "Address to listen on for HTTP traffic")
flag.StringVar(&flagDocRoot, "docroot", flagDocRoot, "Path from which to serve documents. Either this or depotview must be set")
flag.StringVar(&flagDepotViewAddress, "depotview", flagDepotViewAddress, "gRPC endpoint of depotview to serve from Git. Either this or docroot must be set")
flag.StringVar(&flagHackdocURL, "hackdoc_url", flagHackdocURL, "Public URL of hackdoc. If not given, autogenerate from listen path for dev purposes")
flag.StringVar(&flagGitwebURLPattern, "gitweb_url_pattern", flagGitwebURLPattern, "Pattern to sprintf to for URL for viewing a file in Git. First string is ref/rev, second is bare file path (sans //)")
flag.StringVar(&source.FlagGitwebURLPattern, "gitweb_url_pattern", source.FlagGitwebURLPattern, "Pattern to sprintf to for URL for viewing a file in Git. First string is ref/rev, second is bare file path (sans //)")
flag.StringVar(&flagGitwebDefaultBranch, "gitweb_default_rev", flagGitwebDefaultBranch, "Default Git rev to render/link to")
flag.Parse()
@ -40,25 +47,61 @@ func main() {
flagHackdocURL = fmt.Sprintf("http://%s", flagListen)
}
path, err := filepath.Abs(flagDocRoot)
if err != nil {
glog.Fatalf("Could not dereference path %q: %w", path, err)
if flagDocRoot == "" && flagDepotViewAddress == "" {
glog.Errorf("Either -docroot or -depotview must be set")
}
if flagDocRoot != "" && flagDepotViewAddress != "" {
glog.Errorf("Only one of -docroot or -depotview must be set")
}
s := &service{
source: source.NewLocal(path),
m := mirko.New()
if err := m.Listen(); err != nil {
glog.Exitf("Listen(): %v", err)
}
http.HandleFunc("/", s.handler)
var s *service
if flagDocRoot != "" {
path, err := filepath.Abs(flagDocRoot)
if err != nil {
glog.Exitf("Could not dereference path %q: %w", path, err)
}
glog.Infof("Starting in docroot mode for %q -> %q", flagDocRoot, path)
s = &service{
source: source.NewSingleRefProvider(source.NewLocal(path)),
}
} else {
glog.Infof("Starting in depotview mode (server %q)", flagDepotViewAddress)
conn, err := grpc.Dial(flagDepotViewAddress, pki.WithClientHSPKI())
if err != nil {
glog.Exitf("grpc.Dial(%q): %v", flagDepotViewAddress, err)
}
stub := dvpb.NewDepotViewClient(conn)
s = &service{
source: source.NewDepotView(stub),
}
}
mux := http.NewServeMux()
mux.HandleFunc("/", s.handler)
srv := &http.Server{Addr: flagListen, Handler: mux}
glog.Infof("Listening on %q...", flagListen)
if err := http.ListenAndServe(flagListen, nil); err != nil {
glog.Fatal(err)
}
go func() {
if err := srv.ListenAndServe(); err != nil {
glog.Error(err)
}
}()
<-m.Done()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
srv.Shutdown(ctx)
}
type service struct {
source source.Source
source source.SourceProvider
}
func (s *service) handler(w http.ResponseWriter, r *http.Request) {
@ -69,23 +112,60 @@ func (s *service) handler(w http.ResponseWriter, r *http.Request) {
}
glog.Infof("%+v", r.URL.Query())
rev := r.URL.Query().Get("rev")
if rev == "" {
rev = flagGitwebDefaultBranch
ref := r.URL.Query().Get("ref")
if ref == "" {
ref = flagGitwebDefaultBranch
}
ctx := r.Context()
source, err := s.source.Source(ctx, ref)
switch {
case err != nil:
glog.Errorf("Source(%q): %v", ref, err)
handle500(w, r)
return
case source == nil:
handle404(w, r)
return
}
path := r.URL.Path
if match := rePagePath.FindStringSubmatch(path); match != nil {
s.handlePage(w, r, rev, match[1])
req := &request{
w: w,
r: r,
ctx: r.Context(),
ref: ref,
source: source,
}
req.handlePage(match[1])
return
}
handle404(w, r)
}
func logRequest(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) {
result := fmt.Sprintf(format, args...)
glog.Infof("result: %s, remote: %q, ua: %q, referrer: %q, host: %q path: %q", result, r.RemoteAddr, r.Header.Get("User-Agent"), r.Header.Get("Referrer"), r.Host, r.URL.Path)
type request struct {
w http.ResponseWriter
r *http.Request
ctx context.Context
ref string
source source.Source
// rpath is the path requested by the client
rpath string
}
func (r *request) handle500() {
handle500(r.w, r.r)
}
func (r *request) handle404() {
handle404(r.w, r.r)
}
func (r *request) logRequest(format string, args ...interface{}) {
logRequest(r.w, r.r, format, args...)
}
func urlPathToDepotPath(url string) string {
@ -109,70 +189,69 @@ func urlPathToDepotPath(url string) string {
return path
}
func (s *service) handlePageAuto(w http.ResponseWriter, r *http.Request, rev, rpath, dirpath string) {
cfg, err := config.ForPath(s.source, dirpath)
func (r *request) handlePageAuto(dirpath string) {
cfg, err := config.ForPath(r.ctx, r.source, dirpath)
if err != nil {
glog.Errorf("could not get config for path %q: %w", dirpath, err)
handle500(w, r)
r.handle500()
return
}
for _, f := range cfg.DefaultIndex {
fpath := dirpath + f
file, err := s.source.IsFile(fpath)
file, err := r.source.IsFile(r.ctx, fpath)
if err != nil {
glog.Errorf("IsFile(%q): %w", fpath, err)
handle500(w, r)
r.handle500()
return
}
if file {
s.handleMarkdown(w, r, s.source, rev, fpath, cfg)
r.handleMarkdown(fpath, cfg)
return
}
}
handle404(w, r)
r.handle404()
}
func (s *service) handlePage(w http.ResponseWriter, r *http.Request, rev, page string) {
path := urlPathToDepotPath(page)
func (r *request) handlePage(page string) {
r.rpath = urlPathToDepotPath(page)
if strings.HasSuffix(path, "/") {
if strings.HasSuffix(r.rpath, "/") {
// Directory path given, autoresolve.
dirpath := path
if path != "//" {
dirpath = strings.TrimSuffix(path, "/") + "/"
dirpath := r.rpath
if r.rpath != "//" {
dirpath = strings.TrimSuffix(r.rpath, "/") + "/"
}
s.handlePageAuto(w, r, rev, path, dirpath)
r.handlePageAuto(dirpath)
return
}
// Otherwise, try loading the file.
file, err := s.source.IsFile(path)
file, err := r.source.IsFile(r.ctx, r.rpath)
if err != nil {
glog.Errorf("IsFile(%q): %w", path, err)
handle500(w, r)
glog.Errorf("IsFile(%q): %w", r.rpath, err)
r.handle500()
return
}
// File exists, render that.
if file {
parts := strings.Split(path, "/")
parts := strings.Split(r.rpath, "/")
dirpath := strings.Join(parts[:(len(parts)-1)], "/")
cfg, err := config.ForPath(s.source, dirpath)
cfg, err := config.ForPath(r.ctx, r.source, dirpath)
if err != nil {
glog.Errorf("could not get config for path %q: %w", dirpath, err)
handle500(w, r)
r.handle500()
return
}
s.handleMarkdown(w, r, s.source, rev, path, cfg)
r.handleMarkdown(r.rpath, cfg)
return
}
// Otherwise assume directory, try all posibilities.
dirpath := path
if path != "//" {
dirpath = strings.TrimSuffix(path, "/") + "/"
dirpath := r.rpath
if r.rpath != "//" {
dirpath = strings.TrimSuffix(r.rpath, "/") + "/"
}
s.handlePageAuto(w, r, rev, path, dirpath)
r.handlePageAuto(dirpath)
}

50
devtools/hackdoc/markdown.go
View File

@ -1,48 +1,74 @@
package main
import (
"fmt"
"bytes"
"html/template"
"net/http"
"net/url"
"strings"
"code.hackerspace.pl/hscloud/devtools/hackdoc/config"
"code.hackerspace.pl/hscloud/devtools/hackdoc/source"
"github.com/golang/glog"
"gopkg.in/russross/blackfriday.v2"
)
func (s *service) handleMarkdown(w http.ResponseWriter, r *http.Request, src source.Source, branch, path string, cfg *config.Config) {
data, err := src.ReadFile(path)
// renderMarkdown renders markdown to HTML, replacing all relative (intra-hackdoc) links with version that have ref set.
func renderMarkdown(input []byte, ref string) []byte {
r := blackfriday.NewHTMLRenderer(blackfriday.HTMLRendererParameters{
Flags: blackfriday.CommonHTMLFlags,
})
parser := blackfriday.New(blackfriday.WithRenderer(r), blackfriday.WithExtensions(blackfriday.CommonExtensions))
ast := parser.Parse(input)
var buf bytes.Buffer
ast.Walk(func(node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
if ref != "" && entering && node.Type == blackfriday.Link {
dest := string(node.Destination)
u, err := url.Parse(dest)
if err == nil && !u.IsAbs() {
q := u.Query()
q["ref"] = []string{ref}
u.RawQuery = q.Encode()
node.Destination = []byte(u.String())
}
}
return r.RenderNode(&buf, node, entering)
})
return buf.Bytes()
}
func (r *request) handleMarkdown(path string, cfg *config.Config) {
data, err := r.source.ReadFile(r.ctx, path)
if err != nil {
glog.Errorf("ReadFile(%q): %w", err)
handle500(w, r)
r.handle500()
return
}
rendered := blackfriday.Run([]byte(data))
rendered := renderMarkdown([]byte(data), r.ref)
logRequest(w, r, "serving markdown at %s, cfg %+v", path, cfg)
r.logRequest("serving markdown at %s, cfg %+v", path, cfg)
// TODO(q3k): allow markdown files to override which template to load
tmpl, ok := cfg.Templates["default"]
if !ok {
glog.Errorf("No default template found for %s", path)
// TODO(q3k): implement fallback template
w.Write(rendered)
r.w.Write(rendered)
return
}
pathInDepot := strings.TrimPrefix(path, "//")
vars := map[string]interface{}{
"Rendered": template.HTML(rendered),
"Title": path,
"Path": path,
"PathInDepot": strings.TrimPrefix(path, "//"),
"PathInDepot": pathInDepot,
"HackdocURL": flagHackdocURL,
"GitwebURL": fmt.Sprintf(flagGitwebURLPattern, flagGitwebDefaultBranch, strings.TrimPrefix(path, "//")),
"WebLinks": r.source.WebLinks(pathInDepot),
}
err = tmpl.Execute(w, vars)
err = tmpl.Execute(r.w, vars)
if err != nil {
glog.Errorf("Could not execute template for %s: %v", err)
}

3
devtools/hackdoc/source/BUILD.bazel
View File

@ -4,9 +4,10 @@ go_library(
name = "go_default_library",
srcs = [
"source.go",
"source_depotview.go",
"source_local.go",
],
importpath = "code.hackerspace.pl/hscloud/devtools/hackdoc/source",
visibility = ["//visibility:public"],
deps = ["@com_github_golang_glog//:go_default_library"],
deps = ["//devtools/depotview/proto:go_default_library"],
)

38
devtools/hackdoc/source/source.go
View File

@ -1,10 +1,36 @@
package source
type Source interface {
IsFile(path string) (bool, error)
ReadFile(path string) ([]byte, error)
IsDirectory(path string) (bool, error)
import "context"
CacheSet(dependencies []string, key string, value interface{})
CacheGet(key string) interface{}
var (
FlagGitwebURLPattern = "https://gerrit.hackerspace.pl/plugins/gitiles/hscloud/+/%s/%s"
)
type Source interface {
IsFile(ctx context.Context, path string) (bool, error)
ReadFile(ctx context.Context, path string) ([]byte, error)
IsDirectory(ctx context.Context, path string) (bool, error)
WebLinks(fpath string) []WebLink
}
type WebLink struct {
Kind string
LinkLabel string
LinkURL string
}
type SourceProvider interface {
Source(ctx context.Context, rev string) (Source, error)
}
type singleRefProvider struct {
source Source
}
func (s *singleRefProvider) Source(ctx context.Context, rev string) (Source, error) {
return s.source, nil
}
func NewSingleRefProvider(s Source) SourceProvider {
return &singleRefProvider{s}
}

126
devtools/hackdoc/source/source_depotview.go Normal file
View File

@ -0,0 +1,126 @@
package source
import (
"context"
"fmt"
"strconv"
"strings"
dvpb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
)
type DepotViewSourceProvider struct {
stub dvpb.DepotViewClient
}
func NewDepotView(stub dvpb.DepotViewClient) SourceProvider {
return &DepotViewSourceProvider{
stub: stub,
}
}
func changeRef(ref string) int64 {
ref = strings.ToLower(ref)
if !strings.HasPrefix(ref, "change/") && !strings.HasPrefix(ref, "cr/") {
return 0
}
n, err := strconv.ParseInt(strings.SplitN(ref, "/", 2)[1], 10, 64)
if err != nil {
return 0
}
return n
}
func (s *DepotViewSourceProvider) Source(ctx context.Context, ref string) (Source, error) {
var hash string
n := changeRef(ref)
if n != 0 {
res, err := s.stub.ResolveGerritChange(ctx, &dvpb.ResolveGerritChangeRequest{Change: n})
if err != nil {
return nil, err
}
hash = res.Hash
} else {
res, err := s.stub.Resolve(ctx, &dvpb.ResolveRequest{Ref: ref})
if err != nil {
return nil, err
}
hash = res.Hash
}
if hash == "" {
return nil, nil
}
return &depotViewSource{
stub: s.stub,
hash: hash,
change: n,
}, nil
}
type depotViewSource struct {
stub dvpb.DepotViewClient
hash string
change int64
}
func (s *depotViewSource) IsFile(ctx context.Context, path string) (bool, error) {
res, err := s.stub.Stat(ctx, &dvpb.StatRequest{
Hash: s.hash,
Path: path,
})
if err != nil {
return false, err
}
return res.Type == dvpb.StatResponse_TYPE_FILE, nil
}
func (s *depotViewSource) IsDirectory(ctx context.Context, path string) (bool, error) {
res, err := s.stub.Stat(ctx, &dvpb.StatRequest{
Hash: s.hash,
Path: path,
})
if err != nil {
return false, err
}
return res.Type == dvpb.StatResponse_TYPE_DIRECTORY, nil
}
func (s *depotViewSource) ReadFile(ctx context.Context, path string) ([]byte, error) {
var data []byte
srv, err := s.stub.Read(ctx, &dvpb.ReadRequest{
Hash: s.hash,
Path: path,
})
if err != nil {
return nil, err
}
for {
res, err := srv.Recv()
if err != nil {
return nil, err
}
if len(res.Data) == 0 {
break
}
data = append(data, res.Data...)
}
return data, nil
}
func (s *depotViewSource) WebLinks(fpath string) []WebLink {
gitURL := fmt.Sprintf(FlagGitwebURLPattern, s.hash, fpath)
links := []WebLink{
WebLink{Kind: "gitweb", LinkLabel: s.hash[:16], LinkURL: gitURL},
}
if s.change != 0 {
gerritLabel := fmt.Sprintf("change %d", s.change)
gerritLink := fmt.Sprintf("https://gerrit.hackerspace.pl/%d", s.change)
links = append(links, WebLink{Kind: "gerrit", LinkLabel: gerritLabel, LinkURL: gerritLink})
}
return links
}

19
devtools/hackdoc/source/source_local.go
View File

@ -1,6 +1,7 @@
package source
import (
"context"
"fmt"
"io/ioutil"
"os"
@ -29,7 +30,7 @@ func (s *LocalSource) resolve(path string) (string, error) {
return s.root + "/" + path, nil
}
func (s *LocalSource) IsFile(path string) (bool, error) {
func (s *LocalSource) IsFile(ctx context.Context, path string) (bool, error) {
path, err := s.resolve(path)
if err != nil {
return false, err
@ -44,7 +45,7 @@ func (s *LocalSource) IsFile(path string) (bool, error) {
return !stat.IsDir(), nil
}
func (s *LocalSource) ReadFile(path string) ([]byte, error) {
func (s *LocalSource) ReadFile(ctx context.Context, path string) ([]byte, error) {
path, err := s.resolve(path)
if err != nil {
return nil, err
@ -53,7 +54,7 @@ func (s *LocalSource) ReadFile(path string) ([]byte, error) {
return ioutil.ReadFile(path)
}
func (s *LocalSource) IsDirectory(path string) (bool, error) {
func (s *LocalSource) IsDirectory(ctx context.Context, path string) (bool, error) {
path, err := s.resolve(path)
if err != nil {
return false, err
@ -68,11 +69,9 @@ func (s *LocalSource) IsDirectory(path string) (bool, error) {
return stat.IsDir(), nil
}
func (s *LocalSource) CacheSet(dependencies []string, key string, value interface{}) {
// Swallow writes. The local filesystem can always change underneath us and
// we're not tracking anything, so we cannot ever keep caches.
}
func (s *LocalSource) CacheGet(key string) interface{} {
return nil
func (s *LocalSource) WebLinks(fpath string) []WebLink {
gitURL := fmt.Sprintf(FlagGitwebURLPattern, "master", fpath)
return []WebLink{
WebLink{Kind: "gitweb", LinkLabel: "master", LinkURL: gitURL},
}
}

9
devtools/hackdoc/tpl/default.html
View File

@ -85,6 +85,10 @@ body {
color: #b30014;
}
.header span.muted {
color: #666;
}
.footer {
font-size: 0.8em;
color: #ccc;
@ -160,7 +164,10 @@ ul li::before {
<div class="column">
<div class="page">
<div class="header">
<span class="red">hackdoc:</span><span>{{ .Path }}</span> <a href="{{ .GitwebURL }}">[git]</a>
<span class="red">hackdoc:</span><span>{{ .Path }}</span>
{{ range .WebLinks }}
<span class="muted">[{{ .Kind }} <a href="{{ .LinkURL }}">{{ .LinkLabel }}</a>]</span>
{{ end }}
</div>
{{ .Rendered }}
</div>