From f157b4d632b48eedfff8698918f884cc3ed7a082 Mon Sep 17 00:00:00 2001 From: Sergiusz Bazanski Date: Fri, 10 Apr 2020 17:39:43 +0200 Subject: [PATCH] devtools/{depotview,hackdoc}: tie both together Change-Id: I0a1ca3b4fa0e0a074eccbe0f8748839b926db9c1 --- README.md | 2 +- devtools/depotview/proto/depotview.proto | 30 +++- devtools/depotview/service/BUILD.bazel | 6 +- devtools/depotview/service/gerrit.go | 56 +++++++ devtools/depotview/service/service.go | 91 ++++++++-- devtools/hackdoc/BUILD.bazel | 4 + devtools/hackdoc/README.md | 5 + devtools/hackdoc/config/config.go | 27 +-- devtools/hackdoc/helpers.go | 7 + devtools/hackdoc/main.go | 175 ++++++++++++++------ devtools/hackdoc/markdown.go | 50 ++++-- devtools/hackdoc/source/BUILD.bazel | 3 +- devtools/hackdoc/source/source.go | 38 ++++- devtools/hackdoc/source/source_depotview.go | 126 ++++++++++++++ devtools/hackdoc/source/source_local.go | 19 +-- devtools/hackdoc/tpl/default.html | 9 +- 16 files changed, 522 insertions(+), 126 deletions(-) create mode 100644 devtools/depotview/service/gerrit.go create mode 100644 devtools/hackdoc/source/source_depotview.go diff --git a/README.md b/README.md index 7383e891..11a05a7f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Any time you see a `//path/like/this`, it refers to the root of hscloud, ie. the Viewing this documentation -------------------------- -For a please web viewing experience, [see this documentation in hackdoc](https://hackdoc.hackerspace.pl/). This will allow you to read this markdown file (and others) in a pretty, linkable view. +For a pleaseant web viewing experience, [see this documentation in hackdoc](https://hackdoc.hackerspace.pl/). This will allow you to read this markdown file (and others) in a pretty, linkable view. Getting started --------------- diff --git a/devtools/depotview/proto/depotview.proto b/devtools/depotview/proto/depotview.proto index 25849615..b948faed 100644 --- a/devtools/depotview/proto/depotview.proto +++ b/devtools/depotview/proto/depotview.proto @@ -2,6 +2,18 @@ syntax = "proto3"; package depotview; option go_package = "code.hackerspace.pl/hscloud/devtools/depotview/proto"; +service DepotView { + // Resolve a git branch/tag/ref... into a commit hash. + rpc Resolve(ResolveRequest) returns (ResolveResponse); + + // Resolve a gerrit change number into a git commit hash. + rpc ResolveGerritChange(ResolveGerritChangeRequest) returns (ResolveGerritChangeResponse); + + // Minimal file access API. It kinda stinks. + rpc Stat(StatRequest) returns (StatResponse); + rpc Read(ReadRequest) returns (stream ReadResponse); +} + message ResolveRequest { string ref = 1; } @@ -11,6 +23,15 @@ message ResolveResponse { int64 last_checked = 2; } +message ResolveGerritChangeRequest { + int64 change = 1; +} + +message ResolveGerritChangeResponse { + string hash = 1; + int64 last_checked = 2; +} + message StatRequest { string hash = 1; string path = 2; @@ -35,12 +56,3 @@ message ReadResponse { // Chunk of data. Empty once everything has been sent over. bytes data = 1; } - -service DepotView { - // Resolve a git branch/tag/ref... into a commit hash. - rpc Resolve(ResolveRequest) returns (ResolveResponse); - - // Minimal file access API. It kinda stinks. - rpc Stat(StatRequest) returns (StatResponse); - rpc Read(ReadRequest) returns (stream ReadResponse); -} diff --git a/devtools/depotview/service/BUILD.bazel b/devtools/depotview/service/BUILD.bazel index 6f7337ca..056ec307 100644 --- a/devtools/depotview/service/BUILD.bazel +++ b/devtools/depotview/service/BUILD.bazel @@ -2,12 +2,16 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["service.go"], + srcs = [ + "gerrit.go", + "service.go", + ], importpath = "code.hackerspace.pl/hscloud/devtools/depotview/service", visibility = ["//visibility:public"], deps = [ "//devtools/depotview/proto:go_default_library", "@com_github_go_git_go_git_v5//:go_default_library", + "@com_github_go_git_go_git_v5//config:go_default_library", "@com_github_go_git_go_git_v5//plumbing:go_default_library", "@com_github_go_git_go_git_v5//plumbing/filemode:go_default_library", "@com_github_go_git_go_git_v5//plumbing/object:go_default_library", diff --git a/devtools/depotview/service/gerrit.go b/devtools/depotview/service/gerrit.go new file mode 100644 index 00000000..7dab9e63 --- /dev/null +++ b/devtools/depotview/service/gerrit.go @@ -0,0 +1,56 @@ +package service + +import ( + "strconv" + "strings" +) + +type gerritMeta struct { + patchSet int64 + changeId string + commit string +} + +// parseGerritMetadata takes a NoteDB metadata entry and extracts info from it. +func parseGerritMetadata(messages []string) *gerritMeta { + meta := &gerritMeta{} + + for _, message := range messages { + for _, line := range strings.Split(message, "\n") { + line = strings.TrimSpace(line) + if len(line) == 0 { + continue + } + + parts := strings.SplitN(line, ":", 2) + if len(parts) < 2 { + continue + } + k, v := parts[0], strings.TrimSpace(parts[1]) + + switch k { + case "Patch-set": + n, err := strconv.ParseInt(v, 10, 64) + if err != nil { + continue + } + meta.patchSet = n + case "Change-id": + meta.changeId = v + case "Commit": + meta.commit = v + } + } + } + + if meta.patchSet == 0 { + return nil + } + if meta.changeId == "" { + return nil + } + if meta.commit == "" { + return nil + } + return meta +} diff --git a/devtools/depotview/service/service.go b/devtools/depotview/service/service.go index 910bf363..fad20295 100644 --- a/devtools/depotview/service/service.go +++ b/devtools/depotview/service/service.go @@ -2,6 +2,7 @@ package service import ( "context" + "fmt" "io" "regexp" "strings" @@ -13,6 +14,7 @@ import ( "google.golang.org/grpc/status" git "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" @@ -30,9 +32,9 @@ type Service struct { remote string storer storage.Storer - mu sync.Mutex - repo *git.Repository - lastPull time.Time + mu sync.Mutex + repo *git.Repository + lastFetch time.Time } func New(remote string) *Service { @@ -42,10 +44,10 @@ func New(remote string) *Service { } } -func (s *Service) ensureRepo() error { +func (s *Service) ensureRepo(ctx context.Context) error { // Clone repository if necessary. if s.repo == nil { - repo, err := git.Clone(s.storer, nil, &git.CloneOptions{ + repo, err := git.CloneContext(ctx, s.storer, nil, &git.CloneOptions{ URL: s.remote, }) if err != nil { @@ -53,17 +55,24 @@ func (s *Service) ensureRepo() error { return status.Error(codes.Unavailable, "could not clone repository") } s.repo = repo - s.lastPull = time.Now() } // Fetch if necessary. - if time.Since(s.lastPull) > time.Minute { - err := s.repo.Fetch(&git.FetchOptions{}) + if time.Since(s.lastFetch) > 10*time.Second { + glog.Infof("Fetching...") + err := s.repo.FetchContext(ctx, &git.FetchOptions{ + RefSpecs: []config.RefSpec{ + config.RefSpec("+refs/heads/*:refs/remotes/origin/*"), + config.RefSpec("+refs/changes/*:refs/changes/*"), + }, + Force: true, + }) if err != nil && err != git.NoErrAlreadyUpToDate { glog.Errorf("Fetch(): %v", err) } else { - s.lastPull = time.Now() + s.lastFetch = time.Now() } + } return nil @@ -77,22 +86,69 @@ func (s *Service) Resolve(ctx context.Context, req *pb.ResolveRequest) (*pb.Reso return nil, status.Error(codes.InvalidArgument, "ref must be set") } - if err := s.ensureRepo(); err != nil { + if err := s.ensureRepo(ctx); err != nil { return nil, err } h, err := s.repo.ResolveRevision(plumbing.Revision(req.Ref)) switch { case err == plumbing.ErrReferenceNotFound: - return &pb.ResolveResponse{Hash: "", LastChecked: s.lastPull.UnixNano()}, nil + return &pb.ResolveResponse{Hash: "", LastChecked: s.lastFetch.UnixNano()}, nil case err != nil: return nil, status.Errorf(codes.Unavailable, "git resolve error: %v", err) default: - return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastPull.UnixNano()}, nil + return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastFetch.UnixNano()}, nil } } -func (s *Service) getFile(hash, path string, notFoundOkay bool) (*object.File, error) { +func (s *Service) ResolveGerritChange(ctx context.Context, req *pb.ResolveGerritChangeRequest) (*pb.ResolveGerritChangeResponse, error) { + if err := s.ensureRepo(ctx); err != nil { + return nil, err + } + + // I'm totally guessing this, from these examples: + // refs/changes/03/3/meta + // refs/changes/77/77/meta + // refs/changes/47/247/meta + // etc... + shard := fmt.Sprintf("%02d", req.Change%100) + metaRef := fmt.Sprintf("refs/changes/%s/%d/meta", shard, req.Change) + + h, err := s.repo.ResolveRevision(plumbing.Revision(metaRef)) + switch { + case err == plumbing.ErrReferenceNotFound: + return &pb.ResolveGerritChangeResponse{Hash: "", LastChecked: s.lastFetch.UnixNano()}, nil + case err != nil: + return nil, status.Errorf(codes.Unavailable, "git metadata resolve error: %v", err) + } + + c, err := s.repo.CommitObject(*h) + if err != nil { + return nil, status.Errorf(codes.Unavailable, "git error: %v", err) + } + + var messages []string + for { + messages = append([]string{c.Message}, messages...) + + if len(c.ParentHashes) != 1 { + break + } + + c, err = s.repo.CommitObject(c.ParentHashes[0]) + if err != nil { + return nil, status.Errorf(codes.Unavailable, "git error: %v", err) + } + } + + meta := parseGerritMetadata(messages) + if meta == nil { + return nil, status.Errorf(codes.Internal, "could not parse gerrit metadata for ref %q", metaRef) + } + return &pb.ResolveGerritChangeResponse{Hash: meta.commit, LastChecked: s.lastFetch.UnixNano()}, nil +} + +func (s *Service) getFile(ctx context.Context, hash, path string, notFoundOkay bool) (*object.File, error) { if !reHash.MatchString(hash) { return nil, status.Error(codes.InvalidArgument, "hash must be valid full git hash string") } @@ -105,7 +161,7 @@ func (s *Service) getFile(hash, path string, notFoundOkay bool) (*object.File, e return nil, status.Error(codes.InvalidArgument, "path must be a valid unix or depot-style path") } - if err := s.ensureRepo(); err != nil { + if err := s.ensureRepo(ctx); err != nil { return nil, err } @@ -134,7 +190,7 @@ func (s *Service) Stat(ctx context.Context, req *pb.StatRequest) (*pb.StatRespon s.mu.Lock() defer s.mu.Unlock() - file, err := s.getFile(req.Hash, req.Path, true) + file, err := s.getFile(ctx, req.Hash, req.Path, true) if err != nil { return nil, err } @@ -157,7 +213,9 @@ func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error { s.mu.Lock() defer s.mu.Unlock() - file, err := s.getFile(req.Hash, req.Path, false) + ctx := srv.Context() + + file, err := s.getFile(ctx, req.Hash, req.Path, false) if err != nil { return err } @@ -168,7 +226,6 @@ func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error { } defer reader.Close() - ctx := srv.Context() for { if ctx.Err() != nil { return ctx.Err() diff --git a/devtools/hackdoc/BUILD.bazel b/devtools/hackdoc/BUILD.bazel index b29950d6..3988dfe5 100644 --- a/devtools/hackdoc/BUILD.bazel +++ b/devtools/hackdoc/BUILD.bazel @@ -10,10 +10,14 @@ go_library( importpath = "code.hackerspace.pl/hscloud/devtools/hackdoc", visibility = ["//visibility:private"], deps = [ + "//devtools/depotview/proto:go_default_library", "//devtools/hackdoc/config:go_default_library", "//devtools/hackdoc/source:go_default_library", + "//go/mirko:go_default_library", + "//go/pki:go_default_library", "@com_github_golang_glog//:go_default_library", "@in_gopkg_russross_blackfriday_v2//:go_default_library", + "@org_golang_google_grpc//:go_default_library", ], ) diff --git a/devtools/hackdoc/README.md b/devtools/hackdoc/README.md index d3c5187e..45d4486d 100644 --- a/devtools/hackdoc/README.md +++ b/devtools/hackdoc/README.md @@ -8,6 +8,11 @@ Usage Any Markdown submitted to hscloud is visible via hackdoc. Simply go to https://hackdoc.hackerspace.pl/path/to/markdown.md to see it rendered. +You can pass a `?ref=foo` URL parameter to a hackdoc URL to get it to render a particular vesrion of the hscloud monorepo. For example: + +- https://hackdoc.hackerspace.pl/?ref=master for the `master` branch +- https://hackdoc.hackerspace.pl/?ref=change/249 for the the source code at change '249' + Local Rendering --------------- diff --git a/devtools/hackdoc/config/config.go b/devtools/hackdoc/config/config.go index 70b7b469..aba384bc 100644 --- a/devtools/hackdoc/config/config.go +++ b/devtools/hackdoc/config/config.go @@ -1,12 +1,12 @@ package config import ( + "context" "fmt" "html/template" "strings" "github.com/BurntSushi/toml" - "github.com/golang/glog" "code.hackerspace.pl/hscloud/devtools/hackdoc/source" ) @@ -73,24 +73,11 @@ func configFileLocations(path string) []string { return locations } -func ForPath(s source.Source, path string) (*Config, error) { +func ForPath(ctx context.Context, s source.Source, path string) (*Config, error) { if path != "//" { path = strings.TrimRight(path, "/") } - // Try cache. - cacheKey := fmt.Sprintf("config:%s", path) - if v := s.CacheGet(cacheKey); v != nil { - cfg, ok := v.(*Config) - if !ok { - glog.Errorf("Cache key %q corrupted, deleting", cacheKey) - s.CacheSet([]string{}, cacheKey, nil) - } else { - return cfg, nil - } - } - - // Feed cache. cfg := &Config{ Templates: make(map[string]*template.Template), Errors: make(map[string]error), @@ -98,14 +85,14 @@ func ForPath(s source.Source, path string) (*Config, error) { tomlPaths := configFileLocations(path) for _, p := range tomlPaths { - file, err := s.IsFile(p) + file, err := s.IsFile(ctx, p) if err != nil { return nil, fmt.Errorf("IsFile(%q): %w", path, err) } if !file { continue } - data, err := s.ReadFile(p) + data, err := s.ReadFile(ctx, p) if err != nil { return nil, fmt.Errorf("ReadFile(%q): %w", path, err) } @@ -116,7 +103,7 @@ func ForPath(s source.Source, path string) (*Config, error) { continue } - err = cfg.updateFromToml(p, s, c) + err = cfg.updateFromToml(ctx, p, s, c) if err != nil { return nil, fmt.Errorf("updating from %q: %w", p, err) } @@ -125,7 +112,7 @@ func ForPath(s source.Source, path string) (*Config, error) { return cfg, nil } -func (c *Config) updateFromToml(p string, s source.Source, t *configToml) error { +func (c *Config) updateFromToml(ctx context.Context, p string, s source.Source, t *configToml) error { if t.DefaultIndex != nil { c.DefaultIndex = t.DefaultIndex } @@ -134,7 +121,7 @@ func (c *Config) updateFromToml(p string, s source.Source, t *configToml) error tmpl := template.New(k) for _, source := range v.Sources { - data, err := s.ReadFile(source) + data, err := s.ReadFile(ctx, source) if err != nil { c.Errors[p] = fmt.Errorf("reading template file %q: %w", source, err) return nil diff --git a/devtools/hackdoc/helpers.go b/devtools/hackdoc/helpers.go index dd7269ff..a2bd93a9 100644 --- a/devtools/hackdoc/helpers.go +++ b/devtools/hackdoc/helpers.go @@ -3,6 +3,8 @@ package main import ( "fmt" "net/http" + + "github.com/golang/glog" ) func handle404(w http.ResponseWriter, r *http.Request) { @@ -16,3 +18,8 @@ func handle500(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) fmt.Fprintf(w, "500 :(\n") } + +func logRequest(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) { + result := fmt.Sprintf(format, args...) + glog.Infof("result: %s, remote: %q, ua: %q, referrer: %q, host: %q path: %q", result, r.RemoteAddr, r.Header.Get("User-Agent"), r.Header.Get("Referrer"), r.Host, r.URL.Path) +} diff --git a/devtools/hackdoc/main.go b/devtools/hackdoc/main.go index 938a4262..558268b4 100644 --- a/devtools/hackdoc/main.go +++ b/devtools/hackdoc/main.go @@ -1,24 +1,30 @@ package main import ( + "context" "flag" "fmt" "net/http" "path/filepath" "regexp" "strings" + "time" + "code.hackerspace.pl/hscloud/go/mirko" + "code.hackerspace.pl/hscloud/go/pki" "github.com/golang/glog" + "google.golang.org/grpc" + dvpb "code.hackerspace.pl/hscloud/devtools/depotview/proto" "code.hackerspace.pl/hscloud/devtools/hackdoc/config" "code.hackerspace.pl/hscloud/devtools/hackdoc/source" ) var ( flagListen = "127.0.0.1:8080" - flagDocRoot = "./docroot" + flagDocRoot = "" + flagDepotViewAddress = "" flagHackdocURL = "" - flagGitwebURLPattern = "https://gerrit.hackerspace.pl/plugins/gitiles/hscloud/+/%s/%s" flagGitwebDefaultBranch = "master" rePagePath = regexp.MustCompile(`^/([A-Za-z0-9_\-/\. ]*)$`) @@ -29,10 +35,11 @@ func init() { } func main() { - flag.StringVar(&flagListen, "listen", flagListen, "Address to listen on for HTTP traffic") - flag.StringVar(&flagDocRoot, "docroot", flagDocRoot, "Path from which to serve documents") + flag.StringVar(&flagListen, "pub_listen", flagListen, "Address to listen on for HTTP traffic") + flag.StringVar(&flagDocRoot, "docroot", flagDocRoot, "Path from which to serve documents. Either this or depotview must be set") + flag.StringVar(&flagDepotViewAddress, "depotview", flagDepotViewAddress, "gRPC endpoint of depotview to serve from Git. Either this or docroot must be set") flag.StringVar(&flagHackdocURL, "hackdoc_url", flagHackdocURL, "Public URL of hackdoc. If not given, autogenerate from listen path for dev purposes") - flag.StringVar(&flagGitwebURLPattern, "gitweb_url_pattern", flagGitwebURLPattern, "Pattern to sprintf to for URL for viewing a file in Git. First string is ref/rev, second is bare file path (sans //)") + flag.StringVar(&source.FlagGitwebURLPattern, "gitweb_url_pattern", source.FlagGitwebURLPattern, "Pattern to sprintf to for URL for viewing a file in Git. First string is ref/rev, second is bare file path (sans //)") flag.StringVar(&flagGitwebDefaultBranch, "gitweb_default_rev", flagGitwebDefaultBranch, "Default Git rev to render/link to") flag.Parse() @@ -40,25 +47,61 @@ func main() { flagHackdocURL = fmt.Sprintf("http://%s", flagListen) } - path, err := filepath.Abs(flagDocRoot) - if err != nil { - glog.Fatalf("Could not dereference path %q: %w", path, err) + if flagDocRoot == "" && flagDepotViewAddress == "" { + glog.Errorf("Either -docroot or -depotview must be set") + } + if flagDocRoot != "" && flagDepotViewAddress != "" { + glog.Errorf("Only one of -docroot or -depotview must be set") } - s := &service{ - source: source.NewLocal(path), + m := mirko.New() + if err := m.Listen(); err != nil { + glog.Exitf("Listen(): %v", err) } - http.HandleFunc("/", s.handler) + var s *service + if flagDocRoot != "" { + path, err := filepath.Abs(flagDocRoot) + if err != nil { + glog.Exitf("Could not dereference path %q: %w", path, err) + } + glog.Infof("Starting in docroot mode for %q -> %q", flagDocRoot, path) + + s = &service{ + source: source.NewSingleRefProvider(source.NewLocal(path)), + } + } else { + glog.Infof("Starting in depotview mode (server %q)", flagDepotViewAddress) + conn, err := grpc.Dial(flagDepotViewAddress, pki.WithClientHSPKI()) + if err != nil { + glog.Exitf("grpc.Dial(%q): %v", flagDepotViewAddress, err) + } + stub := dvpb.NewDepotViewClient(conn) + s = &service{ + source: source.NewDepotView(stub), + } + } + + mux := http.NewServeMux() + mux.HandleFunc("/", s.handler) + srv := &http.Server{Addr: flagListen, Handler: mux} glog.Infof("Listening on %q...", flagListen) - if err := http.ListenAndServe(flagListen, nil); err != nil { - glog.Fatal(err) - } + go func() { + if err := srv.ListenAndServe(); err != nil { + glog.Error(err) + } + }() + + <-m.Done() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + srv.Shutdown(ctx) + } type service struct { - source source.Source + source source.SourceProvider } func (s *service) handler(w http.ResponseWriter, r *http.Request) { @@ -69,23 +112,60 @@ func (s *service) handler(w http.ResponseWriter, r *http.Request) { } glog.Infof("%+v", r.URL.Query()) - rev := r.URL.Query().Get("rev") - if rev == "" { - rev = flagGitwebDefaultBranch + ref := r.URL.Query().Get("ref") + if ref == "" { + ref = flagGitwebDefaultBranch + } + + ctx := r.Context() + source, err := s.source.Source(ctx, ref) + switch { + case err != nil: + glog.Errorf("Source(%q): %v", ref, err) + handle500(w, r) + return + case source == nil: + handle404(w, r) + return } path := r.URL.Path if match := rePagePath.FindStringSubmatch(path); match != nil { - s.handlePage(w, r, rev, match[1]) + req := &request{ + w: w, + r: r, + ctx: r.Context(), + ref: ref, + source: source, + } + req.handlePage(match[1]) return } handle404(w, r) } -func logRequest(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) { - result := fmt.Sprintf(format, args...) - glog.Infof("result: %s, remote: %q, ua: %q, referrer: %q, host: %q path: %q", result, r.RemoteAddr, r.Header.Get("User-Agent"), r.Header.Get("Referrer"), r.Host, r.URL.Path) +type request struct { + w http.ResponseWriter + r *http.Request + ctx context.Context + + ref string + source source.Source + // rpath is the path requested by the client + rpath string +} + +func (r *request) handle500() { + handle500(r.w, r.r) +} + +func (r *request) handle404() { + handle404(r.w, r.r) +} + +func (r *request) logRequest(format string, args ...interface{}) { + logRequest(r.w, r.r, format, args...) } func urlPathToDepotPath(url string) string { @@ -109,70 +189,69 @@ func urlPathToDepotPath(url string) string { return path } -func (s *service) handlePageAuto(w http.ResponseWriter, r *http.Request, rev, rpath, dirpath string) { - cfg, err := config.ForPath(s.source, dirpath) +func (r *request) handlePageAuto(dirpath string) { + cfg, err := config.ForPath(r.ctx, r.source, dirpath) if err != nil { glog.Errorf("could not get config for path %q: %w", dirpath, err) - handle500(w, r) + r.handle500() return } for _, f := range cfg.DefaultIndex { fpath := dirpath + f - file, err := s.source.IsFile(fpath) + file, err := r.source.IsFile(r.ctx, fpath) if err != nil { glog.Errorf("IsFile(%q): %w", fpath, err) - handle500(w, r) + r.handle500() return } if file { - s.handleMarkdown(w, r, s.source, rev, fpath, cfg) + r.handleMarkdown(fpath, cfg) return } } - - handle404(w, r) + r.handle404() } -func (s *service) handlePage(w http.ResponseWriter, r *http.Request, rev, page string) { - path := urlPathToDepotPath(page) +func (r *request) handlePage(page string) { + r.rpath = urlPathToDepotPath(page) - if strings.HasSuffix(path, "/") { + if strings.HasSuffix(r.rpath, "/") { // Directory path given, autoresolve. - dirpath := path - if path != "//" { - dirpath = strings.TrimSuffix(path, "/") + "/" + dirpath := r.rpath + if r.rpath != "//" { + dirpath = strings.TrimSuffix(r.rpath, "/") + "/" } - s.handlePageAuto(w, r, rev, path, dirpath) + r.handlePageAuto(dirpath) return } // Otherwise, try loading the file. - file, err := s.source.IsFile(path) + file, err := r.source.IsFile(r.ctx, r.rpath) if err != nil { - glog.Errorf("IsFile(%q): %w", path, err) - handle500(w, r) + glog.Errorf("IsFile(%q): %w", r.rpath, err) + r.handle500() return } // File exists, render that. if file { - parts := strings.Split(path, "/") + parts := strings.Split(r.rpath, "/") dirpath := strings.Join(parts[:(len(parts)-1)], "/") - cfg, err := config.ForPath(s.source, dirpath) + cfg, err := config.ForPath(r.ctx, r.source, dirpath) if err != nil { glog.Errorf("could not get config for path %q: %w", dirpath, err) - handle500(w, r) + r.handle500() return } - s.handleMarkdown(w, r, s.source, rev, path, cfg) + r.handleMarkdown(r.rpath, cfg) return } // Otherwise assume directory, try all posibilities. - dirpath := path - if path != "//" { - dirpath = strings.TrimSuffix(path, "/") + "/" + dirpath := r.rpath + if r.rpath != "//" { + dirpath = strings.TrimSuffix(r.rpath, "/") + "/" } - s.handlePageAuto(w, r, rev, path, dirpath) + r.handlePageAuto(dirpath) } diff --git a/devtools/hackdoc/markdown.go b/devtools/hackdoc/markdown.go index da52aa52..911c2c0b 100644 --- a/devtools/hackdoc/markdown.go +++ b/devtools/hackdoc/markdown.go @@ -1,48 +1,74 @@ package main import ( - "fmt" + "bytes" "html/template" - "net/http" + "net/url" "strings" "code.hackerspace.pl/hscloud/devtools/hackdoc/config" - "code.hackerspace.pl/hscloud/devtools/hackdoc/source" "github.com/golang/glog" "gopkg.in/russross/blackfriday.v2" ) -func (s *service) handleMarkdown(w http.ResponseWriter, r *http.Request, src source.Source, branch, path string, cfg *config.Config) { - data, err := src.ReadFile(path) +// renderMarkdown renders markdown to HTML, replacing all relative (intra-hackdoc) links with version that have ref set. +func renderMarkdown(input []byte, ref string) []byte { + r := blackfriday.NewHTMLRenderer(blackfriday.HTMLRendererParameters{ + Flags: blackfriday.CommonHTMLFlags, + }) + + parser := blackfriday.New(blackfriday.WithRenderer(r), blackfriday.WithExtensions(blackfriday.CommonExtensions)) + ast := parser.Parse(input) + + var buf bytes.Buffer + ast.Walk(func(node *blackfriday.Node, entering bool) blackfriday.WalkStatus { + if ref != "" && entering && node.Type == blackfriday.Link { + dest := string(node.Destination) + u, err := url.Parse(dest) + if err == nil && !u.IsAbs() { + q := u.Query() + q["ref"] = []string{ref} + u.RawQuery = q.Encode() + node.Destination = []byte(u.String()) + } + } + return r.RenderNode(&buf, node, entering) + }) + return buf.Bytes() +} + +func (r *request) handleMarkdown(path string, cfg *config.Config) { + data, err := r.source.ReadFile(r.ctx, path) if err != nil { glog.Errorf("ReadFile(%q): %w", err) - handle500(w, r) + r.handle500() return } - rendered := blackfriday.Run([]byte(data)) + rendered := renderMarkdown([]byte(data), r.ref) - logRequest(w, r, "serving markdown at %s, cfg %+v", path, cfg) + r.logRequest("serving markdown at %s, cfg %+v", path, cfg) // TODO(q3k): allow markdown files to override which template to load tmpl, ok := cfg.Templates["default"] if !ok { glog.Errorf("No default template found for %s", path) // TODO(q3k): implement fallback template - w.Write(rendered) + r.w.Write(rendered) return } + pathInDepot := strings.TrimPrefix(path, "//") vars := map[string]interface{}{ "Rendered": template.HTML(rendered), "Title": path, "Path": path, - "PathInDepot": strings.TrimPrefix(path, "//"), + "PathInDepot": pathInDepot, "HackdocURL": flagHackdocURL, - "GitwebURL": fmt.Sprintf(flagGitwebURLPattern, flagGitwebDefaultBranch, strings.TrimPrefix(path, "//")), + "WebLinks": r.source.WebLinks(pathInDepot), } - err = tmpl.Execute(w, vars) + err = tmpl.Execute(r.w, vars) if err != nil { glog.Errorf("Could not execute template for %s: %v", err) } diff --git a/devtools/hackdoc/source/BUILD.bazel b/devtools/hackdoc/source/BUILD.bazel index b896159b..f7f09c62 100644 --- a/devtools/hackdoc/source/BUILD.bazel +++ b/devtools/hackdoc/source/BUILD.bazel @@ -4,9 +4,10 @@ go_library( name = "go_default_library", srcs = [ "source.go", + "source_depotview.go", "source_local.go", ], importpath = "code.hackerspace.pl/hscloud/devtools/hackdoc/source", visibility = ["//visibility:public"], - deps = ["@com_github_golang_glog//:go_default_library"], + deps = ["//devtools/depotview/proto:go_default_library"], ) diff --git a/devtools/hackdoc/source/source.go b/devtools/hackdoc/source/source.go index a79a9209..73d8990b 100644 --- a/devtools/hackdoc/source/source.go +++ b/devtools/hackdoc/source/source.go @@ -1,10 +1,36 @@ package source -type Source interface { - IsFile(path string) (bool, error) - ReadFile(path string) ([]byte, error) - IsDirectory(path string) (bool, error) +import "context" - CacheSet(dependencies []string, key string, value interface{}) - CacheGet(key string) interface{} +var ( + FlagGitwebURLPattern = "https://gerrit.hackerspace.pl/plugins/gitiles/hscloud/+/%s/%s" +) + +type Source interface { + IsFile(ctx context.Context, path string) (bool, error) + ReadFile(ctx context.Context, path string) ([]byte, error) + IsDirectory(ctx context.Context, path string) (bool, error) + WebLinks(fpath string) []WebLink +} + +type WebLink struct { + Kind string + LinkLabel string + LinkURL string +} + +type SourceProvider interface { + Source(ctx context.Context, rev string) (Source, error) +} + +type singleRefProvider struct { + source Source +} + +func (s *singleRefProvider) Source(ctx context.Context, rev string) (Source, error) { + return s.source, nil +} + +func NewSingleRefProvider(s Source) SourceProvider { + return &singleRefProvider{s} } diff --git a/devtools/hackdoc/source/source_depotview.go b/devtools/hackdoc/source/source_depotview.go new file mode 100644 index 00000000..6a256be2 --- /dev/null +++ b/devtools/hackdoc/source/source_depotview.go @@ -0,0 +1,126 @@ +package source + +import ( + "context" + "fmt" + "strconv" + "strings" + + dvpb "code.hackerspace.pl/hscloud/devtools/depotview/proto" +) + +type DepotViewSourceProvider struct { + stub dvpb.DepotViewClient +} + +func NewDepotView(stub dvpb.DepotViewClient) SourceProvider { + return &DepotViewSourceProvider{ + stub: stub, + } +} + +func changeRef(ref string) int64 { + ref = strings.ToLower(ref) + if !strings.HasPrefix(ref, "change/") && !strings.HasPrefix(ref, "cr/") { + return 0 + } + n, err := strconv.ParseInt(strings.SplitN(ref, "/", 2)[1], 10, 64) + if err != nil { + return 0 + } + + return n +} + +func (s *DepotViewSourceProvider) Source(ctx context.Context, ref string) (Source, error) { + var hash string + n := changeRef(ref) + if n != 0 { + res, err := s.stub.ResolveGerritChange(ctx, &dvpb.ResolveGerritChangeRequest{Change: n}) + if err != nil { + return nil, err + } + hash = res.Hash + } else { + res, err := s.stub.Resolve(ctx, &dvpb.ResolveRequest{Ref: ref}) + if err != nil { + return nil, err + } + hash = res.Hash + } + + if hash == "" { + return nil, nil + } + + return &depotViewSource{ + stub: s.stub, + hash: hash, + change: n, + }, nil +} + +type depotViewSource struct { + stub dvpb.DepotViewClient + hash string + change int64 +} + +func (s *depotViewSource) IsFile(ctx context.Context, path string) (bool, error) { + res, err := s.stub.Stat(ctx, &dvpb.StatRequest{ + Hash: s.hash, + Path: path, + }) + if err != nil { + return false, err + } + return res.Type == dvpb.StatResponse_TYPE_FILE, nil +} + +func (s *depotViewSource) IsDirectory(ctx context.Context, path string) (bool, error) { + res, err := s.stub.Stat(ctx, &dvpb.StatRequest{ + Hash: s.hash, + Path: path, + }) + if err != nil { + return false, err + } + return res.Type == dvpb.StatResponse_TYPE_DIRECTORY, nil +} + +func (s *depotViewSource) ReadFile(ctx context.Context, path string) ([]byte, error) { + var data []byte + srv, err := s.stub.Read(ctx, &dvpb.ReadRequest{ + Hash: s.hash, + Path: path, + }) + if err != nil { + return nil, err + } + for { + res, err := srv.Recv() + if err != nil { + return nil, err + } + if len(res.Data) == 0 { + break + } + data = append(data, res.Data...) + } + return data, nil +} + +func (s *depotViewSource) WebLinks(fpath string) []WebLink { + gitURL := fmt.Sprintf(FlagGitwebURLPattern, s.hash, fpath) + links := []WebLink{ + WebLink{Kind: "gitweb", LinkLabel: s.hash[:16], LinkURL: gitURL}, + } + + if s.change != 0 { + gerritLabel := fmt.Sprintf("change %d", s.change) + gerritLink := fmt.Sprintf("https://gerrit.hackerspace.pl/%d", s.change) + links = append(links, WebLink{Kind: "gerrit", LinkLabel: gerritLabel, LinkURL: gerritLink}) + } + + return links +} diff --git a/devtools/hackdoc/source/source_local.go b/devtools/hackdoc/source/source_local.go index 35ad1722..feecd8b7 100644 --- a/devtools/hackdoc/source/source_local.go +++ b/devtools/hackdoc/source/source_local.go @@ -1,6 +1,7 @@ package source import ( + "context" "fmt" "io/ioutil" "os" @@ -29,7 +30,7 @@ func (s *LocalSource) resolve(path string) (string, error) { return s.root + "/" + path, nil } -func (s *LocalSource) IsFile(path string) (bool, error) { +func (s *LocalSource) IsFile(ctx context.Context, path string) (bool, error) { path, err := s.resolve(path) if err != nil { return false, err @@ -44,7 +45,7 @@ func (s *LocalSource) IsFile(path string) (bool, error) { return !stat.IsDir(), nil } -func (s *LocalSource) ReadFile(path string) ([]byte, error) { +func (s *LocalSource) ReadFile(ctx context.Context, path string) ([]byte, error) { path, err := s.resolve(path) if err != nil { return nil, err @@ -53,7 +54,7 @@ func (s *LocalSource) ReadFile(path string) ([]byte, error) { return ioutil.ReadFile(path) } -func (s *LocalSource) IsDirectory(path string) (bool, error) { +func (s *LocalSource) IsDirectory(ctx context.Context, path string) (bool, error) { path, err := s.resolve(path) if err != nil { return false, err @@ -68,11 +69,9 @@ func (s *LocalSource) IsDirectory(path string) (bool, error) { return stat.IsDir(), nil } -func (s *LocalSource) CacheSet(dependencies []string, key string, value interface{}) { - // Swallow writes. The local filesystem can always change underneath us and - // we're not tracking anything, so we cannot ever keep caches. -} - -func (s *LocalSource) CacheGet(key string) interface{} { - return nil +func (s *LocalSource) WebLinks(fpath string) []WebLink { + gitURL := fmt.Sprintf(FlagGitwebURLPattern, "master", fpath) + return []WebLink{ + WebLink{Kind: "gitweb", LinkLabel: "master", LinkURL: gitURL}, + } } diff --git a/devtools/hackdoc/tpl/default.html b/devtools/hackdoc/tpl/default.html index 3bfe62a0..71209627 100644 --- a/devtools/hackdoc/tpl/default.html +++ b/devtools/hackdoc/tpl/default.html @@ -85,6 +85,10 @@ body { color: #b30014; } +.header span.muted { + color: #666; +} + .footer { font-size: 0.8em; color: #ccc; @@ -160,7 +164,10 @@ ul li::before {
- hackdoc:{{ .Path }} [git] + hackdoc:{{ .Path }} + {{ range .WebLinks }} + [{{ .Kind }} {{ .LinkLabel }}] + {{ end }}
{{ .Rendered }}