forked from hswaw/hscloud
devtools/{depotview,hackdoc}: tie both together
Change-Id: I0a1ca3b4fa0e0a074eccbe0f8748839b926db9c1
This commit is contained in:
parent
4c0e9b52c0
commit
f157b4d632
16 changed files with 522 additions and 126 deletions
|
@ -9,7 +9,7 @@ Any time you see a `//path/like/this`, it refers to the root of hscloud, ie. the
|
|||
Viewing this documentation
|
||||
--------------------------
|
||||
|
||||
For a please web viewing experience, [see this documentation in hackdoc](https://hackdoc.hackerspace.pl/). This will allow you to read this markdown file (and others) in a pretty, linkable view.
|
||||
For a pleaseant web viewing experience, [see this documentation in hackdoc](https://hackdoc.hackerspace.pl/). This will allow you to read this markdown file (and others) in a pretty, linkable view.
|
||||
|
||||
Getting started
|
||||
---------------
|
||||
|
|
|
@ -2,6 +2,18 @@ syntax = "proto3";
|
|||
package depotview;
|
||||
option go_package = "code.hackerspace.pl/hscloud/devtools/depotview/proto";
|
||||
|
||||
service DepotView {
|
||||
// Resolve a git branch/tag/ref... into a commit hash.
|
||||
rpc Resolve(ResolveRequest) returns (ResolveResponse);
|
||||
|
||||
// Resolve a gerrit change number into a git commit hash.
|
||||
rpc ResolveGerritChange(ResolveGerritChangeRequest) returns (ResolveGerritChangeResponse);
|
||||
|
||||
// Minimal file access API. It kinda stinks.
|
||||
rpc Stat(StatRequest) returns (StatResponse);
|
||||
rpc Read(ReadRequest) returns (stream ReadResponse);
|
||||
}
|
||||
|
||||
message ResolveRequest {
|
||||
string ref = 1;
|
||||
}
|
||||
|
@ -11,6 +23,15 @@ message ResolveResponse {
|
|||
int64 last_checked = 2;
|
||||
}
|
||||
|
||||
message ResolveGerritChangeRequest {
|
||||
int64 change = 1;
|
||||
}
|
||||
|
||||
message ResolveGerritChangeResponse {
|
||||
string hash = 1;
|
||||
int64 last_checked = 2;
|
||||
}
|
||||
|
||||
message StatRequest {
|
||||
string hash = 1;
|
||||
string path = 2;
|
||||
|
@ -35,12 +56,3 @@ message ReadResponse {
|
|||
// Chunk of data. Empty once everything has been sent over.
|
||||
bytes data = 1;
|
||||
}
|
||||
|
||||
service DepotView {
|
||||
// Resolve a git branch/tag/ref... into a commit hash.
|
||||
rpc Resolve(ResolveRequest) returns (ResolveResponse);
|
||||
|
||||
// Minimal file access API. It kinda stinks.
|
||||
rpc Stat(StatRequest) returns (StatResponse);
|
||||
rpc Read(ReadRequest) returns (stream ReadResponse);
|
||||
}
|
||||
|
|
|
@ -2,12 +2,16 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
|||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["service.go"],
|
||||
srcs = [
|
||||
"gerrit.go",
|
||||
"service.go",
|
||||
],
|
||||
importpath = "code.hackerspace.pl/hscloud/devtools/depotview/service",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//devtools/depotview/proto:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//config:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//plumbing:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//plumbing/filemode:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//plumbing/object:go_default_library",
|
||||
|
|
56
devtools/depotview/service/gerrit.go
Normal file
56
devtools/depotview/service/gerrit.go
Normal file
|
@ -0,0 +1,56 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type gerritMeta struct {
|
||||
patchSet int64
|
||||
changeId string
|
||||
commit string
|
||||
}
|
||||
|
||||
// parseGerritMetadata takes a NoteDB metadata entry and extracts info from it.
|
||||
func parseGerritMetadata(messages []string) *gerritMeta {
|
||||
meta := &gerritMeta{}
|
||||
|
||||
for _, message := range messages {
|
||||
for _, line := range strings.Split(message, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
parts := strings.SplitN(line, ":", 2)
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
k, v := parts[0], strings.TrimSpace(parts[1])
|
||||
|
||||
switch k {
|
||||
case "Patch-set":
|
||||
n, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
meta.patchSet = n
|
||||
case "Change-id":
|
||||
meta.changeId = v
|
||||
case "Commit":
|
||||
meta.commit = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if meta.patchSet == 0 {
|
||||
return nil
|
||||
}
|
||||
if meta.changeId == "" {
|
||||
return nil
|
||||
}
|
||||
if meta.commit == "" {
|
||||
return nil
|
||||
}
|
||||
return meta
|
||||
}
|
|
@ -2,6 +2,7 @@ package service
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
@ -13,6 +14,7 @@ import (
|
|||
"google.golang.org/grpc/status"
|
||||
|
||||
git "github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/filemode"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
|
@ -30,9 +32,9 @@ type Service struct {
|
|||
remote string
|
||||
storer storage.Storer
|
||||
|
||||
mu sync.Mutex
|
||||
repo *git.Repository
|
||||
lastPull time.Time
|
||||
mu sync.Mutex
|
||||
repo *git.Repository
|
||||
lastFetch time.Time
|
||||
}
|
||||
|
||||
func New(remote string) *Service {
|
||||
|
@ -42,10 +44,10 @@ func New(remote string) *Service {
|
|||
}
|
||||
}
|
||||
|
||||
func (s *Service) ensureRepo() error {
|
||||
func (s *Service) ensureRepo(ctx context.Context) error {
|
||||
// Clone repository if necessary.
|
||||
if s.repo == nil {
|
||||
repo, err := git.Clone(s.storer, nil, &git.CloneOptions{
|
||||
repo, err := git.CloneContext(ctx, s.storer, nil, &git.CloneOptions{
|
||||
URL: s.remote,
|
||||
})
|
||||
if err != nil {
|
||||
|
@ -53,17 +55,24 @@ func (s *Service) ensureRepo() error {
|
|||
return status.Error(codes.Unavailable, "could not clone repository")
|
||||
}
|
||||
s.repo = repo
|
||||
s.lastPull = time.Now()
|
||||
}
|
||||
|
||||
// Fetch if necessary.
|
||||
if time.Since(s.lastPull) > time.Minute {
|
||||
err := s.repo.Fetch(&git.FetchOptions{})
|
||||
if time.Since(s.lastFetch) > 10*time.Second {
|
||||
glog.Infof("Fetching...")
|
||||
err := s.repo.FetchContext(ctx, &git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{
|
||||
config.RefSpec("+refs/heads/*:refs/remotes/origin/*"),
|
||||
config.RefSpec("+refs/changes/*:refs/changes/*"),
|
||||
},
|
||||
Force: true,
|
||||
})
|
||||
if err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
glog.Errorf("Fetch(): %v", err)
|
||||
} else {
|
||||
s.lastPull = time.Now()
|
||||
s.lastFetch = time.Now()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
|
@ -77,22 +86,69 @@ func (s *Service) Resolve(ctx context.Context, req *pb.ResolveRequest) (*pb.Reso
|
|||
return nil, status.Error(codes.InvalidArgument, "ref must be set")
|
||||
}
|
||||
|
||||
if err := s.ensureRepo(); err != nil {
|
||||
if err := s.ensureRepo(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
h, err := s.repo.ResolveRevision(plumbing.Revision(req.Ref))
|
||||
switch {
|
||||
case err == plumbing.ErrReferenceNotFound:
|
||||
return &pb.ResolveResponse{Hash: "", LastChecked: s.lastPull.UnixNano()}, nil
|
||||
return &pb.ResolveResponse{Hash: "", LastChecked: s.lastFetch.UnixNano()}, nil
|
||||
case err != nil:
|
||||
return nil, status.Errorf(codes.Unavailable, "git resolve error: %v", err)
|
||||
default:
|
||||
return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastPull.UnixNano()}, nil
|
||||
return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastFetch.UnixNano()}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) getFile(hash, path string, notFoundOkay bool) (*object.File, error) {
|
||||
func (s *Service) ResolveGerritChange(ctx context.Context, req *pb.ResolveGerritChangeRequest) (*pb.ResolveGerritChangeResponse, error) {
|
||||
if err := s.ensureRepo(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// I'm totally guessing this, from these examples:
|
||||
// refs/changes/03/3/meta
|
||||
// refs/changes/77/77/meta
|
||||
// refs/changes/47/247/meta
|
||||
// etc...
|
||||
shard := fmt.Sprintf("%02d", req.Change%100)
|
||||
metaRef := fmt.Sprintf("refs/changes/%s/%d/meta", shard, req.Change)
|
||||
|
||||
h, err := s.repo.ResolveRevision(plumbing.Revision(metaRef))
|
||||
switch {
|
||||
case err == plumbing.ErrReferenceNotFound:
|
||||
return &pb.ResolveGerritChangeResponse{Hash: "", LastChecked: s.lastFetch.UnixNano()}, nil
|
||||
case err != nil:
|
||||
return nil, status.Errorf(codes.Unavailable, "git metadata resolve error: %v", err)
|
||||
}
|
||||
|
||||
c, err := s.repo.CommitObject(*h)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
|
||||
}
|
||||
|
||||
var messages []string
|
||||
for {
|
||||
messages = append([]string{c.Message}, messages...)
|
||||
|
||||
if len(c.ParentHashes) != 1 {
|
||||
break
|
||||
}
|
||||
|
||||
c, err = s.repo.CommitObject(c.ParentHashes[0])
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
meta := parseGerritMetadata(messages)
|
||||
if meta == nil {
|
||||
return nil, status.Errorf(codes.Internal, "could not parse gerrit metadata for ref %q", metaRef)
|
||||
}
|
||||
return &pb.ResolveGerritChangeResponse{Hash: meta.commit, LastChecked: s.lastFetch.UnixNano()}, nil
|
||||
}
|
||||
|
||||
func (s *Service) getFile(ctx context.Context, hash, path string, notFoundOkay bool) (*object.File, error) {
|
||||
if !reHash.MatchString(hash) {
|
||||
return nil, status.Error(codes.InvalidArgument, "hash must be valid full git hash string")
|
||||
}
|
||||
|
@ -105,7 +161,7 @@ func (s *Service) getFile(hash, path string, notFoundOkay bool) (*object.File, e
|
|||
return nil, status.Error(codes.InvalidArgument, "path must be a valid unix or depot-style path")
|
||||
}
|
||||
|
||||
if err := s.ensureRepo(); err != nil {
|
||||
if err := s.ensureRepo(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
@ -134,7 +190,7 @@ func (s *Service) Stat(ctx context.Context, req *pb.StatRequest) (*pb.StatRespon
|
|||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
file, err := s.getFile(req.Hash, req.Path, true)
|
||||
file, err := s.getFile(ctx, req.Hash, req.Path, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -157,7 +213,9 @@ func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error {
|
|||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
file, err := s.getFile(req.Hash, req.Path, false)
|
||||
ctx := srv.Context()
|
||||
|
||||
file, err := s.getFile(ctx, req.Hash, req.Path, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -168,7 +226,6 @@ func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error {
|
|||
}
|
||||
defer reader.Close()
|
||||
|
||||
ctx := srv.Context()
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
|
|
|
@ -10,10 +10,14 @@ go_library(
|
|||
importpath = "code.hackerspace.pl/hscloud/devtools/hackdoc",
|
||||
visibility = ["//visibility:private"],
|
||||
deps = [
|
||||
"//devtools/depotview/proto:go_default_library",
|
||||
"//devtools/hackdoc/config:go_default_library",
|
||||
"//devtools/hackdoc/source:go_default_library",
|
||||
"//go/mirko:go_default_library",
|
||||
"//go/pki:go_default_library",
|
||||
"@com_github_golang_glog//:go_default_library",
|
||||
"@in_gopkg_russross_blackfriday_v2//:go_default_library",
|
||||
"@org_golang_google_grpc//:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
@ -8,6 +8,11 @@ Usage
|
|||
|
||||
Any Markdown submitted to hscloud is visible via hackdoc. Simply go to https://hackdoc.hackerspace.pl/path/to/markdown.md to see it rendered.
|
||||
|
||||
You can pass a `?ref=foo` URL parameter to a hackdoc URL to get it to render a particular vesrion of the hscloud monorepo. For example:
|
||||
|
||||
- https://hackdoc.hackerspace.pl/?ref=master for the `master` branch
|
||||
- https://hackdoc.hackerspace.pl/?ref=change/249 for the the source code at change '249'
|
||||
|
||||
Local Rendering
|
||||
---------------
|
||||
|
||||
|
|
|
@ -1,12 +1,12 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"strings"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/golang/glog"
|
||||
|
||||
"code.hackerspace.pl/hscloud/devtools/hackdoc/source"
|
||||
)
|
||||
|
@ -73,24 +73,11 @@ func configFileLocations(path string) []string {
|
|||
return locations
|
||||
}
|
||||
|
||||
func ForPath(s source.Source, path string) (*Config, error) {
|
||||
func ForPath(ctx context.Context, s source.Source, path string) (*Config, error) {
|
||||
if path != "//" {
|
||||
path = strings.TrimRight(path, "/")
|
||||
}
|
||||
|
||||
// Try cache.
|
||||
cacheKey := fmt.Sprintf("config:%s", path)
|
||||
if v := s.CacheGet(cacheKey); v != nil {
|
||||
cfg, ok := v.(*Config)
|
||||
if !ok {
|
||||
glog.Errorf("Cache key %q corrupted, deleting", cacheKey)
|
||||
s.CacheSet([]string{}, cacheKey, nil)
|
||||
} else {
|
||||
return cfg, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Feed cache.
|
||||
cfg := &Config{
|
||||
Templates: make(map[string]*template.Template),
|
||||
Errors: make(map[string]error),
|
||||
|
@ -98,14 +85,14 @@ func ForPath(s source.Source, path string) (*Config, error) {
|
|||
|
||||
tomlPaths := configFileLocations(path)
|
||||
for _, p := range tomlPaths {
|
||||
file, err := s.IsFile(p)
|
||||
file, err := s.IsFile(ctx, p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("IsFile(%q): %w", path, err)
|
||||
}
|
||||
if !file {
|
||||
continue
|
||||
}
|
||||
data, err := s.ReadFile(p)
|
||||
data, err := s.ReadFile(ctx, p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ReadFile(%q): %w", path, err)
|
||||
}
|
||||
|
@ -116,7 +103,7 @@ func ForPath(s source.Source, path string) (*Config, error) {
|
|||
continue
|
||||
}
|
||||
|
||||
err = cfg.updateFromToml(p, s, c)
|
||||
err = cfg.updateFromToml(ctx, p, s, c)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("updating from %q: %w", p, err)
|
||||
}
|
||||
|
@ -125,7 +112,7 @@ func ForPath(s source.Source, path string) (*Config, error) {
|
|||
return cfg, nil
|
||||
}
|
||||
|
||||
func (c *Config) updateFromToml(p string, s source.Source, t *configToml) error {
|
||||
func (c *Config) updateFromToml(ctx context.Context, p string, s source.Source, t *configToml) error {
|
||||
if t.DefaultIndex != nil {
|
||||
c.DefaultIndex = t.DefaultIndex
|
||||
}
|
||||
|
@ -134,7 +121,7 @@ func (c *Config) updateFromToml(p string, s source.Source, t *configToml) error
|
|||
tmpl := template.New(k)
|
||||
|
||||
for _, source := range v.Sources {
|
||||
data, err := s.ReadFile(source)
|
||||
data, err := s.ReadFile(ctx, source)
|
||||
if err != nil {
|
||||
c.Errors[p] = fmt.Errorf("reading template file %q: %w", source, err)
|
||||
return nil
|
||||
|
|
|
@ -3,6 +3,8 @@ package main
|
|||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/golang/glog"
|
||||
)
|
||||
|
||||
func handle404(w http.ResponseWriter, r *http.Request) {
|
||||
|
@ -16,3 +18,8 @@ func handle500(w http.ResponseWriter, r *http.Request) {
|
|||
w.WriteHeader(http.StatusNotFound)
|
||||
fmt.Fprintf(w, "500 :(\n")
|
||||
}
|
||||
|
||||
func logRequest(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) {
|
||||
result := fmt.Sprintf(format, args...)
|
||||
glog.Infof("result: %s, remote: %q, ua: %q, referrer: %q, host: %q path: %q", result, r.RemoteAddr, r.Header.Get("User-Agent"), r.Header.Get("Referrer"), r.Host, r.URL.Path)
|
||||
}
|
||||
|
|
|
@ -1,24 +1,30 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.hackerspace.pl/hscloud/go/mirko"
|
||||
"code.hackerspace.pl/hscloud/go/pki"
|
||||
"github.com/golang/glog"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
dvpb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
|
||||
"code.hackerspace.pl/hscloud/devtools/hackdoc/config"
|
||||
"code.hackerspace.pl/hscloud/devtools/hackdoc/source"
|
||||
)
|
||||
|
||||
var (
|
||||
flagListen = "127.0.0.1:8080"
|
||||
flagDocRoot = "./docroot"
|
||||
flagDocRoot = ""
|
||||
flagDepotViewAddress = ""
|
||||
flagHackdocURL = ""
|
||||
flagGitwebURLPattern = "https://gerrit.hackerspace.pl/plugins/gitiles/hscloud/+/%s/%s"
|
||||
flagGitwebDefaultBranch = "master"
|
||||
|
||||
rePagePath = regexp.MustCompile(`^/([A-Za-z0-9_\-/\. ]*)$`)
|
||||
|
@ -29,10 +35,11 @@ func init() {
|
|||
}
|
||||
|
||||
func main() {
|
||||
flag.StringVar(&flagListen, "listen", flagListen, "Address to listen on for HTTP traffic")
|
||||
flag.StringVar(&flagDocRoot, "docroot", flagDocRoot, "Path from which to serve documents")
|
||||
flag.StringVar(&flagListen, "pub_listen", flagListen, "Address to listen on for HTTP traffic")
|
||||
flag.StringVar(&flagDocRoot, "docroot", flagDocRoot, "Path from which to serve documents. Either this or depotview must be set")
|
||||
flag.StringVar(&flagDepotViewAddress, "depotview", flagDepotViewAddress, "gRPC endpoint of depotview to serve from Git. Either this or docroot must be set")
|
||||
flag.StringVar(&flagHackdocURL, "hackdoc_url", flagHackdocURL, "Public URL of hackdoc. If not given, autogenerate from listen path for dev purposes")
|
||||
flag.StringVar(&flagGitwebURLPattern, "gitweb_url_pattern", flagGitwebURLPattern, "Pattern to sprintf to for URL for viewing a file in Git. First string is ref/rev, second is bare file path (sans //)")
|
||||
flag.StringVar(&source.FlagGitwebURLPattern, "gitweb_url_pattern", source.FlagGitwebURLPattern, "Pattern to sprintf to for URL for viewing a file in Git. First string is ref/rev, second is bare file path (sans //)")
|
||||
flag.StringVar(&flagGitwebDefaultBranch, "gitweb_default_rev", flagGitwebDefaultBranch, "Default Git rev to render/link to")
|
||||
flag.Parse()
|
||||
|
||||
|
@ -40,25 +47,61 @@ func main() {
|
|||
flagHackdocURL = fmt.Sprintf("http://%s", flagListen)
|
||||
}
|
||||
|
||||
path, err := filepath.Abs(flagDocRoot)
|
||||
if err != nil {
|
||||
glog.Fatalf("Could not dereference path %q: %w", path, err)
|
||||
if flagDocRoot == "" && flagDepotViewAddress == "" {
|
||||
glog.Errorf("Either -docroot or -depotview must be set")
|
||||
}
|
||||
if flagDocRoot != "" && flagDepotViewAddress != "" {
|
||||
glog.Errorf("Only one of -docroot or -depotview must be set")
|
||||
}
|
||||
|
||||
s := &service{
|
||||
source: source.NewLocal(path),
|
||||
m := mirko.New()
|
||||
if err := m.Listen(); err != nil {
|
||||
glog.Exitf("Listen(): %v", err)
|
||||
}
|
||||
|
||||
http.HandleFunc("/", s.handler)
|
||||
var s *service
|
||||
if flagDocRoot != "" {
|
||||
path, err := filepath.Abs(flagDocRoot)
|
||||
if err != nil {
|
||||
glog.Exitf("Could not dereference path %q: %w", path, err)
|
||||
}
|
||||
glog.Infof("Starting in docroot mode for %q -> %q", flagDocRoot, path)
|
||||
|
||||
s = &service{
|
||||
source: source.NewSingleRefProvider(source.NewLocal(path)),
|
||||
}
|
||||
} else {
|
||||
glog.Infof("Starting in depotview mode (server %q)", flagDepotViewAddress)
|
||||
conn, err := grpc.Dial(flagDepotViewAddress, pki.WithClientHSPKI())
|
||||
if err != nil {
|
||||
glog.Exitf("grpc.Dial(%q): %v", flagDepotViewAddress, err)
|
||||
}
|
||||
stub := dvpb.NewDepotViewClient(conn)
|
||||
s = &service{
|
||||
source: source.NewDepotView(stub),
|
||||
}
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/", s.handler)
|
||||
srv := &http.Server{Addr: flagListen, Handler: mux}
|
||||
|
||||
glog.Infof("Listening on %q...", flagListen)
|
||||
if err := http.ListenAndServe(flagListen, nil); err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
go func() {
|
||||
if err := srv.ListenAndServe(); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
<-m.Done()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
srv.Shutdown(ctx)
|
||||
|
||||
}
|
||||
|
||||
type service struct {
|
||||
source source.Source
|
||||
source source.SourceProvider
|
||||
}
|
||||
|
||||
func (s *service) handler(w http.ResponseWriter, r *http.Request) {
|
||||
|
@ -69,23 +112,60 @@ func (s *service) handler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
glog.Infof("%+v", r.URL.Query())
|
||||
rev := r.URL.Query().Get("rev")
|
||||
if rev == "" {
|
||||
rev = flagGitwebDefaultBranch
|
||||
ref := r.URL.Query().Get("ref")
|
||||
if ref == "" {
|
||||
ref = flagGitwebDefaultBranch
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
source, err := s.source.Source(ctx, ref)
|
||||
switch {
|
||||
case err != nil:
|
||||
glog.Errorf("Source(%q): %v", ref, err)
|
||||
handle500(w, r)
|
||||
return
|
||||
case source == nil:
|
||||
handle404(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
path := r.URL.Path
|
||||
|
||||
if match := rePagePath.FindStringSubmatch(path); match != nil {
|
||||
s.handlePage(w, r, rev, match[1])
|
||||
req := &request{
|
||||
w: w,
|
||||
r: r,
|
||||
ctx: r.Context(),
|
||||
ref: ref,
|
||||
source: source,
|
||||
}
|
||||
req.handlePage(match[1])
|
||||
return
|
||||
}
|
||||
handle404(w, r)
|
||||
}
|
||||
|
||||
func logRequest(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) {
|
||||
result := fmt.Sprintf(format, args...)
|
||||
glog.Infof("result: %s, remote: %q, ua: %q, referrer: %q, host: %q path: %q", result, r.RemoteAddr, r.Header.Get("User-Agent"), r.Header.Get("Referrer"), r.Host, r.URL.Path)
|
||||
type request struct {
|
||||
w http.ResponseWriter
|
||||
r *http.Request
|
||||
ctx context.Context
|
||||
|
||||
ref string
|
||||
source source.Source
|
||||
// rpath is the path requested by the client
|
||||
rpath string
|
||||
}
|
||||
|
||||
func (r *request) handle500() {
|
||||
handle500(r.w, r.r)
|
||||
}
|
||||
|
||||
func (r *request) handle404() {
|
||||
handle404(r.w, r.r)
|
||||
}
|
||||
|
||||
func (r *request) logRequest(format string, args ...interface{}) {
|
||||
logRequest(r.w, r.r, format, args...)
|
||||
}
|
||||
|
||||
func urlPathToDepotPath(url string) string {
|
||||
|
@ -109,70 +189,69 @@ func urlPathToDepotPath(url string) string {
|
|||
return path
|
||||
}
|
||||
|
||||
func (s *service) handlePageAuto(w http.ResponseWriter, r *http.Request, rev, rpath, dirpath string) {
|
||||
cfg, err := config.ForPath(s.source, dirpath)
|
||||
func (r *request) handlePageAuto(dirpath string) {
|
||||
cfg, err := config.ForPath(r.ctx, r.source, dirpath)
|
||||
if err != nil {
|
||||
glog.Errorf("could not get config for path %q: %w", dirpath, err)
|
||||
handle500(w, r)
|
||||
r.handle500()
|
||||
return
|
||||
}
|
||||
for _, f := range cfg.DefaultIndex {
|
||||
fpath := dirpath + f
|
||||
file, err := s.source.IsFile(fpath)
|
||||
file, err := r.source.IsFile(r.ctx, fpath)
|
||||
if err != nil {
|
||||
glog.Errorf("IsFile(%q): %w", fpath, err)
|
||||
handle500(w, r)
|
||||
r.handle500()
|
||||
return
|
||||
}
|
||||
|
||||
if file {
|
||||
s.handleMarkdown(w, r, s.source, rev, fpath, cfg)
|
||||
r.handleMarkdown(fpath, cfg)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
handle404(w, r)
|
||||
r.handle404()
|
||||
}
|
||||
|
||||
func (s *service) handlePage(w http.ResponseWriter, r *http.Request, rev, page string) {
|
||||
path := urlPathToDepotPath(page)
|
||||
func (r *request) handlePage(page string) {
|
||||
r.rpath = urlPathToDepotPath(page)
|
||||
|
||||
if strings.HasSuffix(path, "/") {
|
||||
if strings.HasSuffix(r.rpath, "/") {
|
||||
// Directory path given, autoresolve.
|
||||
dirpath := path
|
||||
if path != "//" {
|
||||
dirpath = strings.TrimSuffix(path, "/") + "/"
|
||||
dirpath := r.rpath
|
||||
if r.rpath != "//" {
|
||||
dirpath = strings.TrimSuffix(r.rpath, "/") + "/"
|
||||
}
|
||||
s.handlePageAuto(w, r, rev, path, dirpath)
|
||||
r.handlePageAuto(dirpath)
|
||||
return
|
||||
}
|
||||
|
||||
// Otherwise, try loading the file.
|
||||
file, err := s.source.IsFile(path)
|
||||
file, err := r.source.IsFile(r.ctx, r.rpath)
|
||||
if err != nil {
|
||||
glog.Errorf("IsFile(%q): %w", path, err)
|
||||
handle500(w, r)
|
||||
glog.Errorf("IsFile(%q): %w", r.rpath, err)
|
||||
r.handle500()
|
||||
return
|
||||
}
|
||||
|
||||
// File exists, render that.
|
||||
if file {
|
||||
parts := strings.Split(path, "/")
|
||||
parts := strings.Split(r.rpath, "/")
|
||||
dirpath := strings.Join(parts[:(len(parts)-1)], "/")
|
||||
cfg, err := config.ForPath(s.source, dirpath)
|
||||
cfg, err := config.ForPath(r.ctx, r.source, dirpath)
|
||||
if err != nil {
|
||||
glog.Errorf("could not get config for path %q: %w", dirpath, err)
|
||||
handle500(w, r)
|
||||
r.handle500()
|
||||
return
|
||||
}
|
||||
s.handleMarkdown(w, r, s.source, rev, path, cfg)
|
||||
r.handleMarkdown(r.rpath, cfg)
|
||||
return
|
||||
}
|
||||
|
||||
// Otherwise assume directory, try all posibilities.
|
||||
dirpath := path
|
||||
if path != "//" {
|
||||
dirpath = strings.TrimSuffix(path, "/") + "/"
|
||||
dirpath := r.rpath
|
||||
if r.rpath != "//" {
|
||||
dirpath = strings.TrimSuffix(r.rpath, "/") + "/"
|
||||
}
|
||||
s.handlePageAuto(w, r, rev, path, dirpath)
|
||||
r.handlePageAuto(dirpath)
|
||||
}
|
||||
|
|
|
@ -1,48 +1,74 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"bytes"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"code.hackerspace.pl/hscloud/devtools/hackdoc/config"
|
||||
"code.hackerspace.pl/hscloud/devtools/hackdoc/source"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"gopkg.in/russross/blackfriday.v2"
|
||||
)
|
||||
|
||||
func (s *service) handleMarkdown(w http.ResponseWriter, r *http.Request, src source.Source, branch, path string, cfg *config.Config) {
|
||||
data, err := src.ReadFile(path)
|
||||
// renderMarkdown renders markdown to HTML, replacing all relative (intra-hackdoc) links with version that have ref set.
|
||||
func renderMarkdown(input []byte, ref string) []byte {
|
||||
r := blackfriday.NewHTMLRenderer(blackfriday.HTMLRendererParameters{
|
||||
Flags: blackfriday.CommonHTMLFlags,
|
||||
})
|
||||
|
||||
parser := blackfriday.New(blackfriday.WithRenderer(r), blackfriday.WithExtensions(blackfriday.CommonExtensions))
|
||||
ast := parser.Parse(input)
|
||||
|
||||
var buf bytes.Buffer
|
||||
ast.Walk(func(node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
|
||||
if ref != "" && entering && node.Type == blackfriday.Link {
|
||||
dest := string(node.Destination)
|
||||
u, err := url.Parse(dest)
|
||||
if err == nil && !u.IsAbs() {
|
||||
q := u.Query()
|
||||
q["ref"] = []string{ref}
|
||||
u.RawQuery = q.Encode()
|
||||
node.Destination = []byte(u.String())
|
||||
}
|
||||
}
|
||||
return r.RenderNode(&buf, node, entering)
|
||||
})
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func (r *request) handleMarkdown(path string, cfg *config.Config) {
|
||||
data, err := r.source.ReadFile(r.ctx, path)
|
||||
if err != nil {
|
||||
glog.Errorf("ReadFile(%q): %w", err)
|
||||
handle500(w, r)
|
||||
r.handle500()
|
||||
return
|
||||
}
|
||||
|
||||
rendered := blackfriday.Run([]byte(data))
|
||||
rendered := renderMarkdown([]byte(data), r.ref)
|
||||
|
||||
logRequest(w, r, "serving markdown at %s, cfg %+v", path, cfg)
|
||||
r.logRequest("serving markdown at %s, cfg %+v", path, cfg)
|
||||
|
||||
// TODO(q3k): allow markdown files to override which template to load
|
||||
tmpl, ok := cfg.Templates["default"]
|
||||
if !ok {
|
||||
glog.Errorf("No default template found for %s", path)
|
||||
// TODO(q3k): implement fallback template
|
||||
w.Write(rendered)
|
||||
r.w.Write(rendered)
|
||||
return
|
||||
}
|
||||
|
||||
pathInDepot := strings.TrimPrefix(path, "//")
|
||||
vars := map[string]interface{}{
|
||||
"Rendered": template.HTML(rendered),
|
||||
"Title": path,
|
||||
"Path": path,
|
||||
"PathInDepot": strings.TrimPrefix(path, "//"),
|
||||
"PathInDepot": pathInDepot,
|
||||
"HackdocURL": flagHackdocURL,
|
||||
"GitwebURL": fmt.Sprintf(flagGitwebURLPattern, flagGitwebDefaultBranch, strings.TrimPrefix(path, "//")),
|
||||
"WebLinks": r.source.WebLinks(pathInDepot),
|
||||
}
|
||||
err = tmpl.Execute(w, vars)
|
||||
err = tmpl.Execute(r.w, vars)
|
||||
if err != nil {
|
||||
glog.Errorf("Could not execute template for %s: %v", err)
|
||||
}
|
||||
|
|
|
@ -4,9 +4,10 @@ go_library(
|
|||
name = "go_default_library",
|
||||
srcs = [
|
||||
"source.go",
|
||||
"source_depotview.go",
|
||||
"source_local.go",
|
||||
],
|
||||
importpath = "code.hackerspace.pl/hscloud/devtools/hackdoc/source",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["@com_github_golang_glog//:go_default_library"],
|
||||
deps = ["//devtools/depotview/proto:go_default_library"],
|
||||
)
|
||||
|
|
|
@ -1,10 +1,36 @@
|
|||
package source
|
||||
|
||||
type Source interface {
|
||||
IsFile(path string) (bool, error)
|
||||
ReadFile(path string) ([]byte, error)
|
||||
IsDirectory(path string) (bool, error)
|
||||
import "context"
|
||||
|
||||
CacheSet(dependencies []string, key string, value interface{})
|
||||
CacheGet(key string) interface{}
|
||||
var (
|
||||
FlagGitwebURLPattern = "https://gerrit.hackerspace.pl/plugins/gitiles/hscloud/+/%s/%s"
|
||||
)
|
||||
|
||||
type Source interface {
|
||||
IsFile(ctx context.Context, path string) (bool, error)
|
||||
ReadFile(ctx context.Context, path string) ([]byte, error)
|
||||
IsDirectory(ctx context.Context, path string) (bool, error)
|
||||
WebLinks(fpath string) []WebLink
|
||||
}
|
||||
|
||||
type WebLink struct {
|
||||
Kind string
|
||||
LinkLabel string
|
||||
LinkURL string
|
||||
}
|
||||
|
||||
type SourceProvider interface {
|
||||
Source(ctx context.Context, rev string) (Source, error)
|
||||
}
|
||||
|
||||
type singleRefProvider struct {
|
||||
source Source
|
||||
}
|
||||
|
||||
func (s *singleRefProvider) Source(ctx context.Context, rev string) (Source, error) {
|
||||
return s.source, nil
|
||||
}
|
||||
|
||||
func NewSingleRefProvider(s Source) SourceProvider {
|
||||
return &singleRefProvider{s}
|
||||
}
|
||||
|
|
126
devtools/hackdoc/source/source_depotview.go
Normal file
126
devtools/hackdoc/source/source_depotview.go
Normal file
|
@ -0,0 +1,126 @@
|
|||
package source
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
dvpb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
|
||||
)
|
||||
|
||||
type DepotViewSourceProvider struct {
|
||||
stub dvpb.DepotViewClient
|
||||
}
|
||||
|
||||
func NewDepotView(stub dvpb.DepotViewClient) SourceProvider {
|
||||
return &DepotViewSourceProvider{
|
||||
stub: stub,
|
||||
}
|
||||
}
|
||||
|
||||
func changeRef(ref string) int64 {
|
||||
ref = strings.ToLower(ref)
|
||||
if !strings.HasPrefix(ref, "change/") && !strings.HasPrefix(ref, "cr/") {
|
||||
return 0
|
||||
}
|
||||
n, err := strconv.ParseInt(strings.SplitN(ref, "/", 2)[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return n
|
||||
}
|
||||
|
||||
func (s *DepotViewSourceProvider) Source(ctx context.Context, ref string) (Source, error) {
|
||||
var hash string
|
||||
n := changeRef(ref)
|
||||
if n != 0 {
|
||||
res, err := s.stub.ResolveGerritChange(ctx, &dvpb.ResolveGerritChangeRequest{Change: n})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hash = res.Hash
|
||||
} else {
|
||||
res, err := s.stub.Resolve(ctx, &dvpb.ResolveRequest{Ref: ref})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hash = res.Hash
|
||||
}
|
||||
|
||||
if hash == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return &depotViewSource{
|
||||
stub: s.stub,
|
||||
hash: hash,
|
||||
change: n,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type depotViewSource struct {
|
||||
stub dvpb.DepotViewClient
|
||||
hash string
|
||||
change int64
|
||||
}
|
||||
|
||||
func (s *depotViewSource) IsFile(ctx context.Context, path string) (bool, error) {
|
||||
res, err := s.stub.Stat(ctx, &dvpb.StatRequest{
|
||||
Hash: s.hash,
|
||||
Path: path,
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return res.Type == dvpb.StatResponse_TYPE_FILE, nil
|
||||
}
|
||||
|
||||
func (s *depotViewSource) IsDirectory(ctx context.Context, path string) (bool, error) {
|
||||
res, err := s.stub.Stat(ctx, &dvpb.StatRequest{
|
||||
Hash: s.hash,
|
||||
Path: path,
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return res.Type == dvpb.StatResponse_TYPE_DIRECTORY, nil
|
||||
}
|
||||
|
||||
func (s *depotViewSource) ReadFile(ctx context.Context, path string) ([]byte, error) {
|
||||
var data []byte
|
||||
srv, err := s.stub.Read(ctx, &dvpb.ReadRequest{
|
||||
Hash: s.hash,
|
||||
Path: path,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
res, err := srv.Recv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(res.Data) == 0 {
|
||||
break
|
||||
}
|
||||
data = append(data, res.Data...)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (s *depotViewSource) WebLinks(fpath string) []WebLink {
|
||||
gitURL := fmt.Sprintf(FlagGitwebURLPattern, s.hash, fpath)
|
||||
links := []WebLink{
|
||||
WebLink{Kind: "gitweb", LinkLabel: s.hash[:16], LinkURL: gitURL},
|
||||
}
|
||||
|
||||
if s.change != 0 {
|
||||
gerritLabel := fmt.Sprintf("change %d", s.change)
|
||||
gerritLink := fmt.Sprintf("https://gerrit.hackerspace.pl/%d", s.change)
|
||||
links = append(links, WebLink{Kind: "gerrit", LinkLabel: gerritLabel, LinkURL: gerritLink})
|
||||
}
|
||||
|
||||
return links
|
||||
}
|
|
@ -1,6 +1,7 @@
|
|||
package source
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
@ -29,7 +30,7 @@ func (s *LocalSource) resolve(path string) (string, error) {
|
|||
return s.root + "/" + path, nil
|
||||
}
|
||||
|
||||
func (s *LocalSource) IsFile(path string) (bool, error) {
|
||||
func (s *LocalSource) IsFile(ctx context.Context, path string) (bool, error) {
|
||||
path, err := s.resolve(path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
|
@ -44,7 +45,7 @@ func (s *LocalSource) IsFile(path string) (bool, error) {
|
|||
return !stat.IsDir(), nil
|
||||
}
|
||||
|
||||
func (s *LocalSource) ReadFile(path string) ([]byte, error) {
|
||||
func (s *LocalSource) ReadFile(ctx context.Context, path string) ([]byte, error) {
|
||||
path, err := s.resolve(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -53,7 +54,7 @@ func (s *LocalSource) ReadFile(path string) ([]byte, error) {
|
|||
return ioutil.ReadFile(path)
|
||||
}
|
||||
|
||||
func (s *LocalSource) IsDirectory(path string) (bool, error) {
|
||||
func (s *LocalSource) IsDirectory(ctx context.Context, path string) (bool, error) {
|
||||
path, err := s.resolve(path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
|
@ -68,11 +69,9 @@ func (s *LocalSource) IsDirectory(path string) (bool, error) {
|
|||
return stat.IsDir(), nil
|
||||
}
|
||||
|
||||
func (s *LocalSource) CacheSet(dependencies []string, key string, value interface{}) {
|
||||
// Swallow writes. The local filesystem can always change underneath us and
|
||||
// we're not tracking anything, so we cannot ever keep caches.
|
||||
}
|
||||
|
||||
func (s *LocalSource) CacheGet(key string) interface{} {
|
||||
return nil
|
||||
func (s *LocalSource) WebLinks(fpath string) []WebLink {
|
||||
gitURL := fmt.Sprintf(FlagGitwebURLPattern, "master", fpath)
|
||||
return []WebLink{
|
||||
WebLink{Kind: "gitweb", LinkLabel: "master", LinkURL: gitURL},
|
||||
}
|
||||
}
|
||||
|
|
|
@ -85,6 +85,10 @@ body {
|
|||
color: #b30014;
|
||||
}
|
||||
|
||||
.header span.muted {
|
||||
color: #666;
|
||||
}
|
||||
|
||||
.footer {
|
||||
font-size: 0.8em;
|
||||
color: #ccc;
|
||||
|
@ -160,7 +164,10 @@ ul li::before {
|
|||
<div class="column">
|
||||
<div class="page">
|
||||
<div class="header">
|
||||
<span class="red">hackdoc:</span><span>{{ .Path }}</span> <a href="{{ .GitwebURL }}">[git]</a>
|
||||
<span class="red">hackdoc:</span><span>{{ .Path }}</span>
|
||||
{{ range .WebLinks }}
|
||||
<span class="muted">[{{ .Kind }} <a href="{{ .LinkURL }}">{{ .LinkLabel }}</a>]</span>
|
||||
{{ end }}
|
||||
</div>
|
||||
{{ .Rendered }}
|
||||
</div>
|
||||
|
|
Loading…
Reference in a new issue