forked from hswaw/hscloud
devtools/depotview: init
This is a small service for accessing git repos read-only over gRPC. It's going to be used to allow hackdoc to render arbitrary versions of hscloud. Change-Id: Ib3c5eb5a8bc679e8062142e6fa30505d9550e2famaster
parent
c881cf3c22
commit
4c0e9b52c0
91
WORKSPACE
91
WORKSPACE
|
|
@ -183,23 +183,12 @@ http_file(
|
|||
http_archive(
|
||||
name = "io_bazel_rules_go",
|
||||
urls = [
|
||||
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.21.3/rules_go-v0.21.3.tar.gz",
|
||||
"https://github.com/bazelbuild/rules_go/releases/download/v0.21.3/rules_go-v0.21.3.tar.gz",
|
||||
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.22.2/rules_go-v0.22.2.tar.gz",
|
||||
"https://github.com/bazelbuild/rules_go/releases/download/v0.22.2/rules_go-v0.22.2.tar.gz",
|
||||
],
|
||||
sha256 = "af04c969321e8f428f63ceb73463d6ea817992698974abeff0161e069cd08bd6",
|
||||
sha256 = "142dd33e38b563605f0d20e89d9ef9eda0fc3cb539a14be1bdb1350de2eda659",
|
||||
)
|
||||
|
||||
# Invoke go_rules_dependencies depending on host platform.
|
||||
load("//tools:go_sdk.bzl", "gen_imports")
|
||||
|
||||
gen_imports(name = "go_sdk_imports")
|
||||
|
||||
load("@go_sdk_imports//:imports.bzl", "load_go_sdk")
|
||||
|
||||
load_go_sdk()
|
||||
|
||||
# Go Gazelle rules
|
||||
|
||||
http_archive(
|
||||
name = "bazel_gazelle",
|
||||
urls = [
|
||||
|
|
@ -208,9 +197,20 @@ http_archive(
|
|||
],
|
||||
sha256 = "d8c45ee70ec39a57e7a05e5027c32b1576cc7f16d9dd37135b0eddde45cf1b10",
|
||||
)
|
||||
|
||||
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
|
||||
|
||||
go_repository(
|
||||
name = "org_golang_x_net",
|
||||
commit = "d3edc9973b7eb1fb302b0ff2c62357091cea9a30",
|
||||
importpath = "golang.org/x/net",
|
||||
)
|
||||
|
||||
# Invoke go_rules_dependencies depending on host platform.
|
||||
load("//tools:go_sdk.bzl", "gen_imports")
|
||||
gen_imports(name = "go_sdk_imports")
|
||||
load("@go_sdk_imports//:imports.bzl", "load_go_sdk")
|
||||
load_go_sdk()
|
||||
|
||||
gazelle_dependencies()
|
||||
|
||||
# For devtools/gerrit/gerrit-oauth-provider
|
||||
|
|
@ -480,12 +480,6 @@ go_repository(
|
|||
tag = "v0.38.0",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "org_golang_x_net",
|
||||
commit = "13f9640d40b9",
|
||||
importpath = "golang.org/x/net",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_stackexchange_wmi",
|
||||
commit = "cbe66965904dbe8a6cd589e2298e5d8b986bd7dd",
|
||||
|
|
@ -1963,3 +1957,58 @@ go_repository(
|
|||
commit = "7bfe4c7ecddb3666a94b053b422cdd8f5aaa3615",
|
||||
importpath = "github.com/shurcooL/sanitized_anchor_name",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_go_git_go_billy_v5",
|
||||
commit = "d7a8afccaed297c30f8dff5724dbe422b491dd0d",
|
||||
importpath = "github.com/go-git/go-billy/v5",
|
||||
remote = "https://github.com/go-git/go-billy",
|
||||
vcs = "git",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_go_git_go_git_v5",
|
||||
commit = "3127ad9a44a2ee935502816065dfe39f494f583d",
|
||||
importpath = "github.com/go-git/go-git/v5",
|
||||
remote = "https://github.com/go-git/go-git",
|
||||
vcs = "git",
|
||||
build_extra_args = [
|
||||
"-known_import=github.com/go-git/go-billy/v5",
|
||||
],
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_go_git_gcfg",
|
||||
commit = "22f18f9a74d34e3b1a7d59cfa33043bc50ebe376",
|
||||
importpath = "github.com/go-git/gcfg",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "in_gopkg_warnings_v0",
|
||||
commit = "ec4a0fea49c7b46c2aeb0b51aac55779c607e52b",
|
||||
importpath = "gopkg.in/warnings.v0",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_emirpasic_gods",
|
||||
commit = "80e934ed68b9084f386ae25f74f839aaecfb54d8",
|
||||
importpath = "github.com/emirpasic/gods",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_jbenet_go_context",
|
||||
commit = "d14ea06fba99483203c19d92cfcd13ebe73135f4",
|
||||
importpath = "github.com/jbenet/go-context",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_kevinburke_ssh_config",
|
||||
commit = "01f96b0aa0cdcaa93f9495f89bbc6cb5a992ce6e",
|
||||
importpath = "github.com/kevinburke/ssh_config",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_xanzy_ssh_agent",
|
||||
commit = "6a3e2ff9e7c564f36873c2e36413f634534f1c44",
|
||||
importpath = "github.com/xanzy/ssh-agent",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["main.go"],
|
||||
importpath = "code.hackerspace.pl/hscloud/devtools/depotview",
|
||||
visibility = ["//visibility:private"],
|
||||
deps = [
|
||||
"//devtools/depotview/proto:go_default_library",
|
||||
"//devtools/depotview/service:go_default_library",
|
||||
"//go/mirko:go_default_library",
|
||||
"@com_github_golang_glog//:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_binary(
|
||||
name = "depotview",
|
||||
embed = [":go_default_library"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
depotview
|
||||
=========
|
||||
|
||||
Git-as-a-service over gRPC. Useful to get read-only access to hscloud.
|
||||
|
||||
Development
|
||||
-----------
|
||||
|
||||
$ bazel run //devtools/depotview -- -hspki_disable
|
||||
$ grpcurl -plaintext -d '{"ref": "master"}' 127.0.0.1:4200 depotview.DepotView.Resolve
|
||||
{
|
||||
"hash": "154baf1cf6ed99ae5b2849f512ea4d58dbbf199e",
|
||||
"lastChecked": 1586377071253733703
|
||||
}
|
||||
$ grpcurl -plaintext -d '{"hash": "154baf1cf6ed99ae5b2849f512ea4d58dbbf199e", "path": "//README"}' 127.0.0.1:4200 depotview.DepotView.Read
|
||||
{
|
||||
"data": "SFNDbG...."
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
|
||||
"code.hackerspace.pl/hscloud/go/mirko"
|
||||
"github.com/golang/glog"
|
||||
|
||||
pb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
|
||||
"code.hackerspace.pl/hscloud/devtools/depotview/service"
|
||||
)
|
||||
|
||||
var (
|
||||
flagRemote = "https://gerrit.hackerspace.pl/hscloud"
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.StringVar(&flagRemote, "git_remote", flagRemote, "Address of Git repository to serve")
|
||||
flag.Parse()
|
||||
|
||||
m := mirko.New()
|
||||
if err := m.Listen(); err != nil {
|
||||
glog.Exitf("Listen(): %v", err)
|
||||
}
|
||||
|
||||
s := service.New(flagRemote)
|
||||
pb.RegisterDepotViewServer(m.GRPC(), s)
|
||||
|
||||
if err := m.Serve(); err != nil {
|
||||
glog.Exitf("Serve(): %v", err)
|
||||
}
|
||||
|
||||
<-m.Done()
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_library")
|
||||
load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
|
||||
|
||||
proto_library(
|
||||
name = "proto_proto",
|
||||
srcs = ["depotview.proto"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
go_proto_library(
|
||||
name = "proto_go_proto",
|
||||
compilers = ["@io_bazel_rules_go//proto:go_grpc"],
|
||||
importpath = "code.hackerspace.pl/hscloud/devtools/depotview/proto",
|
||||
proto = ":proto_proto",
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
embed = [":proto_go_proto"],
|
||||
importpath = "code.hackerspace.pl/hscloud/devtools/depotview/proto",
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
syntax = "proto3";
|
||||
package depotview;
|
||||
option go_package = "code.hackerspace.pl/hscloud/devtools/depotview/proto";
|
||||
|
||||
message ResolveRequest {
|
||||
string ref = 1;
|
||||
}
|
||||
|
||||
message ResolveResponse {
|
||||
string hash = 1;
|
||||
int64 last_checked = 2;
|
||||
}
|
||||
|
||||
message StatRequest {
|
||||
string hash = 1;
|
||||
string path = 2;
|
||||
}
|
||||
|
||||
message StatResponse {
|
||||
enum Type {
|
||||
TYPE_INVALID = 0;
|
||||
TYPE_NOT_PRESENT = 1;
|
||||
TYPE_FILE = 2;
|
||||
TYPE_DIRECTORY = 3;
|
||||
};
|
||||
Type type = 1;
|
||||
}
|
||||
|
||||
message ReadRequest {
|
||||
string hash = 1;
|
||||
string path = 2;
|
||||
}
|
||||
|
||||
message ReadResponse {
|
||||
// Chunk of data. Empty once everything has been sent over.
|
||||
bytes data = 1;
|
||||
}
|
||||
|
||||
service DepotView {
|
||||
// Resolve a git branch/tag/ref... into a commit hash.
|
||||
rpc Resolve(ResolveRequest) returns (ResolveResponse);
|
||||
|
||||
// Minimal file access API. It kinda stinks.
|
||||
rpc Stat(StatRequest) returns (StatResponse);
|
||||
rpc Read(ReadRequest) returns (stream ReadResponse);
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["service.go"],
|
||||
importpath = "code.hackerspace.pl/hscloud/devtools/depotview/service",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//devtools/depotview/proto:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//plumbing:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//plumbing/filemode:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//plumbing/object:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//storage:go_default_library",
|
||||
"@com_github_go_git_go_git_v5//storage/memory:go_default_library",
|
||||
"@com_github_golang_glog//:go_default_library",
|
||||
"@org_golang_google_grpc//codes:go_default_library",
|
||||
"@org_golang_google_grpc//status:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["service_test.go"],
|
||||
embed = [":go_default_library"],
|
||||
deps = ["//devtools/depotview/proto:go_default_library"],
|
||||
)
|
||||
|
|
@ -0,0 +1,221 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
git "github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/filemode"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/go-git/go-git/v5/storage"
|
||||
"github.com/go-git/go-git/v5/storage/memory"
|
||||
|
||||
pb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
|
||||
)
|
||||
|
||||
var (
|
||||
reHash = regexp.MustCompile(`[a-f0-9]{40,64}`)
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
remote string
|
||||
storer storage.Storer
|
||||
|
||||
mu sync.Mutex
|
||||
repo *git.Repository
|
||||
lastPull time.Time
|
||||
}
|
||||
|
||||
func New(remote string) *Service {
|
||||
return &Service{
|
||||
remote: remote,
|
||||
storer: memory.NewStorage(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) ensureRepo() error {
|
||||
// Clone repository if necessary.
|
||||
if s.repo == nil {
|
||||
repo, err := git.Clone(s.storer, nil, &git.CloneOptions{
|
||||
URL: s.remote,
|
||||
})
|
||||
if err != nil {
|
||||
glog.Errorf("Clone(%q): %v", s.remote, err)
|
||||
return status.Error(codes.Unavailable, "could not clone repository")
|
||||
}
|
||||
s.repo = repo
|
||||
s.lastPull = time.Now()
|
||||
}
|
||||
|
||||
// Fetch if necessary.
|
||||
if time.Since(s.lastPull) > time.Minute {
|
||||
err := s.repo.Fetch(&git.FetchOptions{})
|
||||
if err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
glog.Errorf("Fetch(): %v", err)
|
||||
} else {
|
||||
s.lastPull = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) Resolve(ctx context.Context, req *pb.ResolveRequest) (*pb.ResolveResponse, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if req.Ref == "" {
|
||||
return nil, status.Error(codes.InvalidArgument, "ref must be set")
|
||||
}
|
||||
|
||||
if err := s.ensureRepo(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
h, err := s.repo.ResolveRevision(plumbing.Revision(req.Ref))
|
||||
switch {
|
||||
case err == plumbing.ErrReferenceNotFound:
|
||||
return &pb.ResolveResponse{Hash: "", LastChecked: s.lastPull.UnixNano()}, nil
|
||||
case err != nil:
|
||||
return nil, status.Errorf(codes.Unavailable, "git resolve error: %v", err)
|
||||
default:
|
||||
return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastPull.UnixNano()}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) getFile(hash, path string, notFoundOkay bool) (*object.File, error) {
|
||||
if !reHash.MatchString(hash) {
|
||||
return nil, status.Error(codes.InvalidArgument, "hash must be valid full git hash string")
|
||||
}
|
||||
if path == "" {
|
||||
return nil, status.Error(codes.InvalidArgument, "path must be set")
|
||||
}
|
||||
|
||||
path = pathNormalize(path)
|
||||
if path == "" {
|
||||
return nil, status.Error(codes.InvalidArgument, "path must be a valid unix or depot-style path")
|
||||
}
|
||||
|
||||
if err := s.ensureRepo(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c, err := s.repo.CommitObject(plumbing.NewHash(hash))
|
||||
switch {
|
||||
case err == plumbing.ErrObjectNotFound:
|
||||
return nil, status.Error(codes.NotFound, "hash not found")
|
||||
case err != nil:
|
||||
return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
|
||||
}
|
||||
|
||||
file, err := c.File(path)
|
||||
switch {
|
||||
case err == object.ErrFileNotFound && !notFoundOkay:
|
||||
return nil, status.Error(codes.NotFound, "file not found")
|
||||
case err == object.ErrFileNotFound && notFoundOkay:
|
||||
return nil, nil
|
||||
case err != nil:
|
||||
return nil, status.Errorf(codes.Unavailable, "git error: %v", err)
|
||||
}
|
||||
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (s *Service) Stat(ctx context.Context, req *pb.StatRequest) (*pb.StatResponse, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
file, err := s.getFile(req.Hash, req.Path, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if file == nil {
|
||||
return &pb.StatResponse{Type: pb.StatResponse_TYPE_NOT_PRESENT}, nil
|
||||
}
|
||||
|
||||
switch {
|
||||
case file.Mode == filemode.Dir:
|
||||
return &pb.StatResponse{Type: pb.StatResponse_TYPE_DIRECTORY}, nil
|
||||
case file.Mode.IsFile():
|
||||
return &pb.StatResponse{Type: pb.StatResponse_TYPE_FILE}, nil
|
||||
default:
|
||||
return nil, status.Errorf(codes.Unimplemented, "unknown file type %o", file.Mode)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
file, err := s.getFile(req.Hash, req.Path, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reader, err := file.Reader()
|
||||
if err != nil {
|
||||
return status.Errorf(codes.Unavailable, "file read error: %v", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
ctx := srv.Context()
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// 1 MB read
|
||||
chunk := make([]byte, 16*1024)
|
||||
n, err := reader.Read(chunk)
|
||||
switch {
|
||||
case err == io.EOF:
|
||||
n = 0
|
||||
case err != nil:
|
||||
return status.Errorf(codes.Unavailable, "file read error: %v", err)
|
||||
}
|
||||
|
||||
err = srv.Send(&pb.ReadResponse{Data: chunk[:n]})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func pathNormalize(path string) string {
|
||||
leadingSlashes := 0
|
||||
for _, c := range path {
|
||||
if c != '/' {
|
||||
break
|
||||
}
|
||||
leadingSlashes += 1
|
||||
}
|
||||
|
||||
// Only foo/bar, /foo/bar, and //foo/bar paths allowed.
|
||||
if leadingSlashes > 2 {
|
||||
return ""
|
||||
}
|
||||
path = path[leadingSlashes:]
|
||||
|
||||
// No trailing slashes allowed.
|
||||
if strings.HasSuffix(path, "/") {
|
||||
return ""
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
pb "code.hackerspace.pl/hscloud/devtools/depotview/proto"
|
||||
)
|
||||
|
||||
func TestIntegration(t *testing.T) {
|
||||
// TODO(q3k); bring up fake git
|
||||
s := New("https://gerrit.hackerspace.pl/hscloud")
|
||||
ctx := context.Background()
|
||||
|
||||
res, err := s.Resolve(ctx, &pb.ResolveRequest{Ref: "master"})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve(master): %v", err)
|
||||
}
|
||||
|
||||
if len(res.Hash) != 40 {
|
||||
t.Fatalf("Resolve returned odd hash: %q", res.Hash)
|
||||
}
|
||||
|
||||
res2, err := s.Stat(ctx, &pb.StatRequest{Hash: res.Hash, Path: "//WORKSPACE"})
|
||||
if err != nil {
|
||||
t.Fatalf("Stat(//WORKSPACE): %v", err)
|
||||
}
|
||||
|
||||
if want, got := pb.StatResponse_TYPE_FILE, res2.Type; want != got {
|
||||
t.Fatalf("Stat(//WORKSPACE): got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue