diff --git a/WORKSPACE b/WORKSPACE index d4d45b12..bf25c0c2 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -183,23 +183,12 @@ http_file( http_archive( name = "io_bazel_rules_go", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.21.3/rules_go-v0.21.3.tar.gz", - "https://github.com/bazelbuild/rules_go/releases/download/v0.21.3/rules_go-v0.21.3.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.22.2/rules_go-v0.22.2.tar.gz", + "https://github.com/bazelbuild/rules_go/releases/download/v0.22.2/rules_go-v0.22.2.tar.gz", ], - sha256 = "af04c969321e8f428f63ceb73463d6ea817992698974abeff0161e069cd08bd6", + sha256 = "142dd33e38b563605f0d20e89d9ef9eda0fc3cb539a14be1bdb1350de2eda659", ) -# Invoke go_rules_dependencies depending on host platform. -load("//tools:go_sdk.bzl", "gen_imports") - -gen_imports(name = "go_sdk_imports") - -load("@go_sdk_imports//:imports.bzl", "load_go_sdk") - -load_go_sdk() - -# Go Gazelle rules - http_archive( name = "bazel_gazelle", urls = [ @@ -208,9 +197,20 @@ http_archive( ], sha256 = "d8c45ee70ec39a57e7a05e5027c32b1576cc7f16d9dd37135b0eddde45cf1b10", ) - load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") +go_repository( + name = "org_golang_x_net", + commit = "d3edc9973b7eb1fb302b0ff2c62357091cea9a30", + importpath = "golang.org/x/net", +) + +# Invoke go_rules_dependencies depending on host platform. +load("//tools:go_sdk.bzl", "gen_imports") +gen_imports(name = "go_sdk_imports") +load("@go_sdk_imports//:imports.bzl", "load_go_sdk") +load_go_sdk() + gazelle_dependencies() # For devtools/gerrit/gerrit-oauth-provider @@ -480,12 +480,6 @@ go_repository( tag = "v0.38.0", ) -go_repository( - name = "org_golang_x_net", - commit = "13f9640d40b9", - importpath = "golang.org/x/net", -) - go_repository( name = "com_github_stackexchange_wmi", commit = "cbe66965904dbe8a6cd589e2298e5d8b986bd7dd", @@ -1963,3 +1957,58 @@ go_repository( commit = "7bfe4c7ecddb3666a94b053b422cdd8f5aaa3615", importpath = "github.com/shurcooL/sanitized_anchor_name", ) + +go_repository( + name = "com_github_go_git_go_billy_v5", + commit = "d7a8afccaed297c30f8dff5724dbe422b491dd0d", + importpath = "github.com/go-git/go-billy/v5", + remote = "https://github.com/go-git/go-billy", + vcs = "git", +) + +go_repository( + name = "com_github_go_git_go_git_v5", + commit = "3127ad9a44a2ee935502816065dfe39f494f583d", + importpath = "github.com/go-git/go-git/v5", + remote = "https://github.com/go-git/go-git", + vcs = "git", + build_extra_args = [ + "-known_import=github.com/go-git/go-billy/v5", + ], +) + +go_repository( + name = "com_github_go_git_gcfg", + commit = "22f18f9a74d34e3b1a7d59cfa33043bc50ebe376", + importpath = "github.com/go-git/gcfg", +) + +go_repository( + name = "in_gopkg_warnings_v0", + commit = "ec4a0fea49c7b46c2aeb0b51aac55779c607e52b", + importpath = "gopkg.in/warnings.v0", +) + +go_repository( + name = "com_github_emirpasic_gods", + commit = "80e934ed68b9084f386ae25f74f839aaecfb54d8", + importpath = "github.com/emirpasic/gods", +) + +go_repository( + name = "com_github_jbenet_go_context", + commit = "d14ea06fba99483203c19d92cfcd13ebe73135f4", + importpath = "github.com/jbenet/go-context", +) + +go_repository( + name = "com_github_kevinburke_ssh_config", + commit = "01f96b0aa0cdcaa93f9495f89bbc6cb5a992ce6e", + importpath = "github.com/kevinburke/ssh_config", +) + +go_repository( + name = "com_github_xanzy_ssh_agent", + commit = "6a3e2ff9e7c564f36873c2e36413f634534f1c44", + importpath = "github.com/xanzy/ssh-agent", +) diff --git a/devtools/depotview/BUILD.bazel b/devtools/depotview/BUILD.bazel new file mode 100644 index 00000000..223f64cc --- /dev/null +++ b/devtools/depotview/BUILD.bazel @@ -0,0 +1,20 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "code.hackerspace.pl/hscloud/devtools/depotview", + visibility = ["//visibility:private"], + deps = [ + "//devtools/depotview/proto:go_default_library", + "//devtools/depotview/service:go_default_library", + "//go/mirko:go_default_library", + "@com_github_golang_glog//:go_default_library", + ], +) + +go_binary( + name = "depotview", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/devtools/depotview/README.md b/devtools/depotview/README.md new file mode 100644 index 00000000..9b806a66 --- /dev/null +++ b/devtools/depotview/README.md @@ -0,0 +1,19 @@ +depotview +========= + +Git-as-a-service over gRPC. Useful to get read-only access to hscloud. + +Development +----------- + + $ bazel run //devtools/depotview -- -hspki_disable + $ grpcurl -plaintext -d '{"ref": "master"}' 127.0.0.1:4200 depotview.DepotView.Resolve + { + "hash": "154baf1cf6ed99ae5b2849f512ea4d58dbbf199e", + "lastChecked": 1586377071253733703 + } + $ grpcurl -plaintext -d '{"hash": "154baf1cf6ed99ae5b2849f512ea4d58dbbf199e", "path": "//README"}' 127.0.0.1:4200 depotview.DepotView.Read + { + "data": "SFNDbG...." + } + diff --git a/devtools/depotview/main.go b/devtools/depotview/main.go new file mode 100644 index 00000000..7b4aed4b --- /dev/null +++ b/devtools/depotview/main.go @@ -0,0 +1,34 @@ +package main + +import ( + "flag" + + "code.hackerspace.pl/hscloud/go/mirko" + "github.com/golang/glog" + + pb "code.hackerspace.pl/hscloud/devtools/depotview/proto" + "code.hackerspace.pl/hscloud/devtools/depotview/service" +) + +var ( + flagRemote = "https://gerrit.hackerspace.pl/hscloud" +) + +func main() { + flag.StringVar(&flagRemote, "git_remote", flagRemote, "Address of Git repository to serve") + flag.Parse() + + m := mirko.New() + if err := m.Listen(); err != nil { + glog.Exitf("Listen(): %v", err) + } + + s := service.New(flagRemote) + pb.RegisterDepotViewServer(m.GRPC(), s) + + if err := m.Serve(); err != nil { + glog.Exitf("Serve(): %v", err) + } + + <-m.Done() +} diff --git a/devtools/depotview/proto/BUILD.bazel b/devtools/depotview/proto/BUILD.bazel new file mode 100644 index 00000000..47df9208 --- /dev/null +++ b/devtools/depotview/proto/BUILD.bazel @@ -0,0 +1,23 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") + +proto_library( + name = "proto_proto", + srcs = ["depotview.proto"], + visibility = ["//visibility:public"], +) + +go_proto_library( + name = "proto_go_proto", + compilers = ["@io_bazel_rules_go//proto:go_grpc"], + importpath = "code.hackerspace.pl/hscloud/devtools/depotview/proto", + proto = ":proto_proto", + visibility = ["//visibility:public"], +) + +go_library( + name = "go_default_library", + embed = [":proto_go_proto"], + importpath = "code.hackerspace.pl/hscloud/devtools/depotview/proto", + visibility = ["//visibility:public"], +) diff --git a/devtools/depotview/proto/depotview.proto b/devtools/depotview/proto/depotview.proto new file mode 100644 index 00000000..25849615 --- /dev/null +++ b/devtools/depotview/proto/depotview.proto @@ -0,0 +1,46 @@ +syntax = "proto3"; +package depotview; +option go_package = "code.hackerspace.pl/hscloud/devtools/depotview/proto"; + +message ResolveRequest { + string ref = 1; +} + +message ResolveResponse { + string hash = 1; + int64 last_checked = 2; +} + +message StatRequest { + string hash = 1; + string path = 2; +} + +message StatResponse { + enum Type { + TYPE_INVALID = 0; + TYPE_NOT_PRESENT = 1; + TYPE_FILE = 2; + TYPE_DIRECTORY = 3; + }; + Type type = 1; +} + +message ReadRequest { + string hash = 1; + string path = 2; +} + +message ReadResponse { + // Chunk of data. Empty once everything has been sent over. + bytes data = 1; +} + +service DepotView { + // Resolve a git branch/tag/ref... into a commit hash. + rpc Resolve(ResolveRequest) returns (ResolveResponse); + + // Minimal file access API. It kinda stinks. + rpc Stat(StatRequest) returns (StatResponse); + rpc Read(ReadRequest) returns (stream ReadResponse); +} diff --git a/devtools/depotview/service/BUILD.bazel b/devtools/depotview/service/BUILD.bazel new file mode 100644 index 00000000..6f7337ca --- /dev/null +++ b/devtools/depotview/service/BUILD.bazel @@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["service.go"], + importpath = "code.hackerspace.pl/hscloud/devtools/depotview/service", + visibility = ["//visibility:public"], + deps = [ + "//devtools/depotview/proto:go_default_library", + "@com_github_go_git_go_git_v5//:go_default_library", + "@com_github_go_git_go_git_v5//plumbing:go_default_library", + "@com_github_go_git_go_git_v5//plumbing/filemode:go_default_library", + "@com_github_go_git_go_git_v5//plumbing/object:go_default_library", + "@com_github_go_git_go_git_v5//storage:go_default_library", + "@com_github_go_git_go_git_v5//storage/memory:go_default_library", + "@com_github_golang_glog//:go_default_library", + "@org_golang_google_grpc//codes:go_default_library", + "@org_golang_google_grpc//status:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["service_test.go"], + embed = [":go_default_library"], + deps = ["//devtools/depotview/proto:go_default_library"], +) diff --git a/devtools/depotview/service/service.go b/devtools/depotview/service/service.go new file mode 100644 index 00000000..910bf363 --- /dev/null +++ b/devtools/depotview/service/service.go @@ -0,0 +1,221 @@ +package service + +import ( + "context" + "io" + "regexp" + "strings" + "sync" + "time" + + "github.com/golang/glog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + git "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage" + "github.com/go-git/go-git/v5/storage/memory" + + pb "code.hackerspace.pl/hscloud/devtools/depotview/proto" +) + +var ( + reHash = regexp.MustCompile(`[a-f0-9]{40,64}`) +) + +type Service struct { + remote string + storer storage.Storer + + mu sync.Mutex + repo *git.Repository + lastPull time.Time +} + +func New(remote string) *Service { + return &Service{ + remote: remote, + storer: memory.NewStorage(), + } +} + +func (s *Service) ensureRepo() error { + // Clone repository if necessary. + if s.repo == nil { + repo, err := git.Clone(s.storer, nil, &git.CloneOptions{ + URL: s.remote, + }) + if err != nil { + glog.Errorf("Clone(%q): %v", s.remote, err) + return status.Error(codes.Unavailable, "could not clone repository") + } + s.repo = repo + s.lastPull = time.Now() + } + + // Fetch if necessary. + if time.Since(s.lastPull) > time.Minute { + err := s.repo.Fetch(&git.FetchOptions{}) + if err != nil && err != git.NoErrAlreadyUpToDate { + glog.Errorf("Fetch(): %v", err) + } else { + s.lastPull = time.Now() + } + } + + return nil +} + +func (s *Service) Resolve(ctx context.Context, req *pb.ResolveRequest) (*pb.ResolveResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if req.Ref == "" { + return nil, status.Error(codes.InvalidArgument, "ref must be set") + } + + if err := s.ensureRepo(); err != nil { + return nil, err + } + + h, err := s.repo.ResolveRevision(plumbing.Revision(req.Ref)) + switch { + case err == plumbing.ErrReferenceNotFound: + return &pb.ResolveResponse{Hash: "", LastChecked: s.lastPull.UnixNano()}, nil + case err != nil: + return nil, status.Errorf(codes.Unavailable, "git resolve error: %v", err) + default: + return &pb.ResolveResponse{Hash: h.String(), LastChecked: s.lastPull.UnixNano()}, nil + } +} + +func (s *Service) getFile(hash, path string, notFoundOkay bool) (*object.File, error) { + if !reHash.MatchString(hash) { + return nil, status.Error(codes.InvalidArgument, "hash must be valid full git hash string") + } + if path == "" { + return nil, status.Error(codes.InvalidArgument, "path must be set") + } + + path = pathNormalize(path) + if path == "" { + return nil, status.Error(codes.InvalidArgument, "path must be a valid unix or depot-style path") + } + + if err := s.ensureRepo(); err != nil { + return nil, err + } + + c, err := s.repo.CommitObject(plumbing.NewHash(hash)) + switch { + case err == plumbing.ErrObjectNotFound: + return nil, status.Error(codes.NotFound, "hash not found") + case err != nil: + return nil, status.Errorf(codes.Unavailable, "git error: %v", err) + } + + file, err := c.File(path) + switch { + case err == object.ErrFileNotFound && !notFoundOkay: + return nil, status.Error(codes.NotFound, "file not found") + case err == object.ErrFileNotFound && notFoundOkay: + return nil, nil + case err != nil: + return nil, status.Errorf(codes.Unavailable, "git error: %v", err) + } + + return file, nil +} + +func (s *Service) Stat(ctx context.Context, req *pb.StatRequest) (*pb.StatResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + + file, err := s.getFile(req.Hash, req.Path, true) + if err != nil { + return nil, err + } + + if file == nil { + return &pb.StatResponse{Type: pb.StatResponse_TYPE_NOT_PRESENT}, nil + } + + switch { + case file.Mode == filemode.Dir: + return &pb.StatResponse{Type: pb.StatResponse_TYPE_DIRECTORY}, nil + case file.Mode.IsFile(): + return &pb.StatResponse{Type: pb.StatResponse_TYPE_FILE}, nil + default: + return nil, status.Errorf(codes.Unimplemented, "unknown file type %o", file.Mode) + } +} + +func (s *Service) Read(req *pb.ReadRequest, srv pb.DepotView_ReadServer) error { + s.mu.Lock() + defer s.mu.Unlock() + + file, err := s.getFile(req.Hash, req.Path, false) + if err != nil { + return err + } + + reader, err := file.Reader() + if err != nil { + return status.Errorf(codes.Unavailable, "file read error: %v", err) + } + defer reader.Close() + + ctx := srv.Context() + for { + if ctx.Err() != nil { + return ctx.Err() + } + + // 1 MB read + chunk := make([]byte, 16*1024) + n, err := reader.Read(chunk) + switch { + case err == io.EOF: + n = 0 + case err != nil: + return status.Errorf(codes.Unavailable, "file read error: %v", err) + } + + err = srv.Send(&pb.ReadResponse{Data: chunk[:n]}) + if err != nil { + return err + } + + if n == 0 { + break + } + } + + return nil +} + +func pathNormalize(path string) string { + leadingSlashes := 0 + for _, c := range path { + if c != '/' { + break + } + leadingSlashes += 1 + } + + // Only foo/bar, /foo/bar, and //foo/bar paths allowed. + if leadingSlashes > 2 { + return "" + } + path = path[leadingSlashes:] + + // No trailing slashes allowed. + if strings.HasSuffix(path, "/") { + return "" + } + + return path +} diff --git a/devtools/depotview/service/service_test.go b/devtools/depotview/service/service_test.go new file mode 100644 index 00000000..8ea07645 --- /dev/null +++ b/devtools/depotview/service/service_test.go @@ -0,0 +1,32 @@ +package service + +import ( + "context" + "testing" + + pb "code.hackerspace.pl/hscloud/devtools/depotview/proto" +) + +func TestIntegration(t *testing.T) { + // TODO(q3k); bring up fake git + s := New("https://gerrit.hackerspace.pl/hscloud") + ctx := context.Background() + + res, err := s.Resolve(ctx, &pb.ResolveRequest{Ref: "master"}) + if err != nil { + t.Fatalf("Resolve(master): %v", err) + } + + if len(res.Hash) != 40 { + t.Fatalf("Resolve returned odd hash: %q", res.Hash) + } + + res2, err := s.Stat(ctx, &pb.StatRequest{Hash: res.Hash, Path: "//WORKSPACE"}) + if err != nil { + t.Fatalf("Stat(//WORKSPACE): %v", err) + } + + if want, got := pb.StatResponse_TYPE_FILE, res2.Type; want != got { + t.Fatalf("Stat(//WORKSPACE): got %v, want %v", got, want) + } +}