Merge changes I46da0fc4,Ib187de6d

* changes:
  cluster/admitomatic: finish up service
  cluster/admitomatic: finish up ingress admission logic
This commit is contained in:
q3k 2021-01-31 11:56:34 +00:00 committed by Gerrit Code Review
commit 2fbd0710f5
8 changed files with 441 additions and 6 deletions

View file

@ -5,13 +5,18 @@ go_library(
srcs = [
"ingress.go",
"main.go",
"service.go",
],
importpath = "code.hackerspace.pl/hscloud/cluster/admitomatic",
visibility = ["//visibility:private"],
deps = [
"//cluster/admitomatic/config:go_default_library",
"//go/mirko:go_default_library",
"@com_github_golang_glog//:go_default_library",
"@io_k8s_api//admission/v1beta1:go_default_library",
"@io_k8s_api//networking/v1beta1:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
"@org_golang_google_protobuf//encoding/prototext:go_default_library",
],
)
@ -25,4 +30,10 @@ go_test(
name = "go_default_test",
srcs = ["ingress_test.go"],
embed = [":go_default_library"],
deps = [
"@io_k8s_api//admission/v1beta1:go_default_library",
"@io_k8s_api//networking/v1beta1:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
"@io_k8s_apimachinery//pkg/runtime:go_default_library",
],
)

View file

@ -0,0 +1,23 @@
load("@rules_proto//proto:defs.bzl", "proto_library")
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
proto_library(
name = "config_proto",
srcs = ["config.proto"],
visibility = ["//visibility:public"],
)
go_proto_library(
name = "config_go_proto",
importpath = "code.hackerspace.pl/hscloud/cluster/admitomatic/config",
proto = ":config_proto",
visibility = ["//visibility:public"],
)
go_library(
name = "go_default_library",
embed = [":config_go_proto"],
importpath = "code.hackerspace.pl/hscloud/cluster/admitomatic/config",
visibility = ["//visibility:public"],
)

View file

@ -0,0 +1,37 @@
syntax = "proto3";
package config;
option go_package = "code.hackerspace.pl/hscloud/cluster/admitomatic/config";
// Admitomatic configuration, passed as a text proto, for
// example:
//
// $ cat sample.pb.text
// allow_domain { namespace: "example" dns: "*.example.com" }
// allow_domain {
// namespace: "personal-q3k" dns: "foo.q3k.org"
// }
// allow_domain {
// namespace: "personal-q3k" dns: "bar.q3k.org"
// }
//
message Config {
// List of domains that are allowed to be configured as
// ingresses in a given namespace. If a domain does not
// appear in this list, it will be allowed to run in any
// namespace.
repeated AllowDomain allow_domain = 1;
}
message AllowDomain {
// namespace is a kubernetes namespace. An empty string is
// treated as the 'default' namespace.
string namespace = 1;
// dns is a domain name like 'example.com' or a wildcard
// like '*.foo.example.com'.
// Wildcards match domains at any level beneath the root,
// so the example above would match 'bar.foo.example.com'
// and 'baz.bar.foo.example.com'. However, they do not
// catch the root itself, ie. the above would not catch
// 'foo.example.com'.
string dns = 2;
}

View file

@ -0,0 +1,7 @@
allow_domain { namespace: "example" dns: "*.example.com" }
allow_domain {
namespace: "personal-q3k" dns: "foo.q3k.org"
}
allow_domain {
namespace: "personal-q3k" dns: "bar.q3k.org"
}

View file

@ -1,10 +1,14 @@
package main
import (
"encoding/json"
"fmt"
"strings"
"github.com/golang/glog"
admission "k8s.io/api/admission/v1beta1"
networking "k8s.io/api/networking/v1beta1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// ingressFilter is a filter which allows or denies the creation of an ingress
@ -125,6 +129,99 @@ func (i *ingressFilter) admit(req *admission.AdmissionRequest) (*admission.Admis
if req.Kind.Group != "networking.k8s.io" || req.Kind.Kind != "Ingress" {
return nil, fmt.Errorf("not an ingress")
}
// TODO(q3k); implement
return nil, fmt.Errorf("unimplemented")
result := func(s string, args ...interface{}) (*admission.AdmissionResponse, error) {
res := &admission.AdmissionResponse{
UID: req.UID,
}
if s == "" {
res.Allowed = true
} else {
res.Allowed = false
res.Result = &meta.Status{
Code: 403,
Message: fmt.Sprintf("admitomatic: %s", fmt.Sprintf(s, args...)),
}
}
return res, nil
}
// Permit any actions on critical system namespaes. See:
// https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/
// “Avoiding operating on the kube-system namespace”
if req.Namespace == "kube-system" {
return result("")
}
switch req.Operation {
case "CREATE":
case "UPDATE":
default:
// We only care about creations/updates, everything else is referred to plain RBAC.
return result("")
}
ingress := networking.Ingress{}
err := json.Unmarshal(req.Object.Raw, &ingress)
if err != nil {
glog.Errorf("Unmarshaling Ingress failed: %v", err)
return result("invalid object")
}
// Check TLS config for hosts.
for j, t := range ingress.Spec.TLS {
for k, h := range t.Hosts {
if strings.Contains(h, "*") {
// TODO(q3k): support wildcards
return result("wildcard host %q (%d in TLS entry %d) is not permitted", h, k, j)
}
if !i.domainAllowed(req.Namespace, h) {
return result("host %q (%d) in TLS entry %d is not allowed in namespace %q", h, k, j, req.Namespace)
}
}
}
// Check rules for hosts.
for j, r := range ingress.Spec.Rules {
h := r.Host
// Per IngressRule spec:
// If the host is unspecified, the Ingress routes all traffic based
// on the specified IngressRuleValue. Host can be "precise" which is
// a domain name without the terminating dot of a network host (e.g.
// "foo.bar.com") or "wildcard", which is a domain name prefixed with
// a single wildcard label (e.g. "*.foo.com").
//
// We reject everything other than precise hosts.
if h == "" {
return result("empty host %q (in rule %d) is not permitted", h, j)
}
if strings.Contains(h, "*") {
// TODO(q3k): support wildcards
return result("wildcard host %q (in rule %d) is not permitted", h, j)
}
if !i.domainAllowed(req.Namespace, h) {
return result("host %q (in rule %d) is not allowed in namespace %q", h, j, req.Namespace)
}
}
// Only allow a trusted subset of n-i-c annotations.
// TODO(q3k): allow opt-out for some namespaces
allowed := map[string]bool{
"proxy-body-size": true,
"ssl-redirect": true,
"backend-protocol": true,
}
prefix := "nginx.ingress.kubernetes.io/"
for k, _ := range ingress.Annotations {
if !strings.HasPrefix(k, prefix) {
continue
}
k = strings.TrimPrefix(k, prefix)
if !allowed[k] {
return result("forbidden annotation %q", k)
}
}
// All clear, accept this Ingress.
return result("")
}

View file

@ -1,6 +1,15 @@
package main
import "testing"
import (
"encoding/json"
"strings"
"testing"
admission "k8s.io/api/admission/v1beta1"
networking "k8s.io/api/networking/v1beta1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
runtime "k8s.io/apimachinery/pkg/runtime"
)
func TestPatterns(t *testing.T) {
f := ingressFilter{}
@ -76,3 +85,121 @@ func TestMatch(t *testing.T) {
}
}
}
func TestIngressPermitted(t *testing.T) {
f := ingressFilter{}
// Errors discarded, tested in TestPatterns.
f.allow("matrix", "matrix.hackerspace.pl")
f.allow("ceph-waw3", "*.hackerspace.pl")
f.allow("personal-q3k", "*.k0.q3k.org")
f.allow("personal-vuko", "shells.vuko.pl")
f.allow("minecraft", "*.k0.q3k.org")
mkReq := func(ns string, annotations map[string]string, is *networking.IngressSpec) *admission.AdmissionRequest {
i := &networking.Ingress{
Spec: *is,
}
i.Annotations = annotations
raw, err := json.Marshal(i)
if err != nil {
t.Fatalf("marshaling test ingress: %v", err)
}
return &admission.AdmissionRequest{
UID: "test",
Kind: meta.GroupVersionKind{
Group: "networking.k8s.io",
Version: "v1beta1",
Kind: "Ingress",
},
Namespace: ns,
Operation: "CREATE",
Object: runtime.RawExtension{
Raw: raw,
},
}
}
for i, el := range []struct {
req *admission.AdmissionRequest
err string
}{
// 0: unrelated domain, should be allowed
{mkReq("default", nil, &networking.IngressSpec{
Rules: []networking.IngressRule{
{Host: "example.com"},
},
TLS: []networking.IngressTLS{
{
Hosts: []string{"example.com"},
},
},
}), ""},
// 1: permitted restricted domain, should be allowed
{mkReq("matrix", nil, &networking.IngressSpec{
Rules: []networking.IngressRule{
{Host: "matrix.hackerspace.pl"},
},
TLS: []networking.IngressTLS{
{
Hosts: []string{"matrix.hackerspace.pl"},
},
},
}), ""},
// 2: forbidden restricted domain, should be rejected
{mkReq("personal-hacker", nil, &networking.IngressSpec{
Rules: []networking.IngressRule{
{Host: "matrix.hackerspace.pl"},
},
TLS: []networking.IngressTLS{
{
Hosts: []string{"matrix.hackerspace.pl"},
},
},
}), "not allowed in namespace"},
// 3: weird ingress but okay
{mkReq("personal-hacker", nil, &networking.IngressSpec{}), ""},
// 4: janky annotations, should be rejected
{mkReq("matrix", map[string]string{
"nginx.ingress.kubernetes.io/configuration-snippet": "omghax",
}, &networking.IngressSpec{
Rules: []networking.IngressRule{
{Host: "matrix.hackerspace.pl"},
},
TLS: []networking.IngressTLS{
{
Hosts: []string{"matrix.hackerspace.pl"},
},
},
}), "forbidden annotation"},
// 5: accepted annotations, should be allowed
{mkReq("matrix", map[string]string{
"nginx.ingress.kubernetes.io/proxy-body-size": "2137",
"foo.q3k.org/bar": "baz",
}, &networking.IngressSpec{
Rules: []networking.IngressRule{
{Host: "matrix.hackerspace.pl"},
},
TLS: []networking.IngressTLS{
{
Hosts: []string{"matrix.hackerspace.pl"},
},
},
}), ""},
} {
res, err := f.admit(el.req)
if err != nil {
t.Errorf("test %d: admit: %v", i, err)
}
if el.err == "" {
if !res.Allowed {
t.Errorf("test %d: wanted allow, got %q", i, res.Result.Message)
}
} else {
if res.Allowed {
t.Errorf("test %d: wanted %q, got allowed", i, el.err)
} else if !strings.Contains(res.Result.Message, el.err) {
t.Errorf("test %d: wanted %q, got %q", i, el.err, res.Result.Message)
}
}
}
}

View file

@ -3,6 +3,7 @@ package main
import (
"context"
"flag"
"io/ioutil"
"net/http"
"time"
@ -11,13 +12,29 @@ import (
)
var (
flagListen = "127.0.0.1:8080"
flagListen = "127.0.0.1:8080"
flagConfig = ""
flagTLSKey = ""
flagTLSCert = ""
)
func main() {
flag.StringVar(&flagListen, "pub_listen", flagListen, "Address to listen on for HTTP traffic")
flag.StringVar(&flagListen, "admitomatic_listen", flagListen, "Address to listen on for HTTP traffic")
flag.StringVar(&flagTLSKey, "admitomatic_tls_key", flagTLSKey, "TLS key to serve HTTP with")
flag.StringVar(&flagTLSCert, "admitomatic_tls_cert", flagTLSCert, "TLS certificate to serve HTTP with")
flag.StringVar(&flagConfig, "admitomatic_config", flagConfig, "Config path (prototext format)")
flag.Parse()
if flagConfig == "" {
glog.Exitf("-admitomatic_config must be set")
}
if flagTLSKey == "" {
glog.Exitf("-admitomatic_tls_key must be set")
}
if flagTLSCert == "" {
glog.Exitf("-admitomatic_tls_cert must be set")
}
m := mirko.New()
if err := m.Listen(); err != nil {
glog.Exitf("Listen(): %v", err)
@ -27,13 +44,24 @@ func main() {
glog.Exitf("Serve(): %v", err)
}
configData, err := ioutil.ReadFile(flagConfig)
if err != nil {
glog.Exitf("Could not read config: %v", err)
}
s, err := newService(configData)
if err != nil {
glog.Exitf("Could not start service: %v", err)
}
mux := http.NewServeMux()
mux.HandleFunc("/", s.handler)
// TODO(q3k): implement admission controller
srv := &http.Server{Addr: flagListen, Handler: mux}
glog.Infof("Listening on %q...", flagListen)
go func() {
if err := srv.ListenAndServe(); err != nil {
if err := srv.ListenAndServeTLS(flagTLSCert, flagTLSKey); err != nil {
glog.Error(err)
}
}()

View file

@ -0,0 +1,105 @@
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"github.com/golang/glog"
"google.golang.org/protobuf/encoding/prototext"
admission "k8s.io/api/admission/v1beta1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
pb "code.hackerspace.pl/hscloud/cluster/admitomatic/config"
)
type service struct {
ingress ingressFilter
}
// newService creates an admitomatic service from a given prototext config.
func newService(configuration []byte) (*service, error) {
var cfg pb.Config
if err := prototext.Unmarshal(configuration, &cfg); err != nil {
return nil, fmt.Errorf("parsing config: %v", err)
}
s := service{}
for i, ad := range cfg.AllowDomain {
if ad.Namespace == "" {
ad.Namespace = "default"
}
if ad.Dns == "" {
return nil, fmt.Errorf("config entry %d: dns must be set", i)
}
if err := s.ingress.allow(ad.Namespace, ad.Dns); err != nil {
return nil, fmt.Errorf("config entry %d: %v", i, err)
}
glog.Infof("Ingress: allowing %s in %s", ad.Dns, ad.Namespace)
}
return &s, nil
}
// handler is the main HTTP handler of the admitomatic service. It servers the
// AdmissionReview API, and is called by the Kubernetes API server to
// permit/deny creation/updating of resources.
func (s *service) handler(w http.ResponseWriter, r *http.Request) {
var body []byte
if r.Body != nil {
if data, err := ioutil.ReadAll(r.Body); err == nil {
body = data
}
}
if r.Method != "POST" {
glog.Errorf("%s %s: invalid method", r.Method, r.URL)
return
}
contentType := r.Header.Get("Content-Type")
if contentType != "application/json" {
glog.Errorf("%s %s: invalid content-type", r.Method, r.URL)
return
}
var review admission.AdmissionReview
if err := json.Unmarshal(body, &review); err != nil {
glog.Errorf("%s %s: cannot decode: %v", r.Method, r.URL, err)
return
}
if review.Kind != "AdmissionReview" {
glog.Errorf("%s %s: invalid Kind (%q)", r.Method, r.URL, review.Kind)
return
}
var err error
req := review.Request
resp := &admission.AdmissionResponse{
UID: req.UID,
Allowed: true,
}
switch {
case req.Kind.Group == "networking.k8s.io" && req.Kind.Kind == "Ingress":
resp, err = s.ingress.admit(req)
if err != nil {
glog.Errorf("%s %s %s: %v", req.Operation, req.Name, req.Namespace, err)
// Fail safe.
// TODO(q3k): monitor this?
resp = &admission.AdmissionResponse{
UID: req.UID,
Allowed: false,
Result: &meta.Status{
Code: 500,
Message: "admitomatic: internal server error",
},
}
}
}
glog.Infof("%s %s %s in %s: %v (%v)", req.Operation, req.Kind.Kind, req.Name, req.Namespace, resp.Allowed, resp.Result)
review.Response = resp
json.NewEncoder(w).Encode(review)
}