bgpwtf/cccampix: add IRR daemon

We add a small IRR service for getting a parsed RPSL from IRRs. For now,
we only support RIPE and ARIN, and only the following attributes:
 - remarks
 - import
 - export

Since RPSL/RFC2622 is fucking insane, there is no guarantee that the
parser, especially the import/export parser, is correct. But it should
be good enough for our use. We even throw in some tests for good
measure.

    $ grpcurl -format text -plaintext -d 'as: "26625"' 127.0.0.1:4200 ix.IRR.Query
    source: SOURCE_ARIN
    attributes: <
      import: <
        expressions: <
          peering: "AS6083"
          actions: "pref=10"
        >
        filter: "ANY"
      >
    >
    attributes: <
      import: <
        expressions: <
          peering: "AS12491"
          actions: "pref=10"
        >
        filter: "ANY"
      >
    >

Change-Id: I8b240ffe2cd3553a25ce33dbd3917c0aef64e804
changes/03/3/1
q3k 2019-08-02 01:25:31 +02:00
parent 0607abae1d
commit 6eaaaf9bab
14 changed files with 836 additions and 0 deletions

View File

@ -599,3 +599,15 @@ go_repository(
commit = "c182affec369e30f25d3eb8cd8a478dee585ae7d",
importpath = "github.com/matttproud/golang_protobuf_extensions",
)
go_repository(
name = "com_github_golang_collections_go_datastructures",
commit = "59788d5eb2591d3497ffb8fafed2f16fe00e7775",
importpath = "github.com/golang-collections/go-datastructures",
)
go_repository(
name = "com_github_go_test_deep",
commit = "cf67d735e69b4a4d50cdf571a92b0144786080f7",
importpath = "github.com/go-test/deep",
)

View File

@ -0,0 +1,22 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
go_library(
name = "go_default_library",
srcs = ["main.go"],
importpath = "code.hackerspace.pl/hscloud/bgpwtf/cccampix/irr",
visibility = ["//visibility:private"],
deps = [
"//bgpwtf/cccampix/irr/provider:go_default_library",
"//bgpwtf/cccampix/proto:go_default_library",
"//go/mirko:go_default_library",
"@com_github_golang_glog//:go_default_library",
"@org_golang_google_grpc//codes:go_default_library",
"@org_golang_google_grpc//status:go_default_library",
],
)
go_binary(
name = "irr",
embed = [":go_default_library"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,69 @@
irr
===
A proxy to access IRR RPSL data. It queries IANA for the responsible IRR, then the IRR directly.
It currently support ARIN and RIPE.
It currently supports querying for information about an aut-num, and returns the following attributes:
- remarks
- import
- export
Example
-------
$ grpcurl -format text -plaintext -d 'as: "26625"' 127.0.0.1:4220 ix.IRR.Query
source: SOURCE_ARIN
attributes: <
import: <
expressions: <
peering: "AS6083"
actions: "pref=10"
>
filter: "ANY"
>
>
attributes: <
import: <
expressions: <
peering: "AS12491"
actions: "pref=10"
>
filter: "ANY"
>
>
attributes: <
import: <
expressions: <
peering: "AS20459"
actions: "pref=10"
>
filter: "ANY"
>
>
attributes: <
export: <
expressions: <
peering: "AS6083"
>
filter: "AS26625"
>
>
attributes: <
export: <
expressions: <
peering: "AS12491"
>
filter: "AS26625"
>
>
attributes: <
export: <
expressions: <
peering: "AS20459"
>
filter: "AS26625"
>
>

View File

@ -0,0 +1,76 @@
package main
import (
"context"
"flag"
"strconv"
"strings"
"github.com/golang/glog"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"code.hackerspace.pl/hscloud/bgpwtf/cccampix/irr/provider"
pb "code.hackerspace.pl/hscloud/bgpwtf/cccampix/proto"
"code.hackerspace.pl/hscloud/go/mirko"
)
type service struct {
iana *provider.IANA
providers map[provider.IRR]provider.Provider
}
func main() {
flag.Parse()
mi := mirko.New()
if err := mi.Listen(); err != nil {
glog.Exitf("Listen failed: %v", err)
}
s := &service{
iana: provider.NewIANA(),
providers: map[provider.IRR]provider.Provider{
provider.IRR_RIPE: provider.NewRIPE(),
provider.IRR_ARIN: provider.NewARIN(),
},
}
pb.RegisterIRRServer(mi.GRPC(), s)
if err := mi.Serve(); err != nil {
glog.Exitf("Serve failed: %v", err)
}
<-mi.Done()
}
// Query returns parsed RPSL data for a given aut-num objects in any of the
// supported IRRs.
func (s *service) Query(ctx context.Context, req *pb.IRRQueryRequest) (*pb.IRRQueryResponse, error) {
if req.As == "" {
return nil, status.Error(codes.InvalidArgument, "as must be given")
}
req.As = strings.ToLower(req.As)
if strings.HasPrefix(req.As, "as") {
req.As = req.As[2:]
}
asn, err := strconv.ParseUint(req.As, 10, 64)
if err != nil {
return nil, status.Error(codes.InvalidArgument, "as is invalid")
}
irr, err := s.iana.Who(ctx, asn)
if err != nil {
return nil, status.Errorf(codes.Unavailable, "could not find AS block delegation from IANA: %v", err)
}
prov, ok := s.providers[irr]
if !ok {
return nil, status.Errorf(codes.NotFound, "AS belongs to unhandled IRR %s", irr.String())
}
res, err := prov.Query(ctx, asn)
return res, err
}

View File

@ -0,0 +1,30 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"arin.go",
"iana.go",
"provider.go",
"ripe.go",
"rpsl.go",
],
importpath = "code.hackerspace.pl/hscloud/bgpwtf/cccampix/irr/provider",
visibility = ["//visibility:public"],
deps = [
"//bgpwtf/cccampix/irr/whois:go_default_library",
"//bgpwtf/cccampix/proto:go_default_library",
"@com_github_golang_collections_go_datastructures//augmentedtree:go_default_library",
"@com_github_golang_glog//:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["rpsl_test.go"],
embed = [":go_default_library"],
deps = [
"//bgpwtf/cccampix/proto:go_default_library",
"@com_github_go_test_deep//:go_default_library",
],
)

View File

@ -0,0 +1,78 @@
package provider
// Support for the ARIN IRR.
// ARIN is special. We have to query them via whois. It's also not the same
// whois as you usually see. And also not many autnums (even big players like
// AS15169 and AS112) have IRR entries at all.
import (
"context"
"fmt"
"strings"
"code.hackerspace.pl/hscloud/bgpwtf/cccampix/irr/whois"
pb "code.hackerspace.pl/hscloud/bgpwtf/cccampix/proto"
)
const ARINWhois = "rr.arin.net:43"
type arin struct {
}
func NewARIN() Provider {
return &arin{}
}
func (r *arin) Query(ctx context.Context, asn uint64) (*pb.IRRQueryResponse, error) {
data, err := whois.Query(ctx, ARINWhois, fmt.Sprintf("AS%d", asn))
if err != nil {
return nil, fmt.Errorf("could not contact ARIN IRR: %v", err)
}
lines := strings.Split(data, "\n")
// Convert possibly 'continued' RPSL entries into single-line entries.
// eg.
// import: from AS6083
// action pref=10;
// accept ANY
// into
// 'import', 'from AS6083, action pref=10; accept ANY'
attrs := []rpslRawAttribute{}
for _, line := range lines {
if strings.HasPrefix(strings.TrimSpace(line), "%") {
// Comment
continue
}
if strings.TrimSpace(line) == "" {
// Empty line
continue
}
if strings.HasPrefix(line, " ") {
// Continuation
if len(attrs) < 1 {
return nil, fmt.Errorf("unparseable IRR, continuation with no previous atribute name: %q", line)
}
attrs[len(attrs)-1].value += " " + strings.TrimSpace(line)
} else {
parts := strings.SplitN(line, ":", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("unparseable IRR, line with no attribute key: %q", line)
}
name := strings.TrimSpace(parts[0])
value := strings.TrimSpace(parts[1])
attrs = append(attrs, rpslRawAttribute{
name: name,
value: value,
})
}
}
return &pb.IRRQueryResponse{
Source: pb.IRRQueryResponse_SOURCE_ARIN,
Attributes: parseAttributes(attrs),
}, nil
}

View File

@ -0,0 +1,163 @@
package provider
// IANA is not a full provider - we use it to determine the relevant IRR for a given aut-num.
import (
"context"
"fmt"
"strconv"
"strings"
"github.com/golang-collections/go-datastructures/augmentedtree"
"github.com/golang/glog"
"code.hackerspace.pl/hscloud/bgpwtf/cccampix/irr/whois"
)
// Possible IRRs that AS blocks can be delegated to.
type IRR int
const (
IRR_UNKNOWN IRR = iota
IRR_ARIN
IRR_RIPE
)
func (i IRR) String() string {
return []string{
"UNKNOWN",
"ARIN",
"RIPE",
}[i]
}
// IANA access service.
type IANA struct {
// Interval tree of AS block delegation to IRRs. We use this to not
// keep hitting the IANA whois unnecessariy.
cache augmentedtree.Tree
// The tree library needs intervals to have a unique ID. We use a counter
// for this effect.
id uint64
}
func NewIANA() *IANA {
return &IANA{
cache: augmentedtree.New(1),
}
}
func (i *IANA) nextID() uint64 {
res := i.id
i.id += 1
return res
}
// delegation implements the Interval interface for the interval tree.
type delegation struct {
to IRR
id uint64
low int64
high int64
}
func (d *delegation) LowAtDimension(n uint64) int64 {
if n != 1 {
panic(fmt.Sprintf("dimension too high (%d)", n))
}
return d.low
}
func (d *delegation) HighAtDimension(n uint64) int64 {
if n != 1 {
panic(fmt.Sprintf("dimension too high (%d)", n))
}
return d.high
}
func (d *delegation) OverlapsAtDimension(i augmentedtree.Interval, n uint64) bool {
if n != 1 {
return false
}
if i.LowAtDimension(1) <= d.HighAtDimension(1) && i.HighAtDimension(1) >= d.LowAtDimension(1) {
return true
}
return false
}
func (d *delegation) ID() uint64 {
return d.id
}
// Who returns the responsible IRR (or UNKNOWN) for a given AS.
func (i *IANA) Who(ctx context.Context, asn uint64) (IRR, error) {
q := &delegation{
id: i.nextID(),
low: int64(asn),
high: int64(asn),
}
res := i.cache.Query(q)
if len(res) > 0 {
return res[0].(*delegation).to, nil
}
// No cache entry, query whois.
glog.Infof("Cache miss for AS%d", asn)
data, err := whois.Query(ctx, "whois.iana.org:43", fmt.Sprintf("AS%d", asn))
if err != nil {
return IRR_UNKNOWN, err
}
// We not only find the responsible IRR, but also the delegation bounds
// to feed the interval tree.
lines := strings.Split(data, "\n")
var lower int64
var upper int64
var irr IRR
for _, line := range lines {
parts := strings.Fields(line)
if len(parts) == 2 && parts[0] == "as-block:" {
bounds := strings.Split(parts[1], "-")
if len(bounds) != 2 {
return IRR_UNKNOWN, fmt.Errorf("unparseable as-block: %v", parts[1])
}
lower, err = strconv.ParseInt(bounds[0], 10, 64)
if err != nil {
return IRR_UNKNOWN, fmt.Errorf("unparseable as-block: %v", parts[1])
}
upper, err = strconv.ParseInt(bounds[1], 10, 64)
if err != nil {
return IRR_UNKNOWN, fmt.Errorf("unparseable as-block: %v", parts[1])
}
}
if len(parts) > 2 && parts[0] == "organisation:" {
switch {
case strings.Contains(line, "RIPE NCC"):
irr = IRR_RIPE
case strings.Contains(line, "ARIN"):
irr = IRR_ARIN
}
}
}
if lower == 0 || upper == 0 {
return IRR_UNKNOWN, nil
}
glog.Infof("Saving %d-%d AS blocks delegation (%s) to cache", lower, upper, irr.String())
// Insert into tree.
q = &delegation{
id: i.nextID(),
to: irr,
low: lower,
high: upper + 1,
}
i.cache.Add(q)
return irr, nil
}

View File

@ -0,0 +1,13 @@
package provider
import (
"context"
pb "code.hackerspace.pl/hscloud/bgpwtf/cccampix/proto"
)
// Provider is the interface exposed to the service by IRR proxies.
type Provider interface {
// Return a proto response for a given AS.
Query(ctx context.Context, as uint64) (*pb.IRRQueryResponse, error)
}

View File

@ -0,0 +1,81 @@
package provider
// Support for the RIPE IRR.
// We use the RIPE REST DB API.
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
pb "code.hackerspace.pl/hscloud/bgpwtf/cccampix/proto"
)
type ripeResponse struct {
Objects struct {
Object []ripeObject `json:"object"`
} `json:"objects"`
}
type ripeObject struct {
Type string `json:"type"`
Attributes struct {
Attribute []ripeAttribute `json:"attribute"`
} `json:"attributes"`
}
type ripeAttribute struct {
Name string `json:"name"`
Value string `json:"value"`
}
type ripe struct {
}
func NewRIPE() Provider {
return &ripe{}
}
func (r *ripe) Query(ctx context.Context, as uint64) (*pb.IRRQueryResponse, error) {
req, err := http.NewRequest("GET", fmt.Sprintf("http://rest.db.ripe.net/ripe/aut-num/AS%d.json", as), nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
client := http.DefaultClient
res, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("could not run GET to RIPE: %v", err)
}
defer res.Body.Close()
bytes, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, fmt.Errorf("could not read response from RIPE: %v", err)
}
data := ripeResponse{}
err = json.Unmarshal(bytes, &data)
if err != nil {
return nil, fmt.Errorf("could not decode response from RIPE: %v", err)
}
if len(data.Objects.Object) != 1 {
return nil, fmt.Errorf("could not retriev aut-num from RIPE")
}
attributes := make([]rpslRawAttribute, len(data.Objects.Object[0].Attributes.Attribute))
for i, attr := range data.Objects.Object[0].Attributes.Attribute {
attributes[i].name = attr.Name
attributes[i].value = attr.Value
}
return &pb.IRRQueryResponse{
Source: pb.IRRQueryResponse_SOURCE_RIPE,
Attributes: parseAttributes(attributes),
}, nil
}

View File

@ -0,0 +1,146 @@
package provider
import (
"strings"
pb "code.hackerspace.pl/hscloud/bgpwtf/cccampix/proto"
)
// A raw, unparsed RPSL name/value pair.
type rpslRawAttribute struct {
name string
value string
}
// parseAttributes converts raw RPSL attributes into parsed, structured protos.
func parseAttributes(attrs []rpslRawAttribute) []*pb.IRRAttribute {
res := []*pb.IRRAttribute{}
for _, attr := range attrs {
switch attr.name {
case "remarks":
res = append(res, &pb.IRRAttribute{
Value: &pb.IRRAttribute_Remarks{attr.value},
})
case "import":
ie := parseImportExport(attr.value)
if ie != nil {
res = append(res, &pb.IRRAttribute{
Value: &pb.IRRAttribute_Import{ie},
})
}
case "export":
ie := parseImportExport(attr.value)
if ie != nil {
res = append(res, &pb.IRRAttribute{
Value: &pb.IRRAttribute_Export{ie},
})
}
}
}
return res
}
// parseImportExport tries to parse the RPSL subset for import/export
// attributes.
// It's a hand-written single-pass parser which makes it not very good.
// See rpsl_test.go for examples.
func parseImportExport(expression string) *pb.IRRAttribute_ImportExport {
popToken := func(s string) (string, string) {
s = strings.TrimSpace(s)
fields := strings.Fields(s)
if len(fields) < 2 {
return "", s
}
return fields[0], strings.TrimSpace(s[len(fields[0]):])
}
res := &pb.IRRAttribute_ImportExport{}
expr := expression
var t string
t, expr = popToken(expr)
for {
switch t {
case "":
break
case "protocol":
t, expr = popToken(expr)
res.ProtocolFrom = t
t, expr = popToken(expr)
continue
case "into":
t, expr = popToken(expr)
res.ProtocolInto = t
t, expr = popToken(expr)
continue
case "from":
fallthrough
case "to":
t, expr = popToken(expr)
peering := t
actions := []string{}
router_us := ""
router_them := ""
t2, expr2 := popToken(expr)
for {
switch t2 {
case "at":
t2, expr2 = popToken(expr2)
router_us = t2
t2, expr2 = popToken(expr2)
continue
case "action":
parts := strings.SplitN(expr2, ";", 2)
if len(parts) != 2 {
// malformed action, no ';' found - bail
return nil
}
actions = append(actions, parts[0])
expr2 = strings.TrimSpace(parts[1])
t2, expr2 = popToken(expr2)
continue
case "":
fallthrough
case "accept":
fallthrough
case "announce":
t = t2
expr = expr2
break
default:
if router_them == "" {
router_them = t2
t2, expr2 = popToken(expr2)
continue
}
t = t2
expr = expr2
break
}
break
}
res.Expressions = append(res.Expressions, &pb.IRRAttribute_ImportExport_Expression{
Peering: peering,
RouterUs: router_us,
RouterThem: router_them,
Actions: actions,
})
continue
case "accept":
fallthrough
case "announce":
res.Filter = expr
t = ""
continue
default:
return nil
}
break
}
return res
}

View File

@ -0,0 +1,64 @@
package provider
import (
"testing"
pb "code.hackerspace.pl/hscloud/bgpwtf/cccampix/proto"
"github.com/go-test/deep"
)
func TestParseImportExport(t *testing.T) {
tests := []struct {
ut string
want *pb.IRRAttribute_ImportExport
}{
{
ut: `
from AS10674 209.251.128.177 at 216.155.103.20 action pref = 1;
accept ANY AND NOT AS-ACCELERATION-CUST
`,
want: &pb.IRRAttribute_ImportExport{
Expressions: []*pb.IRRAttribute_ImportExport_Expression{
{
Peering: "AS10674",
RouterUs: "216.155.103.20",
RouterThem: "209.251.128.177",
Actions: []string{"pref = 1"},
},
},
Filter: "ANY AND NOT AS-ACCELERATION-CUST",
},
},
{
ut: `
to AS201054 94.246.185.174 at 94.246.185.175 announce AS-BGPWTF
`,
want: &pb.IRRAttribute_ImportExport{
Expressions: []*pb.IRRAttribute_ImportExport_Expression{
{
Peering: "AS201054",
RouterUs: "94.246.185.175",
RouterThem: "94.246.185.174",
Actions: []string{},
},
},
Filter: "AS-BGPWTF",
},
},
{
// Invalid - unterminated action.
ut: `
to AS201054 94.246.185.174 at 94.246.185.175 action foo = bar
accept ANY AND NOT AS-ACCELERATION-CUST
`,
want: nil,
},
}
for _, test := range tests {
res := parseImportExport(test.ut)
if diff := deep.Equal(test.want, res); diff != nil {
t.Error(diff)
}
}
}

View File

@ -0,0 +1,8 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = ["whois.go"],
importpath = "code.hackerspace.pl/hscloud/bgpwtf/cccampix/irr/whois",
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,32 @@
package whois
// Support for the WHOIS protocol.
import (
"context"
"fmt"
"io/ioutil"
"net"
"strings"
)
// Query returns a semi-raw response from a WHOIS server for a given query.
// We only convert \r\n to \n, and then do no other transformation to the data.
func Query(ctx context.Context, server string, query string) (string, error) {
var d net.Dialer
conn, err := d.DialContext(ctx, "tcp", server)
if err != nil {
return "", fmt.Errorf("while dialing %q: %v", server, err)
}
defer conn.Close()
fmt.Fprintf(conn, "%s\r\n", query)
data, err := ioutil.ReadAll(conn)
if err != nil {
return "", fmt.Errorf("while receiving data from %q: %v", server, err)
}
return strings.ReplaceAll(string(data), "\r", ""), nil
}

View File

@ -39,3 +39,45 @@ service PeeringDBProxy {
// GetIXMembers returns information about membership of a given PeeringDB IX.
rpc GetIXMembers(GetIXMembersRequest) returns (GetIXMembersResponse);
}
message IRRQueryRequest {
// AS to query for. This needs be the AS number of the AS, possibly
// prefixed with 'as'/'AS'.
string as = 1;
}
message IRRAttribute {
message ImportExport {
message Expression {
string peering = 1;
string router_us = 2;
string router_them = 3;
repeated string actions = 4;
}
string protocol_from = 1;
string protocol_into = 2;
repeated Expression expressions = 3;
string filter = 4;
}
oneof value {
string remarks = 1;
ImportExport import = 2;
ImportExport export = 3;
}
}
message IRRQueryResponse {
enum Source {
SOURCE_INVALID = 0;
SOURCE_RIPE = 1;
SOURCE_ARIN = 2;
}
Source source = 1;
repeated IRRAttribute attributes = 2;
}
service IRR {
// Query returns parsed RPSL data from supported IRRs for a given aut-num.
rpc Query(IRRQueryRequest) returns (IRRQueryResponse);
}