# Main configuration file for edge01.waw.bgp.wtf. # This includes everything needed to run the machine, except for hardware # configuration, which is defined in //bgpwtf/machines/ # edge01.waw.bgp.wtf-hardware.nix. # # Any changes here can be tested in a local NixOS test by running the following: # # nix-build -A bgpwtf.machines.tests.edge01-waw # # To deploy changes, see //ops:machines.nix. { config, pkgs, ... }: with builtins; let passwords = import ./secrets/plain/passwords.nix; in rec { networking.hostName = "edge01"; networking.domain = "waw.bgp.wtf"; imports = [ ./modules/router.nix ./modules/anchorvm.nix # Private configuration data - notably, customer data. ./secrets/plain/edge01.waw.bgp.wtf-private.nix ]; # TODO(q3k): make this generic, move to modules/router.nix. services.unbound = { enable = true; settings = { server = { interface = [ "185.236.240.1" "2a0d:eb00:2137::1" "127.0.0.1" ]; access-control = [ "185.236.240.0/22 allow" "2a0d:eb00::0/29 allow" "127.0.0.0/8 allow" ]; outgoing-interface = [ "185.236.240.1" "2a0d:eb00:2137::1" ]; cache-max-negative-ttl = [ "30" ]; local-zone = [ # Disable DoH in Firefox "\"use-application-dns.net\" static" ]; }; }; }; hscloud.rsh.enable = true; networking.wireguard.interfaces = { wg-fmt = { ips = [ "185.236.240.68/31" "2a0d:eb00:2137:1::e/127" ]; allowedIPsAsRoutes = false; listenPort = 51820; generatePrivateKeyFile = true; privateKeyFile = "/root/fmt-wg"; peers = [ { publicKey = "zxL/1Jr0LLwJwXDm8ZOWkuY3ZkHO3sC7TdSBh89CsWc="; allowedIPs = [ "0.0.0.0/0" "::/0" ]; endpoint = "[2a00:6340:4000:10::10]:41521"; } ]; }; }; hscloud.renameInterfaces = { # Link to Nitronet CPE. e1-nnet.mac = "ac:1f:6b:1c:d7:ae"; # Link to HSWAW Customs. e2-customs.mac = "ac:1f:6b:1c:d7:af"; # Link to management switch. e3-mgmt.mac = "ac:1f:6b:1c:d7:b0"; # Link to oob1. e4-oob.mac = "ac:1f:6b:1c:d7:b1"; e5.mac = "ac:1f:6b:1c:d7:b2"; e6.mac = "ac:1f:6b:1c:d7:b3"; # Link to dcsw01.hswaw.net e7-dcsw.mac = "ac:1f:6b:1c:db:06"; e8.mac = "ac:1f:6b:1c:db:07"; }; networking.interfaces.e7-dcsw.mtu = 9000; networking.vlans = { "vl-globalmix" = { interface = "e1-nnet"; id = 466; }; "vl-polmix" = { interface = "e1-nnet"; id = 2486; }; "vl-openpeering" = { interface = "e1-nnet"; id = 992; }; "vl-dcsw-l3" = { interface = "e7-dcsw"; id = 4001; }; "vl-dist-l3" = { interface = "e7-dcsw"; id = 3006; }; # Extra vlans contained in //bgpwtf/machines/secrets/plain/edge01.waw.bgp.wtf-private.nix }; networking.interfaces = { lo = { ipv4.addresses = [ { address = "185.236.240.1"; prefixLength = 32; } ]; ipv6.addresses = [ { address = "2a0d:eb00:2137::1"; prefixLength = 64; } ]; }; ## EPIX links via Nitronet. "vl-globalmix" = { ipv4.addresses = [ { address = "185.235.70.45"; prefixLength = 31; } ]; ipv6.addresses = [ { address = "2001:67c:778:fd40::b9eb:462d"; prefixLength = 127; } ]; }; "vl-polmix" = { ipv4.addresses = [ { address = "94.246.185.175"; prefixLength = 31; } ]; ipv6.addresses = [ { address = "2001:67c:778:fa40::5ef6:b9af"; prefixLength = 127; } ]; }; "vl-openpeering" = { ipv4.addresses = [ { address = "89.46.145.61"; prefixLength = 21; } ]; ipv6.addresses = [ { address = "2001:678:3ac::313"; prefixLength = 48; } ]; }; ## L3/mgmt links.. # To customs.hackerspace.pl. "e2-customs" = { ipv4.addresses = [ { address = "185.236.240.4"; prefixLength = 31; } ]; ipv6.addresses = [ { address = "2a0d:eb00:2137:1::2"; prefixLength = 127; } ]; }; # To mgmt. "e3-mgmt" = { ipv4.addresses = [ { address = "10.10.10.1"; prefixLength = 24; } ]; }; # To obb1. "e4-oob" = { ipv4.addresses = [ { address = "185.236.240.74"; prefixLength = 29; } ]; }; # To dcsw01, L3 (BGP). "vl-dcsw-l3" = { mtu = 9000; ipv4.addresses = [ { address = "185.236.240.6"; prefixLength = 31; } ]; ipv6.addresses = [ { address = "2a0d:eb00:2137:1::6"; prefixLength = 127; } ]; }; # To dist02, L3 (BGP). "vl-dist-l3" = { ipv4.addresses = [ { address = "185.236.240.14"; prefixLength = 31; } ]; ipv6.addresses = [ { address = "2a0d:eb00:2137:1::a"; prefixLength = 127; } ]; }; # VM bridge "br0" = { ipv4.addresses = [ { address = "185.236.240.17"; prefixLength = 29; } ]; ipv6.addresses = [ { address = "2a0d:eb00:2137:3::1"; prefixLength = 64; } ]; }; # Extra interface configs contained in //bgpwtf/machines/secrets/plain/edge01.waw.bgp.wtf-private.nix }; networking.bridges = { "br0" = { interfaces = []; }; }; hscloud.anchorvm = { bridge = "br0"; }; hscloud.routing.enable = true; hscloud.routing.routerID = "185.236.240.1"; hscloud.routing.asn = 204880; # Use default master4/master6 tables so that `birdc show route` works. hscloud.routing.tables.master.program = true; hscloud.routing.tables.master.programSourceV4 = "185.236.240.1"; hscloud.routing.tables.master.programSourceV6 = "2a0d:eb00:2137::1"; hscloud.routing.extra = '' function net_martian_v4() { return net ~ [ 169.254.0.0/16+, 172.16.0.0/12+, 192.168.0.0/16+, 10.0.0.0/8+, 127.0.0.0/8+, 224.0.0.0/4+, 240.0.0.0/4+, 0.0.0.0/32-, 0.0.0.0/0{25,32}, 0.0.0.0/0{0,7} ]; } function net_as204480_waw_v4() { return net ~ [ 185.236.240.0/23+ ]; } function net_martian_v6() { return net ~ [ fc00::/7+, fec0::/10+, ::/128-, ::/0{0,15}, ::/0{49,128} ]; } function net_as204480_waw_v6() { return net ~ [ 2a0d:eb00::/32 ]; } ''; hscloud.routing.originate = { # WAW prefixes, exposed into internet BGP table. v4.waw = { table = "internet"; address = "185.236.240.0"; prefixLength = 23; }; v6.waw = { table = "internet"; address = "2a0d:eb00::"; prefixLength = 32; }; # Default gateway via us, exposed into aggregated table. v4.default = { table = "aggregate"; address = "0.0.0.0"; prefixLength = 0; }; v6.default = { table = "aggregate"; address = "::"; prefixLength = 0; }; # Loopbacks for IGP table. # Alternatively we could add 'lo' as a stub interface into IGP OSPF, but # that would also add 127.0.0.1... v4.loopbacks = { table = "igp"; address = "185.236.240.1"; prefixLength = 32; }; v6.loopbacks = { table = "igp"; address = "2a0d:eb00:2137::1"; prefixLength = 128; }; }; hscloud.routing.pipe = let copySourcesToKernel = sources: table: extra: { table = "master"; peerTable = table; filterIn = '' ${extra} ${concatStringsSep "\n" (map (v: "if source = RTS_${v} then accept;") sources)} reject; ''; }; in { v4."internet_to_kernel" = copySourcesToKernel ["BGP" "OSPF"] "internet" ""; v4."aggregate_to_kernel" = copySourcesToKernel ["BGP" "OSPF"] "aggregate" '' # Static v4 routes for customers. if proto ~ "static_static_ipv4_customer_*" then accept; ''; v6."internet_to_kernel" = copySourcesToKernel ["BGP" "OSPF"] "internet" ""; v6."aggregate_to_kernel" = copySourcesToKernel ["BGP" "OSPF"] "aggregate" '' # Static v6 routes for customers. if proto ~ "static_static_ipv6_customer_*" then accept; ''; }; hscloud.routing.ospf.v6.main = { area."0.0.0.0".interfaces = { "e2-customs" = { type = "bcast"; }; "e4-oob" = { type = "bcast"; stub = true; }; }; table = "aggregate"; filterIn = '' # hswaw prefix from e2-customs if net ~ [ 2a0d:eb00:4242::/48+ ] then accept; # e2-customs link if net ~ [ 2a0d:eb00:2137:1::2/127+ ] then accept; ''; }; hscloud.routing.ospf.v4.main = { area."0.0.0.0".interfaces = { "e4-oob" = { type = "bcast"; stub = true; }; }; table = "aggregate"; filterIn = '' # e4-oob link if net ~ [ 185.236.240.72/29+ ] then accept; ''; }; hscloud.routing.ospf.v6.igp = { area."0.0.0.0".interfaces = { "wg-fmt" = { type = "ptmp"; neighbors = [ "2a0d:eb00:2137:1::f" ]; }; }; table = "igp"; filterIn = '' # fmt networks if net ~ [ 2a0d:eb01::/48+ ] then accept; ''; }; hscloud.routing.bgpSessions.v4 = let filterInUpstream = '' if net_martian_v4() then reject; if net_as204480_waw_v4() then reject; accept; ''; filterOutUpstream = '' # Accept AS204880-announced prefixes. if (net ~ [ 185.236.240.0/22+ ]) then accept; reject; ''; in { "waw_globalmix" = { description = "UPSTREAM EPIX.WAR GlobalMix"; table = "internet"; local = "185.235.70.45"; neighbors = [ { address = "185.235.70.44"; asn = 62081; } ]; prepend = 2; pref = 100; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; "waw_polmix" = { description = "UPSTREAM EPIX.WAR PolMix"; table = "internet"; local = "94.246.185.175"; neighbors = [ { address = "94.246.185.174"; asn = 201054; } ]; prepend = 1; pref = 200; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; "waw_openpeering" = { description = "IXP EPIX.WAR OpenPeering"; table = "internet"; local = "89.46.145.61"; neighbors = [ { address = "89.46.144.11"; asn = 48850; } { address = "89.46.144.12"; asn = 48850; } ]; prepend = 0; pref = 300; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; "waw_google" = { description = "PEER Google AS15169 (EPIX)"; table = "internet"; local = "89.46.145.61"; neighbors = [ # TODO(q3k): secretify the password. { address = "89.46.144.185"; asn = 15169; password = passwords."edge01.waw-bgp-google"; } ]; prepend = 0; pref = 300; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; "waw_cloudflare" = { description = "PEER Cloudflare AS13335 (EPIX)"; table = "internet"; local = "89.46.145.61"; neighbors = [ { address = "89.46.144.83"; asn = 13335; } ]; prepend = 0; pref = 300; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; # hscloud spine switch (dcsw01.hswaw.net). "waw_hscloud" = { description = "AGGREGATE CUSTOMER hscloud/dcsw01"; table = "aggregate"; local = "185.236.240.6"; asn = 65000; neighbors = [ { address = "185.236.240.7"; asn = 65001; } ]; filterIn = '' # wieloryb prefix if net ~ [ 185.236.240.8/31+ ] then accept; # dcsw01 l2 general purpose if net ~ [ 185.236.240.24/29+ ] then accept; # hscloud l2 general purpose if net ~ [ 185.236.240.32/28+ ] then accept; # k0 metallb pools if net ~ [ 185.236.240.48/28+, 185.236.240.112/28+ ] then accept; # dcsw01.hswaw.net / dcr03sw48.hswaw.net if net ~ [ 185.236.240.66/31 ] then accept; # dcr03 mgmt if net ~ [ 10.10.32.0/24 ] then accept; reject; ''; }; # bgp.wtf internet customer router on W2A, floor 3 (dist02.bgp.wtf). "waw_dist02" = { description = "AGGREGATE CUSTOMER bgpwtf/dist02"; table = "aggregate"; local = "185.236.240.14"; asn = 65000; neighbors = [ { address = "185.236.240.15"; asn = 65002; } ]; filterIn = '' # dist02 customer routed if net ~ [ 185.236.240.80/28+ ] then accept; reject; ''; }; # backup LTE link to edge01.fra "fra_edge01" = { description = "IBGP edge01.fra"; table = "internet"; local = "185.236.240.74"; direct = true; neighbors = [ { address = "185.236.240.75"; asn = 204880; } ]; pref = 50; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; }; hscloud.routing.bgpSessions.v6 = let filterInUpstream = '' if net_martian_v6() then reject; if net_as204480_waw_v6() then reject; accept; ''; filterOutUpstream = '' # Accept AS204880-announced prefixes. if (net ~ [ 2a0d:eb00::/29+ ]) then accept; reject; ''; in { "waw_globalmix" = { description = "UPSTREAM EPIX.WAR GlobalMix"; table = "internet"; local = "2001:67c:778:fd40::b9eb:462d"; neighbors = [ { address = "2001:67c:778:fd40::b9eb:462c"; asn = 62081; } ]; prepend = 2; pref = 100; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; "waw_polmix" = { description = "UPSTREAM EPIX.WAR PolMix"; table = "internet"; local = "2001:67c:778:fa40::5ef6:b9af"; neighbors = [ { address = "2001:67c:778:fa40::5ef6:b9ae"; asn = 201054; } ]; prepend = 1; pref = 200; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; "waw_openpeering" = { description = "IXP EPIX.WAR OpenPeering"; table = "internet"; local = "2001:678:3ac::313"; neighbors = [ { address = "2001:678:3ac::11"; asn = 48850; } { address = "2001:678:3ac::12"; asn = 48850; } ]; prepend = 0; pref = 300; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; "waw_google" = { description = "PEER Google AS15169 (EPIX)"; table = "internet"; local = "2001:678:3ac::313"; neighbors = [ { address = "2001:678:3ac::185"; asn = 15169; password = passwords."edge01.waw-bgp-google"; } ]; prepend = 0; pref = 300; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; "waw_cloudflare" = { description = "PEER Cloudflare AS13335 (EPIX)"; table = "internet"; local = "2001:678:3ac::313"; neighbors = [ { address = "2001:678:3ac::83"; asn = 13335; } ]; prepend = 0; pref = 300; filterIn = filterInUpstream; filterOut = filterOutUpstream; }; # hscloud spine switch (dcsw01.hswaw.net). "waw_hscloud" = { description = "AGGREGATE CUSTOMER dcsw01.hswaw.net"; table = "aggregate"; local = "2a0d:eb00:2137:1::6"; asn = 65000; neighbors = [ { address = "2a0d:eb00:2137:1::7"; asn = 65001; } ]; filterIn = '' # dcsw01 l2 general purpose if net ~ [ 2a0d:eb00:2137::/48+ ] then accept; # customer if net ~ [ 2a0d:eb00:8004::/48+ ] then accept; reject; ''; }; # bgp.wtf internet customer router on W2A, floor 3 (dist02.bgp.wtf). "waw_dist02" = { description = "AGGREGATE CUSTOMER dist02.bgp.wtf"; table = "aggregate"; local = "2a0d:eb00:2137:1::a"; asn = 65000; neighbors = [ { address = "2a0d:eb00:2137:1::b"; asn = 65002; } ]; filterIn = '' # dist02 customers. if net ~ [ 2a0d:eb00:8002::/48 ] then accept; reject; ''; }; }; }