check in checkinator into hswaw/checkinator

repository: https://code.hackerspace.pl/checkinator
revision: 713c7e6c1a8fd6147522c1a5e3067898a1d8bf7a

Change-Id: I1bd2975a46ec0d9a89d6594fb4b9d49832001627
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1219
Reviewed-by: q3k <q3k@hackerspace.pl>

hswaw/checkinator/.gitignore

@@ -0,0 +1,8 @@
+at.cfg
+at.db
+*.egg-info
+venv
+**/*_pb2*.py
+*-config.yaml
+result
+cert-*

hswaw/checkinator/README.rst

@@ -0,0 +1,32 @@
+`Warsaw Hackerspace`_  presence tracker hosted on https://at.hackersapce.pl. It
+uses dhcpd.leases file to track MAC adressess of devices connected to hs LAN
+network.
+
+.. _Warsaw Hackerspace: https://hackerspace.pl
+
+Setup
+-----
+.. code:: bash
+
+    cp config.yaml.dist config.yaml
+
+    # edit config file using your favourite editor
+    $EDITOR config.yaml
+
+    # create new database file (or copy existing one)
+    sqlite3 at.db < dbsetup.sql
+
+    # create python virtual environment
+    python3 -m venv vevnv
+    ./venv/bin/python3 -m pip install -r requirements
+    ./venv/bin/python3 -m pip install gunicorn
+
+Running
+-------
+.. code:: bash
+
+   ./venv/bin/gunicorn run:app
+
+When running on OpenBSD make sure to pass '--no-sendfile' argument to gunicorn
+command. This will prevent AttributeError on os.sendfile that seems to be
+missing in this marvelous OS-es python3 stdlib.

hswaw/checkinator/at/cmd.py

@@ -0,0 +1,36 @@
+import argparse
+from pathlib import Path
+from at.dhcp import parse_isc_dhcpd_leases
+from time import time
+
+def list():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("leases", type=Path, help="leases file")
+    parser.add_argument("--timeout", type=int, default=None, help="timeout in minutes")
+    args = parser.parse_args()
+
+    with open(args.leases) as f:
+        offset, devices = parse_isc_dhcpd_leases(f)
+    if args.timeout is not None:
+        devices.purge_stale(args.timeout * 60)
+    print("Found devices:")
+    for device in devices._devices.values():
+        print(device._replace(atime = time() - device.atime))
+
+import grpc
+from .tracker_pb2 import ClientsRequest
+from .tracker_pb2_grpc import DhcpTrackerStub
+
+def format_mac(raw: bytes) -> str:
+    return ':'.join(f'{b:02x}' for b in raw)
+
+def tracker_list():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("address", default='unix:///tmp/checkinator.sock', nargs='?', help="tracker grpc address")
+    parser.add_argument("--timeout", type=int, default=None, help="timeout in minutes")
+    args = parser.parse_args()
+    with grpc.insecure_channel(args.address) as channel:
+        stub = DhcpTrackerStub(channel)
+        response = stub.GetClients(ClientsRequest())
+    for client in response.clients:
+        print(format_mac(client.hw_address), client.last_seen, client.ip_address, client.client_hostname )

hswaw/checkinator/at/dhcp.py

@@ -0,0 +1,234 @@
+import threading
+import os
+import io
+
+import logging
+
+from typing import Tuple, List, Optional, NamedTuple
+from time import sleep, time, mktime
+from datetime import datetime, timezone
+
+logger = logging.getLogger(__name__)
+
+def strfts(ts, format='%d/%m/%Y %H:%M'):
+    return datetime.fromtimestamp(ts).strftime(format)
+
+class DhcpLease(NamedTuple):
+    hwaddr: Optional[str]
+    atime: Optional[float]
+    ip: Optional[str]
+    name: Optional[str]
+
+
+class ActiveDevices:
+    def __init__(self):
+        self._devices = {}
+
+    def purge_stale(self, timeout):
+        now = time()
+        for device in list(self._devices.values()):
+            if now - device.atime > timeout:
+                del self._devices[device.hwaddr]
+
+    def add(self, lease: DhcpLease) -> bool:
+        if lease.atime is None:
+            lease = lease._replace(atime=time())
+        if lease.hwaddr not in self._devices or self._devices[lease.hwaddr].atime < lease.atime:
+            self._devices[lease.hwaddr] = lease
+            return True
+        return False
+
+    def update(self, devices) -> List[str]:
+        '''Add entries from another ActiveDevices instance
+
+        Args:
+            devices: list of entries to be added
+
+        Returns: list of updated enties
+        '''
+
+        updated = []
+        for device in devices._devices.values():
+            if self.add(device):
+                updated.append(device)
+        return updated
+
+class Updater(threading.Thread):
+    def __init__(self,  timeout, logger=logger, *a, **kw):
+        self.timeout = timeout
+        self.lock = threading.Lock()
+        self.logger = logger
+        self.active = ActiveDevices()
+        threading.Thread.__init__(self, *a, **kw)
+        self.daemon = True
+
+    def get_active_devices(self):
+        with self.lock:
+            self.active.purge_stale(self.timeout)
+            return dict(self.active._devices)
+
+    def get_device(self, ip):
+        with self.lock:
+            active_devices = iter(self.get_active_devices().values())
+        for device in active_devices:
+            if device.ip == ip:
+                return device.hwaddr, device.name
+        return None, None
+
+    def update(self, devices: ActiveDevices):
+        for device in devices._devices.values():
+            with self.lock:
+                changed = self.active.add(device)
+            if changed:
+                self.logger.info('updated %s with atime %s and ip %s',
+                    device.hwaddr, strfts(device.atime), device.ip)
+
+#class CapUpdater(Updater):
+#    def __init__(self, cap_file, *a, **kw):
+#        self.cap_file = cap_file
+#        Updater.__init__(self, *a, **kw)
+#
+#    def run(self):
+#        while True:
+#            try:
+#                with open(self.cap_file, 'r', buffering=0) as f:
+#                    self.logger.info('Updater ready on cap file %s',
+#                                    self.cap_file)
+#                    lines = [l.strip() for l in f.read().split('\n')]
+#                    for hwaddr in lines:
+#                        if hwaddr:
+#                            self.update(hwaddr)
+#                self.logger.warning('Cap file %s closed, reopening',
+#                                   self.cap_file)
+#            except Exception as e:
+#                self.logger.error('Updater got an exception:\n' +
+#                                 traceback.format_exc(e))
+#                sleep(10.0)
+
+
+class MtimeUpdater(Updater):
+    def __init__(self, lease_file, *a, **kw):
+        self.lease_file = lease_file
+        self.position = 0
+        self.last_modified = 0
+        Updater.__init__(self, *a, **kw)
+
+    def file_changed(self, f):
+        """Callback on changed lease file
+
+        Args:
+            f: Lease file. File offset can be used to skip already parsed lines.
+
+        Returns: New byte offset pointing after last parsed byte.
+        """
+        return f.tell()
+
+    def _trigger_update(self):
+        self.logger.info('Lease file changed, updating')
+        with open(self.lease_file, 'r') as f:
+            f.seek(self.position)
+            self.position = self.file_changed(f)
+
+    def run(self):
+        """Periodicaly check if file has changed
+
+        From ISC DHCPD manual:
+
+            New leases are appended to the end of the dhcpd.leases file.  In
+            order to prevent the file from becoming arbitrarily large, from
+            time to time dhcpd creates a new dhcpd.leases file from its in-core
+            lease database. Once this file has been written to disk, the old
+            file is renamed dhcpd.leases~, and the new file is renamed
+            dhcpd.leases.
+        """
+        while True:
+            try:
+                stat = os.stat(self.lease_file)
+                mtime = stat.st_mtime
+                size = stat.st_size
+                if size < self.position:
+                    self.logger.info('leases file changed - reseting pointer')
+                    self.position = 0
+                try:
+                    # checking if DHCPD performed cleanup
+                    # cleanup during operation seems to be currently broken
+                    # on customs so this could never execute
+                    purge_time = os.stat(self.lease_file + '~').st_mtime
+                    if purge_time > self.last_modified:
+                        self.logger.info('leases file purged - reseting pointer')
+                        self.position = 0
+                except FileNotFoundError:
+                    pass
+                if mtime > self.last_modified:
+                    self._trigger_update()
+                self.last_modified = mtime
+                sleep(5.0)
+            except Exception:
+                self.logger.exception('Exception in updater')
+                sleep(10.0)
+
+
+class DnsmasqUpdater(MtimeUpdater):
+    def file_changed(self, f):
+        raise NotImplementedError(
+            "This was not tested after adding differential update")
+        for line in f:
+            ts, hwaddr, ip, name, client_id = line.split(' ')
+            self.update(hwaddr, int(ts), ip, name)
+        return f.tell()
+
+def parse_isc_dhcpd_leases(leases_file: io.TextIOBase) -> Tuple[int, ActiveDevices]:
+    """Parse ISC dhcpd server leases file
+
+    Args:
+        leases_file: opened leases file. To skip already parsed part use seek
+            before calling.
+
+    Returns: Byte offset (as returned by tell()) of last parsed entry and
+        dictionary of parsed leases
+    """
+    leases = ActiveDevices()
+
+    ip: Optional[str] = None
+    hwaddr: Optional[str] = None
+    atime: Optional[float] = None
+    name: Optional[str] = None
+
+    lease = False
+    offset = leases_file.tell()
+    while True:
+        # using readline because iter(file) blocks file.tell usage
+        line = leases_file.readline()
+        if not line:
+            return offset, leases
+        line = line.split('#')[0]
+        cmd = line.strip().split()
+        if not cmd:
+            continue
+        if lease:
+            field = cmd[0]
+            if(field == 'starts'):
+                dt = datetime.strptime(' '.join(cmd[2:]),
+                                       '%Y/%m/%d %H:%M:%S;')
+                atime = dt.replace(tzinfo=timezone.utc).timestamp()
+            if(field == 'client-hostname'):
+                name = cmd[1][1:-2]
+            if(field == 'hardware'):
+                hwaddr = cmd[2][:-1]
+            if(field.startswith('}')):
+                offset = leases_file.tell()
+                lease = False
+                if hwaddr is not None and atime is not None:
+                    leases.add(DhcpLease(hwaddr, atime, ip, name))
+                hwaddr, atime = None, None
+        elif cmd[0] == 'lease':
+            ip = cmd[1]
+            hwaddr, atime, name = None, None, None
+            lease = True
+
+
+class DhcpdUpdater(MtimeUpdater):
+    def file_changed(self, f):
+        offset, devices = parse_isc_dhcpd_leases(f)
+        self.update(devices)
+        return offset

hswaw/checkinator/at/templates/account.html

@@ -0,0 +1,34 @@
+{% extends "basic.html" %}
+{% block content %}
+<a href="/">Back to homepage</a>
+<h2>Account settings</h2>
+{% for msg in get_flashed_messages(True) %}
+  <p class="{{ msg[0] }}">{{ msg[1] }}</p>
+{% endfor %}
+<h3>Claimed devices</h3>
+<table class="devices">
+  <tr>
+    <th>MAC</th>
+    <th>Device name</th>
+    <th>Visible</th>
+    <th>Toggle visibility</th>
+    <th>Delete</th>
+  </tr>
+{% for device in devices %}
+  <tr>
+    <td>{{ device.hwaddr }}</td>
+    <td>{{ device.name }}</td>
+    {% if device.ignored %}
+      <td class="invisible">invisible</td>
+      <td><a href="devices/{{ device.hwaddr }}/show">make visible</a></td>
+    {% else %}
+    <td class="visible">visible</td>
+      <td><a href="devices/{{ device.hwaddr }}/hide">make invisible</a></td>
+    {% endif %}
+    <td><a href="devices/{{ device.hwaddr }}/delete">delete device</a></td>
+  </tr>
+  </tbody>
+{% endfor%}
+</table>
+<p><a href="/claim">claim this device</a>
+{% endblock %}

hswaw/checkinator/at/templates/admin.html

@@ -0,0 +1,13 @@
+{% extends "basic.html" %}
+{% block content %}
+<table class="devices">
+  <tr>
+    <th>MAC</th>
+    <th>Device type</th>
+  </tr>
+  {% for key, l in data.items() %}
+      {% for item in l %}
+          <tr><td>{{ item }}</td><td>{{ key }}</td></tr>
+      {% endfor %}
+  {% endfor %}
+{% endblock %}

hswaw/checkinator/at/templates/basic.html

@@ -0,0 +1,24 @@
+<!doctype html>
+<html>
+  <head>
+    {% block head %}
+    <link rel="stylesheet" type="text/css" href="/static/css/basic.css">
+    <title>{% block title %}Now at hackerspace{% endblock %}</title>
+    {% endblock %}
+  </head>
+  <body>
+    {% block body %}
+      <div class="login">
+      {% if current_user.is_authenticated %}
+        logged in as {{ current_user.id }} |
+        <a href="account">account</a> | 
+        <a href="{{ url_for('spaceauth.logout') }}">log out</a>
+      {% else %}
+        <a href="{{ url_for('spaceauth.login') }}">login</a>
+      {% endif %}
+      </div>
+      {% block content %}
+      {% endblock %}
+    {% endblock %}
+  </body>
+</html>

hswaw/checkinator/at/templates/claim.html

@@ -0,0 +1,21 @@
+{% extends "basic.html" %}
+{% block content %}
+  <h2>Claiming a device</h2>
+  {% if not hwaddr %}
+    <p class="error">Unknown MAC. Are you sure you're in the hackerspace?</p>
+  {% else %}
+  You are about to claim <strong>{{ hwaddr }}</strong> as <strong>{{ current_user.id }}</strong>. Do you wish to continue?
+  <table>
+    <form action="" method="post">
+    <label><tr>
+      <td>Device name (optional):</td>
+      <td><input type="text" name="name" value="{{ name }}"></td>
+    </tr></label>
+    <tr>
+      <td><input type="submit" value="yes"></td>
+      </form>
+      <td><a href="/"><button>no</button></button></td>
+    </tr>
+  </table>
+  {% endif %}
+{% endblock %}

hswaw/checkinator/at/templates/invalid_ip.html

@@ -0,0 +1,16 @@
+{% extends "basic.html" %}
+{% block content %}
+  <h2>Claiming a device</h2>
+  <p class="error">Your IP address is outside of hackerspace LAN network. You might want to connect to HS WiFi and disable vpn's and mobile data.</p>
+
+  <p>
+    Make sure you:
+    <ul>
+        <li>connected to HS lan network</li>
+        <li>disabled VPN connections</li>
+        <li>use customs DNS server 10.8.1.2</li>
+    </ul>
+  </p>
+
+  <p> your IP: {{ ip_address }} </p>
+{% endblock %}

hswaw/checkinator/at/templates/login.html

@@ -0,0 +1,17 @@
+<html>
+<body>
+<h2>Login</h2>
+{% for error in get_flashed_messages() %}
+<p class="error">{{ error }}</p>
+{% endfor %}
+<form action="" method="POST">
+<table>
+<label><tr><td>login</td><td><input type="text" name="login" value="{{ login }}"></td></tr></label>
+<label><tr><td>password</td><td><input type="password" name="password"></td></tr></label>
+{% if goto %}
+<input type="hidden" name="goto" value="{{ goto }}">
+{% endif %}
+<tr><td></td><td><input type="submit" value="login"></input></td></tr>
+</form>
+</body>
+</html>

hswaw/checkinator/at/templates/main.html

@@ -0,0 +1,34 @@
+{% extends "basic.html" %}
+{% block title %}
+Now at hackerspace
+{% endblock %}
+{% block content %}
+  <h2>Now at hackerspace!</h2>
+  Recently at <a href="http://www.hackerspace.pl">hackerspace</a>:
+  <ul>
+  {% for user, timestamp in users %}
+    <li>
+      <a href="{{ user | wikiurl }}">
+      {{ user }} ({{ timestamp|strfts() }})
+      </a>
+    </li>
+  {% endfor %}
+  </ul>
+  {% trans n_unk=unknown|length %}
+  <p>There is {{ n_unk }} unknown device operating. </p>
+  {% pluralize %}
+  <p>There are {{ n_unk }} unknown devices operating.</p> 
+  {% endtrans %}
+  {% trans n_kek=kektops|length %}
+  <p>There is {{ n_kek }} unknown kektop operating.</p>
+  {% pluralize %}
+  <p>There are {{ n_kek }} unknown kektops operating.</p>
+  {% endtrans %}
+  {% trans n_esp=esps|length %}
+  <p>There is {{ n_esp }} unknown ESP operating.</p>
+  {% pluralize %}
+  <p>There are {{ n_esp }} unknown ESPs operating.</p>
+  {% endtrans %}
+  <hr>
+  <a href="claim">Claim this device!</a>
+{% endblock %}

hswaw/checkinator/at/templates/post_claim.html

@@ -0,0 +1,11 @@
+{% extends "basic.html" %}
+{% block content %}
+{% if error %}
+<h2>Error!</h2>
+<p class="error">{{ error }}</p>
+{% else %}
+<h2>Success!</h2>
+Congratulations, you just claimed this device!
+<a href="/">go back</a>
+{% endif %}
+{% endblock %}

hswaw/checkinator/at/templates/register.html

@@ -0,0 +1,36 @@
+{% extends "basic.html" %}
+{% block content %}
+<h2>Register a new account</h2>
+{% for error in get_flashed_messages() %}
+<p class="error">{{ error }}</p>
+{% endfor %}
+<form action="" method="POST">
+  <table>
+    <label><tr>
+      <td>login</td>
+      <td><input type="text" name="login" value="{{ login }}"></td>
+    </tr></label>
+    <label><tr>
+      <td>password</td>
+      <td><input type="password" name="password"></td>
+    </tr></label>
+    <label><tr>
+      <td>confirm password</td>
+      <td><input type="password" name="password2"></td>
+    </tr></label>
+    <label><tr>
+      <td>homepage url</td>
+      <td><input type="text" name="url" value="{{ url }}"></td>
+    </tr></label>
+    <label><tr>
+      <td>use wiki page as url</td>
+      <td><input type="checkbox" name="wiki" 
+        value="yes" {% if wiki == 'yes' %}checked="yes"{% endif %}></td>
+    </tr</label>
+    <tr>
+      <td></td>
+      <td><input type="submit" value="register"></input></td>
+    </tr>
+  </table>
+</form>
+{% endblock %}

hswaw/checkinator/at/tracker.proto

@@ -0,0 +1,31 @@
+syntax = "proto3";
+
+service DhcpTracker {
+  /* get list of clients detected in LAN network */
+  rpc GetClients (ClientsRequest) returns (DhcpClients) {};
+
+  /* get Layer 2 addess (MAC) for LAN ip address (v4 or v6) */
+  rpc GetHwAddr (HwAddrRequest) returns (HwAddrResponse) {};
+}
+
+message ClientsRequest {
+}
+
+message DhcpClient {
+  bytes hw_address = 1;
+  string last_seen = 2;
+  string client_hostname = 3;
+  string ip_address = 4;
+}
+
+message DhcpClients {
+  repeated DhcpClient clients = 1;
+}
+
+message HwAddrRequest {
+  string ip_address = 1; // IPv4 or IPv6 address
+}
+
+message HwAddrResponse {
+  bytes hw_address = 1; // MAC address
+}

hswaw/checkinator/at/tracker.py

@@ -0,0 +1,110 @@
+from at.dhcp import DhcpdUpdater, DhcpLease
+from pathlib import Path
+import yaml
+import grpc
+import json
+import re
+import subprocess
+import logging
+from concurrent import futures
+from datetime import datetime
+
+from .tracker_pb2 import DhcpClient, DhcpClients, HwAddrResponse
+from .tracker_pb2_grpc import DhcpTrackerServicer, add_DhcpTrackerServicer_to_server
+
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument("--verbose", help="output more info", action="store_true")
+parser.add_argument("config", type=Path, help="input file")
+
+logging.basicConfig(level=logging.INFO)
+
+def lease_to_client(lease: DhcpLease) -> DhcpClient:
+    return DhcpClient(
+        hw_address = bytes.fromhex(lease.hwaddr.replace(':', '')),
+        last_seen = datetime.utcfromtimestamp(lease.atime).isoformat(),
+        client_hostname = lease.name,
+        ip_address = lease.ip
+    )
+
+class DhcpTrackerServicer(DhcpTrackerServicer):
+    def __init__(self, tracker: DhcpdUpdater, *args, **kwargs):
+        self._tracker = tracker
+        super().__init__(*args, **kwargs)
+
+    def _authorize(self, context):
+        auth = context.auth_context()
+        ctype = auth.get('transport_security_type', 'local')
+        print(ctype)
+        if ctype == [b'ssl']:
+            if b'at.hackerspace.pl' not in context.peer_identities():
+                context.abort(
+                    grpc.StatusCode.PERMISSION_DENIED,
+                    (
+                        "Only at.hackespace.pl is allowed to access raw "
+                        "clients addresses"
+                    )
+                )
+        elif ctype == 'local':
+            # connection from local unix socket is trusted by default
+            pass
+        else:
+            context.abort(
+                grpc.StatusCode.PERMISSION_DENIED,
+                f"Unknown transport type: {ctype}"
+            )
+
+    def GetClients(self, request, context):
+        self._authorize(context)
+
+        clients = [
+            lease_to_client(c) for c in self._tracker.get_active_devices().values()]
+        return DhcpClients(clients = clients)
+
+    def GetHwAddr(self, request, context):
+        self._authorize(context)
+        ip_address = str(request.ip_address)
+        if not re.fullmatch('[0-9a-fA-F:.]*', ip_address):
+            raise ValueError(f'Invalid ip address: {ip_address!r}')
+        logging.info(f'running ip neigh on {ip_address}')
+        r = subprocess.run(['ip', '-json', 'neigh', 'show', ip_address], check=True, capture_output=True)
+        neighs = json.loads(r.stdout)
+        if neighs:
+            return HwAddrResponse(hw_address=bytes.fromhex(neighs[0]['lladdr'].replace(':', '')))
+        return HwAddrResponse(hw_address=None)
+
+def server():
+    args = parser.parse_args()
+    
+    config = yaml.safe_load(args.config.read_text())
+    tracker = DhcpdUpdater(config['LEASE_FILE'], config['TIMEOUT'])
+    tracker.start()
+    
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    add_DhcpTrackerServicer_to_server(DhcpTrackerServicer(tracker), server)
+
+
+    tls_address = config.get("GRPC_TLS_ADDRESS", None)
+    if tls_address:
+        cert_dir = Path(config.get('GRPC_TLS_CERT_DIR', 'cert'))
+        ca_cert = Path(config.get('GRPC_TLS_CA_CERT', 'ca.pem')).read_bytes()
+
+        server_credentials = grpc.ssl_server_credentials(
+            private_key_certificate_chain_pairs = ((
+                cert_dir.joinpath('key.pem').read_bytes(),
+                cert_dir.joinpath('cert.pem').read_bytes()
+            ),),
+            root_certificates = ca_cert,
+            require_client_auth = True
+        )
+        
+        server.add_secure_port(config.get('GRPC_TLS_ADDRESS', '[::]:2847'), server_credentials)
+
+    unix_socket = config.get('GRPC_UNIX_SOCKET', False)
+    if unix_socket:
+        server.add_insecure_port(f'unix://{unix_socket}')
+
+    if tls_address or unix_socket:
+        print('starting grpc server ...')
+        server.start()
+        server.wait_for_termination()

hswaw/checkinator/at/web.py

@@ -0,0 +1,303 @@
+import json
+import sqlite3
+from pathlib import Path
+from datetime import datetime
+from typing import NamedTuple, Iterable, Iterator, List
+from functools import wraps
+from flask import Flask, render_template, abort, g, \
+    redirect, request, url_for, make_response, send_file, \
+    Response
+from base64 import b64decode
+
+
+from spaceauth import SpaceAuth, login_required, current_user, cap_required
+
+# device infomation stored in database
+class DeviceInfo(NamedTuple):
+    hwaddr: str
+    name: str
+    owner: str
+    ignored: bool
+
+def v4addr():
+    if request.headers.getlist("X-Forwarded-For"):
+        r_addr = request.headers.getlist("X-Forwarded-For")[-1]
+    else:
+        r_addr = request.remote_addr
+    if r_addr.startswith('::ffff:'):
+        r_addr = r_addr[7:]
+    return r_addr
+
+
+def req_to_ctx():
+    return dict(iter(request.form.items()))
+
+def get_device_info(conn: sqlite3.Connection, hwaddr: str) -> DeviceInfo:
+    return list(get_device_infos(conn, (hwaddr,)))[0]
+
+
+def get_device_infos(conn: sqlite3.Connection, hwaddrs: Iterable[str]) -> Iterator[DeviceInfo]:
+    hwaddrs = list(hwaddrs)
+    in_clause = '({})'.format(', '.join(['?'] * len(hwaddrs)))
+    stmt = '''select hwaddr, name, ignored, owner from
+        devices where devices.hwaddr in ''' + in_clause
+    for row in conn.execute(stmt, hwaddrs):
+        owner = row['owner'] or ''
+        ignored = row['ignored']
+        yield DeviceInfo(row['hwaddr'], row['name'], owner, ignored)
+
+def app(instance_path, devices_api, config):
+    app = Flask('at', instance_path=instance_path, instance_relative_config=True)
+    app.config.update(config)
+    app.jinja_env.add_extension('jinja2.ext.i18n')
+    app.jinja_env.install_null_translations()
+    app.updater = devices_api
+    
+    if app.config.get('PROXY_FIX'):
+        from werkzeug.contrib.fixers import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app)
+    
+    app.space_auth = SpaceAuth(app)
+
+
+    def auth_get_user():
+        if config.get('DEBUG', False):
+            if "User" in request.headers:
+                return request.headers.get("User")
+            if "Authorization" in request.headers:
+                raw = b64decode(request.headers.get('Authorization').split(' ')[1])
+                app.logger.info(f'Raw authorization: {raw!s}')
+                return raw.decode().split(':')[0]
+            app.logger.info(request.headers)
+            raise Exception('username not supplied')
+        else:
+            return current_user.id
+
+    def auth_login_required(f):
+        if config.get('DEBUG', False):
+            @wraps(f)
+            def wrapper(*args, **kwargs):
+                try:
+                    auth_get_user()
+                except Exception:
+                    app.logger.exception("auth get exception")
+                    response = make_response('', 401)
+                    response.headers['WWW-Authenticate'] = 'Basic realm="at.hackerspace.pl", charset="UTF-8"'
+                    return response
+                return f(*args, **kwargs)
+            return wrapper
+        else:
+            return login_required(f)
+
+
+    def restrict_ip(prefixes : List[str] = [], exclude : List[str] = []):
+        def decorator(f):
+            @wraps(f)
+            def func(*a, **kw):
+                r_addr = v4addr()
+                if r_addr in exclude:
+                    app.logger.info('got IP %s, rejecting', r_addr)
+                    return render_template('invalid_ip.html', ip_address=r_addr), 403
+
+                for prefix in prefixes:
+                    if r_addr.startswith(prefix):
+                        break
+                else:
+                    app.logger.info('got IP %s, rejecting', r_addr)
+                    return render_template('invalid_ip.html', ip_address=r_addr), 403
+    
+                return f(*a, **kw)
+            return func
+        return decorator
+    
+    
+    @app.template_filter('strfts')
+    def strfts(ts, format='%d/%m/%Y %H:%M'):
+        return datetime.fromtimestamp(ts).strftime(format)
+    
+    
+    @app.template_filter('wikiurl')
+    def wikiurl(user):
+        return app.config['WIKI_URL'].replace("${login}", user)
+    
+    
+    @app.before_request
+    def make_connection():
+        conn = sqlite3.connect(app.config['DB'])
+        conn.row_factory = sqlite3.Row
+        conn.isolation_level = None  # for autocommit mode
+        g.db = conn
+    
+    
+    @app.teardown_request
+    def close_connection(exception):
+        g.db.close()
+    
+    
+    @app.route('/')
+    def main_view():
+        return render_template('main.html', **now_at())
+
+    @app.route('/metrics')
+    def metrics_view():
+        """Render count of different entities, per kind, in Prometheus format."""
+        now = now_at()
+        lines = [
+            "# HELP entities present at the hackerspace according to checkinator / DHCP",
+            "# TYPE people gauge",
+        ]
+        for kind, devices in now.items():
+            # The kind is being directly text-pasted into the metric below -
+            # let's make sure a new kind with special characters doesn't mess
+            # things up too much.
+            if '"' in kind:
+                continue
+            # Not using formatting, as the Prometheus format contains '{' and
+            # '}' characters which throw off Python's .format().
+            line = 'checkinator_now_present_entities{entity_kind="' + kind + '"} '
+            line += str(len(devices))
+            lines.append(line)
+        return Response('\n'.join(lines), mimetype='text/plain')
+    
+    @app.route('/api')
+    def list_all():
+        data = now_at()
+    
+        def prettify_user(xxx_todo_changeme):
+            (user, atime) = xxx_todo_changeme
+            if user == 'greenmaker':
+                user = 'dreammaker'
+            return {
+                'login': user,
+                'timestamp': atime,
+                'pretty_time': strfts(atime),
+            }
+        result = {}
+        result['users'] = list(map(prettify_user, data.pop('users')))
+        result.update((k, len(v)) for k, v in list(data.items()))
+        res = make_response(json.dumps(result), 200)
+        res.headers['Access-Control-Allow-Origin'] = '*'
+        return res
+
+    def now_at():
+        result = dict()
+        devices = app.updater.get_active_devices()
+        macs = list(devices.keys())
+
+        identified_devices = list(get_device_infos(g.db, macs))
+        unknown = set(macs) - set(d.hwaddr for d in identified_devices)
+
+        # das kektop sorting maschine
+        # identify special devices
+        for name, prefixes in app.config['SPECIAL_DEVICES'].items():
+            result[name] = set()
+            prefixes = tuple(prefixes) # startswith accepts tuple as argument
+            for hwaddr in list(unknown):
+                if hwaddr.startswith(prefixes):
+                    result[name].add(hwaddr)
+                    unknown.discard(hwaddr)
+    
+        result['unknown'] = unknown
+    
+        users = {}
+        for info in identified_devices:
+            # append device to user
+            last_seen = users.get(info.owner, 0)
+            if not info.ignored:
+                last_seen = max(last_seen, devices[info.hwaddr].atime)
+                users[info.owner] = last_seen
+
+        result['users'] = sorted(users.items(), key=lambda u_l: (u_l[1], u_l[0]), reverse=True)
+    
+        return result
+    
+    
+    restrict_to_hs = restrict_ip(prefixes=app.config['CLAIMABLE_PREFIXES'],
+                                 exclude=app.config['CLAIMABLE_EXCLUDE'])
+    
+    
+    @app.route('/claim', methods=['GET'])
+    @restrict_to_hs
+    @auth_login_required
+    def claim_form():
+        hwaddr, name = app.updater.get_device(v4addr())
+        return render_template('claim.html', hwaddr=hwaddr, name=name)
+
+    @app.route('/my_ip', methods=['GET'])
+    def get_my_ip():
+        ip = v4addr()
+        hwaddr, name = app.updater.get_device(ip)
+        return f'ip: {ip!r}\nhwaddr: {hwaddr!r}\nhostname: {name!r}\n'
+    
+    @app.route('/claim', methods=['POST'])
+    @restrict_to_hs
+    @auth_login_required
+    def claim():
+        hwaddr, lease_name = app.updater.get_device(v4addr())
+        ctx = None
+        if not hwaddr:
+            ctx = dict(error='Invalid device.')
+        else:
+            login = auth_get_user()
+            try:
+                g.db.execute('''
+    insert into devices (hwaddr, name, owner, ignored) values (?, ?, ?, ?)''',
+                             [hwaddr, request.form['name'], login, False])
+                ctx = {}
+            except sqlite3.Error:
+                error = 'Could not add device! Perhaps someone claimed it?'
+                ctx = dict(error=error)
+        return render_template('post_claim.html', **ctx)
+    
+    def get_user_devices(conn, user):
+        devs = conn.execute('select hwaddr, name, ignored from devices where\
+     owner = ?', [user])
+        device_info = []
+        for row in devs:
+            di = DeviceInfo(row['hwaddr'], row['name'], user, row['ignored'])
+            device_info.append(di)
+        return device_info
+    
+    
+    @app.route('/account', methods=['GET'])
+    @auth_login_required
+    def account():
+        devices = get_user_devices(g.db, auth_get_user())
+        return render_template('account.html', devices=devices)
+    
+    
+    def set_ignored(conn, hwaddr, user, value):
+        return conn.execute('''
+    update devices set ignored = ? where hwaddr = ? and owner = ?''',
+                            [value, hwaddr, user])
+    
+    
+    def delete_device(conn, hwaddr, user):
+        return conn.execute('''
+    delete from devices where hwaddr = ? and owner = ?''',
+                            [hwaddr, user])
+    
+    @app.route('/devices/<id>/<action>/')
+    @auth_login_required
+    def device(id, action):
+        user = auth_get_user()
+        if action == 'hide':
+            set_ignored(g.db, id, user, True)
+        if action == 'show':
+            set_ignored(g.db, id, user, False)
+        if action == 'delete':
+            delete_device(g.db, id, user)
+        return redirect(url_for('account'))
+    
+    
+    @app.route('/admin')
+    @cap_required('staff')
+    def admin():
+        data = now_at()
+        return render_template('admin.html', data=data)
+
+    @app.route('/static/css/basic.css')
+    def css():
+        return send_file(str(Path('./static/css/basic.css').absolute()))
+    
+    return app

hswaw/checkinator/at/webapp.py

@@ -0,0 +1,104 @@
+"""Entry point for running flask application"""
+
+import at.web 
+from at.dhcp import DhcpdUpdater
+from pathlib import Path
+import yaml
+import os
+import ipaddress
+from typing import Tuple, Optional, Dict
+
+import grpc
+from at.tracker_pb2 import ClientsRequest, HwAddrRequest
+from at.tracker_pb2_grpc import DhcpTrackerStub
+from at.dhcp import DhcpLease
+from datetime import datetime
+
+
+def format_mac(raw: bytes) -> str:
+    return ':'.join(f'{b:02x}' for b in raw)
+
+def mac_from_ipv6(address : ipaddress.IPv6Address):
+    if not isinstance(address, ipaddress.IPv6Address):
+        raise ValueError(f"not an IPv6 address: {address}")
+    raw = address.packed[8:]
+    if raw[3:5] != bytes([0xff, 0xfe]):
+        raise ValueError(f"not MAC based IPv6 Address: {address}")
+    mac = bytes([raw[0] ^ 0x02, *raw[1:3], *raw[5:]])
+    return mac
+
+class DevicesApi:
+    def __init__(self, grpc_channel):
+        self._api = DhcpTrackerStub(grpc_channel)
+
+
+    def get_active_devices(self) -> Dict[str, DhcpLease]:
+        devices = self._api.GetClients(ClientsRequest())
+        return { 
+            format_mac(d.hw_address): DhcpLease(
+                hwaddr=format_mac(d.hw_address),
+                atime=datetime.fromisoformat(d.last_seen).timestamp(),
+                ip=d.ip_address,
+                name=d.client_hostname
+            ) for d in devices.clients
+        }
+
+    def get_device(self, ip: str) -> Tuple[Optional[str], Optional[str]]:
+        hw_address = self._api.GetHwAddr(HwAddrRequest(ip_address=ip)).hw_address
+        if hw_address is not None:
+            devices = self._api.GetClients(ClientsRequest())
+            for device in devices.clients:
+                if device.hw_address == hw_address:
+                    return format_mac(hw_address), device.client_hostname
+            return format_mac(hw_address), ""
+
+        address = ipaddress.ip_address(ip)
+        if isinstance(address, ipaddress.IPv6Address):
+            try:
+                mac = mac_from_ipv6(address)
+            except ValueError:
+                pass
+            else:
+                return ( format_mac(mac), "" )
+
+        return None, None
+
+
+config_path = Path(os.environ.get("CHECKINATOR_WEB_CONFIG", 'web-config.yaml'))
+config = yaml.safe_load(config_path.read_text())
+config.update(yaml.safe_load(Path(config["SECRETS_FILE"]).read_text()))
+
+
+tls_address = config.get("GRPC_TLS_ADDRESS", False)
+unix_socket = config.get('GRPC_UNIX_SOCKET', False)
+if tls_address:
+    print("using secure channel")
+    ca_cert = Path(config.get('GRPC_TLS_CA_CERT')).read_bytes()
+    cert_dir = Path(config.get('GRPC_TLS_CERT_DIR'))
+
+    channel_credential = grpc.ssl_channel_credentials(
+        root_certificates = ca_cert,
+        private_key = cert_dir.joinpath('key.pem').read_bytes(),
+        certificate_chain = cert_dir.joinpath('cert.pem').read_bytes(),
+    )
+
+    options = [
+        ('grpc.ssl_target_name_override', 'at.customs.hackerspace.pl')
+    ]
+    channel = grpc.secure_channel(config.get('GRPC_TLS_ADDRESS'), channel_credential, options=options)
+elif unix_socket:
+    channel = grpc.insecure_channel(f'unix://{unix_socket}')
+else:
+    raise Exception("no GRPC_TLS_ADDRESS or GRPC_UNIX_SOCKET set in config file")
+
+app = at.web.app(Path(__file__).parent, DevicesApi(channel), config)
+
+def run_debug():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=8080, help="http port")
+    parser.add_argument("--ip", type=str, default='127.0.0.1', help="http port")
+    
+    args = parser.parse_args()
+    
+    app.run(args.ip, args.port, debug=True)

hswaw/checkinator/cap.py

@@ -0,0 +1,30 @@
+import logging
+import pcapy
+import struct
+
+interface = 'wlan0'
+target = './dhcp-cap'
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+logger.addHandler(logging.StreamHandler())
+
+def hwaddr_ascii(packet):
+    # picking up MAC directly from ethernet frame
+    return ':'.join('%02x' % ord(c) for c in packet[6:12])
+
+def capture_dhcp(itf):
+    f = open(target, 'w')
+    reader = pcapy.open_live(itf, 4096, False, 5000)
+    reader.setfilter('udp dst port 67')
+    def callback(header, packet):
+        hwaddr = hwaddr_ascii(packet)
+        logger.info('Captured dhcp request from %s', hwaddr)
+        f.write(hwaddr + '\n')
+        f.flush()
+    try:
+        while True:
+            reader.dispatch(1, callback)
+    except KeyboardInterrupt:
+        pass
+
+capture_dhcp('wlan0')

hswaw/checkinator/config.dist.yaml

@@ -0,0 +1,43 @@
+DB: 'at.db'
+DEBUG: false
+CAP_FILE: './dhcp-cap'
+LEASE_FILE: './dhcpd.leases'
+TIMEOUT: 1500
+
+WIKI_URL: 'https://wiki.hackerspace.pl/people:%(login)s:start'
+
+CLAIMABLE_PREFIX: '10.8.0.'
+CLAIMABLE_EXCLUDE: [ ]
+
+SECRET_KEY: 'CHANGEME'
+
+SPACEAUTH_CONSUMER_KEY: 'checkinator'
+SPACEAUTH_CONSUMER_SECRET: 'CHANGEME'
+
+SPECIAL_DEVICES:
+  'kektops':
+    - '90:e6:ba:84'
+  'esps':
+    - 'ec:fa:bc'
+    - 'dc:4f:22'
+    - 'd8:a0:1d'
+    - 'b4:e6:2d'
+    - 'ac:d0:74'
+    - 'a4:7b:9d'
+    - 'a0:20:a6'
+    - '90:97:d5'
+    - '68:c6:3a'
+    - '60:01:94'
+    - '5c:cf:7f'
+    - '54:5a:a6'
+    - '30:ae:a4'
+    - '2c:3a:e8'
+    - '24:b2:de'
+    - '24:0a:c4'
+    - '18:fe:34'
+    - '38:2b:78'
+    - 'bc:dd:c2:'
+  'vms':
+    - '52:54:00' # craptrap VMs
+
+PROXY_FIX: false

hswaw/checkinator/dbsetup.sql

@@ -0,0 +1,6 @@
+create table devices (
+  hwaddr character(17) primary key,
+  name varchar(50),
+  owner varchar(100) not null, 
+  ignored boolean
+);

hswaw/checkinator/default.nix

@@ -0,0 +1,30 @@
+{ pkgs ? (import <nixpkgs> {}).c.unstable_2020-05}:
+
+let
+  spaceauth = pkgs.callPackage "${pkgs.fetchgit {
+    url = "http://code.hackerspace.pl/vuko/nix-spaceauth";
+    rev = "1c289eafe041d7730a834bb437b7173ca4b9e2c9";
+    sha256 = "0f2mhbkm92rlx3a1il3wfr4bq6xghdiajczgg349v6a01iazm4qz";
+  }}/spaceauth.nix" {};
+in pkgs.python3Packages.buildPythonPackage {
+  pname = "checkinator";
+  version = "0.2";
+
+  doCheck = false;
+  src = ./.;
+
+  propagatedBuildInputs = with pkgs; [
+    python3Packages.gunicorn
+    python3Packages.flask
+    python3Packages.pyyaml
+    python3Packages.isodate
+    python3Packages.requests
+    python3Packages.requests-unixsocket
+    python3Packages.grpcio
+    python3Packages.grpcio-tools
+    python3Packages.setuptools
+    python3Packages.protobuf
+    spaceauth
+    iproute
+  ];
+}

hswaw/checkinator/requirements.txt

@@ -0,0 +1,17 @@
+blinker==1.4
+certifi==2017.7.27.1
+chardet==3.0.4
+click==6.7
+Flask==0.12.2
+Flask-Login==0.4.0
+Flask-OAuthlib==0.9.4
+-e git+https://code.hackerspace.pl/informatic/flask-spaceauth@4dd1c63912297d499dcd5631879e45dc6aa1819d#egg=Flask_SpaceAuth
+idna==2.6
+itsdangerous==0.24
+Jinja2==2.9.6
+MarkupSafe==1.0
+oauthlib==2.0.4
+requests==2.18.4
+requests-oauthlib==0.8.0
+urllib3==1.22
+Werkzeug==0.12.2

hswaw/checkinator/run.py

@@ -0,0 +1,3 @@
+import at.webapp
+
+app = at.webapp.app

hswaw/checkinator/setup.py

@@ -0,0 +1,42 @@
+from setuptools import setup
+import grpc_tools.protoc
+import pkg_resources
+from pathlib import Path
+
+setupdir = Path(__file__).parent
+
+proto_include = pkg_resources.resource_filename('grpc_tools', '_proto')
+
+compiled_proto = Path('at/tracker_pb2.py')
+if compiled_proto.exists():
+    compiled_proto.unlink()
+
+grpc_tools.protoc.main([
+    'grpc_tools.protoc',
+    f'-I{setupdir!s}',
+    '--python_out=./',
+    '--grpc_python_out=./',
+    'at/tracker.proto'
+])
+
+assert compiled_proto.exists()
+
+setup(
+    name='hswaw-at',
+    version='0.1',
+    description='warsaw hackerspace checkinator',
+
+    packages=['at'],
+    package_data={"at": ["templates/*"]},
+    python_requires='>=3.6,',
+    install_requires=['Flask', 'requests', 'flask-spaceauth', 'pyyaml', 'grpcio', 'protobuf'],
+    entry_points={
+        'console_scripts': [
+            'checkinator-list=at.cmd:list',
+            'checkinator-tracker=at.tracker:server',
+            'checkinator-tracker-list=at.cmd:tracker_list',
+            'checkinator-tracker-get-hwaddr=at.cmd:tracker_get_hwaddr',
+            'checkinator-web-debug=at.webapp:run_debug'
+        ],
+    },
+)

hswaw/checkinator/static/css/basic.css

@@ -0,0 +1,46 @@
+body {
+  margin: 5% auto;
+  background: #f8f8f8;
+  color: #222;
+  font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+  font-size: 16px;
+  line-height: 1.8;
+  text-shadow: 0 1px 0 #ffffff;
+  max-width: 73%;
+}
+
+code {
+  background: white;
+}
+
+a {
+  border-bottom: 1px solid #222;
+  color: #222;
+  text-decoration: none;
+}
+
+a:hover {
+  border-bottom: 0;
+}
+
+p.error {
+  color: red
+}
+
+p.message {
+  color: green
+}
+
+table.devices, .devices td, .devices th {
+  padding: .5em;
+  border: 1px solid black;
+  border-collapse: collapse;
+}
+
+td.invisible {
+  background-color: #fbb
+}
+
+td.visible {
+  background-color: #bfb
+}

hswaw/checkinator/tracker-config.dist.yaml

@@ -0,0 +1,13 @@
+# path to dhcpd lease file 
+LEASE_FILE: './dhcpd.leases'
+
+# timeout for old leases
+TIMEOUT: 1500
+
+# optional - local trusted socket
+GRPC_UNIX_SOCKET: "unix://tmp/checkinator.sock"
+
+# optional - remote authenticated (TLS cert) socket 
+GRPC_TLS_CERT_DIR: "./cert-tracker"
+GRPC_TLS_CA_CERT: "./ca.pem"
+GRPC_TLS_ADDRESS: "[::1]:2847"

hswaw/checkinator/web-config.dist.yaml

@@ -0,0 +1,52 @@
+# local sqlite db for storing user and MAC 
+DB: 'at.db'
+
+# debug option interpreted by flask app
+DEBUG: true
+
+# url to member wiki page
+# "${login}" string is replaced by member login (uid)
+WIKI_URL: 'https://wiki.hackerspace.pl/people:%(login)s:start'
+
+CLAIMABLE_PREFIXES:
+  - '10.8.0.'
+  - '2a0d:eb00:4242:0:'
+
+CLAIMABLE_EXCLUDE: [ ]
+
+SECRETS_FILE: "web-secrets.yaml"
+
+SPECIAL_DEVICES:
+  'kektops':
+    - '90:e6:ba:84'
+  'esps':
+    - 'ec:fa:bc'
+    - 'dc:4f:22'
+    - 'd8:a0:1d'
+    - 'b4:e6:2d'
+    - 'ac:d0:74'
+    - 'a4:7b:9d'
+    - 'a0:20:a6'
+    - '90:97:d5'
+    - '68:c6:3a'
+    - '60:01:94'
+    - '5c:cf:7f'
+    - '54:5a:a6'
+    - '30:ae:a4'
+    - '2c:3a:e8'
+    - '24:b2:de'
+    - '24:0a:c4'
+    - '18:fe:34'
+    - '38:2b:78'
+    - 'bc:dd:c2'
+    - 'cc:50:e3'
+  'vms':
+    - '52:54:00' # craptrap VMs
+
+PROXY_FIX: true
+
+#GRPC_UNIX_SOCKET: "./checkinator.sock"
+
+GRPC_TLS_CERT_DIR: "./cert-webapp"
+GRPC_TLS_CA_CERT: "./ca.pem"
+GRPC_TLS_ADDRESS: '[::1]:2847'

hswaw/checkinator/web-secrets.dist.yaml

@@ -0,0 +1,3 @@
+SECRET_KEY: 'CHANGEME'
+SPACEAUTH_CONSUMER_KEY: 'checkinator'
+SPACEAUTH_CONSUMER_SECRET: 'CHANGEME'

hswaw/machines/customs.hackerspace.pl/checkinator-repo.json

@@ -1,5 +0,0 @@
-{
-    "url": "http://code.hackerspace.pl/checkinator",
-    "rev": "713c7e6c1a8fd6147522c1a5e3067898a1d8bf7a",
-    "sha256": "1vhz9jd0hfa0d1hihgkarf6w7z8yqvz4dzk42wzwk0rs25qlcavi"
-}

hswaw/machines/customs.hackerspace.pl/checkinator.nix

@@ -11,7 +11,7 @@ in pkgs.python3Packages.buildPythonPackage {
   version = "0.2";
 
   doCheck = false;
-  src = pkgs.fetchgit (builtins.fromJSON (builtins.readFile ./checkinator-repo.json));
+  src = ../../../hswaw/checkinator;
 
   patches = [
     ./checkinator-werkzeug.patch