diff options
author | vuko <vuko@hackerspace.pl> | 2020-06-13 12:10:32 +0200 |
---|---|---|
committer | vuko <vuko@hackerspace.pl> | 2020-06-13 12:10:32 +0200 |
commit | f5cde26030a66608628c3581784074753b6ad679 (patch) | |
tree | 2ce10ced5f2c9dc02e4f6a33cc7813075daee5cb | |
parent | 35e30cd74ae17584eedbee5aa50a1c7fa843baf9 (diff) | |
parent | d3ab4653825d080677869bfbe64cf0e9da30eabd (diff) | |
download | checkinator-f5cde26030a66608628c3581784074753b6ad679.tar.gz checkinator-f5cde26030a66608628c3581784074753b6ad679.tar.bz2 checkinator-f5cde26030a66608628c3581784074753b6ad679.tar.xz checkinator-f5cde26030a66608628c3581784074753b6ad679.zip |
optimize leases parsing - skip already parsed entries
-rw-r--r-- | at.py | 102 |
1 files changed, 73 insertions, 29 deletions
@@ -13,7 +13,7 @@ from flask import Flask, render_template, abort, g, \ from datetime import datetime from time import sleep, time, mktime from collections import namedtuple -from urllib import urlencode +from urllib.parse import urlencode from spaceauth import SpaceAuth, login_required, current_user, cap_required @@ -58,7 +58,7 @@ def restrict_ip(prefix='', exclude=[]): def req_to_ctx(): - return dict(request.form.iteritems()) + return dict(iter(request.form.items())) @app.template_filter('strfts') @@ -112,19 +112,18 @@ class Updater(threading.Thread): def purge_stale(self): now = time() - for addr, (atime, ip, name) in self.active.items(): + for addr, (atime, ip, name) in list(self.active.items()): if now - atime > self.timeout: del self.active[addr] def get_active_devices(self): - self.lock.acquire() - self.purge_stale() - r = dict(self.active) - self.lock.release() + with self.lock: + self.purge_stale() + r = dict(self.active) return r def get_device(self, ip): - active_devices = self.get_active_devices().iteritems() + active_devices = iter(self.get_active_devices().items()) for hwaddr, (atime, dip, name) in active_devices: if ip == dip: return hwaddr, name @@ -135,12 +134,11 @@ class Updater(threading.Thread): atime -= self.lease_offset else: atime = time() - self.lock.acquire() - if hwaddr not in self.active or self.active[hwaddr][0] < atime: - self.active[hwaddr] = (atime, ip, name) - app.logger.info('updated %s with atime %s and ip %s', - hwaddr, strfts(atime), ip) - self.lock.release() + with self.lock: + if hwaddr not in self.active or self.active[hwaddr][0] < atime: + self.active[hwaddr] = (atime, ip, name) + app.logger.info('updated %s with atime %s and ip %s', + hwaddr, strfts(atime), ip) class CapUpdater(Updater): @@ -169,36 +167,73 @@ class CapUpdater(Updater): class MtimeUpdater(Updater): def __init__(self, lease_file, *a, **kw): self.lease_file = lease_file + self.position = 0 self.last_modified = 0 Updater.__init__(self, *a, **kw) def file_changed(self, f): - pass + """Callback on changed lease file + + Args: + f: Lease file. File offset can be used to skip already parsed lines. + + Returns: New byte offset pointing after last parsed byte. + """ + return f.tell() def _trigger_update(self): app.logger.info('Lease file changed, updating') with open(self.lease_file, 'r') as f: - self.file_changed(f) + f.seek(self.position) + self.position = self.file_changed(f) def run(self): + """Periodicaly check if file has changed + + From ISC DHCPD manual: + + New leases are appended to the end of the dhcpd.leases file. In + order to prevent the file from becoming arbitrarily large, from + time to time dhcpd creates a new dhcpd.leases file from its in-core + lease database. Once this file has been written to disk, the old + file is renamed dhcpd.leases~, and the new file is renamed + dhcpd.leases. + """ while True: try: - mtime = os.stat(self.lease_file).st_mtime + stat = os.stat(self.lease_file) + mtime = stat.st_mtime + size = stat.st_size + if size < self.position: + app.logger.info('leases file changed - reseting pointer') + self.position = 0 + try: + # checking if DHCPD performed cleanup + # cleanup during operation seems to be currently broken + # on customs so this could never execute + purge_time = os.stat(self.lease_file + '~').st_mtime + if purge_time > self.last_modified: + app.logger.info('leases file purged - reseting pointer') + self.position = 0 + except FileNotFoundError: + pass if mtime > self.last_modified: self._trigger_update() self.last_modified = mtime - sleep(3.0) + sleep(5.0) except Exception as e: - app.logger.error('Updater got an exception:\n' + - traceback.format_exc(e)) + app.logger.exception('Exception in updater') sleep(10.0) class DnsmasqUpdater(MtimeUpdater): def file_changed(self, f): + raise NotImplementedError( + "This was not tested after adding differential update") for line in f: ts, hwaddr, ip, name, client_id = line.split(' ') self.update(hwaddr, int(ts), ip, name) + return f.tell() class DhcpdUpdater(MtimeUpdater): @@ -208,7 +243,12 @@ class DhcpdUpdater(MtimeUpdater): ip = None hwaddr = None atime = None - for line in f: + offset = f.tell() + while True: + # using readline because iter(file) blocks file.tell usage + line = f.readline() + if not line: + return offset line = line.split('#')[0] cmd = line.strip().split() if not cmd: @@ -224,6 +264,7 @@ class DhcpdUpdater(MtimeUpdater): if(field == 'hardware'): hwaddr = cmd[2][:-1] if(field.startswith('}')): + offset = f.tell() lease = False if hwaddr is not None and atime is not None: self.update(hwaddr, atime, ip, name) @@ -243,15 +284,18 @@ def main_view(): def list_all(): data = now_at() - def prettify_user((user, atime)): + def prettify_user(xxx_todo_changeme): + (user, atime) = xxx_todo_changeme + if user == 'greenmaker': + user = 'dreammaker' return { 'login': user, 'timestamp': atime, 'pretty_time': strfts(atime), } result = {} - result['users'] = map(prettify_user, data.pop('users')) - result.update((k, len(v)) for k, v in data.items()) + result['users'] = list(map(prettify_user, data.pop('users'))) + result.update((k, len(v)) for k, v in list(data.items())) res = make_response(json.dumps(result), 200) res.headers['Access-Control-Allow-Origin'] = '*' return res @@ -260,11 +304,11 @@ def list_all(): def now_at(): result = dict() devices = app.updater.get_active_devices() - device_infos = list(get_device_infos(g.db, devices.keys())) - device_infos.sort(key=lambda di: devices.__getitem__) + device_infos = list(get_device_infos(g.db, list(devices.keys()))) + device_infos.sort(key=lambda di: devices[di.hwaddr]) unknown = set(devices.keys()) - set(d.hwaddr for d in device_infos) # das kektop sorting maschine - for name, prefixes in app.config['SPECIAL_DEVICES'].items(): + for name, prefixes in list(app.config['SPECIAL_DEVICES'].items()): result[name] = set() for u in unknown.copy(): if u.startswith(prefixes): @@ -275,9 +319,9 @@ def now_at(): users = {} for info in device_infos: - if info.owner not in users: + if info.owner not in users and not info.ignored: users[info.owner] = devices[info.hwaddr][0] - result['users'] = sorted(users.items(), key=lambda (u, a): a, reverse=True) + result['users'] = sorted(list(users.items()), key=lambda u_a: u_a[1], reverse=True) return result |