summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorvuko <vuko@hackerspace.pl>2020-06-13 12:10:32 +0200
committervuko <vuko@hackerspace.pl>2020-06-13 12:10:32 +0200
commitf5cde26030a66608628c3581784074753b6ad679 (patch)
tree2ce10ced5f2c9dc02e4f6a33cc7813075daee5cb
parent35e30cd74ae17584eedbee5aa50a1c7fa843baf9 (diff)
parentd3ab4653825d080677869bfbe64cf0e9da30eabd (diff)
downloadcheckinator-f5cde26030a66608628c3581784074753b6ad679.tar.gz
checkinator-f5cde26030a66608628c3581784074753b6ad679.tar.bz2
checkinator-f5cde26030a66608628c3581784074753b6ad679.zip
optimize leases parsing - skip already parsed entries
-rw-r--r--at.py102
1 files changed, 73 insertions, 29 deletions
diff --git a/at.py b/at.py
index 1fd0b5c..5a06ef2 100644
--- a/at.py
+++ b/at.py
@@ -13,7 +13,7 @@ from flask import Flask, render_template, abort, g, \
from datetime import datetime
from time import sleep, time, mktime
from collections import namedtuple
-from urllib import urlencode
+from urllib.parse import urlencode
from spaceauth import SpaceAuth, login_required, current_user, cap_required
@@ -58,7 +58,7 @@ def restrict_ip(prefix='', exclude=[]):
def req_to_ctx():
- return dict(request.form.iteritems())
+ return dict(iter(request.form.items()))
@app.template_filter('strfts')
@@ -112,19 +112,18 @@ class Updater(threading.Thread):
def purge_stale(self):
now = time()
- for addr, (atime, ip, name) in self.active.items():
+ for addr, (atime, ip, name) in list(self.active.items()):
if now - atime > self.timeout:
del self.active[addr]
def get_active_devices(self):
- self.lock.acquire()
- self.purge_stale()
- r = dict(self.active)
- self.lock.release()
+ with self.lock:
+ self.purge_stale()
+ r = dict(self.active)
return r
def get_device(self, ip):
- active_devices = self.get_active_devices().iteritems()
+ active_devices = iter(self.get_active_devices().items())
for hwaddr, (atime, dip, name) in active_devices:
if ip == dip:
return hwaddr, name
@@ -135,12 +134,11 @@ class Updater(threading.Thread):
atime -= self.lease_offset
else:
atime = time()
- self.lock.acquire()
- if hwaddr not in self.active or self.active[hwaddr][0] < atime:
- self.active[hwaddr] = (atime, ip, name)
- app.logger.info('updated %s with atime %s and ip %s',
- hwaddr, strfts(atime), ip)
- self.lock.release()
+ with self.lock:
+ if hwaddr not in self.active or self.active[hwaddr][0] < atime:
+ self.active[hwaddr] = (atime, ip, name)
+ app.logger.info('updated %s with atime %s and ip %s',
+ hwaddr, strfts(atime), ip)
class CapUpdater(Updater):
@@ -169,36 +167,73 @@ class CapUpdater(Updater):
class MtimeUpdater(Updater):
def __init__(self, lease_file, *a, **kw):
self.lease_file = lease_file
+ self.position = 0
self.last_modified = 0
Updater.__init__(self, *a, **kw)
def file_changed(self, f):
- pass
+ """Callback on changed lease file
+
+ Args:
+ f: Lease file. File offset can be used to skip already parsed lines.
+
+ Returns: New byte offset pointing after last parsed byte.
+ """
+ return f.tell()
def _trigger_update(self):
app.logger.info('Lease file changed, updating')
with open(self.lease_file, 'r') as f:
- self.file_changed(f)
+ f.seek(self.position)
+ self.position = self.file_changed(f)
def run(self):
+ """Periodicaly check if file has changed
+
+ From ISC DHCPD manual:
+
+ New leases are appended to the end of the dhcpd.leases file. In
+ order to prevent the file from becoming arbitrarily large, from
+ time to time dhcpd creates a new dhcpd.leases file from its in-core
+ lease database. Once this file has been written to disk, the old
+ file is renamed dhcpd.leases~, and the new file is renamed
+ dhcpd.leases.
+ """
while True:
try:
- mtime = os.stat(self.lease_file).st_mtime
+ stat = os.stat(self.lease_file)
+ mtime = stat.st_mtime
+ size = stat.st_size
+ if size < self.position:
+ app.logger.info('leases file changed - reseting pointer')
+ self.position = 0
+ try:
+ # checking if DHCPD performed cleanup
+ # cleanup during operation seems to be currently broken
+ # on customs so this could never execute
+ purge_time = os.stat(self.lease_file + '~').st_mtime
+ if purge_time > self.last_modified:
+ app.logger.info('leases file purged - reseting pointer')
+ self.position = 0
+ except FileNotFoundError:
+ pass
if mtime > self.last_modified:
self._trigger_update()
self.last_modified = mtime
- sleep(3.0)
+ sleep(5.0)
except Exception as e:
- app.logger.error('Updater got an exception:\n' +
- traceback.format_exc(e))
+ app.logger.exception('Exception in updater')
sleep(10.0)
class DnsmasqUpdater(MtimeUpdater):
def file_changed(self, f):
+ raise NotImplementedError(
+ "This was not tested after adding differential update")
for line in f:
ts, hwaddr, ip, name, client_id = line.split(' ')
self.update(hwaddr, int(ts), ip, name)
+ return f.tell()
class DhcpdUpdater(MtimeUpdater):
@@ -208,7 +243,12 @@ class DhcpdUpdater(MtimeUpdater):
ip = None
hwaddr = None
atime = None
- for line in f:
+ offset = f.tell()
+ while True:
+ # using readline because iter(file) blocks file.tell usage
+ line = f.readline()
+ if not line:
+ return offset
line = line.split('#')[0]
cmd = line.strip().split()
if not cmd:
@@ -224,6 +264,7 @@ class DhcpdUpdater(MtimeUpdater):
if(field == 'hardware'):
hwaddr = cmd[2][:-1]
if(field.startswith('}')):
+ offset = f.tell()
lease = False
if hwaddr is not None and atime is not None:
self.update(hwaddr, atime, ip, name)
@@ -243,15 +284,18 @@ def main_view():
def list_all():
data = now_at()
- def prettify_user((user, atime)):
+ def prettify_user(xxx_todo_changeme):
+ (user, atime) = xxx_todo_changeme
+ if user == 'greenmaker':
+ user = 'dreammaker'
return {
'login': user,
'timestamp': atime,
'pretty_time': strfts(atime),
}
result = {}
- result['users'] = map(prettify_user, data.pop('users'))
- result.update((k, len(v)) for k, v in data.items())
+ result['users'] = list(map(prettify_user, data.pop('users')))
+ result.update((k, len(v)) for k, v in list(data.items()))
res = make_response(json.dumps(result), 200)
res.headers['Access-Control-Allow-Origin'] = '*'
return res
@@ -260,11 +304,11 @@ def list_all():
def now_at():
result = dict()
devices = app.updater.get_active_devices()
- device_infos = list(get_device_infos(g.db, devices.keys()))
- device_infos.sort(key=lambda di: devices.__getitem__)
+ device_infos = list(get_device_infos(g.db, list(devices.keys())))
+ device_infos.sort(key=lambda di: devices[di.hwaddr])
unknown = set(devices.keys()) - set(d.hwaddr for d in device_infos)
# das kektop sorting maschine
- for name, prefixes in app.config['SPECIAL_DEVICES'].items():
+ for name, prefixes in list(app.config['SPECIAL_DEVICES'].items()):
result[name] = set()
for u in unknown.copy():
if u.startswith(prefixes):
@@ -275,9 +319,9 @@ def now_at():
users = {}
for info in device_infos:
- if info.owner not in users:
+ if info.owner not in users and not info.ignored:
users[info.owner] = devices[info.hwaddr][0]
- result['users'] = sorted(users.items(), key=lambda (u, a): a, reverse=True)
+ result['users'] = sorted(list(users.items()), key=lambda u_a: u_a[1], reverse=True)
return result