optimize leases parsing - skip already parsed entries

1 files changed, 73 insertions, 29 deletions

diff --git a/at.py b/at.py
index 1fd0b5c..5a06ef2 100644
--- a/at.py
+++ b/at.py
@@ -13,7 +13,7 @@ from flask import Flask, render_template, abort, g, \
 from datetime import datetime
 from time import sleep, time, mktime
 from collections import namedtuple
-from urllib import urlencode
+from urllib.parse import urlencode
 
 from spaceauth import SpaceAuth, login_required, current_user, cap_required
 
@@ -58,7 +58,7 @@ def restrict_ip(prefix='', exclude=[]):
 
 
 def req_to_ctx():
-    return dict(request.form.iteritems())
+    return dict(iter(request.form.items()))
 
 
 @app.template_filter('strfts')
@@ -112,19 +112,18 @@ class Updater(threading.Thread):
 
     def purge_stale(self):
         now = time()
-        for addr, (atime, ip, name) in self.active.items():
+        for addr, (atime, ip, name) in list(self.active.items()):
             if now - atime > self.timeout:
                 del self.active[addr]
 
     def get_active_devices(self):
-        self.lock.acquire()
-        self.purge_stale()
-        r = dict(self.active)
-        self.lock.release()
+        with self.lock:
+            self.purge_stale()
+            r = dict(self.active)
         return r
 
     def get_device(self, ip):
-        active_devices = self.get_active_devices().iteritems()
+        active_devices = iter(self.get_active_devices().items())
         for hwaddr, (atime, dip, name) in active_devices:
             if ip == dip:
                 return hwaddr, name
@@ -135,12 +134,11 @@ class Updater(threading.Thread):
             atime -= self.lease_offset
         else:
             atime = time()
-        self.lock.acquire()
-        if hwaddr not in self.active or self.active[hwaddr][0] < atime:
-            self.active[hwaddr] = (atime, ip, name)
-            app.logger.info('updated %s with atime %s and ip %s',
-                            hwaddr, strfts(atime), ip)
-        self.lock.release()
+        with self.lock:
+            if hwaddr not in self.active or self.active[hwaddr][0] < atime:
+                self.active[hwaddr] = (atime, ip, name)
+                app.logger.info('updated %s with atime %s and ip %s',
+                    hwaddr, strfts(atime), ip)
 
 
 class CapUpdater(Updater):
@@ -169,36 +167,73 @@ class CapUpdater(Updater):
 class MtimeUpdater(Updater):
     def __init__(self, lease_file, *a, **kw):
         self.lease_file = lease_file
+        self.position = 0
         self.last_modified = 0
         Updater.__init__(self, *a, **kw)
 
     def file_changed(self, f):
-        pass
+        """Callback on changed lease file
+
+        Args:
+            f: Lease file. File offset can be used to skip already parsed lines.
+
+        Returns: New byte offset pointing after last parsed byte.
+        """
+        return f.tell()
 
     def _trigger_update(self):
         app.logger.info('Lease file changed, updating')
         with open(self.lease_file, 'r') as f:
-            self.file_changed(f)
+            f.seek(self.position)
+            self.position = self.file_changed(f)
 
     def run(self):
+        """Periodicaly check if file has changed
+
+        From ISC DHCPD manual:
+
+            New leases are appended to the end of the dhcpd.leases file.  In
+            order to prevent the file from becoming arbitrarily large, from
+            time to time dhcpd creates a new dhcpd.leases file from its in-core
+            lease database. Once this file has been written to disk, the old
+            file is renamed dhcpd.leases~, and the new file is renamed
+            dhcpd.leases.
+        """
         while True:
             try:
-                mtime = os.stat(self.lease_file).st_mtime
+                stat = os.stat(self.lease_file)
+                mtime = stat.st_mtime
+                size = stat.st_size
+                if size < self.position:
+                    app.logger.info('leases file changed - reseting pointer')
+                    self.position = 0
+                try:
+                    # checking if DHCPD performed cleanup
+                    # cleanup during operation seems to be currently broken
+                    # on customs so this could never execute
+                    purge_time = os.stat(self.lease_file + '~').st_mtime
+                    if purge_time > self.last_modified:
+                        app.logger.info('leases file purged - reseting pointer')
+                        self.position = 0
+                except FileNotFoundError:
+                    pass
                 if mtime > self.last_modified:
                     self._trigger_update()
                 self.last_modified = mtime
-                sleep(3.0)
+                sleep(5.0)
             except Exception as e:
-                app.logger.error('Updater got an exception:\n' +
-                                 traceback.format_exc(e))
+                app.logger.exception('Exception in updater')
                 sleep(10.0)
 
 
 class DnsmasqUpdater(MtimeUpdater):
     def file_changed(self, f):
+        raise NotImplementedError(
+            "This was not tested after adding differential update")
         for line in f:
             ts, hwaddr, ip, name, client_id = line.split(' ')
             self.update(hwaddr, int(ts), ip, name)
+        return f.tell()
 
 
 class DhcpdUpdater(MtimeUpdater):
@@ -208,7 +243,12 @@ class DhcpdUpdater(MtimeUpdater):
         ip = None
         hwaddr = None
         atime = None
-        for line in f:
+        offset = f.tell()
+        while True:
+            # using readline because iter(file) blocks file.tell usage
+            line = f.readline()
+            if not line:
+                return offset
             line = line.split('#')[0]
             cmd = line.strip().split()
             if not cmd:
@@ -224,6 +264,7 @@ class DhcpdUpdater(MtimeUpdater):
                 if(field == 'hardware'):
                     hwaddr = cmd[2][:-1]
                 if(field.startswith('}')):
+                    offset = f.tell()
                     lease = False
                     if hwaddr is not None and atime is not None:
                         self.update(hwaddr, atime, ip, name)
@@ -243,15 +284,18 @@ def main_view():
 def list_all():
     data = now_at()
 
-    def prettify_user((user, atime)):
+    def prettify_user(xxx_todo_changeme):
+        (user, atime) = xxx_todo_changeme
+        if user == 'greenmaker':
+            user = 'dreammaker'
         return {
             'login': user,
             'timestamp': atime,
             'pretty_time': strfts(atime),
         }
     result = {}
-    result['users'] = map(prettify_user, data.pop('users'))
-    result.update((k, len(v)) for k, v in data.items())
+    result['users'] = list(map(prettify_user, data.pop('users')))
+    result.update((k, len(v)) for k, v in list(data.items()))
     res = make_response(json.dumps(result), 200)
     res.headers['Access-Control-Allow-Origin'] = '*'
     return res
@@ -260,11 +304,11 @@ def list_all():
 def now_at():
     result = dict()
     devices = app.updater.get_active_devices()
-    device_infos = list(get_device_infos(g.db, devices.keys()))
-    device_infos.sort(key=lambda di: devices.__getitem__)
+    device_infos = list(get_device_infos(g.db, list(devices.keys())))
+    device_infos.sort(key=lambda di: devices[di.hwaddr])
     unknown = set(devices.keys()) - set(d.hwaddr for d in device_infos)
     # das kektop sorting maschine
-    for name, prefixes in app.config['SPECIAL_DEVICES'].items():
+    for name, prefixes in list(app.config['SPECIAL_DEVICES'].items()):
         result[name] = set()
         for u in unknown.copy():
             if u.startswith(prefixes):
@@ -275,9 +319,9 @@ def now_at():
 
     users = {}
     for info in device_infos:
-        if info.owner not in users:
+        if info.owner not in users and not info.ignored:
             users[info.owner] = devices[info.hwaddr][0]
-    result['users'] = sorted(users.items(), key=lambda (u, a): a, reverse=True)
+    result['users'] = sorted(list(users.items()), key=lambda u_a: u_a[1], reverse=True)
 
     return result