events: Rewrite to output to sql database instead of XML files

memberPlumbing/events.py

@@ -1,12 +1,48 @@
 #!/usr/bin/env python3
-import os
+
 import re
+from datetime import datetime
 
 import requests
 from lxml import etree
+from peewee import (
+    BooleanField,
+    CharField,
+    CompositeKey,
+    DateTimeField,
+    IntegerField,
+    Model,
+    MySQLDatabase,
+    TextField,
+)
+
+import passwords
 
 from .common import doors
-from .hid.DoorController import E, E_plain
+
+database = MySQLDatabase(
+    **passwords.MEMBERSHIPWORKS_DB,
+    **{"charset": "utf8", "sql_mode": "PIPES_AS_CONCAT", "use_unicode": True,}
+)
+
+
+class HIDEvent(Model):
+    doorName = CharField()
+    timestamp = DateTimeField()
+    eventType = IntegerField()
+    readerAddress = IntegerField()
+    cardholderID = IntegerField(null=True)
+    commandStatus = BooleanField(null=True)
+    forename = TextField(null=True)
+    surname = TextField(null=True)
+    ioState = BooleanField(null=True)
+    newTime = DateTimeField(null=True)
+    oldTime = DateTimeField(null=True)
+    rawCardNumber = CharField(max_length=8, null=True)
+
+    class Meta:
+        primary_key = CompositeKey("doorName", "timestamp", "eventType")
+        database = database
 
 
 def getStrings(door):
@@ -25,57 +61,45 @@ def getStrings(door):
     print({int(g.group(1)): g.group(2) for g in strings})
 
 
+@database.atomic()
 def getMessages(door):
-    # get parameters for messages to get?
+    last_event = (
-    # honestly not really sure why this is required, their API is confusing
+        HIDEvent.select(HIDEvent.timestamp)
-    parXMLIn = E_plain.VertXMessage(E.EventMessages({"action": "LR"}))
+        .where(HIDEvent.doorName == door.name)
-    parXMLOut = door.doXMLRequest(parXMLIn)
+        .order_by(HIDEvent.timestamp.desc())
-    etree.dump(parXMLOut)
+        .first()
-
-    if os.path.exists("logs/" + door.name + ".xml"):
-        # read last log
-        tree = etree.ElementTree(file="logs/" + door.name + ".xml")
-        root = tree.getroot()
-        recordCount = int(parXMLOut[0].attrib["historyRecordMarker"]) - int(
-            root[0][0].attrib["recordMarker"]
-        )
-    else:
-        # first run for this door
-        root = None
-        recordCount = 1000
-
-    if recordCount == 0:
-        print("No records to get!")
-        return
-    print("Getting", recordCount, "records")
-    # get the actual messages
-    eventsXMLIn = E_plain.VertXMessage(
-        E.EventMessages(
-            {
-                "action": "LR",
-                "recordCount": str(recordCount),
-                "historyRecordMarker": parXMLOut[0].attrib["historyRecordMarker"],
-                "historyTimestamp": parXMLOut[0].attrib["historyTimestamp"],
-            }
-        )
     )
-    eventsXMLOut = door.doXMLRequest(eventsXMLIn)
+    if last_event is not None:
-    # TODO: handle modeRecords=true
+        last_ts = last_event.timestamp
-
-    for index, event in enumerate(eventsXMLOut[0]):
-        event.attrib["recordMarker"] = str(
-            int(parXMLOut[0].attrib["historyRecordMarker"]) - index
-        )
-
-    if root is None:
-        tree = etree.ElementTree(eventsXMLOut)
     else:
-        for event in reversed(eventsXMLOut[0]):
+        last_ts = datetime(2010, 1, 1)
-            root[0].insert(0, event)
+
-    tree.write("logs/" + door.name + ".xml")
+    events = door.get_events(last_ts)
+
+    HIDEvent.insert_many(
+        [
+            {
+                # fill with None, then overwrite with real contents
+                **{k: None for k in HIDEvent._meta.fields.keys()},
+                **event.attrib,
+                "doorName": door.name,
+            }
+            for event in events
+        ]
+    ).on_conflict_ignore().execute()
+    return events
+
+
+def dups(events):
+    timestamps = [e.attrib["timestamp"] for e in events]
+    dups = set([x for x in timestamps if timestamps.count(x) > 1])
+    for e in events:
+        if e.attrib["timestamp"] in dups:
+            etree.dump(e)
 
 
 def main():
+    HIDEvent.create_table()
     for door in doors.values():
         getMessages(door)
 

memberPlumbing/hid/DoorController.py

@@ -1,4 +1,5 @@
 import csv
+from datetime import datetime
 from io import StringIO
 
 import requests
@@ -155,7 +156,7 @@ class DoorController:
         )
         return self.doXMLRequest(el)
 
-    def get_records(self, req, count, params={}):
+    def get_records(self, req, count, params={}, stopFunction=None):
         result = []
         recordCount = 0
         moreRecords = True
@@ -166,7 +167,7 @@ class DoorController:
         # again in the next request. I suspect this probably ends
         # poorly if the numbers line up poorly (ie an exact multiple
         # of the returned record limit)
-        while moreRecords:
+        while moreRecords and (stopFunction is None or stopFunction(result)):
             res = self.doXMLRequest(
                 ROOT(
                     req(
@@ -194,6 +195,21 @@ class DoorController:
     def get_credentials(self):
         return self.get_records(E.Credentials, 1000)
 
+    def get_events(self, threshold):
+        def event_newer_than_threshold(event):
+            return datetime.fromisoformat(event.attrib["timestamp"]) > threshold
+
+        def last_event_newer_than_threshold(events):
+            return (not events) or event_newer_than_threshold(events[-1])
+
+        return [
+            event
+            for event in self.get_records(
+                E.EventMessages, 10000, stopFunction=last_event_newer_than_threshold
+            )
+            if event_newer_than_threshold(event)
+        ]
+
     def get_lock(self):
         el = ROOT(E.Doors({"action": "LR", "responseFormat": "status"}))
         xml = self.doXMLRequest(el)