From a53ad5edd43af7f6d64ec44cfe3dda4b8496b01a Mon Sep 17 00:00:00 2001 From: Adam Goldsmith Date: Sun, 12 Apr 2020 01:21:50 -0400 Subject: [PATCH] events: Rewrite to output to sql database instead of XML files --- memberPlumbing/events.py | 118 ++++++++++++++++----------- memberPlumbing/hid/DoorController.py | 20 ++++- 2 files changed, 89 insertions(+), 49 deletions(-) diff --git a/memberPlumbing/events.py b/memberPlumbing/events.py index d180a65..99a1e32 100755 --- a/memberPlumbing/events.py +++ b/memberPlumbing/events.py @@ -1,12 +1,48 @@ #!/usr/bin/env python3 -import os + import re +from datetime import datetime import requests from lxml import etree +from peewee import ( + BooleanField, + CharField, + CompositeKey, + DateTimeField, + IntegerField, + Model, + MySQLDatabase, + TextField, +) + +import passwords from .common import doors -from .hid.DoorController import E, E_plain + +database = MySQLDatabase( + **passwords.MEMBERSHIPWORKS_DB, + **{"charset": "utf8", "sql_mode": "PIPES_AS_CONCAT", "use_unicode": True,} +) + + +class HIDEvent(Model): + doorName = CharField() + timestamp = DateTimeField() + eventType = IntegerField() + readerAddress = IntegerField() + cardholderID = IntegerField(null=True) + commandStatus = BooleanField(null=True) + forename = TextField(null=True) + surname = TextField(null=True) + ioState = BooleanField(null=True) + newTime = DateTimeField(null=True) + oldTime = DateTimeField(null=True) + rawCardNumber = CharField(max_length=8, null=True) + + class Meta: + primary_key = CompositeKey("doorName", "timestamp", "eventType") + database = database def getStrings(door): @@ -25,57 +61,45 @@ def getStrings(door): print({int(g.group(1)): g.group(2) for g in strings}) +@database.atomic() def getMessages(door): - # get parameters for messages to get? - # honestly not really sure why this is required, their API is confusing - parXMLIn = E_plain.VertXMessage(E.EventMessages({"action": "LR"})) - parXMLOut = door.doXMLRequest(parXMLIn) - etree.dump(parXMLOut) - - if os.path.exists("logs/" + door.name + ".xml"): - # read last log - tree = etree.ElementTree(file="logs/" + door.name + ".xml") - root = tree.getroot() - recordCount = int(parXMLOut[0].attrib["historyRecordMarker"]) - int( - root[0][0].attrib["recordMarker"] - ) - else: - # first run for this door - root = None - recordCount = 1000 - - if recordCount == 0: - print("No records to get!") - return - print("Getting", recordCount, "records") - # get the actual messages - eventsXMLIn = E_plain.VertXMessage( - E.EventMessages( - { - "action": "LR", - "recordCount": str(recordCount), - "historyRecordMarker": parXMLOut[0].attrib["historyRecordMarker"], - "historyTimestamp": parXMLOut[0].attrib["historyTimestamp"], - } - ) + last_event = ( + HIDEvent.select(HIDEvent.timestamp) + .where(HIDEvent.doorName == door.name) + .order_by(HIDEvent.timestamp.desc()) + .first() ) - eventsXMLOut = door.doXMLRequest(eventsXMLIn) - # TODO: handle modeRecords=true - - for index, event in enumerate(eventsXMLOut[0]): - event.attrib["recordMarker"] = str( - int(parXMLOut[0].attrib["historyRecordMarker"]) - index - ) - - if root is None: - tree = etree.ElementTree(eventsXMLOut) + if last_event is not None: + last_ts = last_event.timestamp else: - for event in reversed(eventsXMLOut[0]): - root[0].insert(0, event) - tree.write("logs/" + door.name + ".xml") + last_ts = datetime(2010, 1, 1) + + events = door.get_events(last_ts) + + HIDEvent.insert_many( + [ + { + # fill with None, then overwrite with real contents + **{k: None for k in HIDEvent._meta.fields.keys()}, + **event.attrib, + "doorName": door.name, + } + for event in events + ] + ).on_conflict_ignore().execute() + return events + + +def dups(events): + timestamps = [e.attrib["timestamp"] for e in events] + dups = set([x for x in timestamps if timestamps.count(x) > 1]) + for e in events: + if e.attrib["timestamp"] in dups: + etree.dump(e) def main(): + HIDEvent.create_table() for door in doors.values(): getMessages(door) diff --git a/memberPlumbing/hid/DoorController.py b/memberPlumbing/hid/DoorController.py index 63bf874..41e8361 100644 --- a/memberPlumbing/hid/DoorController.py +++ b/memberPlumbing/hid/DoorController.py @@ -1,4 +1,5 @@ import csv +from datetime import datetime from io import StringIO import requests @@ -155,7 +156,7 @@ class DoorController: ) return self.doXMLRequest(el) - def get_records(self, req, count, params={}): + def get_records(self, req, count, params={}, stopFunction=None): result = [] recordCount = 0 moreRecords = True @@ -166,7 +167,7 @@ class DoorController: # again in the next request. I suspect this probably ends # poorly if the numbers line up poorly (ie an exact multiple # of the returned record limit) - while moreRecords: + while moreRecords and (stopFunction is None or stopFunction(result)): res = self.doXMLRequest( ROOT( req( @@ -194,6 +195,21 @@ class DoorController: def get_credentials(self): return self.get_records(E.Credentials, 1000) + def get_events(self, threshold): + def event_newer_than_threshold(event): + return datetime.fromisoformat(event.attrib["timestamp"]) > threshold + + def last_event_newer_than_threshold(events): + return (not events) or event_newer_than_threshold(events[-1]) + + return [ + event + for event in self.get_records( + E.EventMessages, 10000, stopFunction=last_event_newer_than_threshold + ) + if event_newer_than_threshold(event) + ] + def get_lock(self): el = ROOT(E.Doors({"action": "LR", "responseFormat": "status"})) xml = self.doXMLRequest(el)