events: Rewrite to output to sql database instead of XML files

This commit is contained in:
Adam Goldsmith 2020-04-12 01:21:50 -04:00
parent afd6ffbdc0
commit a53ad5edd4
2 changed files with 89 additions and 49 deletions

View File

@ -1,12 +1,48 @@
#!/usr/bin/env python3
import os
import re
from datetime import datetime
import requests
from lxml import etree
from peewee import (
BooleanField,
CharField,
CompositeKey,
DateTimeField,
IntegerField,
Model,
MySQLDatabase,
TextField,
)
import passwords
from .common import doors
from .hid.DoorController import E, E_plain
database = MySQLDatabase(
**passwords.MEMBERSHIPWORKS_DB,
**{"charset": "utf8", "sql_mode": "PIPES_AS_CONCAT", "use_unicode": True,}
)
class HIDEvent(Model):
doorName = CharField()
timestamp = DateTimeField()
eventType = IntegerField()
readerAddress = IntegerField()
cardholderID = IntegerField(null=True)
commandStatus = BooleanField(null=True)
forename = TextField(null=True)
surname = TextField(null=True)
ioState = BooleanField(null=True)
newTime = DateTimeField(null=True)
oldTime = DateTimeField(null=True)
rawCardNumber = CharField(max_length=8, null=True)
class Meta:
primary_key = CompositeKey("doorName", "timestamp", "eventType")
database = database
def getStrings(door):
@ -25,57 +61,45 @@ def getStrings(door):
print({int(g.group(1)): g.group(2) for g in strings})
@database.atomic()
def getMessages(door):
# get parameters for messages to get?
# honestly not really sure why this is required, their API is confusing
parXMLIn = E_plain.VertXMessage(E.EventMessages({"action": "LR"}))
parXMLOut = door.doXMLRequest(parXMLIn)
etree.dump(parXMLOut)
if os.path.exists("logs/" + door.name + ".xml"):
# read last log
tree = etree.ElementTree(file="logs/" + door.name + ".xml")
root = tree.getroot()
recordCount = int(parXMLOut[0].attrib["historyRecordMarker"]) - int(
root[0][0].attrib["recordMarker"]
)
else:
# first run for this door
root = None
recordCount = 1000
if recordCount == 0:
print("No records to get!")
return
print("Getting", recordCount, "records")
# get the actual messages
eventsXMLIn = E_plain.VertXMessage(
E.EventMessages(
{
"action": "LR",
"recordCount": str(recordCount),
"historyRecordMarker": parXMLOut[0].attrib["historyRecordMarker"],
"historyTimestamp": parXMLOut[0].attrib["historyTimestamp"],
}
)
last_event = (
HIDEvent.select(HIDEvent.timestamp)
.where(HIDEvent.doorName == door.name)
.order_by(HIDEvent.timestamp.desc())
.first()
)
eventsXMLOut = door.doXMLRequest(eventsXMLIn)
# TODO: handle modeRecords=true
for index, event in enumerate(eventsXMLOut[0]):
event.attrib["recordMarker"] = str(
int(parXMLOut[0].attrib["historyRecordMarker"]) - index
)
if root is None:
tree = etree.ElementTree(eventsXMLOut)
if last_event is not None:
last_ts = last_event.timestamp
else:
for event in reversed(eventsXMLOut[0]):
root[0].insert(0, event)
tree.write("logs/" + door.name + ".xml")
last_ts = datetime(2010, 1, 1)
events = door.get_events(last_ts)
HIDEvent.insert_many(
[
{
# fill with None, then overwrite with real contents
**{k: None for k in HIDEvent._meta.fields.keys()},
**event.attrib,
"doorName": door.name,
}
for event in events
]
).on_conflict_ignore().execute()
return events
def dups(events):
timestamps = [e.attrib["timestamp"] for e in events]
dups = set([x for x in timestamps if timestamps.count(x) > 1])
for e in events:
if e.attrib["timestamp"] in dups:
etree.dump(e)
def main():
HIDEvent.create_table()
for door in doors.values():
getMessages(door)

View File

@ -1,4 +1,5 @@
import csv
from datetime import datetime
from io import StringIO
import requests
@ -155,7 +156,7 @@ class DoorController:
)
return self.doXMLRequest(el)
def get_records(self, req, count, params={}):
def get_records(self, req, count, params={}, stopFunction=None):
result = []
recordCount = 0
moreRecords = True
@ -166,7 +167,7 @@ class DoorController:
# again in the next request. I suspect this probably ends
# poorly if the numbers line up poorly (ie an exact multiple
# of the returned record limit)
while moreRecords:
while moreRecords and (stopFunction is None or stopFunction(result)):
res = self.doXMLRequest(
ROOT(
req(
@ -194,6 +195,21 @@ class DoorController:
def get_credentials(self):
return self.get_records(E.Credentials, 1000)
def get_events(self, threshold):
def event_newer_than_threshold(event):
return datetime.fromisoformat(event.attrib["timestamp"]) > threshold
def last_event_newer_than_threshold(events):
return (not events) or event_newer_than_threshold(events[-1])
return [
event
for event in self.get_records(
E.EventMessages, 10000, stopFunction=last_event_newer_than_threshold
)
if event_newer_than_threshold(event)
]
def get_lock(self):
el = ROOT(E.Doors({"action": "LR", "responseFormat": "status"}))
xml = self.doXMLRequest(el)