events: Rewrite to output to sql database instead of XML files
This commit is contained in:
parent
afd6ffbdc0
commit
a53ad5edd4
@ -1,12 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import requests
|
||||
from lxml import etree
|
||||
from peewee import (
|
||||
BooleanField,
|
||||
CharField,
|
||||
CompositeKey,
|
||||
DateTimeField,
|
||||
IntegerField,
|
||||
Model,
|
||||
MySQLDatabase,
|
||||
TextField,
|
||||
)
|
||||
|
||||
import passwords
|
||||
|
||||
from .common import doors
|
||||
from .hid.DoorController import E, E_plain
|
||||
|
||||
database = MySQLDatabase(
|
||||
**passwords.MEMBERSHIPWORKS_DB,
|
||||
**{"charset": "utf8", "sql_mode": "PIPES_AS_CONCAT", "use_unicode": True,}
|
||||
)
|
||||
|
||||
|
||||
class HIDEvent(Model):
|
||||
doorName = CharField()
|
||||
timestamp = DateTimeField()
|
||||
eventType = IntegerField()
|
||||
readerAddress = IntegerField()
|
||||
cardholderID = IntegerField(null=True)
|
||||
commandStatus = BooleanField(null=True)
|
||||
forename = TextField(null=True)
|
||||
surname = TextField(null=True)
|
||||
ioState = BooleanField(null=True)
|
||||
newTime = DateTimeField(null=True)
|
||||
oldTime = DateTimeField(null=True)
|
||||
rawCardNumber = CharField(max_length=8, null=True)
|
||||
|
||||
class Meta:
|
||||
primary_key = CompositeKey("doorName", "timestamp", "eventType")
|
||||
database = database
|
||||
|
||||
|
||||
def getStrings(door):
|
||||
@ -25,57 +61,45 @@ def getStrings(door):
|
||||
print({int(g.group(1)): g.group(2) for g in strings})
|
||||
|
||||
|
||||
@database.atomic()
|
||||
def getMessages(door):
|
||||
# get parameters for messages to get?
|
||||
# honestly not really sure why this is required, their API is confusing
|
||||
parXMLIn = E_plain.VertXMessage(E.EventMessages({"action": "LR"}))
|
||||
parXMLOut = door.doXMLRequest(parXMLIn)
|
||||
etree.dump(parXMLOut)
|
||||
|
||||
if os.path.exists("logs/" + door.name + ".xml"):
|
||||
# read last log
|
||||
tree = etree.ElementTree(file="logs/" + door.name + ".xml")
|
||||
root = tree.getroot()
|
||||
recordCount = int(parXMLOut[0].attrib["historyRecordMarker"]) - int(
|
||||
root[0][0].attrib["recordMarker"]
|
||||
)
|
||||
else:
|
||||
# first run for this door
|
||||
root = None
|
||||
recordCount = 1000
|
||||
|
||||
if recordCount == 0:
|
||||
print("No records to get!")
|
||||
return
|
||||
print("Getting", recordCount, "records")
|
||||
# get the actual messages
|
||||
eventsXMLIn = E_plain.VertXMessage(
|
||||
E.EventMessages(
|
||||
{
|
||||
"action": "LR",
|
||||
"recordCount": str(recordCount),
|
||||
"historyRecordMarker": parXMLOut[0].attrib["historyRecordMarker"],
|
||||
"historyTimestamp": parXMLOut[0].attrib["historyTimestamp"],
|
||||
}
|
||||
)
|
||||
last_event = (
|
||||
HIDEvent.select(HIDEvent.timestamp)
|
||||
.where(HIDEvent.doorName == door.name)
|
||||
.order_by(HIDEvent.timestamp.desc())
|
||||
.first()
|
||||
)
|
||||
eventsXMLOut = door.doXMLRequest(eventsXMLIn)
|
||||
# TODO: handle modeRecords=true
|
||||
|
||||
for index, event in enumerate(eventsXMLOut[0]):
|
||||
event.attrib["recordMarker"] = str(
|
||||
int(parXMLOut[0].attrib["historyRecordMarker"]) - index
|
||||
)
|
||||
|
||||
if root is None:
|
||||
tree = etree.ElementTree(eventsXMLOut)
|
||||
if last_event is not None:
|
||||
last_ts = last_event.timestamp
|
||||
else:
|
||||
for event in reversed(eventsXMLOut[0]):
|
||||
root[0].insert(0, event)
|
||||
tree.write("logs/" + door.name + ".xml")
|
||||
last_ts = datetime(2010, 1, 1)
|
||||
|
||||
events = door.get_events(last_ts)
|
||||
|
||||
HIDEvent.insert_many(
|
||||
[
|
||||
{
|
||||
# fill with None, then overwrite with real contents
|
||||
**{k: None for k in HIDEvent._meta.fields.keys()},
|
||||
**event.attrib,
|
||||
"doorName": door.name,
|
||||
}
|
||||
for event in events
|
||||
]
|
||||
).on_conflict_ignore().execute()
|
||||
return events
|
||||
|
||||
|
||||
def dups(events):
|
||||
timestamps = [e.attrib["timestamp"] for e in events]
|
||||
dups = set([x for x in timestamps if timestamps.count(x) > 1])
|
||||
for e in events:
|
||||
if e.attrib["timestamp"] in dups:
|
||||
etree.dump(e)
|
||||
|
||||
|
||||
def main():
|
||||
HIDEvent.create_table()
|
||||
for door in doors.values():
|
||||
getMessages(door)
|
||||
|
||||
|
@ -1,4 +1,5 @@
|
||||
import csv
|
||||
from datetime import datetime
|
||||
from io import StringIO
|
||||
|
||||
import requests
|
||||
@ -155,7 +156,7 @@ class DoorController:
|
||||
)
|
||||
return self.doXMLRequest(el)
|
||||
|
||||
def get_records(self, req, count, params={}):
|
||||
def get_records(self, req, count, params={}, stopFunction=None):
|
||||
result = []
|
||||
recordCount = 0
|
||||
moreRecords = True
|
||||
@ -166,7 +167,7 @@ class DoorController:
|
||||
# again in the next request. I suspect this probably ends
|
||||
# poorly if the numbers line up poorly (ie an exact multiple
|
||||
# of the returned record limit)
|
||||
while moreRecords:
|
||||
while moreRecords and (stopFunction is None or stopFunction(result)):
|
||||
res = self.doXMLRequest(
|
||||
ROOT(
|
||||
req(
|
||||
@ -194,6 +195,21 @@ class DoorController:
|
||||
def get_credentials(self):
|
||||
return self.get_records(E.Credentials, 1000)
|
||||
|
||||
def get_events(self, threshold):
|
||||
def event_newer_than_threshold(event):
|
||||
return datetime.fromisoformat(event.attrib["timestamp"]) > threshold
|
||||
|
||||
def last_event_newer_than_threshold(events):
|
||||
return (not events) or event_newer_than_threshold(events[-1])
|
||||
|
||||
return [
|
||||
event
|
||||
for event in self.get_records(
|
||||
E.EventMessages, 10000, stopFunction=last_event_newer_than_threshold
|
||||
)
|
||||
if event_newer_than_threshold(event)
|
||||
]
|
||||
|
||||
def get_lock(self):
|
||||
el = ROOT(E.Doors({"action": "LR", "responseFormat": "status"}))
|
||||
xml = self.doXMLRequest(el)
|
||||
|
Reference in New Issue
Block a user