sqlExport: Use insert_many() to bulk insert transactions

This fixes an issue that seems to have only manifested on newer Python
versions (maybe >3.9, after update to Debian 12)

.gitignore

@@ -1,2 +1,3 @@
 __pycache__/
-/passwords.py
+/venv/
+/config.yaml

README.md

@@ -0,0 +1,39 @@
+# Claremont MakerSpace Member Plumbing
+
+This repo contains a set of scripts to sync data around for the Claremont MakerSpace. They primarily revolve around pulling member data from [MembershipWorks](https://membershipworks.com/) and pushing it out to various systems at the Space.
+
+## Setup
+
+This project uses [Poetry](https://python-poetry.org/) for dependency management. Typical usage is first running `poetry install` to create a virtualenv and install dependencies, then running `poetry run <script>` to start a specific script.
+
+## Config
+
+Many of the scripts use data from a `config.yaml` in the current working directory when they are run. There is an example config in [`config.example.yaml`](./config.example.yaml) which has been stripped of authentication information.
+
+## Scripts
+
+The primary entry points have scripts entries (`tool.poetry.scripts`) in [`pyproject.toml`](./pyproject.toml). They assume that they are being run from a module, so must be run with `poetry run <script>` or `python -m memberPlumbing.<script>`.
+
+### `doorUpdater`
+
+Retrieves member information from MembershipWorks and pushes it out to the HID Edge Evo SOLO controllers that do access control at the Space. Configuration lives in `config.yaml`.
+
+### `ucsAccounts`
+
+Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space.
+
+### `sqlExport`
+
+Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined with [peewee](peewee-orm.com) in [`memberPlumbing/mw_models.py`](./memberPlumbing/mw_models.py).
+
+### `upcomingEvents`
+
+Retrieves upcoming events from MembershipWorks and formats them for a WordPress post.
+
+### `hidEvents`
+
+Retrieves events from the HID Evo Solo door controllers, and pushes them to a SQL database.
+
+## Systemd
+
+There are systemd units in the [`systemd`](./systemd/) folder, which can be used to run the various scripts regularly.

common.py

@@ -1,78 +0,0 @@
-import csv
-from ruamel.yaml import YAML
-import urllib3
-import os
-import sys
-from io import StringIO
-import requests
-from hid.DoorController import DoorController
-
-from passwords import (DOOR_USERNAME, DOOR_PASSWORD,
-                       MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD)
-
-# it's fine, ssl certs are for losers anyway
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-try:
-    with open(os.path.dirname(os.path.abspath(__file__)) + "/config.yaml") as f:
-        config = YAML().load(f)
-except NameError:
-    with open("config.yaml") as f:
-        config = YAML().load(f)
-
-doors = {doorName: DoorController(doorData['ip'],
-                                  DOOR_USERNAME, DOOR_PASSWORD,
-                                  name=doorName, access=doorData['access'])
-         for doorName, doorData in config["doors"].items()}
-
-memberLevels = config['memberLevels']
-doorSpecificSchedules = config['doorSpecificSchedules']
-
-def getMembershipworksData(folders, columns):
-    """ Pull the members csv from the membershipworks api
-        folders: a list of the names of the folders to get
-                 (see folder_map in this function for mapping to ids)
-        columns: which columns to get"""
-    BASE_URL = "https://api.membershipworks.com/v1/"
-    folder_map = {'members': '5ae37979f033bfe8534f8799',
-                  'staff': '5771675edcdf126302a2f6b9',
-                  'misc': '5b69ee9bf033bf8e7346c434'}
-
-    # login
-    r = requests.post(BASE_URL + 'usr',
-                      data={"_st": "all",
-                            "eml": MEMBERSHIPWORKS_USERNAME,
-                            "org": "10000",
-                            "pwd": MEMBERSHIPWORKS_PASSWORD})
-    if r.status_code != 200 or 'SF' not in r.json():
-        print("MembershipWorks Login Error: ", r.status_code, r.reason)
-        print(r.text)
-        sys.exit(1)
-    login_data = r.json()
-
-    # get list of member/staff IDs
-    r = requests.get(BASE_URL + "ylp",
-                     params={"SF": login_data['SF'],
-                             "lbl": ",".join([folder_map[f] for f in folders]),
-                             "org": login_data['org'],
-                             "var": "_id,nam,ctc"})
-    if r.status_code != 200 or 'usr' not in r.json():
-        print("MembershipWorks User Listing Error: ", r.status_code, r.reason)
-        print(r.text)
-        sys.exit(1)
-    ids = [user['uid'] for user in r.json()['usr']]
-
-    # get members CSV
-    # TODO: maybe can just use previous get instead? would return JSON
-    r = requests.post(BASE_URL + "csv",
-                      params={"SF": login_data['SF']},
-                      data={"_rt": "946702800", # unknown
-                            "mux": "", # unknown
-                            "tid": ",".join(ids), # ids of members to get
-                            "var": columns})
-    if r.status_code != 200:
-        print("MembershipWorks CSV Generation Error: ", r.status_code, r.reason)
-        print(r.text)
-        sys.exit(1)
-
-    return list(csv.DictReader(StringIO(r.text)))

config.example.yaml

@@ -1,4 +1,4 @@
-doors:
+doorControllers:
   Studio Space: {ip: 172.18.51.11, access: Studio Space}
   Front Door: {ip: 172.18.51.12, access: Front Door}
   Metal Shop: {ip: 172.18.51.13, access: Metal Shop}
@@ -26,3 +26,26 @@ doorSpecificSchedules:
     - Metal Shop
     - Wood Shop
     - Wood Shop Rear
+
+
+DOOR_USERNAME: ""
+DOOR_PASSWORD: ""
+
+MEMBERSHIPWORKS_USERNAME: ""
+MEMBERSHIPWORKS_PASSWORD: ""
+
+# arguments for https://udm-rest-client.readthedocs.io/en/latest/udm_rest_client.html#udm_rest_client.udm.UDM
+UCS:
+  url: ""
+  username: ""
+  password: ""
+
+MEMBERSHIPWORKS_DB:
+  database: ""
+  user: ""
+  password: ""
+
+HID_DB:
+  database: ""
+  user: ""
+  password: ""

doorUpdater.py

@@ -1,245 +0,0 @@
-#!/usr/bin/env python3
-
-from common import (doors, getMembershipworksData, memberLevels,
-                    doorSpecificSchedules)
-from hid.Credential import Credential
-from hid.DoorController import ROOT, E
-
-
-class Member():
-    def __init__(self, forename="", surname="", membershipWorksID="",
-                 middleName="", email="", phone="",
-                 cardholderID=None, doorAccess=[], onHold=False,
-                 credentials=set(), levels=[], extraLevels=[], schedules=[]):
-        self.forename = forename
-        self.surname = surname
-        self.membershipWorksID = membershipWorksID
-        self.middleName = middleName
-        self.email = email
-        self.phone = phone
-        self.cardholderID = cardholderID
-        self.doorAccess = doorAccess
-        self.onHold = onHold
-        self.credentials = credentials
-        self.levels = levels
-        self.schedules = schedules
-
-    def __repr__(self):
-        return f"""Name: {self.forename} | {self.middleName} | {self.surname}
-MembershipWorks ID: {self.membershipWorksID}
-Email: {self.email}
-Phone: {self.phone}
-Cardholder ID: {self.cardholderID}
-doorAccess: {self.doorAccess}
-On Hold? {self.onHold}
-Credentials: {self.credentials}
-Levels: {self.levels}
-Schedules: {self.schedules}
-"""
-
-
-class MembershipworksMember(Member):
-    def __init__(self, data):
-        super().__init__(data["First Name"],
-                         data["Last Name"],
-                         membershipWorksID=data["Account ID"],
-                         email=data["Email"],
-                         phone=data["Phone"],
-                         onHold=data["Account on Hold"] != "")
-
-        if data["Access Card Number"] != "":
-            self.credentials = set([Credential(
-                code=(data["Access Card Facility Code"],
-                      data["Access Card Number"]))])
-        else:
-            self.credentials = set()
-
-        levels = {k: v for k, v in memberLevels.items() if data[k] == k}
-        self.levels = list(levels.keys())
-        self.schedules = list(levels.values())
-
-        self.extraLevels = {
-            schedule: sum((doors for prop, doors in props.items()
-                           if data[prop] == "Y"),
-                          [])
-            for schedule, props in doorSpecificSchedules.items()
-        }
-
-        self.doorAccess = [
-            door for door, doorData in doors.items()
-            if data["Access " + doorData.access + "?"] == "Y"
-        ]
-
-    def to_DoorMember(self, door):
-        doorLevels = [k for k, v in self.extraLevels.items() if door.name in v]
-
-        if door.name not in self.doorAccess or self.onHold:
-            schedules = []
-        else:
-            schedules = self.schedules + doorLevels
-
-        dm = DoorMember(door, **self.__dict__)  # this is probably terrible
-        dm.levels = self.levels + doorLevels
-        dm.schedules = schedules
-
-        return dm
-
-
-class DoorMember(Member):
-    def __init__(self, door, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.door = door
-
-    @classmethod
-    def from_cardholder(cls, data, door):
-        ch = cls(door=door,
-                 forename=data.get('forename', ""),
-                 surname=data.get('surname', ""),
-                 membershipWorksID=data.attrib.get('custom2', ""),
-                 middleName=data.attrib.get('middleName', ""),
-                 email=data.attrib.get('email', ""),
-                 phone=data.attrib.get('phone', ""),
-                 cardholderID=data.attrib['cardholderID'])
-
-        ch.credentials = set(
-            Credential(hex=(c.attrib['rawCardNumber']))
-            for c in data.findall('{*}Credential'))
-
-        ch.levels = data.attrib.get('custom1', "").split('|')
-        ch.schedules = [r.attrib['scheduleName']
-                        for r in data.findall('{*}Role')]
-
-        return ch
-
-    def attribs(self):
-        return {"forename": self.forename,
-                "surname": self.surname,
-                "middleName": self.middleName,
-                "email": self.email,
-                "phone": self.phone,
-                "custom1":  "|".join(self.levels).replace("&", "and"),
-                "custom2": self.membershipWorksID}
-
-    def make_schedules(self, schedulesMap):
-        roles = [
-            E.Role({"roleID": self.cardholderID,
-                    "scheduleID": schedulesMap[schedule],
-                    "resourceID": "0"})
-            for schedule in self.schedules]
-
-        return E.RoleSet({"action": "UD", "roleSetID": self.cardholderID},
-                         E.Roles(*roles))
-
-    def make_credentials(self, newCredentials, cardFormats):
-        out = [
-            E.Credential(
-                {"formatName": str(credential.code[0]),
-                 "cardNumber": str(credential.code[1]),
-                 "formatID": cardFormats[str(credential.code[0])],
-                 "isCard": "true",
-                 "cardholderID": self.cardholderID})
-            for credential in newCredentials]
-
-        return E.Credentials({"action": "AD"}, *out)
-
-
-def update_door(door, members):
-    cardFormats = door.get_cardFormats()
-    cardholders = {member.membershipWorksID: member
-                   for member in [DoorMember.from_cardholder(ch, door)
-                                  for ch in door.get_cardholders()]}
-    schedulesMap = door.get_scheduleMap()
-    allCredentials = set(Credential(hex=c.attrib['rawCardNumber'])
-                         for c in door.get_credentials())
-
-    # TODO: can I combine requests?
-    for membershipworksMember in members:
-        member = membershipworksMember.to_DoorMember(door)
-        # cardholder exists, compare contents
-        if member.membershipWorksID in cardholders:
-            ch = cardholders.pop(member.membershipWorksID)
-            member.cardholderID = ch.cardholderID
-            if member.attribs() != ch.attribs():  # update cardholder attributes
-                print(f"- Updating profile for {member.forename} {member.surname}")
-                print(f"  - Old: {ch.attribs()}")
-                print(f"  - New: {member.attribs()}")
-                door.doXMLRequest(ROOT(
-                    E.Cardholders(
-                        {"action": "UD", "cardholderID": member.cardholderID},
-                        E.CardHolder(member.attribs()))))
-
-            if member.credentials != ch.credentials:
-                print(f"- Updating card for {member.forename} {member.surname}")
-                print(f"  - {ch.credentials} -> {member.credentials}")
-
-                oldCards = ch.credentials
-                newCards = member.credentials
-
-                allNewCards = set(
-                    card for m in members if m != membershipworksMember
-                    for card in m.credentials)
-
-                # cards removed, and won't be reassigned to someone else
-                for card in (oldCards - newCards) - allNewCards:
-                    door.doXMLRequest(ROOT(E.Credentials(
-                        {"action": "UD",
-                         "rawCardNumber": card.hex,
-                         "isCard": "true"},
-                        E.Credential({"cardholderID": ""}))))
-
-                if newCards - oldCards:  # cards added
-                    for card in newCards & allNewCards:  # new card exists in another member
-                        print([m for m in members
-                              for card in m.credentials
-                               if card in newCards])
-                        raise Exception(f"Duplicate Card in input data! {card}")
-
-                    # card existed in door, and needs to be reassigned
-                    for card in newCards & allCredentials:
-                        door.doXMLRequest(ROOT(E.Credentials(
-                            {"action": "UD",
-                             "rawCardNumber": card.hex,
-                             "isCard": "true"},
-                            E.Credential({"cardholderID": member.cardholderID}))))
-
-                    # cards that never existed, and need to be created
-                    if newCards - allCredentials:
-                        door.doXMLRequest(ROOT(member.make_credentials(
-                            newCards - allCredentials, cardFormats)))
-
-            if member.schedules != ch.schedules:
-                print("- Updating schedule for" +
-                      f" {member.forename} {member.surname}:" +
-                      f" {ch.schedules} -> {member.schedules}")
-                door.doXMLRequest(ROOT(
-                    member.make_schedules(schedulesMap)))
-        else:  # cardholder did not exist, so add them
-            print(f"- Adding Member:")
-            print(member)
-            resp = door.doXMLRequest(ROOT(
-                E.Cardholders(
-                    {"action": "AD"},
-                    E.Cardholder(member.attribs()))))
-            member.cardholderID = resp.find('{*}Cardholders/{*}Cardholder') \
-                                      .attrib["cardholderID"]
-            door.doXMLRequest(ROOT(
-                member.make_credentials(member.credentials, cardFormats),
-                member.make_schedules(schedulesMap)))
-
-    # TODO: delete cardholders that are no longer members?
-
-
-def main():
-    memberData = getMembershipworksData(
-        ['members', 'staff', 'misc'],
-        "_id,nam,phn,eml,lvl,lbl,xws,xms,xsc,xas,xfd,xac,xcf,xeh,xse")
-
-    members = [MembershipworksMember(m) for m in memberData]
-
-    for door in doors.values():
-        print(door.name, door.ip)
-        update_door(door, members)
-
-
-if __name__ == '__main__':
-    main()

doorUtil.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-from common import doors
-
-def forEachDoor(fxn):
-    for door in doors.values():
-        print(door.name)
-        fxn(door)
-
-#forEachDoor(lambda door: door.sendCardFormat("A901146A-244", 1, 244))
-#forEachDoor(lambda door: door.sendSchedules())

events.py

@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-import datetime
-import itertools
-import re
-
-import numpy as np
-import pandas as pd
-
-import requests
-from lxml import etree
-
-from common import doors
-from hid.DoorController import ROOT, E
-
-
-def getStrings(door):
-    """Parses out the message strings from source."""
-    r = requests.get('https://' + door.ip + '/html/en_EN/en_EN.js',
-                     auth=requests.auth.HTTPDigestAuth(door.username,
-                                                       door.password),
-                     verify=False)
-    regex = re.compile(r'([0-9]+)="([^"]*)')
-    strings = [regex.search(s) for s in r.text.split(';')
-               if s.startswith('localeStrings.eventDetails')]
-    print({int(g.group(1)): g.group(2) for g in strings})
-
-def getMessages(door):
-    events = None
-    recordCount = 0
-    moreRecords = True
-
-    while moreRecords:
-        res = door.doXMLRequest(ROOT(
-            E.EventMessages({
-                "action": "LR",
-                "recordCount": str(1000 - recordCount),
-                "recordOffset": str(recordCount),
-            })))
-        if events is None:
-            events = res[0]
-        else:
-            for event in res[0]:
-                events.append(event)
-
-        recordCount += int(res[0].get('recordCount'))
-        moreRecords = res[0].get('moreRecords') == 'true'
-
-        print(recordCount, moreRecords)
-
-    etree.dump(events, pretty_print=True)
-
-    return events
-
-# def stats(events):
-#     eventsByDay = {k: list(v) for k, v in
-#                    itertools.groupby(sorted(events, key=get_day), key=get_day)}
-
-#     print({k: len(v) for k, v in eventsByDay.items()})
-
-#     #print([get_day(e) for e in events])
-
-def main():
-    for door in doors.values():
-        getMessages(door)
-
-if __name__ == '__main__':
-    main()
-
-events = getMessages(doors["Studio Space"])
-
-# stats(events)
-
-df = pd.DataFrame([dict(e.attrib) for e in events])
-idx = pd.to_datetime(df['timestamp'], format='%Y-%m-%dT%H:%M:%S')
-df = df.set_index(pd.DatetimeIndex(idx.values)).drop('timestamp', axis=1)
-print()
-print(df[df.eventType == '2020'].dropna(axis=1, how='all').head())
-
-entriesPerDay = df[df.eventType == '2020'] \
-    .dropna(axis=1, how='all') \
-    .resample('1D') \
-    .count()['eventType']
-
-entriesPerDay.index = entriesPerDay.index.map(lambda t: t.strftime('%Y-%m-%d'))
-
-print(df.groupby(by=['forename', 'surname']).size().sort_values())
-
-entriesPerDay.plot(kind='bar')

hid/DoorController.py

@@ -1,166 +0,0 @@
-import csv
-from io import StringIO
-
-import requests
-from lxml import etree
-from lxml.builder import ElementMaker
-
-E_plain = ElementMaker(nsmap={"hid": "http://www.hidglobal.com/VertX"})
-E = ElementMaker(namespace="http://www.hidglobal.com/VertX",
-                 nsmap={"hid": "http://www.hidglobal.com/VertX"})
-E_corp = ElementMaker(namespace="http://www.hidcorp.com/VertX", #stupid
-                 nsmap={"hid": "http://www.hidcorp.com/VertX"})
-ROOT = E_plain.VertXMessage
-
-fieldnames = "CardNumber,CardFormat,PinRequired,PinCode,ExtendedAccess,ExpiryDate,Forename,Initial,Surname,Email,Phone,Custom1,Custom2,Schedule1,Schedule2,Schedule3,Schedule4,Schedule5,Schedule6,Schedule7,Schedule8".split(",")
-
-class RemoteError(Exception):
-    def __init__(self, r):
-        super().__init__(
-            f"Door Updating Error: {r.status_code} {r.reason}\n{r.text}")
-
-class DoorController():
-    def __init__(self, ip, username, password, name="", access=""):
-        self.ip = ip
-        self.username = username
-        self.password = password
-        self.name = name
-        self.access = access
-
-    def doImport(self, params=None, files=None):
-        """Send a request to the door control import script"""
-        r = requests.post(
-            'https://' + self.ip + '/cgi-bin/import.cgi',
-            params=params,
-            files=files,
-            auth=requests.auth.HTTPDigestAuth(self.username, self.password),
-            timeout=60,
-            verify=False) # ignore insecure SSL
-        xml = etree.XML(r.content)
-        if r.status_code != 200 \
-           or len(xml.findall("{http://www.hidglobal.com/VertX}Error")) > 0:
-            raise RemoteError(r)
-
-    def doCSVImport(self, csv):
-        """Do the CSV import procedure on a door control"""
-        self.doImport({"task": "importInit"})
-        self.doImport({"task": "importCardsPeople", "name": "cardspeopleschedule.csv"},
-                        {"importCardsPeopleButton": ("cardspeopleschedule.csv", csv, 'text/csv')})
-        self.doImport({"task": "importDone"})
-
-    def doXMLRequest(self, xml, prefix=b'<?xml version="1.0" encoding="UTF-8"?>'):
-        if not isinstance(xml, bytes):
-            xml = etree.tostring(xml)
-        r = requests.get(
-            'https://' + self.ip + '/cgi-bin/vertx_xml.cgi',
-            params={'XML': prefix + xml},
-            auth=requests.auth.HTTPDigestAuth(self.username, self.password),
-            verify=False)
-        resp_xml = etree.XML(r.content)
-        # probably meed to be more sane about this
-        if r.status_code != 200 \
-           or len(resp_xml.findall("{*}Error")) > 0:
-            raise RemoteError(r)
-        return resp_xml
-
-    def get_scheduleMap(self):
-        schedules = self.doXMLRequest(
-            ROOT(E.Schedules({"action": "LR",
-                              "recordOffset": "0",
-                              "recordCount": "8"})))
-        return {fmt.attrib["scheduleName"]: fmt.attrib["scheduleID"]
-                for fmt in schedules[0]}
-
-    def get_schedules(self):
-        # TODO: might be able to do in one request
-        schedules = self.doXMLRequest(ROOT(
-            E.Schedules({"action": "LR"})))
-        etree.dump(schedules)
-
-        data = self.doXMLRequest(ROOT(
-            *[E.Schedules({"action": "LR",
-                           "scheduleID": schedule.attrib["scheduleID"]})
-              for schedule in schedules[0]]))
-        return ROOT(E_corp.Schedules({"action": "AD"},
-                           *[s[0] for s in data]))
-
-    def set_schedules(self, schedules):
-        # clear all people
-        outString = StringIO()
-        writer = csv.DictWriter(outString, fieldnames)
-        writer.writeheader()
-        writer.writerow({})
-        outString.seek(0)
-        self.doCSVImport(outString)
-
-        # clear all schedules
-        delXML = ROOT(
-            *[E.Schedules({"action": "DD", "scheduleID": str(ii)})
-             for ii in range(1, 8)])
-        try:
-            self.doXMLRequest(delXML)
-        except RemoteError:
-            # don't care about failure to delete, they probably just didn't exist
-            pass
-
-        # load new schedules
-        self.doXMLRequest(schedules)
-
-    def get_cardFormats(self):
-        cardFormats = self.doXMLRequest(
-            ROOT(E.CardFormats({"action": "LR",
-                                "responseFormat": "expanded"})))
-
-        return {fmt[0].attrib["value"]: fmt.attrib["formatID"]
-                for fmt in cardFormats[0].findall('{*}CardFormat[{*}FixedField]')}
-
-    def set_cardFormat(self, formatName, templateID, facilityCode):
-        # TODO: add ability to delete formats
-        # delete example: <hid:CardFormats action="DD" formatID="7-1-244"/>
-
-        el = ROOT(
-            E.CardFormats({"action": "AD"},
-                E.CardFormat({"formatName": formatName,
-                              "templateID": str(templateID)},
-                    E.FixedField({"value": str(facilityCode)}))))
-        return self.doXMLRequest(el)
-
-    def get_records(self, req, count, params={}):
-        result = []
-        recordCount = 0
-        moreRecords = True
-
-        while moreRecords:
-            res = self.doXMLRequest(ROOT(
-                req({
-                    "action": "LR",
-                    "recordCount": str(count - recordCount),
-                    "recordOffset": str(recordCount),
-                    **params
-                })))
-            result += list(res[0])
-
-            recordCount += int(res[0].get('recordCount'))
-            moreRecords = res[0].get('moreRecords') == 'true'
-
-        return result
-
-    def get_cardholders(self):
-        return self.get_records(E.Cardholders, 1000,
-                                {"responseFormat": "expanded"})
-
-    def get_credentials(self):
-        return self.get_records(E.Credentials, 1000)
-
-    def get_lock(self):
-        el = ROOT(
-            E.Doors({"action": "LR", "responseFormat": "status"}))
-        xml = self.doXMLRequest(el)
-        relayState = xml.find('./{*}Doors/{*}Door').attrib['relayState']
-        return "unlocked" if relayState == "set" else "locked"
-
-    def set_lock(self, lock=True):
-        el = ROOT(
-            E.Doors({"action": "CM",
-                     "command": "lockDoor" if lock else "unlockDoor"}))
-        return self.doXMLRequest(el)

memberPlumbing/MembershipWorks.py

@@ -0,0 +1,265 @@
+import csv
+from io import StringIO
+
+import requests
+import datetime
+
+BASE_URL = "https://api.membershipworks.com"
+
+# extracted from `SF._is.crm` in https://cdn.membershipworks.com/all.js
+CRM = {
+    0: "Note",
+    4: "Profile Updated",
+    8: "Scheduled/Reminder Email",
+    9: "Renewal Notice",
+    10: "Join Date",
+    11: "Next Renewal Date",
+    12: "Membership Payment",
+    13: "Donation",
+    14: "Event Activity",
+    15: "Conversation",
+    16: "Contact Change",
+    17: "Label Change",
+    18: "Other Payment",
+    19: "Cart Payment",
+    20: "Payment Failed",
+    21: "Billing Updated",
+    22: "Form Checkout",
+    23: "Event Payment",
+    24: "Invoice",
+    25: "Invoice Payment",
+    26: "Renewal",
+    27: "Payment",
+}
+
+# Types of fields, extracted from a html snippet in all.js + some guessing
+typ = {
+    1: "Text input",
+    2: "Password",  # inferred from data
+    3: "Simple text area",
+    4: "Rich text area",
+    7: "Address",
+    8: "Check box",
+    9: "Select",
+    11: "Display value stored in field (ie. read only)",
+    12: "Required waiver/terms",
+}
+
+# more constants, this time extracted from the members csv export in all.js
+staticFlags = {
+    "pos": {"lbl": "Position/relation (contacts)"},
+    "nte": {"lbl": "Admin note (contacts)"},
+    "pwd": {"lbl": "Password"},
+    "lgo": {"lbl": "Business card image URLs"},
+    "pfx": {"lbl": "Profile gallery image URLs"},
+    "lvl": {"lbl": "Membership levels"},
+    "aon": {"lbl": "Membership add-ons"},
+    "lbl": {"lbl": "Labels"},
+    "joi": {"lbl": "Join date"},
+    "end": {"lbl": "Renewal date"},
+    "spy": {"lbl": "Billing method"},
+    "rid": {"lbl": "Auto recurring billing ID"},
+    "ipa": {"lbl": "IP address"},
+    "_id": {"lbl": "Account ID"},
+}
+
+
+class MembershipWorksRemoteError(Exception):
+    def __init__(self, reason, r):
+        super().__init__(
+            f"Error when attempting {reason}: {r.status_code} {r.reason}\n{r.text}"
+        )
+
+
+class MembershipWorks:
+    def __init__(self):
+        self.sess = requests.Session()
+        self.org_info = None
+        self.auth_token = None
+        self.org_num = None
+
+    def login(self, username, password):
+        """Authenticate against the membershipworks api"""
+        r = self.sess.post(
+            BASE_URL + "/v2/account/session",
+            data={"eml": username, "pwd": password},
+            headers={"X-Org": "10000"},
+        )
+        if r.status_code != 200 or "SF" not in r.json():
+            raise MembershipWorksRemoteError("login", r)
+        self.org_info = r.json()
+        self.auth_token = self.org_info["SF"]
+        self.org_num = self.org_info["org"]
+        self.sess.headers.update(
+            {
+                "X-Org": str(self.org_num),
+                "X-Role": "admin",
+                "Authorization": "Bearer " + self.auth_token,
+            }
+        )
+
+    def _inject_auth(self, kwargs):
+        # TODO: should probably be a decorator or something
+        if self.auth_token is None:
+            raise RuntimeError("Not Logged in to MembershipWorks")
+        # add auth token to params
+        if "params" not in kwargs:
+            kwargs["params"] = {}
+        kwargs["params"]["SF"] = self.auth_token
+
+    def _get_v1(self, *args, **kwargs):
+        self._inject_auth(kwargs)
+        # TODO: should probably do some error handling in here
+        return requests.get(*args, **kwargs)
+
+    def _post_v1(self, *args, **kwargs):
+        self._inject_auth(kwargs)
+        # TODO: should probably do some error handling in here
+        return requests.post(*args, **kwargs)
+
+    def _all_fields(self):
+        """Parse out a list of fields from the org data.
+
+        Is this terrible? Yes. Also, not dissimilar to how MW does it
+        in all.js.
+        """
+        fields = staticFlags.copy()
+
+        # TODO: this will take the later option, if the same field
+        # used in mulitple places. I don't know which lbl is used for
+        # csv export
+
+        # anm: member signup, acc: member manage, adm: admin manage
+        for screen_type in ["anm", "acc", "adm"]:
+            for box in self.org_info["tpl"][screen_type]:
+                for element in box["box"]:
+                    if type(element["dat"]) != str:
+                        for field in element["dat"]:
+                            if "_id" in field:
+                                if field["_id"] not in fields:
+                                    fields[field["_id"]] = field
+
+        return fields
+
+    def _parse_flags(self):
+        """Parse the flags out of the org data.
+
+        This is terrible, and there might be a better way to do this.
+        """
+        ret = {"folders": {}, "levels": {}, "addons": {}, "labels": {}}
+
+        for dek in self.org_info["dek"]:
+            # TODO: there must be a better way. this is stupid
+            if dek["dek"] == 1:
+                ret["folders"][dek["lbl"]] = dek["did"]
+            elif "cur" in dek:
+                ret["levels"][dek["lbl"]] = dek["did"]
+            elif "mux" in dek:
+                ret["addons"][dek["lbl"]] = dek["did"]
+            else:
+                ret["labels"][dek["lbl"]] = dek["did"]
+
+        return ret
+
+    def get_member_ids(self, folders):
+        folder_map = self._parse_flags()["folders"]
+
+        r = self.sess.get(
+            BASE_URL + "/v2/accounts",
+            params={"dek": ",".join([folder_map[f] for f in folders])},
+        )
+        if r.status_code != 200 or "usr" not in r.json():
+            raise MembershipWorksRemoteError("user listing", r)
+
+        # get list of member ID matching the search
+        # dedup with set() to work around people with alt uids
+        # TODO: figure out why people have alt uids
+        return set(user["uid"] for user in r.json()["usr"])
+
+    # TODO: has issues with aliasing header names:
+    #       ex: "Personal Studio Space" Label vs Membership Addon/Field
+    def get_members(self, folders, columns):
+        """Pull the members csv from the membershipworks api
+        folders: a list of the names of the folders to get
+                 (see folder_map in this function for mapping to ids)
+        columns: which columns to get"""
+        ids = self.get_member_ids(folders)
+
+        # get members CSV
+        # TODO: maybe can just use previous get instead? would return JSON
+        r = self._post_v1(
+            BASE_URL + "/v1/csv",
+            data={
+                "_rt": "946702800",  # unknown
+                "mux": "",  # unknown
+                "tid": ",".join(ids),  # ids of members to get
+                "var": columns,
+            },
+        )
+        if r.status_code != 200:
+            raise MembershipWorksRemoteError("csv generation", r)
+
+        if r.text[0] == "\ufeff":
+            r.encoding = r.encoding + "-sig"
+
+        return list(csv.DictReader(StringIO(r.text)))
+
+    def get_transactions(self, start_date, end_date, json=False):
+        """Get the transactions between start_date and end_date
+
+        Dates can be datetime.date or datetime.datetime
+
+        json gets a different version of the transactions list,
+        which contains a different set information
+        """
+        r = self._get_v1(
+            BASE_URL + "/v1/csv",
+            params={
+                "crm": ",".join(str(k) for k in CRM.keys()),
+                **({"txl": ""} if json else {}),
+                "sdp": start_date.strftime("%s"),
+                "edp": end_date.strftime("%s"),
+            },
+        )
+        if r.status_code != 200:
+            raise MembershipWorksRemoteError("csv generation", r)
+        if json:
+            return r.json()
+        else:
+            if r.text[0] == "\ufeff":
+                r.encoding = r.encoding + "-sig"
+
+            return list(csv.DictReader(StringIO(r.text)))
+
+    def get_all_members(self):
+        """Get all the data for all the members"""
+        folders = self._parse_flags()["folders"].keys()
+        fields = self._all_fields()
+        members = self.get_members(folders, ",".join(fields.keys()))
+        return members
+
+    def get_events_list(self, start_date: datetime.datetime):
+        """Retrive a list of events since start_date"""
+        r = self.sess.get(
+            BASE_URL + "/v2/events",
+            params={
+                "sdp": start_date.strftime("%s"),
+            },
+        )
+        return r.json()
+
+    def get_event_by_eid(self, eid: str):
+        """Retrieve a specific event by its event id (eid)"""
+        r = self.sess.get(
+            BASE_URL + "/v2/event",
+            params={"eid": eid},
+        )
+        return r.json()
+
+    def get_event_by_url(self, url: str):
+        """Retrieve a specific event by its url"""
+        r = self.sess.get(
+            BASE_URL + "/v2/event",
+            params={"url": url},
+        )
+        return r.json()

memberPlumbing/config.py

@@ -0,0 +1,36 @@
+from ruamel.yaml import YAML
+
+from .hid.DoorController import DoorController
+from .MembershipWorks import MembershipWorks
+
+
+class Config:
+    def __init__(self, path="config.yaml"):
+        with open(path) as f:
+            self._data = YAML().load(f)
+        self.__dict__.update(self._data)
+
+        # lazy init, because this actually talks to an external server
+        self._membershipworks = None
+
+    @property
+    def doors(self):
+        return {
+            doorName: DoorController(
+                doorData["ip"],
+                self.DOOR_USERNAME,
+                self.DOOR_PASSWORD,
+                name=doorName,
+                access=doorData["access"],
+            )
+            for doorName, doorData in self.doorControllers.items()
+        }
+
+    @property
+    def membershipworks(self):
+        if not self._membershipworks:
+            self._membershipworks = MembershipWorks()
+            self._membershipworks.login(
+                self.MEMBERSHIPWORKS_USERNAME, self.MEMBERSHIPWORKS_PASSWORD
+            )
+        return self._membershipworks

memberPlumbing/doorUpdater.py

@@ -0,0 +1,370 @@
+#!/usr/bin/env python3
+
+import copy
+
+from .config import Config
+from .hid.Credential import Credential
+from .hid.DoorController import ROOT, E
+
+
+class Member:
+    def __init__(
+        self,
+        forename="",
+        surname="",
+        membershipWorksID="",
+        middleName="",
+        email="",
+        phone="",
+        cardholderID=None,
+        doorAccess=[],
+        credentials=set(),
+        levels=[],
+        extraLevels=[],
+        schedules=[],
+    ):
+        self.forename = forename
+        self.surname = surname
+        self.membershipWorksID = membershipWorksID
+        self.middleName = middleName
+        self.email = email
+        self.phone = phone
+        self.cardholderID = cardholderID
+        self.doorAccess = doorAccess
+        self.credentials = credentials
+        self.levels = levels
+        self.schedules = schedules
+
+    def __str__(self):
+        return f"""Name: {self.forename} | {self.middleName} | {self.surname}
+MembershipWorks ID: {self.membershipWorksID}
+Email: {self.email}
+Phone: {self.phone}
+Cardholder ID: {self.cardholderID}
+doorAccess: {self.doorAccess}
+Credentials: {self.credentials}
+Levels: {self.levels}
+Schedules: {self.schedules}
+"""
+
+
+class MembershipworksMember(Member):
+    def __init__(self, config, data, formerMember=False):
+        super().__init__(
+            data["First Name"],
+            data["Last Name"],
+            membershipWorksID=data["Account ID"],
+            email=data["Email"],
+            phone=data["Phone"],
+        )
+
+        if data["Access Card Number"] != "":
+            self.credentials = set(
+                [
+                    Credential(
+                        code=(
+                            data["Access Card Facility Code"],
+                            data["Access Card Number"],
+                        )
+                    )
+                ]
+            )
+        else:
+            self.credentials = set()
+
+        self.onHold = (
+            data["Account on Hold"] != ""
+            or data["CMS Membership on hold"] == "CMS Membership on hold"
+        )
+        self.formerMember = formerMember
+
+        levels = {k: v for k, v in config.memberLevels.items() if data[k] == k}
+        self.levels = list(levels.keys())
+        self.schedules = list(levels.values())
+
+        self.extraLevels = {
+            schedule: sum(
+                (doors for prop, doors in props.items() if data[prop] == "Y"), []
+            )
+            for schedule, props in config.doorSpecificSchedules.items()
+        }
+
+        self.doorAccess = [
+            door
+            for door, doorData in config.doors.items()
+            if data["Access " + doorData.access + "?"] == "Y"
+        ]
+
+    def to_DoorMember(self, door):
+        doorLevels = [k for k, v in self.extraLevels.items() if door.name in v]
+
+        schedules = []
+        if door.name in self.doorAccess and not self.onHold and not self.formerMember:
+            schedules = self.schedules + doorLevels
+
+        dm = DoorMember(
+            door,
+            forename=self.forename,
+            surname=self.surname,
+            membershipWorksID=self.membershipWorksID,
+            email=self.email,
+            phone=self.phone,
+            levels=self.levels + doorLevels,
+            doorAccess=self.doorAccess,
+            credentials=self.credentials,
+            schedules=schedules,
+        )
+
+        return dm
+
+    def __str__(self):
+        return (
+            super().__str__()
+            + f"""OnHold? {self.onHold}
+Former Member? {self.formerMember}
+"""
+        )
+
+
+class DoorMember(Member):
+    def __init__(self, door, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.door = door
+
+    @classmethod
+    def from_cardholder(cls, data, door):
+        ch = cls(
+            door=door,
+            forename=data.get("forename", ""),
+            surname=data.get("surname", ""),
+            membershipWorksID=data.attrib.get("custom2", ""),
+            middleName=data.attrib.get("middleName", ""),
+            email=data.attrib.get("email", ""),
+            phone=data.attrib.get("phone", ""),
+            cardholderID=data.attrib["cardholderID"],
+        )
+
+        ch.credentials = set(
+            Credential(hex=(c.attrib["rawCardNumber"]))
+            for c in data.findall("{*}Credential")
+        )
+
+        ch.levels = data.attrib.get("custom1", "").split("|")
+        ch.schedules = [r.attrib["scheduleName"] for r in data.findall("{*}Role")]
+
+        return ch
+
+    def attribs(self):
+        return {
+            "forename": self.forename,
+            "surname": self.surname,
+            "middleName": self.middleName,
+            "email": self.email,
+            "phone": self.phone,
+            "custom1": "|".join(self.levels).replace("&", "and"),
+            "custom2": self.membershipWorksID,
+        }
+
+    def make_schedules(self, schedulesMap):
+        roles = [
+            E.Role(
+                {
+                    "roleID": self.cardholderID,
+                    "scheduleID": schedulesMap[schedule],
+                    "resourceID": "0",
+                }
+            )
+            for schedule in self.schedules
+        ]
+
+        return E.RoleSet(
+            {"action": "UD", "roleSetID": self.cardholderID}, E.Roles(*roles)
+        )
+
+    def make_credentials(self, newCredentials, cardFormats):
+        out = [
+            E.Credential(
+                {
+                    "formatName": str(credential.code[0]),
+                    "cardNumber": str(credential.code[1]),
+                    "formatID": cardFormats[str(credential.code[0])],
+                    "isCard": "true",
+                    "cardholderID": self.cardholderID,
+                }
+            )
+            for credential in newCredentials
+        ]
+
+        return E.Credentials({"action": "AD"}, *out)
+
+
+def update_door(door, members):
+    cardFormats = door.get_cardFormats()
+    cardholders = {
+        member.membershipWorksID: member
+        for member in [
+            DoorMember.from_cardholder(ch, door) for ch in door.get_cardholders()
+        ]
+    }
+    schedulesMap = door.get_scheduleMap()
+    allCredentials = set(
+        Credential(hex=c.attrib["rawCardNumber"]) for c in door.get_credentials()
+    )
+
+    # TODO: can I combine requests?
+    for membershipworksMember in members:
+        member = membershipworksMember.to_DoorMember(door)
+        # cardholder did not exist, so add them
+        if member.membershipWorksID not in cardholders:
+            print("- Adding Member {member.forename} {member.surname}:")
+            print(f"  - {member.attribs()}")
+            resp = door.doXMLRequest(
+                ROOT(E.Cardholders({"action": "AD"}, E.Cardholder(member.attribs())))
+            )
+            member.cardholderID = resp.find("{*}Cardholders/{*}Cardholder").attrib[
+                "cardholderID"
+            ]
+
+            # create a dummy ch to force an update
+            # TODO: probably a cleaner way to do this
+            ch = copy.copy(member)
+            ch.schedules = []
+            ch.credentials = set()
+        # cardholder exists, compare contents
+        else:
+            ch = cardholders.pop(member.membershipWorksID)
+            member.cardholderID = ch.cardholderID
+
+        if member.attribs() != ch.attribs():  # update cardholder attributes
+            print(f"- Updating profile for {member.forename} {member.surname}")
+            print(f"  - Old: {ch.attribs()}")
+            print(f"  - New: {member.attribs()}")
+            door.doXMLRequest(
+                ROOT(
+                    E.Cardholders(
+                        {"action": "UD", "cardholderID": member.cardholderID},
+                        E.CardHolder(member.attribs()),
+                    )
+                )
+            )
+
+        if member.credentials != ch.credentials:
+            print(f"- Updating card for {member.forename} {member.surname}")
+            print(f"  - {ch.credentials} -> {member.credentials}")
+
+            oldCards = ch.credentials
+            newCards = member.credentials
+
+            allNewCards = set(
+                card
+                for m in members
+                if m != membershipworksMember
+                for card in m.credentials
+            )
+
+            # cards removed, and won't be reassigned to someone else
+            for card in (oldCards - newCards) - allNewCards:
+                door.doXMLRequest(
+                    ROOT(
+                        E.Credentials(
+                            {
+                                "action": "UD",
+                                "rawCardNumber": card.hex,
+                                "isCard": "true",
+                            },
+                            E.Credential({"cardholderID": ""}),
+                        )
+                    )
+                )
+
+            if newCards - oldCards:  # cards added
+                for card in newCards & allNewCards:  # new card exists in another member
+                    print(
+                        [
+                            m
+                            for m in members
+                            for card in m.credentials
+                            if card in newCards
+                        ]
+                    )
+                    raise Exception(f"Duplicate Card in input data! {card}")
+
+                # card existed in door, and needs to be reassigned
+                for card in newCards & allCredentials:
+                    door.doXMLRequest(
+                        ROOT(
+                            E.Credentials(
+                                {
+                                    "action": "UD",
+                                    "rawCardNumber": card.hex,
+                                    "isCard": "true",
+                                },
+                                E.Credential({"cardholderID": member.cardholderID}),
+                            )
+                        )
+                    )
+
+                # cards that never existed, and need to be created
+                if newCards - allCredentials:
+                    door.doXMLRequest(
+                        ROOT(
+                            member.make_credentials(
+                                newCards - allCredentials, cardFormats
+                            )
+                        )
+                    )
+
+        if member.schedules != ch.schedules:
+            print(
+                "- Updating schedule for"
+                + f" {member.forename} {member.surname}:"
+                + f" {ch.schedules} -> {member.schedules}"
+            )
+            door.doXMLRequest(ROOT(member.make_schedules(schedulesMap)))
+
+    # TODO: delete cardholders that are no longer members?
+
+
+def main():
+    config = Config()
+    membershipworks = config.membershipworks
+    membershipworks_attributes = (
+        "_id,nam,phn,eml,lvl,lbl,xws,xms,xsc,xas,xfd,xac,xcf,xeh,xse"
+    )
+
+    memberData = membershipworks.get_members(
+        ["Members", "CMS Staff", "Misc. Access"], membershipworks_attributes
+    )
+    members = [MembershipworksMember(config, m) for m in memberData]
+
+    formerMemberData = membershipworks.get_members(
+        ["Former Members"], membershipworks_attributes
+    )
+    formerMembers = [
+        MembershipworksMember(config, m, formerMember=True) for m in formerMemberData
+    ]
+
+    for formerMember in formerMembers:
+        member = next(
+            (
+                m
+                for m in members
+                if m.membershipWorksID == formerMember.membershipWorksID
+            ),
+            None,
+        )
+
+        # member exists in another folder
+        if member is not None:
+            member.formerMember = True
+        else:  # member is only a former member
+            formerMember.formerMember = True
+            members.append(formerMember)
+
+    for door in config.doors.values():
+        print(door.name, door.ip)
+        update_door(door, members)
+
+
+if __name__ == "__main__":
+    main()

memberPlumbing/hid/Credential.py

@@ -3,6 +3,7 @@ import bitstring
 # Reference for H10301 card format:
 # https://www.hidglobal.com/sites/default/files/hid-understanding_card_data_formats-wp-en.pdf
 
+
 class Credential:
     def __init__(self, code=None, hex=None):
         if code is None and hex is None:
@@ -11,8 +12,10 @@ class Credential:
             raise TypeError("Cannot set both code and hex for a Credential")
         elif code is not None:
             self.bits = bitstring.pack(
-                '0b000000, 0b0, uint:8=facility, uint:16=number, 0b0',
+                "0b000000, 0b0, uint:8=facility, uint:16=number, 0b0",
-                facility=code[0], number=code[1])
+                facility=code[0],
+                number=code[1],
+            )
             self.bits[6] = self.bits[7:19].count(1) % 2  # even parity
             self.bits[31] = not (self.bits[19:31].count(1) % 2)  # odd parity
         elif hex is not None:

memberPlumbing/hid/DoorController.py

@@ -0,0 +1,223 @@
+import csv
+from datetime import datetime
+from io import StringIO
+
+import requests
+import urllib3
+from lxml import etree
+from lxml.builder import ElementMaker
+
+E_plain = ElementMaker(nsmap={"hid": "http://www.hidglobal.com/VertX"})
+E = ElementMaker(
+    namespace="http://www.hidglobal.com/VertX",
+    nsmap={"hid": "http://www.hidglobal.com/VertX"},
+)
+E_corp = ElementMaker(
+    namespace="http://www.hidcorp.com/VertX",  # stupid
+    nsmap={"hid": "http://www.hidcorp.com/VertX"},
+)
+ROOT = E_plain.VertXMessage
+
+fieldnames = "CardNumber,CardFormat,PinRequired,PinCode,ExtendedAccess,ExpiryDate,Forename,Initial,Surname,Email,Phone,Custom1,Custom2,Schedule1,Schedule2,Schedule3,Schedule4,Schedule5,Schedule6,Schedule7,Schedule8".split(
+    ","
+)
+
+# TODO: where should this live?
+# it's fine, ssl certs are for losers anyway
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+class RemoteError(Exception):
+    def __init__(self, r):
+        super().__init__(f"Door Updating Error: {r.status_code} {r.reason}\n{r.text}")
+
+
+class DoorController:
+    def __init__(self, ip, username, password, name="", access=""):
+        self.ip = ip
+        self.username = username
+        self.password = password
+        self.name = name
+        self.access = access
+
+    def doImport(self, params=None, files=None):
+        """Send a request to the door control import script"""
+        r = requests.post(
+            "https://" + self.ip + "/cgi-bin/import.cgi",
+            params=params,
+            files=files,
+            auth=requests.auth.HTTPDigestAuth(self.username, self.password),
+            timeout=60,
+            verify=False,
+        )  # ignore insecure SSL
+        xml = etree.XML(r.content)
+        if (
+            r.status_code != 200
+            or len(xml.findall("{http://www.hidglobal.com/VertX}Error")) > 0
+        ):
+            raise RemoteError(r)
+
+    def doCSVImport(self, csv):
+        """Do the CSV import procedure on a door control"""
+        self.doImport({"task": "importInit"})
+        self.doImport(
+            {"task": "importCardsPeople", "name": "cardspeopleschedule.csv"},
+            {"importCardsPeopleButton": ("cardspeopleschedule.csv", csv, "text/csv")},
+        )
+        self.doImport({"task": "importDone"})
+
+    def doXMLRequest(self, xml, prefix=b'<?xml version="1.0" encoding="UTF-8"?>'):
+        if not isinstance(xml, bytes):
+            xml = etree.tostring(xml)
+        r = requests.get(
+            "https://" + self.ip + "/cgi-bin/vertx_xml.cgi",
+            params={"XML": prefix + xml},
+            auth=requests.auth.HTTPDigestAuth(self.username, self.password),
+            verify=False,
+        )
+        resp_xml = etree.XML(r.content)
+        # probably meed to be more sane about this
+        if r.status_code != 200 or len(resp_xml.findall("{*}Error")) > 0:
+            raise RemoteError(r)
+        return resp_xml
+
+    def get_scheduleMap(self):
+        schedules = self.doXMLRequest(
+            ROOT(E.Schedules({"action": "LR", "recordOffset": "0", "recordCount": "8"}))
+        )
+        return {
+            fmt.attrib["scheduleName"]: fmt.attrib["scheduleID"] for fmt in schedules[0]
+        }
+
+    def get_schedules(self):
+        # TODO: might be able to do in one request
+        schedules = self.doXMLRequest(ROOT(E.Schedules({"action": "LR"})))
+        etree.dump(schedules)
+
+        data = self.doXMLRequest(
+            ROOT(
+                *[
+                    E.Schedules(
+                        {"action": "LR", "scheduleID": schedule.attrib["scheduleID"]}
+                    )
+                    for schedule in schedules[0]
+                ]
+            )
+        )
+        return ROOT(E_corp.Schedules({"action": "AD"}, *[s[0] for s in data]))
+
+    def set_schedules(self, schedules):
+        # clear all people
+        outString = StringIO()
+        writer = csv.DictWriter(outString, fieldnames)
+        writer.writeheader()
+        writer.writerow({})
+        outString.seek(0)
+        self.doCSVImport(outString)
+
+        # clear all schedules
+        delXML = ROOT(
+            *[
+                E.Schedules({"action": "DD", "scheduleID": str(ii)})
+                for ii in range(1, 8)
+            ]
+        )
+        try:
+            self.doXMLRequest(delXML)
+        except RemoteError:
+            # don't care about failure to delete, they probably just didn't exist
+            pass
+
+        # load new schedules
+        self.doXMLRequest(schedules)
+
+    def get_cardFormats(self):
+        cardFormats = self.doXMLRequest(
+            ROOT(E.CardFormats({"action": "LR", "responseFormat": "expanded"}))
+        )
+
+        return {
+            fmt[0].attrib["value"]: fmt.attrib["formatID"]
+            for fmt in cardFormats[0].findall("{*}CardFormat[{*}FixedField]")
+        }
+
+    def set_cardFormat(self, formatName, templateID, facilityCode):
+        # TODO: add ability to delete formats
+        # delete example: <hid:CardFormats action="DD" formatID="7-1-244"/>
+
+        el = ROOT(
+            E.CardFormats(
+                {"action": "AD"},
+                E.CardFormat(
+                    {"formatName": formatName, "templateID": str(templateID)},
+                    E.FixedField({"value": str(facilityCode)}),
+                ),
+            )
+        )
+        return self.doXMLRequest(el)
+
+    def get_records(self, req, count, params={}, stopFunction=None):
+        result = []
+        recordCount = 0
+        moreRecords = True
+
+        # note: all the "+/-1" bits are to work around a bug where the
+        # last returned entry is incomplete. There is probably a
+        # better way to do this, but for now I just get the last entry
+        # again in the next request. I suspect this probably ends
+        # poorly if the numbers line up poorly (ie an exact multiple
+        # of the returned record limit)
+        while moreRecords and (stopFunction is None or stopFunction(result)):
+            res = self.doXMLRequest(
+                ROOT(
+                    req(
+                        {
+                            "action": "LR",
+                            "recordCount": str(count - recordCount + 1),
+                            "recordOffset": str(
+                                recordCount - 1 if recordCount > 0 else 0
+                            ),
+                            **params,
+                        }
+                    )
+                )
+            )
+            result = result[:-1] + list(res[0])
+
+            recordCount += int(res[0].get("recordCount")) - 1
+            moreRecords = res[0].get("moreRecords") == "true"
+
+        return result
+
+    def get_cardholders(self):
+        return self.get_records(E.Cardholders, 1000, {"responseFormat": "expanded"})
+
+    def get_credentials(self):
+        return self.get_records(E.Credentials, 1000)
+
+    def get_events(self, threshold):
+        def event_newer_than_threshold(event):
+            return datetime.fromisoformat(event.attrib["timestamp"]) > threshold
+
+        def last_event_newer_than_threshold(events):
+            return (not events) or event_newer_than_threshold(events[-1])
+
+        return [
+            event
+            for event in self.get_records(
+                E.EventMessages, 10000, stopFunction=last_event_newer_than_threshold
+            )
+            if event_newer_than_threshold(event)
+        ]
+
+    def get_lock(self):
+        el = ROOT(E.Doors({"action": "LR", "responseFormat": "status"}))
+        xml = self.doXMLRequest(el)
+        relayState = xml.find("./{*}Doors/{*}Door").attrib["relayState"]
+        return "unlocked" if relayState == "set" else "locked"
+
+    def set_lock(self, lock=True):
+        el = ROOT(
+            E.Doors({"action": "CM", "command": "lockDoor" if lock else "unlockDoor"})
+        )
+        return self.doXMLRequest(el)

memberPlumbing/hidEvents.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+import re
+from datetime import datetime
+
+import requests
+from lxml import etree
+from peewee import (
+    BooleanField,
+    CharField,
+    CompositeKey,
+    DateTimeField,
+    IntegerField,
+    Model,
+    MySQLDatabase,
+    TextField,
+)
+
+from .config import Config
+
+database = MySQLDatabase(None)
+
+
+class HIDEvent(Model):
+    doorName = CharField()
+    timestamp = DateTimeField()
+    eventType = IntegerField()
+    readerAddress = IntegerField()
+    cardholderID = IntegerField(null=True)
+    commandStatus = BooleanField(null=True)
+    forename = TextField(null=True)
+    surname = TextField(null=True)
+    ioState = BooleanField(null=True)
+    newTime = DateTimeField(null=True)
+    oldTime = DateTimeField(null=True)
+    rawCardNumber = CharField(max_length=8, null=True)
+
+    class Meta:
+        primary_key = CompositeKey("doorName", "timestamp", "eventType")
+        database = database
+
+
+def getStrings(door):
+    """Parses out the message strings from source."""
+    r = requests.get(
+        "https://" + door.ip + "/html/en_EN/en_EN.js",
+        auth=requests.auth.HTTPDigestAuth(door.username, door.password),
+        verify=False,
+    )
+    regex = re.compile(r'([0-9]+)="([^"]*)')
+    strings = [
+        regex.search(s)
+        for s in r.text.split(";")
+        if s.startswith("localeStrings.eventDetails")
+    ]
+    print({int(g.group(1)): g.group(2) for g in strings})
+
+
+@database.atomic()
+def getMessages(door):
+    last_event = (
+        HIDEvent.select(HIDEvent.timestamp)
+        .where(HIDEvent.doorName == door.name)
+        .order_by(HIDEvent.timestamp.desc())
+        .first()
+    )
+    if last_event is not None:
+        last_ts = last_event.timestamp
+    else:
+        last_ts = datetime(2010, 1, 1)
+
+    events = door.get_events(last_ts)
+
+    HIDEvent.insert_many(
+        [
+            {
+                # fill with None, then overwrite with real contents
+                **{k: None for k in HIDEvent._meta.fields.keys()},
+                **event.attrib,
+                "doorName": door.name,
+            }
+            for event in events
+        ]
+    ).on_conflict_ignore().execute()
+    return events
+
+
+def dups(events):
+    timestamps = [e.attrib["timestamp"] for e in events]
+    dups = set([x for x in timestamps if timestamps.count(x) > 1])
+    for e in events:
+        if e.attrib["timestamp"] in dups:
+            etree.dump(e)
+
+
+def main():
+    config = Config()
+    database.init(
+        **config.HID_DB,
+        **{
+            "charset": "utf8",
+            "sql_mode": "PIPES_AS_CONCAT",
+            "use_unicode": True,
+        },
+    )
+    HIDEvent.create_table()
+    for door in config.doors.values():
+        getMessages(door)
+
+
+if __name__ == "__main__":
+    main()

memberPlumbing/mw_models.py

@@ -0,0 +1,249 @@
+from datetime import datetime
+
+from peewee import (
+    BooleanField,
+    CompositeKey,
+    DateField,
+    DateTimeField,
+    DecimalField,
+    CharField,
+    FixedCharField,
+    ForeignKeyField,
+    Model,
+    MySQLDatabase,
+    TextField,
+)
+
+database = MySQLDatabase(None)
+
+
+class BaseModel(Model):
+    _csv_headers_override = {}
+    _date_fields = {}
+
+    def insert_instance(self):
+        return self.insert(**self.__data__)
+
+    def upsert_instance(self):
+        return self.insert_instance().on_conflict(
+            action="update", preserve=list(self._meta.fields.values())
+        )
+
+    def magic_save(self):
+        if self._meta.primary_key is False:
+            self.get_or_create(**self.__data__)
+        else:
+            self.upsert_instance().execute()
+
+    @classmethod
+    def _headers_map(cls):
+        return {field.column_name: name for name, field in cls._meta.fields.items()}
+
+    @classmethod
+    def _remap_headers(cls, data):
+        # print(data)
+        hmap = cls._headers_map()
+        hmap.update(cls._csv_headers_override)
+        for k, v in data.items():
+            if k in hmap:
+                yield hmap.get(k), v
+
+    @classmethod
+    def from_csv_dict(cls, data):
+        data = data.copy()
+
+        # parse date fields to datetime objects
+        for field, fmt in cls._date_fields.items():
+            if data[field]:
+                data[field] = datetime.strptime(str(data[field]), fmt)
+            else:
+                # convert empty string to None to make NULL in SQL
+                data[field] = None
+
+        return cls(**dict(cls._remap_headers(data)))
+
+    class Meta:
+        database = database
+
+
+# TODO: is this still a temporal table?
+class Member(BaseModel):
+    uid = FixedCharField(24, primary_key=True)
+    year_of_birth = TextField(column_name="Year of Birth", null=True)
+    account_name = TextField(column_name="Account Name", null=True)
+    first_name = TextField(column_name="First Name", null=True)
+    last_name = TextField(column_name="Last Name", null=True)
+    phone = TextField(column_name="Phone", null=True)
+    email = TextField(column_name="Email", null=True)
+    volunteer_email = TextField(column_name="Volunteer Email", null=True)
+    address_street = TextField(column_name="Address (Street)", null=True)
+    address_city = TextField(column_name="Address (City)", null=True)
+    address_state_province = TextField(
+        column_name="Address (State/Province)", null=True
+    )
+    address_postal_code = TextField(column_name="Address (Postal Code)", null=True)
+    address_country = TextField(column_name="Address (Country)", null=True)
+    profile_description = TextField(column_name="Profile description", null=True)
+    website = TextField(column_name="Website", null=True)
+    fax = TextField(column_name="Fax", null=True)
+    contact_person = TextField(column_name="Contact Person", null=True)
+    password = TextField(column_name="Password", null=True)
+    position_relation = TextField(column_name="Position/relation", null=True)
+    parent_account_id = TextField(column_name="Parent Account ID", null=True)
+    gift_membership_purchased_by = TextField(
+        column_name="Gift Membership purchased by", null=True
+    )
+    purchased_gift_membership_for = TextField(
+        column_name="Purchased Gift Membership for", null=True
+    )
+    closet_storage = TextField(column_name="Closet Storage #", null=True)
+    storage_shelf = TextField(column_name="Storage Shelf #", null=True)
+    personal_studio_space = TextField(column_name="Personal Studio Space #", null=True)
+    access_permitted_shops_during_extended_hours = BooleanField(
+        column_name="Access Permitted Shops During Extended Hours?"
+    )
+    normal_access_permitted_during_covid19_limited_operations = BooleanField(
+        column_name="Normal Access Permitted During COVID-19 Limited Operations"
+    )
+    access_permitted_during_covid19_staffed_period_only = BooleanField(
+        column_name="Access Permitted During COVID-19 Staffed Period Only"
+    )
+    access_front_door_and_studio_space_during_extended_hours = BooleanField(
+        column_name="Access Front Door and Studio Space During Extended Hours?"
+    )
+    access_wood_shop = BooleanField(column_name="Access Wood Shop?")
+    access_metal_shop = BooleanField(column_name="Access Metal Shop?")
+    access_storage_closet = BooleanField(column_name="Access Storage Closet?")
+    access_studio_space = BooleanField(column_name="Access Studio Space?")
+    access_front_door = BooleanField(column_name="Access Front Door?")
+    access_card_number = TextField(column_name="Access Card Number", null=True)
+    access_card_facility_code = TextField(
+        column_name="Access Card Facility Code", null=True
+    )
+    auto_billing_id = TextField(column_name="Auto Billing ID", null=True)
+    billing_method = TextField(column_name="Billing Method", null=True)
+    renewal_date = DateField(column_name="Renewal Date", null=True)
+    join_date = DateField(column_name="Join Date", null=True)
+    admin_note = TextField(column_name="Admin note", null=True)
+    profile_gallery_image_url = TextField(
+        column_name="Profile gallery image URL", null=True
+    )
+    business_card_image_url = TextField(
+        column_name="Business card image URL", null=True
+    )
+    instagram = TextField(column_name="Instagram", null=True)
+    pinterest = TextField(column_name="Pinterest", null=True)
+    youtube = TextField(column_name="Youtube", null=True)
+    yelp = TextField(column_name="Yelp", null=True)
+    google = TextField(column_name="Google+", null=True)
+    bbb = TextField(column_name="BBB", null=True)
+    twitter = TextField(column_name="Twitter", null=True)
+    facebook = TextField(column_name="Facebook", null=True)
+    linked_in = TextField(column_name="LinkedIn", null=True)
+    do_not_show_street_address_in_profile = TextField(
+        column_name="Do not show street address in profile", null=True
+    )
+    do_not_list_in_directory = TextField(
+        column_name="Do not list in directory", null=True
+    )
+    how_did_you_hear = TextField(column_name="HowDidYouHear", null=True)
+    authorize_charge = TextField(column_name="authorizeCharge", null=True)
+    policy_agreement = TextField(column_name="policyAgreement", null=True)
+    waiver_form_signed_and_on_file_date = DateField(
+        column_name="Waiver form signed and on file date.", null=True
+    )
+    membership_agreement_signed_and_on_file_date = DateField(
+        column_name="Membership Agreement signed and on file date.", null=True
+    )
+    ip_address = TextField(column_name="IP Address", null=True)
+    audit_date = DateField(column_name="Audit Date", null=True)
+    agreement_version = TextField(column_name="Agreement Version", null=True)
+    paperwork_status = TextField(column_name="Paperwork status", null=True)
+    membership_agreement_dated = BooleanField(column_name="Membership agreement dated")
+    membership_agreement_acknowledgement_page_filled_out = BooleanField(
+        column_name="Membership Agreement Acknowledgement Page Filled Out"
+    )
+    membership_agreement_signed = BooleanField(
+        column_name="Membership Agreement Signed"
+    )
+    liability_form_filled_out = BooleanField(column_name="Liability Form Filled Out")
+    self_certify_essential_business = BooleanField(
+        column_name="selfCertifyEssentialBusiness"
+    )
+    accepted_covid19_policy = BooleanField(column_name="Accepted COVID-19 Policy")
+
+    _csv_headers_override = {
+        "Account ID": "uid",
+        "Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:": "how_did_you_hear",
+        "Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.": "authorize_charge",
+        "I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.": "policy_agreement",
+        "Access Permitted Using Membership Level Schedule During COVID-19 Limited Operations": "normal_access_permitted_during_covid19_limited_operations",
+        "I hereby certify that I am involved in Essential Business as defined by the State of New Hampshire and that I will follow the practices identified in the State of New Hampshire Exhibit C to Emergency Order #40 (Section E. Manufacturing).": "self_certify_essential_business",
+        "I have read, and agree to abide by the terms, conditions, policies and procedures  contained in the Claremont MakerSpace COVID-19 Policies and Procedures.": "accepted_covid19_policy",
+    }
+
+    _date_fields = {
+        "Join Date": "%b %d, %Y",
+        "Renewal Date": "%b %d, %Y",
+        "Audit Date": "%m/%d/%Y",
+        "Membership Agreement signed and on file date.": "%m/%d/%Y",
+        "Waiver form signed and on file date.": "%m/%d/%Y",
+    }
+
+    class Meta:
+        table_name = "members"
+
+
+class Flag(BaseModel):
+    id = FixedCharField(24, primary_key=True)
+    name = TextField(null=True)
+    type = CharField(6)
+
+
+class MemberFlag(BaseModel):
+    uid = ForeignKeyField(Member, column_name="uid", backref="flags")
+    flag_id = ForeignKeyField(Flag, backref="members")
+
+    class Meta:
+        primary_key = CompositeKey("flag_id", "uid")
+
+
+class Transaction(BaseModel):
+    sid = FixedCharField(27, null=True)
+    uid = ForeignKeyField(Member, column_name="uid", backref="transactions", null=True)
+    timestamp = DateTimeField()
+    type = TextField(null=True)
+    sum = DecimalField(13, 4, null=True)
+    fee = DecimalField(13, 4, null=True)
+    event_id = TextField(null=True)
+    for_ = TextField(column_name="For", null=True)
+    items = TextField(column_name="Items", null=True)
+    discount_code = TextField(column_name="Discount Code", null=True)
+    note = TextField(column_name="Note", null=True)
+    name = TextField(column_name="Name", null=True)
+    contact_person = TextField(column_name="Contact Person", null=True)
+    full_address = TextField(column_name="Full Address", null=True)
+    street = TextField(column_name="Street", null=True)
+    city = TextField(column_name="City", null=True)
+    state_province = TextField(column_name="State/Province", null=True)
+    postal_code = TextField(column_name="Postal Code", null=True)
+    country = TextField(column_name="Country", null=True)
+    phone = TextField(column_name="Phone", null=True)
+    email = TextField(column_name="Email", null=True)
+
+    _csv_headers_override = {
+        "_dp": "timestamp",
+        "Transaction Type": "type",
+        "Event/Form Name": "for_",
+    }
+
+    @classmethod
+    def from_csv_dict(cls, data):
+        txn = data.copy()
+        # can't use '%s' format string, have to use the special function
+        txn["_dp"] = datetime.fromtimestamp(txn["_dp"])
+        return super().from_csv_dict(txn)
+
+    class Meta:
+        table_name = "transactions"
+        primary_key = False

memberPlumbing/sqlExport.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+
+from .config import Config
+from .mw_models import Member, Flag, MemberFlag, Transaction, database
+
+
+@database.atomic()
+def do_import(config):
+    membershipworks = config.membershipworks
+    print("Creating tables")
+    database.create_tables([Member, Flag, MemberFlag, Transaction])
+
+    print("Updating flags (labels, levels, and addons)")
+    flags = membershipworks._parse_flags()
+    Flag.insert_many(
+        [
+            {"id": v, "name": k, "type": typ[:-1]}
+            for typ, flags_of_type in flags.items()
+            for k, v in flags_of_type.items()
+        ]
+    ).on_conflict(action="update", preserve=[Flag.name, Flag.type]).execute()
+
+    print("Getting folder membership...")
+    folders = {
+        folder_id: membershipworks.get_member_ids([folder_name])
+        for folder_name, folder_id in membershipworks._parse_flags()["folders"].items()
+    }
+
+    print("Getting/Updating members...")
+    members = membershipworks.get_all_members()
+    for m in members:
+        # replace flags by booleans
+        for flag in [dek["lbl"] for dek in membershipworks.org_info["dek"]]:
+            if flag in m:
+                m[flag] = m[flag] == flag
+
+        for field_id, field in membershipworks._all_fields().items():
+            # convert checkboxes to real booleans
+            if field.get("typ") == 8 and field["lbl"] in m:  # check box
+                m[field["lbl"]] = True if m[field["lbl"]] == "Y" else False
+
+    for member in members:
+        # create/update member
+        Member.from_csv_dict(member).magic_save()
+
+        # update member's flags
+        for type, flags in membershipworks._parse_flags().items():
+            for flag, id in flags.items():
+                ml = MemberFlag(uid=member["Account ID"], flag_id=id)
+                if (type == "folders" and member["Account ID"] in folders[id]) or (
+                    type != "folders" and member[flag]
+                ):
+                    ml.magic_save()
+                else:
+                    ml.delete_instance()
+
+    print("Getting/Updating transactions...")
+    # Deduping these is hard, so just recreate the data every time
+    Transaction.truncate_table()
+    now = datetime.now()
+    start_date = datetime(2010, 1, 1)
+    transactions_csv = membershipworks.get_transactions(start_date, now)
+    transactions_json = membershipworks.get_transactions(start_date, now, json=True)
+    # this is terrible, but as long as the dates are the same, should be fiiiine
+    transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
+    assert all(
+        [
+            t["Account ID"] == t.get("uid", "") and t["Payment ID"] == t.get("sid", "")
+            for t in transactions
+        ]
+    )
+
+    Transaction.insert_many(
+        [
+            Transaction.from_csv_dict(transaction).__data__
+            for transaction in transactions
+        ]
+    ).execute()
+
+    # TODO: folders, levels, addons
+
+
+def main():
+    config = Config()
+    database.init(
+        **config.MEMBERSHIPWORKS_DB,
+        **{
+            "charset": "utf8",
+            "sql_mode": "PIPES_AS_CONCAT",
+            "use_unicode": True,
+        },
+    )
+
+    do_import(config)
+
+
+if __name__ == "__main__":
+    main()

memberPlumbing/ucsAccounts.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+import asyncio
+import random
+import re
+import string
+
+from udm_rest_client.udm import UDM
+from udm_rest_client.exceptions import NoObject, UdmError
+
+from .config import Config
+
+USER_BASE = "cn=users,dc=sawtooth,dc=claremontmakerspace,dc=org"
+GROUP_BASE = "cn=groups,dc=sawtooth,dc=claremontmakerspace,dc=org"
+GROUPS_REGEX = "|".join(
+    ["Certified: .*", "Access .*\?", "CMS .*", "Volunteer: .*", "Database .*"]
+)
+RAND_PW_LEN = 20
+
+
+# From an API error message:
+#   A group name must start and end with a letter, number or underscore.
+#   In between additionally spaces, dashes and dots are allowed.
+def sanitize_group_name(name):
+    sanitized_body = re.sub(r"[^0-9A-Za-z_ -.]", ".", name)
+    sanitized_start_end = re.sub("^[^0-9A-Za-z_]|[^0-9A-Za-z_]$", "_", sanitized_body)
+
+    return "MW_" + sanitized_start_end
+
+
+# From an API error message: "Username must only contain numbers, letters and dots!"
+def sanitize_user_name(name):
+    return re.sub(r"[^0-9a-z.]", ".", name.lower()).strip(".")
+
+
+async def make_groups(group_mod, members):
+    existing_group_names = [g.props.name async for g in group_mod.search()]
+
+    groups = [
+        sanitize_group_name(group_name)
+        for group_name in members[0].keys()
+        if re.match(GROUPS_REGEX, group_name) is not None
+    ]
+    for group_name in groups:
+        if group_name not in existing_group_names:
+            group = await group_mod.new()
+            group.props.name = group_name
+            await group.save()
+
+
+async def _main():
+    config = Config()
+
+    members = config.membershipworks.get_members(
+        ["Members", "CMS Staff"], "lvl,phn,eml,lbl,nam,end,_id"
+    )
+
+    async with UDM(**config.UCS) as udm:
+        user_mod = udm.get("users/user")
+        group_mod = udm.get("groups/group")
+
+        await make_groups(group_mod, members)
+
+        for member in members:
+            username = sanitize_user_name(member["Account Name"])
+
+            try:  # try to get an existing user to update
+                user = await user_mod.get(f"uid={username},{USER_BASE}")
+            except NoObject:  # create a new user
+                # TODO: search by employeeNumber and rename users when needed
+                user = await user_mod.new()
+
+                # set a random password and ensure it is changed at next login
+                user.props.password = "".join(
+                    random.choice(string.ascii_letters + string.digits)
+                    for x in range(0, RAND_PW_LEN)
+                )
+                user.props.pwdChangeNextLogin = True
+
+            user.props.update(
+                {
+                    "title": "",  # Title
+                    "firstname": member["First Name"],
+                    "lastname": member["Last Name"],  # (c)
+                    "username": username,  # (cmr)
+                    "description": "",  # Description
+                    # "password": "", # (c) Password
+                    # "mailPrimaryAddress": member["Email"], # Primary e-mail address
+                    # "displayName": "", # Display name
+                    # "birthday": "", # Birthdate
+                    # "jpegPhoto": "", # Picture of the user (JPEG format)
+                    "employeeNumber": member["Account ID"],
+                    # "employeeType": "", # Employee type
+                    "homedrive": "H:",  # Windows home drive
+                    "sambahome": "\\\\ucs\\" + username,  # Windows home path
+                    "profilepath": "%LOGONSERVER%\\%USERNAME%\\windows-profiles\\default",  # Windows profile directory
+                    "disabled": member["Account on Hold"] != "",
+                    # "userexpiry": member["Renewal Date"],
+                    # "pwdChangeNextLogin": "1", # User has to change password on next login
+                    # "sambaLogonHours": "", # Permitted times for Windows logins
+                    "e-mail": [member["Email"]],  # ([]) E-mail address
+                    "phone": [member["Phone"]],  # Telephone number
+                    # "PasswordRecoveryMobile": member["Phone"], # Mobile phone number
+                    "PasswordRecoveryEmail": member["Email"],
+                }
+            )
+
+            new_groups = [
+                "cn=" + sanitize_group_name(group) + "," + GROUP_BASE
+                for group, value in member.items()
+                if re.match(GROUPS_REGEX, group) is not None and value != ""
+            ]
+            # groups not from this script
+            other_old_groups = [
+                g for g in user.props.groups if not g[3:].startswith("MW_")
+            ]
+            user.props.groups = other_old_groups + new_groups
+
+            try:
+                await user.save()
+            except UdmError:
+                print("Failed to save user", username)
+                print(user.props)
+                raise
+
+
+def main():
+    asyncio.run(_main())
+
+
+if __name__ == "__main__":
+    main()

memberPlumbing/upcomingEvents.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+import sys
+
+import pyclip
+
+from .config import Config
+
+TIME_FMT = "%l:%M%P"
+DATETIME_FMT = "%a %b %e %Y, " + TIME_FMT
+
+
+def format_datetime_range(start_ts: int, end_ts: int):
+    start = datetime.fromtimestamp(start_ts)
+    end = datetime.fromtimestamp(end_ts)
+    start_str = start.strftime(DATETIME_FMT)
+    if start.date() == end.date():
+        end_str = end.strftime(TIME_FMT)
+    else:
+        # TODO: this probably implies multiple instances. Should read
+        #       RRULE or similar from the event notes
+        end_str = end.strftime(DATETIME_FMT)
+    return f"{start_str} — {end_str}"
+
+
+def format_event(event_details, truncate: bool):
+    try:
+        url = (
+            "https://claremontmakerspace.org/events/#!event/register/"
+            + event_details["url"]
+        )
+        if "lgo" in event_details:
+            img = f"""<img class="alignleft" width="400" src="{event_details['lgo']['l']}">"""
+        else:
+            img = ""
+        # print(json.dumps(event_details))
+        out = f"""<h2 style="text-align: center;">
+  <a href="{url}">{img}{event_details['ttl']}</a>
+</h2>
+<div><i>{format_datetime_range(event_details['sdp'], event_details['edp'])}</i></div>
+"""
+        if not truncate:
+            out += f"""
+<div>
+  {event_details['dtl']}
+</div>
+
+<a href="{url}">Register for this class now!</a>"""
+        return out
+    except KeyError as e:
+        print(
+            f"Event '{event_details.get('ttl')}' missing required property: '{e.args[0]}'"
+        )
+        raise
+
+
+def format_section(title: str, blurb: str, events, truncate: bool):
+    # skip empty sections
+    if not events:
+        return ""
+
+    events_list = "\n<hr />\n\n".join(format_event(event, truncate) for event in events)
+
+    return f"""<h1>{title}</h1>
+<h4><i>{blurb}</i></h4>
+{events_list}
+"""
+
+
+def generate_post():
+    config = Config()
+    now = datetime.now()
+
+    membershipworks = config.membershipworks
+    events = membershipworks.get_events_list(now)
+    if "error" in events:
+        print("Error:", events["error"])
+        return
+
+    ongoing_events = []
+    full_events = []
+    upcoming_events = []
+    for event in events["evt"]:
+        try:
+            # ignore hidden events
+            if event["cal"] == 0:
+                continue
+            event_details = membershipworks.get_event_by_eid(event["eid"])
+
+            # registration has already ended
+            if (
+                "erd" in event_details
+                and datetime.fromtimestamp(event_details["erd"]) < now
+            ):
+                ongoing_events.append(event_details)
+            # class is full
+            elif event_details["cnt"] >= event_details["cap"]:
+                full_events.append(event_details)
+            else:
+                upcoming_events.append(event_details)
+        except KeyError as e:
+            print(
+                f"Event '{event.get('ttl')}' missing required property: '{e.args[0]}'"
+            )
+            raise
+
+    # header
+    yield """<p><img class="aligncenter size-medium wp-image-2319" src="https://claremontmakerspace.org/wp-content/uploads/2019/03/CMS-Logo-b-y-g-300x168.png" alt="" width="300" height="168" /></a></p>
+<p>Greetings Upper Valley Makers:</p>
+<p>We have an exciting list of upcoming classes at the Claremont MakerSpace that we think might interest you.</p>
+<strong>For most classes and events, CMS MEMBERSHIP IS NOT REQUIRED.</strong> That said, members receive a discount on registration and there are some classes/events that are for members only (this will be clearly noted in the event description).
+
+<strong>Class policies</strong> (liability waiver, withdrawal, cancellation, etc.) can be found <a href="https://claremontmakerspace.org/class-policies/" data-wpel-link="internal">here</a>.
+
+<strong>Instructors:</strong> Interested in teaching a class at CMS? Please fill out our <a href="https://claremontmakerspace.org/cms-class-proposal-form/" data-wpel-link="internal">Class Proposal Form</a>.
+
+<strong>Tours:</strong> Want to see what the Claremont MakerSpace is all about? Tours are by appointment only. <a href="https://tickets.claremontmakerspace.org/open.php" target="_blank" rel="noreferrer noopener external" data-wpel-link="external">Contact Us</a> to schedule your tour where you can learn about all the awesome tools that the CMS offers access to, as well as how membership, classes, and studio spaces work.
+
+<hr />
+"""
+
+    yield format_section(
+        "Upcoming Events",
+        "Events that are currently open for registration.",
+        upcoming_events,
+        truncate=False,
+    )
+
+    yield format_section(
+        "<hr />Just Missed",
+        "These classes are currently full at time of writing. If you are interested, please check the event's page; spots occasionally open up. Keep an eye on this newsletter to see when these classes are offered again.",
+        full_events,
+        truncate=True,
+    )
+
+    yield format_section(
+        "<hr />Ongoing Events",
+        "These events are ongoing. Registration is currently closed, but these events may be offered again in the future.",
+        ongoing_events,
+        truncate=True,
+    )
+
+    # footer
+    yield """<div style="clear: both;">
+  <hr />
+
+  <div>Happy Makin’!</div>
+  <div>We are grateful for all of the public support that our 501(c)(3), non-profit organization receives. If you’d like to make a donation,please visit the <a href="https://claremontmakerspace.org/support/"><strong>Support Us page</strong></a> of our website.</div>
+</div>
+"""
+
+
+def main():
+    result = "\n".join(generate_post())
+    print(result)
+    pyclip.copy(result)
+    print("Copied to clipboard!", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

membershipViewer.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-import re
-import http
-
-from flask import Flask, render_template, request
-
-app = Flask(__name__)
-
-from common import *
-from hid.DoorController import DoorController
-
-def parse_list(member, regex):
-    data_list = []
-    for key, value in member.items():
-        match = re.match(regex, key)
-        if match is not None and value != '':
-            data_list.append(match.group(1))
-
-    return ", ".join(data_list)
-
-def parse_members(members):
-    data = []
-    for member in members:
-        props = {
-            'Name': member['Account Name'],
-            'Renewal Date': member['Renewal Date'],
-            'Card Number': member['Access Card Facility Code'] + '-' \
-                         + member['Access Card Number'],
-            'Account on Hold': "Yes" if member['Account on Hold'] != '' else "No" }
-
-        props['Certifications'] = parse_list(member, 'Certified: (.*)')
-        props['Door Access'] = parse_list(member, 'Access (.*)\?')
-        props['Memebership Level'] = parse_list(member, 'CMS (.*)')
-
-        data.append(props)
-
-    return data
-
-@app.route("/")
-def main():
-    # maybe not now: membership agreement signed
-    # TODO: renewal date check
-    term = request.args.get('term', '')
-
-    if len(term) < 3:
-        return render_template("members.html",
-                               error="Enter at least 3 characters to search")
-
-    data = getMembershipworksData(
-        ['members', 'staff'],
-        "lvl,xws,xms,xsc,xas,xfd,xac,phn,eml,lbl,xcf,nam,end")
-
-    members = parse_members(data)
-    members = [member for member in members
-               if term.lower() in member['Name'].lower()]
-    headers = ['Name', 'Certifications', 'Door Access', 'Memebership Level',
-               'Card Number', 'Renewal Date', 'Account on Hold']
-
-    if len(members) > 4:
-        return render_template(
-            "members.html", error="Too many results, please be more specific.")
-
-    return render_template("members.html", headers=headers, members=members)
-
-@app.route('/frontDoor/<lock>', methods=['POST'])
-def unlockLockDoor(lock):
-    doors['Front Door'].lockOrUnlockDoor(lock != 'unlock')
-    return ('', http.HTTPStatus.NO_CONTENT)
-
-if __name__ == "__main__":
-    app.run(debug=True, host='0.0.0.0')

pdm.toml

@@ -0,0 +1,2 @@
+[strategy]
+save = "compatible"

pyproject.toml

@@ -0,0 +1,52 @@
+[project]
+name = "memberPlumbing"
+version = "0.1.0"
+description = ""
+authors = [
+    {name = "Adam Goldsmith", email = "adam@adamgoldsmith.name"},
+]
+requires-python = ">=3.11,<4.0"
+dependencies = [
+    "requests~=2.31",
+    "ruamel-yaml~=0.17",
+    "bitstring~=4.1",
+    "lxml~=4.9",
+    "peewee~=3.16",
+    "mysqlclient~=2.1",
+    "udm-rest-client~=1.2",
+    "pyclip~=0.7",
+    "recurrent~=0.4",
+]
+
+[tool.pdm]
+[tool.pdm.dev-dependencies]
+dev = [
+    "black~=23.3",
+    "isort~=5.11",
+]
+
+[tool.pdm.build]
+includes = [
+    "memberPlumbing",
+]
+
+[build-system]
+requires = ["pdm-backend"]
+build-backend = "pdm.backend"
+
+[project.scripts]
+doorUpdater = "memberPlumbing.doorUpdater:main"
+hidEvents = "memberPlumbing.hidEvents:main"
+sqlExport = "memberPlumbing.sqlExport:main"
+ucsAccounts = "memberPlumbing.ucsAccounts:main"
+upcomingEvents = "memberPlumbing.upcomingEvents:main"
+
+[tool.black]
+line-length = 88
+
+[tool.isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+line_length = 88

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

schedules.xml

@@ -1,65 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<VertXMessage  xmlns:hid="http://www.hidglobal.com/VertX">
-  <hid:Schedules action="AD">
-    <hid:Schedule scheduleID="1" scheduleName="7x24">
-      <hid:DayOfWeekInterval dayOfWeek="0" startTime="00:00:00" endTime="23:59:59"/>
-      <hid:DayOfWeekInterval dayOfWeek="1" startTime="00:00:00" endTime="23:59:59"/>
-      <hid:DayOfWeekInterval dayOfWeek="2" startTime="00:00:00" endTime="23:59:59"/>
-      <hid:DayOfWeekInterval dayOfWeek="3" startTime="00:00:00" endTime="23:59:59"/>
-      <hid:DayOfWeekInterval dayOfWeek="4" startTime="00:00:00" endTime="23:59:59"/>
-      <hid:DayOfWeekInterval dayOfWeek="5" startTime="00:00:00" endTime="23:59:59"/>
-      <hid:DayOfWeekInterval dayOfWeek="6" startTime="00:00:00" endTime="23:59:59"/>
-    </hid:Schedule>
-
-    <hid:Schedule scheduleID="2" scheduleName="Weekends Only">
-      <hid:DayOfWeekInterval dayOfWeek="0" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="6" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:SpecialDayInterval dayOfMonth="1" month="1" specialDayID="3" specialDayName="New Years Day" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:SpecialDayInterval dayOfMonth="25" month="12" specialDayID="2" specialDayName="Christmas Day" startTime="08:00:00" endTime="21:00:00"/>
-    </hid:Schedule>
-
-    <hid:Schedule scheduleID="3" scheduleName="Weekdays Only">
-      <hid:DayOfWeekInterval dayOfWeek="1" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="2" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="3" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="4" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="5" startTime="08:00:00" endTime="21:00:00"/>
-    </hid:Schedule>
-
-    <hid:Schedule scheduleID="4" scheduleName="Unlimited">
-      <hid:DayOfWeekInterval dayOfWeek="0" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="1" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="2" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="3" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="4" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="5" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="6" startTime="08:00:00" endTime="21:00:00"/>
-    </hid:Schedule>
-
-    <hid:Schedule scheduleID="5" scheduleName="Nights and Weekends">
-      <hid:DayOfWeekInterval dayOfWeek="0" startTime="08:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="1" startTime="15:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="2" startTime="15:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="3" startTime="15:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="4" startTime="15:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="5" startTime="15:00:00" endTime="21:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="6" startTime="08:00:00" endTime="21:00:00"/>
-    </hid:Schedule>
-
-    <hid:Schedule scheduleID="6" scheduleName="Front Door Unlocker">
-      <hid:DayOfWeekInterval dayOfWeek="0" startTime="08:00:00" endTime="17:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="1" startTime="08:00:00" endTime="17:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="2" startTime="08:00:00" endTime="17:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="3" startTime="08:00:00" endTime="17:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="4" startTime="08:00:00" endTime="17:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="5" startTime="08:00:00" endTime="17:00:00"/>
-      <hid:DayOfWeekInterval dayOfWeek="6" startTime="08:00:00" endTime="17:00:00"/>
-    </hid:Schedule>
-  </hid:Schedules>
-
-  <hid:SpecialDays action="UD">
-    <hid:SpecialDay specialDayID="1" specialDayName="New Years Day" dayOfMonth="1" month="1"/>
-    <hid:SpecialDay specialDayID="2" specialDayName="Independance Day" dayOfMonth="4" month="7"/>
-    <hid:SpecialDay specialDayID="3" specialDayName="Christmas Day" dayOfMonth="25" month="12"/>
-  </hid:SpecialDays>
-</VertXMessage>

systemd/doorUpdater.service

@@ -5,5 +5,6 @@ OnFailure=status-email-admin@%n.service
 [Service]
 User=adam
 Type=oneshot
-WorkingDirectory=/home/adam/hidDoorWriter/
+TimeoutStartSec=600
-ExecStart=/home/adam/hidDoorWriter/doorUpdater.py
+WorkingDirectory=/home/adam/memberPlumbing/
+ExecStart=/usr/bin/python3 -u /home/adam/.poetry/bin/poetry run doorUpdater

systemd/hidEvents.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Pull events from door controllers into local database
+OnFailure=status-email-admin@%n.service
+After=mariadb.service
+Requires=mariadb.service
+
+[Service]
+User=adam
+Type=oneshot
+TimeoutStartSec=600
+WorkingDirectory=/home/adam/memberPlumbing/
+ExecStart=/usr/bin/python3 -u /home/adam/.poetry/bin/poetry run hidEvents

systemd/hidEvents.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Hourly door controller events sync
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

systemd/membershipworksSQL.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Sync Membershipworks with local database
+OnFailure=status-email-admin@%n.service
+After=mariadb.service
+Requires=mariadb.service
+
+[Service]
+User=adam
+Type=oneshot
+TimeoutStartSec=600
+WorkingDirectory=/home/adam/memberPlumbing/
+ExecStart=/usr/bin/python3 -u /home/adam/.poetry/bin/poetry run sqlExport

systemd/membershipworksSQL.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Hourly Membershipworks database sync
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

systemd/ucsAccounts.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Update UCS Accounts
+OnFailure=status-email-admin@%n.service
+
+[Service]
+User=adam
+Type=oneshot
+TimeoutStartSec=600
+WorkingDirectory=/home/adam/memberPlumbing/
+ExecStart=/usr/bin/python3 -u /home/adam/.poetry/bin/poetry run ucsAccounts

systemd/ucsAccounts.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Hourly UCS Accounts update
+
+[Timer]
+OnCalendar=*:0/15
+Persistent=true
+
+[Install]
+WantedBy=timers.target

templates/members.html

@@ -1,54 +0,0 @@
-<html>
-  <head>
-    <title> CMS Members </title>
-    <style>
-     table {
-       border-collapse: collapse;
-     }
-
-     td, th{
-       border: 1px solid black;
-     }
-
-     td {
-       padding: 3px;
-     }
-
-     tr:nth-child(even) {
-       background-color: #ccc;
-     }
-
-     tr[onHold=Yes] {
-       background-color: #f66;
-     }
-    </style>
-  </head>
-  <body>
-    <form action="./frontDoor/lock" method="post">
-      <button>Lock Front Door</button>
-      <button formaction="./frontDoor/unlock">Unlock Front Door</button>
-    </form>
-    <form>
-      <input type="text" name="term" />
-      <button>Search</button>
-    </form>
-    {% if headers is not none %}
-    <table>
-      <tr>
-        {% for header in headers %}
-        <th> {{ header }} </th>
-        {% endfor %}
-      </tr>
-      {% for member in members %}
-      <tr onHold ="{{ member['Account on Hold'] }}">
-        {% for header in headers %}
-        <td> {{ member[header] }} </td>
-        {% endfor %}
-      </tr>
-      {% endfor %}
-    </div>
-    </table>
-    {% endif %}
-    <span class="error">{{ error }}</span>
-  </body>
-</html>

ucsAccounts.py

@@ -1,89 +0,0 @@
-#!/usr/bin/env python3
-import random
-import re
-import string
-import subprocess
-
-from common import getMembershipworksData
-
-LDAP_BASE = "cn=users,dc=sawtooth,dc=claremontmakerspace,dc=org"
-GROUP_BASE = "cn=groups,dc=sawtooth,dc=claremontmakerspace,dc=org"
-GROUPS_REGEX = "|".join(['Certified: .*', 'Access .*\?', 'CMS .*', 'Volunteer: .*'])
-RAND_PW_LEN = 20
-
-def makeGroups(members):
-    groups = [key.replace(':', '.').replace('?', '')
-              for key in members[0].keys()
-              if re.match(GROUPS_REGEX, key) is not None]
-    for group in groups:
-        subprocess.call(["udm", "groups/group", "create",
-                         "--position", GROUP_BASE,
-                         "--set", "name=" + group])
-
-def makeSets(props):
-    return sum([["--set", key + "=" + value]
-                for key, value in props.items()], [])
-
-def makeAppendGroups(member):
-    groups = [key.replace(':', '.').replace('?', '')
-              for key, value in member.items()
-              if re.match(GROUPS_REGEX, key) is not None and value != '']
-    return sum([["--append", "groups=cn=" + group + ',' + GROUP_BASE]
-                for group in groups], [])
-
-def main():
-    members = getMembershipworksData(['members', 'staff'],
-                                     "lvl,phn,eml,lbl,nam,end,_id")
-    makeGroups(members)
-
-    for member in members:
-        randomPass = ''.join(random.choice(string.ascii_letters + string.digits)
-                             for x in range(0, RAND_PW_LEN))
-        username = member["Account Name"].lower().replace(" ", ".")
-
-        props = {
-            "title": "", # Title
-            "firstname": member["First Name"],
-            "lastname": member["Last Name"], # (c)
-            "username": username, # (cmr)
-            "description": "", # Description
-            "password": randomPass, # (c) Password
-            #"mailPrimaryAddress": member["Email"], # Primary e-mail address
-            #"displayName": "", # Display name
-            #"birthday": "", # Birthdate
-            #"jpegPhoto": "", # Picture of the user (JPEG format)
-
-            "employeeNumber": member["Account ID"],
-            #"employeeType": "", # Employee type
-
-            "homedrive": "H:", # Windows home drive
-            "sambahome": "\\\\ucs\\" + username, # Windows home path
-            "profilepath": "%LOGONSERVER%\\%USERNAME%\\windows-profiles\\default", # Windows profile directory
-
-            "disabled": "1" if member["Account on Hold"] != "" else "0",
-            #"userexpiry": member["Renewal Date"],
-
-            "pwdChangeNextLogin": "1", # User has to change password on next login
-
-            #"sambaLogonHours": "", # Permitted times for Windows logins
-
-            "e-mail": member["Email"],  # ([]) E-mail address
-            "phone": member["Phone"], # Telephone number
-            #"PasswordRecoveryMobile": member["Phone"], # Mobile phone number
-            "PasswordRecoveryEmail": member["Email"]
-        }
-
-        subprocess.call(["udm", "users/user", "create",
-                         "--position", LDAP_BASE] + makeSets(props))
-
-        # remove props we don't want to reset
-        props.pop("password")
-        props.pop("pwdChangeNextLogin")
-
-        subprocess.call(["udm", "users/user", "modify",
-                         "--dn", "uid=" + username + "," + LDAP_BASE]
-                        + makeSets(props)
-                        + makeAppendGroups(member))
-
-if __name__ == "__main__":
-    main()