sqlExport: Use FROM DUAL in insertDistinct to fix older mariadb

This reverts commit 6c862b9fb41f863cfc5d0ef4ffc863c4fe43d9fd.

Turns out that doesn't work, and mariadb was just silently failing.
Reopens #2.

.gitignore

@@ -1,3 +1,2 @@
 __pycache__/
-/venv/
-/config.yaml
+/passwords.py

README.md

@@ -1,39 +1,33 @@
-# Claremont MakerSpace Member Plumbing
+# Claremont Makerspace Member Plumbing
 
-This repo contains a set of scripts to sync data around for the Claremont MakerSpace. They primarily revolve around pulling member data from [MembershipWorks](https://membershipworks.com/) and pushing it out to various systems at the Space.
+This repo contains a set of scripts to sync data around for the Claremont MakerSpace. They primarilly revolve around pulling member data from [MembershipWorks](https://membershipworks.com/) and pushing it out to various systems at the Space.
 
-## Setup
+## `passwords.py`
 
-This project uses [Poetry](https://python-poetry.org/) for dependency management. Typical usage is first running `poetry install` to create a virtualenv and install dependencies, then running `poetry run <script>` to start a specific script.
+Most of the scripts require a `passwords.py` file, defining the usernames in passwords for various services. This file is not distributed in this repository, for obvious reasons, but here is an example:
 
-## Config
+```python
+DOOR_USERNAME = "user1"
+DOOR_PASSWORD = "password1"
 
-Many of the scripts use data from a `config.yaml` in the current working directory when they are run. There is an example config in [`config.example.yaml`](./config.example.yaml) which has been stripped of authentication information.
+MEMBERSHIPWORKS_USERNAME = "user2"
+MEMBERSHIPWORKS_PASSWORD = "password2"
 
-## Scripts
+MEMBERSHIPWORKS_DB = {
+    "database": "db_name_here",
+    "user": "user3",
+    "password": "password3"
+}
+```
 
-The primary entry points have scripts entries (`tool.poetry.scripts`) in [`pyproject.toml`](./pyproject.toml). They assume that they are being run from a module, so must be run with `poetry run <script>` or `python -m memberPlumbing.<script>`.
+## `doorUpdater.py`
 
-### `doorUpdater`
+Retrieves member information from MembershipWorks and pushes it out to the HID Edge Evo SOLO controllers that do access controll at the Space. Configuration lives in `config.yaml`.
 
-Retrieves member information from MembershipWorks and pushes it out to the HID Edge Evo SOLO controllers that do access control at the Space. Configuration lives in `config.yaml`.
-
-### `ucsAccounts`
+## `ucsAccounts.py`
 
 Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space.
 
-### `sqlExport`
+## `sqlExport.py`
 
-Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined with [peewee](peewee-orm.com) in [`memberPlumbing/mw_models.py`](./memberPlumbing/mw_models.py).
-
-### `upcomingEvents`
-
-Retrieves upcoming events from MembershipWorks and formats them for a WordPress post.
-
-### `hidEvents`
-
-Retrieves events from the HID Evo Solo door controllers, and pushes them to a SQL database.
-
-## Systemd
-
-There are systemd units in the [`systemd`](./systemd/) folder, which can be used to run the various scripts regularly.
+Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined (somewhat oddly, to be fair) in `tableMapping.yaml`.

common.py

@@ -0,0 +1,26 @@
+from ruamel.yaml import YAML
+import os
+
+from lib.hid.DoorController import DoorController
+from lib.MembershipWorks import MembershipWorks
+
+from passwords import DOOR_USERNAME, DOOR_PASSWORD
+from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD
+
+try:
+    with open(os.path.dirname(os.path.abspath(__file__)) + "/config.yaml") as f:
+        config = YAML().load(f)
+except NameError:
+    with open("config.yaml") as f:
+        config = YAML().load(f)
+
+doors = {doorName: DoorController(doorData['ip'],
+                                  DOOR_USERNAME, DOOR_PASSWORD,
+                                  name=doorName, access=doorData['access'])
+         for doorName, doorData in config["doors"].items()}
+
+memberLevels = config['memberLevels']
+doorSpecificSchedules = config['doorSpecificSchedules']
+
+membershipworks = MembershipWorks()
+membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD)

config.yaml

@@ -1,4 +1,4 @@
-doorControllers:
+doors:
   Studio Space: {ip: 172.18.51.11, access: Studio Space}
   Front Door: {ip: 172.18.51.12, access: Front Door}
   Metal Shop: {ip: 172.18.51.13, access: Metal Shop}
@@ -26,26 +26,3 @@ doorSpecificSchedules:
     - Metal Shop
     - Wood Shop
     - Wood Shop Rear
-
-
-DOOR_USERNAME: ""
-DOOR_PASSWORD: ""
-
-MEMBERSHIPWORKS_USERNAME: ""
-MEMBERSHIPWORKS_PASSWORD: ""
-
-# arguments for https://udm-rest-client.readthedocs.io/en/latest/udm_rest_client.html#udm_rest_client.udm.UDM
-UCS:
-  url: ""
-  username: ""
-  password: ""
-
-MEMBERSHIPWORKS_DB:
-  database: ""
-  user: ""
-  password: ""
-
-HID_DB:
-  database: ""
-  user: ""
-  password: ""

doorUpdater.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python3
+
+import copy
+
+from common import doors, membershipworks, memberLevels, doorSpecificSchedules
+from lib.hid.Credential import Credential
+from lib.hid.DoorController import ROOT, E
+
+class Member():
+    def __init__(self, forename="", surname="", membershipWorksID="",
+                 middleName="", email="", phone="",
+                 cardholderID=None, doorAccess=[],
+                 credentials=set(), levels=[], extraLevels=[], schedules=[]):
+        self.forename = forename
+        self.surname = surname
+        self.membershipWorksID = membershipWorksID
+        self.middleName = middleName
+        self.email = email
+        self.phone = phone
+        self.cardholderID = cardholderID
+        self.doorAccess = doorAccess
+        self.credentials = credentials
+        self.levels = levels
+        self.schedules = schedules
+
+    def __str__(self):
+        return f"""Name: {self.forename} | {self.middleName} | {self.surname}
+MembershipWorks ID: {self.membershipWorksID}
+Email: {self.email}
+Phone: {self.phone}
+Cardholder ID: {self.cardholderID}
+doorAccess: {self.doorAccess}
+Credentials: {self.credentials}
+Levels: {self.levels}
+Schedules: {self.schedules}
+"""
+
+
+class MembershipworksMember(Member):
+    def __init__(self, data, formerMember=False):
+        super().__init__(data["First Name"],
+                         data["Last Name"],
+                         membershipWorksID=data["Account ID"],
+                         email=data["Email"],
+                         phone=data["Phone"])
+
+        if data["Access Card Number"] != "":
+            self.credentials = set([Credential(
+                code=(data["Access Card Facility Code"],
+                      data["Access Card Number"]))])
+        else:
+            self.credentials = set()
+
+        self.onHold=data["Account on Hold"] != ""
+        self.limitedOperations=data['Access Permitted During Limited Operations'] == "Y"
+        self.formerMember=formerMember
+
+        levels = {k: v for k, v in memberLevels.items() if data[k] == k}
+        self.levels = list(levels.keys())
+        self.schedules = list(levels.values())
+
+        self.extraLevels = {
+            schedule: sum((doors for prop, doors in props.items()
+                           if data[prop] == "Y"),
+                          [])
+            for schedule, props in doorSpecificSchedules.items()
+        }
+
+        self.doorAccess = [
+            door for door, doorData in doors.items()
+            if data["Access " + doorData.access + "?"] == "Y"
+        ]
+
+    def to_DoorMember(self, door):
+        doorLevels = [k for k, v in self.extraLevels.items() if door.name in v]
+
+        if (door.name not in self.doorAccess
+            or self.onHold
+            or self.formerMember
+            or not self.limitedOperations):
+            schedules = []
+        else:
+            schedules = self.schedules + doorLevels
+
+        dm = DoorMember(door,
+                        forename=self.forename,
+                        surname=self.surname,
+                        membershipWorksID=self.membershipWorksID,
+                        email=self.email,
+                        phone=self.phone,
+                        levels=self.levels + doorLevels,
+                        doorAccess=self.doorAccess,
+                        credentials=self.credentials,
+                        schedules=schedules)
+
+        return dm
+
+    def __str__(self):
+        return super().__str__() + f"""OnHold? {self.onHold}
+Limited Operations Access? {self.limitedOperations}
+Former Member? {self.formerMember}
+"""
+
+
+class DoorMember(Member):
+    def __init__(self, door, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.door = door
+
+    @classmethod
+    def from_cardholder(cls, data, door):
+        ch = cls(door=door,
+                 forename=data.get('forename', ""),
+                 surname=data.get('surname', ""),
+                 membershipWorksID=data.attrib.get('custom2', ""),
+                 middleName=data.attrib.get('middleName', ""),
+                 email=data.attrib.get('email', ""),
+                 phone=data.attrib.get('phone', ""),
+                 cardholderID=data.attrib['cardholderID'])
+
+        ch.credentials = set(
+            Credential(hex=(c.attrib['rawCardNumber']))
+            for c in data.findall('{*}Credential'))
+
+        ch.levels = data.attrib.get('custom1', "").split('|')
+        ch.schedules = [r.attrib['scheduleName']
+                        for r in data.findall('{*}Role')]
+
+        return ch
+
+    def attribs(self):
+        return {"forename": self.forename,
+                "surname": self.surname,
+                "middleName": self.middleName,
+                "email": self.email,
+                "phone": self.phone,
+                "custom1":  "|".join(self.levels).replace("&", "and"),
+                "custom2": self.membershipWorksID}
+
+    def make_schedules(self, schedulesMap):
+        roles = [
+            E.Role({"roleID": self.cardholderID,
+                    "scheduleID": schedulesMap[schedule],
+                    "resourceID": "0"})
+            for schedule in self.schedules]
+
+        return E.RoleSet({"action": "UD", "roleSetID": self.cardholderID},
+                         E.Roles(*roles))
+
+    def make_credentials(self, newCredentials, cardFormats):
+        out = [
+            E.Credential(
+                {"formatName": str(credential.code[0]),
+                 "cardNumber": str(credential.code[1]),
+                 "formatID": cardFormats[str(credential.code[0])],
+                 "isCard": "true",
+                 "cardholderID": self.cardholderID})
+            for credential in newCredentials]
+
+        return E.Credentials({"action": "AD"}, *out)
+
+
+def update_door(door, members):
+    cardFormats = door.get_cardFormats()
+    cardholders = {member.membershipWorksID: member
+                   for member in [DoorMember.from_cardholder(ch, door)
+                                  for ch in door.get_cardholders()]}
+    schedulesMap = door.get_scheduleMap()
+    allCredentials = set(Credential(hex=c.attrib['rawCardNumber'])
+                         for c in door.get_credentials())
+
+    # TODO: can I combine requests?
+    for membershipworksMember in members:
+        member = membershipworksMember.to_DoorMember(door)
+        # cardholder did not exist, so add them
+        if member.membershipWorksID not in cardholders:
+            print("- Adding Member {member.forename} {member.surname}:")
+            print(f"  - {member.attribs()}")
+            resp = door.doXMLRequest(ROOT(
+                E.Cardholders(
+                    {"action": "AD"},
+                    E.Cardholder(member.attribs()))))
+            member.cardholderID = resp.find('{*}Cardholders/{*}Cardholder') \
+                                      .attrib["cardholderID"]
+
+            # create a dummy ch to force an update
+            # TODO: probably a cleaner way to do this
+            ch = copy.copy(member)
+            ch.schedules = []
+            ch.credentials = set()
+        # cardholder exists, compare contents
+        else:
+            ch = cardholders.pop(member.membershipWorksID)
+            member.cardholderID = ch.cardholderID
+
+        if member.attribs() != ch.attribs():  # update cardholder attributes
+            print(f"- Updating profile for {member.forename} {member.surname}")
+            print(f"  - Old: {ch.attribs()}")
+            print(f"  - New: {member.attribs()}")
+            door.doXMLRequest(ROOT(
+                E.Cardholders(
+                    {"action": "UD", "cardholderID": member.cardholderID},
+                    E.CardHolder(member.attribs()))))
+
+        if member.credentials != ch.credentials:
+            print(f"- Updating card for {member.forename} {member.surname}")
+            print(f"  - {ch.credentials} -> {member.credentials}")
+
+            oldCards = ch.credentials
+            newCards = member.credentials
+
+            allNewCards = set(
+            card for m in members if m != membershipworksMember
+                for card in m.credentials)
+
+            # cards removed, and won't be reassigned to someone else
+            for card in (oldCards - newCards) - allNewCards:
+                door.doXMLRequest(ROOT(E.Credentials(
+                    {"action": "UD",
+                     "rawCardNumber": card.hex,
+                     "isCard": "true"},
+                    E.Credential({"cardholderID": ""}))))
+
+            if newCards - oldCards:  # cards added
+                for card in newCards & allNewCards:  # new card exists in another member
+                    print([m for m in members
+                          for card in m.credentials
+                           if card in newCards])
+                    raise Exception(f"Duplicate Card in input data! {card}")
+
+                # card existed in door, and needs to be reassigned
+                for card in newCards & allCredentials:
+                    door.doXMLRequest(ROOT(E.Credentials(
+                        {"action": "UD",
+                         "rawCardNumber": card.hex,
+                         "isCard": "true"},
+                        E.Credential({"cardholderID": member.cardholderID}))))
+
+                # cards that never existed, and need to be created
+                if newCards - allCredentials:
+                    door.doXMLRequest(ROOT(member.make_credentials(
+                        newCards - allCredentials, cardFormats)))
+
+        if member.schedules != ch.schedules:
+            print("- Updating schedule for" +
+                  f" {member.forename} {member.surname}:" +
+                  f" {ch.schedules} -> {member.schedules}")
+            door.doXMLRequest(ROOT(member.make_schedules(schedulesMap)))
+
+    # TODO: delete cardholders that are no longer members?
+
+
+def main():
+    membershipworks_attributes = \
+        "_id,nam,phn,eml,lvl,lbl,xws,xms,xsc,xas,xfd,xac,xcf,xeh,xse,xlo"
+
+    memberData = membershipworks.get_members(
+        ['Members', 'CMS Staff', 'Misc. Access'], membershipworks_attributes)
+    members = [MembershipworksMember(m) for m in memberData]
+
+    formerMemberData = membershipworks.get_members(
+        ['Former Members'], membershipworks_attributes)
+    formerMembers = [MembershipworksMember(m, formerMember=True)
+                     for m in formerMemberData]
+
+    for formerMember in formerMembers:
+        member = next(
+            (m for m in members
+             if m.membershipWorksID == formerMember.membershipWorksID),
+            None)
+
+        # member exists in another folder
+        if member is not None:
+            member.formerMember = True
+        else: # member is only a former member
+            formerMember.formerMember = True
+            members.append(formerMember)
+
+    for door in doors.values():
+        print(door.name, door.ip)
+        update_door(door, members)
+
+
+if __name__ == '__main__':
+    main()

events.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+from collections import defaultdict
+from lxml import etree
+import os
+import re
+import requests
+
+from common import *
+
+def getStrings(door):
+    """Parses out the message strings from source."""
+    r = requests.get('https://' + door.ip + '/html/en_EN/en_EN.js',
+                     auth=requests.auth.HTTPDigestAuth(door.username, door.password),
+                     verify=False)
+    regex = re.compile(r'([0-9]+)="([^"]*)')
+    strings = [regex.search(s) for s in r.text.split(';')
+               if s.startswith('localeStrings.eventDetails')]
+    print({int(g.group(1)): g.group(2) for g in strings})
+
+def getMessages(door):
+    # get parameters for messages to get?
+    # honestly not really sure why this is required, their API is confusing
+    parXMLIn = E_plain.VertXMessage(
+        E.EventMessages({"action": "LR"}))
+    parXMLOut = door.doXMLRequest(parXMLIn)
+    etree.dump(parXMLOut)
+
+    if os.path.exists("logs/" + door.name + ".xml"):
+        # read last log
+        tree = etree.ElementTree(file="logs/" + door.name + ".xml")
+        root = tree.getroot()
+        recordCount = int(parXMLOut[0].attrib["historyRecordMarker"]) - \
+                      int(root[0][0].attrib["recordMarker"])
+    else:
+        # first run for this door
+        root = None
+        recordCount = 1000
+
+    if recordCount == 0:
+        print("No records to get!")
+        return
+    print("Getting", recordCount, "records")
+    # get the actual messages
+    eventsXMLIn = E_plain.VertXMessage(
+        E.EventMessages({"action": "LR",
+                         "recordCount": str(recordCount),
+                         "historyRecordMarker": parXMLOut[0].attrib["historyRecordMarker"],
+                         "historyTimestamp": parXMLOut[0].attrib["historyTimestamp"]}))
+    eventsXMLOut = door.doXMLRequest(eventsXMLIn)
+    #TODO: handle modeRecords=true
+
+    for index, event in enumerate(eventsXMLOut[0]):
+        event.attrib["recordMarker"] = str(int(parXMLOut[0].attrib["historyRecordMarker"]) - index)
+
+    if root is None:
+        tree = etree.ElementTree(eventsXMLOut)
+    else:
+        for event in reversed(eventsXMLOut[0]):
+            root[0].insert(0, event)
+    tree.write("logs/" + doorName + ".xml")
+
+def main():
+    for door in doors.values():
+        getMessages(door)
+
+if __name__ == '__main__':
+    main()

lib/MembershipWorks.py

@@ -0,0 +1,210 @@
+import csv
+from io import StringIO
+import requests
+
+BASE_URL = "https://api.membershipworks.com/v1/"
+
+# extracted from `SF._is.crm` in https://cdn.membershipworks.com/all.js
+CRM = {0: "Note",
+       4: "Profile Updated",
+       8: "Scheduled/Reminder Email",
+       9: "Renewal Notice",
+       10: "Join Date",
+       11: "Next Renewal Date",
+       12: "Membership Payment",
+       13: "Donation",
+       14: "Event Activity",
+       15: "Conversation",
+       16: "Contact Change",
+       17: "Label Change",
+       18: "Other Payment",
+       19: "Cart Payment",
+       20: "Payment Failed",
+       21: "Billing Updated",
+       22: "Form Checkout",
+       23: "Event Payment",
+       24: "Invoice",
+       25: "Invoice Payment",
+       26: "Renewal",
+       27: "Payment"}
+
+# Types of fields, extracted from a html snippet in all.js + some guessing
+typ = {
+    1: "Text input",
+    2: "Password", # inferred from data
+    3: "Simple text area",
+    4: "Rich text area",
+    7: "Address",
+    8: "Check box",
+    9: "Select",
+    11: "Display value stored in field (ie. read only)",
+    12: "Required waiver/terms"}
+
+# more constants, this time extracted from the members csv export in all.js
+staticFlags = {
+    "pos": {"lbl": "Position/relation (contacts)"},
+    "nte": {"lbl": "Admin note (contacts)"},
+    "pwd": {"lbl": "Password"},
+    "lgo": {"lbl": "Business card image URLs"},
+    "pfx": {"lbl": "Profile gallery image URLs"},
+    "lvl": {"lbl": "Membership levels"},
+    "aon": {"lbl": "Membership add-ons"},
+    "lbl": {"lbl": "Labels"},
+    "joi": {"lbl": "Join date"},
+    "end": {"lbl": "Renewal date"},
+    "spy": {"lbl": "Billing method"},
+    "rid": {"lbl": "Auto recurring billing ID"},
+    "ipa": {"lbl": "IP address"},
+    "_id": {"lbl": "Account ID"}
+}
+
+
+class MembershipWorksRemoteError(Exception):
+    def __init__(self, reason, r):
+        super().__init__(
+            f"Error when attempting {reason}: {r.status_code} {r.reason}\n{r.text}")
+
+class MembershipWorks:
+    def __init__(self):
+        self.org_info = None
+        self.auth_token = None
+        self.org_num = None
+
+    def login(self, username, password):
+        """Authenticate against the membershipworks api"""
+        r = requests.post(BASE_URL + 'usr',
+                          data={"_st": "all",
+                                "eml": username,
+                                "org": "10000",
+                                "pwd": password})
+        if r.status_code != 200 or 'SF' not in r.json():
+            raise MembershipWorksRemoteError('login', r)
+        self.org_info = r.json()
+        self.auth_token = self.org_info['SF']
+        self.org_num = self.org_info['org']
+
+    def _inject_auth(self, kwargs):
+        # TODO: should probably be a decorator or something
+        if self.auth_token is None:
+            raise RuntimeError('Not Logged in to MembershipWorks')
+        # add auth token to params
+        if 'params' not in kwargs:
+            kwargs['params'] = {}
+        kwargs['params']["SF"] = self.auth_token
+
+    def _get(self, *args, **kwargs):
+        self._inject_auth(kwargs)
+        # TODO: should probably do some error handling in here
+        return requests.get(*args, **kwargs)
+
+    def _post(self, *args, **kwargs):
+        self._inject_auth(kwargs)
+        # TODO: should probably do some error handling in here
+        return requests.post(*args, **kwargs)
+
+    def _all_fields(self):
+        """Parse out a list of fields from the org data.
+
+        Is this terrible? Yes. Also, not dissimilar to how MW does it
+        in all.js.
+        """
+        fields = staticFlags.copy()
+
+        # TODO: this will take the later option, if the same field
+        # used in mulitple places. I don't know which lbl is used for
+        # csv export
+
+        # anm: member signup, acc: member manage, adm: admin manage
+        for screen_type in ['anm', 'acc', 'adm']:
+            for box in self.org_info['tpl'][screen_type]:
+                for element in box['box']:
+                    if (type(element['dat']) != str):
+                        for field in element['dat']:
+                            if '_id' in field:
+                                if field['_id'] not in fields:
+                                    fields[field['_id']] = field
+
+        return fields
+
+    def _parse_flags(self):
+        """Parse the flags out of the org data.
+
+        This is terrible, and there might be a better way to do this.
+        """
+        ret = {"folders": {},
+               "levels": {},
+               "addons": {},
+               "labels": {}}
+
+        for dek in self.org_info['dek']:
+            # TODO: there must be a better way. this is stupid
+            if dek['dek'] == 1:
+                ret["folders"][dek['lbl']] = dek['_id']
+            elif 'cur' in dek:
+                ret["levels"][dek['lbl']] = dek['_id']
+            elif 'mux' in dek:
+                ret["addons"][dek['lbl']] = dek['_id']
+            else:
+                ret["labels"][dek['lbl']] = dek['_id']
+
+        return ret
+
+    def get_member_ids(self, folders):
+        folder_map = self._parse_flags()["folders"]
+
+        r = self._get(BASE_URL + "ylp", params={
+            "lbl": ",".join([folder_map[f] for f in folders]),
+            "org": self.org_num,
+            "var": "_id,nam,ctc"})
+        if r.status_code != 200 or 'usr' not in r.json():
+            raise MembershipWorksRemoteError('user listing', r)
+
+        # get list of member ID matching the search
+        return [user['uid'] for user in r.json()['usr']]
+
+    # TODO: has issues with aliasing header names:
+    #       ex: "Personal Studio Space" Label vs Membership Addon/Field
+    def get_members(self, folders, columns):
+        """ Pull the members csv from the membershipworks api
+            folders: a list of the names of the folders to get
+                     (see folder_map in this function for mapping to ids)
+            columns: which columns to get"""
+        ids = self.get_member_ids(folders)
+
+        # get members CSV
+        # TODO: maybe can just use previous get instead? would return JSON
+        r = self._post(BASE_URL + "csv",
+                       data={"_rt": "946702800", # unknown
+                             "mux": "", # unknown
+                             "tid": ",".join(ids), # ids of members to get
+                             "var": columns})
+        if r.status_code != 200:
+            raise MembershipWorksRemoteError('csv generation', r)
+        return list(csv.DictReader(StringIO(r.text)))
+
+    def get_transactions(self, start_date, end_date, json=False):
+        """Get the transactions between start_date and end_date
+
+           Dates can be datetime.date or datetime.datetime
+
+           json gets a different version of the transactions list,
+           which contains a different set information
+        """
+        r = self._get(BASE_URL + "csv",
+                 params={'crm': '12,13,14,18,19', # transaction types, see CRM
+                         **({'txl': ''} if json else {}),
+                         'sdp': start_date.strftime('%s'),
+                         'edp': end_date.strftime('%s')})
+        if r.status_code != 200:
+            raise MembershipWorksRemoteError('csv generation', r)
+        if json:
+            return r.json()
+        else:
+            return list(csv.DictReader(StringIO(r.text)))
+
+    def get_all_members(self):
+        """Get all the data for all the members"""
+        folders = self._parse_flags()["folders"].keys()
+        fields = self._all_fields()
+        members = self.get_members(folders, ",".join(fields.keys()))
+        return members

lib/hid/Credential.py

@@ -3,7 +3,6 @@ import bitstring
 # Reference for H10301 card format:
 # https://www.hidglobal.com/sites/default/files/hid-understanding_card_data_formats-wp-en.pdf
 
-
 class Credential:
     def __init__(self, code=None, hex=None):
         if code is None and hex is None:
@@ -12,10 +11,8 @@ class Credential:
             raise TypeError("Cannot set both code and hex for a Credential")
         elif code is not None:
             self.bits = bitstring.pack(
-                "0b000000, 0b0, uint:8=facility, uint:16=number, 0b0",
-                facility=code[0],
-                number=code[1],
-            )
+                '0b000000, 0b0, uint:8=facility, uint:16=number, 0b0',
+                facility=code[0], number=code[1])
             self.bits[6] = self.bits[7:19].count(1) % 2  # even parity
             self.bits[31] = not (self.bits[19:31].count(1) % 2)  # odd parity
         elif hex is not None:

lib/hid/DoorController.py

@@ -0,0 +1,178 @@
+import csv
+from io import StringIO
+import urllib3
+
+import requests
+from lxml import etree
+from lxml.builder import ElementMaker
+
+E_plain = ElementMaker(nsmap={"hid": "http://www.hidglobal.com/VertX"})
+E = ElementMaker(namespace="http://www.hidglobal.com/VertX",
+                 nsmap={"hid": "http://www.hidglobal.com/VertX"})
+E_corp = ElementMaker(namespace="http://www.hidcorp.com/VertX", #stupid
+                 nsmap={"hid": "http://www.hidcorp.com/VertX"})
+ROOT = E_plain.VertXMessage
+
+fieldnames = "CardNumber,CardFormat,PinRequired,PinCode,ExtendedAccess,ExpiryDate,Forename,Initial,Surname,Email,Phone,Custom1,Custom2,Schedule1,Schedule2,Schedule3,Schedule4,Schedule5,Schedule6,Schedule7,Schedule8".split(",")
+
+# TODO: where should this live?
+# it's fine, ssl certs are for losers anyway
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+class RemoteError(Exception):
+    def __init__(self, r):
+        super().__init__(
+            f"Door Updating Error: {r.status_code} {r.reason}\n{r.text}")
+
+class DoorController():
+    def __init__(self, ip, username, password, name="", access=""):
+        self.ip = ip
+        self.username = username
+        self.password = password
+        self.name = name
+        self.access = access
+
+    def doImport(self, params=None, files=None):
+        """Send a request to the door control import script"""
+        r = requests.post(
+            'https://' + self.ip + '/cgi-bin/import.cgi',
+            params=params,
+            files=files,
+            auth=requests.auth.HTTPDigestAuth(self.username, self.password),
+            timeout=60,
+            verify=False) # ignore insecure SSL
+        xml = etree.XML(r.content)
+        if r.status_code != 200 \
+           or len(xml.findall("{http://www.hidglobal.com/VertX}Error")) > 0:
+            raise RemoteError(r)
+
+    def doCSVImport(self, csv):
+        """Do the CSV import procedure on a door control"""
+        self.doImport({"task": "importInit"})
+        self.doImport({"task": "importCardsPeople", "name": "cardspeopleschedule.csv"},
+                        {"importCardsPeopleButton": ("cardspeopleschedule.csv", csv, 'text/csv')})
+        self.doImport({"task": "importDone"})
+
+    def doXMLRequest(self, xml, prefix=b'<?xml version="1.0" encoding="UTF-8"?>'):
+        if not isinstance(xml, bytes):
+            xml = etree.tostring(xml)
+        r = requests.get(
+            'https://' + self.ip + '/cgi-bin/vertx_xml.cgi',
+            params={'XML': prefix + xml},
+            auth=requests.auth.HTTPDigestAuth(self.username, self.password),
+            verify=False)
+        resp_xml = etree.XML(r.content)
+        # probably meed to be more sane about this
+        if r.status_code != 200 \
+           or len(resp_xml.findall("{*}Error")) > 0:
+            raise RemoteError(r)
+        return resp_xml
+
+    def get_scheduleMap(self):
+        schedules = self.doXMLRequest(
+            ROOT(E.Schedules({"action": "LR",
+                              "recordOffset": "0",
+                              "recordCount": "8"})))
+        return {fmt.attrib["scheduleName"]: fmt.attrib["scheduleID"]
+                for fmt in schedules[0]}
+
+    def get_schedules(self):
+        # TODO: might be able to do in one request
+        schedules = self.doXMLRequest(ROOT(
+            E.Schedules({"action": "LR"})))
+        etree.dump(schedules)
+
+        data = self.doXMLRequest(ROOT(
+            *[E.Schedules({"action": "LR",
+                           "scheduleID": schedule.attrib["scheduleID"]})
+              for schedule in schedules[0]]))
+        return ROOT(E_corp.Schedules({"action": "AD"},
+                           *[s[0] for s in data]))
+
+    def set_schedules(self, schedules):
+        # clear all people
+        outString = StringIO()
+        writer = csv.DictWriter(outString, fieldnames)
+        writer.writeheader()
+        writer.writerow({})
+        outString.seek(0)
+        self.doCSVImport(outString)
+
+        # clear all schedules
+        delXML = ROOT(
+            *[E.Schedules({"action": "DD", "scheduleID": str(ii)})
+             for ii in range(1, 8)])
+        try:
+            self.doXMLRequest(delXML)
+        except RemoteError:
+            # don't care about failure to delete, they probably just didn't exist
+            pass
+
+        # load new schedules
+        self.doXMLRequest(schedules)
+
+    def get_cardFormats(self):
+        cardFormats = self.doXMLRequest(
+            ROOT(E.CardFormats({"action": "LR",
+                                "responseFormat": "expanded"})))
+
+        return {fmt[0].attrib["value"]: fmt.attrib["formatID"]
+                for fmt in cardFormats[0].findall('{*}CardFormat[{*}FixedField]')}
+
+    def set_cardFormat(self, formatName, templateID, facilityCode):
+        # TODO: add ability to delete formats
+        # delete example: <hid:CardFormats action="DD" formatID="7-1-244"/>
+
+        el = ROOT(
+            E.CardFormats({"action": "AD"},
+                E.CardFormat({"formatName": formatName,
+                              "templateID": str(templateID)},
+                    E.FixedField({"value": str(facilityCode)}))))
+        return self.doXMLRequest(el)
+
+    def get_records(self, req, count, params={}):
+        result = []
+        recordCount = 0
+        moreRecords = True
+
+        # note: all the "+/-1" bits are to work around a bug where the
+        # last returned entry is incomplete. There is probably a
+        # better way to do this, but for now I just get the last entry
+        # again in the next request. I suspect this probably ends
+        # poorly if the numbers line up poorly (ie an exact multiple
+        # of the returned record limit)
+        while moreRecords:
+            res = self.doXMLRequest(ROOT(
+                req({
+                    "action": "LR",
+                    "recordCount": str(count - recordCount + 1),
+                    "recordOffset": str(recordCount - 1
+                                        if recordCount > 0 else 0),
+                    **params
+                })))
+            result = result[:-1] + list(res[0])
+
+            recordCount += int(res[0].get('recordCount')) - 1
+            moreRecords = res[0].get('moreRecords') == 'true'
+
+        return result
+
+    def get_cardholders(self):
+        return self.get_records(E.Cardholders, 1000,
+                                {"responseFormat": "expanded"})
+
+    def get_credentials(self):
+        return self.get_records(E.Credentials, 1000)
+
+    def get_lock(self):
+        el = ROOT(
+            E.Doors({"action": "LR", "responseFormat": "status"}))
+        xml = self.doXMLRequest(el)
+        relayState = xml.find('./{*}Doors/{*}Door').attrib['relayState']
+        return "unlocked" if relayState == "set" else "locked"
+
+    def set_lock(self, lock=True):
+        el = ROOT(
+            E.Doors({"action": "CM",
+                     "command": "lockDoor" if lock else "unlockDoor"}))
+        return self.doXMLRequest(el)

memberPlumbing/MembershipWorks.py

@@ -1,265 +0,0 @@
-import csv
-from io import StringIO
-
-import requests
-import datetime
-
-BASE_URL = "https://api.membershipworks.com"
-
-# extracted from `SF._is.crm` in https://cdn.membershipworks.com/all.js
-CRM = {
-    0: "Note",
-    4: "Profile Updated",
-    8: "Scheduled/Reminder Email",
-    9: "Renewal Notice",
-    10: "Join Date",
-    11: "Next Renewal Date",
-    12: "Membership Payment",
-    13: "Donation",
-    14: "Event Activity",
-    15: "Conversation",
-    16: "Contact Change",
-    17: "Label Change",
-    18: "Other Payment",
-    19: "Cart Payment",
-    20: "Payment Failed",
-    21: "Billing Updated",
-    22: "Form Checkout",
-    23: "Event Payment",
-    24: "Invoice",
-    25: "Invoice Payment",
-    26: "Renewal",
-    27: "Payment",
-}
-
-# Types of fields, extracted from a html snippet in all.js + some guessing
-typ = {
-    1: "Text input",
-    2: "Password",  # inferred from data
-    3: "Simple text area",
-    4: "Rich text area",
-    7: "Address",
-    8: "Check box",
-    9: "Select",
-    11: "Display value stored in field (ie. read only)",
-    12: "Required waiver/terms",
-}
-
-# more constants, this time extracted from the members csv export in all.js
-staticFlags = {
-    "pos": {"lbl": "Position/relation (contacts)"},
-    "nte": {"lbl": "Admin note (contacts)"},
-    "pwd": {"lbl": "Password"},
-    "lgo": {"lbl": "Business card image URLs"},
-    "pfx": {"lbl": "Profile gallery image URLs"},
-    "lvl": {"lbl": "Membership levels"},
-    "aon": {"lbl": "Membership add-ons"},
-    "lbl": {"lbl": "Labels"},
-    "joi": {"lbl": "Join date"},
-    "end": {"lbl": "Renewal date"},
-    "spy": {"lbl": "Billing method"},
-    "rid": {"lbl": "Auto recurring billing ID"},
-    "ipa": {"lbl": "IP address"},
-    "_id": {"lbl": "Account ID"},
-}
-
-
-class MembershipWorksRemoteError(Exception):
-    def __init__(self, reason, r):
-        super().__init__(
-            f"Error when attempting {reason}: {r.status_code} {r.reason}\n{r.text}"
-        )
-
-
-class MembershipWorks:
-    def __init__(self):
-        self.sess = requests.Session()
-        self.org_info = None
-        self.auth_token = None
-        self.org_num = None
-
-    def login(self, username, password):
-        """Authenticate against the membershipworks api"""
-        r = self.sess.post(
-            BASE_URL + "/v2/account/session",
-            data={"eml": username, "pwd": password},
-            headers={"X-Org": "10000"},
-        )
-        if r.status_code != 200 or "SF" not in r.json():
-            raise MembershipWorksRemoteError("login", r)
-        self.org_info = r.json()
-        self.auth_token = self.org_info["SF"]
-        self.org_num = self.org_info["org"]
-        self.sess.headers.update(
-            {
-                "X-Org": str(self.org_num),
-                "X-Role": "admin",
-                "Authorization": "Bearer " + self.auth_token,
-            }
-        )
-
-    def _inject_auth(self, kwargs):
-        # TODO: should probably be a decorator or something
-        if self.auth_token is None:
-            raise RuntimeError("Not Logged in to MembershipWorks")
-        # add auth token to params
-        if "params" not in kwargs:
-            kwargs["params"] = {}
-        kwargs["params"]["SF"] = self.auth_token
-
-    def _get_v1(self, *args, **kwargs):
-        self._inject_auth(kwargs)
-        # TODO: should probably do some error handling in here
-        return requests.get(*args, **kwargs)
-
-    def _post_v1(self, *args, **kwargs):
-        self._inject_auth(kwargs)
-        # TODO: should probably do some error handling in here
-        return requests.post(*args, **kwargs)
-
-    def _all_fields(self):
-        """Parse out a list of fields from the org data.
-
-        Is this terrible? Yes. Also, not dissimilar to how MW does it
-        in all.js.
-        """
-        fields = staticFlags.copy()
-
-        # TODO: this will take the later option, if the same field
-        # used in mulitple places. I don't know which lbl is used for
-        # csv export
-
-        # anm: member signup, acc: member manage, adm: admin manage
-        for screen_type in ["anm", "acc", "adm"]:
-            for box in self.org_info["tpl"][screen_type]:
-                for element in box["box"]:
-                    if type(element["dat"]) != str:
-                        for field in element["dat"]:
-                            if "_id" in field:
-                                if field["_id"] not in fields:
-                                    fields[field["_id"]] = field
-
-        return fields
-
-    def _parse_flags(self):
-        """Parse the flags out of the org data.
-
-        This is terrible, and there might be a better way to do this.
-        """
-        ret = {"folders": {}, "levels": {}, "addons": {}, "labels": {}}
-
-        for dek in self.org_info["dek"]:
-            # TODO: there must be a better way. this is stupid
-            if dek["dek"] == 1:
-                ret["folders"][dek["lbl"]] = dek["did"]
-            elif "cur" in dek:
-                ret["levels"][dek["lbl"]] = dek["did"]
-            elif "mux" in dek:
-                ret["addons"][dek["lbl"]] = dek["did"]
-            else:
-                ret["labels"][dek["lbl"]] = dek["did"]
-
-        return ret
-
-    def get_member_ids(self, folders):
-        folder_map = self._parse_flags()["folders"]
-
-        r = self.sess.get(
-            BASE_URL + "/v2/accounts",
-            params={"dek": ",".join([folder_map[f] for f in folders])},
-        )
-        if r.status_code != 200 or "usr" not in r.json():
-            raise MembershipWorksRemoteError("user listing", r)
-
-        # get list of member ID matching the search
-        # dedup with set() to work around people with alt uids
-        # TODO: figure out why people have alt uids
-        return set(user["uid"] for user in r.json()["usr"])
-
-    # TODO: has issues with aliasing header names:
-    #       ex: "Personal Studio Space" Label vs Membership Addon/Field
-    def get_members(self, folders, columns):
-        """Pull the members csv from the membershipworks api
-        folders: a list of the names of the folders to get
-                 (see folder_map in this function for mapping to ids)
-        columns: which columns to get"""
-        ids = self.get_member_ids(folders)
-
-        # get members CSV
-        # TODO: maybe can just use previous get instead? would return JSON
-        r = self._post_v1(
-            BASE_URL + "/v1/csv",
-            data={
-                "_rt": "946702800",  # unknown
-                "mux": "",  # unknown
-                "tid": ",".join(ids),  # ids of members to get
-                "var": columns,
-            },
-        )
-        if r.status_code != 200:
-            raise MembershipWorksRemoteError("csv generation", r)
-
-        if r.text[0] == "\ufeff":
-            r.encoding = r.encoding + "-sig"
-
-        return list(csv.DictReader(StringIO(r.text)))
-
-    def get_transactions(self, start_date, end_date, json=False):
-        """Get the transactions between start_date and end_date
-
-        Dates can be datetime.date or datetime.datetime
-
-        json gets a different version of the transactions list,
-        which contains a different set information
-        """
-        r = self._get_v1(
-            BASE_URL + "/v1/csv",
-            params={
-                "crm": ",".join(str(k) for k in CRM.keys()),
-                **({"txl": ""} if json else {}),
-                "sdp": start_date.strftime("%s"),
-                "edp": end_date.strftime("%s"),
-            },
-        )
-        if r.status_code != 200:
-            raise MembershipWorksRemoteError("csv generation", r)
-        if json:
-            return r.json()
-        else:
-            if r.text[0] == "\ufeff":
-                r.encoding = r.encoding + "-sig"
-
-            return list(csv.DictReader(StringIO(r.text)))
-
-    def get_all_members(self):
-        """Get all the data for all the members"""
-        folders = self._parse_flags()["folders"].keys()
-        fields = self._all_fields()
-        members = self.get_members(folders, ",".join(fields.keys()))
-        return members
-
-    def get_events_list(self, start_date: datetime.datetime):
-        """Retrive a list of events since start_date"""
-        r = self.sess.get(
-            BASE_URL + "/v2/events",
-            params={
-                "sdp": start_date.strftime("%s"),
-            },
-        )
-        return r.json()
-
-    def get_event_by_eid(self, eid: str):
-        """Retrieve a specific event by its event id (eid)"""
-        r = self.sess.get(
-            BASE_URL + "/v2/event",
-            params={"eid": eid},
-        )
-        return r.json()
-
-    def get_event_by_url(self, url: str):
-        """Retrieve a specific event by its url"""
-        r = self.sess.get(
-            BASE_URL + "/v2/event",
-            params={"url": url},
-        )
-        return r.json()

memberPlumbing/config.py

@@ -1,36 +0,0 @@
-from ruamel.yaml import YAML
-
-from .hid.DoorController import DoorController
-from .MembershipWorks import MembershipWorks
-
-
-class Config:
-    def __init__(self, path="config.yaml"):
-        with open(path) as f:
-            self._data = YAML().load(f)
-        self.__dict__.update(self._data)
-
-        # lazy init, because this actually talks to an external server
-        self._membershipworks = None
-
-    @property
-    def doors(self):
-        return {
-            doorName: DoorController(
-                doorData["ip"],
-                self.DOOR_USERNAME,
-                self.DOOR_PASSWORD,
-                name=doorName,
-                access=doorData["access"],
-            )
-            for doorName, doorData in self.doorControllers.items()
-        }
-
-    @property
-    def membershipworks(self):
-        if not self._membershipworks:
-            self._membershipworks = MembershipWorks()
-            self._membershipworks.login(
-                self.MEMBERSHIPWORKS_USERNAME, self.MEMBERSHIPWORKS_PASSWORD
-            )
-        return self._membershipworks

memberPlumbing/doorUpdater.py

@@ -1,370 +0,0 @@
-#!/usr/bin/env python3
-
-import copy
-
-from .config import Config
-from .hid.Credential import Credential
-from .hid.DoorController import ROOT, E
-
-
-class Member:
-    def __init__(
-        self,
-        forename="",
-        surname="",
-        membershipWorksID="",
-        middleName="",
-        email="",
-        phone="",
-        cardholderID=None,
-        doorAccess=[],
-        credentials=set(),
-        levels=[],
-        extraLevels=[],
-        schedules=[],
-    ):
-        self.forename = forename
-        self.surname = surname
-        self.membershipWorksID = membershipWorksID
-        self.middleName = middleName
-        self.email = email
-        self.phone = phone
-        self.cardholderID = cardholderID
-        self.doorAccess = doorAccess
-        self.credentials = credentials
-        self.levels = levels
-        self.schedules = schedules
-
-    def __str__(self):
-        return f"""Name: {self.forename} | {self.middleName} | {self.surname}
-MembershipWorks ID: {self.membershipWorksID}
-Email: {self.email}
-Phone: {self.phone}
-Cardholder ID: {self.cardholderID}
-doorAccess: {self.doorAccess}
-Credentials: {self.credentials}
-Levels: {self.levels}
-Schedules: {self.schedules}
-"""
-
-
-class MembershipworksMember(Member):
-    def __init__(self, config, data, formerMember=False):
-        super().__init__(
-            data["First Name"],
-            data["Last Name"],
-            membershipWorksID=data["Account ID"],
-            email=data["Email"],
-            phone=data["Phone"],
-        )
-
-        if data["Access Card Number"] != "":
-            self.credentials = set(
-                [
-                    Credential(
-                        code=(
-                            data["Access Card Facility Code"],
-                            data["Access Card Number"],
-                        )
-                    )
-                ]
-            )
-        else:
-            self.credentials = set()
-
-        self.onHold = (
-            data["Account on Hold"] != ""
-            or data["CMS Membership on hold"] == "CMS Membership on hold"
-        )
-        self.formerMember = formerMember
-
-        levels = {k: v for k, v in config.memberLevels.items() if data[k] == k}
-        self.levels = list(levels.keys())
-        self.schedules = list(levels.values())
-
-        self.extraLevels = {
-            schedule: sum(
-                (doors for prop, doors in props.items() if data[prop] == "Y"), []
-            )
-            for schedule, props in config.doorSpecificSchedules.items()
-        }
-
-        self.doorAccess = [
-            door
-            for door, doorData in config.doors.items()
-            if data["Access " + doorData.access + "?"] == "Y"
-        ]
-
-    def to_DoorMember(self, door):
-        doorLevels = [k for k, v in self.extraLevels.items() if door.name in v]
-
-        schedules = []
-        if door.name in self.doorAccess and not self.onHold and not self.formerMember:
-            schedules = self.schedules + doorLevels
-
-        dm = DoorMember(
-            door,
-            forename=self.forename,
-            surname=self.surname,
-            membershipWorksID=self.membershipWorksID,
-            email=self.email,
-            phone=self.phone,
-            levels=self.levels + doorLevels,
-            doorAccess=self.doorAccess,
-            credentials=self.credentials,
-            schedules=schedules,
-        )
-
-        return dm
-
-    def __str__(self):
-        return (
-            super().__str__()
-            + f"""OnHold? {self.onHold}
-Former Member? {self.formerMember}
-"""
-        )
-
-
-class DoorMember(Member):
-    def __init__(self, door, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.door = door
-
-    @classmethod
-    def from_cardholder(cls, data, door):
-        ch = cls(
-            door=door,
-            forename=data.get("forename", ""),
-            surname=data.get("surname", ""),
-            membershipWorksID=data.attrib.get("custom2", ""),
-            middleName=data.attrib.get("middleName", ""),
-            email=data.attrib.get("email", ""),
-            phone=data.attrib.get("phone", ""),
-            cardholderID=data.attrib["cardholderID"],
-        )
-
-        ch.credentials = set(
-            Credential(hex=(c.attrib["rawCardNumber"]))
-            for c in data.findall("{*}Credential")
-        )
-
-        ch.levels = data.attrib.get("custom1", "").split("|")
-        ch.schedules = [r.attrib["scheduleName"] for r in data.findall("{*}Role")]
-
-        return ch
-
-    def attribs(self):
-        return {
-            "forename": self.forename,
-            "surname": self.surname,
-            "middleName": self.middleName,
-            "email": self.email,
-            "phone": self.phone,
-            "custom1": "|".join(self.levels).replace("&", "and"),
-            "custom2": self.membershipWorksID,
-        }
-
-    def make_schedules(self, schedulesMap):
-        roles = [
-            E.Role(
-                {
-                    "roleID": self.cardholderID,
-                    "scheduleID": schedulesMap[schedule],
-                    "resourceID": "0",
-                }
-            )
-            for schedule in self.schedules
-        ]
-
-        return E.RoleSet(
-            {"action": "UD", "roleSetID": self.cardholderID}, E.Roles(*roles)
-        )
-
-    def make_credentials(self, newCredentials, cardFormats):
-        out = [
-            E.Credential(
-                {
-                    "formatName": str(credential.code[0]),
-                    "cardNumber": str(credential.code[1]),
-                    "formatID": cardFormats[str(credential.code[0])],
-                    "isCard": "true",
-                    "cardholderID": self.cardholderID,
-                }
-            )
-            for credential in newCredentials
-        ]
-
-        return E.Credentials({"action": "AD"}, *out)
-
-
-def update_door(door, members):
-    cardFormats = door.get_cardFormats()
-    cardholders = {
-        member.membershipWorksID: member
-        for member in [
-            DoorMember.from_cardholder(ch, door) for ch in door.get_cardholders()
-        ]
-    }
-    schedulesMap = door.get_scheduleMap()
-    allCredentials = set(
-        Credential(hex=c.attrib["rawCardNumber"]) for c in door.get_credentials()
-    )
-
-    # TODO: can I combine requests?
-    for membershipworksMember in members:
-        member = membershipworksMember.to_DoorMember(door)
-        # cardholder did not exist, so add them
-        if member.membershipWorksID not in cardholders:
-            print("- Adding Member {member.forename} {member.surname}:")
-            print(f"  - {member.attribs()}")
-            resp = door.doXMLRequest(
-                ROOT(E.Cardholders({"action": "AD"}, E.Cardholder(member.attribs())))
-            )
-            member.cardholderID = resp.find("{*}Cardholders/{*}Cardholder").attrib[
-                "cardholderID"
-            ]
-
-            # create a dummy ch to force an update
-            # TODO: probably a cleaner way to do this
-            ch = copy.copy(member)
-            ch.schedules = []
-            ch.credentials = set()
-        # cardholder exists, compare contents
-        else:
-            ch = cardholders.pop(member.membershipWorksID)
-            member.cardholderID = ch.cardholderID
-
-        if member.attribs() != ch.attribs():  # update cardholder attributes
-            print(f"- Updating profile for {member.forename} {member.surname}")
-            print(f"  - Old: {ch.attribs()}")
-            print(f"  - New: {member.attribs()}")
-            door.doXMLRequest(
-                ROOT(
-                    E.Cardholders(
-                        {"action": "UD", "cardholderID": member.cardholderID},
-                        E.CardHolder(member.attribs()),
-                    )
-                )
-            )
-
-        if member.credentials != ch.credentials:
-            print(f"- Updating card for {member.forename} {member.surname}")
-            print(f"  - {ch.credentials} -> {member.credentials}")
-
-            oldCards = ch.credentials
-            newCards = member.credentials
-
-            allNewCards = set(
-                card
-                for m in members
-                if m != membershipworksMember
-                for card in m.credentials
-            )
-
-            # cards removed, and won't be reassigned to someone else
-            for card in (oldCards - newCards) - allNewCards:
-                door.doXMLRequest(
-                    ROOT(
-                        E.Credentials(
-                            {
-                                "action": "UD",
-                                "rawCardNumber": card.hex,
-                                "isCard": "true",
-                            },
-                            E.Credential({"cardholderID": ""}),
-                        )
-                    )
-                )
-
-            if newCards - oldCards:  # cards added
-                for card in newCards & allNewCards:  # new card exists in another member
-                    print(
-                        [
-                            m
-                            for m in members
-                            for card in m.credentials
-                            if card in newCards
-                        ]
-                    )
-                    raise Exception(f"Duplicate Card in input data! {card}")
-
-                # card existed in door, and needs to be reassigned
-                for card in newCards & allCredentials:
-                    door.doXMLRequest(
-                        ROOT(
-                            E.Credentials(
-                                {
-                                    "action": "UD",
-                                    "rawCardNumber": card.hex,
-                                    "isCard": "true",
-                                },
-                                E.Credential({"cardholderID": member.cardholderID}),
-                            )
-                        )
-                    )
-
-                # cards that never existed, and need to be created
-                if newCards - allCredentials:
-                    door.doXMLRequest(
-                        ROOT(
-                            member.make_credentials(
-                                newCards - allCredentials, cardFormats
-                            )
-                        )
-                    )
-
-        if member.schedules != ch.schedules:
-            print(
-                "- Updating schedule for"
-                + f" {member.forename} {member.surname}:"
-                + f" {ch.schedules} -> {member.schedules}"
-            )
-            door.doXMLRequest(ROOT(member.make_schedules(schedulesMap)))
-
-    # TODO: delete cardholders that are no longer members?
-
-
-def main():
-    config = Config()
-    membershipworks = config.membershipworks
-    membershipworks_attributes = (
-        "_id,nam,phn,eml,lvl,lbl,xws,xms,xsc,xas,xfd,xac,xcf,xeh,xse"
-    )
-
-    memberData = membershipworks.get_members(
-        ["Members", "CMS Staff", "Misc. Access"], membershipworks_attributes
-    )
-    members = [MembershipworksMember(config, m) for m in memberData]
-
-    formerMemberData = membershipworks.get_members(
-        ["Former Members"], membershipworks_attributes
-    )
-    formerMembers = [
-        MembershipworksMember(config, m, formerMember=True) for m in formerMemberData
-    ]
-
-    for formerMember in formerMembers:
-        member = next(
-            (
-                m
-                for m in members
-                if m.membershipWorksID == formerMember.membershipWorksID
-            ),
-            None,
-        )
-
-        # member exists in another folder
-        if member is not None:
-            member.formerMember = True
-        else:  # member is only a former member
-            formerMember.formerMember = True
-            members.append(formerMember)
-
-    for door in config.doors.values():
-        print(door.name, door.ip)
-        update_door(door, members)
-
-
-if __name__ == "__main__":
-    main()

memberPlumbing/hid/DoorController.py

@@ -1,223 +0,0 @@
-import csv
-from datetime import datetime
-from io import StringIO
-
-import requests
-import urllib3
-from lxml import etree
-from lxml.builder import ElementMaker
-
-E_plain = ElementMaker(nsmap={"hid": "http://www.hidglobal.com/VertX"})
-E = ElementMaker(
-    namespace="http://www.hidglobal.com/VertX",
-    nsmap={"hid": "http://www.hidglobal.com/VertX"},
-)
-E_corp = ElementMaker(
-    namespace="http://www.hidcorp.com/VertX",  # stupid
-    nsmap={"hid": "http://www.hidcorp.com/VertX"},
-)
-ROOT = E_plain.VertXMessage
-
-fieldnames = "CardNumber,CardFormat,PinRequired,PinCode,ExtendedAccess,ExpiryDate,Forename,Initial,Surname,Email,Phone,Custom1,Custom2,Schedule1,Schedule2,Schedule3,Schedule4,Schedule5,Schedule6,Schedule7,Schedule8".split(
-    ","
-)
-
-# TODO: where should this live?
-# it's fine, ssl certs are for losers anyway
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-
-class RemoteError(Exception):
-    def __init__(self, r):
-        super().__init__(f"Door Updating Error: {r.status_code} {r.reason}\n{r.text}")
-
-
-class DoorController:
-    def __init__(self, ip, username, password, name="", access=""):
-        self.ip = ip
-        self.username = username
-        self.password = password
-        self.name = name
-        self.access = access
-
-    def doImport(self, params=None, files=None):
-        """Send a request to the door control import script"""
-        r = requests.post(
-            "https://" + self.ip + "/cgi-bin/import.cgi",
-            params=params,
-            files=files,
-            auth=requests.auth.HTTPDigestAuth(self.username, self.password),
-            timeout=60,
-            verify=False,
-        )  # ignore insecure SSL
-        xml = etree.XML(r.content)
-        if (
-            r.status_code != 200
-            or len(xml.findall("{http://www.hidglobal.com/VertX}Error")) > 0
-        ):
-            raise RemoteError(r)
-
-    def doCSVImport(self, csv):
-        """Do the CSV import procedure on a door control"""
-        self.doImport({"task": "importInit"})
-        self.doImport(
-            {"task": "importCardsPeople", "name": "cardspeopleschedule.csv"},
-            {"importCardsPeopleButton": ("cardspeopleschedule.csv", csv, "text/csv")},
-        )
-        self.doImport({"task": "importDone"})
-
-    def doXMLRequest(self, xml, prefix=b'<?xml version="1.0" encoding="UTF-8"?>'):
-        if not isinstance(xml, bytes):
-            xml = etree.tostring(xml)
-        r = requests.get(
-            "https://" + self.ip + "/cgi-bin/vertx_xml.cgi",
-            params={"XML": prefix + xml},
-            auth=requests.auth.HTTPDigestAuth(self.username, self.password),
-            verify=False,
-        )
-        resp_xml = etree.XML(r.content)
-        # probably meed to be more sane about this
-        if r.status_code != 200 or len(resp_xml.findall("{*}Error")) > 0:
-            raise RemoteError(r)
-        return resp_xml
-
-    def get_scheduleMap(self):
-        schedules = self.doXMLRequest(
-            ROOT(E.Schedules({"action": "LR", "recordOffset": "0", "recordCount": "8"}))
-        )
-        return {
-            fmt.attrib["scheduleName"]: fmt.attrib["scheduleID"] for fmt in schedules[0]
-        }
-
-    def get_schedules(self):
-        # TODO: might be able to do in one request
-        schedules = self.doXMLRequest(ROOT(E.Schedules({"action": "LR"})))
-        etree.dump(schedules)
-
-        data = self.doXMLRequest(
-            ROOT(
-                *[
-                    E.Schedules(
-                        {"action": "LR", "scheduleID": schedule.attrib["scheduleID"]}
-                    )
-                    for schedule in schedules[0]
-                ]
-            )
-        )
-        return ROOT(E_corp.Schedules({"action": "AD"}, *[s[0] for s in data]))
-
-    def set_schedules(self, schedules):
-        # clear all people
-        outString = StringIO()
-        writer = csv.DictWriter(outString, fieldnames)
-        writer.writeheader()
-        writer.writerow({})
-        outString.seek(0)
-        self.doCSVImport(outString)
-
-        # clear all schedules
-        delXML = ROOT(
-            *[
-                E.Schedules({"action": "DD", "scheduleID": str(ii)})
-                for ii in range(1, 8)
-            ]
-        )
-        try:
-            self.doXMLRequest(delXML)
-        except RemoteError:
-            # don't care about failure to delete, they probably just didn't exist
-            pass
-
-        # load new schedules
-        self.doXMLRequest(schedules)
-
-    def get_cardFormats(self):
-        cardFormats = self.doXMLRequest(
-            ROOT(E.CardFormats({"action": "LR", "responseFormat": "expanded"}))
-        )
-
-        return {
-            fmt[0].attrib["value"]: fmt.attrib["formatID"]
-            for fmt in cardFormats[0].findall("{*}CardFormat[{*}FixedField]")
-        }
-
-    def set_cardFormat(self, formatName, templateID, facilityCode):
-        # TODO: add ability to delete formats
-        # delete example: <hid:CardFormats action="DD" formatID="7-1-244"/>
-
-        el = ROOT(
-            E.CardFormats(
-                {"action": "AD"},
-                E.CardFormat(
-                    {"formatName": formatName, "templateID": str(templateID)},
-                    E.FixedField({"value": str(facilityCode)}),
-                ),
-            )
-        )
-        return self.doXMLRequest(el)
-
-    def get_records(self, req, count, params={}, stopFunction=None):
-        result = []
-        recordCount = 0
-        moreRecords = True
-
-        # note: all the "+/-1" bits are to work around a bug where the
-        # last returned entry is incomplete. There is probably a
-        # better way to do this, but for now I just get the last entry
-        # again in the next request. I suspect this probably ends
-        # poorly if the numbers line up poorly (ie an exact multiple
-        # of the returned record limit)
-        while moreRecords and (stopFunction is None or stopFunction(result)):
-            res = self.doXMLRequest(
-                ROOT(
-                    req(
-                        {
-                            "action": "LR",
-                            "recordCount": str(count - recordCount + 1),
-                            "recordOffset": str(
-                                recordCount - 1 if recordCount > 0 else 0
-                            ),
-                            **params,
-                        }
-                    )
-                )
-            )
-            result = result[:-1] + list(res[0])
-
-            recordCount += int(res[0].get("recordCount")) - 1
-            moreRecords = res[0].get("moreRecords") == "true"
-
-        return result
-
-    def get_cardholders(self):
-        return self.get_records(E.Cardholders, 1000, {"responseFormat": "expanded"})
-
-    def get_credentials(self):
-        return self.get_records(E.Credentials, 1000)
-
-    def get_events(self, threshold):
-        def event_newer_than_threshold(event):
-            return datetime.fromisoformat(event.attrib["timestamp"]) > threshold
-
-        def last_event_newer_than_threshold(events):
-            return (not events) or event_newer_than_threshold(events[-1])
-
-        return [
-            event
-            for event in self.get_records(
-                E.EventMessages, 10000, stopFunction=last_event_newer_than_threshold
-            )
-            if event_newer_than_threshold(event)
-        ]
-
-    def get_lock(self):
-        el = ROOT(E.Doors({"action": "LR", "responseFormat": "status"}))
-        xml = self.doXMLRequest(el)
-        relayState = xml.find("./{*}Doors/{*}Door").attrib["relayState"]
-        return "unlocked" if relayState == "set" else "locked"
-
-    def set_lock(self, lock=True):
-        el = ROOT(
-            E.Doors({"action": "CM", "command": "lockDoor" if lock else "unlockDoor"})
-        )
-        return self.doXMLRequest(el)

memberPlumbing/hidEvents.py

@@ -1,112 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-from datetime import datetime
-
-import requests
-from lxml import etree
-from peewee import (
-    BooleanField,
-    CharField,
-    CompositeKey,
-    DateTimeField,
-    IntegerField,
-    Model,
-    MySQLDatabase,
-    TextField,
-)
-
-from .config import Config
-
-database = MySQLDatabase(None)
-
-
-class HIDEvent(Model):
-    doorName = CharField()
-    timestamp = DateTimeField()
-    eventType = IntegerField()
-    readerAddress = IntegerField()
-    cardholderID = IntegerField(null=True)
-    commandStatus = BooleanField(null=True)
-    forename = TextField(null=True)
-    surname = TextField(null=True)
-    ioState = BooleanField(null=True)
-    newTime = DateTimeField(null=True)
-    oldTime = DateTimeField(null=True)
-    rawCardNumber = CharField(max_length=8, null=True)
-
-    class Meta:
-        primary_key = CompositeKey("doorName", "timestamp", "eventType")
-        database = database
-
-
-def getStrings(door):
-    """Parses out the message strings from source."""
-    r = requests.get(
-        "https://" + door.ip + "/html/en_EN/en_EN.js",
-        auth=requests.auth.HTTPDigestAuth(door.username, door.password),
-        verify=False,
-    )
-    regex = re.compile(r'([0-9]+)="([^"]*)')
-    strings = [
-        regex.search(s)
-        for s in r.text.split(";")
-        if s.startswith("localeStrings.eventDetails")
-    ]
-    print({int(g.group(1)): g.group(2) for g in strings})
-
-
-@database.atomic()
-def getMessages(door):
-    last_event = (
-        HIDEvent.select(HIDEvent.timestamp)
-        .where(HIDEvent.doorName == door.name)
-        .order_by(HIDEvent.timestamp.desc())
-        .first()
-    )
-    if last_event is not None:
-        last_ts = last_event.timestamp
-    else:
-        last_ts = datetime(2010, 1, 1)
-
-    events = door.get_events(last_ts)
-
-    HIDEvent.insert_many(
-        [
-            {
-                # fill with None, then overwrite with real contents
-                **{k: None for k in HIDEvent._meta.fields.keys()},
-                **event.attrib,
-                "doorName": door.name,
-            }
-            for event in events
-        ]
-    ).on_conflict_ignore().execute()
-    return events
-
-
-def dups(events):
-    timestamps = [e.attrib["timestamp"] for e in events]
-    dups = set([x for x in timestamps if timestamps.count(x) > 1])
-    for e in events:
-        if e.attrib["timestamp"] in dups:
-            etree.dump(e)
-
-
-def main():
-    config = Config()
-    database.init(
-        **config.HID_DB,
-        **{
-            "charset": "utf8",
-            "sql_mode": "PIPES_AS_CONCAT",
-            "use_unicode": True,
-        },
-    )
-    HIDEvent.create_table()
-    for door in config.doors.values():
-        getMessages(door)
-
-
-if __name__ == "__main__":
-    main()

memberPlumbing/mw_models.py

@@ -1,249 +0,0 @@
-from datetime import datetime
-
-from peewee import (
-    BooleanField,
-    CompositeKey,
-    DateField,
-    DateTimeField,
-    DecimalField,
-    CharField,
-    FixedCharField,
-    ForeignKeyField,
-    Model,
-    MySQLDatabase,
-    TextField,
-)
-
-database = MySQLDatabase(None)
-
-
-class BaseModel(Model):
-    _csv_headers_override = {}
-    _date_fields = {}
-
-    def insert_instance(self):
-        return self.insert(**self.__data__)
-
-    def upsert_instance(self):
-        return self.insert_instance().on_conflict(
-            action="update", preserve=list(self._meta.fields.values())
-        )
-
-    def magic_save(self):
-        if self._meta.primary_key is False:
-            self.get_or_create(**self.__data__)
-        else:
-            self.upsert_instance().execute()
-
-    @classmethod
-    def _headers_map(cls):
-        return {field.column_name: name for name, field in cls._meta.fields.items()}
-
-    @classmethod
-    def _remap_headers(cls, data):
-        # print(data)
-        hmap = cls._headers_map()
-        hmap.update(cls._csv_headers_override)
-        for k, v in data.items():
-            if k in hmap:
-                yield hmap.get(k), v
-
-    @classmethod
-    def from_csv_dict(cls, data):
-        data = data.copy()
-
-        # parse date fields to datetime objects
-        for field, fmt in cls._date_fields.items():
-            if data[field]:
-                data[field] = datetime.strptime(str(data[field]), fmt)
-            else:
-                # convert empty string to None to make NULL in SQL
-                data[field] = None
-
-        return cls(**dict(cls._remap_headers(data)))
-
-    class Meta:
-        database = database
-
-
-# TODO: is this still a temporal table?
-class Member(BaseModel):
-    uid = FixedCharField(24, primary_key=True)
-    year_of_birth = TextField(column_name="Year of Birth", null=True)
-    account_name = TextField(column_name="Account Name", null=True)
-    first_name = TextField(column_name="First Name", null=True)
-    last_name = TextField(column_name="Last Name", null=True)
-    phone = TextField(column_name="Phone", null=True)
-    email = TextField(column_name="Email", null=True)
-    volunteer_email = TextField(column_name="Volunteer Email", null=True)
-    address_street = TextField(column_name="Address (Street)", null=True)
-    address_city = TextField(column_name="Address (City)", null=True)
-    address_state_province = TextField(
-        column_name="Address (State/Province)", null=True
-    )
-    address_postal_code = TextField(column_name="Address (Postal Code)", null=True)
-    address_country = TextField(column_name="Address (Country)", null=True)
-    profile_description = TextField(column_name="Profile description", null=True)
-    website = TextField(column_name="Website", null=True)
-    fax = TextField(column_name="Fax", null=True)
-    contact_person = TextField(column_name="Contact Person", null=True)
-    password = TextField(column_name="Password", null=True)
-    position_relation = TextField(column_name="Position/relation", null=True)
-    parent_account_id = TextField(column_name="Parent Account ID", null=True)
-    gift_membership_purchased_by = TextField(
-        column_name="Gift Membership purchased by", null=True
-    )
-    purchased_gift_membership_for = TextField(
-        column_name="Purchased Gift Membership for", null=True
-    )
-    closet_storage = TextField(column_name="Closet Storage #", null=True)
-    storage_shelf = TextField(column_name="Storage Shelf #", null=True)
-    personal_studio_space = TextField(column_name="Personal Studio Space #", null=True)
-    access_permitted_shops_during_extended_hours = BooleanField(
-        column_name="Access Permitted Shops During Extended Hours?"
-    )
-    normal_access_permitted_during_covid19_limited_operations = BooleanField(
-        column_name="Normal Access Permitted During COVID-19 Limited Operations"
-    )
-    access_permitted_during_covid19_staffed_period_only = BooleanField(
-        column_name="Access Permitted During COVID-19 Staffed Period Only"
-    )
-    access_front_door_and_studio_space_during_extended_hours = BooleanField(
-        column_name="Access Front Door and Studio Space During Extended Hours?"
-    )
-    access_wood_shop = BooleanField(column_name="Access Wood Shop?")
-    access_metal_shop = BooleanField(column_name="Access Metal Shop?")
-    access_storage_closet = BooleanField(column_name="Access Storage Closet?")
-    access_studio_space = BooleanField(column_name="Access Studio Space?")
-    access_front_door = BooleanField(column_name="Access Front Door?")
-    access_card_number = TextField(column_name="Access Card Number", null=True)
-    access_card_facility_code = TextField(
-        column_name="Access Card Facility Code", null=True
-    )
-    auto_billing_id = TextField(column_name="Auto Billing ID", null=True)
-    billing_method = TextField(column_name="Billing Method", null=True)
-    renewal_date = DateField(column_name="Renewal Date", null=True)
-    join_date = DateField(column_name="Join Date", null=True)
-    admin_note = TextField(column_name="Admin note", null=True)
-    profile_gallery_image_url = TextField(
-        column_name="Profile gallery image URL", null=True
-    )
-    business_card_image_url = TextField(
-        column_name="Business card image URL", null=True
-    )
-    instagram = TextField(column_name="Instagram", null=True)
-    pinterest = TextField(column_name="Pinterest", null=True)
-    youtube = TextField(column_name="Youtube", null=True)
-    yelp = TextField(column_name="Yelp", null=True)
-    google = TextField(column_name="Google+", null=True)
-    bbb = TextField(column_name="BBB", null=True)
-    twitter = TextField(column_name="Twitter", null=True)
-    facebook = TextField(column_name="Facebook", null=True)
-    linked_in = TextField(column_name="LinkedIn", null=True)
-    do_not_show_street_address_in_profile = TextField(
-        column_name="Do not show street address in profile", null=True
-    )
-    do_not_list_in_directory = TextField(
-        column_name="Do not list in directory", null=True
-    )
-    how_did_you_hear = TextField(column_name="HowDidYouHear", null=True)
-    authorize_charge = TextField(column_name="authorizeCharge", null=True)
-    policy_agreement = TextField(column_name="policyAgreement", null=True)
-    waiver_form_signed_and_on_file_date = DateField(
-        column_name="Waiver form signed and on file date.", null=True
-    )
-    membership_agreement_signed_and_on_file_date = DateField(
-        column_name="Membership Agreement signed and on file date.", null=True
-    )
-    ip_address = TextField(column_name="IP Address", null=True)
-    audit_date = DateField(column_name="Audit Date", null=True)
-    agreement_version = TextField(column_name="Agreement Version", null=True)
-    paperwork_status = TextField(column_name="Paperwork status", null=True)
-    membership_agreement_dated = BooleanField(column_name="Membership agreement dated")
-    membership_agreement_acknowledgement_page_filled_out = BooleanField(
-        column_name="Membership Agreement Acknowledgement Page Filled Out"
-    )
-    membership_agreement_signed = BooleanField(
-        column_name="Membership Agreement Signed"
-    )
-    liability_form_filled_out = BooleanField(column_name="Liability Form Filled Out")
-    self_certify_essential_business = BooleanField(
-        column_name="selfCertifyEssentialBusiness"
-    )
-    accepted_covid19_policy = BooleanField(column_name="Accepted COVID-19 Policy")
-
-    _csv_headers_override = {
-        "Account ID": "uid",
-        "Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:": "how_did_you_hear",
-        "Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.": "authorize_charge",
-        "I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.": "policy_agreement",
-        "Access Permitted Using Membership Level Schedule During COVID-19 Limited Operations": "normal_access_permitted_during_covid19_limited_operations",
-        "I hereby certify that I am involved in Essential Business as defined by the State of New Hampshire and that I will follow the practices identified in the State of New Hampshire Exhibit C to Emergency Order #40 (Section E. Manufacturing).": "self_certify_essential_business",
-        "I have read, and agree to abide by the terms, conditions, policies and procedures  contained in the Claremont MakerSpace COVID-19 Policies and Procedures.": "accepted_covid19_policy",
-    }
-
-    _date_fields = {
-        "Join Date": "%b %d, %Y",
-        "Renewal Date": "%b %d, %Y",
-        "Audit Date": "%m/%d/%Y",
-        "Membership Agreement signed and on file date.": "%m/%d/%Y",
-        "Waiver form signed and on file date.": "%m/%d/%Y",
-    }
-
-    class Meta:
-        table_name = "members"
-
-
-class Flag(BaseModel):
-    id = FixedCharField(24, primary_key=True)
-    name = TextField(null=True)
-    type = CharField(6)
-
-
-class MemberFlag(BaseModel):
-    uid = ForeignKeyField(Member, column_name="uid", backref="flags")
-    flag_id = ForeignKeyField(Flag, backref="members")
-
-    class Meta:
-        primary_key = CompositeKey("flag_id", "uid")
-
-
-class Transaction(BaseModel):
-    sid = FixedCharField(27, null=True)
-    uid = ForeignKeyField(Member, column_name="uid", backref="transactions", null=True)
-    timestamp = DateTimeField()
-    type = TextField(null=True)
-    sum = DecimalField(13, 4, null=True)
-    fee = DecimalField(13, 4, null=True)
-    event_id = TextField(null=True)
-    for_ = TextField(column_name="For", null=True)
-    items = TextField(column_name="Items", null=True)
-    discount_code = TextField(column_name="Discount Code", null=True)
-    note = TextField(column_name="Note", null=True)
-    name = TextField(column_name="Name", null=True)
-    contact_person = TextField(column_name="Contact Person", null=True)
-    full_address = TextField(column_name="Full Address", null=True)
-    street = TextField(column_name="Street", null=True)
-    city = TextField(column_name="City", null=True)
-    state_province = TextField(column_name="State/Province", null=True)
-    postal_code = TextField(column_name="Postal Code", null=True)
-    country = TextField(column_name="Country", null=True)
-    phone = TextField(column_name="Phone", null=True)
-    email = TextField(column_name="Email", null=True)
-
-    _csv_headers_override = {
-        "_dp": "timestamp",
-        "Transaction Type": "type",
-        "Event/Form Name": "for_",
-    }
-
-    @classmethod
-    def from_csv_dict(cls, data):
-        txn = data.copy()
-        # can't use '%s' format string, have to use the special function
-        txn["_dp"] = datetime.fromtimestamp(txn["_dp"])
-        return super().from_csv_dict(txn)
-
-    class Meta:
-        table_name = "transactions"
-        primary_key = False

memberPlumbing/sqlExport.py

@@ -1,100 +0,0 @@
-#!/usr/bin/env python3
-
-from datetime import datetime
-
-from .config import Config
-from .mw_models import Member, Flag, MemberFlag, Transaction, database
-
-
-@database.atomic()
-def do_import(config):
-    membershipworks = config.membershipworks
-    print("Creating tables")
-    database.create_tables([Member, Flag, MemberFlag, Transaction])
-
-    print("Updating flags (labels, levels, and addons)")
-    flags = membershipworks._parse_flags()
-    Flag.insert_many(
-        [
-            {"id": v, "name": k, "type": typ[:-1]}
-            for typ, flags_of_type in flags.items()
-            for k, v in flags_of_type.items()
-        ]
-    ).on_conflict(action="update", preserve=[Flag.name, Flag.type]).execute()
-
-    print("Getting folder membership...")
-    folders = {
-        folder_id: membershipworks.get_member_ids([folder_name])
-        for folder_name, folder_id in membershipworks._parse_flags()["folders"].items()
-    }
-
-    print("Getting/Updating members...")
-    members = membershipworks.get_all_members()
-    for m in members:
-        # replace flags by booleans
-        for flag in [dek["lbl"] for dek in membershipworks.org_info["dek"]]:
-            if flag in m:
-                m[flag] = m[flag] == flag
-
-        for field_id, field in membershipworks._all_fields().items():
-            # convert checkboxes to real booleans
-            if field.get("typ") == 8 and field["lbl"] in m:  # check box
-                m[field["lbl"]] = True if m[field["lbl"]] == "Y" else False
-
-    for member in members:
-        # create/update member
-        Member.from_csv_dict(member).magic_save()
-
-        # update member's flags
-        for type, flags in membershipworks._parse_flags().items():
-            for flag, id in flags.items():
-                ml = MemberFlag(uid=member["Account ID"], flag_id=id)
-                if (type == "folders" and member["Account ID"] in folders[id]) or (
-                    type != "folders" and member[flag]
-                ):
-                    ml.magic_save()
-                else:
-                    ml.delete_instance()
-
-    print("Getting/Updating transactions...")
-    # Deduping these is hard, so just recreate the data every time
-    Transaction.truncate_table()
-    now = datetime.now()
-    start_date = datetime(2010, 1, 1)
-    transactions_csv = membershipworks.get_transactions(start_date, now)
-    transactions_json = membershipworks.get_transactions(start_date, now, json=True)
-    # this is terrible, but as long as the dates are the same, should be fiiiine
-    transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
-    assert all(
-        [
-            t["Account ID"] == t.get("uid", "") and t["Payment ID"] == t.get("sid", "")
-            for t in transactions
-        ]
-    )
-
-    Transaction.insert_many(
-        [
-            Transaction.from_csv_dict(transaction).__data__
-            for transaction in transactions
-        ]
-    ).execute()
-
-    # TODO: folders, levels, addons
-
-
-def main():
-    config = Config()
-    database.init(
-        **config.MEMBERSHIPWORKS_DB,
-        **{
-            "charset": "utf8",
-            "sql_mode": "PIPES_AS_CONCAT",
-            "use_unicode": True,
-        },
-    )
-
-    do_import(config)
-
-
-if __name__ == "__main__":
-    main()

memberPlumbing/ucsAccounts.py

@@ -1,131 +0,0 @@
-#!/usr/bin/env python3
-import asyncio
-import random
-import re
-import string
-
-from udm_rest_client.udm import UDM
-from udm_rest_client.exceptions import NoObject, UdmError
-
-from .config import Config
-
-USER_BASE = "cn=users,dc=sawtooth,dc=claremontmakerspace,dc=org"
-GROUP_BASE = "cn=groups,dc=sawtooth,dc=claremontmakerspace,dc=org"
-GROUPS_REGEX = "|".join(
-    ["Certified: .*", "Access .*\?", "CMS .*", "Volunteer: .*", "Database .*"]
-)
-RAND_PW_LEN = 20
-
-
-# From an API error message:
-#   A group name must start and end with a letter, number or underscore.
-#   In between additionally spaces, dashes and dots are allowed.
-def sanitize_group_name(name):
-    sanitized_body = re.sub(r"[^0-9A-Za-z_ -.]", ".", name)
-    sanitized_start_end = re.sub("^[^0-9A-Za-z_]|[^0-9A-Za-z_]$", "_", sanitized_body)
-
-    return "MW_" + sanitized_start_end
-
-
-# From an API error message: "Username must only contain numbers, letters and dots!"
-def sanitize_user_name(name):
-    return re.sub(r"[^0-9a-z.]", ".", name.lower()).strip(".")
-
-
-async def make_groups(group_mod, members):
-    existing_group_names = [g.props.name async for g in group_mod.search()]
-
-    groups = [
-        sanitize_group_name(group_name)
-        for group_name in members[0].keys()
-        if re.match(GROUPS_REGEX, group_name) is not None
-    ]
-    for group_name in groups:
-        if group_name not in existing_group_names:
-            group = await group_mod.new()
-            group.props.name = group_name
-            await group.save()
-
-
-async def _main():
-    config = Config()
-
-    members = config.membershipworks.get_members(
-        ["Members", "CMS Staff"], "lvl,phn,eml,lbl,nam,end,_id"
-    )
-
-    async with UDM(**config.UCS) as udm:
-        user_mod = udm.get("users/user")
-        group_mod = udm.get("groups/group")
-
-        await make_groups(group_mod, members)
-
-        for member in members:
-            username = sanitize_user_name(member["Account Name"])
-
-            try:  # try to get an existing user to update
-                user = await user_mod.get(f"uid={username},{USER_BASE}")
-            except NoObject:  # create a new user
-                # TODO: search by employeeNumber and rename users when needed
-                user = await user_mod.new()
-
-                # set a random password and ensure it is changed at next login
-                user.props.password = "".join(
-                    random.choice(string.ascii_letters + string.digits)
-                    for x in range(0, RAND_PW_LEN)
-                )
-                user.props.pwdChangeNextLogin = True
-
-            user.props.update(
-                {
-                    "title": "",  # Title
-                    "firstname": member["First Name"],
-                    "lastname": member["Last Name"],  # (c)
-                    "username": username,  # (cmr)
-                    "description": "",  # Description
-                    # "password": "", # (c) Password
-                    # "mailPrimaryAddress": member["Email"], # Primary e-mail address
-                    # "displayName": "", # Display name
-                    # "birthday": "", # Birthdate
-                    # "jpegPhoto": "", # Picture of the user (JPEG format)
-                    "employeeNumber": member["Account ID"],
-                    # "employeeType": "", # Employee type
-                    "homedrive": "H:",  # Windows home drive
-                    "sambahome": "\\\\ucs\\" + username,  # Windows home path
-                    "profilepath": "%LOGONSERVER%\\%USERNAME%\\windows-profiles\\default",  # Windows profile directory
-                    "disabled": member["Account on Hold"] != "",
-                    # "userexpiry": member["Renewal Date"],
-                    # "pwdChangeNextLogin": "1", # User has to change password on next login
-                    # "sambaLogonHours": "", # Permitted times for Windows logins
-                    "e-mail": [member["Email"]],  # ([]) E-mail address
-                    "phone": [member["Phone"]],  # Telephone number
-                    # "PasswordRecoveryMobile": member["Phone"], # Mobile phone number
-                    "PasswordRecoveryEmail": member["Email"],
-                }
-            )
-
-            new_groups = [
-                "cn=" + sanitize_group_name(group) + "," + GROUP_BASE
-                for group, value in member.items()
-                if re.match(GROUPS_REGEX, group) is not None and value != ""
-            ]
-            # groups not from this script
-            other_old_groups = [
-                g for g in user.props.groups if not g[3:].startswith("MW_")
-            ]
-            user.props.groups = other_old_groups + new_groups
-
-            try:
-                await user.save()
-            except UdmError:
-                print("Failed to save user", username)
-                print(user.props)
-                raise
-
-
-def main():
-    asyncio.run(_main())
-
-
-if __name__ == "__main__":
-    main()

memberPlumbing/upcomingEvents.py

@@ -1,162 +0,0 @@
-#!/usr/bin/env python3
-
-from datetime import datetime
-import sys
-
-import pyclip
-
-from .config import Config
-
-TIME_FMT = "%l:%M%P"
-DATETIME_FMT = "%a %b %e %Y, " + TIME_FMT
-
-
-def format_datetime_range(start_ts: int, end_ts: int):
-    start = datetime.fromtimestamp(start_ts)
-    end = datetime.fromtimestamp(end_ts)
-    start_str = start.strftime(DATETIME_FMT)
-    if start.date() == end.date():
-        end_str = end.strftime(TIME_FMT)
-    else:
-        # TODO: this probably implies multiple instances. Should read
-        #       RRULE or similar from the event notes
-        end_str = end.strftime(DATETIME_FMT)
-    return f"{start_str} — {end_str}"
-
-
-def format_event(event_details, truncate: bool):
-    try:
-        url = (
-            "https://claremontmakerspace.org/events/#!event/register/"
-            + event_details["url"]
-        )
-        if "lgo" in event_details:
-            img = f"""<img class="alignleft" width="400" src="{event_details['lgo']['l']}">"""
-        else:
-            img = ""
-        # print(json.dumps(event_details))
-        out = f"""<h2 style="text-align: center;">
-  <a href="{url}">{img}{event_details['ttl']}</a>
-</h2>
-<div><i>{format_datetime_range(event_details['sdp'], event_details['edp'])}</i></div>
-"""
-        if not truncate:
-            out += f"""
-<div>
-  {event_details['dtl']}
-</div>
-
-<a href="{url}">Register for this class now!</a>"""
-        return out
-    except KeyError as e:
-        print(
-            f"Event '{event_details.get('ttl')}' missing required property: '{e.args[0]}'"
-        )
-        raise
-
-
-def format_section(title: str, blurb: str, events, truncate: bool):
-    # skip empty sections
-    if not events:
-        return ""
-
-    events_list = "\n<hr />\n\n".join(format_event(event, truncate) for event in events)
-
-    return f"""<h1>{title}</h1>
-<h4><i>{blurb}</i></h4>
-{events_list}
-"""
-
-
-def generate_post():
-    config = Config()
-    now = datetime.now()
-
-    membershipworks = config.membershipworks
-    events = membershipworks.get_events_list(now)
-    if "error" in events:
-        print("Error:", events["error"])
-        return
-
-    ongoing_events = []
-    full_events = []
-    upcoming_events = []
-    for event in events["evt"]:
-        try:
-            # ignore hidden events
-            if event["cal"] == 0:
-                continue
-            event_details = membershipworks.get_event_by_eid(event["eid"])
-
-            # registration has already ended
-            if (
-                "erd" in event_details
-                and datetime.fromtimestamp(event_details["erd"]) < now
-            ):
-                ongoing_events.append(event_details)
-            # class is full
-            elif event_details["cnt"] >= event_details["cap"]:
-                full_events.append(event_details)
-            else:
-                upcoming_events.append(event_details)
-        except KeyError as e:
-            print(
-                f"Event '{event.get('ttl')}' missing required property: '{e.args[0]}'"
-            )
-            raise
-
-    # header
-    yield """<p><img class="aligncenter size-medium wp-image-2319" src="https://claremontmakerspace.org/wp-content/uploads/2019/03/CMS-Logo-b-y-g-300x168.png" alt="" width="300" height="168" /></a></p>
-<p>Greetings Upper Valley Makers:</p>
-<p>We have an exciting list of upcoming classes at the Claremont MakerSpace that we think might interest you.</p>
-<strong>For most classes and events, CMS MEMBERSHIP IS NOT REQUIRED.</strong> That said, members receive a discount on registration and there are some classes/events that are for members only (this will be clearly noted in the event description).
-
-<strong>Class policies</strong> (liability waiver, withdrawal, cancellation, etc.) can be found <a href="https://claremontmakerspace.org/class-policies/" data-wpel-link="internal">here</a>.
-
-<strong>Instructors:</strong> Interested in teaching a class at CMS? Please fill out our <a href="https://claremontmakerspace.org/cms-class-proposal-form/" data-wpel-link="internal">Class Proposal Form</a>.
-
-<strong>Tours:</strong> Want to see what the Claremont MakerSpace is all about? Tours are by appointment only. <a href="https://tickets.claremontmakerspace.org/open.php" target="_blank" rel="noreferrer noopener external" data-wpel-link="external">Contact Us</a> to schedule your tour where you can learn about all the awesome tools that the CMS offers access to, as well as how membership, classes, and studio spaces work.
-
-<hr />
-"""
-
-    yield format_section(
-        "Upcoming Events",
-        "Events that are currently open for registration.",
-        upcoming_events,
-        truncate=False,
-    )
-
-    yield format_section(
-        "<hr />Just Missed",
-        "These classes are currently full at time of writing. If you are interested, please check the event's page; spots occasionally open up. Keep an eye on this newsletter to see when these classes are offered again.",
-        full_events,
-        truncate=True,
-    )
-
-    yield format_section(
-        "<hr />Ongoing Events",
-        "These events are ongoing. Registration is currently closed, but these events may be offered again in the future.",
-        ongoing_events,
-        truncate=True,
-    )
-
-    # footer
-    yield """<div style="clear: both;">
-  <hr />
-
-  <div>Happy Makin’!</div>
-  <div>We are grateful for all of the public support that our 501(c)(3), non-profit organization receives. If you’d like to make a donation,please visit the <a href="https://claremontmakerspace.org/support/"><strong>Support Us page</strong></a> of our website.</div>
-</div>
-"""
-
-
-def main():
-    result = "\n".join(generate_post())
-    print(result)
-    pyclip.copy(result)
-    print("Copied to clipboard!", file=sys.stderr)
-
-
-if __name__ == "__main__":
-    main()

membershipViewer.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+import re
+import http
+
+from flask import Flask, render_template, request
+
+app = Flask(__name__)
+
+from common import doors, membershipworks
+
+def parse_list(member, regex):
+    data_list = []
+    for key, value in member.items():
+        match = re.match(regex, key)
+        if match is not None and value != '':
+            data_list.append(match.group(1))
+
+    return ", ".join(data_list)
+
+def parse_members(members):
+    data = []
+    for member in members:
+        props = {
+            'Name': member['Account Name'],
+            'Renewal Date': member['Renewal Date'],
+            'Card Number': member['Access Card Facility Code'] + '-' \
+                         + member['Access Card Number'],
+            'Account on Hold': "Yes" if member['Account on Hold'] != '' else "No" }
+
+        props['Certifications'] = parse_list(member, 'Certified: (.*)')
+        props['Door Access'] = parse_list(member, 'Access (.*)\?')
+        props['Memebership Level'] = parse_list(member, 'CMS (.*)')
+
+        data.append(props)
+
+    return data
+
+@app.route("/")
+def main():
+    # maybe not now: membership agreement signed
+    # TODO: renewal date check
+    term = request.args.get('term', '')
+
+    if len(term) < 3:
+        return render_template("members.html",
+                               error="Enter at least 3 characters to search")
+
+    data = membershipworks.get_members(
+        ['members', 'staff'],
+        "lvl,xws,xms,xsc,xas,xfd,xac,phn,eml,lbl,xcf,nam,end")
+
+    members = parse_members(data)
+    members = [member for member in members
+               if term.lower() in member['Name'].lower()]
+    headers = ['Name', 'Certifications', 'Door Access', 'Memebership Level',
+               'Card Number', 'Renewal Date', 'Account on Hold']
+
+    if len(members) > 4:
+        return render_template(
+            "members.html", error="Too many results, please be more specific.")
+
+    return render_template("members.html", headers=headers, members=members)
+
+@app.route('/frontDoor/<lock>', methods=['POST'])
+def unlockLockDoor(lock):
+    doors['Front Door'].lockOrUnlockDoor(lock != 'unlock')
+    return ('', http.HTTPStatus.NO_CONTENT)
+
+if __name__ == "__main__":
+    app.run(debug=True, host='0.0.0.0')

pdm.toml

@@ -1,2 +0,0 @@
-[strategy]
-save = "compatible"

pyproject.toml

@@ -1,52 +0,0 @@
-[project]
-name = "memberPlumbing"
-version = "0.1.0"
-description = ""
-authors = [
-    {name = "Adam Goldsmith", email = "adam@adamgoldsmith.name"},
-]
-requires-python = ">=3.11,<4.0"
-dependencies = [
-    "requests~=2.31",
-    "ruamel-yaml~=0.17",
-    "bitstring~=4.1",
-    "lxml~=4.9",
-    "peewee~=3.16",
-    "mysqlclient~=2.1",
-    "udm-rest-client~=1.2",
-    "pyclip~=0.7",
-    "recurrent~=0.4",
-]
-
-[tool.pdm]
-[tool.pdm.dev-dependencies]
-dev = [
-    "black~=23.3",
-    "isort~=5.11",
-]
-
-[tool.pdm.build]
-includes = [
-    "memberPlumbing",
-]
-
-[build-system]
-requires = ["pdm-backend"]
-build-backend = "pdm.backend"
-
-[project.scripts]
-doorUpdater = "memberPlumbing.doorUpdater:main"
-hidEvents = "memberPlumbing.hidEvents:main"
-sqlExport = "memberPlumbing.sqlExport:main"
-ucsAccounts = "memberPlumbing.ucsAccounts:main"
-upcomingEvents = "memberPlumbing.upcomingEvents:main"
-
-[tool.black]
-line-length = 88
-
-[tool.isort]
-multi_line_output = 3
-include_trailing_comma = true
-force_grid_wrap = 0
-use_parentheses = true
-line_length = 88

renovate.json

@@ -1,6 +0,0 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:base"
-  ]
-}

schedules.xml

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<VertXMessage  xmlns:hid="http://www.hidglobal.com/VertX">
+  <hid:Schedules action="AD">
+    <hid:Schedule scheduleID="1" scheduleName="7x24">
+      <hid:DayOfWeekInterval dayOfWeek="0" startTime="00:00:00" endTime="23:59:59"/>
+      <hid:DayOfWeekInterval dayOfWeek="1" startTime="00:00:00" endTime="23:59:59"/>
+      <hid:DayOfWeekInterval dayOfWeek="2" startTime="00:00:00" endTime="23:59:59"/>
+      <hid:DayOfWeekInterval dayOfWeek="3" startTime="00:00:00" endTime="23:59:59"/>
+      <hid:DayOfWeekInterval dayOfWeek="4" startTime="00:00:00" endTime="23:59:59"/>
+      <hid:DayOfWeekInterval dayOfWeek="5" startTime="00:00:00" endTime="23:59:59"/>
+      <hid:DayOfWeekInterval dayOfWeek="6" startTime="00:00:00" endTime="23:59:59"/>
+    </hid:Schedule>
+
+    <hid:Schedule scheduleID="2" scheduleName="Weekends Only">
+      <hid:DayOfWeekInterval dayOfWeek="0" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="6" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:SpecialDayInterval dayOfMonth="1" month="1" specialDayID="3" specialDayName="New Years Day" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:SpecialDayInterval dayOfMonth="25" month="12" specialDayID="2" specialDayName="Christmas Day" startTime="08:00:00" endTime="21:00:00"/>
+    </hid:Schedule>
+
+    <hid:Schedule scheduleID="3" scheduleName="Weekdays Only">
+      <hid:DayOfWeekInterval dayOfWeek="1" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="2" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="3" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="4" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="5" startTime="08:00:00" endTime="21:00:00"/>
+    </hid:Schedule>
+
+    <hid:Schedule scheduleID="4" scheduleName="Unlimited">
+      <hid:DayOfWeekInterval dayOfWeek="0" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="1" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="2" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="3" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="4" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="5" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="6" startTime="08:00:00" endTime="21:00:00"/>
+    </hid:Schedule>
+
+    <hid:Schedule scheduleID="5" scheduleName="Nights and Weekends">
+      <hid:DayOfWeekInterval dayOfWeek="0" startTime="08:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="1" startTime="15:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="2" startTime="15:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="3" startTime="15:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="4" startTime="15:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="5" startTime="15:00:00" endTime="21:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="6" startTime="08:00:00" endTime="21:00:00"/>
+    </hid:Schedule>
+
+    <hid:Schedule scheduleID="6" scheduleName="Front Door Unlocker">
+      <hid:DayOfWeekInterval dayOfWeek="0" startTime="08:00:00" endTime="17:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="1" startTime="08:00:00" endTime="17:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="2" startTime="08:00:00" endTime="17:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="3" startTime="08:00:00" endTime="17:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="4" startTime="08:00:00" endTime="17:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="5" startTime="08:00:00" endTime="17:00:00"/>
+      <hid:DayOfWeekInterval dayOfWeek="6" startTime="08:00:00" endTime="17:00:00"/>
+    </hid:Schedule>
+  </hid:Schedules>
+
+  <hid:SpecialDays action="UD">
+    <hid:SpecialDay specialDayID="1" specialDayName="New Years Day" dayOfMonth="1" month="1"/>
+    <hid:SpecialDay specialDayID="2" specialDayName="Independance Day" dayOfMonth="4" month="7"/>
+    <hid:SpecialDay specialDayID="3" specialDayName="Christmas Day" dayOfMonth="25" month="12"/>
+  </hid:SpecialDays>
+</VertXMessage>

sqlExport.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+import MySQLdb
+import yaml
+
+from common import membershipworks
+from passwords import MEMBERSHIPWORKS_DB
+
+def resolveSource(key, value):
+    if type(value) == str:
+        return value
+    elif type(value) == dict and 'source' in value:
+        return value['source']
+    else:
+        return key
+
+def formatRows(tableMap, data):
+    for d in data:
+        yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
+
+def insertFromTableMap(table, data, tableMap):
+    # TODO: this could probably be done better as a single statement?
+    c.executemany(
+        f"""INSERT INTO {table} ({','.join(f'`{k}`' for k in tableMap.keys())})
+        VALUES ({','.join(len(tableMap) * ['%s'])})
+        ON DUPLICATE KEY UPDATE
+        {', '.join(f'`{k}`=VALUES(`{k}`)' for k in tableMap.keys())};""",
+        list(formatRows(tableMap, data)))
+
+def insertDistinctFromTableMap(table, data, tableMap):
+    # TODO: this could probably be done better as a single statement?
+    c.executemany(
+       f"""INSERT INTO {table} ({','.join(f'`{k}`' for k in tableMap.keys())})
+        SELECT {','.join(len(tableMap) * ['%s'])} FROM DUAL
+        WHERE NOT EXISTS (
+            SELECT 1 from {table}
+            WHERE {' AND '.join(f'`{k}`<=>%s' for k in tableMap.keys())}
+        );""" ,
+        list([r * 2 for r in formatRows(tableMap, data)]))
+
+
+# TODO: delete non-valid labels
+def insertLabels(members):
+    for member in members:
+        for label, label_id in membershipworks._parse_flags()['labels'].items():
+            if member[label]:
+                c.execute("""
+                INSERT INTO member_labels (uid, label_id) VALUES (%s, %s)
+                ON DUPLICATE KEY UPDATE
+                  uid=VALUES(uid), label_id=VALUES(label_id);""",
+                          (member['Account ID'], label_id))
+            else:
+                c.execute('DELETE FROM member_labels WHERE uid=%s && label_id=%s;',
+                          (member['Account ID'], label_id))
+
+
+with open('tableMapping.yaml') as f:
+    tableMapping = yaml.load(f, yaml.SafeLoader)
+
+
+conn =  MySQLdb.connect(**MEMBERSHIPWORKS_DB)
+conn.set_character_set('utf8')
+c = conn.cursor()
+
+def createDefinitionsFromTableMap(tableMap):
+    def resolveColType(value):
+        if type(value) == dict and 'type' in value:
+            return value['type']
+        else:
+            return 'TEXT'
+
+    return ', '.join([f'`{k}` ' + resolveColType(v)
+                      for k, v in tableMap.items()])
+
+try:
+    print("Creating tables...")
+    c.execute('CREATE TABLE IF NOT EXISTS members (' +
+              createDefinitionsFromTableMap(tableMapping['members']) +
+              ') WITH SYSTEM VERSIONING;')
+
+    c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
+              createDefinitionsFromTableMap(tableMapping['transactions']) +
+              ", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));")
+              #-- FOREIGN KEY event_id REFERENCES event eid
+
+    c.execute("""CREATE TABLE IF NOT EXISTS labels (
+    label_id CHAR(24) PRIMARY KEY, label TEXT
+    ) WITH SYSTEM VERSIONING;""")
+
+    c.execute("""CREATE TABLE IF NOT EXISTS member_labels (
+    uid CHAR(24), label_id CHAR(24),
+    PRIMARY KEY(uid, label_id),
+    FOREIGN KEY(uid) REFERENCES members(uid),
+    FOREIGN KEY(label_id) REFERENCES labels(label_id)
+    ) WITH SYSTEM VERSIONING;""")
+
+    print("Updating labels")
+    c.executemany("""INSERT INTO labels (label, label_id) VALUES (%s, %s)
+    ON DUPLICATE KEY UPDATE
+    label=VALUES(label), label_id=VALUES(label_id);""",
+                  membershipworks._parse_flags()['labels'].items())
+
+    print("Getting/Updating members...")
+    members = membershipworks.get_all_members()
+    for m in members:
+        # replace flags by booleans
+        for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
+            if flag in m:
+                m[flag] = m[flag] == flag
+
+        for field_id, field in membershipworks._all_fields().items():
+            # convert checkboxes to real booleans
+            if field.get('typ') == 8 and field['lbl'] in m: # check box
+                m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
+
+    insertFromTableMap('members', members, tableMapping['members'])
+
+    insertLabels(members)
+
+    print("Getting/Updating transactions...")
+    now = datetime.now()
+    transactions_csv = membershipworks.get_transactions(datetime(2010, 1, 1), now)
+    transactions_json = membershipworks.get_transactions(
+        datetime(2010, 1, 1), now, json=True)
+    # this is terrible, but as long as the dates are the same, should be fiiiine
+    transactions = [{**j, **v}
+                    for j, v in zip(transactions_csv, transactions_json)]
+    assert all([t['Account ID'] == t.get('uid', '')
+                and t['Payment ID'] == t.get('sid', '')
+                for t in transactions])
+    insertDistinctFromTableMap(
+        'transactions', transactions, tableMapping['transactions'])
+
+    print("Committing changes...")
+    conn.commit()
+
+except Exception as e:
+    print("Transaction failed, rolling back")
+    conn.rollback();
+    raise e
+
+finally:
+    conn.close()
+
+# TODO: folders, levels, addons

systemd/doorUpdater.service

@@ -5,6 +5,5 @@ OnFailure=status-email-admin@%n.service
 [Service]
 User=adam
 Type=oneshot
-TimeoutStartSec=600
-WorkingDirectory=/home/adam/memberPlumbing/
-ExecStart=/usr/bin/python3 -u /home/adam/.poetry/bin/poetry run doorUpdater
+WorkingDirectory=/home/adam/hidDoorWriter/
+ExecStart=/home/adam/hidDoorWriter/doorUpdater.py

systemd/hidEvents.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=Pull events from door controllers into local database
-OnFailure=status-email-admin@%n.service
-After=mariadb.service
-Requires=mariadb.service
-
-[Service]
-User=adam
-Type=oneshot
-TimeoutStartSec=600
-WorkingDirectory=/home/adam/memberPlumbing/
-ExecStart=/usr/bin/python3 -u /home/adam/.poetry/bin/poetry run hidEvents

systemd/hidEvents.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=Hourly door controller events sync
-
-[Timer]
-OnCalendar=hourly
-Persistent=true
-
-[Install]
-WantedBy=timers.target

systemd/membershipworksSQL.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=Sync Membershipworks with local database
-OnFailure=status-email-admin@%n.service
-After=mariadb.service
-Requires=mariadb.service
-
-[Service]
-User=adam
-Type=oneshot
-TimeoutStartSec=600
-WorkingDirectory=/home/adam/memberPlumbing/
-ExecStart=/usr/bin/python3 -u /home/adam/.poetry/bin/poetry run sqlExport

systemd/membershipworksSQL.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=Hourly Membershipworks database sync
-
-[Timer]
-OnCalendar=hourly
-Persistent=true
-
-[Install]
-WantedBy=timers.target

systemd/ucsAccounts.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=Update UCS Accounts
-OnFailure=status-email-admin@%n.service
-
-[Service]
-User=adam
-Type=oneshot
-TimeoutStartSec=600
-WorkingDirectory=/home/adam/memberPlumbing/
-ExecStart=/usr/bin/python3 -u /home/adam/.poetry/bin/poetry run ucsAccounts

systemd/ucsAccounts.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=Hourly UCS Accounts update
-
-[Timer]
-OnCalendar=*:0/15
-Persistent=true
-
-[Install]
-WantedBy=timers.target

tableMapping.yaml

@@ -0,0 +1,102 @@
+members:
+  'uid': {type: 'CHAR(24) PRIMARY KEY', source: 'Account ID'}
+  'Year of Birth':
+  'Account Name':
+  'First Name':
+  'Last Name':
+  'Phone':
+  'Email':
+
+  'Address (Street)':
+  'Address (City)':
+  'Address (State/Province)':
+  'Address (Postal Code)':
+  'Address (Country)':
+  'Profile description':
+  'Website':
+  'Fax':
+  'Contact Person':
+  'Password':
+  'Position/relation':
+
+  'Parent Account ID':
+
+  'Gift Membership purchased by':
+  'Purchased Gift Membership for':
+
+  'Closet Storage #':
+  'Storage Shelf #':
+  'Personal Studio Space #':
+
+  'Access Permitted Shops During Extended Hours?': {type: 'BOOLEAN'}
+  'Access Front Door and Studio Space During Extended Hours?': {type: 'BOOLEAN'}
+  'Access Wood Shop?': {type: 'BOOLEAN'}
+  'Access Metal Shop?': {type: 'BOOLEAN'}
+  'Access Storage Closet?': {type: 'BOOLEAN'}
+  'Access Studio Space?': {type: 'BOOLEAN'}
+  'Access Front Door?': {type: 'BOOLEAN'}
+  'Access Card Number':
+  'Access Card Facility Code':
+
+  'Auto Billing ID':
+  'Billing Method':
+  'Renewal Date':
+  'Join Date':
+
+  'Admin note':
+
+  'Profile gallery image URL':
+  'Business card image URL':
+  'Instagram':
+  'Pinterest':
+  'Youtube':
+  'Yelp':
+  'Google+':
+  'BBB':
+  'Twitter':
+  'Facebook':
+  'LinkedIn':
+
+  'Do not show street address in profile':
+  'Do not list in directory':
+
+  'HowDidYouHear': 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:'
+  'authorizeCharge': 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.'
+  'policyAgreement': 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.'
+  'Waiver form signed and on file date.':
+  'Membership Agreement signed and on file date.':
+  'Agreement Version':
+  'Paperwork status':
+  'Membership agreement dated': {type: BOOLEAN}
+  'Membership Agreement Acknowledgement Page Filled Out': {type: BOOLEAN}
+  'Membership Agreement Signed': {type: BOOLEAN}
+  'Liability Form Filled Out': {type: BOOLEAN}
+  'Audit Date':
+
+  'IP Address':
+
+transactions:
+  'sid': {type: CHAR(27)}
+  'uid': {type: CHAR(24)}
+  'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp?
+  'type': {source: 'Transaction Type'}
+  'sum': {type: 'DECIMAL(13,4)'}
+  'fee': {type: 'DECIMAL(13,4)'}
+  'event_id': {source: 'eid'}
+  'For':
+  'Items':
+  'Discount Code':
+  'Note':
+
+  # this is painful, but necessary because some users have no uid
+  # TODO: fix this horribleness
+  'Name':
+  'Contact Person':
+  'Full Address':
+  'Street':
+  'City':
+  'State/Province':
+  'Postal Code':
+  'Country':
+  'Phone':
+  'Email':

templates/members.html

@@ -0,0 +1,54 @@
+<html>
+  <head>
+    <title> CMS Members </title>
+    <style>
+     table {
+       border-collapse: collapse;
+     }
+
+     td, th{
+       border: 1px solid black;
+     }
+
+     td {
+       padding: 3px;
+     }
+
+     tr:nth-child(even) {
+       background-color: #ccc;
+     }
+
+     tr[onHold=Yes] {
+       background-color: #f66;
+     }
+    </style>
+  </head>
+  <body>
+    <form action="./frontDoor/lock" method="post">
+      <button>Lock Front Door</button>
+      <button formaction="./frontDoor/unlock">Unlock Front Door</button>
+    </form>
+    <form>
+      <input type="text" name="term" />
+      <button>Search</button>
+    </form>
+    {% if headers is not none %}
+    <table>
+      <tr>
+        {% for header in headers %}
+        <th> {{ header }} </th>
+        {% endfor %}
+      </tr>
+      {% for member in members %}
+      <tr onHold ="{{ member['Account on Hold'] }}">
+        {% for header in headers %}
+        <td> {{ member[header] }} </td>
+        {% endfor %}
+      </tr>
+      {% endfor %}
+    </div>
+    </table>
+    {% endif %}
+    <span class="error">{{ error }}</span>
+  </body>
+</html>

ucsAccounts.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+import random
+import re
+import string
+import subprocess
+
+from common import membershipworks
+
+LDAP_BASE = "cn=users,dc=sawtooth,dc=claremontmakerspace,dc=org"
+GROUP_BASE = "cn=groups,dc=sawtooth,dc=claremontmakerspace,dc=org"
+GROUPS_REGEX = "|".join(['Certified: .*', 'Access .*\?', 'CMS .*', 'Volunteer: .*'])
+RAND_PW_LEN = 20
+
+def makeGroups(members):
+    groups = [key.replace(':', '.').replace('?', '')
+              for key in members[0].keys()
+              if re.match(GROUPS_REGEX, key) is not None]
+    for group in groups:
+        subprocess.call(["udm", "groups/group", "create",
+                         "--position", GROUP_BASE,
+                         "--set", "name=" + group])
+
+def makeSets(props):
+    return sum([["--set", key + "=" + value]
+                for key, value in props.items()], [])
+
+def makeAppendGroups(member):
+    groups = [key.replace(':', '.').replace('?', '')
+              for key, value in member.items()
+              if re.match(GROUPS_REGEX, key) is not None and value != '']
+    return sum([["--append", "groups=cn=" + group + ',' + GROUP_BASE]
+                for group in groups], [])
+
+def main():
+    members = membershipworks.get_members(['Members', 'CMS Staff'],
+                                     "lvl,phn,eml,lbl,nam,end,_id")
+    makeGroups(members)
+
+    for member in members:
+        randomPass = ''.join(random.choice(string.ascii_letters + string.digits)
+                             for x in range(0, RAND_PW_LEN))
+        username = member["Account Name"].lower().replace(" ", ".")
+
+        props = {
+            "title": "", # Title
+            "firstname": member["First Name"],
+            "lastname": member["Last Name"], # (c)
+            "username": username, # (cmr)
+            "description": "", # Description
+            "password": randomPass, # (c) Password
+            #"mailPrimaryAddress": member["Email"], # Primary e-mail address
+            #"displayName": "", # Display name
+            #"birthday": "", # Birthdate
+            #"jpegPhoto": "", # Picture of the user (JPEG format)
+
+            "employeeNumber": member["Account ID"],
+            #"employeeType": "", # Employee type
+
+            "homedrive": "H:", # Windows home drive
+            "sambahome": "\\\\ucs\\" + username, # Windows home path
+            "profilepath": "%LOGONSERVER%\\%USERNAME%\\windows-profiles\\default", # Windows profile directory
+
+            "disabled": "1" if member["Account on Hold"] != "" else "0",
+            #"userexpiry": member["Renewal Date"],
+
+            "pwdChangeNextLogin": "1", # User has to change password on next login
+
+            #"sambaLogonHours": "", # Permitted times for Windows logins
+
+            "e-mail": member["Email"],  # ([]) E-mail address
+            "phone": member["Phone"], # Telephone number
+            #"PasswordRecoveryMobile": member["Phone"], # Mobile phone number
+            "PasswordRecoveryEmail": member["Email"]
+        }
+
+        subprocess.call(["udm", "users/user", "create",
+                         "--position", LDAP_BASE] + makeSets(props))
+
+        # remove props we don't want to reset
+        props.pop("password")
+        props.pop("pwdChangeNextLogin")
+
+        subprocess.call(["udm", "users/user", "modify",
+                         "--dn", "uid=" + username + "," + LDAP_BASE]
+                        + makeSets(props)
+                        + makeAppendGroups(member))
+
+if __name__ == "__main__":
+    main()