Add taskipy and coverage tools

This allows running all of the updater scripts on net-svcs, instead of
running this one on UCS itself

.gitignore

@@ -1,3 +1,7 @@
-__pycache__/
+/.coverage
+/.mypy_cache/
+/memberplumbing.egg-info/
 /venv/
+__pycache__/
+
 /config.yaml

README.md

@@ -4,7 +4,7 @@ This repo contains a set of scripts to sync data around for the Claremont MakerS
 
 ## Setup
 
-This project uses [Poetry](https://python-poetry.org/) for dependency management. Typical usage is first running `poetry install` to create a virtualenv and install dependencies, then running `poetry run <script>` to start a specific script.
+This project uses [Poetry](https://python-poetry.org/) for dependency management. Typical usage is first running `poetry install` to create a virtualenv and install dependencies, then running `poetry run bin/<script>` to start a specific script.
 
 ## Config
 
@@ -12,7 +12,7 @@ Many of the scripts use data from a `config.yaml` in the current working directo
 
 ## Scripts
 
-The primary entry points have scripts entries (`tool.poetry.scripts`) in [`pyproject.toml`](./pyproject.toml). They assume that they are being run from a module, so must be run with `poetry run <script>` or `python -m memberPlumbing.<script>`.
+The primary entry points have scripts in [`bin`](./bin/). They assume that memberPlumbing is in `PYTHONPATH`, which can either be done with `poetry` or manually set.
 
 ### `doorUpdater`
 
@@ -26,10 +26,6 @@ Retrieves member information from MembershipWorks and pushes it out to [UCS](htt
 
 Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined with [peewee](peewee-orm.com) in [`memberPlumbing/mw_models.py`](./memberPlumbing/mw_models.py).
 
-### `upcomingEvents`
-
-Retrieves upcoming events from MembershipWorks and formats them for a WordPress post.
-
 ### `hidEvents`
 
 Retrieves events from the HID Evo Solo door controllers, and pushes them to a SQL database.
@@ -37,3 +33,15 @@ Retrieves events from the HID Evo Solo door controllers, and pushes them to a SQ
 ## Systemd
 
 There are systemd units in the [`systemd`](./systemd/) folder, which can be used to run the various scripts regularly.
+
+## SSL Certificates
+
+The HID Evo Solo door controllers we use have a self-signed certificate, which is included in this repo as [`hidglobal.com.pem`](./hidglobal.com.pem).
+
+If you need to use a different certificate, you can either download it with your browser or the following command (replacing `SERVER` by the address of the door):
+
+```sh
+openssl s_client -connect SERVER:443 </dev/null | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > example.pem
+```
+
+Then set `doorControllerCA_BUNDLE` in `config.yaml` to the path to the created pem file. If your doors have different certificates, (which ours annoyingly don't), you will need to concatenate the certificates together into a single file.

config.example.yaml

@@ -6,6 +6,8 @@ doorControllers:
   Wood Shop Rear: {ip: 172.18.51.15, access: Wood Shop}
   Storage Closet: {ip: 172.18.51.16, access: Storage Closet}
 
+doorControllerCA_BUNDLE: "hidglobal.com.pem"
+
 # {member type: door schedule}
 memberLevels:
   CMS Staff: 7x24

hidglobal.com.pem

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIID9TCCAt2gAwIBAgIJAPBotjnyfGu6MA0GCSqGSIb3DQEBCwUAMIGQMQswCQYD
+VQQGEwJVUzERMA8GA1UECAwIQ29sb3JhZG8xFDASBgNVBAcMC1dlc3RtaW5zdGVy
+MQwwCgYDVQQKDANISUQxDDAKBgNVBAsMA05BUzEWMBQGA1UEAwwNaGlkZ2xvYmFs
+LmNvbTEkMCIGCSqGSIb3DQEJARYVc3VwcG9ydEBoaWRnbG9iYWwuY29tMB4XDTE5
+MDcyMjEwMTQxMFoXDTI5MDcxOTEwMTQxMFowgZAxCzAJBgNVBAYTAlVTMREwDwYD
+VQQIDAhDb2xvcmFkbzEUMBIGA1UEBwwLV2VzdG1pbnN0ZXIxDDAKBgNVBAoMA0hJ
+RDEMMAoGA1UECwwDTkFTMRYwFAYDVQQDDA1oaWRnbG9iYWwuY29tMSQwIgYJKoZI
+hvcNAQkBFhVzdXBwb3J0QGhpZGdsb2JhbC5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDSibuXB9Tn0EdwL2jDig26s/b1D9SX5B4xnZM+xZ4/mE6U
+Meg5xbiTSMiWqtoSMVxG1WJDxogJWxCgZis2qk3AG89PBarg17pBmxPLYyCLricx
+alyNvTJBxYgA/zKagPof6h6UqKOkhsW9qvulEmPe+TKk47pmlZXe+v+1A6PQDY5B
+Y3MtqE23cnZ5nBTVanFAc1vbokMXUCCtRvE1Y/KhvuaJr2VjOSJ/KV3vcdTSCLGc
+W9/n/Fv8udvI/eIkoPNpCUwngm8j3Aa7qN/OSg3SvVvBcl/Ykc08STSyZPMJGBaR
+EuUcAraEBZbUDOCinDS488jKVHAXhrnvzzi7RMlhAgMBAAGjUDBOMB0GA1UdDgQW
+BBSvOzxCjQi86ZUsuW0o4aa7GNAeSTAfBgNVHSMEGDAWgBSvOzxCjQi86ZUsuW0o
+4aa7GNAeSTAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQA/8FCv6x8v
+PHm2Ya/hbZ/S3amdl7/E1illeApNRZodTGCn/rlVSGanCfWYzEY2naHiDC2ImhZq
+NkHK9uvUtxaVQFq5VN6WQMo351J78LLfcoqpKOLGX3b9byFvrw7WporZx3C7yL1U
+LS3oxI/pgavxy1KbOIw/yl+QgV50vlfvQ7sKZ1E5YOrgWLP5nJ9OeEKRdsASJyZS
+Jjl0k/eGaZreSvAZPmx4kaePfbi7DNDA+mNhSFygwt6AakjjVoF2xUZ1F+qwBtER
+GPxdZWldywUYsdBRG1PPvBsMo9ME46HpPdXRIjMge8P01fsaMr/6H86ojWg9uJmH
+cCKtiouo08hL
+-----END CERTIFICATE-----

memberPlumbing/MembershipWorks.py

@@ -2,9 +2,8 @@ import csv
 from io import StringIO
 
 import requests
-import datetime
 
-BASE_URL = "https://api.membershipworks.com"
+BASE_URL = "https://api.membershipworks.com/v1/"
 
 # extracted from `SF._is.crm` in https://cdn.membershipworks.com/all.js
 CRM = {
@@ -73,30 +72,21 @@ class MembershipWorksRemoteError(Exception):
 
 class MembershipWorks:
     def __init__(self):
-        self.sess = requests.Session()
         self.org_info = None
         self.auth_token = None
         self.org_num = None
 
     def login(self, username, password):
         """Authenticate against the membershipworks api"""
-        r = self.sess.post(
+        r = requests.post(
-            BASE_URL + "/v2/account/session",
+            BASE_URL + "usr",
-            data={"eml": username, "pwd": password},
+            data={"_st": "all", "eml": username, "org": "10000", "pwd": password},
-            headers={"X-Org": "10000"},
         )
         if r.status_code != 200 or "SF" not in r.json():
             raise MembershipWorksRemoteError("login", r)
         self.org_info = r.json()
         self.auth_token = self.org_info["SF"]
         self.org_num = self.org_info["org"]
-        self.sess.headers.update(
-            {
-                "X-Org": str(self.org_num),
-                "X-Role": "admin",
-                "Authorization": "Bearer " + self.auth_token,
-            }
-        )
 
     def _inject_auth(self, kwargs):
         # TODO: should probably be a decorator or something
@@ -107,12 +97,12 @@ class MembershipWorks:
             kwargs["params"] = {}
         kwargs["params"]["SF"] = self.auth_token
 
-    def _get_v1(self, *args, **kwargs):
+    def _get(self, *args, **kwargs):
         self._inject_auth(kwargs)
         # TODO: should probably do some error handling in here
         return requests.get(*args, **kwargs)
 
-    def _post_v1(self, *args, **kwargs):
+    def _post(self, *args, **kwargs):
         self._inject_auth(kwargs)
         # TODO: should probably do some error handling in here
         return requests.post(*args, **kwargs)
@@ -151,44 +141,46 @@ class MembershipWorks:
         for dek in self.org_info["dek"]:
             # TODO: there must be a better way. this is stupid
             if dek["dek"] == 1:
-                ret["folders"][dek["lbl"]] = dek["did"]
+                ret["folders"][dek["lbl"]] = dek["_id"]
             elif "cur" in dek:
-                ret["levels"][dek["lbl"]] = dek["did"]
+                ret["levels"][dek["lbl"]] = dek["_id"]
             elif "mux" in dek:
-                ret["addons"][dek["lbl"]] = dek["did"]
+                ret["addons"][dek["lbl"]] = dek["_id"]
             else:
-                ret["labels"][dek["lbl"]] = dek["did"]
+                ret["labels"][dek["lbl"]] = dek["_id"]
 
         return ret
 
     def get_member_ids(self, folders):
         folder_map = self._parse_flags()["folders"]
 
-        r = self.sess.get(
+        r = self._get(
-            BASE_URL + "/v2/accounts",
+            BASE_URL + "ylp",
-            params={"dek": ",".join([folder_map[f] for f in folders])},
+            params={
+                "lbl": ",".join([folder_map[f] for f in folders]),
+                "org": self.org_num,
+                "var": "_id,nam,ctc",
+            },
         )
         if r.status_code != 200 or "usr" not in r.json():
             raise MembershipWorksRemoteError("user listing", r)
 
         # get list of member ID matching the search
-        # dedup with set() to work around people with alt uids
+        return [user["uid"] for user in r.json()["usr"]]
-        # TODO: figure out why people have alt uids
-        return set(user["uid"] for user in r.json()["usr"])
 
     # TODO: has issues with aliasing header names:
     #       ex: "Personal Studio Space" Label vs Membership Addon/Field
     def get_members(self, folders, columns):
-        """Pull the members csv from the membershipworks api
+        """ Pull the members csv from the membershipworks api
-        folders: a list of the names of the folders to get
+            folders: a list of the names of the folders to get
-                 (see folder_map in this function for mapping to ids)
+                     (see folder_map in this function for mapping to ids)
-        columns: which columns to get"""
+            columns: which columns to get"""
         ids = self.get_member_ids(folders)
 
         # get members CSV
         # TODO: maybe can just use previous get instead? would return JSON
-        r = self._post_v1(
+        r = self._post(
-            BASE_URL + "/v1/csv",
+            BASE_URL + "csv",
             data={
                 "_rt": "946702800",  # unknown
                 "mux": "",  # unknown
@@ -198,24 +190,20 @@ class MembershipWorks:
         )
         if r.status_code != 200:
             raise MembershipWorksRemoteError("csv generation", r)
-
-        if r.text[0] == "\ufeff":
-            r.encoding = r.encoding + "-sig"
-
         return list(csv.DictReader(StringIO(r.text)))
 
     def get_transactions(self, start_date, end_date, json=False):
         """Get the transactions between start_date and end_date
 
-        Dates can be datetime.date or datetime.datetime
+           Dates can be datetime.date or datetime.datetime
 
-        json gets a different version of the transactions list,
+           json gets a different version of the transactions list,
-        which contains a different set information
+           which contains a different set information
         """
-        r = self._get_v1(
+        r = self._get(
-            BASE_URL + "/v1/csv",
+            BASE_URL + "csv",
             params={
-                "crm": ",".join(str(k) for k in CRM.keys()),
+                "crm": "12,13,14,18,19",  # transaction types, see CRM
                 **({"txl": ""} if json else {}),
                 "sdp": start_date.strftime("%s"),
                 "edp": end_date.strftime("%s"),
@@ -226,9 +214,6 @@ class MembershipWorks:
         if json:
             return r.json()
         else:
-            if r.text[0] == "\ufeff":
-                r.encoding = r.encoding + "-sig"
-
             return list(csv.DictReader(StringIO(r.text)))
 
     def get_all_members(self):
@@ -237,29 +222,3 @@ class MembershipWorks:
         fields = self._all_fields()
         members = self.get_members(folders, ",".join(fields.keys()))
         return members
-
-    def get_events_list(self, start_date: datetime.datetime):
-        """Retrive a list of events since start_date"""
-        r = self.sess.get(
-            BASE_URL + "/v2/events",
-            params={
-                "sdp": start_date.strftime("%s"),
-            },
-        )
-        return r.json()
-
-    def get_event_by_eid(self, eid: str):
-        """Retrieve a specific event by its event id (eid)"""
-        r = self.sess.get(
-            BASE_URL + "/v2/event",
-            params={"eid": eid},
-        )
-        return r.json()
-
-    def get_event_by_url(self, url: str):
-        """Retrieve a specific event by its url"""
-        r = self.sess.get(
-            BASE_URL + "/v2/event",
-            params={"url": url},
-        )
-        return r.json()

memberPlumbing/config.py

@@ -22,6 +22,7 @@ class Config:
                 self.DOOR_PASSWORD,
                 name=doorName,
                 access=doorData["access"],
+                cert=self.doorControllerCA_BUNDLE
             )
             for doorName, doorData in self.doorControllers.items()
         }

memberPlumbing/doorUpdater.py

@@ -72,9 +72,15 @@ class MembershipworksMember(Member):
         else:
             self.credentials = set()
 
-        self.onHold = (
+        self.onHold = data["Account on Hold"] != ""
-            data["Account on Hold"] != ""
+        self.limitedOperations = (
-            or data["CMS Membership on hold"] == "CMS Membership on hold"
+            data[
+                "Access Permitted Using Membership Level Schedule During COVID-19 Limited Operations"
+            ]
+            == "Y"
+        )
+        self.staffedLimitedOperations = (
+            data["Access Permitted During COVID-19 Staffed Period Only"] == "Y"
         )
         self.formerMember = formerMember
 
@@ -100,7 +106,13 @@ class MembershipworksMember(Member):
 
         schedules = []
         if door.name in self.doorAccess and not self.onHold and not self.formerMember:
-            schedules = self.schedules + doorLevels
+            # members should get their normal schedules
+            if self.limitedOperations or "CMS Staff" in self.levels:
+                schedules = self.schedules + doorLevels
+
+            # members should get only the staffed hours schedule
+            if self.staffedLimitedOperations:
+                schedules += ["Staffed Hours"]
 
         dm = DoorMember(
             door,
@@ -121,6 +133,7 @@ class MembershipworksMember(Member):
         return (
             super().__str__()
             + f"""OnHold? {self.onHold}
+Limited Operations Access? {self.limitedOperations}
 Former Member? {self.formerMember}
 """
         )
@@ -165,48 +178,14 @@ class DoorMember(Member):
             "custom2": self.membershipWorksID,
         }
 
-    def make_schedules(self, schedulesMap):
-        roles = [
-            E.Role(
-                {
-                    "roleID": self.cardholderID,
-                    "scheduleID": schedulesMap[schedule],
-                    "resourceID": "0",
-                }
-            )
-            for schedule in self.schedules
-        ]
-
-        return E.RoleSet(
-            {"action": "UD", "roleSetID": self.cardholderID}, E.Roles(*roles)
-        )
-
-    def make_credentials(self, newCredentials, cardFormats):
-        out = [
-            E.Credential(
-                {
-                    "formatName": str(credential.code[0]),
-                    "cardNumber": str(credential.code[1]),
-                    "formatID": cardFormats[str(credential.code[0])],
-                    "isCard": "true",
-                    "cardholderID": self.cardholderID,
-                }
-            )
-            for credential in newCredentials
-        ]
-
-        return E.Credentials({"action": "AD"}, *out)
-
 
 def update_door(door, members):
-    cardFormats = door.get_cardFormats()
     cardholders = {
         member.membershipWorksID: member
         for member in [
             DoorMember.from_cardholder(ch, door) for ch in door.get_cardholders()
         ]
     }
-    schedulesMap = door.get_scheduleMap()
     allCredentials = set(
         Credential(hex=c.attrib["rawCardNumber"]) for c in door.get_credentials()
     )
@@ -218,9 +197,7 @@ def update_door(door, members):
         if member.membershipWorksID not in cardholders:
             print("- Adding Member {member.forename} {member.surname}:")
             print(f"  - {member.attribs()}")
-            resp = door.doXMLRequest(
+            resp = door.add_cardholder(member.attribs)
-                ROOT(E.Cardholders({"action": "AD"}, E.Cardholder(member.attribs())))
-            )
             member.cardholderID = resp.find("{*}Cardholders/{*}Cardholder").attrib[
                 "cardholderID"
             ]
@@ -239,14 +216,7 @@ def update_door(door, members):
             print(f"- Updating profile for {member.forename} {member.surname}")
             print(f"  - Old: {ch.attribs()}")
             print(f"  - New: {member.attribs()}")
-            door.doXMLRequest(
+            door.update_cardholder(member.cardholderID, member.attribs)
-                ROOT(
-                    E.Cardholders(
-                        {"action": "UD", "cardholderID": member.cardholderID},
-                        E.CardHolder(member.attribs()),
-                    )
-                )
-            )
 
         if member.credentials != ch.credentials:
             print(f"- Updating card for {member.forename} {member.surname}")
@@ -264,18 +234,7 @@ def update_door(door, members):
 
             # cards removed, and won't be reassigned to someone else
             for card in (oldCards - newCards) - allNewCards:
-                door.doXMLRequest(
+                door.assign_credential(card, None)
-                    ROOT(
-                        E.Credentials(
-                            {
-                                "action": "UD",
-                                "rawCardNumber": card.hex,
-                                "isCard": "true",
-                            },
-                            E.Credential({"cardholderID": ""}),
-                        )
-                    )
-                )
 
             if newCards - oldCards:  # cards added
                 for card in newCards & allNewCards:  # new card exists in another member
@@ -291,28 +250,11 @@ def update_door(door, members):
 
                 # card existed in door, and needs to be reassigned
                 for card in newCards & allCredentials:
-                    door.doXMLRequest(
+                    door.assign_credential(card, member.cardholderID)
-                        ROOT(
-                            E.Credentials(
-                                {
-                                    "action": "UD",
-                                    "rawCardNumber": card.hex,
-                                    "isCard": "true",
-                                },
-                                E.Credential({"cardholderID": member.cardholderID}),
-                            )
-                        )
-                    )
 
                 # cards that never existed, and need to be created
                 if newCards - allCredentials:
-                    door.doXMLRequest(
+                    door.add_credentials(newCards - allCredentials, member.cardholderID)
-                        ROOT(
-                            member.make_credentials(
-                                newCards - allCredentials, cardFormats
-                            )
-                        )
-                    )
 
         if member.schedules != ch.schedules:
             print(
@@ -320,7 +262,7 @@ def update_door(door, members):
                 + f" {member.forename} {member.surname}:"
                 + f" {ch.schedules} -> {member.schedules}"
             )
-            door.doXMLRequest(ROOT(member.make_schedules(schedulesMap)))
+            door.set_cardholder_schedules(member.cardholderID, member.schedules)
 
     # TODO: delete cardholders that are no longer members?
 
@@ -329,7 +271,7 @@ def main():
     config = Config()
     membershipworks = config.membershipworks
     membershipworks_attributes = (
-        "_id,nam,phn,eml,lvl,lbl,xws,xms,xsc,xas,xfd,xac,xcf,xeh,xse"
+        "_id,nam,phn,eml,lvl,lbl,xws,xms,xsc,xas,xfd,xac,xcf,xeh,xse,xlo,xxc"
     )
 
     memberData = membershipworks.get_members(

memberPlumbing/hid/Credential.py

@@ -1,11 +1,14 @@
 import bitstring
+from typing import Tuple, Optional
 
 # Reference for H10301 card format:
 # https://www.hidglobal.com/sites/default/files/hid-understanding_card_data_formats-wp-en.pdf
 
 
 class Credential:
-    def __init__(self, code=None, hex=None):
+    def __init__(
+        self, code: Optional[Tuple[int, int]] = None, hex: Optional[str] = None
+    ) -> None:
         if code is None and hex is None:
             raise TypeError("Must set either code or hex for a Credential")
         elif code is not None and hex is not None:
@@ -21,21 +24,24 @@ class Credential:
         elif hex is not None:
             self.bits = bitstring.Bits(hex=hex)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return f"Credential({self.code})"
 
-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
-        return self.bits == other.bits
+        if isinstance(other, Credential):
+            return self.bits == other.bits
+        else:
+            return False
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return self.bits.int
 
     @property
-    def code(self):
+    def code(self) -> Tuple[int, int]:
         facility = self.bits[7:15].uint
         code = self.bits[15:31].uint
         return (facility, code)
 
     @property
-    def hex(self):
+    def hex(self) -> str:
         return self.bits.hex.upper()

memberPlumbing/hid/DoorController.py

@@ -1,11 +1,25 @@
 import csv
 from datetime import datetime
 from io import StringIO
+from typing import (
+    IO,
+    Callable,
+    Iterable,
+    List,
+    Literal,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 import requests
-import urllib3
 from lxml import etree
 from lxml.builder import ElementMaker
+from requests import Session
+from requests.adapters import HTTPAdapter
+
+from .Credential import Credential
 
 E_plain = ElementMaker(nsmap={"hid": "http://www.hidglobal.com/VertX"})
 E = ElementMaker(
@@ -22,33 +36,65 @@ fieldnames = "CardNumber,CardFormat,PinRequired,PinCode,ExtendedAccess,ExpiryDat
     ","
 )
 
-# TODO: where should this live?
+
-# it's fine, ssl certs are for losers anyway
+class HostNameIgnoringAdapter(HTTPAdapter):
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+    def init_poolmanager(self, *args, **kwargs) -> None:
+        super().init_poolmanager(*args, **kwargs, assert_hostname=False)
 
 
 class RemoteError(Exception):
-    def __init__(self, r):
+    def __init__(self, r: requests.Response) -> None:
         super().__init__(f"Door Updating Error: {r.status_code} {r.reason}\n{r.text}")
 
 
 class DoorController:
-    def __init__(self, ip, username, password, name="", access=""):
+    def __init__(
+        self,
+        ip: str,
+        username: str,
+        password: str,
+        name: str = "",
+        access: str = "",
+        cert: Optional[str] = None,
+    ) -> None:
         self.ip = ip
         self.username = username
         self.password = password
         self.name = name
         self.access = access
+        self.session = Session()
+        if cert is not None:
+            self.session.mount("https://", HostNameIgnoringAdapter())
+            self.session.verify = cert
 
-    def doImport(self, params=None, files=None):
+        self._cardFormats: Optional[Mapping[str, str]] = None
+        self._scheduleMap: Optional[Mapping[str, str]] = None
+
+    # lazy evaluated, hopefully won't change for the lifetime of this object
+    @property
+    def cardFormats(self) -> Mapping[str, str]:
+        if not self._cardFormats:
+            self._cardFormats = self.get_cardFormats()
+        return self._cardFormats
+
+    @property
+    def scheduleMap(self) -> Mapping[str, str]:
+        if not self._scheduleMap:
+            self._scheduleMap = self.get_scheduleMap()
+        return self._scheduleMap
+
+    def doImport(
+        self,
+        params: Optional[Mapping[str, str]] = None,
+        files: Optional[Mapping[str, Tuple[str, Union[IO[str], str], str]]] = None,
+    ) -> None:
         """Send a request to the door control import script"""
-        r = requests.post(
+        r = self.session.post(
             "https://" + self.ip + "/cgi-bin/import.cgi",
             params=params,
             files=files,
             auth=requests.auth.HTTPDigestAuth(self.username, self.password),
             timeout=60,
-            verify=False,
         )  # ignore insecure SSL
         xml = etree.XML(r.content)
         if (
@@ -57,7 +103,7 @@ class DoorController:
         ):
             raise RemoteError(r)
 
-    def doCSVImport(self, csv):
+    def doCSVImport(self, csv: Union[IO[str], str]) -> None:
         """Do the CSV import procedure on a door control"""
         self.doImport({"task": "importInit"})
         self.doImport(
@@ -66,22 +112,29 @@ class DoorController:
         )
         self.doImport({"task": "importDone"})
 
-    def doXMLRequest(self, xml, prefix=b'<?xml version="1.0" encoding="UTF-8"?>'):
+    def doXMLRequest(
+        self,
+        xml: Union[etree.Element, bytes],
+        prefix: bytes = b'<?xml version="1.0" encoding="UTF-8"?>',
+    ) -> etree.XML:
         if not isinstance(xml, bytes):
             xml = etree.tostring(xml)
-        r = requests.get(
+        r = self.session.get(
             "https://" + self.ip + "/cgi-bin/vertx_xml.cgi",
             params={"XML": prefix + xml},
             auth=requests.auth.HTTPDigestAuth(self.username, self.password),
-            verify=False,
         )
-        resp_xml = etree.XML(r.content)
+
+        if r.status_code != 200:
+            raise RemoteError(r)
+
         # probably meed to be more sane about this
-        if r.status_code != 200 or len(resp_xml.findall("{*}Error")) > 0:
+        resp_xml = etree.XML(r.content)
+        if len(resp_xml.findall("{*}Error")) > 0:
             raise RemoteError(r)
         return resp_xml
 
-    def get_scheduleMap(self):
+    def get_scheduleMap(self) -> Mapping[str, str]:
         schedules = self.doXMLRequest(
             ROOT(E.Schedules({"action": "LR", "recordOffset": "0", "recordCount": "8"}))
         )
@@ -89,7 +142,7 @@ class DoorController:
             fmt.attrib["scheduleName"]: fmt.attrib["scheduleID"] for fmt in schedules[0]
         }
 
-    def get_schedules(self):
+    def get_schedules(self) -> etree.Element:
         # TODO: might be able to do in one request
         schedules = self.doXMLRequest(ROOT(E.Schedules({"action": "LR"})))
         etree.dump(schedules)
@@ -106,7 +159,7 @@ class DoorController:
         )
         return ROOT(E_corp.Schedules({"action": "AD"}, *[s[0] for s in data]))
 
-    def set_schedules(self, schedules):
+    def set_schedules(self, schedules: etree.Element) -> None:
         # clear all people
         outString = StringIO()
         writer = csv.DictWriter(outString, fieldnames)
@@ -131,7 +184,27 @@ class DoorController:
         # load new schedules
         self.doXMLRequest(schedules)
 
-    def get_cardFormats(self):
+    def set_cardholder_schedules(
+        self, cardholderID: str, schedules: Iterable[str]
+    ) -> etree.XML:
+        roles = [
+            E.Role(
+                {
+                    "roleID": cardholderID,
+                    "scheduleID": self.scheduleMap[schedule],
+                    "resourceID": "0",
+                }
+            )
+            for schedule in schedules
+        ]
+
+        roleSet = E.RoleSet(
+            {"action": "UD", "roleSetID": cardholderID}, E.Roles(*roles)
+        )
+
+        return self.doXMLRequest(ROOT(roleSet))
+
+    def get_cardFormats(self) -> Mapping[str, str]:
         cardFormats = self.doXMLRequest(
             ROOT(E.CardFormats({"action": "LR", "responseFormat": "expanded"}))
         )
@@ -141,7 +214,9 @@ class DoorController:
             for fmt in cardFormats[0].findall("{*}CardFormat[{*}FixedField]")
         }
 
-    def set_cardFormat(self, formatName, templateID, facilityCode):
+    def set_cardFormat(
+        self, formatName: str, templateID: int, facilityCode: int
+    ) -> etree.XML:
         # TODO: add ability to delete formats
         # delete example: <hid:CardFormats action="DD" formatID="7-1-244"/>
 
@@ -156,8 +231,14 @@ class DoorController:
         )
         return self.doXMLRequest(el)
 
-    def get_records(self, req, count, params={}, stopFunction=None):
+    def get_records(
-        result = []
+        self,
+        req: ElementMaker,
+        count: int,
+        params: Mapping[str, str] = {},
+        stopFunction: Optional[Callable[[List[etree.Element]], bool]] = None,
+    ) -> List[etree.Element]:
+        result: List[etree.Element] = []
         recordCount = 0
         moreRecords = True
 
@@ -189,17 +270,76 @@ class DoorController:
 
         return result
 
-    def get_cardholders(self):
+    def get_cardholders(self) -> List[etree.Element]:
         return self.get_records(E.Cardholders, 1000, {"responseFormat": "expanded"})
 
-    def get_credentials(self):
+    def add_cardholder(self, attribs: Mapping[str, str]) -> etree.XML:
+        return self.doXMLRequest(
+            ROOT(E.Cardholders({"action": "AD"}, E.Cardholder(attribs)))
+        )
+
+    def update_cardholder(
+        self, cardholderID: str, attribs: Mapping[str, str]
+    ) -> etree.XML:
+        return self.doXMLRequest(
+            ROOT(
+                E.Cardholders(
+                    {"action": "UD", "cardholderID": cardholderID},
+                    E.CardHolder(attribs),
+                )
+            )
+        )
+
+    def get_credentials(self) -> List[etree.Element]:
         return self.get_records(E.Credentials, 1000)
 
-    def get_events(self, threshold):
+    def add_credentials(
-        def event_newer_than_threshold(event):
+        self, credentials: Iterable[Credential], cardholderID: Optional[str] = None
+    ) -> etree.XML:
+        """Create new Credentials. If a cardholderID is provided, assign the
+        new credentials to that cardholder"""
+        creds = [
+            E.Credential(
+                {
+                    "formatName": str(credential.code[0]),
+                    "cardNumber": str(credential.code[1]),
+                    "formatID": self.cardFormats[str(credential.code[0])],
+                    "isCard": "true",
+                    **({"cardholderID": cardholderID} if cardholderID else {}),
+                }
+            )
+            for credential in credentials
+        ]
+
+        return self.doXMLRequest(ROOT(E.Credentials({"action": "AD"}, *creds)))
+
+    def assign_credential(
+        self, credential: Credential, cardholderID: Optional[str] = None
+    ) -> etree.XML:
+        # empty string removes assignment
+        if cardholderID is None:
+            cardholderID = ""
+
+        return self.doXMLRequest(
+            ROOT(
+                E.Credentials(
+                    {
+                        "action": "UD",
+                        "rawCardNumber": credential.hex,
+                        "isCard": "true",
+                    },
+                    E.Credential({"cardholderID": cardholderID}),
+                )
+            )
+        )
+
+    def get_events(self, threshold: datetime) -> List[etree.Element]:
+        def event_newer_than_threshold(event: etree.Element) -> bool:
             return datetime.fromisoformat(event.attrib["timestamp"]) > threshold
 
-        def last_event_newer_than_threshold(events):
+        def last_event_newer_than_threshold(
+            events: List[etree.Element],
+        ) -> etree.Element:
             return (not events) or event_newer_than_threshold(events[-1])
 
         return [
@@ -210,13 +350,13 @@ class DoorController:
             if event_newer_than_threshold(event)
         ]
 
-    def get_lock(self):
+    def get_lock(self) -> Union[Literal["locked"], Literal["unlocked"]]:
         el = ROOT(E.Doors({"action": "LR", "responseFormat": "status"}))
         xml = self.doXMLRequest(el)
         relayState = xml.find("./{*}Doors/{*}Door").attrib["relayState"]
         return "unlocked" if relayState == "set" else "locked"
 
-    def set_lock(self, lock=True):
+    def set_lock(self, lock: bool = True) -> etree.XML:
         el = ROOT(
             E.Doors({"action": "CM", "command": "lockDoor" if lock else "unlockDoor"})
         )

memberPlumbing/hid/tests/test_Credential.py

@@ -0,0 +1,13 @@
+import pytest
+
+from ..Credential import Credential
+
+
+def test_code_to_hex() -> None:
+    cred = Credential(code=(123, 45678))
+    assert cred.hex == "02F764DD"
+
+
+def test_hex_to_code() -> None:
+    cred = Credential(hex="02F764DD")
+    assert cred.code == (123, 45678)

memberPlumbing/hidEvents.py

@@ -97,11 +97,7 @@ def main():
     config = Config()
     database.init(
         **config.HID_DB,
-        **{
+        **{"charset": "utf8", "sql_mode": "PIPES_AS_CONCAT", "use_unicode": True,}
-            "charset": "utf8",
-            "sql_mode": "PIPES_AS_CONCAT",
-            "use_unicode": True,
-        }
     )
     HIDEvent.create_table()
     for door in config.doors.values():

memberPlumbing/sqlExport.py

@@ -22,12 +22,6 @@ def do_import(config):
         ]
     ).on_conflict(action="update", preserve=[Flag.name, Flag.type]).execute()
 
-    print("Getting folder membership...")
-    folders = {
-        folder_id: membershipworks.get_member_ids([folder_name])
-        for folder_name, folder_id in membershipworks._parse_flags()["folders"].items()
-    }
-
     print("Getting/Updating members...")
     members = membershipworks.get_all_members()
     for m in members:
@@ -47,14 +41,13 @@ def do_import(config):
 
         # update member's flags
         for type, flags in membershipworks._parse_flags().items():
-            for flag, id in flags.items():
+            if type != "folders":  # currently no way to retrieve this info
-                ml = MemberFlag(uid=member["Account ID"], flag_id=id)
+                for flag, id in flags.items():
-                if (type == "folders" and member["Account ID"] in folders[id]) or (
+                    ml = MemberFlag(uid=member["Account ID"], flag_id=id)
-                    type != "folders" and member[flag]
+                    if member[flag]:
-                ):
+                        ml.magic_save()
-                    ml.magic_save()
+                    else:
-                else:
+                        ml.delete_instance()
-                    ml.delete_instance()
 
     print("Getting/Updating transactions...")
     # Deduping these is hard, so just recreate the data every time
@@ -82,11 +75,7 @@ def main():
     config = Config()
     database.init(
         **config.MEMBERSHIPWORKS_DB,
-        **{
+        **{"charset": "utf8", "sql_mode": "PIPES_AS_CONCAT", "use_unicode": True,}
-            "charset": "utf8",
-            "sql_mode": "PIPES_AS_CONCAT",
-            "use_unicode": True,
-        }
     )
 
     do_import(config)

memberPlumbing/tests/json-schemas/usr.schema

@@ -0,0 +1,688 @@
+{
+  "$schema": "http://json-schema.org/schema#",
+  "type": "object",
+  "properties": {
+    "nam": {
+      "type": "string"
+    },
+    "adr": {
+      "type": "object",
+      "properties": {
+        "zip": {
+          "type": "string"
+        },
+        "cit": {
+          "type": "string"
+        },
+        "sta": {
+          "type": "string"
+        },
+        "con": {
+          "type": "string"
+        },
+        "ad1": {
+          "type": "string"
+        },
+        "loc": {
+          "type": "array",
+          "items": {
+            "type": "number"
+          }
+        }
+      },
+      "required": [
+        "ad1",
+        "cit",
+        "con",
+        "loc",
+        "sta",
+        "zip"
+      ]
+    },
+    "org": {
+      "type": "integer"
+    },
+    "cur": {
+      "type": "string"
+    },
+    "ctc": {
+      "type": "string"
+    },
+    "atg": {
+      "type": "string"
+    },
+    "uid": {
+      "type": "string"
+    },
+    "typ": {
+      "type": "string"
+    },
+    "eml": {
+      "type": "string"
+    },
+    "phn": {
+      "type": "string"
+    },
+    "end": {
+      "type": "integer"
+    },
+    "tpl": {
+      "type": "object",
+      "properties": {
+        "anm": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "lbl": {
+                "type": "string"
+              },
+              "box": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "dat": {
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "array",
+                          "items": {
+                            "type": "object",
+                            "properties": {
+                              "_id": {
+                                "type": "string"
+                              },
+                              "typ": {
+                                "type": "integer"
+                              },
+                              "plh": {
+                                "type": "string"
+                              },
+                              "req": {
+                                "type": "integer"
+                              },
+                              "rxp": {
+                                "type": "string"
+                              },
+                              "err": {
+                                "type": "string"
+                              },
+                              "lbl": {
+                                "type": "string"
+                              },
+                              "dtl": {
+                                "type": "string"
+                              }
+                            },
+                            "required": [
+                              "typ"
+                            ]
+                          }
+                        }
+                      ]
+                    },
+                    "ttl": {
+                      "type": "string"
+                    },
+                    "dtl": {
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "dat"
+                  ]
+                }
+              },
+              "dir": {
+                "type": "integer"
+              }
+            },
+            "required": [
+              "box",
+              "lbl"
+            ]
+          }
+        },
+        "acc": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "lbl": {
+                "type": "string"
+              },
+              "box": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "dat": {
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "array",
+                          "items": {
+                            "type": "object",
+                            "properties": {
+                              "_id": {
+                                "type": "string"
+                              },
+                              "typ": {
+                                "type": "integer"
+                              },
+                              "plh": {
+                                "type": "string"
+                              },
+                              "req": {
+                                "type": "integer"
+                              },
+                              "rxp": {
+                                "type": "string"
+                              },
+                              "err": {
+                                "type": "string"
+                              },
+                              "lbl": {
+                                "type": "string"
+                              }
+                            },
+                            "required": [
+                              "_id",
+                              "lbl",
+                              "typ"
+                            ]
+                          }
+                        }
+                      ]
+                    },
+                    "ttl": {
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "dat"
+                  ]
+                }
+              }
+            },
+            "required": [
+              "box",
+              "lbl"
+            ]
+          }
+        },
+        "adm": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "lbl": {
+                "type": "string"
+              },
+              "box": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "ttl": {
+                      "type": "string"
+                    },
+                    "dat": {
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "array",
+                          "items": {
+                            "type": "object",
+                            "properties": {
+                              "_id": {
+                                "type": "string"
+                              },
+                              "typ": {
+                                "type": "integer"
+                              },
+                              "plh": {
+                                "type": "string"
+                              },
+                              "req": {
+                                "type": "integer"
+                              },
+                              "rxp": {
+                                "type": "string"
+                              },
+                              "err": {
+                                "type": "string"
+                              },
+                              "lbl": {
+                                "type": "string"
+                              },
+                              "def": {
+                                "type": "string"
+                              }
+                            },
+                            "required": [
+                              "_id",
+                              "typ"
+                            ]
+                          }
+                        }
+                      ]
+                    },
+                    "dtl": {
+                      "type": "string"
+                    }
+                  },
+                  "required": [
+                    "dat"
+                  ]
+                }
+              }
+            },
+            "required": [
+              "box",
+              "lbl"
+            ]
+          }
+        },
+        "dir": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "lbl": {
+                "type": "string"
+              },
+              "box": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "htm": {
+                      "type": "string"
+                    },
+                    "dat": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            },
+            "required": [
+              "box",
+              "lbl"
+            ]
+          }
+        },
+        "crd": {
+          "type": "object",
+          "properties": {
+            "typ": {
+              "type": "integer"
+            },
+            "dtl": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "dtl",
+            "typ"
+          ]
+        },
+        "ylp": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "dat": {
+                "type": "string"
+              },
+              "ttl": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "dat"
+            ]
+          }
+        },
+        "evt": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "_id": {
+                "type": "string"
+              },
+              "typ": {
+                "type": "integer"
+              },
+              "req": {
+                "type": "integer"
+              },
+              "lbl": {
+                "type": "string"
+              },
+              "rxp": {
+                "type": "string"
+              },
+              "err": {
+                "type": "string"
+              }
+            },
+            "required": [
+              "_id",
+              "lbl",
+              "req",
+              "typ"
+            ]
+          }
+        }
+      },
+      "required": [
+        "acc",
+        "adm",
+        "anm",
+        "crd",
+        "dir",
+        "evt",
+        "ylp"
+      ]
+    },
+    "evg": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "ttl": {
+            "type": "string"
+          },
+          "cal": {
+            "type": "integer"
+          },
+          "opt": {
+            "type": "integer"
+          }
+        },
+        "required": [
+          "cal",
+          "opt",
+          "ttl"
+        ]
+      }
+    },
+    "dcc": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "lbl": {
+            "type": "string"
+          },
+          "amt": {
+            "type": "integer"
+          },
+          "typ": {
+            "type": "integer"
+          },
+          "_id": {
+            "type": "string"
+          },
+          "cnt": {
+            "type": "integer"
+          },
+          "cap": {
+            "type": "integer"
+          }
+        },
+        "required": [
+          "_id",
+          "amt",
+          "lbl",
+          "typ"
+        ]
+      }
+    },
+    "dek": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "did": {
+            "type": "string"
+          },
+          "lbl": {
+            "type": "string"
+          },
+          "dir": {
+            "type": "integer"
+          },
+          "aon": {
+            "type": "integer"
+          },
+          "typ": {
+            "type": "integer"
+          },
+          "dek": {
+            "type": "integer"
+          },
+          "pub": {
+            "type": "integer"
+          },
+          "dtl": {
+            "type": "string"
+          },
+          "mux": {
+            "type": "integer"
+          },
+          "itv": {
+            "type": "string"
+          },
+          "lvl": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "_id": {
+            "type": "string"
+          },
+          "ctc": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "tnm": {
+            "type": "string"
+          },
+          "dnm": {
+            "type": "string"
+          },
+          "cur": {
+            "type": "string"
+          },
+          "_rt": {
+            "type": "integer"
+          },
+          "bil": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "bid": {
+                  "type": "string"
+                },
+                "ttl": {
+                  "type": "string"
+                },
+                "way": {
+                  "type": "integer"
+                },
+                "amt": {
+                  "type": "number"
+                },
+                "itv": {
+                  "type": "string"
+                },
+                "mux": {
+                  "type": "integer"
+                },
+                "pub": {
+                  "type": "integer"
+                },
+                "dcc": {
+                  "type": "object"
+                },
+                "dso": {
+                  "type": "string"
+                }
+              },
+              "required": [
+                "amt",
+                "bid",
+                "itv",
+                "mux",
+                "ttl",
+                "way"
+              ]
+            }
+          },
+          "cat": {
+            "type": "integer"
+          },
+          "put": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "upg": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "nyn": {
+            "type": "integer"
+          },
+          "prp": {
+            "type": "integer"
+          },
+          "psm": {
+            "type": "integer"
+          },
+          "exp": {
+            "type": "string"
+          },
+          "enr": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "day": {
+                  "type": "integer"
+                },
+                "ttl": {
+                  "type": "string"
+                },
+                "dtl": {
+                  "type": "string"
+                }
+              },
+              "required": [
+                "day",
+                "dtl",
+                "ttl"
+              ]
+            }
+          },
+          "wen": {
+            "type": "string"
+          },
+          "pur": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "puk": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "puu": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "pus": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "pup": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "puq": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "required": [
+          "_id",
+          "dek",
+          "did",
+          "dir",
+          "lbl",
+          "pub",
+          "typ"
+        ]
+      }
+    },
+    "SF": {
+      "type": "string"
+    },
+    "_fd": {
+      "type": "integer"
+    },
+    "_ts": {
+      "type": "integer"
+    },
+    "_re": {
+      "type": "integer"
+    }
+  },
+  "required": [
+    "SF",
+    "_fd",
+    "_ts",
+    "adr",
+    "atg",
+    "cur",
+    "dcc",
+    "dek",
+    "eml",
+    "end",
+    "evg",
+    "nam",
+    "org",
+    "tpl",
+    "typ",
+    "uid"
+  ]
+}

memberPlumbing/tests/test_membershipworks.py

@@ -0,0 +1,179 @@
+import datetime
+from unittest.mock import patch
+
+import pytest
+import responses
+
+from ..MembershipWorks import BASE_URL, MembershipWorks, MembershipWorksRemoteError
+
+
+@responses.activate
+def test_login():
+    membershipworks = MembershipWorks()
+
+    responses.add(
+        responses.POST, BASE_URL + "usr", json={"SF": "test1", "org": 10000}, status=200
+    )
+
+    membershipworks.login("test1@example.com", "test2")
+
+    assert membershipworks.auth_token == "test1"
+    assert membershipworks.org_num == 10000
+
+
+@responses.activate
+def test_login_fail():
+    membershipworks = MembershipWorks()
+
+    responses.add(
+        responses.POST, BASE_URL + "usr", json={"error": "Login/password not accepted"},
+    )
+
+    with pytest.raises(MembershipWorksRemoteError) as e:
+        membershipworks.login("test1@example.com", "test2")
+
+        assert (
+            e.exception.args[0]
+            == "Error when attempting login: 200 OK\n"
+            + '{"error":"Login/password not accepted"}'
+        )
+
+
+@pytest.fixture
+def membershipworks():
+    mw = MembershipWorks()
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            responses.POST,
+            BASE_URL + "usr",
+            json={
+                "SF": "test1",
+                "org": 10000,
+                "dek": [{"_id": "12345", "lbl": "test", "dek": 1}],
+            },
+        )
+
+        mw.login("test1@example.com", "test2")
+
+    return mw
+
+
+@responses.activate
+def test_get_member_ids(membershipworks: MembershipWorks):
+    responses.add(
+        responses.GET,
+        BASE_URL + "ylp",
+        json={
+            "typ": "a",
+            "usr": [{"uid": "asdf", "nam": "test", "rnk": 12.345}],
+            "_re": 631,
+        },
+    )
+    ids = membershipworks.get_member_ids(["test"])
+    assert ids == ["asdf"]
+
+
+@responses.activate
+@patch("memberPlumbing.MembershipWorks.MembershipWorks.get_member_ids")
+def test_get_members(mockIDs, membershipworks: MembershipWorks):
+    mockIDs.return_value = ["asdf"]
+
+    responses.add(
+        responses.POST,
+        BASE_URL + "csv",
+        body=""""Account Name","First Name","Last Name","Account ID","Parent Account ID"
+"bob whatever", "bob", "whatever", "aaaaaaaaaaaaaaaaaaaaaaaa", ""
+"george whatever", "george", "whatever", "bbbbbbbbbbbbbbbbbbbbbbbb", ""
+""",
+    )
+
+    members = membershipworks.get_members(["test"], "_id,nam")
+
+    assert members == [
+        {
+            "Account Name": "bob whatever",
+            "First Name": ' "bob"',
+            "Last Name": ' "whatever"',
+            "Account ID": ' "aaaaaaaaaaaaaaaaaaaaaaaa"',
+            "Parent Account ID": ' ""',
+        },
+        {
+            "Account Name": "george whatever",
+            "First Name": ' "george"',
+            "Last Name": ' "whatever"',
+            "Account ID": ' "bbbbbbbbbbbbbbbbbbbbbbbb"',
+            "Parent Account ID": ' ""',
+        },
+    ]
+
+
+@responses.activate
+def test_get_transactions_csv(membershipworks: MembershipWorks):
+    responses.add(
+        responses.GET,
+        BASE_URL + "csv",
+        body=""""Date","Name","Contact Person","Full Address","Street","City","State/Province","Postal Code","Country","Phone","Email","Membership Sub-Total","Event Sub-Total","Donation Sub-Total","Cart Sub-Total","Other Sub-Total","Handling","Total Tax","Transaction Total","Transaction Fee","Total Payment Due","Transaction Type","For","Items","Payment ID","Account ID","Discount Code","Note"
+"Jan 01, 2020","whatever","","PO Box 0, fakeplace NH, US","PO Box 0","fakeplace","NH","","US","000-000-0000","example@example.com",1,0,0,0,0,0,0,1,0.32,0,"Membership","","CMS Membership on hold - Membership on hold","ch_aaaaaaaaaaaaaaaaaaaaaaaa","aaaaaaaaaaaaaaaaaaaaaaaa","",""
+""",
+    )
+
+    transactions = membershipworks.get_transactions(
+        datetime.datetime(2000, 1, 1), datetime.datetime(2021, 1, 1)
+    )
+
+    assert transactions == [
+        {
+            "Date": "Jan 01, 2020",
+            "Name": "whatever",
+            "Contact Person": "",
+            "Full Address": "PO Box 0, fakeplace NH, US",
+            "Street": "PO Box 0",
+            "City": "fakeplace",
+            "State/Province": "NH",
+            "Postal Code": "",
+            "Country": "US",
+            "Phone": "000-000-0000",
+            "Email": "example@example.com",
+            "Membership Sub-Total": "1",
+            "Event Sub-Total": "0",
+            "Donation Sub-Total": "0",
+            "Cart Sub-Total": "0",
+            "Other Sub-Total": "0",
+            "Handling": "0",
+            "Total Tax": "0",
+            "Transaction Total": "1",
+            "Transaction Fee": "0.32",
+            "Total Payment Due": "0",
+            "Transaction Type": "Membership",
+            "For": "",
+            "Items": "CMS Membership on hold - Membership on hold",
+            "Payment ID": "ch_aaaaaaaaaaaaaaaaaaaaaaaa",
+            "Account ID": "aaaaaaaaaaaaaaaaaaaaaaaa",
+            "Discount Code": "",
+            "Note": "",
+        }
+    ]
+
+
+@responses.activate
+def test_get_transactions_json(membershipworks: MembershipWorks):
+    data_json = [
+        {
+            "cur": "usd",
+            "sid": "ch_aaaaaaaaaaaaaaaaaaaaaaaa",
+            "typ": 12,
+            "_dp": 1585100000,
+            "sum": 1,
+            "fee": 0.32,
+            "uid": "aaaaaaaaaaaaaaaaaaaaaaaa",
+            "nam": "whatever",
+        }
+    ]
+    responses.add(responses.GET, BASE_URL + "csv", json=data_json)
+
+    transactions = membershipworks.get_transactions(
+        datetime.datetime(2000, 1, 1), datetime.datetime(2021, 1, 1), json=True
+    )
+    assert "&txl=" in responses.calls[0].request.url  # json flag
+    assert transactions == data_json

memberPlumbing/tests/test_membershipworks_contract.py

@@ -0,0 +1,31 @@
+import json
+import os
+
+import jsonschema
+import requests
+
+import pytest
+
+from ..common import MEMBERSHIPWORKS_PASSWORD, MEMBERSHIPWORKS_USERNAME
+from ..MembershipWorks import BASE_URL
+
+schemas_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "json-schemas")
+
+
+def test_login():
+    r = requests.post(
+        BASE_URL + "usr",
+        data={
+            "_st": "all",
+            "eml": MEMBERSHIPWORKS_USERNAME,
+            "org": "10000",
+            "pwd": MEMBERSHIPWORKS_PASSWORD,
+        },
+    )
+
+    with open(os.path.join(schemas_dir, "usr.schema")) as f:
+        schema = json.load(f)
+
+    usr = r.json()
+
+    jsonschema.validate(usr, schema)

memberPlumbing/ucsAccounts.py

@@ -5,15 +5,13 @@ import re
 import string
 
 from udm_rest_client.udm import UDM
-from udm_rest_client.exceptions import NoObject, UdmError
+from udm_rest_client.exceptions import NoObject
 
 from .config import Config
 
 USER_BASE = "cn=users,dc=sawtooth,dc=claremontmakerspace,dc=org"
 GROUP_BASE = "cn=groups,dc=sawtooth,dc=claremontmakerspace,dc=org"
-GROUPS_REGEX = "|".join(
+GROUPS_REGEX = "|".join(["Certified: .*", "Access .*\?", "CMS .*", "Volunteer: .*"])
-    ["Certified: .*", "Access .*\?", "CMS .*", "Volunteer: .*", "Database .*"]
-)
 RAND_PW_LEN = 20
 
 
@@ -29,7 +27,7 @@ def sanitize_group_name(name):
 
 # From an API error message: "Username must only contain numbers, letters and dots!"
 def sanitize_user_name(name):
-    return re.sub(r"[^0-9a-z.]", ".", name.lower()).strip(".")
+    return re.sub(r"[^0-9a-z.]", ".", name.lower())
 
 
 async def make_groups(group_mod, members):
@@ -54,6 +52,7 @@ async def _main():
         ["Members", "CMS Staff"], "lvl,phn,eml,lbl,nam,end,_id"
     )
 
+    # TODO: security!
     async with UDM(**config.UCS) as udm:
         user_mod = udm.get("users/user")
         group_mod = udm.get("groups/group")
@@ -66,7 +65,7 @@ async def _main():
             try:  # try to get an existing user to update
                 user = await user_mod.get(f"uid={username},{USER_BASE}")
             except NoObject:  # create a new user
-                # TODO: search by employeeNumber and rename users when needed
+                # TODO: search by employeeID and rename users when needed
                 user = await user_mod.new()
 
                 # set a random password and ensure it is changed at next login
@@ -110,17 +109,10 @@ async def _main():
                 if re.match(GROUPS_REGEX, group) is not None and value != ""
             ]
             # groups not from this script
-            other_old_groups = [
+            other_old_groups = [g for g in user.props.groups if g[3:].startswith("MW_")]
-                g for g in user.props.groups if not g[3:].startswith("MW_")
-            ]
             user.props.groups = other_old_groups + new_groups
 
-            try:
+            await user.save()
-                await user.save()
-            except UdmError:
-                print("Failed to save user", username)
-                print(user.props)
-                raise
 
 
 def main():

memberPlumbing/upcomingEvents.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-
-from datetime import datetime
-
-from .config import Config
-from .MembershipWorks import MembershipWorks
-
-
-def format_event(membershipworks: MembershipWorks, event):
-    event_details = membershipworks.get_event_by_eid(event["eid"])
-    url = (
-        "https://claremontmakerspace.org/events/#!event/register/"
-        + event_details["url"]
-    )
-    if "lgo" in event_details:
-        img = f"""<img class="aligncenter" width="500" height="500" src="{event_details['lgo']['l']}">"""
-    else:
-        img = ""
-    # print(json.dumps(event_details))
-    return f"""<h2 style="text-align: center;">
-  <a href="{url}">
-    {img}
-    {event_details['ttl']}
-  </a>
-</h2>
-<div>{event_details['szp']} &mdash; {event_details['ezp']}</div>
-<div>
-  {event_details['dtl']}
-</div>
-
-<a href="{url}">Register for this class now!</a>"""
-
-
-def main():
-    config = Config()
-
-    membershipworks = config.membershipworks
-    events = membershipworks.get_events_list(datetime.now())
-    if "error" in events:
-        print("Error:", events["error"])
-        return
-
-    events_list = "\n<hr />\n\n".join(
-        format_event(membershipworks, event)
-        for event in events["evt"]
-        if event["ttl"] != "[TEMPLATE FOR COPYING]"
-    )
-    header = """<p><img class="aligncenter size-medium wp-image-2319" src="https://claremontmakerspace.org/wp-content/uploads/2019/03/CMS-Logo-b-y-g-300x168.png" alt="" width="300" height="168" /></a></p>
-<p>Greetings Upper Valley Makers:</p>
-<p>We have an exciting list of upcoming classes at the Claremont MakerSpace that we think might interest you.</p>
-<strong>For most classes and events, CMS MEMBERSHIP IS NOT REQUIRED.</strong> That said, members receive a discount on registration and there are some classes/events that are for members only (this will be clearly noted in the event description).
-
-<strong>Class policies</strong> (liability waiver, withdrawal, cancellation, etc.) can be found <a href="https://claremontmakerspace.org/class-policies/" data-wpel-link="internal">here</a>.
-
-<strong>Please note: </strong>The Claremont MakerSpace currently requires masks for all visitors and members.
-
-<strong>Instructors:</strong> Interested in teaching a class at CMS? Please fill out our <a href="https://docs.google.com/forms/d/e/1FAIpQLSdJyEVRJxzIczG784VkOm_DsNyv-VXRXYzlis8qlMdEOvHGpQ/viewform" target="_blank" rel="noreferrer noopener external" data-wpel-link="external">Class Proposal Form</a><a href="https://docs.google.com/forms/d/e/1FAIpQLSdJyEVRJxzIczG784VkOm_DsNyv-VXRXYzlis8qlMdEOvHGpQ/viewform" target="_blank" rel="noreferrer noopener external" data-wpel-link="external">.</a>
-
-<strong>Tours:</strong> Want to see what the Claremont MakerSpace is all about? Tours are by appointment only due to COVID-19 restrictions. <a href="https://tickets.claremontmakerspace.org/open.php" target="_blank" rel="noreferrer noopener external" data-wpel-link="external">Contact Us</a> to schedule your tour where you can learn about all the awesome tools that the CMS offers access to, as well as how membership, classes, and studio spaces work.
-
-<hr />
-"""
-
-    footer = """
-<hr />
-
-<div>Happy Makin’!</div>
-<div>We are grateful for all of the public support that our 501(c)(3), non-profit organization receives. If you’d like to make a donation,please visit the <a href="https://claremontmakerspace.org/support/"><strong>Support Us page</strong></a> of our website.</div>
-</div>
-"""
-
-    print(header, events_list, footer)
-
-
-if __name__ == "__main__":
-    main()

pyproject.toml

@@ -17,23 +17,39 @@ authors = ["Adam Goldsmith <adam@adamgoldsmith.name>"]
 [tool.poetry.dependencies]
 python = "^3.7"
 requests = "^2.23.0"
-"ruamel.yaml" = "^0.17.20"
+"ruamel.yaml" = "^0.16.10"
 bitstring = "^3.1.6"
 lxml = "^4.5.0"
 peewee = "^3.13.2"
-mysqlclient = "^2.1.0"
+mysqlclient = "^1.4.6"
-udm-rest-client = "^1.0.6"
+udm-rest-client = "^0.4.0"
 
 [tool.poetry.dev-dependencies]
-black = "^22.3.0"
+black = "^19.10b0"
-isort = "^5.10.1"
+isort = "^4.3.21"
+taskipy = "^1.2.1"
+pytest = "^5.4.2"
+coverage = {extras = ["toml"], version = "^5.1"}
+
+[tool.taskipy.tasks]
+test = "pytest"
+mypy = "mypy --strict memberPlumbing/hid/ stubs"
+cov = "coverage run && coverage html"
+
+[tool.coverage.run]
+branch = true
+source = ["memberPlumbing"]
+omit = ["*/tests/*"]
+command_line = "-m pytest"
+
+[tool.coverage.report]
+skip_empty = true
 
 [tool.poetry.scripts]
 doorUpdater = 'memberPlumbing.doorUpdater:main'
 hidEvents = 'memberPlumbing.hidEvents:main'
 sqlExport = 'memberPlumbing.sqlExport:main'
 ucsAccounts = 'memberPlumbing.ucsAccounts:main'
-upcomingEvents = 'memberPlumbing.upcomingEvents:main'
 
 [build-system]
 requires = ["poetry>=0.12"]

systemd/ucsAccounts.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=Update UCS Accounts
-OnFailure=status-email-admin@%n.service
-
-[Service]
-User=adam
-Type=oneshot
-TimeoutStartSec=600
-WorkingDirectory=/home/adam/memberPlumbing/
-ExecStart=/usr/bin/python3 -u /home/adam/.poetry/bin/poetry run ucsAccounts

systemd/ucsAccounts.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=Hourly UCS Accounts update
-
-[Timer]
-OnCalendar=*:0/15
-Persistent=true
-
-[Install]
-WantedBy=timers.target