diff --git a/membershipworks/__init__.py b/membershipworks/__init__.py index e69de29..c609103 100644 --- a/membershipworks/__init__.py +++ b/membershipworks/__init__.py @@ -0,0 +1 @@ +from .membershipworks_api import MembershipWorks, MembershipWorksRemoteError diff --git a/membershipworks/management/__init__.py b/membershipworks/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/membershipworks/management/commands/__init__.py b/membershipworks/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/membershipworks/management/commands/scrape_membershipworks.py b/membershipworks/management/commands/scrape_membershipworks.py new file mode 100644 index 0000000..c311a40 --- /dev/null +++ b/membershipworks/management/commands/scrape_membershipworks.py @@ -0,0 +1,48 @@ +from django.conf import settings +from django.core.management.base import BaseCommand +from django.db import transaction + +from membershipworks.models import Member, Flag +from membershipworks import MembershipWorks + + +class Command(BaseCommand): + def flags_for_member(self, csv_member, all_flags): + # update member's flags + for flag in all_flags: + if flag.type != "folder": # currently no way to retrieve this info + if csv_member[flag.name] == flag.name: + yield flag + + def update_flags(self, mw_flags) -> list[Flag]: + for typ, flags_of_type in mw_flags.items(): + for name, id in flags_of_type.items(): + flag = Flag(id=id, name=name, type=typ[:-1]) + flag.save() + yield flag + + @transaction.atomic() + def handle(self, *args, **options): + membershipworks = MembershipWorks() + membershipworks.login( + settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD + ) + + print("Updating flags (labels, levels, and addons)") + flags = list(self.update_flags(membershipworks._parse_flags())) + + print("Getting/Updating members...") + members = membershipworks.get_all_members() + for csv_member in members: + for field_id, field in membershipworks._all_fields().items(): + # convert checkboxes to real booleans + if field.get("typ") == 8 and field["lbl"] in csv_member: + csv_member[field["lbl"]] = ( + True if csv_member[field["lbl"]] == "Y" else False + ) + + # create/update member + member = Member.from_csv_dict(csv_member) + member.clean_fields() + member.save() + member.flags.set(self.flags_for_member(csv_member, flags)) diff --git a/membershipworks/membershipworks_api.py b/membershipworks/membershipworks_api.py new file mode 100644 index 0000000..2349103 --- /dev/null +++ b/membershipworks/membershipworks_api.py @@ -0,0 +1,231 @@ +import csv +from io import StringIO + +import requests + +BASE_URL = "https://api.membershipworks.com" + +# extracted from `SF._is.crm` in https://cdn.membershipworks.com/all.js +CRM = { + 0: "Note", + 4: "Profile Updated", + 8: "Scheduled/Reminder Email", + 9: "Renewal Notice", + 10: "Join Date", + 11: "Next Renewal Date", + 12: "Membership Payment", + 13: "Donation", + 14: "Event Activity", + 15: "Conversation", + 16: "Contact Change", + 17: "Label Change", + 18: "Other Payment", + 19: "Cart Payment", + 20: "Payment Failed", + 21: "Billing Updated", + 22: "Form Checkout", + 23: "Event Payment", + 24: "Invoice", + 25: "Invoice Payment", + 26: "Renewal", + 27: "Payment", +} + +# Types of fields, extracted from a html snippet in all.js + some guessing +typ = { + 1: "Text input", + 2: "Password", # inferred from data + 3: "Simple text area", + 4: "Rich text area", + 7: "Address", + 8: "Check box", + 9: "Select", + 11: "Display value stored in field (ie. read only)", + 12: "Required waiver/terms", +} + +# more constants, this time extracted from the members csv export in all.js +staticFlags = { + "pos": {"lbl": "Position/relation (contacts)"}, + "nte": {"lbl": "Admin note (contacts)"}, + "pwd": {"lbl": "Password"}, + "lgo": {"lbl": "Business card image URLs"}, + "pfx": {"lbl": "Profile gallery image URLs"}, + "lvl": {"lbl": "Membership levels"}, + "aon": {"lbl": "Membership add-ons"}, + "lbl": {"lbl": "Labels"}, + "joi": {"lbl": "Join date"}, + "end": {"lbl": "Renewal date"}, + "spy": {"lbl": "Billing method"}, + "rid": {"lbl": "Auto recurring billing ID"}, + "ipa": {"lbl": "IP address"}, + "_id": {"lbl": "Account ID"}, +} + + +class MembershipWorksRemoteError(Exception): + def __init__(self, reason, r): + super().__init__( + f"Error when attempting {reason}: {r.status_code} {r.reason}\n{r.text}" + ) + + +class MembershipWorks: + def __init__(self): + self.sess = requests.Session() + self.org_info = None + self.auth_token = None + self.org_num = None + + def login(self, username, password): + """Authenticate against the membershipworks api""" + r = self.sess.post( + BASE_URL + "/v2/account/session", + data={"eml": username, "pwd": password}, + headers={"X-Org": "10000"}, + ) + if r.status_code != 200 or "SF" not in r.json(): + raise MembershipWorksRemoteError("login", r) + self.org_info = r.json() + self.auth_token = self.org_info["SF"] + self.org_num = self.org_info["org"] + self.sess.headers.update( + { + "X-Org": str(self.org_num), + "X-Role": "admin", + "Authorization": "Bearer " + self.auth_token, + } + ) + + def _inject_auth(self, kwargs): + # TODO: should probably be a decorator or something + if self.auth_token is None: + raise RuntimeError("Not Logged in to MembershipWorks") + # add auth token to params + if "params" not in kwargs: + kwargs["params"] = {} + kwargs["params"]["SF"] = self.auth_token + + def _get_v1(self, *args, **kwargs): + self._inject_auth(kwargs) + # TODO: should probably do some error handling in here + return requests.get(*args, **kwargs) + + def _post_v1(self, *args, **kwargs): + self._inject_auth(kwargs) + # TODO: should probably do some error handling in here + return requests.post(*args, **kwargs) + + def _all_fields(self): + """Parse out a list of fields from the org data. + + Is this terrible? Yes. Also, not dissimilar to how MW does it + in all.js. + """ + fields = staticFlags.copy() + + # TODO: this will take the later option, if the same field + # used in mulitple places. I don't know which lbl is used for + # csv export + + # anm: member signup, acc: member manage, adm: admin manage + for screen_type in ["anm", "acc", "adm"]: + for box in self.org_info["tpl"][screen_type]: + for element in box["box"]: + if type(element["dat"]) != str: + for field in element["dat"]: + if "_id" in field: + if field["_id"] not in fields: + fields[field["_id"]] = field + + return fields + + def _parse_flags(self): + """Parse the flags out of the org data. + + This is terrible, and there might be a better way to do this. + """ + ret = {"folders": {}, "levels": {}, "addons": {}, "labels": {}} + + for dek in self.org_info["dek"]: + # TODO: there must be a better way. this is stupid + if dek["dek"] == 1: + ret["folders"][dek["lbl"]] = dek["did"] + elif "cur" in dek: + ret["levels"][dek["lbl"]] = dek["did"] + elif "mux" in dek: + ret["addons"][dek["lbl"]] = dek["did"] + else: + ret["labels"][dek["lbl"]] = dek["did"] + + return ret + + def get_member_ids(self, folders): + folder_map = self._parse_flags()["folders"] + + r = self.sess.get( + BASE_URL + "/v2/accounts", + params={"dek": ",".join([folder_map[f] for f in folders])}, + ) + if r.status_code != 200 or "usr" not in r.json(): + raise MembershipWorksRemoteError("user listing", r) + + # get list of member ID matching the search + # dedup with set() to work around people with alt uids + # TODO: figure out why people have alt uids + return set(user["uid"] for user in r.json()["usr"]) + + # TODO: has issues with aliasing header names: + # ex: "Personal Studio Space" Label vs Membership Addon/Field + def get_members(self, folders, columns): + """Pull the members csv from the membershipworks api + folders: a list of the names of the folders to get + (see folder_map in this function for mapping to ids) + columns: which columns to get""" + ids = self.get_member_ids(folders) + + # get members CSV + # TODO: maybe can just use previous get instead? would return JSON + r = self._post_v1( + BASE_URL + "/v1/csv", + data={ + "_rt": "946702800", # unknown + "mux": "", # unknown + "tid": ",".join(ids), # ids of members to get + "var": columns, + }, + ) + if r.status_code != 200: + raise MembershipWorksRemoteError("csv generation", r) + return list(csv.DictReader(StringIO(r.text))) + + def get_transactions(self, start_date, end_date, json=False): + """Get the transactions between start_date and end_date + + Dates can be datetime.date or datetime.datetime + + json gets a different version of the transactions list, + which contains a different set information + """ + r = self._get_v1( + BASE_URL + "/v1/csv", + params={ + "crm": ",".join(str(k) for k in CRM.keys()), + **({"txl": ""} if json else {}), + "sdp": start_date.strftime("%s"), + "edp": end_date.strftime("%s"), + }, + ) + if r.status_code != 200: + raise MembershipWorksRemoteError("csv generation", r) + if json: + return r.json() + else: + return list(csv.DictReader(StringIO(r.text))) + + def get_all_members(self): + """Get all the data for all the members""" + folders = self._parse_flags()["folders"].keys() + fields = self._all_fields() + members = self.get_members(folders, ",".join(fields.keys())) + return members diff --git a/membershipworks/models.py b/membershipworks/models.py index c14eb9f..996affe 100644 --- a/membershipworks/models.py +++ b/membershipworks/models.py @@ -1,4 +1,5 @@ from typing import Optional +from datetime import datetime import django.core.mail.message from django.conf import settings @@ -6,7 +7,47 @@ from django.db import models from django.db.models import Exists, OuterRef -class Flag(models.Model): +class BaseModel(models.Model): + _csv_headers_override = {} + _date_fields = {} + + class Meta: + abstract = True + + @classmethod + def _get_header_map(cls): + header_map = { + field.db_column: field.name + for field in cls._meta.get_fields() + if not field.auto_created + } + header_map.update(cls._csv_headers_override) + + return header_map + + @classmethod + def _remap_headers(cls, data): + header_map = cls._get_header_map() + for k, v in data.items(): + if k in header_map: + yield header_map.get(k), v + + @classmethod + def from_csv_dict(cls, data): + data = data.copy() + + # parse date fields to datetime objects + for field, fmt in cls._date_fields.items(): + if data[field]: + data[field] = datetime.strptime(str(data[field]), fmt) + else: + # convert empty string to None to make NULL in SQL + data[field] = None + + return cls(**dict(cls._remap_headers(data))) + + +class Flag(BaseModel): id = models.CharField(max_length=24, primary_key=True) name = models.TextField(null=True, blank=True) type = models.CharField(max_length=6) @@ -44,7 +85,7 @@ class MemberQuerySet(models.QuerySet): # TODO: is this still a temporal table? -class Member(models.Model): +class Member(BaseModel): objects = MemberQuerySet.as_manager() uid = models.CharField(max_length=24, primary_key=True) @@ -172,6 +213,21 @@ class Member(models.Model): ) flags = models.ManyToManyField(Flag, through="MemberFlag", related_name="members") + _csv_headers_override = { + "Account ID": "uid", + "Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:": "how_did_you_hear", + "Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.": "authorize_charge", + "I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.": "policy_agreement", + } + + _date_fields = { + "Join Date": "%b %d, %Y", + "Renewal Date": "%b %d, %Y", + "Audit Date": "%m/%d/%Y", + "Membership Agreement signed and on file date.": "%m/%d/%Y", + "Waiver form signed and on file date.": "%m/%d/%Y", + } + def __str__(self): return f"{self.account_name}" @@ -205,7 +261,7 @@ class Member(models.Model): ) -class MemberFlag(models.Model): +class MemberFlag(BaseModel): member = models.ForeignKey(Member, on_delete=models.PROTECT, db_column="uid") flag = models.ForeignKey(Flag, on_delete=models.PROTECT)