membershipworks: Add API module and command for scraping data

This commit is contained in:
Adam Goldsmith 2022-02-28 16:06:10 -05:00
parent 6b3113e839
commit dfacf813e2
6 changed files with 339 additions and 3 deletions

View File

@ -0,0 +1 @@
from .membershipworks_api import MembershipWorks, MembershipWorksRemoteError

View File

View File

@ -0,0 +1,48 @@
from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import transaction
from membershipworks.models import Member, Flag
from membershipworks import MembershipWorks
class Command(BaseCommand):
def flags_for_member(self, csv_member, all_flags):
# update member's flags
for flag in all_flags:
if flag.type != "folder": # currently no way to retrieve this info
if csv_member[flag.name] == flag.name:
yield flag
def update_flags(self, mw_flags) -> list[Flag]:
for typ, flags_of_type in mw_flags.items():
for name, id in flags_of_type.items():
flag = Flag(id=id, name=name, type=typ[:-1])
flag.save()
yield flag
@transaction.atomic()
def handle(self, *args, **options):
membershipworks = MembershipWorks()
membershipworks.login(
settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
)
print("Updating flags (labels, levels, and addons)")
flags = list(self.update_flags(membershipworks._parse_flags()))
print("Getting/Updating members...")
members = membershipworks.get_all_members()
for csv_member in members:
for field_id, field in membershipworks._all_fields().items():
# convert checkboxes to real booleans
if field.get("typ") == 8 and field["lbl"] in csv_member:
csv_member[field["lbl"]] = (
True if csv_member[field["lbl"]] == "Y" else False
)
# create/update member
member = Member.from_csv_dict(csv_member)
member.clean_fields()
member.save()
member.flags.set(self.flags_for_member(csv_member, flags))

View File

@ -0,0 +1,231 @@
import csv
from io import StringIO
import requests
BASE_URL = "https://api.membershipworks.com"
# extracted from `SF._is.crm` in https://cdn.membershipworks.com/all.js
CRM = {
0: "Note",
4: "Profile Updated",
8: "Scheduled/Reminder Email",
9: "Renewal Notice",
10: "Join Date",
11: "Next Renewal Date",
12: "Membership Payment",
13: "Donation",
14: "Event Activity",
15: "Conversation",
16: "Contact Change",
17: "Label Change",
18: "Other Payment",
19: "Cart Payment",
20: "Payment Failed",
21: "Billing Updated",
22: "Form Checkout",
23: "Event Payment",
24: "Invoice",
25: "Invoice Payment",
26: "Renewal",
27: "Payment",
}
# Types of fields, extracted from a html snippet in all.js + some guessing
typ = {
1: "Text input",
2: "Password", # inferred from data
3: "Simple text area",
4: "Rich text area",
7: "Address",
8: "Check box",
9: "Select",
11: "Display value stored in field (ie. read only)",
12: "Required waiver/terms",
}
# more constants, this time extracted from the members csv export in all.js
staticFlags = {
"pos": {"lbl": "Position/relation (contacts)"},
"nte": {"lbl": "Admin note (contacts)"},
"pwd": {"lbl": "Password"},
"lgo": {"lbl": "Business card image URLs"},
"pfx": {"lbl": "Profile gallery image URLs"},
"lvl": {"lbl": "Membership levels"},
"aon": {"lbl": "Membership add-ons"},
"lbl": {"lbl": "Labels"},
"joi": {"lbl": "Join date"},
"end": {"lbl": "Renewal date"},
"spy": {"lbl": "Billing method"},
"rid": {"lbl": "Auto recurring billing ID"},
"ipa": {"lbl": "IP address"},
"_id": {"lbl": "Account ID"},
}
class MembershipWorksRemoteError(Exception):
def __init__(self, reason, r):
super().__init__(
f"Error when attempting {reason}: {r.status_code} {r.reason}\n{r.text}"
)
class MembershipWorks:
def __init__(self):
self.sess = requests.Session()
self.org_info = None
self.auth_token = None
self.org_num = None
def login(self, username, password):
"""Authenticate against the membershipworks api"""
r = self.sess.post(
BASE_URL + "/v2/account/session",
data={"eml": username, "pwd": password},
headers={"X-Org": "10000"},
)
if r.status_code != 200 or "SF" not in r.json():
raise MembershipWorksRemoteError("login", r)
self.org_info = r.json()
self.auth_token = self.org_info["SF"]
self.org_num = self.org_info["org"]
self.sess.headers.update(
{
"X-Org": str(self.org_num),
"X-Role": "admin",
"Authorization": "Bearer " + self.auth_token,
}
)
def _inject_auth(self, kwargs):
# TODO: should probably be a decorator or something
if self.auth_token is None:
raise RuntimeError("Not Logged in to MembershipWorks")
# add auth token to params
if "params" not in kwargs:
kwargs["params"] = {}
kwargs["params"]["SF"] = self.auth_token
def _get_v1(self, *args, **kwargs):
self._inject_auth(kwargs)
# TODO: should probably do some error handling in here
return requests.get(*args, **kwargs)
def _post_v1(self, *args, **kwargs):
self._inject_auth(kwargs)
# TODO: should probably do some error handling in here
return requests.post(*args, **kwargs)
def _all_fields(self):
"""Parse out a list of fields from the org data.
Is this terrible? Yes. Also, not dissimilar to how MW does it
in all.js.
"""
fields = staticFlags.copy()
# TODO: this will take the later option, if the same field
# used in mulitple places. I don't know which lbl is used for
# csv export
# anm: member signup, acc: member manage, adm: admin manage
for screen_type in ["anm", "acc", "adm"]:
for box in self.org_info["tpl"][screen_type]:
for element in box["box"]:
if type(element["dat"]) != str:
for field in element["dat"]:
if "_id" in field:
if field["_id"] not in fields:
fields[field["_id"]] = field
return fields
def _parse_flags(self):
"""Parse the flags out of the org data.
This is terrible, and there might be a better way to do this.
"""
ret = {"folders": {}, "levels": {}, "addons": {}, "labels": {}}
for dek in self.org_info["dek"]:
# TODO: there must be a better way. this is stupid
if dek["dek"] == 1:
ret["folders"][dek["lbl"]] = dek["did"]
elif "cur" in dek:
ret["levels"][dek["lbl"]] = dek["did"]
elif "mux" in dek:
ret["addons"][dek["lbl"]] = dek["did"]
else:
ret["labels"][dek["lbl"]] = dek["did"]
return ret
def get_member_ids(self, folders):
folder_map = self._parse_flags()["folders"]
r = self.sess.get(
BASE_URL + "/v2/accounts",
params={"dek": ",".join([folder_map[f] for f in folders])},
)
if r.status_code != 200 or "usr" not in r.json():
raise MembershipWorksRemoteError("user listing", r)
# get list of member ID matching the search
# dedup with set() to work around people with alt uids
# TODO: figure out why people have alt uids
return set(user["uid"] for user in r.json()["usr"])
# TODO: has issues with aliasing header names:
# ex: "Personal Studio Space" Label vs Membership Addon/Field
def get_members(self, folders, columns):
"""Pull the members csv from the membershipworks api
folders: a list of the names of the folders to get
(see folder_map in this function for mapping to ids)
columns: which columns to get"""
ids = self.get_member_ids(folders)
# get members CSV
# TODO: maybe can just use previous get instead? would return JSON
r = self._post_v1(
BASE_URL + "/v1/csv",
data={
"_rt": "946702800", # unknown
"mux": "", # unknown
"tid": ",".join(ids), # ids of members to get
"var": columns,
},
)
if r.status_code != 200:
raise MembershipWorksRemoteError("csv generation", r)
return list(csv.DictReader(StringIO(r.text)))
def get_transactions(self, start_date, end_date, json=False):
"""Get the transactions between start_date and end_date
Dates can be datetime.date or datetime.datetime
json gets a different version of the transactions list,
which contains a different set information
"""
r = self._get_v1(
BASE_URL + "/v1/csv",
params={
"crm": ",".join(str(k) for k in CRM.keys()),
**({"txl": ""} if json else {}),
"sdp": start_date.strftime("%s"),
"edp": end_date.strftime("%s"),
},
)
if r.status_code != 200:
raise MembershipWorksRemoteError("csv generation", r)
if json:
return r.json()
else:
return list(csv.DictReader(StringIO(r.text)))
def get_all_members(self):
"""Get all the data for all the members"""
folders = self._parse_flags()["folders"].keys()
fields = self._all_fields()
members = self.get_members(folders, ",".join(fields.keys()))
return members

View File

@ -1,4 +1,5 @@
from typing import Optional from typing import Optional
from datetime import datetime
import django.core.mail.message import django.core.mail.message
from django.conf import settings from django.conf import settings
@ -6,7 +7,47 @@ from django.db import models
from django.db.models import Exists, OuterRef from django.db.models import Exists, OuterRef
class Flag(models.Model): class BaseModel(models.Model):
_csv_headers_override = {}
_date_fields = {}
class Meta:
abstract = True
@classmethod
def _get_header_map(cls):
header_map = {
field.db_column: field.name
for field in cls._meta.get_fields()
if not field.auto_created
}
header_map.update(cls._csv_headers_override)
return header_map
@classmethod
def _remap_headers(cls, data):
header_map = cls._get_header_map()
for k, v in data.items():
if k in header_map:
yield header_map.get(k), v
@classmethod
def from_csv_dict(cls, data):
data = data.copy()
# parse date fields to datetime objects
for field, fmt in cls._date_fields.items():
if data[field]:
data[field] = datetime.strptime(str(data[field]), fmt)
else:
# convert empty string to None to make NULL in SQL
data[field] = None
return cls(**dict(cls._remap_headers(data)))
class Flag(BaseModel):
id = models.CharField(max_length=24, primary_key=True) id = models.CharField(max_length=24, primary_key=True)
name = models.TextField(null=True, blank=True) name = models.TextField(null=True, blank=True)
type = models.CharField(max_length=6) type = models.CharField(max_length=6)
@ -44,7 +85,7 @@ class MemberQuerySet(models.QuerySet):
# TODO: is this still a temporal table? # TODO: is this still a temporal table?
class Member(models.Model): class Member(BaseModel):
objects = MemberQuerySet.as_manager() objects = MemberQuerySet.as_manager()
uid = models.CharField(max_length=24, primary_key=True) uid = models.CharField(max_length=24, primary_key=True)
@ -172,6 +213,21 @@ class Member(models.Model):
) )
flags = models.ManyToManyField(Flag, through="MemberFlag", related_name="members") flags = models.ManyToManyField(Flag, through="MemberFlag", related_name="members")
_csv_headers_override = {
"Account ID": "uid",
"Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:": "how_did_you_hear",
"Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.": "authorize_charge",
"I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.": "policy_agreement",
}
_date_fields = {
"Join Date": "%b %d, %Y",
"Renewal Date": "%b %d, %Y",
"Audit Date": "%m/%d/%Y",
"Membership Agreement signed and on file date.": "%m/%d/%Y",
"Waiver form signed and on file date.": "%m/%d/%Y",
}
def __str__(self): def __str__(self):
return f"{self.account_name}" return f"{self.account_name}"
@ -205,7 +261,7 @@ class Member(models.Model):
) )
class MemberFlag(models.Model): class MemberFlag(BaseModel):
member = models.ForeignKey(Member, on_delete=models.PROTECT, db_column="uid") member = models.ForeignKey(Member, on_delete=models.PROTECT, db_column="uid")
flag = models.ForeignKey(Flag, on_delete=models.PROTECT) flag = models.ForeignKey(Flag, on_delete=models.PROTECT)