membershipworks: Add API module and command for scraping data
This commit is contained in:
parent
6b3113e839
commit
dfacf813e2
@ -0,0 +1 @@
|
||||
from .membershipworks_api import MembershipWorks, MembershipWorksRemoteError
|
||||
0
membershipworks/management/__init__.py
Normal file
0
membershipworks/management/__init__.py
Normal file
0
membershipworks/management/commands/__init__.py
Normal file
0
membershipworks/management/commands/__init__.py
Normal file
@ -0,0 +1,48 @@
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import transaction
|
||||
|
||||
from membershipworks.models import Member, Flag
|
||||
from membershipworks import MembershipWorks
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
def flags_for_member(self, csv_member, all_flags):
|
||||
# update member's flags
|
||||
for flag in all_flags:
|
||||
if flag.type != "folder": # currently no way to retrieve this info
|
||||
if csv_member[flag.name] == flag.name:
|
||||
yield flag
|
||||
|
||||
def update_flags(self, mw_flags) -> list[Flag]:
|
||||
for typ, flags_of_type in mw_flags.items():
|
||||
for name, id in flags_of_type.items():
|
||||
flag = Flag(id=id, name=name, type=typ[:-1])
|
||||
flag.save()
|
||||
yield flag
|
||||
|
||||
@transaction.atomic()
|
||||
def handle(self, *args, **options):
|
||||
membershipworks = MembershipWorks()
|
||||
membershipworks.login(
|
||||
settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
|
||||
)
|
||||
|
||||
print("Updating flags (labels, levels, and addons)")
|
||||
flags = list(self.update_flags(membershipworks._parse_flags()))
|
||||
|
||||
print("Getting/Updating members...")
|
||||
members = membershipworks.get_all_members()
|
||||
for csv_member in members:
|
||||
for field_id, field in membershipworks._all_fields().items():
|
||||
# convert checkboxes to real booleans
|
||||
if field.get("typ") == 8 and field["lbl"] in csv_member:
|
||||
csv_member[field["lbl"]] = (
|
||||
True if csv_member[field["lbl"]] == "Y" else False
|
||||
)
|
||||
|
||||
# create/update member
|
||||
member = Member.from_csv_dict(csv_member)
|
||||
member.clean_fields()
|
||||
member.save()
|
||||
member.flags.set(self.flags_for_member(csv_member, flags))
|
||||
231
membershipworks/membershipworks_api.py
Normal file
231
membershipworks/membershipworks_api.py
Normal file
@ -0,0 +1,231 @@
|
||||
import csv
|
||||
from io import StringIO
|
||||
|
||||
import requests
|
||||
|
||||
BASE_URL = "https://api.membershipworks.com"
|
||||
|
||||
# extracted from `SF._is.crm` in https://cdn.membershipworks.com/all.js
|
||||
CRM = {
|
||||
0: "Note",
|
||||
4: "Profile Updated",
|
||||
8: "Scheduled/Reminder Email",
|
||||
9: "Renewal Notice",
|
||||
10: "Join Date",
|
||||
11: "Next Renewal Date",
|
||||
12: "Membership Payment",
|
||||
13: "Donation",
|
||||
14: "Event Activity",
|
||||
15: "Conversation",
|
||||
16: "Contact Change",
|
||||
17: "Label Change",
|
||||
18: "Other Payment",
|
||||
19: "Cart Payment",
|
||||
20: "Payment Failed",
|
||||
21: "Billing Updated",
|
||||
22: "Form Checkout",
|
||||
23: "Event Payment",
|
||||
24: "Invoice",
|
||||
25: "Invoice Payment",
|
||||
26: "Renewal",
|
||||
27: "Payment",
|
||||
}
|
||||
|
||||
# Types of fields, extracted from a html snippet in all.js + some guessing
|
||||
typ = {
|
||||
1: "Text input",
|
||||
2: "Password", # inferred from data
|
||||
3: "Simple text area",
|
||||
4: "Rich text area",
|
||||
7: "Address",
|
||||
8: "Check box",
|
||||
9: "Select",
|
||||
11: "Display value stored in field (ie. read only)",
|
||||
12: "Required waiver/terms",
|
||||
}
|
||||
|
||||
# more constants, this time extracted from the members csv export in all.js
|
||||
staticFlags = {
|
||||
"pos": {"lbl": "Position/relation (contacts)"},
|
||||
"nte": {"lbl": "Admin note (contacts)"},
|
||||
"pwd": {"lbl": "Password"},
|
||||
"lgo": {"lbl": "Business card image URLs"},
|
||||
"pfx": {"lbl": "Profile gallery image URLs"},
|
||||
"lvl": {"lbl": "Membership levels"},
|
||||
"aon": {"lbl": "Membership add-ons"},
|
||||
"lbl": {"lbl": "Labels"},
|
||||
"joi": {"lbl": "Join date"},
|
||||
"end": {"lbl": "Renewal date"},
|
||||
"spy": {"lbl": "Billing method"},
|
||||
"rid": {"lbl": "Auto recurring billing ID"},
|
||||
"ipa": {"lbl": "IP address"},
|
||||
"_id": {"lbl": "Account ID"},
|
||||
}
|
||||
|
||||
|
||||
class MembershipWorksRemoteError(Exception):
|
||||
def __init__(self, reason, r):
|
||||
super().__init__(
|
||||
f"Error when attempting {reason}: {r.status_code} {r.reason}\n{r.text}"
|
||||
)
|
||||
|
||||
|
||||
class MembershipWorks:
|
||||
def __init__(self):
|
||||
self.sess = requests.Session()
|
||||
self.org_info = None
|
||||
self.auth_token = None
|
||||
self.org_num = None
|
||||
|
||||
def login(self, username, password):
|
||||
"""Authenticate against the membershipworks api"""
|
||||
r = self.sess.post(
|
||||
BASE_URL + "/v2/account/session",
|
||||
data={"eml": username, "pwd": password},
|
||||
headers={"X-Org": "10000"},
|
||||
)
|
||||
if r.status_code != 200 or "SF" not in r.json():
|
||||
raise MembershipWorksRemoteError("login", r)
|
||||
self.org_info = r.json()
|
||||
self.auth_token = self.org_info["SF"]
|
||||
self.org_num = self.org_info["org"]
|
||||
self.sess.headers.update(
|
||||
{
|
||||
"X-Org": str(self.org_num),
|
||||
"X-Role": "admin",
|
||||
"Authorization": "Bearer " + self.auth_token,
|
||||
}
|
||||
)
|
||||
|
||||
def _inject_auth(self, kwargs):
|
||||
# TODO: should probably be a decorator or something
|
||||
if self.auth_token is None:
|
||||
raise RuntimeError("Not Logged in to MembershipWorks")
|
||||
# add auth token to params
|
||||
if "params" not in kwargs:
|
||||
kwargs["params"] = {}
|
||||
kwargs["params"]["SF"] = self.auth_token
|
||||
|
||||
def _get_v1(self, *args, **kwargs):
|
||||
self._inject_auth(kwargs)
|
||||
# TODO: should probably do some error handling in here
|
||||
return requests.get(*args, **kwargs)
|
||||
|
||||
def _post_v1(self, *args, **kwargs):
|
||||
self._inject_auth(kwargs)
|
||||
# TODO: should probably do some error handling in here
|
||||
return requests.post(*args, **kwargs)
|
||||
|
||||
def _all_fields(self):
|
||||
"""Parse out a list of fields from the org data.
|
||||
|
||||
Is this terrible? Yes. Also, not dissimilar to how MW does it
|
||||
in all.js.
|
||||
"""
|
||||
fields = staticFlags.copy()
|
||||
|
||||
# TODO: this will take the later option, if the same field
|
||||
# used in mulitple places. I don't know which lbl is used for
|
||||
# csv export
|
||||
|
||||
# anm: member signup, acc: member manage, adm: admin manage
|
||||
for screen_type in ["anm", "acc", "adm"]:
|
||||
for box in self.org_info["tpl"][screen_type]:
|
||||
for element in box["box"]:
|
||||
if type(element["dat"]) != str:
|
||||
for field in element["dat"]:
|
||||
if "_id" in field:
|
||||
if field["_id"] not in fields:
|
||||
fields[field["_id"]] = field
|
||||
|
||||
return fields
|
||||
|
||||
def _parse_flags(self):
|
||||
"""Parse the flags out of the org data.
|
||||
|
||||
This is terrible, and there might be a better way to do this.
|
||||
"""
|
||||
ret = {"folders": {}, "levels": {}, "addons": {}, "labels": {}}
|
||||
|
||||
for dek in self.org_info["dek"]:
|
||||
# TODO: there must be a better way. this is stupid
|
||||
if dek["dek"] == 1:
|
||||
ret["folders"][dek["lbl"]] = dek["did"]
|
||||
elif "cur" in dek:
|
||||
ret["levels"][dek["lbl"]] = dek["did"]
|
||||
elif "mux" in dek:
|
||||
ret["addons"][dek["lbl"]] = dek["did"]
|
||||
else:
|
||||
ret["labels"][dek["lbl"]] = dek["did"]
|
||||
|
||||
return ret
|
||||
|
||||
def get_member_ids(self, folders):
|
||||
folder_map = self._parse_flags()["folders"]
|
||||
|
||||
r = self.sess.get(
|
||||
BASE_URL + "/v2/accounts",
|
||||
params={"dek": ",".join([folder_map[f] for f in folders])},
|
||||
)
|
||||
if r.status_code != 200 or "usr" not in r.json():
|
||||
raise MembershipWorksRemoteError("user listing", r)
|
||||
|
||||
# get list of member ID matching the search
|
||||
# dedup with set() to work around people with alt uids
|
||||
# TODO: figure out why people have alt uids
|
||||
return set(user["uid"] for user in r.json()["usr"])
|
||||
|
||||
# TODO: has issues with aliasing header names:
|
||||
# ex: "Personal Studio Space" Label vs Membership Addon/Field
|
||||
def get_members(self, folders, columns):
|
||||
"""Pull the members csv from the membershipworks api
|
||||
folders: a list of the names of the folders to get
|
||||
(see folder_map in this function for mapping to ids)
|
||||
columns: which columns to get"""
|
||||
ids = self.get_member_ids(folders)
|
||||
|
||||
# get members CSV
|
||||
# TODO: maybe can just use previous get instead? would return JSON
|
||||
r = self._post_v1(
|
||||
BASE_URL + "/v1/csv",
|
||||
data={
|
||||
"_rt": "946702800", # unknown
|
||||
"mux": "", # unknown
|
||||
"tid": ",".join(ids), # ids of members to get
|
||||
"var": columns,
|
||||
},
|
||||
)
|
||||
if r.status_code != 200:
|
||||
raise MembershipWorksRemoteError("csv generation", r)
|
||||
return list(csv.DictReader(StringIO(r.text)))
|
||||
|
||||
def get_transactions(self, start_date, end_date, json=False):
|
||||
"""Get the transactions between start_date and end_date
|
||||
|
||||
Dates can be datetime.date or datetime.datetime
|
||||
|
||||
json gets a different version of the transactions list,
|
||||
which contains a different set information
|
||||
"""
|
||||
r = self._get_v1(
|
||||
BASE_URL + "/v1/csv",
|
||||
params={
|
||||
"crm": ",".join(str(k) for k in CRM.keys()),
|
||||
**({"txl": ""} if json else {}),
|
||||
"sdp": start_date.strftime("%s"),
|
||||
"edp": end_date.strftime("%s"),
|
||||
},
|
||||
)
|
||||
if r.status_code != 200:
|
||||
raise MembershipWorksRemoteError("csv generation", r)
|
||||
if json:
|
||||
return r.json()
|
||||
else:
|
||||
return list(csv.DictReader(StringIO(r.text)))
|
||||
|
||||
def get_all_members(self):
|
||||
"""Get all the data for all the members"""
|
||||
folders = self._parse_flags()["folders"].keys()
|
||||
fields = self._all_fields()
|
||||
members = self.get_members(folders, ",".join(fields.keys()))
|
||||
return members
|
||||
@ -1,4 +1,5 @@
|
||||
from typing import Optional
|
||||
from datetime import datetime
|
||||
|
||||
import django.core.mail.message
|
||||
from django.conf import settings
|
||||
@ -6,7 +7,47 @@ from django.db import models
|
||||
from django.db.models import Exists, OuterRef
|
||||
|
||||
|
||||
class Flag(models.Model):
|
||||
class BaseModel(models.Model):
|
||||
_csv_headers_override = {}
|
||||
_date_fields = {}
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
||||
@classmethod
|
||||
def _get_header_map(cls):
|
||||
header_map = {
|
||||
field.db_column: field.name
|
||||
for field in cls._meta.get_fields()
|
||||
if not field.auto_created
|
||||
}
|
||||
header_map.update(cls._csv_headers_override)
|
||||
|
||||
return header_map
|
||||
|
||||
@classmethod
|
||||
def _remap_headers(cls, data):
|
||||
header_map = cls._get_header_map()
|
||||
for k, v in data.items():
|
||||
if k in header_map:
|
||||
yield header_map.get(k), v
|
||||
|
||||
@classmethod
|
||||
def from_csv_dict(cls, data):
|
||||
data = data.copy()
|
||||
|
||||
# parse date fields to datetime objects
|
||||
for field, fmt in cls._date_fields.items():
|
||||
if data[field]:
|
||||
data[field] = datetime.strptime(str(data[field]), fmt)
|
||||
else:
|
||||
# convert empty string to None to make NULL in SQL
|
||||
data[field] = None
|
||||
|
||||
return cls(**dict(cls._remap_headers(data)))
|
||||
|
||||
|
||||
class Flag(BaseModel):
|
||||
id = models.CharField(max_length=24, primary_key=True)
|
||||
name = models.TextField(null=True, blank=True)
|
||||
type = models.CharField(max_length=6)
|
||||
@ -44,7 +85,7 @@ class MemberQuerySet(models.QuerySet):
|
||||
|
||||
|
||||
# TODO: is this still a temporal table?
|
||||
class Member(models.Model):
|
||||
class Member(BaseModel):
|
||||
objects = MemberQuerySet.as_manager()
|
||||
|
||||
uid = models.CharField(max_length=24, primary_key=True)
|
||||
@ -172,6 +213,21 @@ class Member(models.Model):
|
||||
)
|
||||
flags = models.ManyToManyField(Flag, through="MemberFlag", related_name="members")
|
||||
|
||||
_csv_headers_override = {
|
||||
"Account ID": "uid",
|
||||
"Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:": "how_did_you_hear",
|
||||
"Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.": "authorize_charge",
|
||||
"I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.": "policy_agreement",
|
||||
}
|
||||
|
||||
_date_fields = {
|
||||
"Join Date": "%b %d, %Y",
|
||||
"Renewal Date": "%b %d, %Y",
|
||||
"Audit Date": "%m/%d/%Y",
|
||||
"Membership Agreement signed and on file date.": "%m/%d/%Y",
|
||||
"Waiver form signed and on file date.": "%m/%d/%Y",
|
||||
}
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.account_name}"
|
||||
|
||||
@ -205,7 +261,7 @@ class Member(models.Model):
|
||||
)
|
||||
|
||||
|
||||
class MemberFlag(models.Model):
|
||||
class MemberFlag(BaseModel):
|
||||
member = models.ForeignKey(Member, on_delete=models.PROTECT, db_column="uid")
|
||||
flag = models.ForeignKey(Flag, on_delete=models.PROTECT)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user