membershipworks: Add upcoming events generator

This avoids having to deduplicate transactions, at the cost of
hypothetically missing transactions in some unlikely edge cases

cmsmanage/settings/base.py

@@ -37,6 +37,7 @@ INSTALLED_APPS = [
     "rest_framework",
     "rest_framework.authtoken",
     "django_q",
+    "django_bleach",
     "tasks.apps.TasksConfig",
     "rentals.apps.RentalsConfig",
     "membershipworks.apps.MembershipworksConfig",

membershipworks/__init__.py

@@ -0,0 +1 @@
+from .membershipworks_api import MembershipWorks, MembershipWorksRemoteError

membershipworks/admin.py

@@ -1,6 +1,12 @@
 from django.contrib import admin
+from django.contrib.humanize.templatetags.humanize import naturaltime
 
-from .models import Member, Flag
+from django_object_actions import DjangoObjectActions, action
+from django_q.tasks import async_task
+from django_q.models import Task
+
+from .models import Member, Flag, Transaction
+from .tasks.scrape import scrape_membershipworks
 
 
 class ReadOnlyAdmin(admin.ModelAdmin):
@@ -14,20 +20,51 @@ class ReadOnlyAdmin(admin.ModelAdmin):
         return False
 
 
+class BaseMembershipWorksAdmin(DjangoObjectActions, ReadOnlyAdmin):
+    changelist_actions = ("refresh_membershipworks_data",)
+
+    # internal method from DjangoObjectActions
+    def _get_tool_dict(self, tool_name):
+        tool = super(DjangoObjectActions, self)._get_tool_dict(tool_name)
+        if tool_name == "refresh_membershipworks_data":
+            last_run = (
+                Task.objects.filter(group="Scrape Data from MembershipWorks")
+                .order_by("started")
+                .last()
+            )
+            tool["label"] = f"Refresh Data [Last Run {naturaltime(last_run.started)}]"
+        return tool
+
+    @action
+    def refresh_membershipworks_data(self, request, obj):
+        async_task(scrape_membershipworks, group="Scrape Data from MembershipWorks")
+        self.message_user(
+            request,
+            "Queued refresh, please wait a few seconds/minutes then refresh the page",
+        )
+
+
 class MemberFlagInline(admin.TabularInline):
     model = Member.flags.through
 
 
 @admin.register(Member)
-class MemberAdmin(ReadOnlyAdmin):
+class MemberAdmin(BaseMembershipWorksAdmin):
     search_fields = ["^first_name", "^last_name", "^account_name"]
     inlines = [MemberFlagInline]
 
 
 @admin.register(Flag)
-class FlagAdmin(ReadOnlyAdmin):
+class FlagAdmin(BaseMembershipWorksAdmin):
     inlines = [MemberFlagInline]
     list_display = ["name", "type"]
     list_filter = ["type"]
     show_facets = admin.ShowFacets.ALWAYS
     search_fields = ["name"]
+
+
+@admin.register(Transaction)
+class TransactionAdmin(BaseMembershipWorksAdmin):
+    list_display = ["timestamp", "member", "name", "type", "sum", "note"]
+    list_filter = ["type"]
+    show_facets = admin.ShowFacets.ALWAYS

membershipworks/apps.py

@@ -7,8 +7,15 @@ def post_migrate_callback(sender, **kwargs):
 
     from cmsmanage.django_q2_helper import ensure_scheduled
 
+    from .tasks.scrape import scrape_membershipworks
     from .tasks.ucsAccounts import sync_accounts
 
+    ensure_scheduled(
+        "Scrape MembershipWorks Data",
+        scrape_membershipworks,
+        schedule_type=Schedule.HOURLY,
+    )
+
     ensure_scheduled(
         "Sync UCS Accounts",
         sync_accounts,

membershipworks/management/commands/scrape_membershipworks.py

@@ -0,0 +1,8 @@
+from django.core.management.base import BaseCommand
+
+from membershipworks.tasks.scrape import scrape_membershipworks
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        scrape_membershipworks()

membershipworks/membershipworks_api.py

@@ -0,0 +1,265 @@
+import csv
+from io import StringIO
+
+import requests
+import datetime
+
+BASE_URL = "https://api.membershipworks.com"
+
+# extracted from `SF._is.crm` in https://cdn.membershipworks.com/all.js
+CRM = {
+    0: "Note",
+    4: "Profile Updated",
+    8: "Scheduled/Reminder Email",
+    9: "Renewal Notice",
+    10: "Join Date",
+    11: "Next Renewal Date",
+    12: "Membership Payment",
+    13: "Donation",
+    14: "Event Activity",
+    15: "Conversation",
+    16: "Contact Change",
+    17: "Label Change",
+    18: "Other Payment",
+    19: "Cart Payment",
+    20: "Payment Failed",
+    21: "Billing Updated",
+    22: "Form Checkout",
+    23: "Event Payment",
+    24: "Invoice",
+    25: "Invoice Payment",
+    26: "Renewal",
+    27: "Payment",
+}
+
+# Types of fields, extracted from a html snippet in all.js + some guessing
+typ = {
+    1: "Text input",
+    2: "Password",  # inferred from data
+    3: "Simple text area",
+    4: "Rich text area",
+    7: "Address",
+    8: "Check box",
+    9: "Select",
+    11: "Display value stored in field (ie. read only)",
+    12: "Required waiver/terms",
+}
+
+# more constants, this time extracted from the members csv export in all.js
+staticFlags = {
+    "pos": {"lbl": "Position/relation (contacts)"},
+    "nte": {"lbl": "Admin note (contacts)"},
+    "pwd": {"lbl": "Password"},
+    "lgo": {"lbl": "Business card image URLs"},
+    "pfx": {"lbl": "Profile gallery image URLs"},
+    "lvl": {"lbl": "Membership levels"},
+    "aon": {"lbl": "Membership add-ons"},
+    "lbl": {"lbl": "Labels"},
+    "joi": {"lbl": "Join date"},
+    "end": {"lbl": "Renewal date"},
+    "spy": {"lbl": "Billing method"},
+    "rid": {"lbl": "Auto recurring billing ID"},
+    "ipa": {"lbl": "IP address"},
+    "_id": {"lbl": "Account ID"},
+}
+
+
+class MembershipWorksRemoteError(Exception):
+    def __init__(self, reason, r):
+        super().__init__(
+            f"Error when attempting {reason}: {r.status_code} {r.reason}\n{r.text}"
+        )
+
+
+class MembershipWorks:
+    def __init__(self):
+        self.sess = requests.Session()
+        self.org_info = None
+        self.auth_token = None
+        self.org_num = None
+
+    def login(self, username, password):
+        """Authenticate against the membershipworks api"""
+        r = self.sess.post(
+            BASE_URL + "/v2/account/session",
+            data={"eml": username, "pwd": password},
+            headers={"X-Org": "10000"},
+        )
+        if r.status_code != 200 or "SF" not in r.json():
+            raise MembershipWorksRemoteError("login", r)
+        self.org_info = r.json()
+        self.auth_token = self.org_info["SF"]
+        self.org_num = self.org_info["org"]
+        self.sess.headers.update(
+            {
+                "X-Org": str(self.org_num),
+                "X-Role": "admin",
+                "Authorization": "Bearer " + self.auth_token,
+            }
+        )
+
+    def _inject_auth(self, kwargs):
+        # TODO: should probably be a decorator or something
+        if self.auth_token is None:
+            raise RuntimeError("Not Logged in to MembershipWorks")
+        # add auth token to params
+        if "params" not in kwargs:
+            kwargs["params"] = {}
+        kwargs["params"]["SF"] = self.auth_token
+
+    def _get_v1(self, *args, **kwargs):
+        self._inject_auth(kwargs)
+        # TODO: should probably do some error handling in here
+        return requests.get(*args, **kwargs)
+
+    def _post_v1(self, *args, **kwargs):
+        self._inject_auth(kwargs)
+        # TODO: should probably do some error handling in here
+        return requests.post(*args, **kwargs)
+
+    def _all_fields(self):
+        """Parse out a list of fields from the org data.
+
+        Is this terrible? Yes. Also, not dissimilar to how MW does it
+        in all.js.
+        """
+        fields = staticFlags.copy()
+
+        # TODO: this will take the later option, if the same field
+        # used in mulitple places. I don't know which lbl is used for
+        # csv export
+
+        # anm: member signup, acc: member manage, adm: admin manage
+        for screen_type in ["anm", "acc", "adm"]:
+            for box in self.org_info["tpl"][screen_type]:
+                for element in box["box"]:
+                    if type(element["dat"]) != str:
+                        for field in element["dat"]:
+                            if "_id" in field:
+                                if field["_id"] not in fields:
+                                    fields[field["_id"]] = field
+
+        return fields
+
+    def _parse_flags(self):
+        """Parse the flags out of the org data.
+
+        This is terrible, and there might be a better way to do this.
+        """
+        ret = {"folders": {}, "levels": {}, "addons": {}, "labels": {}}
+
+        for dek in self.org_info["dek"]:
+            # TODO: there must be a better way. this is stupid
+            if dek["dek"] == 1:
+                ret["folders"][dek["lbl"]] = dek["did"]
+            elif "cur" in dek:
+                ret["levels"][dek["lbl"]] = dek["did"]
+            elif "mux" in dek:
+                ret["addons"][dek["lbl"]] = dek["did"]
+            else:
+                ret["labels"][dek["lbl"]] = dek["did"]
+
+        return ret
+
+    def get_member_ids(self, folders):
+        folder_map = self._parse_flags()["folders"]
+
+        r = self.sess.get(
+            BASE_URL + "/v2/accounts",
+            params={"dek": ",".join([folder_map[f] for f in folders])},
+        )
+        if r.status_code != 200 or "usr" not in r.json():
+            raise MembershipWorksRemoteError("user listing", r)
+
+        # get list of member ID matching the search
+        # dedup with set() to work around people with alt uids
+        # TODO: figure out why people have alt uids
+        return set(user["uid"] for user in r.json()["usr"])
+
+    # TODO: has issues with aliasing header names:
+    #       ex: "Personal Studio Space" Label vs Membership Addon/Field
+    def get_members(self, folders, columns):
+        """Pull the members csv from the membershipworks api
+        folders: a list of the names of the folders to get
+                 (see folder_map in this function for mapping to ids)
+        columns: which columns to get"""
+        ids = self.get_member_ids(folders)
+
+        # get members CSV
+        # TODO: maybe can just use previous get instead? would return JSON
+        r = self._post_v1(
+            BASE_URL + "/v1/csv",
+            data={
+                "_rt": "946702800",  # unknown
+                "mux": "",  # unknown
+                "tid": ",".join(ids),  # ids of members to get
+                "var": columns,
+            },
+        )
+        if r.status_code != 200:
+            raise MembershipWorksRemoteError("csv generation", r)
+
+        if r.text[0] == "\ufeff":
+            r.encoding = r.encoding + "-sig"
+
+        return list(csv.DictReader(StringIO(r.text)))
+
+    def get_transactions(self, start_date, end_date, json=False):
+        """Get the transactions between start_date and end_date
+
+        Dates can be datetime.date or datetime.datetime
+
+        json gets a different version of the transactions list,
+        which contains a different set information
+        """
+        r = self._get_v1(
+            BASE_URL + "/v1/csv",
+            params={
+                "crm": ",".join(str(k) for k in CRM.keys()),
+                **({"txl": ""} if json else {}),
+                "sdp": start_date.strftime("%s"),
+                "edp": end_date.strftime("%s"),
+            },
+        )
+        if r.status_code != 200:
+            raise MembershipWorksRemoteError("csv generation", r)
+        if json:
+            return r.json()
+        else:
+            if r.text[0] == "\ufeff":
+                r.encoding = r.encoding + "-sig"
+
+            return list(csv.DictReader(StringIO(r.text)))
+
+    def get_all_members(self):
+        """Get all the data for all the members"""
+        folders = self._parse_flags()["folders"].keys()
+        fields = self._all_fields()
+        members = self.get_members(folders, ",".join(fields.keys()))
+        return members
+
+    def get_events_list(self, start_date: datetime.datetime):
+        """Retrive a list of events since start_date"""
+        r = self.sess.get(
+            BASE_URL + "/v2/events",
+            params={
+                "sdp": start_date.strftime("%s"),
+            },
+        )
+        return r.json()
+
+    def get_event_by_eid(self, eid: str):
+        """Retrieve a specific event by its event id (eid)"""
+        r = self.sess.get(
+            BASE_URL + "/v2/event",
+            params={"eid": eid},
+        )
+        return r.json()
+
+    def get_event_by_url(self, url: str):
+        """Retrieve a specific event by its url"""
+        r = self.sess.get(
+            BASE_URL + "/v2/event",
+            params={"url": url},
+        )
+        return r.json()

membershipworks/migrations/0001_initial.py

@@ -1,7 +1,7 @@
-# Generated by Django 4.0.2 on 2022-02-12 05:05
+# Generated by Django 5.0 on 2023-12-20 05:40
 
-from django.db import migrations, models
 import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
@@ -17,12 +17,12 @@ class Migration(migrations.Migration):
                     "id",
                     models.CharField(max_length=24, primary_key=True, serialize=False),
                 ),
-                ("name", models.TextField(null=True)),
+                ("name", models.TextField(blank=True, null=True)),
                 ("type", models.CharField(max_length=6)),
             ],
             options={
                 "db_table": "flag",
-                "managed": False,
+                "ordering": ("name",),
             },
         ),
         migrations.CreateModel(
@@ -34,75 +34,116 @@ class Migration(migrations.Migration):
                 ),
                 (
                     "year_of_birth",
-                    models.TextField(db_column="Year of Birth", null=True),
+                    models.TextField(blank=True, db_column="Year of Birth", null=True),
+                ),
+                (
+                    "account_name",
+                    models.TextField(blank=True, db_column="Account Name", null=True),
+                ),
+                (
+                    "first_name",
+                    models.TextField(blank=True, db_column="First Name", null=True),
+                ),
+                (
+                    "last_name",
+                    models.TextField(blank=True, db_column="Last Name", null=True),
+                ),
+                ("phone", models.TextField(blank=True, db_column="Phone", null=True)),
+                ("email", models.TextField(blank=True, db_column="Email", null=True)),
+                (
+                    "volunteer_email",
+                    models.TextField(
+                        blank=True, db_column="Volunteer Email", null=True
+                    ),
                 ),
-                ("account_name", models.TextField(db_column="Account Name", null=True)),
-                ("first_name", models.TextField(db_column="First Name", null=True)),
-                ("last_name", models.TextField(db_column="Last Name", null=True)),
-                ("phone", models.TextField(db_column="Phone", null=True)),
-                ("email", models.TextField(db_column="Email", null=True)),
                 (
                     "address_street",
-                    models.TextField(db_column="Address (Street)", null=True),
+                    models.TextField(
+                        blank=True, db_column="Address (Street)", null=True
+                    ),
                 ),
                 (
                     "address_city",
-                    models.TextField(db_column="Address (City)", null=True),
+                    models.TextField(blank=True, db_column="Address (City)", null=True),
                 ),
                 (
                     "address_state_province",
-                    models.TextField(db_column="Address (State/Province)", null=True),
+                    models.TextField(
+                        blank=True, db_column="Address (State/Province)", null=True
+                    ),
                 ),
                 (
                     "address_postal_code",
-                    models.TextField(db_column="Address (Postal Code)", null=True),
+                    models.TextField(
+                        blank=True, db_column="Address (Postal Code)", null=True
+                    ),
                 ),
                 (
                     "address_country",
-                    models.TextField(db_column="Address (Country)", null=True),
+                    models.TextField(
+                        blank=True, db_column="Address (Country)", null=True
+                    ),
                 ),
                 (
                     "profile_description",
-                    models.TextField(db_column="Profile description", null=True),
+                    models.TextField(
+                        blank=True, db_column="Profile description", null=True
+                    ),
                 ),
-                ("website", models.TextField(db_column="Website", null=True)),
+                (
-                ("fax", models.TextField(db_column="Fax", null=True)),
+                    "website",
+                    models.TextField(blank=True, db_column="Website", null=True),
+                ),
+                ("fax", models.TextField(blank=True, db_column="Fax", null=True)),
                 (
                     "contact_person",
-                    models.TextField(db_column="Contact Person", null=True),
+                    models.TextField(blank=True, db_column="Contact Person", null=True),
+                ),
+                (
+                    "password",
+                    models.TextField(blank=True, db_column="Password", null=True),
                 ),
-                ("password", models.TextField(db_column="Password", null=True)),
                 (
                     "position_relation",
-                    models.TextField(db_column="Position/relation", null=True),
+                    models.TextField(
+                        blank=True, db_column="Position/relation", null=True
+                    ),
                 ),
                 (
                     "parent_account_id",
-                    models.TextField(db_column="Parent Account ID", null=True),
+                    models.TextField(
+                        blank=True, db_column="Parent Account ID", null=True
+                    ),
                 ),
                 (
                     "gift_membership_purchased_by",
                     models.TextField(
-                        db_column="Gift Membership purchased by", null=True
+                        blank=True, db_column="Gift Membership purchased by", null=True
                     ),
                 ),
                 (
                     "purchased_gift_membership_for",
                     models.TextField(
-                        db_column="Purchased Gift Membership for", null=True
+                        blank=True, db_column="Purchased Gift Membership for", null=True
                     ),
                 ),
                 (
                     "closet_storage",
-                    models.TextField(db_column="Closet Storage #", null=True),
+                    models.TextField(
+                        blank=True, db_column="Closet Storage #", null=True
+                    ),
                 ),
                 (
                     "storage_shelf",
-                    models.TextField(db_column="Storage Shelf #", null=True),
+                    models.TextField(
+                        blank=True, db_column="Storage Shelf #", null=True
+                    ),
                 ),
                 (
                     "personal_studio_space",
-                    models.TextField(db_column="Personal Studio Space #", null=True),
+                    models.TextField(
+                        blank=True, db_column="Personal Studio Space #", null=True
+                    ),
                 ),
                 (
                     "access_permitted_shops_during_extended_hours",
@@ -150,84 +191,145 @@ class Migration(migrations.Migration):
                 ),
                 (
                     "access_card_number",
-                    models.TextField(db_column="Access Card Number", null=True),
+                    models.TextField(
+                        blank=True, db_column="Access Card Number", null=True
+                    ),
                 ),
                 (
                     "access_card_facility_code",
-                    models.TextField(db_column="Access Card Facility Code", null=True),
+                    models.TextField(
+                        blank=True, db_column="Access Card Facility Code", null=True
+                    ),
                 ),
                 (
                     "auto_billing_id",
-                    models.TextField(db_column="Auto Billing ID", null=True),
+                    models.TextField(
+                        blank=True, db_column="Auto Billing ID", null=True
+                    ),
                 ),
                 (
                     "billing_method",
-                    models.TextField(db_column="Billing Method", null=True),
+                    models.TextField(blank=True, db_column="Billing Method", null=True),
+                ),
+                (
+                    "renewal_date",
+                    models.DateField(blank=True, db_column="Renewal Date", null=True),
+                ),
+                (
+                    "join_date",
+                    models.DateField(blank=True, db_column="Join Date", null=True),
+                ),
+                (
+                    "admin_note",
+                    models.TextField(blank=True, db_column="Admin note", null=True),
                 ),
-                ("renewal_date", models.DateField(db_column="Renewal Date", null=True)),
-                ("join_date", models.DateField(db_column="Join Date", null=True)),
-                ("admin_note", models.TextField(db_column="Admin note", null=True)),
                 (
                     "profile_gallery_image_url",
-                    models.TextField(db_column="Profile gallery image URL", null=True),
+                    models.TextField(
+                        blank=True, db_column="Profile gallery image URL", null=True
+                    ),
                 ),
                 (
                     "business_card_image_url",
-                    models.TextField(db_column="Business card image URL", null=True),
+                    models.TextField(
+                        blank=True, db_column="Business card image URL", null=True
+                    ),
+                ),
+                (
+                    "instagram",
+                    models.TextField(blank=True, db_column="Instagram", null=True),
+                ),
+                (
+                    "pinterest",
+                    models.TextField(blank=True, db_column="Pinterest", null=True),
+                ),
+                (
+                    "youtube",
+                    models.TextField(blank=True, db_column="Youtube", null=True),
+                ),
+                ("yelp", models.TextField(blank=True, db_column="Yelp", null=True)),
+                (
+                    "google",
+                    models.TextField(blank=True, db_column="Google+", null=True),
+                ),
+                ("bbb", models.TextField(blank=True, db_column="BBB", null=True)),
+                (
+                    "twitter",
+                    models.TextField(blank=True, db_column="Twitter", null=True),
+                ),
+                (
+                    "facebook",
+                    models.TextField(blank=True, db_column="Facebook", null=True),
+                ),
+                (
+                    "linked_in",
+                    models.TextField(blank=True, db_column="LinkedIn", null=True),
                 ),
-                ("instagram", models.TextField(db_column="Instagram", null=True)),
-                ("pinterest", models.TextField(db_column="Pinterest", null=True)),
-                ("youtube", models.TextField(db_column="Youtube", null=True)),
-                ("yelp", models.TextField(db_column="Yelp", null=True)),
-                ("google", models.TextField(db_column="Google+", null=True)),
-                ("bbb", models.TextField(db_column="BBB", null=True)),
-                ("twitter", models.TextField(db_column="Twitter", null=True)),
-                ("facebook", models.TextField(db_column="Facebook", null=True)),
-                ("linked_in", models.TextField(db_column="LinkedIn", null=True)),
                 (
                     "do_not_show_street_address_in_profile",
                     models.TextField(
-                        db_column="Do not show street address in profile", null=True
+                        blank=True,
+                        db_column="Do not show street address in profile",
+                        null=True,
                     ),
                 ),
                 (
                     "do_not_list_in_directory",
-                    models.TextField(db_column="Do not list in directory", null=True),
+                    models.TextField(
+                        blank=True, db_column="Do not list in directory", null=True
+                    ),
                 ),
                 (
                     "how_did_you_hear",
-                    models.TextField(db_column="HowDidYouHear", null=True),
+                    models.TextField(blank=True, db_column="HowDidYouHear", null=True),
                 ),
                 (
                     "authorize_charge",
-                    models.TextField(db_column="authorizeCharge", null=True),
+                    models.TextField(
+                        blank=True, db_column="authorizeCharge", null=True
+                    ),
                 ),
                 (
                     "policy_agreement",
-                    models.TextField(db_column="policyAgreement", null=True),
+                    models.TextField(
+                        blank=True, db_column="policyAgreement", null=True
+                    ),
                 ),
                 (
                     "waiver_form_signed_and_on_file_date",
                     models.DateField(
-                        db_column="Waiver form signed and on file date.", null=True
+                        blank=True,
+                        db_column="Waiver form signed and on file date.",
+                        null=True,
                     ),
                 ),
                 (
                     "membership_agreement_signed_and_on_file_date",
                     models.DateField(
+                        blank=True,
                         db_column="Membership Agreement signed and on file date.",
                         null=True,
                     ),
                 ),
-                ("ip_address", models.TextField(db_column="IP Address", null=True)),
+                (
-                ("audit_date", models.DateField(db_column="Audit Date", null=True)),
+                    "ip_address",
+                    models.TextField(blank=True, db_column="IP Address", null=True),
+                ),
+                (
+                    "audit_date",
+                    models.DateField(blank=True, db_column="Audit Date", null=True),
+                ),
                 (
                     "agreement_version",
-                    models.TextField(db_column="Agreement Version", null=True),
+                    models.TextField(
+                        blank=True, db_column="Agreement Version", null=True
+                    ),
                 ),
                 (
                     "paperwork_status",
-                    models.TextField(db_column="Paperwork status", null=True),
+                    models.TextField(
+                        blank=True, db_column="Paperwork status", null=True
+                    ),
                 ),
                 (
                     "membership_agreement_dated",
@@ -259,7 +361,6 @@ class Migration(migrations.Migration):
             options={
                 "db_table": "members",
                 "ordering": ("first_name", "last_name"),
-                "managed": False,
             },
         ),
         migrations.CreateModel(
@@ -274,10 +375,127 @@ class Migration(migrations.Migration):
                         verbose_name="ID",
                     ),
                 ),
+                (
+                    "flag",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="membershipworks.flag",
+                    ),
+                ),
+                (
+                    "member",
+                    models.ForeignKey(
+                        db_column="uid",
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="membershipworks.member",
+                    ),
+                ),
             ],
             options={
                 "db_table": "memberflag",
-                "managed": False,
             },
         ),
+        migrations.AddField(
+            model_name="member",
+            name="flags",
+            field=models.ManyToManyField(
+                related_name="members",
+                through="membershipworks.MemberFlag",
+                to="membershipworks.flag",
+            ),
+        ),
+        migrations.CreateModel(
+            name="Transaction",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("sid", models.CharField(blank=True, max_length=27, null=True)),
+                ("timestamp", models.DateTimeField()),
+                ("type", models.TextField(blank=True, null=True)),
+                (
+                    "sum",
+                    models.DecimalField(
+                        blank=True, decimal_places=4, max_digits=13, null=True
+                    ),
+                ),
+                (
+                    "fee",
+                    models.DecimalField(
+                        blank=True, decimal_places=4, max_digits=13, null=True
+                    ),
+                ),
+                ("event_id", models.TextField(blank=True, null=True)),
+                ("for_what", models.TextField(blank=True, db_column="For", null=True)),
+                ("items", models.TextField(blank=True, db_column="Items", null=True)),
+                (
+                    "discount_code",
+                    models.TextField(blank=True, db_column="Discount Code", null=True),
+                ),
+                ("note", models.TextField(blank=True, db_column="Note", null=True)),
+                ("name", models.TextField(blank=True, db_column="Name", null=True)),
+                (
+                    "contact_person",
+                    models.TextField(blank=True, db_column="Contact Person", null=True),
+                ),
+                (
+                    "full_address",
+                    models.TextField(blank=True, db_column="Full Address", null=True),
+                ),
+                ("street", models.TextField(blank=True, db_column="Street", null=True)),
+                ("city", models.TextField(blank=True, db_column="City", null=True)),
+                (
+                    "state_province",
+                    models.TextField(blank=True, db_column="State/Province", null=True),
+                ),
+                (
+                    "postal_code",
+                    models.TextField(blank=True, db_column="Postal Code", null=True),
+                ),
+                (
+                    "country",
+                    models.TextField(blank=True, db_column="Country", null=True),
+                ),
+                ("phone", models.TextField(blank=True, db_column="Phone", null=True)),
+                ("email", models.TextField(blank=True, db_column="Email", null=True)),
+                (
+                    "member",
+                    models.ForeignKey(
+                        blank=True,
+                        db_column="uid",
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="transactions",
+                        to="membershipworks.member",
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "transactions",
+            },
+        ),
+        migrations.AddConstraint(
+            model_name="memberflag",
+            constraint=models.UniqueConstraint(
+                fields=("member", "flag"), name="unique_member_flag"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="member",
+            index=models.Index(fields=["account_name"], name="account_name_idx"),
+        ),
+        migrations.AddIndex(
+            model_name="member",
+            index=models.Index(fields=["first_name"], name="first_name_idx"),
+        ),
+        migrations.AddIndex(
+            model_name="member",
+            index=models.Index(fields=["last_name"], name="last_name_idx"),
+        ),
     ]

membershipworks/migrations/0002_alter_flag_options.py

@@ -1,16 +0,0 @@
-# Generated by Django 4.1.3 on 2023-01-24 02:02
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name="flag",
-            options={"managed": False, "ordering": ("name",)},
-        ),
-    ]

membershipworks/migrations/0002_remove_member_accepted_covid19_policy_and_more.py

@@ -0,0 +1,37 @@
+# Generated by Django 4.0.2 on 2022-03-01 19:33
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("membershipworks", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="member",
+            name="accepted_covid19_policy",
+        ),
+        migrations.RemoveField(
+            model_name="member",
+            name="access_permitted_during_covid19_staffed_period_only",
+        ),
+        migrations.RemoveField(
+            model_name="member",
+            name="gift_membership_purchased_by",
+        ),
+        migrations.RemoveField(
+            model_name="member",
+            name="purchased_gift_membership_for",
+        ),
+        migrations.RemoveField(
+            model_name="member",
+            name="normal_access_permitted_during_covid19_limited_operations",
+        ),
+        migrations.RemoveField(
+            model_name="member",
+            name="self_certify_essential_business",
+        ),
+    ]

membershipworks/migrations/0003_alter_transaction_sid.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0 on 2023-12-20 06:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("membershipworks", "0002_remove_member_accepted_covid19_policy_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="transaction",
+            name="sid",
+            field=models.CharField(blank=True, max_length=256, null=True),
+        ),
+    ]

membershipworks/models.py

@@ -1,21 +1,58 @@
 from typing import Optional
+from datetime import datetime
 
 import django.core.mail.message
 from django.conf import settings
 from django.db import models
 from django.db.models import Exists, OuterRef
+from django.utils import timezone
 
 
-class Flag(models.Model):
+class BaseModel(models.Model):
+    _csv_headers_override = {}
+    _date_fields = {}
+
+    class Meta:
+        abstract = True
+
+    @classmethod
+    def _remap_headers(cls, data):
+        for field in cls._meta.get_fields():
+            # TODO: more robust filtering of fields that don't have a column
+            if field.auto_created or field.many_to_many or not field.concrete:
+                continue
+
+            field_name, csv_header = field.get_attname_column()
+
+            if field_name in cls._csv_headers_override:
+                csv_header = cls._csv_headers_override[field.name]
+
+            yield field_name, data[csv_header]
+
+    @classmethod
+    def from_csv_dict(cls, data):
+        data = data.copy()
+
+        # parse date fields to datetime objects
+        for field, fmt in cls._date_fields.items():
+            if data[field]:
+                data[field] = datetime.strptime(str(data[field]), fmt)
+            else:
+                # convert empty string to None to make NULL in SQL
+                data[field] = None
+
+        return cls(**dict(cls._remap_headers(data)))
+
+
+class Flag(BaseModel):
     id = models.CharField(max_length=24, primary_key=True)
-    name = models.TextField(null=True)
+    name = models.TextField(null=True, blank=True)
     type = models.CharField(max_length=6)
 
     def __str__(self):
         return f"{self.name} ({self.type})"
 
     class Meta:
-        managed = False
         db_table = "flag"
         ordering = ("name",)
 
@@ -45,51 +82,55 @@ class MemberQuerySet(models.QuerySet):
 
 
 # TODO: is this still a temporal table?
-class Member(models.Model):
+class Member(BaseModel):
     objects = MemberQuerySet.as_manager()
 
     uid = models.CharField(max_length=24, primary_key=True)
-    year_of_birth = models.TextField(db_column="Year of Birth", null=True)
+    year_of_birth = models.TextField(db_column="Year of Birth", null=True, blank=True)
-    account_name = models.TextField(db_column="Account Name", null=True)
+    account_name = models.TextField(db_column="Account Name", null=True, blank=True)
-    first_name = models.TextField(db_column="First Name", null=True)
+    first_name = models.TextField(db_column="First Name", null=True, blank=True)
-    last_name = models.TextField(db_column="Last Name", null=True)
+    last_name = models.TextField(db_column="Last Name", null=True, blank=True)
-    phone = models.TextField(db_column="Phone", null=True)
+    phone = models.TextField(db_column="Phone", null=True, blank=True)
-    email = models.TextField(db_column="Email", null=True)
+    email = models.TextField(db_column="Email", null=True, blank=True)
-    volunteer_email = models.TextField(db_column="Volunteer Email", null=True)
+    volunteer_email = models.TextField(
-    address_street = models.TextField(db_column="Address (Street)", null=True)
+        db_column="Volunteer Email", null=True, blank=True
-    address_city = models.TextField(db_column="Address (City)", null=True)
+    )
+    address_street = models.TextField(
+        db_column="Address (Street)", null=True, blank=True
+    )
+    address_city = models.TextField(db_column="Address (City)", null=True, blank=True)
     address_state_province = models.TextField(
-        db_column="Address (State/Province)", null=True
+        db_column="Address (State/Province)", null=True, blank=True
     )
-    address_postal_code = models.TextField(db_column="Address (Postal Code)", null=True)
+    address_postal_code = models.TextField(
-    address_country = models.TextField(db_column="Address (Country)", null=True)
+        db_column="Address (Postal Code)", null=True, blank=True
-    profile_description = models.TextField(db_column="Profile description", null=True)
-    website = models.TextField(db_column="Website", null=True)
-    fax = models.TextField(db_column="Fax", null=True)
-    contact_person = models.TextField(db_column="Contact Person", null=True)
-    password = models.TextField(db_column="Password", null=True)
-    position_relation = models.TextField(db_column="Position/relation", null=True)
-    parent_account_id = models.TextField(db_column="Parent Account ID", null=True)
-    gift_membership_purchased_by = models.TextField(
-        db_column="Gift Membership purchased by", null=True
     )
-    purchased_gift_membership_for = models.TextField(
+    address_country = models.TextField(
-        db_column="Purchased Gift Membership for", null=True
+        db_column="Address (Country)", null=True, blank=True
     )
-    closet_storage = models.TextField(db_column="Closet Storage #", null=True)
+    profile_description = models.TextField(
-    storage_shelf = models.TextField(db_column="Storage Shelf #", null=True)
+        db_column="Profile description", null=True, blank=True
+    )
+    website = models.TextField(db_column="Website", null=True, blank=True)
+    fax = models.TextField(db_column="Fax", null=True, blank=True)
+    contact_person = models.TextField(db_column="Contact Person", null=True, blank=True)
+    password = models.TextField(db_column="Password", null=True, blank=True)
+    position_relation = models.TextField(
+        db_column="Position/relation", null=True, blank=True
+    )
+    parent_account_id = models.TextField(
+        db_column="Parent Account ID", null=True, blank=True
+    )
+    closet_storage = models.TextField(
+        db_column="Closet Storage #", null=True, blank=True
+    )
+    storage_shelf = models.TextField(db_column="Storage Shelf #", null=True, blank=True)
     personal_studio_space = models.TextField(
-        db_column="Personal Studio Space #", null=True
+        db_column="Personal Studio Space #", null=True, blank=True
     )
     access_permitted_shops_during_extended_hours = models.BooleanField(
         db_column="Access Permitted Shops During Extended Hours?"
     )
-    normal_access_permitted_during_covid19_limited_operations = models.BooleanField(
-        db_column="Normal Access Permitted During COVID-19 Limited Operations"
-    )
-    access_permitted_during_covid19_staffed_period_only = models.BooleanField(
-        db_column="Access Permitted During COVID-19 Staffed Period Only"
-    )
     access_front_door_and_studio_space_during_extended_hours = models.BooleanField(
         db_column="Access Front Door and Studio Space During Extended Hours?"
     )
@@ -98,49 +139,63 @@ class Member(models.Model):
     access_storage_closet = models.BooleanField(db_column="Access Storage Closet?")
     access_studio_space = models.BooleanField(db_column="Access Studio Space?")
     access_front_door = models.BooleanField(db_column="Access Front Door?")
-    access_card_number = models.TextField(db_column="Access Card Number", null=True)
+    access_card_number = models.TextField(
-    access_card_facility_code = models.TextField(
+        db_column="Access Card Number", null=True, blank=True
-        db_column="Access Card Facility Code", null=True
     )
-    auto_billing_id = models.TextField(db_column="Auto Billing ID", null=True)
+    access_card_facility_code = models.TextField(
-    billing_method = models.TextField(db_column="Billing Method", null=True)
+        db_column="Access Card Facility Code", null=True, blank=True
-    renewal_date = models.DateField(db_column="Renewal Date", null=True)
+    )
-    join_date = models.DateField(db_column="Join Date", null=True)
+    auto_billing_id = models.TextField(
-    admin_note = models.TextField(db_column="Admin note", null=True)
+        db_column="Auto Billing ID", null=True, blank=True
+    )
+    billing_method = models.TextField(db_column="Billing Method", null=True, blank=True)
+    renewal_date = models.DateField(db_column="Renewal Date", null=True, blank=True)
+    join_date = models.DateField(db_column="Join Date", null=True, blank=True)
+    admin_note = models.TextField(db_column="Admin note", null=True, blank=True)
     profile_gallery_image_url = models.TextField(
-        db_column="Profile gallery image URL", null=True
+        db_column="Profile gallery image URL", null=True, blank=True
     )
     business_card_image_url = models.TextField(
-        db_column="Business card image URL", null=True
+        db_column="Business card image URL", null=True, blank=True
     )
-    instagram = models.TextField(db_column="Instagram", null=True)
+    instagram = models.TextField(db_column="Instagram", null=True, blank=True)
-    pinterest = models.TextField(db_column="Pinterest", null=True)
+    pinterest = models.TextField(db_column="Pinterest", null=True, blank=True)
-    youtube = models.TextField(db_column="Youtube", null=True)
+    youtube = models.TextField(db_column="Youtube", null=True, blank=True)
-    yelp = models.TextField(db_column="Yelp", null=True)
+    yelp = models.TextField(db_column="Yelp", null=True, blank=True)
-    google = models.TextField(db_column="Google+", null=True)
+    google = models.TextField(db_column="Google+", null=True, blank=True)
-    bbb = models.TextField(db_column="BBB", null=True)
+    bbb = models.TextField(db_column="BBB", null=True, blank=True)
-    twitter = models.TextField(db_column="Twitter", null=True)
+    twitter = models.TextField(db_column="Twitter", null=True, blank=True)
-    facebook = models.TextField(db_column="Facebook", null=True)
+    facebook = models.TextField(db_column="Facebook", null=True, blank=True)
-    linked_in = models.TextField(db_column="LinkedIn", null=True)
+    linked_in = models.TextField(db_column="LinkedIn", null=True, blank=True)
     do_not_show_street_address_in_profile = models.TextField(
-        db_column="Do not show street address in profile", null=True
+        db_column="Do not show street address in profile", null=True, blank=True
     )
     do_not_list_in_directory = models.TextField(
-        db_column="Do not list in directory", null=True
+        db_column="Do not list in directory", null=True, blank=True
+    )
+    how_did_you_hear = models.TextField(
+        db_column="HowDidYouHear", null=True, blank=True
+    )
+    authorize_charge = models.TextField(
+        db_column="authorizeCharge", null=True, blank=True
+    )
+    policy_agreement = models.TextField(
+        db_column="policyAgreement", null=True, blank=True
     )
-    how_did_you_hear = models.TextField(db_column="HowDidYouHear", null=True)
-    authorize_charge = models.TextField(db_column="authorizeCharge", null=True)
-    policy_agreement = models.TextField(db_column="policyAgreement", null=True)
     waiver_form_signed_and_on_file_date = models.DateField(
-        db_column="Waiver form signed and on file date.", null=True
+        db_column="Waiver form signed and on file date.", null=True, blank=True
     )
     membership_agreement_signed_and_on_file_date = models.DateField(
-        db_column="Membership Agreement signed and on file date.", null=True
+        db_column="Membership Agreement signed and on file date.", null=True, blank=True
+    )
+    ip_address = models.TextField(db_column="IP Address", null=True, blank=True)
+    audit_date = models.DateField(db_column="Audit Date", null=True, blank=True)
+    agreement_version = models.TextField(
+        db_column="Agreement Version", null=True, blank=True
+    )
+    paperwork_status = models.TextField(
+        db_column="Paperwork status", null=True, blank=True
     )
-    ip_address = models.TextField(db_column="IP Address", null=True)
-    audit_date = models.DateField(db_column="Audit Date", null=True)
-    agreement_version = models.TextField(db_column="Agreement Version", null=True)
-    paperwork_status = models.TextField(db_column="Paperwork status", null=True)
     membership_agreement_dated = models.BooleanField(
         db_column="Membership agreement dated"
     )
@@ -153,23 +208,33 @@ class Member(models.Model):
     liability_form_filled_out = models.BooleanField(
         db_column="Liability Form Filled Out"
     )
-    self_certify_essential_business = models.BooleanField(
-        db_column="selfCertifyEssentialBusiness"
-    )
-    accepted_covid19_policy = models.BooleanField(db_column="Accepted COVID-19 Policy")
     flags = models.ManyToManyField(Flag, through="MemberFlag", related_name="members")
 
+    _csv_headers_override = {
+        "uid": "Account ID",
+        "how_did_you_hear": "Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:",
+        "authorize_charge": "Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.",
+        "policy_agreement": "I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.",
+    }
+
+    _date_fields = {
+        "Join Date": "%b %d, %Y",
+        "Renewal Date": "%b %d, %Y",
+        "Audit Date": "%m/%d/%Y",
+        "Membership Agreement signed and on file date.": "%m/%d/%Y",
+        "Waiver form signed and on file date.": "%m/%d/%Y",
+    }
+
     def __str__(self):
         return f"{self.account_name}"
 
     class Meta:
-        managed = False
         db_table = "members"
         ordering = ("first_name", "last_name")
         indexes = [
-            models.Index(fields=["account_name"]),
+            models.Index(fields=["account_name"], name="account_name_idx"),
-            models.Index(fields=["first_name"]),
+            models.Index(fields=["first_name"], name="first_name_idx"),
-            models.Index(fields=["last_name"]),
+            models.Index(fields=["last_name"], name="last_name_idx"),
         ]
 
     @classmethod
@@ -193,7 +258,7 @@ class Member(models.Model):
         )
 
 
-class MemberFlag(models.Model):
+class MemberFlag(BaseModel):
     member = models.ForeignKey(Member, on_delete=models.PROTECT, db_column="uid")
     flag = models.ForeignKey(Flag, on_delete=models.PROTECT)
 
@@ -201,10 +266,72 @@ class MemberFlag(models.Model):
         return f"{self.member} - {self.flag}"
 
     class Meta:
-        managed = False
         db_table = "memberflag"
         constraints = [
             models.UniqueConstraint(
-                fields=["member", "flag_id"], name="unique_member_flag"
+                fields=["member", "flag"], name="unique_member_flag"
             )
         ]
+
+
+class Transaction(BaseModel):
+    sid = models.CharField(max_length=256, null=True, blank=True)
+    member = models.ForeignKey(
+        Member,
+        on_delete=models.PROTECT,
+        db_column="uid",
+        related_name="transactions",
+        null=True,
+        blank=True,
+    )
+    timestamp = models.DateTimeField()
+    type = models.TextField(null=True, blank=True)
+    sum = models.DecimalField(max_digits=13, decimal_places=4, null=True, blank=True)
+    fee = models.DecimalField(max_digits=13, decimal_places=4, null=True, blank=True)
+    event_id = models.TextField(null=True, blank=True)
+    for_what = models.TextField(db_column="For", null=True, blank=True)
+    items = models.TextField(db_column="Items", null=True, blank=True)
+    discount_code = models.TextField(db_column="Discount Code", null=True, blank=True)
+    note = models.TextField(db_column="Note", null=True, blank=True)
+    name = models.TextField(db_column="Name", null=True, blank=True)
+    contact_person = models.TextField(db_column="Contact Person", null=True, blank=True)
+    full_address = models.TextField(db_column="Full Address", null=True, blank=True)
+    street = models.TextField(db_column="Street", null=True, blank=True)
+    city = models.TextField(db_column="City", null=True, blank=True)
+    state_province = models.TextField(db_column="State/Province", null=True, blank=True)
+    postal_code = models.TextField(db_column="Postal Code", null=True, blank=True)
+    country = models.TextField(db_column="Country", null=True, blank=True)
+    phone = models.TextField(db_column="Phone", null=True, blank=True)
+    email = models.TextField(db_column="Email", null=True, blank=True)
+
+    @classmethod
+    def from_csv_dict(cls, data):
+        txn = data.copy()
+        # can't use '%s' format string, have to use the special function
+        txn["_dp"] = datetime.fromtimestamp(
+            txn["_dp"], tz=timezone.get_current_timezone()
+        )
+        allowed_missing_fields = [
+            "sid",
+            "uid",
+            "eid",
+            "fee",
+            "sum",
+        ]
+        for field in allowed_missing_fields:
+            if field not in txn:
+                txn[field] = None
+        return super().from_csv_dict(txn)
+
+    _csv_headers_override = {
+        "event_id": "eid",
+        "timestamp": "_dp",
+        "type": "Transaction Type",
+        "for_what": "Event/Form Name",
+    }
+
+    def __str__(self):
+        return f"{self.type} [{self.member if self.member else self.name}] {self.timestamp}"
+
+    class Meta:
+        db_table = "transactions"

membershipworks/routers.py

@@ -13,8 +13,8 @@ class MembershipWorksRouter:
         return None
 
     def allow_migrate(self, db, app_label, model_name=None, **hints):
-        if db == self.db:
+        if app_label == self.app_label:
-            return False
+            return db == self.db
         return None
 
     def allow_relation(self, obj1, obj2, **hints):

membershipworks/tasks/scrape.py

@@ -0,0 +1,93 @@
+from datetime import datetime, timedelta
+import logging
+
+from django.conf import settings
+from django.db import transaction
+
+from membershipworks.models import Member, Flag, Transaction
+from membershipworks import MembershipWorks
+
+
+logger = logging.getLogger(__name__)
+
+
+def flags_for_member(csv_member, all_flags, folders):
+    for flag in all_flags:
+        if flag.type == "folder":
+            if csv_member["Account ID"] in folders[flag.id]:
+                yield flag
+        else:
+            if csv_member[flag.name] == flag.name:
+                yield flag
+
+
+def update_flags(mw_flags) -> list[Flag]:
+    for typ, flags_of_type in mw_flags.items():
+        for name, id in flags_of_type.items():
+            flag = Flag(id=id, name=name, type=typ[:-1])
+            flag.save()
+            yield flag
+
+
+def scrape_members(membershipworks: MembershipWorks):
+    logger.info("Updating flags (labels, levels, and addons)")
+    flags = list(update_flags(membershipworks._parse_flags()))
+
+    logger.info("Getting folder membership")
+    folders = {
+        folder_id: membershipworks.get_member_ids([folder_name])
+        for folder_name, folder_id in membershipworks._parse_flags()["folders"].items()
+    }
+
+    logger.info("Getting/Updating members...")
+    members = membershipworks.get_all_members()
+    for csv_member in members:
+        for field_id, field in membershipworks._all_fields().items():
+            # convert checkboxes to real booleans
+            if field.get("typ") == 8 and field["lbl"] in csv_member:
+                csv_member[field["lbl"]] = (
+                    True if csv_member[field["lbl"]] == "Y" else False
+                )
+
+        # create/update member
+        member = Member.from_csv_dict(csv_member)
+        member.clean_fields()
+        member.save()
+        member.flags.set(flags_for_member(csv_member, flags, folders))
+
+
+def scrape_transactions(membershipworks: MembershipWorks):
+    now = datetime.now()
+    start_date = datetime(2010, 1, 1)
+    last_transaction = Transaction.objects.order_by("timestamp").last()
+    if last_transaction is not None:
+        # technically this has the potential to lose
+        # transactions, but it should be incredibly unlikely
+        start_date = last_transaction.timestamp + timedelta(seconds=1)
+
+    logger.info(f"Getting/Updating transactions since {start_date}...")
+
+    transactions_csv = membershipworks.get_transactions(start_date, now)
+    transactions_json = membershipworks.get_transactions(start_date, now, json=True)
+    # this is terrible, but as long as the dates are the same, should be fiiiine
+    transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
+    assert all(
+        [
+            t["Account ID"] == t.get("uid", "") and t["Payment ID"] == t.get("sid", "")
+            for t in transactions
+        ]
+    )
+
+    for csv_transaction in transactions:
+        Transaction.from_csv_dict(csv_transaction).save()
+
+
+@transaction.atomic
+def scrape_membershipworks(*args, **options):
+    membershipworks = MembershipWorks()
+    membershipworks.login(
+        settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
+    )
+
+    scrape_members(membershipworks)
+    scrape_transactions(membershipworks)

membershipworks/templates/membershipworks/upcoming_events.dj.html

@@ -0,0 +1,121 @@
+{% extends "base.dj.html" %}
+
+{% load bleach_tags %}
+
+{% block title %}Upcoming Events{% endblock %}
+{% block content %}
+  <div id="preview">
+    <p>
+      <img class="aligncenter size-medium wp-image-2319"
+           src="https://claremontmakerspace.org/wp-content/uploads/2019/03/CMS-Logo-b-y-g-300x168.png"
+           alt=""
+           width="300"
+           height="168" />
+    </p>
+    <p>Greetings Upper Valley Makers:</p>
+    <p>We have an exciting list of upcoming classes at the Claremont MakerSpace that we think might interest you.</p>
+    <div>
+      <strong>For most classes and events, CMS MEMBERSHIP IS NOT REQUIRED.</strong> That said, members receive a discount
+      on registration and there are some classes/events that are for members only (this will be clearly noted in the event
+      description).
+    </div>
+    <div>
+      <strong>Class policies</strong> (liability waiver, withdrawal, cancellation, etc.) can be found <a href="https://claremontmakerspace.org/class-policies/"
+    data-wpel-link="internal">here</a>.
+    </div>
+    <div>
+      <strong>Instructors:</strong> Interested in teaching a class at CMS? Please fill out our <a href="https://claremontmakerspace.org/cms-class-proposal-form/"
+    data-wpel-link="internal">Class Proposal Form</a>.
+    </div>
+    <div>
+      <strong>Tours:</strong> Want to see what the Claremont MakerSpace is all about? Tours are by appointment only.
+    </div>
+    <a href="https://tickets.claremontmakerspace.org/open.php"
+       target="_blank"
+       rel="noreferrer noopener external"
+       data-wpel-link="external">Contact Us</a> to schedule your tour where you can learn about all the awesome tools that
+    the CMS offers access to, as well as how membership, classes, and studio spaces work.
+    <hr />
+    {% for section in event_sections %}
+      {% if section.events %}
+        <h1>{{ section.title }}</h1>
+        <h4>
+          <i>{{ section.blurb }}</i>
+        </h4>
+        {% for event in section.events %}
+          {% with url="https://claremontmakerspace.org/events/#!event/register/"|add:event.url %}
+            {% spaceless %}
+              <h2 style="text-align: center;">
+                <a href="{{ url }}">
+                  {# djlint:off H006,H013 #}
+                  {% if "lgo" in event %}<img class="alignleft" width="400" src="{{ event.lgo.l }}">{% endif %}
+                  {# djlint:on #}
+                  <span>{{ event.ttl|bleach }}</span>
+                </a>
+              </h2>
+            {% endspaceless %}
+            {% spaceless %}
+              <div>
+                {# wordpress is very annoying with spacing here #}
+                {# djlint:off #}
+                <i>
+                  {# TODO: different dates probably implies multiple instances. Should read RRULE or similar from the event notes #}
+                  {{ event.sdp_dt|date }} {{ event.sdp_dt|time }} &mdash; {% if event.sdp_dt.date != event.edp_dt.date %}{{ event.edp_dt|date }}{% endif %} {{ event.edp_dt|time }}
+                </i>
+                {# djlint:on #}
+              </div>
+            {% endspaceless %}
+            {% if not section.truncate %}
+              <div>{{ event.dtl|bleach:"a,abbr,acronym,b,blockquote,code,em,i,li,ol,strong,ul,p,span,br,div" }}</div>
+              <div>
+                <a href="{{ url }}">Register for this class now!</a>
+              </div>
+            {% endif %}
+            <hr />
+          {% endwith %}
+        {% endfor %}
+      {% endif %}
+    {% endfor %}
+    <div style="clear: both;">
+      <div>Happy Makin’!</div>
+      <div>
+        We are grateful for all of the public support that our 501(c)(3), non-profit organization receives. If you’d
+        like to make a donation,please visit the <a href="https://claremontmakerspace.org/support/"><strong>Support Us
+        page</strong></a> of our website.
+      </div>
+    </div>
+  </div>
+  <div class="position-fixed end-0 bottom-0">
+    <button id="copy-button"
+            type="button"
+            class="btn btn-primary m-3"
+            data-bs-toggle="popover"
+            data-bs-content="Copied!">
+      <i class="bi bi-clipboard-fill"></i> Copy to clipboard
+    </button>
+  </div>
+{% endblock %}
+{% block script %}
+  <script>
+    // TODO: This should use the newer Clipboard API, but Firefox doesn't support it yet
+    // https://developer.mozilla.org/en-US/docs/Web/API/ClipboardItem
+    function copyToClipboard(event) {
+      const cb = e => {
+        e.clipboardData.setData("text/html", document.getElementById("preview").innerHTML);
+        e.clipboardData.setData("text/plain", document.getElementById("preview").innerHTML);
+        e.preventDefault();
+      };
+      document.addEventListener("copy", cb);
+      document.execCommand("copy");
+      document.removeEventListener("copy", cb);
+
+      bootstrap.Popover.getInstance(event.target).show();
+      setTimeout(() => bootstrap.Popover.getInstance(event.target).hide(), 1000);
+    }
+    const button = document.getElementById("copy-button");
+    const popover = new bootstrap.Popover(button, {
+      trigger: "manual"
+    })
+    button.addEventListener("click", copyToClipboard);
+  </script>
+{% endblock %}

membershipworks/urls.py

@@ -1,6 +1,6 @@
 from django.urls import path
 
-from .views import MemberAutocomplete
+from .views import MemberAutocomplete, upcoming_events
 
 app_name = "membershipworks"
 
@@ -10,4 +10,9 @@ urlpatterns = [
         MemberAutocomplete.as_view(),
         name="member-autocomplete",
     ),
+    path(
+        "upcoming-events/",
+        upcoming_events,
+        name="upcoming-events",
+    ),
 ]

membershipworks/views.py

@@ -1,6 +1,15 @@
+from datetime import datetime
+
+from django.conf import settings
+from django.contrib import messages
+from django.contrib.auth.decorators import login_required
+from django.http import HttpResponse
+from django.shortcuts import render
+
 from dal import autocomplete
 
 from .models import Member
+from membershipworks import MembershipWorks
 
 
 class MemberAutocomplete(autocomplete.Select2QuerySetView):
@@ -12,3 +21,84 @@ class MemberAutocomplete(autocomplete.Select2QuerySetView):
             return Member.objects.none()
         else:
             return super().get_queryset()
+
+
+@login_required
+# TODO: permission required?
+def upcoming_events(request):
+    now = datetime.now()
+
+    membershipworks = MembershipWorks()
+    membershipworks.login(
+        settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
+    )
+
+    events = membershipworks.get_events_list(now)
+    if "error" in events:
+        messages.add_message(
+            request,
+            messages.ERROR,
+            f"MembershipWorks Error: {events['error']}",
+        )
+        # TODO: this should probably be an HTTP 500 response
+        return render(request, "base.dj.html")
+
+    ongoing_events = []
+    full_events = []
+    upcoming_events = []
+    for event in events["evt"]:
+        try:
+            # ignore hidden events
+            if event["cal"] == 0:
+                continue
+            event_details = membershipworks.get_event_by_eid(event["eid"])
+
+            # Convert timestamps to datetime objects
+            event_details["sdp_dt"] = datetime.fromtimestamp(event_details["sdp"])
+            event_details["edp_dt"] = datetime.fromtimestamp(event_details["edp"])
+            print(event_details["ttl"])
+
+            # registration has already ended
+            if (
+                "erd" in event_details
+                and datetime.fromtimestamp(event_details["erd"]) < now
+            ):
+                ongoing_events.append(event_details)
+            # class is full
+            elif event_details["cnt"] >= event_details["cap"]:
+                full_events.append(event_details)
+            else:
+                upcoming_events.append(event_details)
+
+        except KeyError as e:
+            messages.add_message(
+                request,
+                messages.ERROR,
+                f"Event '{event.get('ttl')}' missing required property: '{e.args[0]}'",
+            )
+            # TODO: this should probably be an HTTP 500 response
+            return render(request, "base.dj.html")
+
+    context = {
+        "event_sections": [
+            {
+                "title": "Upcoming Events",
+                "blurb": "Events that are currently open for registration.",
+                "events": upcoming_events,
+                "truncate": False,
+            },
+            {
+                "title": "Just Missed",
+                "blurb": "These classes are currently full at time of writing. If you are interested, please check the event's page; spots occasionally open up. Keep an eye on this newsletter to see when these classes are offered again.",
+                "events": full_events,
+                "truncate": True,
+            },
+            {
+                "title": "Ongoing Events",
+                "blurb": "These events are ongoing. Registration is currently closed, but these events may be offered again in the future.",
+                "events": ongoing_events,
+                "truncate": True,
+            },
+        ]
+    }
+    return render(request, "membershipworks/upcoming_events.dj.html", context)

paperwork/migrations/0009_rename_department_list_moderator_flag_to_shop_lead_flag.py

@@ -6,7 +6,7 @@ import django.db.models.deletion
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("membershipworks", "0002_alter_flag_options"),
+        ("membershipworks", "0001_initial"),
         ("paperwork", "0008_remove_certificationdefinition_mailing_list"),
     ]
 

paperwork/migrations/0014_alter_certificationdefinition_options_and_more.py

@@ -6,7 +6,7 @@ import django.db.models.deletion
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("membershipworks", "0002_alter_flag_options"),
+        ("membershipworks", "0001_initial"),
         ("paperwork", "0013_alter_certificationdefinition_certification_name"),
     ]
 

pdm.lock

@@ -5,7 +5,7 @@
 groups = ["default", "debug", "lint", "server", "typing", "dev"]
 strategy = ["cross_platform"]
 lock_version = "4.4"
-content_hash = "sha256:8afb89517cfd55ec9138e679d7df47143d3dda10543096382656cb2028e97b50"
+content_hash = "sha256:4b66341c252a0c283b65ee725342a18f2b71c34811cc7853c75e78fde80815df"
 
 [[package]]
 name = "aiohttp"
@@ -443,6 +443,19 @@ files = [
     {file = "django-autocomplete-light-3.9.7.tar.gz", hash = "sha256:a34f192ac438c4df056dbfd399550799ddc631c4661960134ded924648770373"},
 ]
 
+[[package]]
+name = "django-bleach"
+version = "1.0.0"
+summary = "Easily use bleach with Django models and templates"
+dependencies = [
+    "Django>=1.11",
+    "bleach>=1.5.0",
+]
+files = [
+    {file = "django-bleach-1.0.0.tar.gz", hash = "sha256:2586b90d641d4d7e70ee353570ad33d3625ed4b97036a3ea5b03ea1bb5bbeccd"},
+    {file = "django_bleach-1.0.0-py2.py3-none-any.whl", hash = "sha256:60074a4f4bc8d5200fdb2e03dce16fb4913427698b64570bc3e1a7ea1b8c3cf7"},
+]
+
 [[package]]
 name = "django-debug-toolbar"
 version = "4.2.0"

pyproject.toml

@@ -28,6 +28,7 @@ dependencies = [
     "django-object-actions~=4.2",
     "udm-rest-client~=1.2",
     "openapi-client-udm~=1.0",
+    "django-bleach~=1.0",
 ]
 requires-python = ">=3.11"