cmsmanage/membershipworks/management/commands/scrape_membershipworks.py

from datetime import datetime, timedelta

from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import transaction

from membershipworks.models import Member, Flag, Transaction
from membershipworks import MembershipWorks


class Command(BaseCommand):
    def flags_for_member(self, csv_member, all_flags, folders):
        for flag in all_flags:
            if flag.type == "folder":
                if csv_member["Account ID"] in folders[flag.id]:
                    yield flag
            else:
                if csv_member[flag.name] == flag.name:
                    yield flag

    def update_flags(self, mw_flags) -> list[Flag]:
        for typ, flags_of_type in mw_flags.items():
            for name, id in flags_of_type.items():
                flag = Flag(id=id, name=name, type=typ[:-1])
                flag.save()
                yield flag

    def scrape_members(self, membershipworks: MembershipWorks):
        print("Updating flags (labels, levels, and addons)")
        flags = list(self.update_flags(membershipworks._parse_flags()))

        print("Getting folder membership")
        folders = {
            folder_id: membershipworks.get_member_ids([folder_name])
            for folder_name, folder_id in membershipworks._parse_flags()[
                "folders"
            ].items()
        }

        print("Getting/Updating members...")
        members = membershipworks.get_all_members()
        for csv_member in members:
            for field_id, field in membershipworks._all_fields().items():
                # convert checkboxes to real booleans
                if field.get("typ") == 8 and field["lbl"] in csv_member:
                    csv_member[field["lbl"]] = (
                        True if csv_member[field["lbl"]] == "Y" else False
                    )

            # create/update member
            member = Member.from_csv_dict(csv_member)
            member.clean_fields()
            member.save()
            member.flags.set(self.flags_for_member(csv_member, flags, folders))

    def scrape_transactions(self, membershipworks: MembershipWorks):
        now = datetime.now()
        start_date = datetime(2010, 1, 1)
        last_transaction = Transaction.objects.order_by("timestamp").last()
        if last_transaction is not None:
            # technically this has the potential to lose
            # transactions, but it should be incredibly unlikely
            start_date = last_transaction.timestamp + timedelta(seconds=1)

        print(f"Getting/Updating transactions since {start_date}...")

        transactions_csv = membershipworks.get_transactions(start_date, now)
        transactions_json = membershipworks.get_transactions(start_date, now, json=True)
        # this is terrible, but as long as the dates are the same, should be fiiiine
        transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
        assert all(
            [
                t["Account ID"] == t.get("uid", "")
                and t["Payment ID"] == t.get("sid", "")
                for t in transactions
            ]
        )

        for csv_transaction in transactions:
            Transaction.from_csv_dict(csv_transaction).save()

    @transaction.atomic()
    def handle(self, *args, **options):
        membershipworks = MembershipWorks()
        membershipworks.login(
            settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
        )

        self.scrape_members(membershipworks)
        self.scrape_transactions(membershipworks)