from datetime import datetime, timedelta import logging from django.conf import settings from django.db import transaction from membershipworks.models import Member, Flag, Transaction from membershipworks import MembershipWorks logger = logging.getLogger(__name__) def flags_for_member(csv_member, all_flags, folders): for flag in all_flags: if flag.type == "folder": if csv_member["Account ID"] in folders[flag.id]: yield flag else: if csv_member[flag.name] == flag.name: yield flag def update_flags(mw_flags) -> list[Flag]: for typ, flags_of_type in mw_flags.items(): for name, id in flags_of_type.items(): flag = Flag(id=id, name=name, type=typ[:-1]) flag.save() yield flag def scrape_members(membershipworks: MembershipWorks): logger.info("Updating flags (labels, levels, and addons)") flags = list(update_flags(membershipworks._parse_flags())) logger.info("Getting folder membership") folders = { folder_id: membershipworks.get_member_ids([folder_name]) for folder_name, folder_id in membershipworks._parse_flags()["folders"].items() } logger.info("Getting/Updating members...") members = membershipworks.get_all_members() for csv_member in members: for field_id, field in membershipworks._all_fields().items(): # convert checkboxes to real booleans if field.get("typ") == 8 and field["lbl"] in csv_member: csv_member[field["lbl"]] = ( True if csv_member[field["lbl"]] == "Y" else False ) # create/update member member = Member.from_csv_dict(csv_member) member.clean_fields() member.save() member.flags.set(flags_for_member(csv_member, flags, folders)) def scrape_transactions(membershipworks: MembershipWorks): now = datetime.now() start_date = datetime(2010, 1, 1) last_transaction = Transaction.objects.order_by("timestamp").last() if last_transaction is not None: # technically this has the potential to lose # transactions, but it should be incredibly unlikely start_date = last_transaction.timestamp + timedelta(seconds=1) logger.info(f"Getting/Updating transactions since {start_date}...") transactions_csv = membershipworks.get_transactions(start_date, now) transactions_json = membershipworks.get_transactions(start_date, now, json=True) # this is terrible, but as long as the dates are the same, should be fiiiine transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)] assert all( [ t["Account ID"] == t.get("uid", "") and t["Payment ID"] == t.get("sid", "") for t in transactions ] ) for csv_transaction in transactions: Transaction.from_csv_dict(csv_transaction).save() @transaction.atomic def scrape_membershipworks(*args, **options): membershipworks = MembershipWorks() membershipworks.login( settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD ) scrape_members(membershipworks) scrape_transactions(membershipworks)