cmsmanage/membershipworks/management/commands/scrape_membershipworks.py
Adam Goldsmith 02c9be5ae6 membershipworks: Get only transactions since last in database + 1 second
This avoids having to deduplicate transactions, at the cost of
hypothetically missing transactions in some unlikely edge cases
2023-12-20 13:04:56 -05:00

86 lines
3.3 KiB
Python

from datetime import datetime, timedelta
from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import transaction
from membershipworks.models import Member, Flag, Transaction
from membershipworks import MembershipWorks
class Command(BaseCommand):
def flags_for_member(self, csv_member, all_flags, folders):
for flag in all_flags:
if flag.type == "folder":
if csv_member["Account ID"] in folders[flag.id]:
yield flag
else:
if csv_member[flag.name] == flag.name:
yield flag
def update_flags(self, mw_flags) -> list[Flag]:
for typ, flags_of_type in mw_flags.items():
for name, id in flags_of_type.items():
flag = Flag(id=id, name=name, type=typ[:-1])
flag.save()
yield flag
@transaction.atomic()
def handle(self, *args, **options):
membershipworks = MembershipWorks()
membershipworks.login(
settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
)
print("Updating flags (labels, levels, and addons)")
flags = list(self.update_flags(membershipworks._parse_flags()))
print("Getting folder membership")
folders = {
folder_id: membershipworks.get_member_ids([folder_name])
for folder_name, folder_id in membershipworks._parse_flags()[
"folders"
].items()
}
print("Getting/Updating members...")
members = membershipworks.get_all_members()
for csv_member in members:
for field_id, field in membershipworks._all_fields().items():
# convert checkboxes to real booleans
if field.get("typ") == 8 and field["lbl"] in csv_member:
csv_member[field["lbl"]] = (
True if csv_member[field["lbl"]] == "Y" else False
)
# create/update member
member = Member.from_csv_dict(csv_member)
member.clean_fields()
member.save()
member.flags.set(self.flags_for_member(csv_member, flags, folders))
now = datetime.now()
start_date = datetime(2010, 1, 1)
last_transaction = Transaction.objects.order_by("timestamp").last()
if last_transaction is not None:
# technically this has the potential to lose
# transactions, but it should be incredibly unlikely
start_date = last_transaction.timestamp + timedelta(seconds=1)
print(f"Getting/Updating transactions since {start_date}...")
transactions_csv = membershipworks.get_transactions(start_date, now)
transactions_json = membershipworks.get_transactions(start_date, now, json=True)
# this is terrible, but as long as the dates are the same, should be fiiiine
transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
assert all(
[
t["Account ID"] == t.get("uid", "")
and t["Payment ID"] == t.get("sid", "")
for t in transactions
]
)
for csv_transaction in transactions:
Transaction.from_csv_dict(csv_transaction).save()