cmsmanage/membershipworks/tasks/scrape.py

94 lines
3.2 KiB
Python
Raw Normal View History

from datetime import datetime, timedelta
import logging
from django.conf import settings
from django.db import transaction
from membershipworks.models import Member, Flag, Transaction
from membershipworks import MembershipWorks
logger = logging.getLogger(__name__)
def flags_for_member(csv_member, all_flags, folders):
for flag in all_flags:
if flag.type == "folder":
if csv_member["Account ID"] in folders[flag.id]:
yield flag
else:
if csv_member[flag.name] == flag.name:
yield flag
def update_flags(mw_flags) -> list[Flag]:
for typ, flags_of_type in mw_flags.items():
for name, id in flags_of_type.items():
flag = Flag(id=id, name=name, type=typ[:-1])
flag.save()
yield flag
def scrape_members(membershipworks: MembershipWorks):
logger.info("Updating flags (labels, levels, and addons)")
flags = list(update_flags(membershipworks._parse_flags()))
logger.info("Getting folder membership")
folders = {
folder_id: membershipworks.get_member_ids([folder_name])
for folder_name, folder_id in membershipworks._parse_flags()["folders"].items()
}
logger.info("Getting/Updating members...")
members = membershipworks.get_all_members()
for csv_member in members:
for field_id, field in membershipworks._all_fields().items():
# convert checkboxes to real booleans
if field.get("typ") == 8 and field["lbl"] in csv_member:
csv_member[field["lbl"]] = (
True if csv_member[field["lbl"]] == "Y" else False
)
# create/update member
member = Member.from_csv_dict(csv_member)
member.clean_fields()
member.save()
member.flags.set(flags_for_member(csv_member, flags, folders))
def scrape_transactions(membershipworks: MembershipWorks):
now = datetime.now()
start_date = datetime(2010, 1, 1)
last_transaction = Transaction.objects.order_by("timestamp").last()
if last_transaction is not None:
# technically this has the potential to lose
# transactions, but it should be incredibly unlikely
start_date = last_transaction.timestamp + timedelta(seconds=1)
logger.info(f"Getting/Updating transactions since {start_date}...")
transactions_csv = membershipworks.get_transactions(start_date, now)
transactions_json = membershipworks.get_transactions(start_date, now, json=True)
# this is terrible, but as long as the dates are the same, should be fiiiine
transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
assert all(
[
t["Account ID"] == t.get("uid", "") and t["Payment ID"] == t.get("sid", "")
for t in transactions
]
)
for csv_transaction in transactions:
Transaction.from_csv_dict(csv_transaction).save()
@transaction.atomic
def scrape_membershipworks(*args, **options):
membershipworks = MembershipWorks()
membershipworks.login(
settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
)
scrape_members(membershipworks)
scrape_transactions(membershipworks)