From 18a811ce4443f781ebe6a3a77a2a3c1f78837459 Mon Sep 17 00:00:00 2001 From: Adam Goldsmith Date: Thu, 24 Mar 2022 17:02:52 -0400 Subject: [PATCH] membershipworks: Move scraping logic to tasks module --- .../commands/scrape_membershipworks.py | 86 +---------------- membershipworks/tasks.py | 95 +++++++++++++++++++ 2 files changed, 97 insertions(+), 84 deletions(-) create mode 100644 membershipworks/tasks.py diff --git a/membershipworks/management/commands/scrape_membershipworks.py b/membershipworks/management/commands/scrape_membershipworks.py index 990048a..4a254e2 100644 --- a/membershipworks/management/commands/scrape_membershipworks.py +++ b/membershipworks/management/commands/scrape_membershipworks.py @@ -1,90 +1,8 @@ -from datetime import datetime, timedelta - -from django.conf import settings from django.core.management.base import BaseCommand -from django.db import transaction -from membershipworks.models import Member, Flag, Transaction -from membershipworks import MembershipWorks +from membershipworks import tasks class Command(BaseCommand): - def flags_for_member(self, csv_member, all_flags, folders): - for flag in all_flags: - if flag.type == "folder": - if csv_member["Account ID"] in folders[flag.id]: - yield flag - else: - if csv_member[flag.name] == flag.name: - yield flag - - def update_flags(self, mw_flags) -> list[Flag]: - for typ, flags_of_type in mw_flags.items(): - for name, id in flags_of_type.items(): - flag = Flag(id=id, name=name, type=typ[:-1]) - flag.save() - yield flag - - def scrape_members(self, membershipworks: MembershipWorks): - print("Updating flags (labels, levels, and addons)") - flags = list(self.update_flags(membershipworks._parse_flags())) - - print("Getting folder membership") - folders = { - folder_id: membershipworks.get_member_ids([folder_name]) - for folder_name, folder_id in membershipworks._parse_flags()[ - "folders" - ].items() - } - - print("Getting/Updating members...") - members = membershipworks.get_all_members() - for csv_member in members: - for field_id, field in membershipworks._all_fields().items(): - # convert checkboxes to real booleans - if field.get("typ") == 8 and field["lbl"] in csv_member: - csv_member[field["lbl"]] = ( - True if csv_member[field["lbl"]] == "Y" else False - ) - - # create/update member - member = Member.from_csv_dict(csv_member) - member.clean_fields() - member.save() - member.flags.set(self.flags_for_member(csv_member, flags, folders)) - - def scrape_transactions(self, membershipworks: MembershipWorks): - now = datetime.now() - start_date = datetime(2010, 1, 1) - last_transaction = Transaction.objects.order_by("timestamp").last() - if last_transaction is not None: - # technically this has the potential to lose - # transactions, but it should be incredibly unlikely - start_date = last_transaction.timestamp + timedelta(seconds=1) - - print(f"Getting/Updating transactions since {start_date}...") - - transactions_csv = membershipworks.get_transactions(start_date, now) - transactions_json = membershipworks.get_transactions(start_date, now, json=True) - # this is terrible, but as long as the dates are the same, should be fiiiine - transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)] - assert all( - [ - t["Account ID"] == t.get("uid", "") - and t["Payment ID"] == t.get("sid", "") - for t in transactions - ] - ) - - for csv_transaction in transactions: - Transaction.from_csv_dict(csv_transaction).save() - - @transaction.atomic() def handle(self, *args, **options): - membershipworks = MembershipWorks() - membershipworks.login( - settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD - ) - - self.scrape_members(membershipworks) - self.scrape_transactions(membershipworks) + tasks.scrape_membershipworks() diff --git a/membershipworks/tasks.py b/membershipworks/tasks.py new file mode 100644 index 0000000..14e5174 --- /dev/null +++ b/membershipworks/tasks.py @@ -0,0 +1,95 @@ +from datetime import datetime, timedelta +import logging + +from django.conf import settings +from django.db import transaction + +from django_q.models import Schedule + +from membershipworks.models import Member, Flag, Transaction +from membershipworks import MembershipWorks + + +logger = logging.getLogger(__name__) + + +def flags_for_member(csv_member, all_flags, folders): + for flag in all_flags: + if flag.type == "folder": + if csv_member["Account ID"] in folders[flag.id]: + yield flag + else: + if csv_member[flag.name] == flag.name: + yield flag + + +def update_flags(mw_flags) -> list[Flag]: + for typ, flags_of_type in mw_flags.items(): + for name, id in flags_of_type.items(): + flag = Flag(id=id, name=name, type=typ[:-1]) + flag.save() + yield flag + + +def scrape_members(membershipworks: MembershipWorks): + logger.info("Updating flags (labels, levels, and addons)") + flags = list(update_flags(membershipworks._parse_flags())) + + logger.info("Getting folder membership") + folders = { + folder_id: membershipworks.get_member_ids([folder_name]) + for folder_name, folder_id in membershipworks._parse_flags()["folders"].items() + } + + logger.info("Getting/Updating members...") + members = membershipworks.get_all_members() + for csv_member in members: + for field_id, field in membershipworks._all_fields().items(): + # convert checkboxes to real booleans + if field.get("typ") == 8 and field["lbl"] in csv_member: + csv_member[field["lbl"]] = ( + True if csv_member[field["lbl"]] == "Y" else False + ) + + # create/update member + member = Member.from_csv_dict(csv_member) + member.clean_fields() + member.save() + member.flags.set(flags_for_member(csv_member, flags, folders)) + + +def scrape_transactions(membershipworks: MembershipWorks): + now = datetime.now() + start_date = datetime(2010, 1, 1) + last_transaction = Transaction.objects.order_by("timestamp").last() + if last_transaction is not None: + # technically this has the potential to lose + # transactions, but it should be incredibly unlikely + start_date = last_transaction.timestamp + timedelta(seconds=1) + + logger.info(f"Getting/Updating transactions since {start_date}...") + + transactions_csv = membershipworks.get_transactions(start_date, now) + transactions_json = membershipworks.get_transactions(start_date, now, json=True) + # this is terrible, but as long as the dates are the same, should be fiiiine + transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)] + assert all( + [ + t["Account ID"] == t.get("uid", "") and t["Payment ID"] == t.get("sid", "") + for t in transactions + ] + ) + + for csv_transaction in transactions: + Transaction.from_csv_dict(csv_transaction).save() + + +@transaction.atomic +def scrape_membershipworks(*args, **options): + membershipworks = MembershipWorks() + membershipworks.login( + settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD + ) + + scrape_members(membershipworks) + scrape_transactions(membershipworks)