membershipworks: Move scraping logic to tasks module

membershipworks/management/commands/scrape_membershipworks.py

@@ -1,90 +1,8 @@
-from datetime import datetime, timedelta
-
-from django.conf import settings
 from django.core.management.base import BaseCommand
-from django.db import transaction
 
-from membershipworks.models import Member, Flag, Transaction
+from membershipworks import tasks
-from membershipworks import MembershipWorks
 
 
 class Command(BaseCommand):
-    def flags_for_member(self, csv_member, all_flags, folders):
-        for flag in all_flags:
-            if flag.type == "folder":
-                if csv_member["Account ID"] in folders[flag.id]:
-                    yield flag
-            else:
-                if csv_member[flag.name] == flag.name:
-                    yield flag
-
-    def update_flags(self, mw_flags) -> list[Flag]:
-        for typ, flags_of_type in mw_flags.items():
-            for name, id in flags_of_type.items():
-                flag = Flag(id=id, name=name, type=typ[:-1])
-                flag.save()
-                yield flag
-
-    def scrape_members(self, membershipworks: MembershipWorks):
-        print("Updating flags (labels, levels, and addons)")
-        flags = list(self.update_flags(membershipworks._parse_flags()))
-
-        print("Getting folder membership")
-        folders = {
-            folder_id: membershipworks.get_member_ids([folder_name])
-            for folder_name, folder_id in membershipworks._parse_flags()[
-                "folders"
-            ].items()
-        }
-
-        print("Getting/Updating members...")
-        members = membershipworks.get_all_members()
-        for csv_member in members:
-            for field_id, field in membershipworks._all_fields().items():
-                # convert checkboxes to real booleans
-                if field.get("typ") == 8 and field["lbl"] in csv_member:
-                    csv_member[field["lbl"]] = (
-                        True if csv_member[field["lbl"]] == "Y" else False
-                    )
-
-            # create/update member
-            member = Member.from_csv_dict(csv_member)
-            member.clean_fields()
-            member.save()
-            member.flags.set(self.flags_for_member(csv_member, flags, folders))
-
-    def scrape_transactions(self, membershipworks: MembershipWorks):
-        now = datetime.now()
-        start_date = datetime(2010, 1, 1)
-        last_transaction = Transaction.objects.order_by("timestamp").last()
-        if last_transaction is not None:
-            # technically this has the potential to lose
-            # transactions, but it should be incredibly unlikely
-            start_date = last_transaction.timestamp + timedelta(seconds=1)
-
-        print(f"Getting/Updating transactions since {start_date}...")
-
-        transactions_csv = membershipworks.get_transactions(start_date, now)
-        transactions_json = membershipworks.get_transactions(start_date, now, json=True)
-        # this is terrible, but as long as the dates are the same, should be fiiiine
-        transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
-        assert all(
-            [
-                t["Account ID"] == t.get("uid", "")
-                and t["Payment ID"] == t.get("sid", "")
-                for t in transactions
-            ]
-        )
-
-        for csv_transaction in transactions:
-            Transaction.from_csv_dict(csv_transaction).save()
-
-    @transaction.atomic()
     def handle(self, *args, **options):
-        membershipworks = MembershipWorks()
+        tasks.scrape_membershipworks()
-        membershipworks.login(
-            settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
-        )
-
-        self.scrape_members(membershipworks)
-        self.scrape_transactions(membershipworks)

membershipworks/tasks.py

@@ -0,0 +1,95 @@
+from datetime import datetime, timedelta
+import logging
+
+from django.conf import settings
+from django.db import transaction
+
+from django_q.models import Schedule
+
+from membershipworks.models import Member, Flag, Transaction
+from membershipworks import MembershipWorks
+
+
+logger = logging.getLogger(__name__)
+
+
+def flags_for_member(csv_member, all_flags, folders):
+    for flag in all_flags:
+        if flag.type == "folder":
+            if csv_member["Account ID"] in folders[flag.id]:
+                yield flag
+        else:
+            if csv_member[flag.name] == flag.name:
+                yield flag
+
+
+def update_flags(mw_flags) -> list[Flag]:
+    for typ, flags_of_type in mw_flags.items():
+        for name, id in flags_of_type.items():
+            flag = Flag(id=id, name=name, type=typ[:-1])
+            flag.save()
+            yield flag
+
+
+def scrape_members(membershipworks: MembershipWorks):
+    logger.info("Updating flags (labels, levels, and addons)")
+    flags = list(update_flags(membershipworks._parse_flags()))
+
+    logger.info("Getting folder membership")
+    folders = {
+        folder_id: membershipworks.get_member_ids([folder_name])
+        for folder_name, folder_id in membershipworks._parse_flags()["folders"].items()
+    }
+
+    logger.info("Getting/Updating members...")
+    members = membershipworks.get_all_members()
+    for csv_member in members:
+        for field_id, field in membershipworks._all_fields().items():
+            # convert checkboxes to real booleans
+            if field.get("typ") == 8 and field["lbl"] in csv_member:
+                csv_member[field["lbl"]] = (
+                    True if csv_member[field["lbl"]] == "Y" else False
+                )
+
+        # create/update member
+        member = Member.from_csv_dict(csv_member)
+        member.clean_fields()
+        member.save()
+        member.flags.set(flags_for_member(csv_member, flags, folders))
+
+
+def scrape_transactions(membershipworks: MembershipWorks):
+    now = datetime.now()
+    start_date = datetime(2010, 1, 1)
+    last_transaction = Transaction.objects.order_by("timestamp").last()
+    if last_transaction is not None:
+        # technically this has the potential to lose
+        # transactions, but it should be incredibly unlikely
+        start_date = last_transaction.timestamp + timedelta(seconds=1)
+
+    logger.info(f"Getting/Updating transactions since {start_date}...")
+
+    transactions_csv = membershipworks.get_transactions(start_date, now)
+    transactions_json = membershipworks.get_transactions(start_date, now, json=True)
+    # this is terrible, but as long as the dates are the same, should be fiiiine
+    transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
+    assert all(
+        [
+            t["Account ID"] == t.get("uid", "") and t["Payment ID"] == t.get("sid", "")
+            for t in transactions
+        ]
+    )
+
+    for csv_transaction in transactions:
+        Transaction.from_csv_dict(csv_transaction).save()
+
+
+@transaction.atomic
+def scrape_membershipworks(*args, **options):
+    membershipworks = MembershipWorks()
+    membershipworks.login(
+        settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
+    )
+
+    scrape_members(membershipworks)
+    scrape_transactions(membershipworks)