diff --git a/membershipworks/management/commands/scrape_membershipworks.py b/membershipworks/management/commands/scrape_membershipworks.py index ad6b8db..791a0ab 100644 --- a/membershipworks/management/commands/scrape_membershipworks.py +++ b/membershipworks/management/commands/scrape_membershipworks.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta from django.conf import settings from django.core.management.base import BaseCommand @@ -59,11 +59,16 @@ class Command(BaseCommand): member.save() member.flags.set(self.flags_for_member(csv_member, flags, folders)) - print("Getting/Updating transactions...") - # Deduping these is hard, so just recreate the data every time - Transaction.objects.all().delete() now = datetime.now() start_date = datetime(2010, 1, 1) + last_transaction = Transaction.objects.order_by("timestamp").last() + if last_transaction is not None: + # technically this has the potential to lose + # transactions, but it should be incredibly unlikely + start_date = last_transaction.timestamp + timedelta(seconds=1) + + print(f"Getting/Updating transactions since {start_date}...") + transactions_csv = membershipworks.get_transactions(start_date, now) transactions_json = membershipworks.get_transactions(start_date, now, json=True) # this is terrible, but as long as the dates are the same, should be fiiiine