From 02c9be5ae62fa011230c53a202856f4613450e92 Mon Sep 17 00:00:00 2001 From: Adam Goldsmith Date: Wed, 2 Mar 2022 17:19:36 -0500 Subject: [PATCH] membershipworks: Get only transactions since last in database + 1 second This avoids having to deduplicate transactions, at the cost of hypothetically missing transactions in some unlikely edge cases --- .../management/commands/scrape_membershipworks.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/membershipworks/management/commands/scrape_membershipworks.py b/membershipworks/management/commands/scrape_membershipworks.py index ad6b8db..791a0ab 100644 --- a/membershipworks/management/commands/scrape_membershipworks.py +++ b/membershipworks/management/commands/scrape_membershipworks.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta from django.conf import settings from django.core.management.base import BaseCommand @@ -59,11 +59,16 @@ class Command(BaseCommand): member.save() member.flags.set(self.flags_for_member(csv_member, flags, folders)) - print("Getting/Updating transactions...") - # Deduping these is hard, so just recreate the data every time - Transaction.objects.all().delete() now = datetime.now() start_date = datetime(2010, 1, 1) + last_transaction = Transaction.objects.order_by("timestamp").last() + if last_transaction is not None: + # technically this has the potential to lose + # transactions, but it should be incredibly unlikely + start_date = last_transaction.timestamp + timedelta(seconds=1) + + print(f"Getting/Updating transactions since {start_date}...") + transactions_csv = membershipworks.get_transactions(start_date, now) transactions_json = membershipworks.get_transactions(start_date, now, json=True) # this is terrible, but as long as the dates are the same, should be fiiiine