membershipworks: Make MW api->model BaseModel more generic

membershipworks/models.py

@@ -9,8 +9,9 @@ from django.utils import timezone
 
 
 class BaseModel(models.Model):
-    _csv_headers_override = {}
+    _api_names_override = {}
     _date_fields = {}
+    _allowed_missing_fields = []
 
     class Meta:
         abstract = True
@@ -22,20 +23,30 @@ class BaseModel(models.Model):
             if field.auto_created or field.many_to_many or not field.concrete:
                 continue
 
-            field_name, csv_header = field.get_attname_column()
+            field_name, api_name = field.get_attname_column()
 
-            if field_name in cls._csv_headers_override:
+            if field_name in cls._api_names_override:
-                csv_header = cls._csv_headers_override[field.name]
+                api_name = cls._api_names_override[field_name]
 
-            yield field_name, data[csv_header]
+            yield field_name, data[api_name]
 
     @classmethod
-    def from_csv_dict(cls, data):
+    def from_api_dict(cls, data):
         data = data.copy()
 
+        for field in cls._allowed_missing_fields:
+            if field not in data:
+                data[field] = None
+
         # parse date fields to datetime objects
         for field, fmt in cls._date_fields.items():
             if data[field]:
+                if fmt is None:
+                    # can't use '%s' format string, have to use the special function
+                    data[field] = datetime.fromtimestamp(
+                        data[field], tz=timezone.get_current_timezone()
+                    )
+                else:
                     data[field] = datetime.strptime(str(data[field]), fmt)
             else:
                 # convert empty string to None to make NULL in SQL
@@ -210,7 +221,7 @@ class Member(BaseModel):
     )
     flags = models.ManyToManyField(Flag, through="MemberFlag", related_name="members")
 
-    _csv_headers_override = {
+    _api_names_override = {
         "uid": "Account ID",
         "how_did_you_hear": "Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:",
         "authorize_charge": "Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.",
@@ -307,31 +318,22 @@ class Transaction(BaseModel):
     phone = models.TextField(db_column="Phone", null=True, blank=True)
     email = models.TextField(db_column="Email", null=True, blank=True)
 
-    @classmethod
+    _allowed_missing_fields = [
-    def from_csv_dict(cls, data):
-        txn = data.copy()
-        # can't use '%s' format string, have to use the special function
-        txn["_dp"] = datetime.fromtimestamp(
-            txn["_dp"], tz=timezone.get_current_timezone()
-        )
-        allowed_missing_fields = [
         "sid",
         "uid",
         "eid",
         "fee",
         "sum",
     ]
-        for field in allowed_missing_fields:
+    _api_names_override = {
-            if field not in txn:
-                txn[field] = None
-        return super().from_csv_dict(txn)
-
-    _csv_headers_override = {
         "event_id": "eid",
         "timestamp": "_dp",
         "type": "Transaction Type",
         "for_what": "Event/Form Name",
     }
+    _date_fields = {
+        "_dp": None,
+    }
 
     def __str__(self):
         return f"{self.type} [{self.member if self.member else self.name}] {self.timestamp}"

membershipworks/tasks/scrape.py

@@ -50,7 +50,7 @@ def scrape_members(membershipworks: MembershipWorks):
                 )
 
         # create/update member
-        member = Member.from_csv_dict(csv_member)
+        member = Member.from_api_dict(csv_member)
         member.clean_fields()
         member.save()
         member.flags.set(flags_for_member(csv_member, flags, folders))
@@ -79,7 +79,7 @@ def scrape_transactions(membershipworks: MembershipWorks):
     )
 
     for csv_transaction in transactions:
-        Transaction.from_csv_dict(csv_transaction).save()
+        Transaction.from_api_dict(csv_transaction).save()
 
 
 @transaction.atomic