diff --git a/membershipworks/models.py b/membershipworks/models.py index 268cea9..453bdfe 100644 --- a/membershipworks/models.py +++ b/membershipworks/models.py @@ -9,8 +9,9 @@ from django.utils import timezone class BaseModel(models.Model): - _csv_headers_override = {} + _api_names_override = {} _date_fields = {} + _allowed_missing_fields = [] class Meta: abstract = True @@ -22,21 +23,31 @@ class BaseModel(models.Model): if field.auto_created or field.many_to_many or not field.concrete: continue - field_name, csv_header = field.get_attname_column() + field_name, api_name = field.get_attname_column() - if field_name in cls._csv_headers_override: - csv_header = cls._csv_headers_override[field.name] + if field_name in cls._api_names_override: + api_name = cls._api_names_override[field_name] - yield field_name, data[csv_header] + yield field_name, data[api_name] @classmethod - def from_csv_dict(cls, data): + def from_api_dict(cls, data): data = data.copy() + for field in cls._allowed_missing_fields: + if field not in data: + data[field] = None + # parse date fields to datetime objects for field, fmt in cls._date_fields.items(): if data[field]: - data[field] = datetime.strptime(str(data[field]), fmt) + if fmt is None: + # can't use '%s' format string, have to use the special function + data[field] = datetime.fromtimestamp( + data[field], tz=timezone.get_current_timezone() + ) + else: + data[field] = datetime.strptime(str(data[field]), fmt) else: # convert empty string to None to make NULL in SQL data[field] = None @@ -210,7 +221,7 @@ class Member(BaseModel): ) flags = models.ManyToManyField(Flag, through="MemberFlag", related_name="members") - _csv_headers_override = { + _api_names_override = { "uid": "Account ID", "how_did_you_hear": "Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:", "authorize_charge": "Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.", @@ -307,31 +318,22 @@ class Transaction(BaseModel): phone = models.TextField(db_column="Phone", null=True, blank=True) email = models.TextField(db_column="Email", null=True, blank=True) - @classmethod - def from_csv_dict(cls, data): - txn = data.copy() - # can't use '%s' format string, have to use the special function - txn["_dp"] = datetime.fromtimestamp( - txn["_dp"], tz=timezone.get_current_timezone() - ) - allowed_missing_fields = [ - "sid", - "uid", - "eid", - "fee", - "sum", - ] - for field in allowed_missing_fields: - if field not in txn: - txn[field] = None - return super().from_csv_dict(txn) - - _csv_headers_override = { + _allowed_missing_fields = [ + "sid", + "uid", + "eid", + "fee", + "sum", + ] + _api_names_override = { "event_id": "eid", "timestamp": "_dp", "type": "Transaction Type", "for_what": "Event/Form Name", } + _date_fields = { + "_dp": None, + } def __str__(self): return f"{self.type} [{self.member if self.member else self.name}] {self.timestamp}" diff --git a/membershipworks/tasks/scrape.py b/membershipworks/tasks/scrape.py index ce5f0f8..8cbe0cd 100644 --- a/membershipworks/tasks/scrape.py +++ b/membershipworks/tasks/scrape.py @@ -50,7 +50,7 @@ def scrape_members(membershipworks: MembershipWorks): ) # create/update member - member = Member.from_csv_dict(csv_member) + member = Member.from_api_dict(csv_member) member.clean_fields() member.save() member.flags.set(flags_for_member(csv_member, flags, folders)) @@ -79,7 +79,7 @@ def scrape_transactions(membershipworks: MembershipWorks): ) for csv_transaction in transactions: - Transaction.from_csv_dict(csv_transaction).save() + Transaction.from_api_dict(csv_transaction).save() @transaction.atomic