From 525fd24a2299945ef8b48061607f724bd853066c Mon Sep 17 00:00:00 2001 From: Adam Goldsmith Date: Thu, 2 Apr 2020 15:56:19 -0400 Subject: [PATCH] sqlExport: Add script to export MembershipWorks data to a MariaDB server --- README.md | 4 + lib/mw_models.py | 203 +++++++++++++++++++++++++++++ sqlExport.py | 62 +++++++++ systemd/doorUpdater.service | 5 +- systemd/membershipworksSQL.service | 12 ++ systemd/membershipworksSQL.timer | 9 ++ 6 files changed, 293 insertions(+), 2 deletions(-) create mode 100644 lib/mw_models.py create mode 100755 sqlExport.py create mode 100644 systemd/membershipworksSQL.service create mode 100644 systemd/membershipworksSQL.timer diff --git a/README.md b/README.md index f35a1e6..ff4a322 100644 --- a/README.md +++ b/README.md @@ -27,3 +27,7 @@ Retrieves member information from MembershipWorks and pushes it out to the HID E ## `ucsAccounts.py` Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space. + +## `sqlExport.py` + +Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined with [peewee](peewee-orm.com) in `./lib/mw_models.py`. diff --git a/lib/mw_models.py b/lib/mw_models.py new file mode 100644 index 0000000..13f6eb1 --- /dev/null +++ b/lib/mw_models.py @@ -0,0 +1,203 @@ +from datetime import datetime + +from peewee import (BooleanField, FixedCharField, CompositeKey, DateField, + DateTimeField, DecimalField, ForeignKeyField, Model, + MySQLDatabase, TextField) + +import passwords + +database = MySQLDatabase( + **passwords.MEMBERSHIPWORKS_DB, + **{ + "charset": "utf8", + "sql_mode": "PIPES_AS_CONCAT", + "use_unicode": True, + } +) + +class BaseModel(Model): + _csv_headers_override = {} + _date_fields = {} + + def insert_instance(self): + return self.insert(**self.__data__) + + def upsert_instance(self): + return self.insert_instance() \ + .on_conflict(action="update", + preserve=list(self._meta.fields.values())) + + def magic_save(self): + if self._meta.primary_key is False: + self.get_or_create(**self.__data__) + else: + self.upsert_instance().execute() + + @classmethod + def _headers_map(cls): + return {field.column_name: name + for name, field in cls._meta.fields.items()} + + @classmethod + def _remap_headers(cls, data): + # print(data) + hmap = cls._headers_map() + hmap.update(cls._csv_headers_override) + for k, v in data.items(): + if k in hmap: + yield hmap.get(k), v + + @classmethod + def from_csv_dict(cls, data): + data = data.copy() + + # parse date fields to datetime objects + for field, fmt in cls._date_fields.items(): + if data[field]: + data[field] = datetime.strptime(str(data[field]), fmt) + else: + # convert empty string to None to make NULL in SQL + data[field] = None + + return cls(**dict(cls._remap_headers(data))) + + class Meta: + database = database + +class Label(BaseModel): + label_id = FixedCharField(24, primary_key=True) + label = TextField(null=True) + + class Meta: + table_name = 'labels' + +class Member(BaseModel): + uid = FixedCharField(24, primary_key=True) + year_of_birth = TextField(column_name='Year of Birth', null=True) + account_name = TextField(column_name='Account Name', null=True) + first_name = TextField(column_name='First Name', null=True) + last_name = TextField(column_name='Last Name', null=True) + phone = TextField(column_name='Phone', null=True) + email = TextField(column_name='Email', null=True) + address_street = TextField(column_name='Address (Street)', null=True) + address_city = TextField(column_name='Address (City)', null=True) + address_state_province = TextField(column_name='Address (State/Province)', null=True) + address_postal_code = TextField(column_name='Address (Postal Code)', null=True) + address_country = TextField(column_name='Address (Country)', null=True) + profile_description = TextField(column_name='Profile description', null=True) + website = TextField(column_name='Website', null=True) + fax = TextField(column_name='Fax', null=True) + contact_person = TextField(column_name='Contact Person', null=True) + password = TextField(column_name='Password', null=True) + position_relation = TextField(column_name='Position/relation', null=True) + parent_account_id = TextField(column_name='Parent Account ID', null=True) + gift_membership_purchased_by = TextField(column_name='Gift Membership purchased by', null=True) + purchased_gift_membership_for = TextField(column_name='Purchased Gift Membership for', null=True) + closet_storage = TextField(column_name='Closet Storage #', null=True) + storage_shelf = TextField(column_name='Storage Shelf #', null=True) + personal_studio_space = TextField(column_name='Personal Studio Space #', null=True) + access_permitted_shops_during_extended_hours = BooleanField(column_name='Access Permitted Shops During Extended Hours?', null=True) + access_front_door_and_studio_space_during_extended_hours = BooleanField(column_name='Access Front Door and Studio Space During Extended Hours?', null=True) + access_wood_shop = BooleanField(column_name='Access Wood Shop?', null=True) + access_metal_shop = BooleanField(column_name='Access Metal Shop?', null=True) + access_storage_closet = BooleanField(column_name='Access Storage Closet?', null=True) + access_studio_space = BooleanField(column_name='Access Studio Space?', null=True) + access_front_door = BooleanField(column_name='Access Front Door?', null=True) + access_card_number = TextField(column_name='Access Card Number', null=True) + access_card_facility_code = TextField(column_name='Access Card Facility Code', null=True) + auto_billing_id = TextField(column_name='Auto Billing ID', null=True) + billing_method = TextField(column_name='Billing Method', null=True) + renewal_date = DateField(column_name='Renewal Date', null=True) + join_date = DateField(column_name='Join Date', null=True) + admin_note = TextField(column_name='Admin note', null=True) + profile_gallery_image_url = TextField(column_name='Profile gallery image URL', null=True) + business_card_image_url = TextField(column_name='Business card image URL', null=True) + instagram = TextField(column_name='Instagram', null=True) + pinterest = TextField(column_name='Pinterest', null=True) + youtube = TextField(column_name='Youtube', null=True) + yelp = TextField(column_name='Yelp', null=True) + google = TextField(column_name='Google+', null=True) + bbb = TextField(column_name='BBB', null=True) + twitter = TextField(column_name='Twitter', null=True) + facebook = TextField(column_name='Facebook', null=True) + linked_in = TextField(column_name='LinkedIn', null=True) + do_not_show_street_address_in_profile = TextField(column_name='Do not show street address in profile', null=True) + do_not_list_in_directory = TextField(column_name='Do not list in directory', null=True) + how_did_you_hear = TextField(column_name='HowDidYouHear', null=True) + authorize_charge = TextField(column_name='authorizeCharge', null=True) + policy_agreement = TextField(column_name='policyAgreement', null=True) + waiver_form_signed_and_on_file_date = DateField(column_name='Waiver form signed and on file date.', null=True) + membership_agreement_signed_and_on_file_date = DateField(column_name='Membership Agreement signed and on file date.', null=True) + ip_address = TextField(column_name='IP Address', null=True) + audit_date = DateField(column_name='Audit Date', null=True) + agreement_version = TextField(column_name='Agreement Version', null=True) + paperwork_status = TextField(column_name='Paperwork status', null=True) + membership_agreement_dated = BooleanField(column_name='Membership agreement dated', null=True) + membership_agreement_acknowledgement_page_filled_out = BooleanField(column_name='Membership Agreement Acknowledgement Page Filled Out', null=True) + membership_agreement_signed = BooleanField(column_name='Membership Agreement Signed', null=True) + liability_form_filled_out = BooleanField(column_name='Liability Form Filled Out', null=True) + + _csv_headers_override = { + 'Account ID': 'uid', + 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': 'how_did_you_hear', + 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': 'authorize_charge', + 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': 'policy_agreement' + } + + _date_fields = { + 'Join Date': '%b %d, %Y', + 'Renewal Date': '%b %d, %Y', + 'Audit Date': '%m/%d/%Y', + 'Membership Agreement signed and on file date.': '%m/%d/%Y', + 'Waiver form signed and on file date.': '%m/%d/%Y' + } + + class Meta: + table_name = 'members' + +class MemberLabel(BaseModel): + uid = ForeignKeyField(Member, column_name='uid', backref='labels') + label_id = ForeignKeyField(Label, backref='members') + + class Meta: + table_name = 'member_labels' + primary_key = CompositeKey('label_id', 'uid') + +class Transaction(BaseModel): + sid = FixedCharField(27, null=True) + uid = ForeignKeyField(Member, column_name='uid', backref='transactions', null=True) + timestamp = DateTimeField() + type = TextField(null=True) + sum = DecimalField(13, 4, null=True) + fee = DecimalField(13, 4, null=True) + event_id = TextField(null=True) + for_ = TextField(column_name='For', null=True) + items = TextField(column_name='Items', null=True) + discount_code = TextField(column_name='Discount Code', null=True) + note = TextField(column_name='Note', null=True) + name = TextField(column_name='Name', null=True) + contact_person = TextField(column_name='Contact Person', null=True) + full_address = TextField(column_name='Full Address', null=True) + street = TextField(column_name='Street', null=True) + city = TextField(column_name='City', null=True) + state_province = TextField(column_name='State/Province', null=True) + postal_code = TextField(column_name='Postal Code', null=True) + country = TextField(column_name='Country', null=True) + phone = TextField(column_name='Phone', null=True) + email = TextField(column_name='Email', null=True) + + _csv_headers_override = { + '_dp': 'timestamp', + 'Transaction Type': 'type' + } + + @classmethod + def from_csv_dict(cls, data): + txn = data.copy() + # can't use '%s' format string, have to use the special function + txn['_dp'] = datetime.fromtimestamp(txn['_dp']) + return super().from_csv_dict(txn) + + class Meta: + table_name = 'transactions' + primary_key = False diff --git a/sqlExport.py b/sqlExport.py new file mode 100755 index 0000000..ba5dc8f --- /dev/null +++ b/sqlExport.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +from datetime import datetime + +from common import membershipworks +from lib.mw_models import database, Label, Member, MemberLabel, Transaction + +@database.atomic() +def main(): + print("Creating tables") + database.create_tables([Label, Member, MemberLabel, Transaction]) + + print("Updating labels") + labels = membershipworks._parse_flags()['labels'] + Label \ + .insert_many([{'label_id': v, 'label': k} for k, v in labels.items()]) \ + .on_conflict(action="update", preserve=[Label.label]) \ + .execute() + + print("Getting/Updating members...") + members = membershipworks.get_all_members() + for m in members: + # replace flags by booleans + for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]: + if flag in m: + m[flag] = m[flag] == flag + + for field_id, field in membershipworks._all_fields().items(): + # convert checkboxes to real booleans + if field.get('typ') == 8 and field['lbl'] in m: # check box + m[field['lbl']] = True if m[field['lbl']] == 'Y' else False + + for member in members: + # create/update member + Member.from_csv_dict(member).magic_save() + + # update member's labels + for label, label_id in membershipworks._parse_flags()['labels'].items(): + ml = MemberLabel(uid=member['Account ID'], label_id=label_id) + if member[label]: + ml.magic_save() + else: + ml.delete_instance() + + print("Getting/Updating transactions...") + now = datetime.now() + start_date = datetime(2010, 1, 1) + transactions_csv = membershipworks.get_transactions(start_date, now) + transactions_json = membershipworks.get_transactions(start_date, now, json=True) + # this is terrible, but as long as the dates are the same, should be fiiiine + transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)] + assert all([t['Account ID'] == t.get('uid', '') + and t['Payment ID'] == t.get('sid', '') + for t in transactions]) + + for transaction in transactions: + Transaction.from_csv_dict(transaction).magic_save() + + # TODO: folders, levels, addons + +if __name__ == '__main__': + main() diff --git a/systemd/doorUpdater.service b/systemd/doorUpdater.service index 4287ec3..7fab175 100644 --- a/systemd/doorUpdater.service +++ b/systemd/doorUpdater.service @@ -5,5 +5,6 @@ OnFailure=status-email-admin@%n.service [Service] User=adam Type=oneshot -WorkingDirectory=/home/adam/hidDoorWriter/ -ExecStart=/home/adam/hidDoorWriter/doorUpdater.py +TimeoutStartSec=600 +WorkingDirectory=/home/adam/memberPlumbing/ +ExecStart=/usr/bin/python3 -u /home/adam/memberPlumbing/doorUpdater.py diff --git a/systemd/membershipworksSQL.service b/systemd/membershipworksSQL.service new file mode 100644 index 0000000..ff0b003 --- /dev/null +++ b/systemd/membershipworksSQL.service @@ -0,0 +1,12 @@ +[Unit] +Description=Sync Membershipworks with local database +OnFailure=status-email-admin@%n.service +After=mariadb.service +Requires=mariadb.service + +[Service] +User=adam +Type=oneshot +TimeoutStartSec=600 +WorkingDirectory=/home/adam/memberPlumbing/ +ExecStart=/usr/bin/python3 -u /home/adam/memberPlumbing/sqlExport.py diff --git a/systemd/membershipworksSQL.timer b/systemd/membershipworksSQL.timer new file mode 100644 index 0000000..05461a2 --- /dev/null +++ b/systemd/membershipworksSQL.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Hourly Membershipworks database sync + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target