sqlExport: Add script to export MembershipWorks data to a MariaDB server

README.md

@@ -27,3 +27,7 @@ Retrieves member information from MembershipWorks and pushes it out to the HID E
 ## `ucsAccounts.py`
 
 Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space.
+
+## `sqlExport.py`
+
+Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined with [peewee](peewee-orm.com) in `./lib/mw_models.py`.

lib/mw_models.py

@@ -0,0 +1,203 @@
+from datetime import datetime
+
+from peewee import (BooleanField, FixedCharField, CompositeKey, DateField,
+                    DateTimeField, DecimalField, ForeignKeyField, Model,
+                    MySQLDatabase, TextField)
+
+import passwords
+
+database = MySQLDatabase(
+    **passwords.MEMBERSHIPWORKS_DB,
+    **{
+        "charset": "utf8",
+        "sql_mode": "PIPES_AS_CONCAT",
+        "use_unicode": True,
+    }
+)
+
+class BaseModel(Model):
+    _csv_headers_override = {}
+    _date_fields = {}
+
+    def insert_instance(self):
+        return self.insert(**self.__data__)
+
+    def upsert_instance(self):
+        return self.insert_instance() \
+                   .on_conflict(action="update",
+                                preserve=list(self._meta.fields.values()))
+
+    def magic_save(self):
+        if self._meta.primary_key is False:
+            self.get_or_create(**self.__data__)
+        else:
+            self.upsert_instance().execute()
+
+    @classmethod
+    def _headers_map(cls):
+        return {field.column_name: name
+                for name, field in cls._meta.fields.items()}
+
+    @classmethod
+    def _remap_headers(cls, data):
+        # print(data)
+        hmap = cls._headers_map()
+        hmap.update(cls._csv_headers_override)
+        for k, v in data.items():
+            if k in hmap:
+                yield hmap.get(k), v
+
+    @classmethod
+    def from_csv_dict(cls, data):
+        data = data.copy()
+
+        # parse date fields to datetime objects
+        for field, fmt in cls._date_fields.items():
+            if data[field]:
+                data[field] = datetime.strptime(str(data[field]), fmt)
+            else:
+                # convert empty string to None to make NULL in SQL
+                data[field] = None
+
+        return cls(**dict(cls._remap_headers(data)))
+
+    class Meta:
+        database = database
+
+class Label(BaseModel):
+    label_id = FixedCharField(24, primary_key=True)
+    label = TextField(null=True)
+
+    class Meta:
+        table_name = 'labels'
+
+class Member(BaseModel):
+    uid = FixedCharField(24, primary_key=True)
+    year_of_birth = TextField(column_name='Year of Birth', null=True)
+    account_name = TextField(column_name='Account Name', null=True)
+    first_name = TextField(column_name='First Name', null=True)
+    last_name = TextField(column_name='Last Name', null=True)
+    phone = TextField(column_name='Phone', null=True)
+    email = TextField(column_name='Email', null=True)
+    address_street = TextField(column_name='Address (Street)', null=True)
+    address_city = TextField(column_name='Address (City)', null=True)
+    address_state_province = TextField(column_name='Address (State/Province)', null=True)
+    address_postal_code = TextField(column_name='Address (Postal Code)', null=True)
+    address_country = TextField(column_name='Address (Country)', null=True)
+    profile_description = TextField(column_name='Profile description', null=True)
+    website = TextField(column_name='Website', null=True)
+    fax = TextField(column_name='Fax', null=True)
+    contact_person = TextField(column_name='Contact Person', null=True)
+    password = TextField(column_name='Password', null=True)
+    position_relation = TextField(column_name='Position/relation', null=True)
+    parent_account_id = TextField(column_name='Parent Account ID', null=True)
+    gift_membership_purchased_by = TextField(column_name='Gift Membership purchased by', null=True)
+    purchased_gift_membership_for = TextField(column_name='Purchased Gift Membership for', null=True)
+    closet_storage = TextField(column_name='Closet Storage #', null=True)
+    storage_shelf = TextField(column_name='Storage Shelf #', null=True)
+    personal_studio_space = TextField(column_name='Personal Studio Space #', null=True)
+    access_permitted_shops_during_extended_hours = BooleanField(column_name='Access Permitted Shops During Extended Hours?', null=True)
+    access_front_door_and_studio_space_during_extended_hours = BooleanField(column_name='Access Front Door and Studio Space During Extended Hours?', null=True)
+    access_wood_shop = BooleanField(column_name='Access Wood Shop?', null=True)
+    access_metal_shop = BooleanField(column_name='Access Metal Shop?', null=True)
+    access_storage_closet = BooleanField(column_name='Access Storage Closet?', null=True)
+    access_studio_space = BooleanField(column_name='Access Studio Space?', null=True)
+    access_front_door = BooleanField(column_name='Access Front Door?', null=True)
+    access_card_number = TextField(column_name='Access Card Number', null=True)
+    access_card_facility_code = TextField(column_name='Access Card Facility Code', null=True)
+    auto_billing_id = TextField(column_name='Auto Billing ID', null=True)
+    billing_method = TextField(column_name='Billing Method', null=True)
+    renewal_date = DateField(column_name='Renewal Date', null=True)
+    join_date = DateField(column_name='Join Date', null=True)
+    admin_note = TextField(column_name='Admin note', null=True)
+    profile_gallery_image_url = TextField(column_name='Profile gallery image URL', null=True)
+    business_card_image_url = TextField(column_name='Business card image URL', null=True)
+    instagram = TextField(column_name='Instagram', null=True)
+    pinterest = TextField(column_name='Pinterest', null=True)
+    youtube = TextField(column_name='Youtube', null=True)
+    yelp = TextField(column_name='Yelp', null=True)
+    google = TextField(column_name='Google+', null=True)
+    bbb = TextField(column_name='BBB', null=True)
+    twitter = TextField(column_name='Twitter', null=True)
+    facebook = TextField(column_name='Facebook', null=True)
+    linked_in = TextField(column_name='LinkedIn', null=True)
+    do_not_show_street_address_in_profile = TextField(column_name='Do not show street address in profile', null=True)
+    do_not_list_in_directory = TextField(column_name='Do not list in directory', null=True)
+    how_did_you_hear = TextField(column_name='HowDidYouHear', null=True)
+    authorize_charge = TextField(column_name='authorizeCharge', null=True)
+    policy_agreement = TextField(column_name='policyAgreement', null=True)
+    waiver_form_signed_and_on_file_date = DateField(column_name='Waiver form signed and on file date.', null=True)
+    membership_agreement_signed_and_on_file_date = DateField(column_name='Membership Agreement signed and on file date.', null=True)
+    ip_address = TextField(column_name='IP Address', null=True)
+    audit_date = DateField(column_name='Audit Date', null=True)
+    agreement_version = TextField(column_name='Agreement Version', null=True)
+    paperwork_status = TextField(column_name='Paperwork status', null=True)
+    membership_agreement_dated = BooleanField(column_name='Membership agreement dated', null=True)
+    membership_agreement_acknowledgement_page_filled_out = BooleanField(column_name='Membership Agreement Acknowledgement Page Filled Out', null=True)
+    membership_agreement_signed = BooleanField(column_name='Membership Agreement Signed', null=True)
+    liability_form_filled_out = BooleanField(column_name='Liability Form Filled Out', null=True)
+
+    _csv_headers_override = {
+        'Account ID': 'uid',
+        'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': 'how_did_you_hear',
+        'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': 'authorize_charge',
+        'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': 'policy_agreement'
+    }
+
+    _date_fields = {
+        'Join Date': '%b %d, %Y',
+        'Renewal Date': '%b %d, %Y',
+        'Audit Date': '%m/%d/%Y',
+        'Membership Agreement signed and on file date.': '%m/%d/%Y',
+        'Waiver form signed and on file date.': '%m/%d/%Y'
+    }
+
+    class Meta:
+        table_name = 'members'
+
+class MemberLabel(BaseModel):
+    uid = ForeignKeyField(Member, column_name='uid', backref='labels')
+    label_id = ForeignKeyField(Label, backref='members')
+
+    class Meta:
+        table_name = 'member_labels'
+        primary_key = CompositeKey('label_id', 'uid')
+
+class Transaction(BaseModel):
+    sid = FixedCharField(27, null=True)
+    uid = ForeignKeyField(Member, column_name='uid', backref='transactions', null=True)
+    timestamp = DateTimeField()
+    type = TextField(null=True)
+    sum = DecimalField(13, 4, null=True)
+    fee = DecimalField(13, 4, null=True)
+    event_id = TextField(null=True)
+    for_ = TextField(column_name='For', null=True)
+    items = TextField(column_name='Items', null=True)
+    discount_code = TextField(column_name='Discount Code', null=True)
+    note = TextField(column_name='Note', null=True)
+    name = TextField(column_name='Name', null=True)
+    contact_person = TextField(column_name='Contact Person', null=True)
+    full_address = TextField(column_name='Full Address', null=True)
+    street = TextField(column_name='Street', null=True)
+    city = TextField(column_name='City', null=True)
+    state_province = TextField(column_name='State/Province', null=True)
+    postal_code = TextField(column_name='Postal Code', null=True)
+    country = TextField(column_name='Country', null=True)
+    phone = TextField(column_name='Phone', null=True)
+    email = TextField(column_name='Email', null=True)
+
+    _csv_headers_override = {
+        '_dp': 'timestamp',
+        'Transaction Type': 'type'
+    }
+
+    @classmethod
+    def from_csv_dict(cls, data):
+        txn = data.copy()
+        # can't use '%s' format string, have to use the special function
+        txn['_dp'] = datetime.fromtimestamp(txn['_dp'])
+        return super().from_csv_dict(txn)
+
+    class Meta:
+        table_name = 'transactions'
+        primary_key = False

sqlExport.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+
+from common import membershipworks
+from lib.mw_models import database, Label, Member, MemberLabel, Transaction
+
+@database.atomic()
+def main():
+    print("Creating tables")
+    database.create_tables([Label, Member, MemberLabel, Transaction])
+
+    print("Updating labels")
+    labels = membershipworks._parse_flags()['labels']
+    Label \
+      .insert_many([{'label_id': v, 'label': k} for k, v in labels.items()]) \
+      .on_conflict(action="update", preserve=[Label.label]) \
+      .execute()
+
+    print("Getting/Updating members...")
+    members = membershipworks.get_all_members()
+    for m in members:
+        # replace flags by booleans
+        for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
+            if flag in m:
+                m[flag] = m[flag] == flag
+
+        for field_id, field in membershipworks._all_fields().items():
+            # convert checkboxes to real booleans
+            if field.get('typ') == 8 and field['lbl'] in m: # check box
+                m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
+
+    for member in members:
+        # create/update member
+        Member.from_csv_dict(member).magic_save()
+
+        # update member's labels
+        for label, label_id in membershipworks._parse_flags()['labels'].items():
+            ml = MemberLabel(uid=member['Account ID'], label_id=label_id)
+            if member[label]:
+                ml.magic_save()
+            else:
+                ml.delete_instance()
+
+    print("Getting/Updating transactions...")
+    now = datetime.now()
+    start_date = datetime(2010, 1, 1)
+    transactions_csv = membershipworks.get_transactions(start_date, now)
+    transactions_json = membershipworks.get_transactions(start_date, now, json=True)
+    # this is terrible, but as long as the dates are the same, should be fiiiine
+    transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
+    assert all([t['Account ID'] == t.get('uid', '')
+                and t['Payment ID'] == t.get('sid', '')
+                for t in transactions])
+
+    for transaction in transactions:
+        Transaction.from_csv_dict(transaction).magic_save()
+
+    # TODO: folders, levels, addons
+
+if __name__ == '__main__':
+    main()

systemd/doorUpdater.service

@@ -5,5 +5,6 @@ OnFailure=status-email-admin@%n.service
 [Service]
 User=adam
 Type=oneshot
-WorkingDirectory=/home/adam/hidDoorWriter/
-ExecStart=/home/adam/hidDoorWriter/doorUpdater.py
+TimeoutStartSec=600
+WorkingDirectory=/home/adam/memberPlumbing/
+ExecStart=/usr/bin/python3 -u /home/adam/memberPlumbing/doorUpdater.py

systemd/membershipworksSQL.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Sync Membershipworks with local database
+OnFailure=status-email-admin@%n.service
+After=mariadb.service
+Requires=mariadb.service
+
+[Service]
+User=adam
+Type=oneshot
+TimeoutStartSec=600
+WorkingDirectory=/home/adam/memberPlumbing/
+ExecStart=/usr/bin/python3 -u /home/adam/memberPlumbing/sqlExport.py

systemd/membershipworksSQL.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Hourly Membershipworks database sync
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]
+WantedBy=timers.target