sqlExport: Add script to export MembershipWorks data to a MariaDB server

This commit is contained in:
Adam Goldsmith 2020-04-02 15:56:19 -04:00
parent cf84086446
commit 525fd24a22
6 changed files with 293 additions and 2 deletions

View File

@ -27,3 +27,7 @@ Retrieves member information from MembershipWorks and pushes it out to the HID E
## `ucsAccounts.py` ## `ucsAccounts.py`
Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space. Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space.
## `sqlExport.py`
Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined with [peewee](peewee-orm.com) in `./lib/mw_models.py`.

203
lib/mw_models.py Normal file
View File

@ -0,0 +1,203 @@
from datetime import datetime
from peewee import (BooleanField, FixedCharField, CompositeKey, DateField,
DateTimeField, DecimalField, ForeignKeyField, Model,
MySQLDatabase, TextField)
import passwords
database = MySQLDatabase(
**passwords.MEMBERSHIPWORKS_DB,
**{
"charset": "utf8",
"sql_mode": "PIPES_AS_CONCAT",
"use_unicode": True,
}
)
class BaseModel(Model):
_csv_headers_override = {}
_date_fields = {}
def insert_instance(self):
return self.insert(**self.__data__)
def upsert_instance(self):
return self.insert_instance() \
.on_conflict(action="update",
preserve=list(self._meta.fields.values()))
def magic_save(self):
if self._meta.primary_key is False:
self.get_or_create(**self.__data__)
else:
self.upsert_instance().execute()
@classmethod
def _headers_map(cls):
return {field.column_name: name
for name, field in cls._meta.fields.items()}
@classmethod
def _remap_headers(cls, data):
# print(data)
hmap = cls._headers_map()
hmap.update(cls._csv_headers_override)
for k, v in data.items():
if k in hmap:
yield hmap.get(k), v
@classmethod
def from_csv_dict(cls, data):
data = data.copy()
# parse date fields to datetime objects
for field, fmt in cls._date_fields.items():
if data[field]:
data[field] = datetime.strptime(str(data[field]), fmt)
else:
# convert empty string to None to make NULL in SQL
data[field] = None
return cls(**dict(cls._remap_headers(data)))
class Meta:
database = database
class Label(BaseModel):
label_id = FixedCharField(24, primary_key=True)
label = TextField(null=True)
class Meta:
table_name = 'labels'
class Member(BaseModel):
uid = FixedCharField(24, primary_key=True)
year_of_birth = TextField(column_name='Year of Birth', null=True)
account_name = TextField(column_name='Account Name', null=True)
first_name = TextField(column_name='First Name', null=True)
last_name = TextField(column_name='Last Name', null=True)
phone = TextField(column_name='Phone', null=True)
email = TextField(column_name='Email', null=True)
address_street = TextField(column_name='Address (Street)', null=True)
address_city = TextField(column_name='Address (City)', null=True)
address_state_province = TextField(column_name='Address (State/Province)', null=True)
address_postal_code = TextField(column_name='Address (Postal Code)', null=True)
address_country = TextField(column_name='Address (Country)', null=True)
profile_description = TextField(column_name='Profile description', null=True)
website = TextField(column_name='Website', null=True)
fax = TextField(column_name='Fax', null=True)
contact_person = TextField(column_name='Contact Person', null=True)
password = TextField(column_name='Password', null=True)
position_relation = TextField(column_name='Position/relation', null=True)
parent_account_id = TextField(column_name='Parent Account ID', null=True)
gift_membership_purchased_by = TextField(column_name='Gift Membership purchased by', null=True)
purchased_gift_membership_for = TextField(column_name='Purchased Gift Membership for', null=True)
closet_storage = TextField(column_name='Closet Storage #', null=True)
storage_shelf = TextField(column_name='Storage Shelf #', null=True)
personal_studio_space = TextField(column_name='Personal Studio Space #', null=True)
access_permitted_shops_during_extended_hours = BooleanField(column_name='Access Permitted Shops During Extended Hours?', null=True)
access_front_door_and_studio_space_during_extended_hours = BooleanField(column_name='Access Front Door and Studio Space During Extended Hours?', null=True)
access_wood_shop = BooleanField(column_name='Access Wood Shop?', null=True)
access_metal_shop = BooleanField(column_name='Access Metal Shop?', null=True)
access_storage_closet = BooleanField(column_name='Access Storage Closet?', null=True)
access_studio_space = BooleanField(column_name='Access Studio Space?', null=True)
access_front_door = BooleanField(column_name='Access Front Door?', null=True)
access_card_number = TextField(column_name='Access Card Number', null=True)
access_card_facility_code = TextField(column_name='Access Card Facility Code', null=True)
auto_billing_id = TextField(column_name='Auto Billing ID', null=True)
billing_method = TextField(column_name='Billing Method', null=True)
renewal_date = DateField(column_name='Renewal Date', null=True)
join_date = DateField(column_name='Join Date', null=True)
admin_note = TextField(column_name='Admin note', null=True)
profile_gallery_image_url = TextField(column_name='Profile gallery image URL', null=True)
business_card_image_url = TextField(column_name='Business card image URL', null=True)
instagram = TextField(column_name='Instagram', null=True)
pinterest = TextField(column_name='Pinterest', null=True)
youtube = TextField(column_name='Youtube', null=True)
yelp = TextField(column_name='Yelp', null=True)
google = TextField(column_name='Google+', null=True)
bbb = TextField(column_name='BBB', null=True)
twitter = TextField(column_name='Twitter', null=True)
facebook = TextField(column_name='Facebook', null=True)
linked_in = TextField(column_name='LinkedIn', null=True)
do_not_show_street_address_in_profile = TextField(column_name='Do not show street address in profile', null=True)
do_not_list_in_directory = TextField(column_name='Do not list in directory', null=True)
how_did_you_hear = TextField(column_name='HowDidYouHear', null=True)
authorize_charge = TextField(column_name='authorizeCharge', null=True)
policy_agreement = TextField(column_name='policyAgreement', null=True)
waiver_form_signed_and_on_file_date = DateField(column_name='Waiver form signed and on file date.', null=True)
membership_agreement_signed_and_on_file_date = DateField(column_name='Membership Agreement signed and on file date.', null=True)
ip_address = TextField(column_name='IP Address', null=True)
audit_date = DateField(column_name='Audit Date', null=True)
agreement_version = TextField(column_name='Agreement Version', null=True)
paperwork_status = TextField(column_name='Paperwork status', null=True)
membership_agreement_dated = BooleanField(column_name='Membership agreement dated', null=True)
membership_agreement_acknowledgement_page_filled_out = BooleanField(column_name='Membership Agreement Acknowledgement Page Filled Out', null=True)
membership_agreement_signed = BooleanField(column_name='Membership Agreement Signed', null=True)
liability_form_filled_out = BooleanField(column_name='Liability Form Filled Out', null=True)
_csv_headers_override = {
'Account ID': 'uid',
'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': 'how_did_you_hear',
'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': 'authorize_charge',
'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': 'policy_agreement'
}
_date_fields = {
'Join Date': '%b %d, %Y',
'Renewal Date': '%b %d, %Y',
'Audit Date': '%m/%d/%Y',
'Membership Agreement signed and on file date.': '%m/%d/%Y',
'Waiver form signed and on file date.': '%m/%d/%Y'
}
class Meta:
table_name = 'members'
class MemberLabel(BaseModel):
uid = ForeignKeyField(Member, column_name='uid', backref='labels')
label_id = ForeignKeyField(Label, backref='members')
class Meta:
table_name = 'member_labels'
primary_key = CompositeKey('label_id', 'uid')
class Transaction(BaseModel):
sid = FixedCharField(27, null=True)
uid = ForeignKeyField(Member, column_name='uid', backref='transactions', null=True)
timestamp = DateTimeField()
type = TextField(null=True)
sum = DecimalField(13, 4, null=True)
fee = DecimalField(13, 4, null=True)
event_id = TextField(null=True)
for_ = TextField(column_name='For', null=True)
items = TextField(column_name='Items', null=True)
discount_code = TextField(column_name='Discount Code', null=True)
note = TextField(column_name='Note', null=True)
name = TextField(column_name='Name', null=True)
contact_person = TextField(column_name='Contact Person', null=True)
full_address = TextField(column_name='Full Address', null=True)
street = TextField(column_name='Street', null=True)
city = TextField(column_name='City', null=True)
state_province = TextField(column_name='State/Province', null=True)
postal_code = TextField(column_name='Postal Code', null=True)
country = TextField(column_name='Country', null=True)
phone = TextField(column_name='Phone', null=True)
email = TextField(column_name='Email', null=True)
_csv_headers_override = {
'_dp': 'timestamp',
'Transaction Type': 'type'
}
@classmethod
def from_csv_dict(cls, data):
txn = data.copy()
# can't use '%s' format string, have to use the special function
txn['_dp'] = datetime.fromtimestamp(txn['_dp'])
return super().from_csv_dict(txn)
class Meta:
table_name = 'transactions'
primary_key = False

62
sqlExport.py Executable file
View File

@ -0,0 +1,62 @@
#!/usr/bin/env python3
from datetime import datetime
from common import membershipworks
from lib.mw_models import database, Label, Member, MemberLabel, Transaction
@database.atomic()
def main():
print("Creating tables")
database.create_tables([Label, Member, MemberLabel, Transaction])
print("Updating labels")
labels = membershipworks._parse_flags()['labels']
Label \
.insert_many([{'label_id': v, 'label': k} for k, v in labels.items()]) \
.on_conflict(action="update", preserve=[Label.label]) \
.execute()
print("Getting/Updating members...")
members = membershipworks.get_all_members()
for m in members:
# replace flags by booleans
for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
if flag in m:
m[flag] = m[flag] == flag
for field_id, field in membershipworks._all_fields().items():
# convert checkboxes to real booleans
if field.get('typ') == 8 and field['lbl'] in m: # check box
m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
for member in members:
# create/update member
Member.from_csv_dict(member).magic_save()
# update member's labels
for label, label_id in membershipworks._parse_flags()['labels'].items():
ml = MemberLabel(uid=member['Account ID'], label_id=label_id)
if member[label]:
ml.magic_save()
else:
ml.delete_instance()
print("Getting/Updating transactions...")
now = datetime.now()
start_date = datetime(2010, 1, 1)
transactions_csv = membershipworks.get_transactions(start_date, now)
transactions_json = membershipworks.get_transactions(start_date, now, json=True)
# this is terrible, but as long as the dates are the same, should be fiiiine
transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
assert all([t['Account ID'] == t.get('uid', '')
and t['Payment ID'] == t.get('sid', '')
for t in transactions])
for transaction in transactions:
Transaction.from_csv_dict(transaction).magic_save()
# TODO: folders, levels, addons
if __name__ == '__main__':
main()

View File

@ -5,5 +5,6 @@ OnFailure=status-email-admin@%n.service
[Service] [Service]
User=adam User=adam
Type=oneshot Type=oneshot
WorkingDirectory=/home/adam/hidDoorWriter/ TimeoutStartSec=600
ExecStart=/home/adam/hidDoorWriter/doorUpdater.py WorkingDirectory=/home/adam/memberPlumbing/
ExecStart=/usr/bin/python3 -u /home/adam/memberPlumbing/doorUpdater.py

View File

@ -0,0 +1,12 @@
[Unit]
Description=Sync Membershipworks with local database
OnFailure=status-email-admin@%n.service
After=mariadb.service
Requires=mariadb.service
[Service]
User=adam
Type=oneshot
TimeoutStartSec=600
WorkingDirectory=/home/adam/memberPlumbing/
ExecStart=/usr/bin/python3 -u /home/adam/memberPlumbing/sqlExport.py

View File

@ -0,0 +1,9 @@
[Unit]
Description=Hourly Membershipworks database sync
[Timer]
OnCalendar=hourly
Persistent=true
[Install]
WantedBy=timers.target