sqlExport: Add script to export MembershipWorks data to a MariaDB server
This commit is contained in:
parent
cf84086446
commit
525fd24a22
@ -27,3 +27,7 @@ Retrieves member information from MembershipWorks and pushes it out to the HID E
|
||||
## `ucsAccounts.py`
|
||||
|
||||
Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space.
|
||||
|
||||
## `sqlExport.py`
|
||||
|
||||
Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined with [peewee](peewee-orm.com) in `./lib/mw_models.py`.
|
||||
|
203
lib/mw_models.py
Normal file
203
lib/mw_models.py
Normal file
@ -0,0 +1,203 @@
|
||||
from datetime import datetime
|
||||
|
||||
from peewee import (BooleanField, FixedCharField, CompositeKey, DateField,
|
||||
DateTimeField, DecimalField, ForeignKeyField, Model,
|
||||
MySQLDatabase, TextField)
|
||||
|
||||
import passwords
|
||||
|
||||
database = MySQLDatabase(
|
||||
**passwords.MEMBERSHIPWORKS_DB,
|
||||
**{
|
||||
"charset": "utf8",
|
||||
"sql_mode": "PIPES_AS_CONCAT",
|
||||
"use_unicode": True,
|
||||
}
|
||||
)
|
||||
|
||||
class BaseModel(Model):
|
||||
_csv_headers_override = {}
|
||||
_date_fields = {}
|
||||
|
||||
def insert_instance(self):
|
||||
return self.insert(**self.__data__)
|
||||
|
||||
def upsert_instance(self):
|
||||
return self.insert_instance() \
|
||||
.on_conflict(action="update",
|
||||
preserve=list(self._meta.fields.values()))
|
||||
|
||||
def magic_save(self):
|
||||
if self._meta.primary_key is False:
|
||||
self.get_or_create(**self.__data__)
|
||||
else:
|
||||
self.upsert_instance().execute()
|
||||
|
||||
@classmethod
|
||||
def _headers_map(cls):
|
||||
return {field.column_name: name
|
||||
for name, field in cls._meta.fields.items()}
|
||||
|
||||
@classmethod
|
||||
def _remap_headers(cls, data):
|
||||
# print(data)
|
||||
hmap = cls._headers_map()
|
||||
hmap.update(cls._csv_headers_override)
|
||||
for k, v in data.items():
|
||||
if k in hmap:
|
||||
yield hmap.get(k), v
|
||||
|
||||
@classmethod
|
||||
def from_csv_dict(cls, data):
|
||||
data = data.copy()
|
||||
|
||||
# parse date fields to datetime objects
|
||||
for field, fmt in cls._date_fields.items():
|
||||
if data[field]:
|
||||
data[field] = datetime.strptime(str(data[field]), fmt)
|
||||
else:
|
||||
# convert empty string to None to make NULL in SQL
|
||||
data[field] = None
|
||||
|
||||
return cls(**dict(cls._remap_headers(data)))
|
||||
|
||||
class Meta:
|
||||
database = database
|
||||
|
||||
class Label(BaseModel):
|
||||
label_id = FixedCharField(24, primary_key=True)
|
||||
label = TextField(null=True)
|
||||
|
||||
class Meta:
|
||||
table_name = 'labels'
|
||||
|
||||
class Member(BaseModel):
|
||||
uid = FixedCharField(24, primary_key=True)
|
||||
year_of_birth = TextField(column_name='Year of Birth', null=True)
|
||||
account_name = TextField(column_name='Account Name', null=True)
|
||||
first_name = TextField(column_name='First Name', null=True)
|
||||
last_name = TextField(column_name='Last Name', null=True)
|
||||
phone = TextField(column_name='Phone', null=True)
|
||||
email = TextField(column_name='Email', null=True)
|
||||
address_street = TextField(column_name='Address (Street)', null=True)
|
||||
address_city = TextField(column_name='Address (City)', null=True)
|
||||
address_state_province = TextField(column_name='Address (State/Province)', null=True)
|
||||
address_postal_code = TextField(column_name='Address (Postal Code)', null=True)
|
||||
address_country = TextField(column_name='Address (Country)', null=True)
|
||||
profile_description = TextField(column_name='Profile description', null=True)
|
||||
website = TextField(column_name='Website', null=True)
|
||||
fax = TextField(column_name='Fax', null=True)
|
||||
contact_person = TextField(column_name='Contact Person', null=True)
|
||||
password = TextField(column_name='Password', null=True)
|
||||
position_relation = TextField(column_name='Position/relation', null=True)
|
||||
parent_account_id = TextField(column_name='Parent Account ID', null=True)
|
||||
gift_membership_purchased_by = TextField(column_name='Gift Membership purchased by', null=True)
|
||||
purchased_gift_membership_for = TextField(column_name='Purchased Gift Membership for', null=True)
|
||||
closet_storage = TextField(column_name='Closet Storage #', null=True)
|
||||
storage_shelf = TextField(column_name='Storage Shelf #', null=True)
|
||||
personal_studio_space = TextField(column_name='Personal Studio Space #', null=True)
|
||||
access_permitted_shops_during_extended_hours = BooleanField(column_name='Access Permitted Shops During Extended Hours?', null=True)
|
||||
access_front_door_and_studio_space_during_extended_hours = BooleanField(column_name='Access Front Door and Studio Space During Extended Hours?', null=True)
|
||||
access_wood_shop = BooleanField(column_name='Access Wood Shop?', null=True)
|
||||
access_metal_shop = BooleanField(column_name='Access Metal Shop?', null=True)
|
||||
access_storage_closet = BooleanField(column_name='Access Storage Closet?', null=True)
|
||||
access_studio_space = BooleanField(column_name='Access Studio Space?', null=True)
|
||||
access_front_door = BooleanField(column_name='Access Front Door?', null=True)
|
||||
access_card_number = TextField(column_name='Access Card Number', null=True)
|
||||
access_card_facility_code = TextField(column_name='Access Card Facility Code', null=True)
|
||||
auto_billing_id = TextField(column_name='Auto Billing ID', null=True)
|
||||
billing_method = TextField(column_name='Billing Method', null=True)
|
||||
renewal_date = DateField(column_name='Renewal Date', null=True)
|
||||
join_date = DateField(column_name='Join Date', null=True)
|
||||
admin_note = TextField(column_name='Admin note', null=True)
|
||||
profile_gallery_image_url = TextField(column_name='Profile gallery image URL', null=True)
|
||||
business_card_image_url = TextField(column_name='Business card image URL', null=True)
|
||||
instagram = TextField(column_name='Instagram', null=True)
|
||||
pinterest = TextField(column_name='Pinterest', null=True)
|
||||
youtube = TextField(column_name='Youtube', null=True)
|
||||
yelp = TextField(column_name='Yelp', null=True)
|
||||
google = TextField(column_name='Google+', null=True)
|
||||
bbb = TextField(column_name='BBB', null=True)
|
||||
twitter = TextField(column_name='Twitter', null=True)
|
||||
facebook = TextField(column_name='Facebook', null=True)
|
||||
linked_in = TextField(column_name='LinkedIn', null=True)
|
||||
do_not_show_street_address_in_profile = TextField(column_name='Do not show street address in profile', null=True)
|
||||
do_not_list_in_directory = TextField(column_name='Do not list in directory', null=True)
|
||||
how_did_you_hear = TextField(column_name='HowDidYouHear', null=True)
|
||||
authorize_charge = TextField(column_name='authorizeCharge', null=True)
|
||||
policy_agreement = TextField(column_name='policyAgreement', null=True)
|
||||
waiver_form_signed_and_on_file_date = DateField(column_name='Waiver form signed and on file date.', null=True)
|
||||
membership_agreement_signed_and_on_file_date = DateField(column_name='Membership Agreement signed and on file date.', null=True)
|
||||
ip_address = TextField(column_name='IP Address', null=True)
|
||||
audit_date = DateField(column_name='Audit Date', null=True)
|
||||
agreement_version = TextField(column_name='Agreement Version', null=True)
|
||||
paperwork_status = TextField(column_name='Paperwork status', null=True)
|
||||
membership_agreement_dated = BooleanField(column_name='Membership agreement dated', null=True)
|
||||
membership_agreement_acknowledgement_page_filled_out = BooleanField(column_name='Membership Agreement Acknowledgement Page Filled Out', null=True)
|
||||
membership_agreement_signed = BooleanField(column_name='Membership Agreement Signed', null=True)
|
||||
liability_form_filled_out = BooleanField(column_name='Liability Form Filled Out', null=True)
|
||||
|
||||
_csv_headers_override = {
|
||||
'Account ID': 'uid',
|
||||
'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': 'how_did_you_hear',
|
||||
'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': 'authorize_charge',
|
||||
'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': 'policy_agreement'
|
||||
}
|
||||
|
||||
_date_fields = {
|
||||
'Join Date': '%b %d, %Y',
|
||||
'Renewal Date': '%b %d, %Y',
|
||||
'Audit Date': '%m/%d/%Y',
|
||||
'Membership Agreement signed and on file date.': '%m/%d/%Y',
|
||||
'Waiver form signed and on file date.': '%m/%d/%Y'
|
||||
}
|
||||
|
||||
class Meta:
|
||||
table_name = 'members'
|
||||
|
||||
class MemberLabel(BaseModel):
|
||||
uid = ForeignKeyField(Member, column_name='uid', backref='labels')
|
||||
label_id = ForeignKeyField(Label, backref='members')
|
||||
|
||||
class Meta:
|
||||
table_name = 'member_labels'
|
||||
primary_key = CompositeKey('label_id', 'uid')
|
||||
|
||||
class Transaction(BaseModel):
|
||||
sid = FixedCharField(27, null=True)
|
||||
uid = ForeignKeyField(Member, column_name='uid', backref='transactions', null=True)
|
||||
timestamp = DateTimeField()
|
||||
type = TextField(null=True)
|
||||
sum = DecimalField(13, 4, null=True)
|
||||
fee = DecimalField(13, 4, null=True)
|
||||
event_id = TextField(null=True)
|
||||
for_ = TextField(column_name='For', null=True)
|
||||
items = TextField(column_name='Items', null=True)
|
||||
discount_code = TextField(column_name='Discount Code', null=True)
|
||||
note = TextField(column_name='Note', null=True)
|
||||
name = TextField(column_name='Name', null=True)
|
||||
contact_person = TextField(column_name='Contact Person', null=True)
|
||||
full_address = TextField(column_name='Full Address', null=True)
|
||||
street = TextField(column_name='Street', null=True)
|
||||
city = TextField(column_name='City', null=True)
|
||||
state_province = TextField(column_name='State/Province', null=True)
|
||||
postal_code = TextField(column_name='Postal Code', null=True)
|
||||
country = TextField(column_name='Country', null=True)
|
||||
phone = TextField(column_name='Phone', null=True)
|
||||
email = TextField(column_name='Email', null=True)
|
||||
|
||||
_csv_headers_override = {
|
||||
'_dp': 'timestamp',
|
||||
'Transaction Type': 'type'
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_csv_dict(cls, data):
|
||||
txn = data.copy()
|
||||
# can't use '%s' format string, have to use the special function
|
||||
txn['_dp'] = datetime.fromtimestamp(txn['_dp'])
|
||||
return super().from_csv_dict(txn)
|
||||
|
||||
class Meta:
|
||||
table_name = 'transactions'
|
||||
primary_key = False
|
62
sqlExport.py
Executable file
62
sqlExport.py
Executable file
@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from common import membershipworks
|
||||
from lib.mw_models import database, Label, Member, MemberLabel, Transaction
|
||||
|
||||
@database.atomic()
|
||||
def main():
|
||||
print("Creating tables")
|
||||
database.create_tables([Label, Member, MemberLabel, Transaction])
|
||||
|
||||
print("Updating labels")
|
||||
labels = membershipworks._parse_flags()['labels']
|
||||
Label \
|
||||
.insert_many([{'label_id': v, 'label': k} for k, v in labels.items()]) \
|
||||
.on_conflict(action="update", preserve=[Label.label]) \
|
||||
.execute()
|
||||
|
||||
print("Getting/Updating members...")
|
||||
members = membershipworks.get_all_members()
|
||||
for m in members:
|
||||
# replace flags by booleans
|
||||
for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
|
||||
if flag in m:
|
||||
m[flag] = m[flag] == flag
|
||||
|
||||
for field_id, field in membershipworks._all_fields().items():
|
||||
# convert checkboxes to real booleans
|
||||
if field.get('typ') == 8 and field['lbl'] in m: # check box
|
||||
m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
|
||||
|
||||
for member in members:
|
||||
# create/update member
|
||||
Member.from_csv_dict(member).magic_save()
|
||||
|
||||
# update member's labels
|
||||
for label, label_id in membershipworks._parse_flags()['labels'].items():
|
||||
ml = MemberLabel(uid=member['Account ID'], label_id=label_id)
|
||||
if member[label]:
|
||||
ml.magic_save()
|
||||
else:
|
||||
ml.delete_instance()
|
||||
|
||||
print("Getting/Updating transactions...")
|
||||
now = datetime.now()
|
||||
start_date = datetime(2010, 1, 1)
|
||||
transactions_csv = membershipworks.get_transactions(start_date, now)
|
||||
transactions_json = membershipworks.get_transactions(start_date, now, json=True)
|
||||
# this is terrible, but as long as the dates are the same, should be fiiiine
|
||||
transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
|
||||
assert all([t['Account ID'] == t.get('uid', '')
|
||||
and t['Payment ID'] == t.get('sid', '')
|
||||
for t in transactions])
|
||||
|
||||
for transaction in transactions:
|
||||
Transaction.from_csv_dict(transaction).magic_save()
|
||||
|
||||
# TODO: folders, levels, addons
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
@ -5,5 +5,6 @@ OnFailure=status-email-admin@%n.service
|
||||
[Service]
|
||||
User=adam
|
||||
Type=oneshot
|
||||
WorkingDirectory=/home/adam/hidDoorWriter/
|
||||
ExecStart=/home/adam/hidDoorWriter/doorUpdater.py
|
||||
TimeoutStartSec=600
|
||||
WorkingDirectory=/home/adam/memberPlumbing/
|
||||
ExecStart=/usr/bin/python3 -u /home/adam/memberPlumbing/doorUpdater.py
|
||||
|
12
systemd/membershipworksSQL.service
Normal file
12
systemd/membershipworksSQL.service
Normal file
@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Sync Membershipworks with local database
|
||||
OnFailure=status-email-admin@%n.service
|
||||
After=mariadb.service
|
||||
Requires=mariadb.service
|
||||
|
||||
[Service]
|
||||
User=adam
|
||||
Type=oneshot
|
||||
TimeoutStartSec=600
|
||||
WorkingDirectory=/home/adam/memberPlumbing/
|
||||
ExecStart=/usr/bin/python3 -u /home/adam/memberPlumbing/sqlExport.py
|
9
systemd/membershipworksSQL.timer
Normal file
9
systemd/membershipworksSQL.timer
Normal file
@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=Hourly Membershipworks database sync
|
||||
|
||||
[Timer]
|
||||
OnCalendar=hourly
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
Reference in New Issue
Block a user