sqlExport: Add script to export MembershipWorks data to a MariaDB server
This commit is contained in:
parent
cf84086446
commit
525fd24a22
@ -27,3 +27,7 @@ Retrieves member information from MembershipWorks and pushes it out to the HID E
|
|||||||
## `ucsAccounts.py`
|
## `ucsAccounts.py`
|
||||||
|
|
||||||
Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space.
|
Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space.
|
||||||
|
|
||||||
|
## `sqlExport.py`
|
||||||
|
|
||||||
|
Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined with [peewee](peewee-orm.com) in `./lib/mw_models.py`.
|
||||||
|
203
lib/mw_models.py
Normal file
203
lib/mw_models.py
Normal file
@ -0,0 +1,203 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from peewee import (BooleanField, FixedCharField, CompositeKey, DateField,
|
||||||
|
DateTimeField, DecimalField, ForeignKeyField, Model,
|
||||||
|
MySQLDatabase, TextField)
|
||||||
|
|
||||||
|
import passwords
|
||||||
|
|
||||||
|
database = MySQLDatabase(
|
||||||
|
**passwords.MEMBERSHIPWORKS_DB,
|
||||||
|
**{
|
||||||
|
"charset": "utf8",
|
||||||
|
"sql_mode": "PIPES_AS_CONCAT",
|
||||||
|
"use_unicode": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
class BaseModel(Model):
|
||||||
|
_csv_headers_override = {}
|
||||||
|
_date_fields = {}
|
||||||
|
|
||||||
|
def insert_instance(self):
|
||||||
|
return self.insert(**self.__data__)
|
||||||
|
|
||||||
|
def upsert_instance(self):
|
||||||
|
return self.insert_instance() \
|
||||||
|
.on_conflict(action="update",
|
||||||
|
preserve=list(self._meta.fields.values()))
|
||||||
|
|
||||||
|
def magic_save(self):
|
||||||
|
if self._meta.primary_key is False:
|
||||||
|
self.get_or_create(**self.__data__)
|
||||||
|
else:
|
||||||
|
self.upsert_instance().execute()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _headers_map(cls):
|
||||||
|
return {field.column_name: name
|
||||||
|
for name, field in cls._meta.fields.items()}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _remap_headers(cls, data):
|
||||||
|
# print(data)
|
||||||
|
hmap = cls._headers_map()
|
||||||
|
hmap.update(cls._csv_headers_override)
|
||||||
|
for k, v in data.items():
|
||||||
|
if k in hmap:
|
||||||
|
yield hmap.get(k), v
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_csv_dict(cls, data):
|
||||||
|
data = data.copy()
|
||||||
|
|
||||||
|
# parse date fields to datetime objects
|
||||||
|
for field, fmt in cls._date_fields.items():
|
||||||
|
if data[field]:
|
||||||
|
data[field] = datetime.strptime(str(data[field]), fmt)
|
||||||
|
else:
|
||||||
|
# convert empty string to None to make NULL in SQL
|
||||||
|
data[field] = None
|
||||||
|
|
||||||
|
return cls(**dict(cls._remap_headers(data)))
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
database = database
|
||||||
|
|
||||||
|
class Label(BaseModel):
|
||||||
|
label_id = FixedCharField(24, primary_key=True)
|
||||||
|
label = TextField(null=True)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
table_name = 'labels'
|
||||||
|
|
||||||
|
class Member(BaseModel):
|
||||||
|
uid = FixedCharField(24, primary_key=True)
|
||||||
|
year_of_birth = TextField(column_name='Year of Birth', null=True)
|
||||||
|
account_name = TextField(column_name='Account Name', null=True)
|
||||||
|
first_name = TextField(column_name='First Name', null=True)
|
||||||
|
last_name = TextField(column_name='Last Name', null=True)
|
||||||
|
phone = TextField(column_name='Phone', null=True)
|
||||||
|
email = TextField(column_name='Email', null=True)
|
||||||
|
address_street = TextField(column_name='Address (Street)', null=True)
|
||||||
|
address_city = TextField(column_name='Address (City)', null=True)
|
||||||
|
address_state_province = TextField(column_name='Address (State/Province)', null=True)
|
||||||
|
address_postal_code = TextField(column_name='Address (Postal Code)', null=True)
|
||||||
|
address_country = TextField(column_name='Address (Country)', null=True)
|
||||||
|
profile_description = TextField(column_name='Profile description', null=True)
|
||||||
|
website = TextField(column_name='Website', null=True)
|
||||||
|
fax = TextField(column_name='Fax', null=True)
|
||||||
|
contact_person = TextField(column_name='Contact Person', null=True)
|
||||||
|
password = TextField(column_name='Password', null=True)
|
||||||
|
position_relation = TextField(column_name='Position/relation', null=True)
|
||||||
|
parent_account_id = TextField(column_name='Parent Account ID', null=True)
|
||||||
|
gift_membership_purchased_by = TextField(column_name='Gift Membership purchased by', null=True)
|
||||||
|
purchased_gift_membership_for = TextField(column_name='Purchased Gift Membership for', null=True)
|
||||||
|
closet_storage = TextField(column_name='Closet Storage #', null=True)
|
||||||
|
storage_shelf = TextField(column_name='Storage Shelf #', null=True)
|
||||||
|
personal_studio_space = TextField(column_name='Personal Studio Space #', null=True)
|
||||||
|
access_permitted_shops_during_extended_hours = BooleanField(column_name='Access Permitted Shops During Extended Hours?', null=True)
|
||||||
|
access_front_door_and_studio_space_during_extended_hours = BooleanField(column_name='Access Front Door and Studio Space During Extended Hours?', null=True)
|
||||||
|
access_wood_shop = BooleanField(column_name='Access Wood Shop?', null=True)
|
||||||
|
access_metal_shop = BooleanField(column_name='Access Metal Shop?', null=True)
|
||||||
|
access_storage_closet = BooleanField(column_name='Access Storage Closet?', null=True)
|
||||||
|
access_studio_space = BooleanField(column_name='Access Studio Space?', null=True)
|
||||||
|
access_front_door = BooleanField(column_name='Access Front Door?', null=True)
|
||||||
|
access_card_number = TextField(column_name='Access Card Number', null=True)
|
||||||
|
access_card_facility_code = TextField(column_name='Access Card Facility Code', null=True)
|
||||||
|
auto_billing_id = TextField(column_name='Auto Billing ID', null=True)
|
||||||
|
billing_method = TextField(column_name='Billing Method', null=True)
|
||||||
|
renewal_date = DateField(column_name='Renewal Date', null=True)
|
||||||
|
join_date = DateField(column_name='Join Date', null=True)
|
||||||
|
admin_note = TextField(column_name='Admin note', null=True)
|
||||||
|
profile_gallery_image_url = TextField(column_name='Profile gallery image URL', null=True)
|
||||||
|
business_card_image_url = TextField(column_name='Business card image URL', null=True)
|
||||||
|
instagram = TextField(column_name='Instagram', null=True)
|
||||||
|
pinterest = TextField(column_name='Pinterest', null=True)
|
||||||
|
youtube = TextField(column_name='Youtube', null=True)
|
||||||
|
yelp = TextField(column_name='Yelp', null=True)
|
||||||
|
google = TextField(column_name='Google+', null=True)
|
||||||
|
bbb = TextField(column_name='BBB', null=True)
|
||||||
|
twitter = TextField(column_name='Twitter', null=True)
|
||||||
|
facebook = TextField(column_name='Facebook', null=True)
|
||||||
|
linked_in = TextField(column_name='LinkedIn', null=True)
|
||||||
|
do_not_show_street_address_in_profile = TextField(column_name='Do not show street address in profile', null=True)
|
||||||
|
do_not_list_in_directory = TextField(column_name='Do not list in directory', null=True)
|
||||||
|
how_did_you_hear = TextField(column_name='HowDidYouHear', null=True)
|
||||||
|
authorize_charge = TextField(column_name='authorizeCharge', null=True)
|
||||||
|
policy_agreement = TextField(column_name='policyAgreement', null=True)
|
||||||
|
waiver_form_signed_and_on_file_date = DateField(column_name='Waiver form signed and on file date.', null=True)
|
||||||
|
membership_agreement_signed_and_on_file_date = DateField(column_name='Membership Agreement signed and on file date.', null=True)
|
||||||
|
ip_address = TextField(column_name='IP Address', null=True)
|
||||||
|
audit_date = DateField(column_name='Audit Date', null=True)
|
||||||
|
agreement_version = TextField(column_name='Agreement Version', null=True)
|
||||||
|
paperwork_status = TextField(column_name='Paperwork status', null=True)
|
||||||
|
membership_agreement_dated = BooleanField(column_name='Membership agreement dated', null=True)
|
||||||
|
membership_agreement_acknowledgement_page_filled_out = BooleanField(column_name='Membership Agreement Acknowledgement Page Filled Out', null=True)
|
||||||
|
membership_agreement_signed = BooleanField(column_name='Membership Agreement Signed', null=True)
|
||||||
|
liability_form_filled_out = BooleanField(column_name='Liability Form Filled Out', null=True)
|
||||||
|
|
||||||
|
_csv_headers_override = {
|
||||||
|
'Account ID': 'uid',
|
||||||
|
'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': 'how_did_you_hear',
|
||||||
|
'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': 'authorize_charge',
|
||||||
|
'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': 'policy_agreement'
|
||||||
|
}
|
||||||
|
|
||||||
|
_date_fields = {
|
||||||
|
'Join Date': '%b %d, %Y',
|
||||||
|
'Renewal Date': '%b %d, %Y',
|
||||||
|
'Audit Date': '%m/%d/%Y',
|
||||||
|
'Membership Agreement signed and on file date.': '%m/%d/%Y',
|
||||||
|
'Waiver form signed and on file date.': '%m/%d/%Y'
|
||||||
|
}
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
table_name = 'members'
|
||||||
|
|
||||||
|
class MemberLabel(BaseModel):
|
||||||
|
uid = ForeignKeyField(Member, column_name='uid', backref='labels')
|
||||||
|
label_id = ForeignKeyField(Label, backref='members')
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
table_name = 'member_labels'
|
||||||
|
primary_key = CompositeKey('label_id', 'uid')
|
||||||
|
|
||||||
|
class Transaction(BaseModel):
|
||||||
|
sid = FixedCharField(27, null=True)
|
||||||
|
uid = ForeignKeyField(Member, column_name='uid', backref='transactions', null=True)
|
||||||
|
timestamp = DateTimeField()
|
||||||
|
type = TextField(null=True)
|
||||||
|
sum = DecimalField(13, 4, null=True)
|
||||||
|
fee = DecimalField(13, 4, null=True)
|
||||||
|
event_id = TextField(null=True)
|
||||||
|
for_ = TextField(column_name='For', null=True)
|
||||||
|
items = TextField(column_name='Items', null=True)
|
||||||
|
discount_code = TextField(column_name='Discount Code', null=True)
|
||||||
|
note = TextField(column_name='Note', null=True)
|
||||||
|
name = TextField(column_name='Name', null=True)
|
||||||
|
contact_person = TextField(column_name='Contact Person', null=True)
|
||||||
|
full_address = TextField(column_name='Full Address', null=True)
|
||||||
|
street = TextField(column_name='Street', null=True)
|
||||||
|
city = TextField(column_name='City', null=True)
|
||||||
|
state_province = TextField(column_name='State/Province', null=True)
|
||||||
|
postal_code = TextField(column_name='Postal Code', null=True)
|
||||||
|
country = TextField(column_name='Country', null=True)
|
||||||
|
phone = TextField(column_name='Phone', null=True)
|
||||||
|
email = TextField(column_name='Email', null=True)
|
||||||
|
|
||||||
|
_csv_headers_override = {
|
||||||
|
'_dp': 'timestamp',
|
||||||
|
'Transaction Type': 'type'
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_csv_dict(cls, data):
|
||||||
|
txn = data.copy()
|
||||||
|
# can't use '%s' format string, have to use the special function
|
||||||
|
txn['_dp'] = datetime.fromtimestamp(txn['_dp'])
|
||||||
|
return super().from_csv_dict(txn)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
table_name = 'transactions'
|
||||||
|
primary_key = False
|
62
sqlExport.py
Executable file
62
sqlExport.py
Executable file
@ -0,0 +1,62 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from common import membershipworks
|
||||||
|
from lib.mw_models import database, Label, Member, MemberLabel, Transaction
|
||||||
|
|
||||||
|
@database.atomic()
|
||||||
|
def main():
|
||||||
|
print("Creating tables")
|
||||||
|
database.create_tables([Label, Member, MemberLabel, Transaction])
|
||||||
|
|
||||||
|
print("Updating labels")
|
||||||
|
labels = membershipworks._parse_flags()['labels']
|
||||||
|
Label \
|
||||||
|
.insert_many([{'label_id': v, 'label': k} for k, v in labels.items()]) \
|
||||||
|
.on_conflict(action="update", preserve=[Label.label]) \
|
||||||
|
.execute()
|
||||||
|
|
||||||
|
print("Getting/Updating members...")
|
||||||
|
members = membershipworks.get_all_members()
|
||||||
|
for m in members:
|
||||||
|
# replace flags by booleans
|
||||||
|
for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
|
||||||
|
if flag in m:
|
||||||
|
m[flag] = m[flag] == flag
|
||||||
|
|
||||||
|
for field_id, field in membershipworks._all_fields().items():
|
||||||
|
# convert checkboxes to real booleans
|
||||||
|
if field.get('typ') == 8 and field['lbl'] in m: # check box
|
||||||
|
m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
|
||||||
|
|
||||||
|
for member in members:
|
||||||
|
# create/update member
|
||||||
|
Member.from_csv_dict(member).magic_save()
|
||||||
|
|
||||||
|
# update member's labels
|
||||||
|
for label, label_id in membershipworks._parse_flags()['labels'].items():
|
||||||
|
ml = MemberLabel(uid=member['Account ID'], label_id=label_id)
|
||||||
|
if member[label]:
|
||||||
|
ml.magic_save()
|
||||||
|
else:
|
||||||
|
ml.delete_instance()
|
||||||
|
|
||||||
|
print("Getting/Updating transactions...")
|
||||||
|
now = datetime.now()
|
||||||
|
start_date = datetime(2010, 1, 1)
|
||||||
|
transactions_csv = membershipworks.get_transactions(start_date, now)
|
||||||
|
transactions_json = membershipworks.get_transactions(start_date, now, json=True)
|
||||||
|
# this is terrible, but as long as the dates are the same, should be fiiiine
|
||||||
|
transactions = [{**j, **v} for j, v in zip(transactions_csv, transactions_json)]
|
||||||
|
assert all([t['Account ID'] == t.get('uid', '')
|
||||||
|
and t['Payment ID'] == t.get('sid', '')
|
||||||
|
for t in transactions])
|
||||||
|
|
||||||
|
for transaction in transactions:
|
||||||
|
Transaction.from_csv_dict(transaction).magic_save()
|
||||||
|
|
||||||
|
# TODO: folders, levels, addons
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
@ -5,5 +5,6 @@ OnFailure=status-email-admin@%n.service
|
|||||||
[Service]
|
[Service]
|
||||||
User=adam
|
User=adam
|
||||||
Type=oneshot
|
Type=oneshot
|
||||||
WorkingDirectory=/home/adam/hidDoorWriter/
|
TimeoutStartSec=600
|
||||||
ExecStart=/home/adam/hidDoorWriter/doorUpdater.py
|
WorkingDirectory=/home/adam/memberPlumbing/
|
||||||
|
ExecStart=/usr/bin/python3 -u /home/adam/memberPlumbing/doorUpdater.py
|
||||||
|
12
systemd/membershipworksSQL.service
Normal file
12
systemd/membershipworksSQL.service
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Sync Membershipworks with local database
|
||||||
|
OnFailure=status-email-admin@%n.service
|
||||||
|
After=mariadb.service
|
||||||
|
Requires=mariadb.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
User=adam
|
||||||
|
Type=oneshot
|
||||||
|
TimeoutStartSec=600
|
||||||
|
WorkingDirectory=/home/adam/memberPlumbing/
|
||||||
|
ExecStart=/usr/bin/python3 -u /home/adam/memberPlumbing/sqlExport.py
|
9
systemd/membershipworksSQL.timer
Normal file
9
systemd/membershipworksSQL.timer
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Hourly Membershipworks database sync
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=hourly
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
Reference in New Issue
Block a user