forked from CMS/memberPlumbing
Compare commits
19 Commits
master
...
sql-export
Author | SHA1 | Date | |
---|---|---|---|
f84f357056 | |||
bdf3b84c74 | |||
d5ecf50943 | |||
925e1abe80 | |||
74827ac9ad | |||
ae0cf430b5 | |||
17c9da6f9b | |||
44e560cd34 | |||
721000ce43 | |||
b417821703 | |||
65b79cad99 | |||
6ab2b7bd9c | |||
a6e596cfd4 | |||
516813b895 | |||
01e08e1007 | |||
b82da10073 | |||
8edaaf0df9 | |||
8b54dc7fe3 | |||
38ba551f1e |
@ -27,3 +27,7 @@ Retrieves member information from MembershipWorks and pushes it out to the HID E
|
||||
## `ucsAccounts.py`
|
||||
|
||||
Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space.
|
||||
|
||||
## `sqlExport.py`
|
||||
|
||||
Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined (somewhat oddly, to be fair) in `tableMapping.yaml`.
|
||||
|
146
sqlExport.py
Executable file
146
sqlExport.py
Executable file
@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from datetime import datetime
|
||||
import MySQLdb
|
||||
import yaml
|
||||
|
||||
from common import membershipworks
|
||||
from passwords import MEMBERSHIPWORKS_DB
|
||||
|
||||
def resolveSource(key, value):
|
||||
if type(value) == str:
|
||||
return value
|
||||
elif type(value) == dict and 'source' in value:
|
||||
return value['source']
|
||||
else:
|
||||
return key
|
||||
|
||||
def formatRows(tableMap, data):
|
||||
for d in data:
|
||||
yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
|
||||
|
||||
def insertFromTableMap(table, data, tableMap):
|
||||
# TODO: this could probably be done better as a single statement?
|
||||
c.executemany(
|
||||
f"""INSERT INTO {table} ({','.join(f'`{k}`' for k in tableMap.keys())})
|
||||
VALUES ({','.join(len(tableMap) * ['%s'])})
|
||||
ON DUPLICATE KEY UPDATE
|
||||
{', '.join(f'`{k}`=VALUES(`{k}`)' for k in tableMap.keys())};""",
|
||||
list(formatRows(tableMap, data)))
|
||||
|
||||
def insertDistinctFromTableMap(table, data, tableMap):
|
||||
# TODO: this could probably be done better as a single statement?
|
||||
c.executemany(
|
||||
f"""INSERT INTO {table} ({','.join(f'`{k}`' for k in tableMap.keys())})
|
||||
SELECT {','.join(len(tableMap) * ['%s'])} FROM DUAL
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 from {table}
|
||||
WHERE {' AND '.join(f'`{k}`<=>%s' for k in tableMap.keys())}
|
||||
);""" ,
|
||||
list([r * 2 for r in formatRows(tableMap, data)]))
|
||||
|
||||
|
||||
# TODO: delete non-valid labels
|
||||
def insertLabels(members):
|
||||
for member in members:
|
||||
for label, label_id in membershipworks._parse_flags()['labels'].items():
|
||||
if member[label]:
|
||||
c.execute("""
|
||||
INSERT INTO member_labels (uid, label_id) VALUES (%s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
uid=VALUES(uid), label_id=VALUES(label_id);""",
|
||||
(member['Account ID'], label_id))
|
||||
else:
|
||||
c.execute('DELETE FROM member_labels WHERE uid=%s && label_id=%s;',
|
||||
(member['Account ID'], label_id))
|
||||
|
||||
|
||||
with open('tableMapping.yaml') as f:
|
||||
tableMapping = yaml.load(f, yaml.SafeLoader)
|
||||
|
||||
|
||||
conn = MySQLdb.connect(**MEMBERSHIPWORKS_DB)
|
||||
conn.set_character_set('utf8')
|
||||
c = conn.cursor()
|
||||
|
||||
def createDefinitionsFromTableMap(tableMap):
|
||||
def resolveColType(value):
|
||||
if type(value) == dict and 'type' in value:
|
||||
return value['type']
|
||||
else:
|
||||
return 'TEXT'
|
||||
|
||||
return ', '.join([f'`{k}` ' + resolveColType(v)
|
||||
for k, v in tableMap.items()])
|
||||
|
||||
try:
|
||||
print("Creating tables...")
|
||||
c.execute('CREATE TABLE IF NOT EXISTS members (' +
|
||||
createDefinitionsFromTableMap(tableMapping['members']) +
|
||||
') WITH SYSTEM VERSIONING;')
|
||||
|
||||
c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
|
||||
createDefinitionsFromTableMap(tableMapping['transactions']) +
|
||||
", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));")
|
||||
#-- FOREIGN KEY event_id REFERENCES event eid
|
||||
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS labels (
|
||||
label_id CHAR(24) PRIMARY KEY, label TEXT
|
||||
) WITH SYSTEM VERSIONING;""")
|
||||
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS member_labels (
|
||||
uid CHAR(24), label_id CHAR(24),
|
||||
PRIMARY KEY(uid, label_id),
|
||||
FOREIGN KEY(uid) REFERENCES members(uid),
|
||||
FOREIGN KEY(label_id) REFERENCES labels(label_id)
|
||||
) WITH SYSTEM VERSIONING;""")
|
||||
|
||||
print("Updating labels")
|
||||
c.executemany("""INSERT INTO labels (label, label_id) VALUES (%s, %s)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
label=VALUES(label), label_id=VALUES(label_id);""",
|
||||
membershipworks._parse_flags()['labels'].items())
|
||||
|
||||
print("Getting/Updating members...")
|
||||
members = membershipworks.get_all_members()
|
||||
for m in members:
|
||||
# replace flags by booleans
|
||||
for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
|
||||
if flag in m:
|
||||
m[flag] = m[flag] == flag
|
||||
|
||||
for field_id, field in membershipworks._all_fields().items():
|
||||
# convert checkboxes to real booleans
|
||||
if field.get('typ') == 8 and field['lbl'] in m: # check box
|
||||
m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
|
||||
|
||||
insertFromTableMap('members', members, tableMapping['members'])
|
||||
|
||||
insertLabels(members)
|
||||
|
||||
print("Getting/Updating transactions...")
|
||||
now = datetime.now()
|
||||
transactions_csv = membershipworks.get_transactions(datetime(2010, 1, 1), now)
|
||||
transactions_json = membershipworks.get_transactions(
|
||||
datetime(2010, 1, 1), now, json=True)
|
||||
# this is terrible, but as long as the dates are the same, should be fiiiine
|
||||
transactions = [{**j, **v}
|
||||
for j, v in zip(transactions_csv, transactions_json)]
|
||||
assert all([t['Account ID'] == t.get('uid', '')
|
||||
and t['Payment ID'] == t.get('sid', '')
|
||||
for t in transactions])
|
||||
insertDistinctFromTableMap(
|
||||
'transactions', transactions, tableMapping['transactions'])
|
||||
|
||||
print("Committing changes...")
|
||||
conn.commit()
|
||||
|
||||
except Exception as e:
|
||||
print("Transaction failed, rolling back")
|
||||
conn.rollback();
|
||||
raise e
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# TODO: folders, levels, addons
|
102
tableMapping.yaml
Normal file
102
tableMapping.yaml
Normal file
@ -0,0 +1,102 @@
|
||||
members:
|
||||
'uid': {type: 'CHAR(24) PRIMARY KEY', source: 'Account ID'}
|
||||
'Year of Birth':
|
||||
'Account Name':
|
||||
'First Name':
|
||||
'Last Name':
|
||||
'Phone':
|
||||
'Email':
|
||||
|
||||
'Address (Street)':
|
||||
'Address (City)':
|
||||
'Address (State/Province)':
|
||||
'Address (Postal Code)':
|
||||
'Address (Country)':
|
||||
'Profile description':
|
||||
'Website':
|
||||
'Fax':
|
||||
'Contact Person':
|
||||
'Password':
|
||||
'Position/relation':
|
||||
|
||||
'Parent Account ID':
|
||||
|
||||
'Gift Membership purchased by':
|
||||
'Purchased Gift Membership for':
|
||||
|
||||
'Closet Storage #':
|
||||
'Storage Shelf #':
|
||||
'Personal Studio Space #':
|
||||
|
||||
'Access Permitted Shops During Extended Hours?': {type: 'BOOLEAN'}
|
||||
'Access Front Door and Studio Space During Extended Hours?': {type: 'BOOLEAN'}
|
||||
'Access Wood Shop?': {type: 'BOOLEAN'}
|
||||
'Access Metal Shop?': {type: 'BOOLEAN'}
|
||||
'Access Storage Closet?': {type: 'BOOLEAN'}
|
||||
'Access Studio Space?': {type: 'BOOLEAN'}
|
||||
'Access Front Door?': {type: 'BOOLEAN'}
|
||||
'Access Card Number':
|
||||
'Access Card Facility Code':
|
||||
|
||||
'Auto Billing ID':
|
||||
'Billing Method':
|
||||
'Renewal Date':
|
||||
'Join Date':
|
||||
|
||||
'Admin note':
|
||||
|
||||
'Profile gallery image URL':
|
||||
'Business card image URL':
|
||||
'Instagram':
|
||||
'Pinterest':
|
||||
'Youtube':
|
||||
'Yelp':
|
||||
'Google+':
|
||||
'BBB':
|
||||
'Twitter':
|
||||
'Facebook':
|
||||
'LinkedIn':
|
||||
|
||||
'Do not show street address in profile':
|
||||
'Do not list in directory':
|
||||
|
||||
'HowDidYouHear': 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:'
|
||||
'authorizeCharge': 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.'
|
||||
'policyAgreement': 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.'
|
||||
'Waiver form signed and on file date.':
|
||||
'Membership Agreement signed and on file date.':
|
||||
'Agreement Version':
|
||||
'Paperwork status':
|
||||
'Membership agreement dated': {type: BOOLEAN}
|
||||
'Membership Agreement Acknowledgement Page Filled Out': {type: BOOLEAN}
|
||||
'Membership Agreement Signed': {type: BOOLEAN}
|
||||
'Liability Form Filled Out': {type: BOOLEAN}
|
||||
'Audit Date':
|
||||
|
||||
'IP Address':
|
||||
|
||||
transactions:
|
||||
'sid': {type: CHAR(27)}
|
||||
'uid': {type: CHAR(24)}
|
||||
'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp?
|
||||
'type': {source: 'Transaction Type'}
|
||||
'sum': {type: 'DECIMAL(13,4)'}
|
||||
'fee': {type: 'DECIMAL(13,4)'}
|
||||
'event_id': {source: 'eid'}
|
||||
'For':
|
||||
'Items':
|
||||
'Discount Code':
|
||||
'Note':
|
||||
|
||||
# this is painful, but necessary because some users have no uid
|
||||
# TODO: fix this horribleness
|
||||
'Name':
|
||||
'Contact Person':
|
||||
'Full Address':
|
||||
'Street':
|
||||
'City':
|
||||
'State/Province':
|
||||
'Postal Code':
|
||||
'Country':
|
||||
'Phone':
|
||||
'Email':
|
Loading…
Reference in New Issue
Block a user