forked from CMS/memberPlumbing
114 lines
3.7 KiB
Python
114 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
|
|
from datetime import datetime
|
|
import MySQLdb
|
|
import yaml
|
|
|
|
from common import membershipworks
|
|
|
|
|
|
def insertFromTableMap(table, data, tableMap):
|
|
def resolveSource(key, value):
|
|
if type(value) == str:
|
|
return value
|
|
elif type(value) == dict and 'source' in value:
|
|
return value['source']
|
|
else:
|
|
return key
|
|
|
|
def formatRows(data):
|
|
for d in data:
|
|
yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
|
|
|
|
# TODO: this could probably be done better as a single statement?
|
|
# note: this only works if in python >= 3.7 where `dict` is ordered
|
|
c.executemany('REPLACE INTO ' + table + ' (' +
|
|
','.join(f'`{k}`' for k in tableMap.keys()) +
|
|
') VALUES (' +
|
|
','.join(len(tableMap) * ['%s']) +
|
|
');',
|
|
list(formatRows(data)))
|
|
|
|
# TODO: delete non-valid labels
|
|
def insertLabels(members):
|
|
def formatLabels(labels):
|
|
for member in members:
|
|
for label, label_id in membershipworks._parse_flags()['labels'].items():
|
|
if member[label]:
|
|
yield member['Account ID'], label_id
|
|
|
|
c.executemany('REPLACE INTO member_labels (uid, label_id) VALUES (%s, %s);',
|
|
list(formatLabels(members)))
|
|
|
|
with open('tableMapping.yaml') as f:
|
|
tableMapping = yaml.load(f, yaml.SafeLoader)
|
|
|
|
|
|
conn = MySQLdb.connect(user='membershipworks', password='asdf', database='membershipworks')
|
|
c = conn.cursor()
|
|
|
|
def createDefinitionsFromTableMap(tableMap):
|
|
def resolveColType(value):
|
|
if type(value) == dict and 'type' in value:
|
|
return value['type']
|
|
else:
|
|
return 'TEXT'
|
|
|
|
return ', '.join([f'`{k}` ' + resolveColType(v)
|
|
for k, v in tableMap.items()])
|
|
|
|
try:
|
|
c.execute('CREATE TABLE IF NOT EXISTS members (' +
|
|
createDefinitionsFromTableMap(tableMapping['members']) +
|
|
') WITH SYSTEM VERSIONING;')
|
|
|
|
c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
|
|
createDefinitionsFromTableMap(tableMapping['transactions']) +
|
|
", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));")
|
|
#-- FOREIGN KEY event_id REFERENCES event eid
|
|
|
|
c.execute("""CREATE TABLE IF NOT EXISTS labels (
|
|
label_id CHAR(24) PRIMARY KEY, label TEXT
|
|
) WITH SYSTEM VERSIONING;""")
|
|
|
|
c.execute("""CREATE TABLE IF NOT EXISTS member_labels (
|
|
uid CHAR(24), label_id CHAR(24),
|
|
PRIMARY KEY(uid, label_id),
|
|
FOREIGN KEY(uid) REFERENCES members(uid),
|
|
FOREIGN KEY(label_id) REFERENCES labels(label_id)
|
|
) WITH SYSTEM VERSIONING;""")
|
|
|
|
members = membershipworks.get_all_members()
|
|
for m in members:
|
|
# replace flags by booleans
|
|
for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
|
|
if flag in m:
|
|
m[flag] = m[flag] == flag
|
|
|
|
for field_id, field in membershipworks._all_fields().items():
|
|
# convert checkboxes to real booleans
|
|
if field.get('typ') == 8 and field['lbl'] in m: # check box
|
|
m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
|
|
|
|
c.executemany("REPLACE INTO labels (label, label_id) VALUES (%s, %s)",
|
|
membershipworks._parse_flags()['labels'].items())
|
|
|
|
insertFromTableMap('members', members, tableMapping['members'])
|
|
|
|
insertLabels(members)
|
|
|
|
transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
|
|
insertFromTableMap('transactions', transactions, tableMapping['transactions'])
|
|
|
|
conn.commit()
|
|
|
|
except Exception as e:
|
|
print("Transaction failed, rolling back")
|
|
conn.rollback();
|
|
raise e
|
|
|
|
finally:
|
|
conn.close()
|
|
|
|
# TODO: folders, levels, addons
|