#!/usr/bin/env python3 from datetime import datetime import MySQLdb import yaml from common import membershipworks from passwords import MEMBERSHIPWORKS_DB def insertFromTableMap(table, data, tableMap): def resolveSource(key, value): if type(value) == str: return value elif type(value) == dict and 'source' in value: return value['source'] else: return key def formatRows(data): for d in data: yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()] # TODO: this could probably be done better as a single statement? # note: this only works if in python >= 3.7 where `dict` is ordered c.executemany( 'INSERT INTO ' + table + ' (' + ','.join(f'`{k}`' for k in tableMap.keys()) + ') VALUES (' + ','.join(len(tableMap) * ['%s']) + ') ' + 'ON DUPLICATE KEY UPDATE ' + ', '.join(f'`{k}`=VALUES(`{k}`)' for k in tableMap.keys()) + ';', list(formatRows(data))) # TODO: delete non-valid labels def insertLabels(members): for member in members: for label, label_id in membershipworks._parse_flags()['labels'].items(): if member[label]: c.execute(""" INSERT INTO member_labels (uid, label_id) VALUES (%s, %s) ON DUPLICATE KEY UPDATE uid=VALUES(uid), label_id=VALUES(label_id);""", (member['Account ID'], label_id)) else: c.execute('DELETE FROM member_labels WHERE uid=%s && label_id=%s;', (member['Account ID'], label_id)) with open('tableMapping.yaml') as f: tableMapping = yaml.load(f, yaml.SafeLoader) conn = MySQLdb.connect(**MEMBERSHIPWORKS_DB) conn.set_character_set('utf8') c = conn.cursor() def createDefinitionsFromTableMap(tableMap): def resolveColType(value): if type(value) == dict and 'type' in value: return value['type'] else: return 'TEXT' return ', '.join([f'`{k}` ' + resolveColType(v) for k, v in tableMap.items()]) try: print("Creating tables...") c.execute('CREATE TABLE IF NOT EXISTS members (' + createDefinitionsFromTableMap(tableMapping['members']) + ') WITH SYSTEM VERSIONING;') c.execute("CREATE TABLE IF NOT EXISTS transactions (" + createDefinitionsFromTableMap(tableMapping['transactions']) + ", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));") #-- FOREIGN KEY event_id REFERENCES event eid c.execute("""CREATE TABLE IF NOT EXISTS labels ( label_id CHAR(24) PRIMARY KEY, label TEXT ) WITH SYSTEM VERSIONING;""") c.execute("""CREATE TABLE IF NOT EXISTS member_labels ( uid CHAR(24), label_id CHAR(24), PRIMARY KEY(uid, label_id), FOREIGN KEY(uid) REFERENCES members(uid), FOREIGN KEY(label_id) REFERENCES labels(label_id) ) WITH SYSTEM VERSIONING;""") print("Updating labels") c.executemany("""INSERT INTO labels (label, label_id) VALUES (%s, %s) ON DUPLICATE KEY UPDATE label=VALUES(label), label_id=VALUES(label_id);""", membershipworks._parse_flags()['labels'].items()) print("Getting/Updating members...") members = membershipworks.get_all_members() for m in members: # replace flags by booleans for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]: if flag in m: m[flag] = m[flag] == flag for field_id, field in membershipworks._all_fields().items(): # convert checkboxes to real booleans if field.get('typ') == 8 and field['lbl'] in m: # check box m[field['lbl']] = True if m[field['lbl']] == 'Y' else False insertFromTableMap('members', members, tableMapping['members']) insertLabels(members) print("Getting/Updating transactions...") transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now()) insertFromTableMap('transactions', transactions, tableMapping['transactions']) print("Committing changes...") conn.commit() except Exception as e: print("Transaction failed, rolling back") conn.rollback(); raise e finally: conn.close() # TODO: folders, levels, addons