memberPlumbing/sqlExport.py

#!/usr/bin/env python3

from datetime import datetime
import MySQLdb
import yaml

from common import membershipworks
from passwords import MEMBERSHIPWORKS_DB


def insertFromTableMap(table, data, tableMap):
    def resolveSource(key, value):
        if type(value) == str:
            return value
        elif type(value) == dict and 'source' in value:
            return value['source']
        else:
            return key

    def formatRows(data):
        for d in data:
            yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]

    # TODO: this could probably be done better as a single statement?
    # note: this only works if in python >= 3.7 where `dict` is ordered
    c.executemany(
        'INSERT INTO ' + table + ' (' +
        ','.join(f'`{k}`' for k in tableMap.keys()) +
        ') VALUES (' + ','.join(len(tableMap) * ['%s']) + ') ' +
        'ON DUPLICATE KEY UPDATE ' +
        ', '.join(f'`{k}`=VALUES(`{k}`)' for k in tableMap.keys()) + ';',
        list(formatRows(data)))

# TODO: delete non-valid labels
def insertLabels(members):
    for member in members:
        for label, label_id in membershipworks._parse_flags()['labels'].items():
            if member[label]:
                c.execute("""
                INSERT INTO member_labels (uid, label_id) VALUES (%s, %s)
                ON DUPLICATE KEY UPDATE
                  uid=VALUES(uid), label_id=VALUES(label_id);""",
                          (member['Account ID'], label_id))
            else:
                c.execute('DELETE FROM member_labels WHERE uid=%s && label_id=%s;',
                          (member['Account ID'], label_id))


with open('tableMapping.yaml') as f:
    tableMapping = yaml.load(f, yaml.SafeLoader)


conn =  MySQLdb.connect(**MEMBERSHIPWORKS_DB)
c = conn.cursor()

def createDefinitionsFromTableMap(tableMap):
    def resolveColType(value):
        if type(value) == dict and 'type' in value:
            return value['type']
        else:
            return 'TEXT'

    return ', '.join([f'`{k}` ' + resolveColType(v)
                      for k, v in tableMap.items()])

try:
    c.execute('CREATE TABLE IF NOT EXISTS members (' +
              createDefinitionsFromTableMap(tableMapping['members']) +
              ') WITH SYSTEM VERSIONING;')

    c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
              createDefinitionsFromTableMap(tableMapping['transactions']) +
              ", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));")
              #-- FOREIGN KEY event_id REFERENCES event eid

    c.execute("""CREATE TABLE IF NOT EXISTS labels (
    label_id CHAR(24) PRIMARY KEY, label TEXT
    ) WITH SYSTEM VERSIONING;""")

    c.execute("""CREATE TABLE IF NOT EXISTS member_labels (
    uid CHAR(24), label_id CHAR(24),
    PRIMARY KEY(uid, label_id),
    FOREIGN KEY(uid) REFERENCES members(uid),
    FOREIGN KEY(label_id) REFERENCES labels(label_id)
    ) WITH SYSTEM VERSIONING;""")

    members = membershipworks.get_all_members()
    for m in members:
        # replace flags by booleans
        for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
            if flag in m:
                m[flag] = m[flag] == flag

        for field_id, field in membershipworks._all_fields().items():
            # convert checkboxes to real booleans
            if field.get('typ') == 8 and field['lbl'] in m: # check box
                m[field['lbl']] = True if m[field['lbl']] == 'Y' else False

    c.executemany("""INSERT INTO labels (label, label_id) VALUES (%s, %s)
    ON DUPLICATE KEY UPDATE
    label=VALUES(label), label_id=VALUES(label_id);""",
                  membershipworks._parse_flags()['labels'].items())

    insertFromTableMap('members', members, tableMapping['members'])

    insertLabels(members)

    transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
    insertFromTableMap('transactions', transactions, tableMapping['transactions'])

    conn.commit()

except Exception as e:
    print("Transaction failed, rolling back")
    conn.rollback();
    raise e

finally:
    conn.close()

# TODO: folders, levels, addons