memberPlumbing/sqlExport.py

#!/usr/bin/env python3

from datetime import datetime
import MySQLdb
import yaml

from common import membershipworks


def insertFromTableMap(table, data, tableMap):
    def resolveSource(key, value):
        if type(value) == str:
            return value
        elif type(value) == dict and 'source' in value:
            return value['source']
        else:
            return key

    def formatRows(data):
        for d in data:
            yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]

    # TODO: this could probably be done better as a single statement?
    # note: this only works if in python >= 3.7 where `dict` is ordered
    c.executemany('REPLACE INTO ' + table + ' (' +
                  ','.join(f'`{k}`' for k in tableMap.keys()) +
                  ') VALUES (' +
                  ','.join(len(tableMap) * ['%s']) +
                  ');',
                  list(formatRows(data)))

# TODO: delete non-valid labels
def insertLabels(members):
    def formatLabels(labels):
        for member in members:
            for label, label_id in membershipworks._parse_flags()['labels'].items():
                if member[label]:
                    yield member['Account ID'], label_id

    c.executemany('REPLACE INTO member_labels (uid, label_id) VALUES (%s, %s);',
                  list(formatLabels(members)))

with open('tableMapping.yaml') as f:
    tableMapping = yaml.load(f, yaml.SafeLoader)


conn =  MySQLdb.connect(user='membershipworks', password='asdf', database='membershipworks')
c = conn.cursor()

def createDefinitionsFromTableMap(tableMap):
    def resolveColType(value):
        if type(value) == dict and 'type' in value:
            return value['type']
        else:
            return 'TEXT'

    return ', '.join([f'`{k}` ' + resolveColType(v)
                      for k, v in tableMap.items()])

try:
    c.execute('CREATE TABLE IF NOT EXISTS members (' +
              createDefinitionsFromTableMap(tableMapping['members']) +
              ') WITH SYSTEM VERSIONING;')

    c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
              createDefinitionsFromTableMap(tableMapping['transactions']) +
              ", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));")
              #-- FOREIGN KEY event_id REFERENCES event eid

    c.execute("""CREATE TABLE IF NOT EXISTS labels (
    label_id CHAR(24) PRIMARY KEY, label TEXT
    ) WITH SYSTEM VERSIONING;""")

    c.execute("""CREATE TABLE IF NOT EXISTS member_labels (
    uid CHAR(24), label_id CHAR(24),
    PRIMARY KEY(uid, label_id),
    FOREIGN KEY(uid) REFERENCES members(uid),
    FOREIGN KEY(label_id) REFERENCES labels(label_id)
    ) WITH SYSTEM VERSIONING;""")

    members = membershipworks.get_all_members()
    for m in members:
        # replace flags by booleans
        for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
            if flag in m:
                m[flag] = m[flag] == flag

        for field_id, field in membershipworks._all_fields().items():
            # convert checkboxes to real booleans
            if field.get('typ') == 8 and field['lbl'] in m: # check box
                m[field['lbl']] = True if m[field['lbl']] == 'Y' else False

    c.executemany("REPLACE INTO labels (label, label_id) VALUES (%s, %s)",
                  membershipworks._parse_flags()['labels'].items())

    insertFromTableMap('members', members, tableMapping['members'])

    insertLabels(members)

    transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
    insertFromTableMap('transactions', transactions, tableMapping['transactions'])

    conn.commit()

except Exception as e:
    print("Transaction failed, rolling back")
    conn.rollback();
    raise e

finally:
    conn.close()

# TODO: folders, levels, addons