memberPlumbing/sqlExport.py

#!/usr/bin/env python3

from datetime import datetime
import sqlite3
import yaml

#from common import membershipworks

# TODO: TEMP. remove later:
#from lib.MembershipWorks import MembershipWorks
from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD
membershipworks = MembershipWorks()
membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD)

def insertFromTableMap(table, data, tableMap):
    def formatRows(data):
        for d in data:
            yield [
                d.get(v.get('source', k) if v is not None else k) for k, v in tableMap.items()
            ]

    c.execute('BEGIN')
    # NOTE: this only works if in python >= 3.7 where `dict` is ordered
    c.executemany('INSERT OR REPLACE INTO ' + table + ' (' +
                  ','.join('"' + k + '"' for k in tableMap.keys()) +
                  ') values (' +
                  ','.join(len(tableMap) * '?') +
                  ');',
                  list(formatRows(data)))
    c.execute('END')

def insertLabels(members):
    def formatLabels(labels):
        for member in members:
            for label in membershipworks._parse_flags()['labels']:
                if member[label]:
                    yield member['Account ID'], label

    c.execute('BEGIN')
    c.executemany('INSERT OR REPLACE INTO labels (uid, label) values (?, ?);',
                  list(formatLabels(members)))
    c.execute('END')

with open('tableMapping.yaml') as f:
    tableMapping = yaml.load(f, yaml.SafeLoader)

with sqlite3.connect('membershipworks.db') as conn:
    c = conn.cursor()

    c.execute('CREATE TABLE IF NOT EXISTS members (' +
              ', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
                         for k, v in tableMapping['members'].items()]) +
              ');')

    c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
              ', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
                         for k, v in tableMapping['transactions'].items()]) +
              """, FOREIGN KEY(uid) REFERENCES member(uid)
              -- FOREIGN KEY(event_id) REFERENCES event(eid)
              );
              """)

    c.execute("""CREATE TABLE IF NOT EXISTS labels (
    uid TEXT, label Text,
    PRIMARY KEY(uid, label),
    FOREIGN KEY(uid) REFERENCES member(uid)
    );""")


    members = membershipworks.get_all_members()
    for m in members:
        # replace flags by booleans
        for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
            if flag in m:
                m[flag] = m[flag] == flag

        for field_id, field in membershipworks._all_fields().items():
            # convert checkboxes to real booleans
            if field.get('typ') == 8 and field['lbl'] in m: # check box
                m[field['lbl']] = True if m[field['lbl']] == 'Y' else False

    insertFromTableMap('members', members, tableMapping['members'])
    insertLabels(members)

    transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
    insertFromTableMap('transactions', transactions, tableMapping['transactions'])

    # TODO: folders, levels, addons