memberPlumbing/sqlExport.py

#!/usr/bin/env python3

from datetime import datetime
import sqlite3

#from common import membershipworks

# TODO: TEMP. remove later:
#from lib.MembershipWorks import MembershipWorks
from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD
membershipworks = MembershipWorks()
membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD)

membersTableColumns = {
    'uid': {'type': 'TEXT PRIMARY KEY', 'label': 'Account ID'},
    'Year of Birth': {},
    'Account Name': {},
    'First Name': {},
    'Last Name': {},
    'Phone': {},
    'Email': {},

    'Address (Street)': {},
    'Address (City)': {},
    'Address (State/Province)': {},
    'Address (Postal Code)': {},
    'Address (Country)': {},
    'Profile description': {},
    'Website': {},
    'Fax': {},
    'Contact Person': {},
    'Password': {},
    'Position/relation': {},

    'Parent Account ID': {},

    'Gift Membership purchased by': {},
    'Purchased Gift Membership for': {},

    'Closet Storage #': {},
    'Storage Shelf #': {},
    'Personal Studio Space #': {},

    'Access Permitted Shops During Extended Hours?': {'type': 'BOOLEAN'},
    'Access Front Door and Studio Space During Extended Hours?': {'type': 'BOOLEAN'},
    'Access Wood Shop?': {'type': 'BOOLEAN'},
    'Access Metal Shop?': {'type': 'BOOLEAN'},
    'Access Storage Closet?': {'type': 'BOOLEAN'},
    'Access Studio Space?': {'type': 'BOOLEAN'},
    'Access Front Door?': {'type': 'BOOLEAN'},
    'Access Card Number': {'type': 'INTEGER'},
    'Access Card Facility Code': {'type': 'INTEGER'},

    'Auto Billing ID': {},
    'Billing Method': {},
    'Renewal Date': {},
    'Join Date': {},

    'Admin note': {},

    'Profile gallery image URL': {},
    'Business card image URL': {},
    'Instagram': {},
    'Pinterest': {},
    'Youtube': {},
    'Yelp': {},
    'Google+': {},
    'BBB': {},
    'Twitter': {},
    'Facebook': {},
    'LinkedIn': {},

    'Do not show street address in profile': {},
    'Do not list in directory': {},

    'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': {},
    'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': {},
    'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': {},
    'Waiver form signed and on file date.': {},
    'Membership Agreement signed and on file date.': {},

    'IP Address': {},
}

def insertMembers(members):
    def formatMembers(members):
        for m in members:
            yield [
                m[v.get('label', k)] for k, v in membersTableColumns.items()
            ]

    c.execute('BEGIN')
    # NOTE: this only works if in python >= 3.7 where `dict` is ordered
    c.executemany('INSERT OR REPLACE INTO members (' +
                  ','.join('"' + k + '"' for k in membersTableColumns.keys()) +
                  ') values (' +
                  ','.join(len(membersTableColumns) * '?') +
                  ');',
                  list(formatMembers(members)))
    c.execute('END')

def insertLabels(members):
    def formatLabels(labels):
        for member in members:
            for label in membershipworks._parse_flags()['labels']:
                if member[label]:
                    yield member['Account ID'], label

    c.execute('BEGIN')
    c.executemany('INSERT OR REPLACE INTO labels (uid, label) values (?, ?);',
                  list(formatLabels(members)))
    c.execute('END')

def insertTransactions(transactions):
    def formatTransactions(transactions):
        for t in transactions:
            yield [
                t["sid"],
                t["_dp"], # timestamp
                t["typ"], # type
                t["cur"], # currency
                t["sum"],
                t.get("fee"),
                t["uid"],
                t["nam"], # name
                t.get("eid"), # event_id
                t.get("ttl") # "For"
            ]

    c.execute('BEGIN')
    c.executemany('INSERT OR REPLACE INTO transactions (sid, timestamp, type, currency, sum, fee, uid, name, event_id, ttl) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);',
                  list(formatTransactions(transactions)))
    c.execute('END')


with sqlite3.connect('membershipworks.db') as conn:
    c = conn.cursor()

    c.execute('CREATE TABLE IF NOT EXISTS members (' +
              ', '.join([f'"{k}" {v.get("type", "")}'
                         for k, v in membersTableColumns.items()]) +
              ')')

    c.execute("""CREATE TABLE IF NOT EXISTS transactions (
    sid TEXT PRIMARY KEY,
    timestamp INTEGER NOT NULL,
    type INTEGER NOT NULL,
    currency TEXT NOT NULL,
    sum INTEGER NOT NULL,
    fee INTEGER,
    uid TEXT NOT NULL,
    name TEXT NOT NULL,
    event_id TEXT,
    ttl TEXT,
    FOREIGN KEY(uid) REFERENCES member(uid)
    -- FOREIGN KEY(event_id) REFERENCES event(eid)
    );
    """)

    # TODO: define a pk constraint
    c.execute("""CREATE TABLE IF NOT EXISTS labels (
    uid TEXT, label Text,
    FOREIGN KEY(uid) REFERENCES member(uid),
    PRIMARY KEY(uid, label)
    );
    """)


    members = membershipworks.get_all_members()
    for m in members:
        # replace flags by booleans
        for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
            if flag in m:
                m[flag] = m[flag] == flag

        for field_id, field in membershipworks._all_fields().items():
            # convert checkboxes to real booleans
            if field.get('typ') == 8 and field['lbl'] in m: # check box
                m[field['lbl']] = True if m[field['lbl']] == 'Y' else False

    insertMembers(members)
    insertLabels(members)

    transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
    insertTransactions(transactions)

    # TODO: folders, levels, addons