From 38ba551f1ef906878688582e773c3fcf1cee4ae8 Mon Sep 17 00:00:00 2001 From: Adam Goldsmith Date: Fri, 21 Feb 2020 17:25:53 -0500 Subject: [PATCH] sqlExport: WIP: sort of working export to SQL --- sqlExport.py | 187 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 sqlExport.py diff --git a/sqlExport.py b/sqlExport.py new file mode 100644 index 0000000..32d233f --- /dev/null +++ b/sqlExport.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 + +from datetime import datetime +import sqlite3 + +#from common import membershipworks + +# TODO: TEMP. remove later: +#from lib.MembershipWorks import MembershipWorks +from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD +membershipworks = MembershipWorks() +membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD) + +membersTableColumns = { + 'uid': {'type': 'TEXT PRIMARY KEY', 'label': 'Account ID'}, + 'Year of Birth': {}, + 'Account Name': {}, + 'First Name': {}, + 'Last Name': {}, + 'Phone': {}, + 'Email': {}, + + 'Address (Street)': {}, + 'Address (City)': {}, + 'Address (State/Province)': {}, + 'Address (Postal Code)': {}, + 'Address (Country)': {}, + 'Profile description': {}, + 'Website': {}, + 'Fax': {}, + 'Contact Person': {}, + 'Password': {}, + 'Position/relation': {}, + + 'Parent Account ID': {}, + + 'Gift Membership purchased by': {}, + 'Purchased Gift Membership for': {}, + + 'Closet Storage #': {}, + 'Storage Shelf #': {}, + 'Personal Studio Space #': {}, + + 'Access Permitted Shops During Extended Hours?': {'type': 'BOOLEAN'}, + 'Access Front Door and Studio Space During Extended Hours?': {'type': 'BOOLEAN'}, + 'Access Wood Shop?': {'type': 'BOOLEAN'}, + 'Access Metal Shop?': {'type': 'BOOLEAN'}, + 'Access Storage Closet?': {'type': 'BOOLEAN'}, + 'Access Studio Space?': {'type': 'BOOLEAN'}, + 'Access Front Door?': {'type': 'BOOLEAN'}, + 'Access Card Number': {'type': 'INTEGER'}, + 'Access Card Facility Code': {'type': 'INTEGER'}, + + 'Auto Billing ID': {}, + 'Billing Method': {}, + 'Renewal Date': {}, + 'Join Date': {}, + + 'Admin note': {}, + + 'Profile gallery image URL': {}, + 'Business card image URL': {}, + 'Instagram': {}, + 'Pinterest': {}, + 'Youtube': {}, + 'Yelp': {}, + 'Google+': {}, + 'BBB': {}, + 'Twitter': {}, + 'Facebook': {}, + 'LinkedIn': {}, + + 'Do not show street address in profile': {}, + 'Do not list in directory': {}, + + 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': {}, + 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': {}, + 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': {}, + 'Waiver form signed and on file date.': {}, + 'Membership Agreement signed and on file date.': {}, + + 'IP Address': {}, +} + +def insertMembers(members): + def formatMembers(members): + for m in members: + yield [ + m[v.get('label', k)] for k, v in membersTableColumns.items() + ] + + c.execute('BEGIN') + # NOTE: this only works if in python >= 3.7 where `dict` is ordered + c.executemany('INSERT OR REPLACE INTO members (' + + ','.join('"' + k + '"' for k in membersTableColumns.keys()) + + ') values (' + + ','.join(len(membersTableColumns) * '?') + + ');', + list(formatMembers(members))) + c.execute('END') + +def insertLabels(members): + def formatLabels(labels): + for member in members: + for label in membershipworks._parse_flags()['labels']: + if member[label]: + yield member['Account ID'], label + + c.execute('BEGIN') + c.executemany('INSERT OR REPLACE INTO labels (uid, label) values (?, ?);', + list(formatLabels(members))) + c.execute('END') + +def insertTransactions(transactions): + def formatTransactions(transactions): + for t in transactions: + yield [ + t["sid"], + t["_dp"], # timestamp + t["typ"], # type + t["cur"], # currency + t["sum"], + t.get("fee"), + t["uid"], + t["nam"], # name + t.get("eid"), # event_id + t.get("ttl") # "For" + ] + + c.execute('BEGIN') + c.executemany('INSERT OR REPLACE INTO transactions (sid, timestamp, type, currency, sum, fee, uid, name, event_id, ttl) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);', + list(formatTransactions(transactions))) + c.execute('END') + + +with sqlite3.connect('membershipworks.db') as conn: + c = conn.cursor() + + c.execute('CREATE TABLE IF NOT EXISTS members (' + + ', '.join([f'"{k}" {v.get("type", "")}' + for k, v in membersTableColumns.items()]) + + ')') + + c.execute("""CREATE TABLE IF NOT EXISTS transactions ( + sid TEXT PRIMARY KEY, + timestamp INTEGER NOT NULL, + type INTEGER NOT NULL, + currency TEXT NOT NULL, + sum INTEGER NOT NULL, + fee INTEGER, + uid TEXT NOT NULL, + name TEXT NOT NULL, + event_id TEXT, + ttl TEXT, + FOREIGN KEY(uid) REFERENCES member(uid) + -- FOREIGN KEY(event_id) REFERENCES event(eid) + ); + """) + + # TODO: define a pk constraint + c.execute("""CREATE TABLE IF NOT EXISTS labels ( + uid TEXT, label Text, + FOREIGN KEY(uid) REFERENCES member(uid), + PRIMARY KEY(uid, label) + ); + """) + + + members = membershipworks.get_all_members() + for m in members: + # replace flags by booleans + for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]: + if flag in m: + m[flag] = m[flag] == flag + + for field_id, field in membershipworks._all_fields().items(): + # convert checkboxes to real booleans + if field.get('typ') == 8 and field['lbl'] in m: # check box + m[field['lbl']] = True if m[field['lbl']] == 'Y' else False + + insertMembers(members) + insertLabels(members) + + transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now()) + insertTransactions(transactions) + + # TODO: folders, levels, addons