From 8b54dc7fe31768d692a6c948cbc840e9ab92d69f Mon Sep 17 00:00:00 2001 From: Adam Goldsmith Date: Sat, 15 Feb 2020 12:08:34 -0500 Subject: [PATCH] sqlExport: Merge the logic of members and transactions tables, w/ mappings defined in yaml --- sqlExport.py | 151 ++++++++-------------------------------------- tableMapping.yaml | 81 +++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 125 deletions(-) create mode 100644 tableMapping.yaml diff --git a/sqlExport.py b/sqlExport.py index 32d233f..5b81432 100644 --- a/sqlExport.py +++ b/sqlExport.py @@ -2,6 +2,7 @@ from datetime import datetime import sqlite3 +import yaml #from common import membershipworks @@ -11,92 +12,21 @@ from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD membershipworks = MembershipWorks() membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD) -membersTableColumns = { - 'uid': {'type': 'TEXT PRIMARY KEY', 'label': 'Account ID'}, - 'Year of Birth': {}, - 'Account Name': {}, - 'First Name': {}, - 'Last Name': {}, - 'Phone': {}, - 'Email': {}, - - 'Address (Street)': {}, - 'Address (City)': {}, - 'Address (State/Province)': {}, - 'Address (Postal Code)': {}, - 'Address (Country)': {}, - 'Profile description': {}, - 'Website': {}, - 'Fax': {}, - 'Contact Person': {}, - 'Password': {}, - 'Position/relation': {}, - - 'Parent Account ID': {}, - - 'Gift Membership purchased by': {}, - 'Purchased Gift Membership for': {}, - - 'Closet Storage #': {}, - 'Storage Shelf #': {}, - 'Personal Studio Space #': {}, - - 'Access Permitted Shops During Extended Hours?': {'type': 'BOOLEAN'}, - 'Access Front Door and Studio Space During Extended Hours?': {'type': 'BOOLEAN'}, - 'Access Wood Shop?': {'type': 'BOOLEAN'}, - 'Access Metal Shop?': {'type': 'BOOLEAN'}, - 'Access Storage Closet?': {'type': 'BOOLEAN'}, - 'Access Studio Space?': {'type': 'BOOLEAN'}, - 'Access Front Door?': {'type': 'BOOLEAN'}, - 'Access Card Number': {'type': 'INTEGER'}, - 'Access Card Facility Code': {'type': 'INTEGER'}, - - 'Auto Billing ID': {}, - 'Billing Method': {}, - 'Renewal Date': {}, - 'Join Date': {}, - - 'Admin note': {}, - - 'Profile gallery image URL': {}, - 'Business card image URL': {}, - 'Instagram': {}, - 'Pinterest': {}, - 'Youtube': {}, - 'Yelp': {}, - 'Google+': {}, - 'BBB': {}, - 'Twitter': {}, - 'Facebook': {}, - 'LinkedIn': {}, - - 'Do not show street address in profile': {}, - 'Do not list in directory': {}, - - 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': {}, - 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': {}, - 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': {}, - 'Waiver form signed and on file date.': {}, - 'Membership Agreement signed and on file date.': {}, - - 'IP Address': {}, -} - -def insertMembers(members): - def formatMembers(members): - for m in members: +def insertFromTableMap(table, data, tableMap): + def formatRows(data): + for d in data: yield [ - m[v.get('label', k)] for k, v in membersTableColumns.items() + d.get(v.get('source', k) if v is not None else k) for k, v in tableMap.items() ] c.execute('BEGIN') # NOTE: this only works if in python >= 3.7 where `dict` is ordered - c.executemany('INSERT OR REPLACE INTO members (' + - ','.join('"' + k + '"' for k in membersTableColumns.keys()) + + c.executemany('INSERT OR REPLACE INTO ' + table + ' (' + + ','.join('"' + k + '"' for k in tableMap.keys()) + ') values (' + - ','.join(len(membersTableColumns) * '?') + + ','.join(len(tableMap) * '?') + ');', - list(formatMembers(members))) + list(formatRows(data))) c.execute('END') def insertLabels(members): @@ -111,59 +41,30 @@ def insertLabels(members): list(formatLabels(members))) c.execute('END') -def insertTransactions(transactions): - def formatTransactions(transactions): - for t in transactions: - yield [ - t["sid"], - t["_dp"], # timestamp - t["typ"], # type - t["cur"], # currency - t["sum"], - t.get("fee"), - t["uid"], - t["nam"], # name - t.get("eid"), # event_id - t.get("ttl") # "For" - ] - - c.execute('BEGIN') - c.executemany('INSERT OR REPLACE INTO transactions (sid, timestamp, type, currency, sum, fee, uid, name, event_id, ttl) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);', - list(formatTransactions(transactions))) - c.execute('END') - +with open('tableMapping.yaml') as f: + tableMapping = yaml.load(f, yaml.SafeLoader) with sqlite3.connect('membershipworks.db') as conn: c = conn.cursor() c.execute('CREATE TABLE IF NOT EXISTS members (' + - ', '.join([f'"{k}" {v.get("type", "")}' - for k, v in membersTableColumns.items()]) + - ')') + ', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}' + for k, v in tableMapping['members'].items()]) + + ');') - c.execute("""CREATE TABLE IF NOT EXISTS transactions ( - sid TEXT PRIMARY KEY, - timestamp INTEGER NOT NULL, - type INTEGER NOT NULL, - currency TEXT NOT NULL, - sum INTEGER NOT NULL, - fee INTEGER, - uid TEXT NOT NULL, - name TEXT NOT NULL, - event_id TEXT, - ttl TEXT, - FOREIGN KEY(uid) REFERENCES member(uid) - -- FOREIGN KEY(event_id) REFERENCES event(eid) - ); - """) + c.execute("CREATE TABLE IF NOT EXISTS transactions (" + + ', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}' + for k, v in tableMapping['transactions'].items()]) + + """, FOREIGN KEY(uid) REFERENCES member(uid) + -- FOREIGN KEY(event_id) REFERENCES event(eid) + ); + """) - # TODO: define a pk constraint c.execute("""CREATE TABLE IF NOT EXISTS labels ( uid TEXT, label Text, - FOREIGN KEY(uid) REFERENCES member(uid), - PRIMARY KEY(uid, label) - ); - """) + PRIMARY KEY(uid, label), + FOREIGN KEY(uid) REFERENCES member(uid) + );""") members = membershipworks.get_all_members() @@ -178,10 +79,10 @@ with sqlite3.connect('membershipworks.db') as conn: if field.get('typ') == 8 and field['lbl'] in m: # check box m[field['lbl']] = True if m[field['lbl']] == 'Y' else False - insertMembers(members) + insertFromTableMap('members', members, tableMapping['members']) insertLabels(members) transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now()) - insertTransactions(transactions) + insertFromTableMap('transactions', transactions, tableMapping['transactions']) # TODO: folders, levels, addons diff --git a/tableMapping.yaml b/tableMapping.yaml new file mode 100644 index 0000000..4438ddb --- /dev/null +++ b/tableMapping.yaml @@ -0,0 +1,81 @@ +members: + 'uid': {type: 'TEXT PRIMARY KEY', source: 'Account ID'} + 'Year of Birth': + 'Account Name': + 'First Name': + 'Last Name': + 'Phone': + 'Email': + + 'Address (Street)': + 'Address (City)': + 'Address (State/Province)': + 'Address (Postal Code)': + 'Address (Country)': + 'Profile description': + 'Website': + 'Fax': + 'Contact Person': + 'Password': + 'Position/relation': + + 'Parent Account ID': + + 'Gift Membership purchased by': + 'Purchased Gift Membership for': + + 'Closet Storage #': + 'Storage Shelf #': + 'Personal Studio Space #': + + 'Access Permitted Shops During Extended Hours?': {type: 'BOOLEAN'} + 'Access Front Door and Studio Space During Extended Hours?': {type: 'BOOLEAN'} + 'Access Wood Shop?': {type: 'BOOLEAN'} + 'Access Metal Shop?': {type: 'BOOLEAN'} + 'Access Storage Closet?': {type: 'BOOLEAN'} + 'Access Studio Space?': {type: 'BOOLEAN'} + 'Access Front Door?': {type: 'BOOLEAN'} + 'Access Card Number': {type: 'INTEGER'} + 'Access Card Facility Code': {type: 'INTEGER'} + + 'Auto Billing ID': + 'Billing Method': + 'Renewal Date': + 'Join Date': + + 'Admin note': + + 'Profile gallery image URL': + 'Business card image URL': + 'Instagram': + 'Pinterest': + 'Youtube': + 'Yelp': + 'Google+': + 'BBB': + 'Twitter': + 'Facebook': + 'LinkedIn': + + 'Do not show street address in profile': + 'Do not list in directory': + + 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': + 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': + 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': + 'Waiver form signed and on file date.': + 'Membership Agreement signed and on file date.': + + 'IP Address': + +transactions: + 'sid': {type: TEXT PRIMARY KEY} + 'timestamp': {type: INTEGER, source: '_dp'} + 'type': {type: INTEGER, source: 'typ'} # transaction type + 'currency': {source: 'cur'} + 'sum': {type: INTEGER} + 'fee': {type: INTEGER} + 'uid': + 'name': {source: 'nam'} + 'event_id': {source: 'eid'} + 'ttl': # 'For'