From 8edaaf0df9643bcd5abde9be47cf5107b4ba4325 Mon Sep 17 00:00:00 2001 From: Adam Goldsmith Date: Tue, 18 Feb 2020 19:11:14 -0500 Subject: [PATCH] sqlExport: Migrate to mariadb, improve types --- sqlExport.py | 101 +++++++++++++++++++++++++++++----------------- tableMapping.yaml | 22 +++++----- 2 files changed, 74 insertions(+), 49 deletions(-) diff --git a/sqlExport.py b/sqlExport.py index 5b81432..c263b12 100644 --- a/sqlExport.py +++ b/sqlExport.py @@ -1,71 +1,82 @@ #!/usr/bin/env python3 from datetime import datetime -import sqlite3 +import MySQLdb import yaml -#from common import membershipworks +from common import membershipworks -# TODO: TEMP. remove later: -#from lib.MembershipWorks import MembershipWorks -from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD -membershipworks = MembershipWorks() -membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD) def insertFromTableMap(table, data, tableMap): + def resolveSource(key, value): + if type(value) == str: + return value + elif type(value) == dict and 'source' in value: + return value['source'] + else: + return key + def formatRows(data): for d in data: - yield [ - d.get(v.get('source', k) if v is not None else k) for k, v in tableMap.items() - ] + yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()] - c.execute('BEGIN') - # NOTE: this only works if in python >= 3.7 where `dict` is ordered - c.executemany('INSERT OR REPLACE INTO ' + table + ' (' + - ','.join('"' + k + '"' for k in tableMap.keys()) + - ') values (' + - ','.join(len(tableMap) * '?') + + # TODO: this could probably be done better as a single statement? + # note: this only works if in python >= 3.7 where `dict` is ordered + c.executemany('REPLACE INTO ' + table + ' (' + + ','.join(f'`{k}`' for k in tableMap.keys()) + + ') VALUES (' + + ','.join(len(tableMap) * ['%s']) + ');', list(formatRows(data))) - c.execute('END') +# TODO: delete non-valid labels def insertLabels(members): def formatLabels(labels): for member in members: - for label in membershipworks._parse_flags()['labels']: + for label, label_id in membershipworks._parse_flags()['labels'].items(): if member[label]: - yield member['Account ID'], label + yield member['Account ID'], label_id - c.execute('BEGIN') - c.executemany('INSERT OR REPLACE INTO labels (uid, label) values (?, ?);', + c.executemany('REPLACE INTO member_labels (uid, label_id) VALUES (%s, %s);', list(formatLabels(members))) - c.execute('END') with open('tableMapping.yaml') as f: tableMapping = yaml.load(f, yaml.SafeLoader) -with sqlite3.connect('membershipworks.db') as conn: - c = conn.cursor() +conn = MySQLdb.connect(user='membershipworks', password='asdf', database='membershipworks') +c = conn.cursor() + +def createDefinitionsFromTableMap(tableMap): + def resolveColType(value): + if type(value) == dict and 'type' in value: + return value['type'] + else: + return 'TEXT' + + return ', '.join([f'`{k}` ' + resolveColType(v) + for k, v in tableMap.items()]) + +try: c.execute('CREATE TABLE IF NOT EXISTS members (' + - ', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}' - for k, v in tableMapping['members'].items()]) + - ');') + createDefinitionsFromTableMap(tableMapping['members']) + + ') WITH SYSTEM VERSIONING;') c.execute("CREATE TABLE IF NOT EXISTS transactions (" + - ', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}' - for k, v in tableMapping['transactions'].items()]) + - """, FOREIGN KEY(uid) REFERENCES member(uid) - -- FOREIGN KEY(event_id) REFERENCES event(eid) - ); - """) + createDefinitionsFromTableMap(tableMapping['transactions']) + + ", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));") + #-- FOREIGN KEY event_id REFERENCES event eid c.execute("""CREATE TABLE IF NOT EXISTS labels ( - uid TEXT, label Text, - PRIMARY KEY(uid, label), - FOREIGN KEY(uid) REFERENCES member(uid) - );""") + label_id CHAR(24) PRIMARY KEY, label TEXT + ) WITH SYSTEM VERSIONING;""") + c.execute("""CREATE TABLE IF NOT EXISTS member_labels ( + uid CHAR(24), label_id CHAR(24), + PRIMARY KEY(uid, label_id), + FOREIGN KEY(uid) REFERENCES members(uid), + FOREIGN KEY(label_id) REFERENCES labels(label_id) + ) WITH SYSTEM VERSIONING;""") members = membershipworks.get_all_members() for m in members: @@ -79,10 +90,24 @@ with sqlite3.connect('membershipworks.db') as conn: if field.get('typ') == 8 and field['lbl'] in m: # check box m[field['lbl']] = True if m[field['lbl']] == 'Y' else False + c.executemany("REPLACE INTO labels (label, label_id) VALUES (%s, %s)", + membershipworks._parse_flags()['labels'].items()) + insertFromTableMap('members', members, tableMapping['members']) + insertLabels(members) transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now()) insertFromTableMap('transactions', transactions, tableMapping['transactions']) - # TODO: folders, levels, addons + conn.commit() + +except Exception as e: + print("Transaction failed, rolling back") + conn.rollback(); + raise e + +finally: + conn.close() + +# TODO: folders, levels, addons diff --git a/tableMapping.yaml b/tableMapping.yaml index 4438ddb..c4e7e47 100644 --- a/tableMapping.yaml +++ b/tableMapping.yaml @@ -1,5 +1,5 @@ members: - 'uid': {type: 'TEXT PRIMARY KEY', source: 'Account ID'} + 'uid': {type: 'CHAR(24) PRIMARY KEY', source: 'Account ID'} 'Year of Birth': 'Account Name': 'First Name': @@ -35,8 +35,8 @@ members: 'Access Storage Closet?': {type: 'BOOLEAN'} 'Access Studio Space?': {type: 'BOOLEAN'} 'Access Front Door?': {type: 'BOOLEAN'} - 'Access Card Number': {type: 'INTEGER'} - 'Access Card Facility Code': {type: 'INTEGER'} + 'Access Card Number': + 'Access Card Facility Code': 'Auto Billing ID': 'Billing Method': @@ -60,22 +60,22 @@ members: 'Do not show street address in profile': 'Do not list in directory': - 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': - 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': - 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': + 'HowDidYouHear': 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:' + 'authorizeCharge': 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.' + 'policyAgreement': 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.' 'Waiver form signed and on file date.': 'Membership Agreement signed and on file date.': 'IP Address': transactions: - 'sid': {type: TEXT PRIMARY KEY} - 'timestamp': {type: INTEGER, source: '_dp'} + 'sid': {type: CHAR(27) PRIMARY KEY} + 'uid': {type: CHAR(24)} + 'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp? 'type': {type: INTEGER, source: 'typ'} # transaction type 'currency': {source: 'cur'} - 'sum': {type: INTEGER} - 'fee': {type: INTEGER} - 'uid': + 'sum': {type: 'DECIMAL(13,4)'} + 'fee': {type: 'DECIMAL(13,4)'} 'name': {source: 'nam'} 'event_id': {source: 'eid'} 'ttl': # 'For'