steve.goldsmith/memberPlumbing
1
0
Fork 0
forked from CMS/memberPlumbing
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sqlExport: Migrate to mariadb, improve types

Browse Source
This commit is contained in:
Adam Goldsmith 2020-02-18 19:11:14 -05:00
parent 8b54dc7fe3
commit 8edaaf0df9
2 changed files with 74 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
sqlExport.py
View File

@ -1,71 +1,82 @@
#!/usr/bin/env python3
from datetime import datetime
import sqlite3
import MySQLdb
import yaml
#from common import membershipworks
from common import membershipworks
# TODO: TEMP. remove later:
#from lib.MembershipWorks import MembershipWorks
from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD
membershipworks = MembershipWorks()
membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD)
def insertFromTableMap(table, data, tableMap):
def resolveSource(key, value):
if type(value) == str:
return value
elif type(value) == dict and 'source' in value:
return value['source']
else:
return key
def formatRows(data):
for d in data:
yield [
d.get(v.get('source', k) if v is not None else k) for k, v in tableMap.items()
]
yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
c.execute('BEGIN')
# NOTE: this only works if in python >= 3.7 where `dict` is ordered
c.executemany('INSERT OR REPLACE INTO ' + table + ' (' +
','.join('"' + k + '"' for k in tableMap.keys()) +
') values (' +
','.join(len(tableMap) * '?') +
# TODO: this could probably be done better as a single statement?
# note: this only works if in python >= 3.7 where `dict` is ordered
c.executemany('REPLACE INTO ' + table + ' (' +
','.join(f'`{k}`' for k in tableMap.keys()) +
') VALUES (' +
','.join(len(tableMap) * ['%s']) +
');',
list(formatRows(data)))
c.execute('END')
# TODO: delete non-valid labels
def insertLabels(members):
def formatLabels(labels):
for member in members:
for label in membershipworks._parse_flags()['labels']:
for label, label_id in membershipworks._parse_flags()['labels'].items():
if member[label]:
yield member['Account ID'], label
yield member['Account ID'], label_id
c.execute('BEGIN')
c.executemany('INSERT OR REPLACE INTO labels (uid, label) values (?, ?);',
c.executemany('REPLACE INTO member_labels (uid, label_id) VALUES (%s, %s);',
list(formatLabels(members)))
c.execute('END')
with open('tableMapping.yaml') as f:
tableMapping = yaml.load(f, yaml.SafeLoader)
with sqlite3.connect('membershipworks.db') as conn:
conn = MySQLdb.connect(user='membershipworks', password='asdf', database='membershipworks')
c = conn.cursor()
def createDefinitionsFromTableMap(tableMap):
def resolveColType(value):
if type(value) == dict and 'type' in value:
return value['type']
else:
return 'TEXT'
return ', '.join([f'`{k}` ' + resolveColType(v)
for k, v in tableMap.items()])
try:
c.execute('CREATE TABLE IF NOT EXISTS members (' +
', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
for k, v in tableMapping['members'].items()]) +
');')
createDefinitionsFromTableMap(tableMapping['members']) +
') WITH SYSTEM VERSIONING;')
c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
for k, v in tableMapping['transactions'].items()]) +
""", FOREIGN KEY(uid) REFERENCES member(uid)
-- FOREIGN KEY(event_id) REFERENCES event(eid)
);
""")
createDefinitionsFromTableMap(tableMapping['transactions']) +
", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));")
#-- FOREIGN KEY event_id REFERENCES event eid
c.execute("""CREATE TABLE IF NOT EXISTS labels (
uid TEXT, label Text,
PRIMARY KEY(uid, label),
FOREIGN KEY(uid) REFERENCES member(uid)
);""")
label_id CHAR(24) PRIMARY KEY, label TEXT
) WITH SYSTEM VERSIONING;""")
c.execute("""CREATE TABLE IF NOT EXISTS member_labels (
uid CHAR(24), label_id CHAR(24),
PRIMARY KEY(uid, label_id),
FOREIGN KEY(uid) REFERENCES members(uid),
FOREIGN KEY(label_id) REFERENCES labels(label_id)
) WITH SYSTEM VERSIONING;""")
members = membershipworks.get_all_members()
for m in members:
@ -79,10 +90,24 @@ with sqlite3.connect('membershipworks.db') as conn:
if field.get('typ') == 8 and field['lbl'] in m: # check box
m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
c.executemany("REPLACE INTO labels (label, label_id) VALUES (%s, %s)",
membershipworks._parse_flags()['labels'].items())
insertFromTableMap('members', members, tableMapping['members'])
insertLabels(members)
transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
insertFromTableMap('transactions', transactions, tableMapping['transactions'])
conn.commit()
except Exception as e:
print("Transaction failed, rolling back")
conn.rollback();
raise e
finally:
conn.close()
# TODO: folders, levels, addons

22
tableMapping.yaml
View File

@ -1,5 +1,5 @@
members:
'uid': {type: 'TEXT PRIMARY KEY', source: 'Account ID'}
'uid': {type: 'CHAR(24) PRIMARY KEY', source: 'Account ID'}
'Year of Birth':
'Account Name':
'First Name':
@ -35,8 +35,8 @@ members:
'Access Storage Closet?': {type: 'BOOLEAN'}
'Access Studio Space?': {type: 'BOOLEAN'}
'Access Front Door?': {type: 'BOOLEAN'}
'Access Card Number': {type: 'INTEGER'}
'Access Card Facility Code': {type: 'INTEGER'}
'Access Card Number':
'Access Card Facility Code':
'Auto Billing ID':
'Billing Method':
@ -60,22 +60,22 @@ members:
'Do not show street address in profile':
'Do not list in directory':
'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:':
'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.':
'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.':
'HowDidYouHear': 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:'
'authorizeCharge': 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.'
'policyAgreement': 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.'
'Waiver form signed and on file date.':
'Membership Agreement signed and on file date.':
'IP Address':
transactions:
'sid': {type: TEXT PRIMARY KEY}
'timestamp': {type: INTEGER, source: '_dp'}
'sid': {type: CHAR(27) PRIMARY KEY}
'uid': {type: CHAR(24)}
'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp?
'type': {type: INTEGER, source: 'typ'} # transaction type
'currency': {source: 'cur'}
'sum': {type: INTEGER}
'fee': {type: INTEGER}
'uid':
'sum': {type: 'DECIMAL(13,4)'}
'fee': {type: 'DECIMAL(13,4)'}
'name': {source: 'nam'}
'event_id': {source: 'eid'}
'ttl': # 'For'