sqlExport: Migrate to mariadb, improve types
This commit is contained in:
parent
8b54dc7fe3
commit
8edaaf0df9
101
sqlExport.py
101
sqlExport.py
@ -1,71 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from datetime import datetime
|
||||
import sqlite3
|
||||
import MySQLdb
|
||||
import yaml
|
||||
|
||||
#from common import membershipworks
|
||||
from common import membershipworks
|
||||
|
||||
# TODO: TEMP. remove later:
|
||||
#from lib.MembershipWorks import MembershipWorks
|
||||
from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD
|
||||
membershipworks = MembershipWorks()
|
||||
membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD)
|
||||
|
||||
def insertFromTableMap(table, data, tableMap):
|
||||
def resolveSource(key, value):
|
||||
if type(value) == str:
|
||||
return value
|
||||
elif type(value) == dict and 'source' in value:
|
||||
return value['source']
|
||||
else:
|
||||
return key
|
||||
|
||||
def formatRows(data):
|
||||
for d in data:
|
||||
yield [
|
||||
d.get(v.get('source', k) if v is not None else k) for k, v in tableMap.items()
|
||||
]
|
||||
yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
|
||||
|
||||
c.execute('BEGIN')
|
||||
# NOTE: this only works if in python >= 3.7 where `dict` is ordered
|
||||
c.executemany('INSERT OR REPLACE INTO ' + table + ' (' +
|
||||
','.join('"' + k + '"' for k in tableMap.keys()) +
|
||||
') values (' +
|
||||
','.join(len(tableMap) * '?') +
|
||||
# TODO: this could probably be done better as a single statement?
|
||||
# note: this only works if in python >= 3.7 where `dict` is ordered
|
||||
c.executemany('REPLACE INTO ' + table + ' (' +
|
||||
','.join(f'`{k}`' for k in tableMap.keys()) +
|
||||
') VALUES (' +
|
||||
','.join(len(tableMap) * ['%s']) +
|
||||
');',
|
||||
list(formatRows(data)))
|
||||
c.execute('END')
|
||||
|
||||
# TODO: delete non-valid labels
|
||||
def insertLabels(members):
|
||||
def formatLabels(labels):
|
||||
for member in members:
|
||||
for label in membershipworks._parse_flags()['labels']:
|
||||
for label, label_id in membershipworks._parse_flags()['labels'].items():
|
||||
if member[label]:
|
||||
yield member['Account ID'], label
|
||||
yield member['Account ID'], label_id
|
||||
|
||||
c.execute('BEGIN')
|
||||
c.executemany('INSERT OR REPLACE INTO labels (uid, label) values (?, ?);',
|
||||
c.executemany('REPLACE INTO member_labels (uid, label_id) VALUES (%s, %s);',
|
||||
list(formatLabels(members)))
|
||||
c.execute('END')
|
||||
|
||||
with open('tableMapping.yaml') as f:
|
||||
tableMapping = yaml.load(f, yaml.SafeLoader)
|
||||
|
||||
with sqlite3.connect('membershipworks.db') as conn:
|
||||
c = conn.cursor()
|
||||
|
||||
conn = MySQLdb.connect(user='membershipworks', password='asdf', database='membershipworks')
|
||||
c = conn.cursor()
|
||||
|
||||
def createDefinitionsFromTableMap(tableMap):
|
||||
def resolveColType(value):
|
||||
if type(value) == dict and 'type' in value:
|
||||
return value['type']
|
||||
else:
|
||||
return 'TEXT'
|
||||
|
||||
return ', '.join([f'`{k}` ' + resolveColType(v)
|
||||
for k, v in tableMap.items()])
|
||||
|
||||
try:
|
||||
c.execute('CREATE TABLE IF NOT EXISTS members (' +
|
||||
', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
|
||||
for k, v in tableMapping['members'].items()]) +
|
||||
');')
|
||||
createDefinitionsFromTableMap(tableMapping['members']) +
|
||||
') WITH SYSTEM VERSIONING;')
|
||||
|
||||
c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
|
||||
', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
|
||||
for k, v in tableMapping['transactions'].items()]) +
|
||||
""", FOREIGN KEY(uid) REFERENCES member(uid)
|
||||
-- FOREIGN KEY(event_id) REFERENCES event(eid)
|
||||
);
|
||||
""")
|
||||
createDefinitionsFromTableMap(tableMapping['transactions']) +
|
||||
", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));")
|
||||
#-- FOREIGN KEY event_id REFERENCES event eid
|
||||
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS labels (
|
||||
uid TEXT, label Text,
|
||||
PRIMARY KEY(uid, label),
|
||||
FOREIGN KEY(uid) REFERENCES member(uid)
|
||||
);""")
|
||||
label_id CHAR(24) PRIMARY KEY, label TEXT
|
||||
) WITH SYSTEM VERSIONING;""")
|
||||
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS member_labels (
|
||||
uid CHAR(24), label_id CHAR(24),
|
||||
PRIMARY KEY(uid, label_id),
|
||||
FOREIGN KEY(uid) REFERENCES members(uid),
|
||||
FOREIGN KEY(label_id) REFERENCES labels(label_id)
|
||||
) WITH SYSTEM VERSIONING;""")
|
||||
|
||||
members = membershipworks.get_all_members()
|
||||
for m in members:
|
||||
@ -79,10 +90,24 @@ with sqlite3.connect('membershipworks.db') as conn:
|
||||
if field.get('typ') == 8 and field['lbl'] in m: # check box
|
||||
m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
|
||||
|
||||
c.executemany("REPLACE INTO labels (label, label_id) VALUES (%s, %s)",
|
||||
membershipworks._parse_flags()['labels'].items())
|
||||
|
||||
insertFromTableMap('members', members, tableMapping['members'])
|
||||
|
||||
insertLabels(members)
|
||||
|
||||
transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
|
||||
insertFromTableMap('transactions', transactions, tableMapping['transactions'])
|
||||
|
||||
# TODO: folders, levels, addons
|
||||
conn.commit()
|
||||
|
||||
except Exception as e:
|
||||
print("Transaction failed, rolling back")
|
||||
conn.rollback();
|
||||
raise e
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# TODO: folders, levels, addons
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
members:
|
||||
'uid': {type: 'TEXT PRIMARY KEY', source: 'Account ID'}
|
||||
'uid': {type: 'CHAR(24) PRIMARY KEY', source: 'Account ID'}
|
||||
'Year of Birth':
|
||||
'Account Name':
|
||||
'First Name':
|
||||
@ -35,8 +35,8 @@ members:
|
||||
'Access Storage Closet?': {type: 'BOOLEAN'}
|
||||
'Access Studio Space?': {type: 'BOOLEAN'}
|
||||
'Access Front Door?': {type: 'BOOLEAN'}
|
||||
'Access Card Number': {type: 'INTEGER'}
|
||||
'Access Card Facility Code': {type: 'INTEGER'}
|
||||
'Access Card Number':
|
||||
'Access Card Facility Code':
|
||||
|
||||
'Auto Billing ID':
|
||||
'Billing Method':
|
||||
@ -60,22 +60,22 @@ members:
|
||||
'Do not show street address in profile':
|
||||
'Do not list in directory':
|
||||
|
||||
'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:':
|
||||
'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.':
|
||||
'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.':
|
||||
'HowDidYouHear': 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:'
|
||||
'authorizeCharge': 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.'
|
||||
'policyAgreement': 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.'
|
||||
'Waiver form signed and on file date.':
|
||||
'Membership Agreement signed and on file date.':
|
||||
|
||||
'IP Address':
|
||||
|
||||
transactions:
|
||||
'sid': {type: TEXT PRIMARY KEY}
|
||||
'timestamp': {type: INTEGER, source: '_dp'}
|
||||
'sid': {type: CHAR(27) PRIMARY KEY}
|
||||
'uid': {type: CHAR(24)}
|
||||
'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp?
|
||||
'type': {type: INTEGER, source: 'typ'} # transaction type
|
||||
'currency': {source: 'cur'}
|
||||
'sum': {type: INTEGER}
|
||||
'fee': {type: INTEGER}
|
||||
'uid':
|
||||
'sum': {type: 'DECIMAL(13,4)'}
|
||||
'fee': {type: 'DECIMAL(13,4)'}
|
||||
'name': {source: 'nam'}
|
||||
'event_id': {source: 'eid'}
|
||||
'ttl': # 'For'
|
||||
|
||||
Reference in New Issue
Block a user