sqlExport: Merge the logic of members and transactions tables, w/ mappings defined in yaml
This commit is contained in:
parent
38ba551f1e
commit
8b54dc7fe3
151
sqlExport.py
151
sqlExport.py
@ -2,6 +2,7 @@
|
||||
|
||||
from datetime import datetime
|
||||
import sqlite3
|
||||
import yaml
|
||||
|
||||
#from common import membershipworks
|
||||
|
||||
@ -11,92 +12,21 @@ from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD
|
||||
membershipworks = MembershipWorks()
|
||||
membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD)
|
||||
|
||||
membersTableColumns = {
|
||||
'uid': {'type': 'TEXT PRIMARY KEY', 'label': 'Account ID'},
|
||||
'Year of Birth': {},
|
||||
'Account Name': {},
|
||||
'First Name': {},
|
||||
'Last Name': {},
|
||||
'Phone': {},
|
||||
'Email': {},
|
||||
|
||||
'Address (Street)': {},
|
||||
'Address (City)': {},
|
||||
'Address (State/Province)': {},
|
||||
'Address (Postal Code)': {},
|
||||
'Address (Country)': {},
|
||||
'Profile description': {},
|
||||
'Website': {},
|
||||
'Fax': {},
|
||||
'Contact Person': {},
|
||||
'Password': {},
|
||||
'Position/relation': {},
|
||||
|
||||
'Parent Account ID': {},
|
||||
|
||||
'Gift Membership purchased by': {},
|
||||
'Purchased Gift Membership for': {},
|
||||
|
||||
'Closet Storage #': {},
|
||||
'Storage Shelf #': {},
|
||||
'Personal Studio Space #': {},
|
||||
|
||||
'Access Permitted Shops During Extended Hours?': {'type': 'BOOLEAN'},
|
||||
'Access Front Door and Studio Space During Extended Hours?': {'type': 'BOOLEAN'},
|
||||
'Access Wood Shop?': {'type': 'BOOLEAN'},
|
||||
'Access Metal Shop?': {'type': 'BOOLEAN'},
|
||||
'Access Storage Closet?': {'type': 'BOOLEAN'},
|
||||
'Access Studio Space?': {'type': 'BOOLEAN'},
|
||||
'Access Front Door?': {'type': 'BOOLEAN'},
|
||||
'Access Card Number': {'type': 'INTEGER'},
|
||||
'Access Card Facility Code': {'type': 'INTEGER'},
|
||||
|
||||
'Auto Billing ID': {},
|
||||
'Billing Method': {},
|
||||
'Renewal Date': {},
|
||||
'Join Date': {},
|
||||
|
||||
'Admin note': {},
|
||||
|
||||
'Profile gallery image URL': {},
|
||||
'Business card image URL': {},
|
||||
'Instagram': {},
|
||||
'Pinterest': {},
|
||||
'Youtube': {},
|
||||
'Yelp': {},
|
||||
'Google+': {},
|
||||
'BBB': {},
|
||||
'Twitter': {},
|
||||
'Facebook': {},
|
||||
'LinkedIn': {},
|
||||
|
||||
'Do not show street address in profile': {},
|
||||
'Do not list in directory': {},
|
||||
|
||||
'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': {},
|
||||
'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': {},
|
||||
'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': {},
|
||||
'Waiver form signed and on file date.': {},
|
||||
'Membership Agreement signed and on file date.': {},
|
||||
|
||||
'IP Address': {},
|
||||
}
|
||||
|
||||
def insertMembers(members):
|
||||
def formatMembers(members):
|
||||
for m in members:
|
||||
def insertFromTableMap(table, data, tableMap):
|
||||
def formatRows(data):
|
||||
for d in data:
|
||||
yield [
|
||||
m[v.get('label', k)] for k, v in membersTableColumns.items()
|
||||
d.get(v.get('source', k) if v is not None else k) for k, v in tableMap.items()
|
||||
]
|
||||
|
||||
c.execute('BEGIN')
|
||||
# NOTE: this only works if in python >= 3.7 where `dict` is ordered
|
||||
c.executemany('INSERT OR REPLACE INTO members (' +
|
||||
','.join('"' + k + '"' for k in membersTableColumns.keys()) +
|
||||
c.executemany('INSERT OR REPLACE INTO ' + table + ' (' +
|
||||
','.join('"' + k + '"' for k in tableMap.keys()) +
|
||||
') values (' +
|
||||
','.join(len(membersTableColumns) * '?') +
|
||||
','.join(len(tableMap) * '?') +
|
||||
');',
|
||||
list(formatMembers(members)))
|
||||
list(formatRows(data)))
|
||||
c.execute('END')
|
||||
|
||||
def insertLabels(members):
|
||||
@ -111,59 +41,30 @@ def insertLabels(members):
|
||||
list(formatLabels(members)))
|
||||
c.execute('END')
|
||||
|
||||
def insertTransactions(transactions):
|
||||
def formatTransactions(transactions):
|
||||
for t in transactions:
|
||||
yield [
|
||||
t["sid"],
|
||||
t["_dp"], # timestamp
|
||||
t["typ"], # type
|
||||
t["cur"], # currency
|
||||
t["sum"],
|
||||
t.get("fee"),
|
||||
t["uid"],
|
||||
t["nam"], # name
|
||||
t.get("eid"), # event_id
|
||||
t.get("ttl") # "For"
|
||||
]
|
||||
|
||||
c.execute('BEGIN')
|
||||
c.executemany('INSERT OR REPLACE INTO transactions (sid, timestamp, type, currency, sum, fee, uid, name, event_id, ttl) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);',
|
||||
list(formatTransactions(transactions)))
|
||||
c.execute('END')
|
||||
|
||||
with open('tableMapping.yaml') as f:
|
||||
tableMapping = yaml.load(f, yaml.SafeLoader)
|
||||
|
||||
with sqlite3.connect('membershipworks.db') as conn:
|
||||
c = conn.cursor()
|
||||
|
||||
c.execute('CREATE TABLE IF NOT EXISTS members (' +
|
||||
', '.join([f'"{k}" {v.get("type", "")}'
|
||||
for k, v in membersTableColumns.items()]) +
|
||||
')')
|
||||
', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
|
||||
for k, v in tableMapping['members'].items()]) +
|
||||
');')
|
||||
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS transactions (
|
||||
sid TEXT PRIMARY KEY,
|
||||
timestamp INTEGER NOT NULL,
|
||||
type INTEGER NOT NULL,
|
||||
currency TEXT NOT NULL,
|
||||
sum INTEGER NOT NULL,
|
||||
fee INTEGER,
|
||||
uid TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
event_id TEXT,
|
||||
ttl TEXT,
|
||||
FOREIGN KEY(uid) REFERENCES member(uid)
|
||||
-- FOREIGN KEY(event_id) REFERENCES event(eid)
|
||||
);
|
||||
""")
|
||||
c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
|
||||
', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
|
||||
for k, v in tableMapping['transactions'].items()]) +
|
||||
""", FOREIGN KEY(uid) REFERENCES member(uid)
|
||||
-- FOREIGN KEY(event_id) REFERENCES event(eid)
|
||||
);
|
||||
""")
|
||||
|
||||
# TODO: define a pk constraint
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS labels (
|
||||
uid TEXT, label Text,
|
||||
FOREIGN KEY(uid) REFERENCES member(uid),
|
||||
PRIMARY KEY(uid, label)
|
||||
);
|
||||
""")
|
||||
PRIMARY KEY(uid, label),
|
||||
FOREIGN KEY(uid) REFERENCES member(uid)
|
||||
);""")
|
||||
|
||||
|
||||
members = membershipworks.get_all_members()
|
||||
@ -178,10 +79,10 @@ with sqlite3.connect('membershipworks.db') as conn:
|
||||
if field.get('typ') == 8 and field['lbl'] in m: # check box
|
||||
m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
|
||||
|
||||
insertMembers(members)
|
||||
insertFromTableMap('members', members, tableMapping['members'])
|
||||
insertLabels(members)
|
||||
|
||||
transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
|
||||
insertTransactions(transactions)
|
||||
insertFromTableMap('transactions', transactions, tableMapping['transactions'])
|
||||
|
||||
# TODO: folders, levels, addons
|
||||
|
||||
81
tableMapping.yaml
Normal file
81
tableMapping.yaml
Normal file
@ -0,0 +1,81 @@
|
||||
members:
|
||||
'uid': {type: 'TEXT PRIMARY KEY', source: 'Account ID'}
|
||||
'Year of Birth':
|
||||
'Account Name':
|
||||
'First Name':
|
||||
'Last Name':
|
||||
'Phone':
|
||||
'Email':
|
||||
|
||||
'Address (Street)':
|
||||
'Address (City)':
|
||||
'Address (State/Province)':
|
||||
'Address (Postal Code)':
|
||||
'Address (Country)':
|
||||
'Profile description':
|
||||
'Website':
|
||||
'Fax':
|
||||
'Contact Person':
|
||||
'Password':
|
||||
'Position/relation':
|
||||
|
||||
'Parent Account ID':
|
||||
|
||||
'Gift Membership purchased by':
|
||||
'Purchased Gift Membership for':
|
||||
|
||||
'Closet Storage #':
|
||||
'Storage Shelf #':
|
||||
'Personal Studio Space #':
|
||||
|
||||
'Access Permitted Shops During Extended Hours?': {type: 'BOOLEAN'}
|
||||
'Access Front Door and Studio Space During Extended Hours?': {type: 'BOOLEAN'}
|
||||
'Access Wood Shop?': {type: 'BOOLEAN'}
|
||||
'Access Metal Shop?': {type: 'BOOLEAN'}
|
||||
'Access Storage Closet?': {type: 'BOOLEAN'}
|
||||
'Access Studio Space?': {type: 'BOOLEAN'}
|
||||
'Access Front Door?': {type: 'BOOLEAN'}
|
||||
'Access Card Number': {type: 'INTEGER'}
|
||||
'Access Card Facility Code': {type: 'INTEGER'}
|
||||
|
||||
'Auto Billing ID':
|
||||
'Billing Method':
|
||||
'Renewal Date':
|
||||
'Join Date':
|
||||
|
||||
'Admin note':
|
||||
|
||||
'Profile gallery image URL':
|
||||
'Business card image URL':
|
||||
'Instagram':
|
||||
'Pinterest':
|
||||
'Youtube':
|
||||
'Yelp':
|
||||
'Google+':
|
||||
'BBB':
|
||||
'Twitter':
|
||||
'Facebook':
|
||||
'LinkedIn':
|
||||
|
||||
'Do not show street address in profile':
|
||||
'Do not list in directory':
|
||||
|
||||
'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:':
|
||||
'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.':
|
||||
'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.':
|
||||
'Waiver form signed and on file date.':
|
||||
'Membership Agreement signed and on file date.':
|
||||
|
||||
'IP Address':
|
||||
|
||||
transactions:
|
||||
'sid': {type: TEXT PRIMARY KEY}
|
||||
'timestamp': {type: INTEGER, source: '_dp'}
|
||||
'type': {type: INTEGER, source: 'typ'} # transaction type
|
||||
'currency': {source: 'cur'}
|
||||
'sum': {type: INTEGER}
|
||||
'fee': {type: INTEGER}
|
||||
'uid':
|
||||
'name': {source: 'nam'}
|
||||
'event_id': {source: 'eid'}
|
||||
'ttl': # 'For'
|
||||
Reference in New Issue
Block a user