sqlExport: Merge the logic of members and transactions tables, w/ mappings defined in yaml
This commit is contained in:
parent
38ba551f1e
commit
8b54dc7fe3
145
sqlExport.py
145
sqlExport.py
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
import yaml
|
||||||
|
|
||||||
#from common import membershipworks
|
#from common import membershipworks
|
||||||
|
|
||||||
@ -11,92 +12,21 @@ from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD
|
|||||||
membershipworks = MembershipWorks()
|
membershipworks = MembershipWorks()
|
||||||
membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD)
|
membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD)
|
||||||
|
|
||||||
membersTableColumns = {
|
def insertFromTableMap(table, data, tableMap):
|
||||||
'uid': {'type': 'TEXT PRIMARY KEY', 'label': 'Account ID'},
|
def formatRows(data):
|
||||||
'Year of Birth': {},
|
for d in data:
|
||||||
'Account Name': {},
|
|
||||||
'First Name': {},
|
|
||||||
'Last Name': {},
|
|
||||||
'Phone': {},
|
|
||||||
'Email': {},
|
|
||||||
|
|
||||||
'Address (Street)': {},
|
|
||||||
'Address (City)': {},
|
|
||||||
'Address (State/Province)': {},
|
|
||||||
'Address (Postal Code)': {},
|
|
||||||
'Address (Country)': {},
|
|
||||||
'Profile description': {},
|
|
||||||
'Website': {},
|
|
||||||
'Fax': {},
|
|
||||||
'Contact Person': {},
|
|
||||||
'Password': {},
|
|
||||||
'Position/relation': {},
|
|
||||||
|
|
||||||
'Parent Account ID': {},
|
|
||||||
|
|
||||||
'Gift Membership purchased by': {},
|
|
||||||
'Purchased Gift Membership for': {},
|
|
||||||
|
|
||||||
'Closet Storage #': {},
|
|
||||||
'Storage Shelf #': {},
|
|
||||||
'Personal Studio Space #': {},
|
|
||||||
|
|
||||||
'Access Permitted Shops During Extended Hours?': {'type': 'BOOLEAN'},
|
|
||||||
'Access Front Door and Studio Space During Extended Hours?': {'type': 'BOOLEAN'},
|
|
||||||
'Access Wood Shop?': {'type': 'BOOLEAN'},
|
|
||||||
'Access Metal Shop?': {'type': 'BOOLEAN'},
|
|
||||||
'Access Storage Closet?': {'type': 'BOOLEAN'},
|
|
||||||
'Access Studio Space?': {'type': 'BOOLEAN'},
|
|
||||||
'Access Front Door?': {'type': 'BOOLEAN'},
|
|
||||||
'Access Card Number': {'type': 'INTEGER'},
|
|
||||||
'Access Card Facility Code': {'type': 'INTEGER'},
|
|
||||||
|
|
||||||
'Auto Billing ID': {},
|
|
||||||
'Billing Method': {},
|
|
||||||
'Renewal Date': {},
|
|
||||||
'Join Date': {},
|
|
||||||
|
|
||||||
'Admin note': {},
|
|
||||||
|
|
||||||
'Profile gallery image URL': {},
|
|
||||||
'Business card image URL': {},
|
|
||||||
'Instagram': {},
|
|
||||||
'Pinterest': {},
|
|
||||||
'Youtube': {},
|
|
||||||
'Yelp': {},
|
|
||||||
'Google+': {},
|
|
||||||
'BBB': {},
|
|
||||||
'Twitter': {},
|
|
||||||
'Facebook': {},
|
|
||||||
'LinkedIn': {},
|
|
||||||
|
|
||||||
'Do not show street address in profile': {},
|
|
||||||
'Do not list in directory': {},
|
|
||||||
|
|
||||||
'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:': {},
|
|
||||||
'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.': {},
|
|
||||||
'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.': {},
|
|
||||||
'Waiver form signed and on file date.': {},
|
|
||||||
'Membership Agreement signed and on file date.': {},
|
|
||||||
|
|
||||||
'IP Address': {},
|
|
||||||
}
|
|
||||||
|
|
||||||
def insertMembers(members):
|
|
||||||
def formatMembers(members):
|
|
||||||
for m in members:
|
|
||||||
yield [
|
yield [
|
||||||
m[v.get('label', k)] for k, v in membersTableColumns.items()
|
d.get(v.get('source', k) if v is not None else k) for k, v in tableMap.items()
|
||||||
]
|
]
|
||||||
|
|
||||||
c.execute('BEGIN')
|
c.execute('BEGIN')
|
||||||
# NOTE: this only works if in python >= 3.7 where `dict` is ordered
|
# NOTE: this only works if in python >= 3.7 where `dict` is ordered
|
||||||
c.executemany('INSERT OR REPLACE INTO members (' +
|
c.executemany('INSERT OR REPLACE INTO ' + table + ' (' +
|
||||||
','.join('"' + k + '"' for k in membersTableColumns.keys()) +
|
','.join('"' + k + '"' for k in tableMap.keys()) +
|
||||||
') values (' +
|
') values (' +
|
||||||
','.join(len(membersTableColumns) * '?') +
|
','.join(len(tableMap) * '?') +
|
||||||
');',
|
');',
|
||||||
list(formatMembers(members)))
|
list(formatRows(data)))
|
||||||
c.execute('END')
|
c.execute('END')
|
||||||
|
|
||||||
def insertLabels(members):
|
def insertLabels(members):
|
||||||
@ -111,59 +41,30 @@ def insertLabels(members):
|
|||||||
list(formatLabels(members)))
|
list(formatLabels(members)))
|
||||||
c.execute('END')
|
c.execute('END')
|
||||||
|
|
||||||
def insertTransactions(transactions):
|
with open('tableMapping.yaml') as f:
|
||||||
def formatTransactions(transactions):
|
tableMapping = yaml.load(f, yaml.SafeLoader)
|
||||||
for t in transactions:
|
|
||||||
yield [
|
|
||||||
t["sid"],
|
|
||||||
t["_dp"], # timestamp
|
|
||||||
t["typ"], # type
|
|
||||||
t["cur"], # currency
|
|
||||||
t["sum"],
|
|
||||||
t.get("fee"),
|
|
||||||
t["uid"],
|
|
||||||
t["nam"], # name
|
|
||||||
t.get("eid"), # event_id
|
|
||||||
t.get("ttl") # "For"
|
|
||||||
]
|
|
||||||
|
|
||||||
c.execute('BEGIN')
|
|
||||||
c.executemany('INSERT OR REPLACE INTO transactions (sid, timestamp, type, currency, sum, fee, uid, name, event_id, ttl) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);',
|
|
||||||
list(formatTransactions(transactions)))
|
|
||||||
c.execute('END')
|
|
||||||
|
|
||||||
|
|
||||||
with sqlite3.connect('membershipworks.db') as conn:
|
with sqlite3.connect('membershipworks.db') as conn:
|
||||||
c = conn.cursor()
|
c = conn.cursor()
|
||||||
|
|
||||||
c.execute('CREATE TABLE IF NOT EXISTS members (' +
|
c.execute('CREATE TABLE IF NOT EXISTS members (' +
|
||||||
', '.join([f'"{k}" {v.get("type", "")}'
|
', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
|
||||||
for k, v in membersTableColumns.items()]) +
|
for k, v in tableMapping['members'].items()]) +
|
||||||
')')
|
');')
|
||||||
|
|
||||||
c.execute("""CREATE TABLE IF NOT EXISTS transactions (
|
c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
|
||||||
sid TEXT PRIMARY KEY,
|
', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
|
||||||
timestamp INTEGER NOT NULL,
|
for k, v in tableMapping['transactions'].items()]) +
|
||||||
type INTEGER NOT NULL,
|
""", FOREIGN KEY(uid) REFERENCES member(uid)
|
||||||
currency TEXT NOT NULL,
|
|
||||||
sum INTEGER NOT NULL,
|
|
||||||
fee INTEGER,
|
|
||||||
uid TEXT NOT NULL,
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
event_id TEXT,
|
|
||||||
ttl TEXT,
|
|
||||||
FOREIGN KEY(uid) REFERENCES member(uid)
|
|
||||||
-- FOREIGN KEY(event_id) REFERENCES event(eid)
|
-- FOREIGN KEY(event_id) REFERENCES event(eid)
|
||||||
);
|
);
|
||||||
""")
|
""")
|
||||||
|
|
||||||
# TODO: define a pk constraint
|
|
||||||
c.execute("""CREATE TABLE IF NOT EXISTS labels (
|
c.execute("""CREATE TABLE IF NOT EXISTS labels (
|
||||||
uid TEXT, label Text,
|
uid TEXT, label Text,
|
||||||
FOREIGN KEY(uid) REFERENCES member(uid),
|
PRIMARY KEY(uid, label),
|
||||||
PRIMARY KEY(uid, label)
|
FOREIGN KEY(uid) REFERENCES member(uid)
|
||||||
);
|
);""")
|
||||||
""")
|
|
||||||
|
|
||||||
|
|
||||||
members = membershipworks.get_all_members()
|
members = membershipworks.get_all_members()
|
||||||
@ -178,10 +79,10 @@ with sqlite3.connect('membershipworks.db') as conn:
|
|||||||
if field.get('typ') == 8 and field['lbl'] in m: # check box
|
if field.get('typ') == 8 and field['lbl'] in m: # check box
|
||||||
m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
|
m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
|
||||||
|
|
||||||
insertMembers(members)
|
insertFromTableMap('members', members, tableMapping['members'])
|
||||||
insertLabels(members)
|
insertLabels(members)
|
||||||
|
|
||||||
transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
|
transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
|
||||||
insertTransactions(transactions)
|
insertFromTableMap('transactions', transactions, tableMapping['transactions'])
|
||||||
|
|
||||||
# TODO: folders, levels, addons
|
# TODO: folders, levels, addons
|
||||||
|
|||||||
81
tableMapping.yaml
Normal file
81
tableMapping.yaml
Normal file
@ -0,0 +1,81 @@
|
|||||||
|
members:
|
||||||
|
'uid': {type: 'TEXT PRIMARY KEY', source: 'Account ID'}
|
||||||
|
'Year of Birth':
|
||||||
|
'Account Name':
|
||||||
|
'First Name':
|
||||||
|
'Last Name':
|
||||||
|
'Phone':
|
||||||
|
'Email':
|
||||||
|
|
||||||
|
'Address (Street)':
|
||||||
|
'Address (City)':
|
||||||
|
'Address (State/Province)':
|
||||||
|
'Address (Postal Code)':
|
||||||
|
'Address (Country)':
|
||||||
|
'Profile description':
|
||||||
|
'Website':
|
||||||
|
'Fax':
|
||||||
|
'Contact Person':
|
||||||
|
'Password':
|
||||||
|
'Position/relation':
|
||||||
|
|
||||||
|
'Parent Account ID':
|
||||||
|
|
||||||
|
'Gift Membership purchased by':
|
||||||
|
'Purchased Gift Membership for':
|
||||||
|
|
||||||
|
'Closet Storage #':
|
||||||
|
'Storage Shelf #':
|
||||||
|
'Personal Studio Space #':
|
||||||
|
|
||||||
|
'Access Permitted Shops During Extended Hours?': {type: 'BOOLEAN'}
|
||||||
|
'Access Front Door and Studio Space During Extended Hours?': {type: 'BOOLEAN'}
|
||||||
|
'Access Wood Shop?': {type: 'BOOLEAN'}
|
||||||
|
'Access Metal Shop?': {type: 'BOOLEAN'}
|
||||||
|
'Access Storage Closet?': {type: 'BOOLEAN'}
|
||||||
|
'Access Studio Space?': {type: 'BOOLEAN'}
|
||||||
|
'Access Front Door?': {type: 'BOOLEAN'}
|
||||||
|
'Access Card Number': {type: 'INTEGER'}
|
||||||
|
'Access Card Facility Code': {type: 'INTEGER'}
|
||||||
|
|
||||||
|
'Auto Billing ID':
|
||||||
|
'Billing Method':
|
||||||
|
'Renewal Date':
|
||||||
|
'Join Date':
|
||||||
|
|
||||||
|
'Admin note':
|
||||||
|
|
||||||
|
'Profile gallery image URL':
|
||||||
|
'Business card image URL':
|
||||||
|
'Instagram':
|
||||||
|
'Pinterest':
|
||||||
|
'Youtube':
|
||||||
|
'Yelp':
|
||||||
|
'Google+':
|
||||||
|
'BBB':
|
||||||
|
'Twitter':
|
||||||
|
'Facebook':
|
||||||
|
'LinkedIn':
|
||||||
|
|
||||||
|
'Do not show street address in profile':
|
||||||
|
'Do not list in directory':
|
||||||
|
|
||||||
|
'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:':
|
||||||
|
'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.':
|
||||||
|
'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.':
|
||||||
|
'Waiver form signed and on file date.':
|
||||||
|
'Membership Agreement signed and on file date.':
|
||||||
|
|
||||||
|
'IP Address':
|
||||||
|
|
||||||
|
transactions:
|
||||||
|
'sid': {type: TEXT PRIMARY KEY}
|
||||||
|
'timestamp': {type: INTEGER, source: '_dp'}
|
||||||
|
'type': {type: INTEGER, source: 'typ'} # transaction type
|
||||||
|
'currency': {source: 'cur'}
|
||||||
|
'sum': {type: INTEGER}
|
||||||
|
'fee': {type: INTEGER}
|
||||||
|
'uid':
|
||||||
|
'name': {source: 'nam'}
|
||||||
|
'event_id': {source: 'eid'}
|
||||||
|
'ttl': # 'For'
|
||||||
Reference in New Issue
Block a user