steve.goldsmith/memberPlumbing
1
0
Fork 0
forked from CMS/memberPlumbing
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sqlExport: WIP: Even worse fix for missing transactions

Browse Source
This commit is contained in:
Adam Goldsmith 2020-04-01 17:36:06 -04:00
parent 925e1abe80
commit d5ecf50943
2 changed files with 26 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
sqlExport.py Normal file → Executable file
View File

@ -7,9 +7,7 @@ import yaml
from common import membershipworks from common import membershipworks
from passwords import MEMBERSHIPWORKS_DB from passwords import MEMBERSHIPWORKS_DB
def resolveSource(key, value):
def insertFromTableMap(table, data, tableMap):
def resolveSource(key, value):
if type(value) == str: if type(value) == str:
return value return value
elif type(value) == dict and 'source' in value: elif type(value) == dict and 'source' in value:
@ -17,19 +15,31 @@ def insertFromTableMap(table, data, tableMap):
else: else:
return key return key
def formatRows(data): def formatRows(tableMap, data):
for d in data: for d in data:
yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()] yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
def insertFromTableMap(table, data, tableMap):
# TODO: this could probably be done better as a single statement? # TODO: this could probably be done better as a single statement?
# note: this only works if in python >= 3.7 where `dict` is ordered
c.executemany( c.executemany(
'INSERT INTO ' + table + ' (' + 'INSERT INTO ' + table + ' (' +
','.join(f'`{k}`' for k in tableMap.keys()) + ','.join(f'`{k}`' for k in tableMap.keys()) +
') VALUES (' + ','.join(len(tableMap) * ['%s']) + ') ' + ') VALUES (' + ','.join(len(tableMap) * ['%s']) + ') ' +
'ON DUPLICATE KEY UPDATE ' + 'ON DUPLICATE KEY UPDATE ' +
', '.join(f'`{k}`=VALUES(`{k}`)' for k in tableMap.keys()) + ';', ', '.join(f'`{k}`=VALUES(`{k}`)' for k in tableMap.keys()) + ';',
list(formatRows(data))) list(formatRows(tableMap, data)))
def insertFromTableMapWithoutDups(table, data, tableMap):
# TODO: this could probably be done better as a single statement?
c.executemany(
'INSERT INTO ' + table + ' (' +
','.join(f'`{k}`' for k in tableMap.keys()) +
') SELECT ' + ','.join(len(tableMap) * ['%s']) +
' WHERE NOT EXISTS (SELECT 1 from ' + table +
' WHERE ' + ' AND '.join(f'`{k}`<=>%s' for k in tableMap.keys()) +
');' ,
list([r * 2 for r in formatRows(tableMap, data)]))
# TODO: delete non-valid labels # TODO: delete non-valid labels
def insertLabels(members): def insertLabels(members):
@ -120,7 +130,7 @@ try:
assert all([t['Account ID'] == t.get('uid', '') assert all([t['Account ID'] == t.get('uid', '')
and t['Payment ID'] == t.get('sid', '') and t['Payment ID'] == t.get('sid', '')
for t in transactions]) for t in transactions])
insertFromTableMap('transactions', transactions, tableMapping['transactions']) insertFromTableMapWithoutDups('transactions', transactions, tableMapping['transactions'])
print("Committing changes...") print("Committing changes...")
conn.commit() conn.commit()

3
tableMapping.yaml
View File

@ -78,8 +78,7 @@ members:
transactions: transactions:
'sid': {type: CHAR(27)} 'sid': {type: CHAR(27)}
'uid': {type: CHAR(24)} 'uid': {type: CHAR(24)}
# TODO: this is a terrible PK 'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp?
'timestamp': {type: 'INT(11) PRIMARY KEY', source: '_dp'} # TODO: should be a real timestamp?
'type': {source: 'Transaction Type'} 'type': {source: 'Transaction Type'}
'sum': {type: 'DECIMAL(13,4)'} 'sum': {type: 'DECIMAL(13,4)'}
'fee': {type: 'DECIMAL(13,4)'} 'fee': {type: 'DECIMAL(13,4)'}