sqlExport: WIP: Even worse fix for missing transactions
This commit is contained in:
parent
925e1abe80
commit
d5ecf50943
40
sqlExport.py
Normal file → Executable file
40
sqlExport.py
Normal file → Executable file
@ -7,29 +7,39 @@ import yaml
|
||||
from common import membershipworks
|
||||
from passwords import MEMBERSHIPWORKS_DB
|
||||
|
||||
def resolveSource(key, value):
|
||||
if type(value) == str:
|
||||
return value
|
||||
elif type(value) == dict and 'source' in value:
|
||||
return value['source']
|
||||
else:
|
||||
return key
|
||||
|
||||
def formatRows(tableMap, data):
|
||||
for d in data:
|
||||
yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
|
||||
|
||||
def insertFromTableMap(table, data, tableMap):
|
||||
def resolveSource(key, value):
|
||||
if type(value) == str:
|
||||
return value
|
||||
elif type(value) == dict and 'source' in value:
|
||||
return value['source']
|
||||
else:
|
||||
return key
|
||||
|
||||
def formatRows(data):
|
||||
for d in data:
|
||||
yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
|
||||
|
||||
# TODO: this could probably be done better as a single statement?
|
||||
# note: this only works if in python >= 3.7 where `dict` is ordered
|
||||
c.executemany(
|
||||
'INSERT INTO ' + table + ' (' +
|
||||
','.join(f'`{k}`' for k in tableMap.keys()) +
|
||||
') VALUES (' + ','.join(len(tableMap) * ['%s']) + ') ' +
|
||||
'ON DUPLICATE KEY UPDATE ' +
|
||||
', '.join(f'`{k}`=VALUES(`{k}`)' for k in tableMap.keys()) + ';',
|
||||
list(formatRows(data)))
|
||||
list(formatRows(tableMap, data)))
|
||||
|
||||
def insertFromTableMapWithoutDups(table, data, tableMap):
|
||||
# TODO: this could probably be done better as a single statement?
|
||||
c.executemany(
|
||||
'INSERT INTO ' + table + ' (' +
|
||||
','.join(f'`{k}`' for k in tableMap.keys()) +
|
||||
') SELECT ' + ','.join(len(tableMap) * ['%s']) +
|
||||
' WHERE NOT EXISTS (SELECT 1 from ' + table +
|
||||
' WHERE ' + ' AND '.join(f'`{k}`<=>%s' for k in tableMap.keys()) +
|
||||
');' ,
|
||||
list([r * 2 for r in formatRows(tableMap, data)]))
|
||||
|
||||
|
||||
# TODO: delete non-valid labels
|
||||
def insertLabels(members):
|
||||
@ -120,7 +130,7 @@ try:
|
||||
assert all([t['Account ID'] == t.get('uid', '')
|
||||
and t['Payment ID'] == t.get('sid', '')
|
||||
for t in transactions])
|
||||
insertFromTableMap('transactions', transactions, tableMapping['transactions'])
|
||||
insertFromTableMapWithoutDups('transactions', transactions, tableMapping['transactions'])
|
||||
|
||||
print("Committing changes...")
|
||||
conn.commit()
|
||||
|
@ -78,8 +78,7 @@ members:
|
||||
transactions:
|
||||
'sid': {type: CHAR(27)}
|
||||
'uid': {type: CHAR(24)}
|
||||
# TODO: this is a terrible PK
|
||||
'timestamp': {type: 'INT(11) PRIMARY KEY', source: '_dp'} # TODO: should be a real timestamp?
|
||||
'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp?
|
||||
'type': {source: 'Transaction Type'}
|
||||
'sum': {type: 'DECIMAL(13,4)'}
|
||||
'fee': {type: 'DECIMAL(13,4)'}
|
||||
|
Reference in New Issue
Block a user