sqlExport: WIP: Even worse fix for missing transactions

sqlExport.py

@@ -7,29 +7,39 @@ import yaml
 from common import membershipworks
 from passwords import MEMBERSHIPWORKS_DB
 
+def resolveSource(key, value):
+    if type(value) == str:
+        return value
+    elif type(value) == dict and 'source' in value:
+        return value['source']
+    else:
+        return key
+
+def formatRows(tableMap, data):
+    for d in data:
+        yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
 
 def insertFromTableMap(table, data, tableMap):
-    def resolveSource(key, value):
-        if type(value) == str:
-            return value
-        elif type(value) == dict and 'source' in value:
-            return value['source']
-        else:
-            return key
-
-    def formatRows(data):
-        for d in data:
-            yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
-
     # TODO: this could probably be done better as a single statement?
-    # note: this only works if in python >= 3.7 where `dict` is ordered
     c.executemany(
         'INSERT INTO ' + table + ' (' +
         ','.join(f'`{k}`' for k in tableMap.keys()) +
         ') VALUES (' + ','.join(len(tableMap) * ['%s']) + ') ' +
         'ON DUPLICATE KEY UPDATE ' +
         ', '.join(f'`{k}`=VALUES(`{k}`)' for k in tableMap.keys()) + ';',
-        list(formatRows(data)))
+        list(formatRows(tableMap, data)))
+
+def insertFromTableMapWithoutDups(table, data, tableMap):
+    # TODO: this could probably be done better as a single statement?
+    c.executemany(
+        'INSERT INTO ' + table + ' (' +
+        ','.join(f'`{k}`' for k in tableMap.keys()) +
+        ') SELECT ' + ','.join(len(tableMap) * ['%s']) +
+        ' WHERE NOT EXISTS (SELECT 1 from ' + table +
+        ' WHERE ' + ' AND '.join(f'`{k}`<=>%s' for k in tableMap.keys()) +
+        ');' ,
+        list([r * 2 for r in formatRows(tableMap, data)]))
+
 
 # TODO: delete non-valid labels
 def insertLabels(members):
@@ -120,7 +130,7 @@ try:
     assert all([t['Account ID'] == t.get('uid', '')
                 and t['Payment ID'] == t.get('sid', '')
                 for t in transactions])
-    insertFromTableMap('transactions', transactions, tableMapping['transactions'])
+    insertFromTableMapWithoutDups('transactions', transactions, tableMapping['transactions'])
 
     print("Committing changes...")
     conn.commit()

tableMapping.yaml

@@ -78,8 +78,7 @@ members:
 transactions:
   'sid': {type: CHAR(27)}
   'uid': {type: CHAR(24)}
-  # TODO: this is a terrible PK
+  'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp?
-  'timestamp': {type: 'INT(11) PRIMARY KEY', source: '_dp'} # TODO: should be a real timestamp?
   'type': {source: 'Transaction Type'}
   'sum': {type: 'DECIMAL(13,4)'}
   'fee': {type: 'DECIMAL(13,4)'}