From d5ecf50943e1a848dabc6f834b13f45cdf90bb9c Mon Sep 17 00:00:00 2001
From: Adam Goldsmith <adam@adamgoldsmith.name>
Date: Wed, 1 Apr 2020 17:36:06 -0400
Subject: [PATCH] sqlExport: WIP: Even worse fix for missing transactions

---
 sqlExport.py      | 40 +++++++++++++++++++++++++---------------
 tableMapping.yaml |  3 +--
 2 files changed, 26 insertions(+), 17 deletions(-)
 mode change 100644 => 100755 sqlExport.py

diff --git a/sqlExport.py b/sqlExport.py
old mode 100644
new mode 100755
index 9f294fb..2e37f51
--- a/sqlExport.py
+++ b/sqlExport.py
@@ -7,29 +7,39 @@ import yaml
 from common import membershipworks
 from passwords import MEMBERSHIPWORKS_DB
 
+def resolveSource(key, value):
+    if type(value) == str:
+        return value
+    elif type(value) == dict and 'source' in value:
+        return value['source']
+    else:
+        return key
+
+def formatRows(tableMap, data):
+    for d in data:
+        yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
 
 def insertFromTableMap(table, data, tableMap):
-    def resolveSource(key, value):
-        if type(value) == str:
-            return value
-        elif type(value) == dict and 'source' in value:
-            return value['source']
-        else:
-            return key
-
-    def formatRows(data):
-        for d in data:
-            yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
-
     # TODO: this could probably be done better as a single statement?
-    # note: this only works if in python >= 3.7 where `dict` is ordered
     c.executemany(
         'INSERT INTO ' + table + ' (' +
         ','.join(f'`{k}`' for k in tableMap.keys()) +
         ') VALUES (' + ','.join(len(tableMap) * ['%s']) + ') ' +
         'ON DUPLICATE KEY UPDATE ' +
         ', '.join(f'`{k}`=VALUES(`{k}`)' for k in tableMap.keys()) + ';',
-        list(formatRows(data)))
+        list(formatRows(tableMap, data)))
+
+def insertFromTableMapWithoutDups(table, data, tableMap):
+    # TODO: this could probably be done better as a single statement?
+    c.executemany(
+        'INSERT INTO ' + table + ' (' +
+        ','.join(f'`{k}`' for k in tableMap.keys()) +
+        ') SELECT ' + ','.join(len(tableMap) * ['%s']) +
+        ' WHERE NOT EXISTS (SELECT 1 from ' + table +
+        ' WHERE ' + ' AND '.join(f'`{k}`<=>%s' for k in tableMap.keys()) +
+        ');' ,
+        list([r * 2 for r in formatRows(tableMap, data)]))
+
 
 # TODO: delete non-valid labels
 def insertLabels(members):
@@ -120,7 +130,7 @@ try:
     assert all([t['Account ID'] == t.get('uid', '')
                 and t['Payment ID'] == t.get('sid', '')
                 for t in transactions])
-    insertFromTableMap('transactions', transactions, tableMapping['transactions'])
+    insertFromTableMapWithoutDups('transactions', transactions, tableMapping['transactions'])
 
     print("Committing changes...")
     conn.commit()
diff --git a/tableMapping.yaml b/tableMapping.yaml
index 0ad28c9..11369f7 100644
--- a/tableMapping.yaml
+++ b/tableMapping.yaml
@@ -78,8 +78,7 @@ members:
 transactions:
   'sid': {type: CHAR(27)}
   'uid': {type: CHAR(24)}
-  # TODO: this is a terrible PK
-  'timestamp': {type: 'INT(11) PRIMARY KEY', source: '_dp'} # TODO: should be a real timestamp?
+  'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp?
   'type': {source: 'Transaction Type'}
   'sum': {type: 'DECIMAL(13,4)'}
   'fee': {type: 'DECIMAL(13,4)'}