sqlExport: Migrate to mariadb, improve types

sqlExport.py

@@ -1,71 +1,82 @@
 #!/usr/bin/env python3
 
 from datetime import datetime
-import sqlite3
+import MySQLdb
 import yaml
 
-#from common import membershipworks
+from common import membershipworks
 
-# TODO: TEMP. remove later:
-#from lib.MembershipWorks import MembershipWorks
-from passwords import MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD
-membershipworks = MembershipWorks()
-membershipworks.login(MEMBERSHIPWORKS_USERNAME, MEMBERSHIPWORKS_PASSWORD)
 
 def insertFromTableMap(table, data, tableMap):
+    def resolveSource(key, value):
+        if type(value) == str:
+            return value
+        elif type(value) == dict and 'source' in value:
+            return value['source']
+        else:
+            return key
+
     def formatRows(data):
         for d in data:
-            yield [
+            yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
-                d.get(v.get('source', k) if v is not None else k) for k, v in tableMap.items()
-            ]
 
-    c.execute('BEGIN')
+    # TODO: this could probably be done better as a single statement?
-    # NOTE: this only works if in python >= 3.7 where `dict` is ordered
+    # note: this only works if in python >= 3.7 where `dict` is ordered
-    c.executemany('INSERT OR REPLACE INTO ' + table + ' (' +
+    c.executemany('REPLACE INTO ' + table + ' (' +
-                  ','.join('"' + k + '"' for k in tableMap.keys()) +
+                  ','.join(f'`{k}`' for k in tableMap.keys()) +
-                  ') values (' +
+                  ') VALUES (' +
-                  ','.join(len(tableMap) * '?') +
+                  ','.join(len(tableMap) * ['%s']) +
                   ');',
                   list(formatRows(data)))
-    c.execute('END')
 
+# TODO: delete non-valid labels
 def insertLabels(members):
     def formatLabels(labels):
         for member in members:
-            for label in membershipworks._parse_flags()['labels']:
+            for label, label_id in membershipworks._parse_flags()['labels'].items():
                 if member[label]:
-                    yield member['Account ID'], label
+                    yield member['Account ID'], label_id
 
-    c.execute('BEGIN')
+    c.executemany('REPLACE INTO member_labels (uid, label_id) VALUES (%s, %s);',
-    c.executemany('INSERT OR REPLACE INTO labels (uid, label) values (?, ?);',
                   list(formatLabels(members)))
-    c.execute('END')
 
 with open('tableMapping.yaml') as f:
     tableMapping = yaml.load(f, yaml.SafeLoader)
 
-with sqlite3.connect('membershipworks.db') as conn:
-    c = conn.cursor()
 
+conn =  MySQLdb.connect(user='membershipworks', password='asdf', database='membershipworks')
+c = conn.cursor()
+
+def createDefinitionsFromTableMap(tableMap):
+    def resolveColType(value):
+        if type(value) == dict and 'type' in value:
+            return value['type']
+        else:
+            return 'TEXT'
+
+    return ', '.join([f'`{k}` ' + resolveColType(v)
+                      for k, v in tableMap.items()])
+
+try:
     c.execute('CREATE TABLE IF NOT EXISTS members (' +
-              ', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
+              createDefinitionsFromTableMap(tableMapping['members']) +
-                         for k, v in tableMapping['members'].items()]) +
+              ') WITH SYSTEM VERSIONING;')
-              ');')
 
     c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
-              ', '.join([f'"{k}" {v.get("type", "") if v is not None else ""}'
+              createDefinitionsFromTableMap(tableMapping['transactions']) +
-                         for k, v in tableMapping['transactions'].items()]) +
+              ", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));")
-              """, FOREIGN KEY(uid) REFERENCES member(uid)
+              #-- FOREIGN KEY event_id REFERENCES event eid
-              -- FOREIGN KEY(event_id) REFERENCES event(eid)
-              );
-              """)
 
     c.execute("""CREATE TABLE IF NOT EXISTS labels (
-    uid TEXT, label Text,
+    label_id CHAR(24) PRIMARY KEY, label TEXT
-    PRIMARY KEY(uid, label),
+    ) WITH SYSTEM VERSIONING;""")
-    FOREIGN KEY(uid) REFERENCES member(uid)
-    );""")
 
+    c.execute("""CREATE TABLE IF NOT EXISTS member_labels (
+    uid CHAR(24), label_id CHAR(24),
+    PRIMARY KEY(uid, label_id),
+    FOREIGN KEY(uid) REFERENCES members(uid),
+    FOREIGN KEY(label_id) REFERENCES labels(label_id)
+    ) WITH SYSTEM VERSIONING;""")
 
     members = membershipworks.get_all_members()
     for m in members:
@@ -79,10 +90,24 @@ with sqlite3.connect('membershipworks.db') as conn:
             if field.get('typ') == 8 and field['lbl'] in m: # check box
                 m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
 
+    c.executemany("REPLACE INTO labels (label, label_id) VALUES (%s, %s)",
+                  membershipworks._parse_flags()['labels'].items())
+
     insertFromTableMap('members', members, tableMapping['members'])
+
     insertLabels(members)
 
     transactions = membershipworks.get_transactions(datetime(2020, 1, 1), datetime.now())
     insertFromTableMap('transactions', transactions, tableMapping['transactions'])
 
-    # TODO: folders, levels, addons
+    conn.commit()
+
+except Exception as e:
+    print("Transaction failed, rolling back")
+    conn.rollback();
+    raise e
+
+finally:
+    conn.close()
+
+# TODO: folders, levels, addons

tableMapping.yaml

@@ -1,5 +1,5 @@
 members:
-  'uid': {type: 'TEXT PRIMARY KEY', source: 'Account ID'}
+  'uid': {type: 'CHAR(24) PRIMARY KEY', source: 'Account ID'}
   'Year of Birth':
   'Account Name':
   'First Name':
@@ -35,8 +35,8 @@ members:
   'Access Storage Closet?': {type: 'BOOLEAN'}
   'Access Studio Space?': {type: 'BOOLEAN'}
   'Access Front Door?': {type: 'BOOLEAN'}
-  'Access Card Number': {type: 'INTEGER'}
+  'Access Card Number':
-  'Access Card Facility Code': {type: 'INTEGER'}
+  'Access Card Facility Code':
 
   'Auto Billing ID':
   'Billing Method':
@@ -60,22 +60,22 @@ members:
   'Do not show street address in profile':
   'Do not list in directory':
 
-  'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:':
+  'HowDidYouHear': 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:'
-  'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.':
+  'authorizeCharge': 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.'
-  'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.':
+  'policyAgreement': 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.'
   'Waiver form signed and on file date.':
   'Membership Agreement signed and on file date.':
 
   'IP Address':
 
 transactions:
-  'sid': {type: TEXT PRIMARY KEY}
+  'sid': {type: CHAR(27) PRIMARY KEY}
-  'timestamp': {type: INTEGER, source: '_dp'}
+  'uid': {type: CHAR(24)}
+  'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp?
   'type': {type: INTEGER, source: 'typ'} # transaction type
   'currency': {source: 'cur'}
-  'sum': {type: INTEGER}
+  'sum': {type: 'DECIMAL(13,4)'}
-  'fee': {type: INTEGER}
+  'fee': {type: 'DECIMAL(13,4)'}
-  'uid':
   'name': {source: 'nam'}
   'event_id': {source: 'eid'}
   'ttl': # 'For'