sqlExport: Use FROM DUAL in insertDistinct to fix older mariadb

This reverts commit 6c862b9fb41f863cfc5d0ef4ffc863c4fe43d9fd.

Turns out that doesn't work, and mariadb was just silently failing.
Reopens #2.

README.md

@@ -27,3 +27,7 @@ Retrieves member information from MembershipWorks and pushes it out to the HID E
 ## `ucsAccounts.py`
 
 Retrieves member information from MembershipWorks and pushes it out to [UCS](https://www.univention.com/products/ucs/), which we use as a domain controller for the Windows computers at the Space.
+
+## `sqlExport.py`
+
+Retrieves account and transaction information from MembershipWorks, and pushes it to a MariaDB database for use in other projects. Schemas are defined (somewhat oddly, to be fair) in `tableMapping.yaml`.

sqlExport.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+import MySQLdb
+import yaml
+
+from common import membershipworks
+from passwords import MEMBERSHIPWORKS_DB
+
+def resolveSource(key, value):
+    if type(value) == str:
+        return value
+    elif type(value) == dict and 'source' in value:
+        return value['source']
+    else:
+        return key
+
+def formatRows(tableMap, data):
+    for d in data:
+        yield [d.get(resolveSource(k, v)) for k, v in tableMap.items()]
+
+def insertFromTableMap(table, data, tableMap):
+    # TODO: this could probably be done better as a single statement?
+    c.executemany(
+        f"""INSERT INTO {table} ({','.join(f'`{k}`' for k in tableMap.keys())})
+        VALUES ({','.join(len(tableMap) * ['%s'])})
+        ON DUPLICATE KEY UPDATE
+        {', '.join(f'`{k}`=VALUES(`{k}`)' for k in tableMap.keys())};""",
+        list(formatRows(tableMap, data)))
+
+def insertDistinctFromTableMap(table, data, tableMap):
+    # TODO: this could probably be done better as a single statement?
+    c.executemany(
+       f"""INSERT INTO {table} ({','.join(f'`{k}`' for k in tableMap.keys())})
+        SELECT {','.join(len(tableMap) * ['%s'])} FROM DUAL
+        WHERE NOT EXISTS (
+            SELECT 1 from {table}
+            WHERE {' AND '.join(f'`{k}`<=>%s' for k in tableMap.keys())}
+        );""" ,
+        list([r * 2 for r in formatRows(tableMap, data)]))
+
+
+# TODO: delete non-valid labels
+def insertLabels(members):
+    for member in members:
+        for label, label_id in membershipworks._parse_flags()['labels'].items():
+            if member[label]:
+                c.execute("""
+                INSERT INTO member_labels (uid, label_id) VALUES (%s, %s)
+                ON DUPLICATE KEY UPDATE
+                  uid=VALUES(uid), label_id=VALUES(label_id);""",
+                          (member['Account ID'], label_id))
+            else:
+                c.execute('DELETE FROM member_labels WHERE uid=%s && label_id=%s;',
+                          (member['Account ID'], label_id))
+
+
+with open('tableMapping.yaml') as f:
+    tableMapping = yaml.load(f, yaml.SafeLoader)
+
+
+conn =  MySQLdb.connect(**MEMBERSHIPWORKS_DB)
+conn.set_character_set('utf8')
+c = conn.cursor()
+
+def createDefinitionsFromTableMap(tableMap):
+    def resolveColType(value):
+        if type(value) == dict and 'type' in value:
+            return value['type']
+        else:
+            return 'TEXT'
+
+    return ', '.join([f'`{k}` ' + resolveColType(v)
+                      for k, v in tableMap.items()])
+
+try:
+    print("Creating tables...")
+    c.execute('CREATE TABLE IF NOT EXISTS members (' +
+              createDefinitionsFromTableMap(tableMapping['members']) +
+              ') WITH SYSTEM VERSIONING;')
+
+    c.execute("CREATE TABLE IF NOT EXISTS transactions (" +
+              createDefinitionsFromTableMap(tableMapping['transactions']) +
+              ", CONSTRAINT `fk_member_uid` FOREIGN KEY (uid) REFERENCES members(uid));")
+              #-- FOREIGN KEY event_id REFERENCES event eid
+
+    c.execute("""CREATE TABLE IF NOT EXISTS labels (
+    label_id CHAR(24) PRIMARY KEY, label TEXT
+    ) WITH SYSTEM VERSIONING;""")
+
+    c.execute("""CREATE TABLE IF NOT EXISTS member_labels (
+    uid CHAR(24), label_id CHAR(24),
+    PRIMARY KEY(uid, label_id),
+    FOREIGN KEY(uid) REFERENCES members(uid),
+    FOREIGN KEY(label_id) REFERENCES labels(label_id)
+    ) WITH SYSTEM VERSIONING;""")
+
+    print("Updating labels")
+    c.executemany("""INSERT INTO labels (label, label_id) VALUES (%s, %s)
+    ON DUPLICATE KEY UPDATE
+    label=VALUES(label), label_id=VALUES(label_id);""",
+                  membershipworks._parse_flags()['labels'].items())
+
+    print("Getting/Updating members...")
+    members = membershipworks.get_all_members()
+    for m in members:
+        # replace flags by booleans
+        for flag in [dek['lbl'] for dek in membershipworks.org_info['dek']]:
+            if flag in m:
+                m[flag] = m[flag] == flag
+
+        for field_id, field in membershipworks._all_fields().items():
+            # convert checkboxes to real booleans
+            if field.get('typ') == 8 and field['lbl'] in m: # check box
+                m[field['lbl']] = True if m[field['lbl']] == 'Y' else False
+
+    insertFromTableMap('members', members, tableMapping['members'])
+
+    insertLabels(members)
+
+    print("Getting/Updating transactions...")
+    now = datetime.now()
+    transactions_csv = membershipworks.get_transactions(datetime(2010, 1, 1), now)
+    transactions_json = membershipworks.get_transactions(
+        datetime(2010, 1, 1), now, json=True)
+    # this is terrible, but as long as the dates are the same, should be fiiiine
+    transactions = [{**j, **v}
+                    for j, v in zip(transactions_csv, transactions_json)]
+    assert all([t['Account ID'] == t.get('uid', '')
+                and t['Payment ID'] == t.get('sid', '')
+                for t in transactions])
+    insertDistinctFromTableMap(
+        'transactions', transactions, tableMapping['transactions'])
+
+    print("Committing changes...")
+    conn.commit()
+
+except Exception as e:
+    print("Transaction failed, rolling back")
+    conn.rollback();
+    raise e
+
+finally:
+    conn.close()
+
+# TODO: folders, levels, addons

tableMapping.yaml

@@ -0,0 +1,102 @@
+members:
+  'uid': {type: 'CHAR(24) PRIMARY KEY', source: 'Account ID'}
+  'Year of Birth':
+  'Account Name':
+  'First Name':
+  'Last Name':
+  'Phone':
+  'Email':
+
+  'Address (Street)':
+  'Address (City)':
+  'Address (State/Province)':
+  'Address (Postal Code)':
+  'Address (Country)':
+  'Profile description':
+  'Website':
+  'Fax':
+  'Contact Person':
+  'Password':
+  'Position/relation':
+
+  'Parent Account ID':
+
+  'Gift Membership purchased by':
+  'Purchased Gift Membership for':
+
+  'Closet Storage #':
+  'Storage Shelf #':
+  'Personal Studio Space #':
+
+  'Access Permitted Shops During Extended Hours?': {type: 'BOOLEAN'}
+  'Access Front Door and Studio Space During Extended Hours?': {type: 'BOOLEAN'}
+  'Access Wood Shop?': {type: 'BOOLEAN'}
+  'Access Metal Shop?': {type: 'BOOLEAN'}
+  'Access Storage Closet?': {type: 'BOOLEAN'}
+  'Access Studio Space?': {type: 'BOOLEAN'}
+  'Access Front Door?': {type: 'BOOLEAN'}
+  'Access Card Number':
+  'Access Card Facility Code':
+
+  'Auto Billing ID':
+  'Billing Method':
+  'Renewal Date':
+  'Join Date':
+
+  'Admin note':
+
+  'Profile gallery image URL':
+  'Business card image URL':
+  'Instagram':
+  'Pinterest':
+  'Youtube':
+  'Yelp':
+  'Google+':
+  'BBB':
+  'Twitter':
+  'Facebook':
+  'LinkedIn':
+
+  'Do not show street address in profile':
+  'Do not list in directory':
+
+  'HowDidYouHear': 'Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:'
+  'authorizeCharge': 'Yes - I authorize TwinState MakerSpaces, Inc. to charge my credit card for the membership and other options that I have selected.'
+  'policyAgreement': 'I have read the Claremont MakerSpace Membership Agreement & Policies, and agree to all terms stated therein.'
+  'Waiver form signed and on file date.':
+  'Membership Agreement signed and on file date.':
+  'Agreement Version':
+  'Paperwork status':
+  'Membership agreement dated': {type: BOOLEAN}
+  'Membership Agreement Acknowledgement Page Filled Out': {type: BOOLEAN}
+  'Membership Agreement Signed': {type: BOOLEAN}
+  'Liability Form Filled Out': {type: BOOLEAN}
+  'Audit Date':
+
+  'IP Address':
+
+transactions:
+  'sid': {type: CHAR(27)}
+  'uid': {type: CHAR(24)}
+  'timestamp': {type: 'INT(11)', source: '_dp'} # TODO: should be a real timestamp?
+  'type': {source: 'Transaction Type'}
+  'sum': {type: 'DECIMAL(13,4)'}
+  'fee': {type: 'DECIMAL(13,4)'}
+  'event_id': {source: 'eid'}
+  'For':
+  'Items':
+  'Discount Code':
+  'Note':
+
+  # this is painful, but necessary because some users have no uid
+  # TODO: fix this horribleness
+  'Name':
+  'Contact Person':
+  'Full Address':
+  'Street':
+  'City':
+  'State/Province':
+  'Postal Code':
+  'Country':
+  'Phone':
+  'Email':