Add django-postgres-metrics

MariaDB has become far too annoying/buggy, and there are some neat
features only available in PostgreSQL

.gitea/workflows/test.yml

@@ -9,21 +9,12 @@ jobs:
     runs-on: ubuntu-latest
     container: catthehacker/ubuntu:act-latest
     services:
-      mariadb:
+      postgres:
-        # TODO: this is pinned to avoid what apears to be a bug with
+        image: postgres:15
-        # MariaDB >= 10.11.9, and collation issues with 11.x.x
-        image: mariadb:10.11.8
         env:
-          MARIADB_ROOT_PASSWORD: whatever
+          POSTGRES_PASSWORD: whatever
         healthcheck:
-          test:
+          test: ["CMD-SHELL", "pg_isready"]
-            [
-              "CMD",
-              "healthcheck.sh",
-              "--su-mysql",
-              "--connect",
-              "--innodb_initialized",
-            ]
     steps:
       - uses: actions/checkout@v4
       - name: Setup PDM
@@ -35,7 +26,7 @@ jobs:
       - name: Install apt dependencies
         run: >-
           sudo apt-get update &&
-          sudo apt-get -y install build-essential python3-dev libldap2-dev libsasl2-dev mariadb-client
+          sudo apt-get -y install build-essential python3-dev libldap2-dev libsasl2-dev
       - name: Install python dependencies
         run: pdm sync -d -G dev
 

cmsmanage/settings.py

@@ -28,11 +28,16 @@ class Base(Configuration):
     @classmethod
     def setup(cls):
         super().setup()
-        cls.DATABASES["default"]["OPTIONS"] = {"charset": "utf8mb4"}
+
+        # TODO: this is a nasty hack, since the connection
+        #       pool doesn't seem to work well with django-q2
+        if "qcluster" not in sys.argv:
+            cls.DATABASES["default"]["OPTIONS"] = {"pool": True}
 
     INSTALLED_APPS = [
         "dal",
         "dal_select2",
+        "postgres_metrics.apps.PostgresMetrics",
         "django.contrib.admin",
         "django.contrib.auth",
         "django.contrib.contenttypes",
@@ -52,9 +57,9 @@ class Base(Configuration):
         "django_tables2",
         "django_filters",
         "django_db_views",
-        "django_mysql",
         "django_sendfile",
         "django_bootstrap5",
+        "simple_history",
         # "tasks.apps.TasksConfig",
         "rentals.apps.RentalsConfig",
         "membershipworks.apps.MembershipworksConfig",
@@ -106,9 +111,6 @@ class Base(Configuration):
 
     WSGI_APPLICATION = "cmsmanage.wsgi.application"
 
-    # mysql.W003 (unique CharField length) is irrelevant on MariaDB >= 10.4.3
-    SILENCED_SYSTEM_CHECKS = ["mysql.W003"]
-
     # Default URL to redirect to after authentication
     LOGIN_REDIRECT_URL = "/"
     LOGIN_URL = "/auth/login/"
@@ -213,6 +215,9 @@ class Base(Configuration):
     # CMSManage specific stuff
     WIKI_URL = values.URLValue("https://wiki.claremontmakerspace.org")
 
+    # ID of flag for Members folder in MembershipWorks
+    MW_MEMBERS_FOLDER_ID = "5771675edcdf126302a2f6b9"
+
 
 class NonCIBase(Base):
     """required for all but CI"""
@@ -367,13 +372,10 @@ class CI(Base):
 
     DATABASES = {
         "default": {
-            "ENGINE": "django.db.backends.mysql",
+            "ENGINE": "django.db.backends.postgresql",
-            "HOST": "mariadb",
+            "HOST": "postgres",
-            "NAME": "CMS_Database",
+            "NAME": "cms",
-            "USER": "root",
+            "USER": "postgres",
             "PASSWORD": "whatever",
-            "OPTIONS": {
-                "charset": "utf8mb4",
-            },
         }
     }

cmsmanage/urls.py

@@ -39,6 +39,7 @@ urlpatterns = [
     path("paperwork/", include("paperwork.urls")),
     path("doorcontrol/", include("doorcontrol.urls")),
     path("api/v1/", include((router.urls, "api"), namespace="v1")),
+    path("admin/postgres-metrics/", include("postgres_metrics.urls")),
     path("admin/", admin.site.urls),
     path(
         "auth/",

doorcontrol/admin.py

@@ -44,9 +44,6 @@ class HIDEventAdmin(DjangoObjectActions, admin.ModelAdmin):
     readonly_fields = ["decoded_card_number"]
     changelist_actions = ("refresh_all_doors",)
 
-    def get_queryset(self, request):
-        return super().get_queryset(request).with_decoded_card_number()
-
     @admin.display(boolean=True)
     def _is_red(self, obj):
         return obj.is_red

doorcontrol/hid/Credential.py

@@ -1,41 +1,62 @@
+import dataclasses
+from typing import Literal
+
 import bitstring
 
 # Reference for H10301 card format:
 # https://www.hidglobal.com/sites/default/files/hid-understanding_card_data_formats-wp-en.pdf
 
 
+class InvalidHexCode(Exception):
+    pass
+
+
+class Not26Bit(InvalidHexCode):
+    def __init__(self) -> None:
+        super().__init__("Card number > 26 bits")
+
+
+class InvalidParity(InvalidHexCode):
+    def __init__(self, even_odd: Literal["even", "odd"]) -> None:
+        super().__init__(f"Bad {even_odd} parity")
+
+
+@dataclasses.dataclass
 class Credential:
-    def __init__(self, code=None, hex_code=None):
+    bits: bitstring.Bits
-        if code is None and hex_code is None:
-            raise TypeError("Must set either code or hex for a Credential")
-        elif code is not None and hex_code is not None:
-            raise TypeError("Cannot set both code and hex for a Credential")
-        elif code is not None:
-            self.bits = bitstring.pack(
-                "0b000000, 0b0, uint:8=facility, uint:16=number, 0b0",
-                facility=code[0],
-                number=code[1],
-            )
-            self.bits[6] = self.bits[7:19].count(1) % 2  # even parity
-            self.bits[31] = not (self.bits[19:31].count(1) % 2)  # odd parity
-        elif hex_code is not None:
-            self.bits = bitstring.Bits(hex=hex_code)
 
-    def __repr__(self):
+    @classmethod
-        return f"Credential({self.code})"
+    def from_code(cls, facility=int, card_number=int) -> "Credential":
+        bits = bitstring.pack(
+            "0b000000, 0b0, uint:8=facility, uint:16=card_number, 0b0",
+            facility=facility,
+            card_number=card_number,
+        )
+        bits[6] = bits[7:19].count(1) % 2  # even parity
+        bits[31] = bits[19:31].count(0) % 2  # odd parity
+        return cls(bits)
 
-    def __eq__(self, other):
+    @classmethod
-        return self.bits == other.bits
+    def from_hex(cls, hex_code: str) -> "Credential":
+        bits = bitstring.Bits(hex=hex_code)
 
-    def __hash__(self):
+        if bits[:6].any(1):
-        return self.bits.int
+            raise Not26Bit
+        if bits[6] != bits[7:19].count(1) % 2:
+            raise InvalidParity("even")
+        if bits[31] != (bits[19:31].count(0) % 2):
+            raise InvalidParity("odd")
+
+        return cls(bits)
 
     @property
-    def code(self):
+    def facility_code(self) -> int:
-        facility = self.bits[7:15].uint
+        return self.bits[7:15].uint
-        code = self.bits[15:31].uint
-        return (facility, code)
 
     @property
-    def hex(self):
+    def card_number(self) -> int:
+        return self.bits[15:31].uint
+
+    @property
+    def hex(self) -> str:
         return self.bits.hex.upper()

doorcontrol/hid/DoorController.py

@@ -2,6 +2,7 @@ import contextlib
 import csv
 from datetime import datetime
 from io import StringIO
+from itertools import takewhile
 
 import requests
 import urllib3
@@ -33,6 +34,14 @@ class RemoteError(Exception):
         super().__init__(f"Door Updating Error: {r.status_code} {r.reason}\n{r.text}")
 
 
+class UnsupportedPageSize(Exception):
+    def __init__(self, page_size: int) -> None:
+        super().__init__(
+            f"Page size {page_size} greater than supported by controller. "
+            "(controller returned moreRecords=true)"
+        )
+
+
 class DoorController:
     def __init__(self, ip, username, password):
         self.ip = ip
@@ -152,48 +161,44 @@ class DoorController:
         )
         return self.doXMLRequest(el)
 
-    def get_records(self, req, count, params=None, stopFunction=None):
+    def get_records(
-        recordCount = 0
+        self,
-        moreRecords = True
+        req,
+        count_attr: str,
+        params: dict[str, str] | None = None,
+        page_size: int = 100,
+    ):
+        dr = self.doXMLRequest(ROOT(req({"action": "DR"})))
 
-        # note: all the "+/-1" bits are to work around a bug where the
+        for offset in range(0, int(dr[0].attrib[count_attr]), page_size):
-        # last returned entry is incomplete. There is probably a
-        # better way to do this, but for now I just get the last entry
-        # again in the next request. I suspect this probably ends
-        # poorly if the numbers line up poorly (ie an exact multiple
-        # of the returned record limit)
-        while True:
             res = self.doXMLRequest(
                 ROOT(
                     req(
                         {
                             "action": "LR",
-                            "recordCount": str(count - recordCount + 1),
+                            "recordCount": str(page_size),
-                            "recordOffset": str(
+                            "recordOffset": str(offset),
-                                recordCount - 1 if recordCount > 0 else 0
-                            ),
                             **(params or {}),
                         }
                     )
                 )
             )
-            recordCount += int(res[0].get("recordCount")) - 1
-            moreRecords = res[0].get("moreRecords") == "true"
 
-            if moreRecords and (stopFunction is None or stopFunction(list(res[0]))):
+            # The web interface does sub-pagination when needed, but that is very messy.
-                yield list(res[0])[:-1]
+            # See previous versions of this function for an example :)
-            else:
+            if res[0].attrib["moreRecords"] != "false":
-                yield list(res[0])
+                raise UnsupportedPageSize(page_size)
-                break
+
+            yield list(res[0])
 
     def get_cardholders(self):
         for page in self.get_records(
-            E.Cardholders, 1000, {"responseFormat": "expanded"}
+            E.Cardholders, "cardholdersInUse", params={"responseFormat": "expanded"}
         ):
             yield from page
 
     def get_credentials(self):
-        for page in self.get_records(E.Credentials, 1000):
+        for page in self.get_records(E.Credentials, "credentialsInUse"):
             yield from page
 
     def update_credential(self, rawCardNumber: str, cardholderID: str):
@@ -210,19 +215,17 @@ class DoorController:
             )
         )
 
-    def get_events(self, threshold):
+    def get_events(self, threshold: datetime):
         def event_newer_than_threshold(event):
             return datetime.fromisoformat(event.attrib["timestamp"]) > threshold
 
-        # These door controllers only store 5000 events max
+        # smaller page size empirically determined
-        for page in self.get_records(
+        for page in self.get_records(E.EventMessages, "eventsInUse", page_size=25):
-            E.EventMessages,
+            events = list(takewhile(event_newer_than_threshold, page))
-            5000,
-            stopFunction=lambda events: event_newer_than_threshold(events[-1]),
-        ):
-            events = [event for event in page if event_newer_than_threshold(event)]
             if events:
                 yield events
+            else:
+                break
 
     def get_lock(self):
         el = ROOT(E.Doors({"action": "LR", "responseFormat": "status"}))

doorcontrol/hid/tests/test_Credential.py

@@ -0,0 +1,20 @@
+from unittest import TestCase
+
+from hypothesis import given
+from hypothesis import strategies as st
+
+from doorcontrol.hid.Credential import Credential
+
+
+class CredentialTestCase(TestCase):
+    @given(facility_code=st.integers(0, 0xFF), card_number=st.integers(0, 0xFFFF))
+    def test_code_round_trip(self, facility_code: int, card_number: int):
+        cred = Credential.from_code(facility_code, card_number)
+        self.assertEqual(cred.facility_code, facility_code)
+        self.assertEqual(cred.card_number, card_number)
+
+    @given(facility_code=st.integers(0, 0xFF), card_number=st.integers(0, 0xFFFF))
+    def test_to_hex_round_trip(self, facility_code: int, card_number: int):
+        cred = Credential.from_code(facility_code, card_number)
+        hex_cred = Credential.from_hex(cred.hex)
+        self.assertEqual(cred, hex_cred)

doorcontrol/migrations/0001_initial.py

@@ -1,14 +1,40 @@
-# Generated by Django 4.1.3 on 2023-01-25 02:18
+# Generated by Django 5.1 on 2024-08-21 18:31
 
+import django.db.models.deletion
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
     initial = True
 
-    dependencies = []
+    dependencies = [
+        ("membershipworks", "0001_initial"),
+    ]
 
     operations = [
+        migrations.CreateModel(
+            name="Door",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=64, unique=True)),
+                ("ip", models.GenericIPAddressField(protocol="IPv4")),
+                (
+                    "access_field",
+                    models.TextField(
+                        help_text="Membershipworks field that grants members access to this door",
+                        max_length=128,
+                    ),
+                ),
+            ],
+        ),
         migrations.CreateModel(
             name="HIDEvent",
             fields=[
@@ -21,7 +47,6 @@ class Migration(migrations.Migration):
                         verbose_name="ID",
                     ),
                 ),
-                ("door_name", models.CharField(db_column="doorName", max_length=64)),
                 ("timestamp", models.DateTimeField()),
                 (
                     "event_type",
@@ -88,16 +113,173 @@ class Migration(migrations.Migration):
                         blank=True, db_column="rawCardNumber", max_length=8, null=True
                     ),
                 ),
+                (
+                    "door",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="doorcontrol.door",
+                    ),
+                ),
+                (
+                    "is_red",
+                    models.GeneratedField(
+                        db_persist=True,
+                        expression=models.Q(
+                            (
+                                "event_type__in",
+                                [
+                                    1022,
+                                    1023,
+                                    2024,
+                                    2029,
+                                    2036,
+                                    2042,
+                                    2043,
+                                    2046,
+                                    4041,
+                                    4042,
+                                    4043,
+                                    4044,
+                                    4045,
+                                ],
+                            )
+                        ),
+                        output_field=models.BooleanField(),
+                    ),
+                ),
             ],
             options={
                 "db_table": "hidevent",
                 "ordering": ("-timestamp",),
+                "constraints": [
+                    models.UniqueConstraint(
+                        fields=("door", "timestamp", "event_type"),
+                        name="unique_hidevent",
+                    )
+                ],
             },
         ),
-        migrations.AddConstraint(
+        migrations.CreateModel(
-            model_name="hidevent",
+            name="DoorCardholderMember",
-            constraint=models.UniqueConstraint(
+            fields=[
-                fields=("door_name", "timestamp", "event_type"), name="unique_hidevent"
+                (
-            ),
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("cardholder_id", models.IntegerField()),
+                (
+                    "door",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="doorcontrol.door",
+                    ),
+                ),
+                (
+                    "member",
+                    models.ForeignKey(
+                        db_constraint=False,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="membershipworks.member",
+                    ),
+                ),
+            ],
+            options={
+                "constraints": [
+                    models.UniqueConstraint(
+                        fields=("door", "cardholder_id"),
+                        name="unique_door_cardholder_id",
+                    ),
+                    models.UniqueConstraint(
+                        fields=("door", "member"), name="unique_door_member"
+                    ),
+                ],
+            },
+        ),
+        migrations.CreateModel(
+            name="Schedule",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=255, unique=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="FlagScheduleRule",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("doors", models.ManyToManyField(to="doorcontrol.door")),
+                (
+                    "flag",
+                    models.ForeignKey(
+                        blank=True,
+                        db_constraint=False,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="membershipworks.flag",
+                    ),
+                ),
+                (
+                    "schedule",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="doorcontrol.schedule",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="AttributeScheduleRule",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "access_field",
+                    models.CharField(
+                        help_text="Membershipworks field that grants members access to this door using this schedule.",
+                        max_length=128,
+                    ),
+                ),
+                ("doors", models.ManyToManyField(to="doorcontrol.door")),
+                (
+                    "schedule",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="doorcontrol.schedule",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
         ),
     ]

doorcontrol/migrations/0002_door_remove_hidevent_door_name_and_more.py

@@ -1,74 +0,0 @@
-# Generated by Django 4.2.5 on 2023-09-19 04:20
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-def link_events_to_doors(apps, schema_editor):
-    HIDEvent = apps.get_model("doorcontrol", "HIDEvent")
-    Door = apps.get_model("doorcontrol", "Door")
-    for event in HIDEvent.objects.all():
-        door, created = Door.objects.get_or_create(name=event.door_name)
-        event.door = door
-        event.save()
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("doorcontrol", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name="hidevent",
-            options={"ordering": ("-timestamp",)},
-        ),
-        migrations.CreateModel(
-            name="Door",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("name", models.CharField(max_length=64, unique=True)),
-            ],
-        ),
-        # create nullable foreign key to door
-        migrations.AddField(
-            model_name="hidevent",
-            name="door",
-            field=models.ForeignKey(
-                on_delete=django.db.models.deletion.CASCADE,
-                to="doorcontrol.door",
-                null=True,
-            ),
-        ),
-        # create new Doors and link them to HID Events
-        migrations.RunPython(link_events_to_doors, atomic=True),
-        # make door foreign key not nullable
-        migrations.AlterField(
-            model_name="hidevent",
-            name="door",
-            field=models.ForeignKey(
-                on_delete=django.db.models.deletion.CASCADE, to="doorcontrol.door"
-            ),
-        ),
-        # remove old constaint
-        migrations.RemoveConstraint(model_name="hidevent", name="unique_hidevent"),
-        # remove old name field
-        migrations.RemoveField(
-            model_name="hidevent",
-            name="door_name",
-        ),
-        migrations.AddConstraint(
-            model_name="hidevent",
-            constraint=models.UniqueConstraint(
-                fields=("door", "timestamp", "event_type"), name="unique_hidevent"
-            ),
-        ),
-    ]

doorcontrol/migrations/0003_door_ip.py

@@ -1,18 +0,0 @@
-# Generated by Django 4.2.5 on 2023-09-19 15:19
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("doorcontrol", "0002_door_remove_hidevent_door_name_and_more"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="door",
-            name="ip",
-            field=models.GenericIPAddressField(default="", protocol="IPv4"),
-            preserve_default=False,
-        ),
-    ]

doorcontrol/migrations/0004_hidevent_is_red.py

@@ -1,40 +0,0 @@
-# Generated by Django 5.0 on 2023-12-04 16:53
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("doorcontrol", "0003_door_ip"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="hidevent",
-            name="is_red",
-            field=models.GeneratedField(
-                db_persist=False,
-                expression=models.Q(
-                    (
-                        "event_type__in",
-                        [
-                            1022,
-                            1023,
-                            2024,
-                            2029,
-                            2036,
-                            2042,
-                            2043,
-                            2046,
-                            4041,
-                            4042,
-                            4043,
-                            4044,
-                            4045,
-                        ],
-                    )
-                ),
-                output_field=models.BooleanField(),
-            ),
-        ),
-    ]

doorcontrol/migrations/0005_doorcardholdermember_and_more.py

@@ -1,56 +0,0 @@
-# Generated by Django 5.0.1 on 2024-02-09 16:37
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("doorcontrol", "0004_hidevent_is_red"),
-        ("membershipworks", "0014_remove_eventext_details_timestamp"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="DoorCardholderMember",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("cardholder_id", models.IntegerField()),
-                (
-                    "door",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="doorcontrol.door",
-                    ),
-                ),
-                (
-                    "member",
-                    models.ForeignKey(
-                        db_constraint=False,
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="membershipworks.member",
-                    ),
-                ),
-            ],
-        ),
-        migrations.AddConstraint(
-            model_name="doorcardholdermember",
-            constraint=models.UniqueConstraint(
-                fields=("door", "cardholder_id"), name="unique_door_cardholder_id"
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="doorcardholdermember",
-            constraint=models.UniqueConstraint(
-                fields=("door", "member"), name="unique_door_member"
-            ),
-        ),
-    ]

doorcontrol/migrations/0006_schedule_door_access_field_flagschedulerule_and_more.py

@@ -1,106 +0,0 @@
-# Generated by Django 5.0.2 on 2024-02-23 18:36
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("doorcontrol", "0005_doorcardholdermember_and_more"),
-        ("membershipworks", "0015_eventmeetingtime_end_after_start"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="Schedule",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("name", models.CharField(max_length=255, unique=True)),
-            ],
-        ),
-        migrations.AddField(
-            model_name="door",
-            name="access_field",
-            field=models.TextField(
-                default="CHANGE ME",
-                help_text="Membershipworks field that grants members access to this door",
-                max_length=128,
-            ),
-            preserve_default=False,
-        ),
-        migrations.CreateModel(
-            name="FlagScheduleRule",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("doors", models.ManyToManyField(to="doorcontrol.door")),
-                (
-                    "flag",
-                    models.ForeignKey(
-                        blank=True,
-                        db_constraint=False,
-                        null=True,
-                        on_delete=django.db.models.deletion.PROTECT,
-                        to="membershipworks.flag",
-                    ),
-                ),
-                (
-                    "schedule",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="doorcontrol.schedule",
-                    ),
-                ),
-            ],
-            options={
-                "abstract": False,
-            },
-        ),
-        migrations.CreateModel(
-            name="AttributeScheduleRule",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                (
-                    "access_field",
-                    models.CharField(
-                        help_text="Membershipworks field that grants members access to this door using this schedule.",
-                        max_length=128,
-                    ),
-                ),
-                ("doors", models.ManyToManyField(to="doorcontrol.door")),
-                (
-                    "schedule",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="doorcontrol.schedule",
-                    ),
-                ),
-            ],
-            options={
-                "abstract": False,
-            },
-        ),
-    ]

doorcontrol/models.py

@@ -3,14 +3,14 @@ from typing import Self
 
 from django.conf import settings
 from django.db import models
-from django.db.models import F, Func, OuterRef, Q, Subquery
+from django.db.models import OuterRef, Q, Subquery
-from django.db.models.functions import Mod
 from django.utils import timezone
 from django.utils.functional import cached_property
 
 from membershipworks.models import Flag as MembershipWorksFlag
 from membershipworks.models import Member
 
+from .hid.Credential import Credential, InvalidHexCode
 from .hid.DoorController import DoorController
 
 
@@ -102,42 +102,6 @@ class AttributeScheduleRule(AbstractScheduleRule):
 
 
 class HIDEventQuerySet(models.QuerySet):
-    def with_decoded_card_number(self):
-        # TODO: CONV and BIT_COUNT are MySQL/MariaDB specific
-        class Conv(Func):
-            function = "CONV"
-            arity = 3
-            # This is technically not true, but fine for my purposes
-            output_field = models.IntegerField()
-
-        class BitCount(Func):
-            function = "BIT_COUNT"
-            arity = 1
-
-        return (
-            self.alias(card_number=Conv(F("raw_card_number"), 16, 10))
-            .alias(more_than_26_bits=F("card_number").bitrightshift(26))
-            .annotate(card_is_26_bit=Q(more_than_26_bits=0))
-            .alias(
-                parity_a=Mod(
-                    BitCount(F("card_number").bitrightshift(1).bitand(0xFFF)), 2
-                ),
-                parity_b=Mod(
-                    BitCount(F("card_number").bitrightshift(13).bitand(0xFFF)), 2
-                ),
-            )
-            .annotate(
-                card_is_valid_26_bit=~Q(parity_a=F("card_number").bitand(1))
-                & Q(parity_b=F("card_number").bitrightshift(25).bitand(1))
-            )
-            .annotate(
-                card_number_26_bit=F("card_number").bitrightshift(1).bitand(0xFFFF),
-                card_facility_code_26_bit=F("card_number")
-                .bitrightshift(17)
-                .bitand(0xFF),
-            )
-        )
-
     def with_member_id(self):
         return self.annotate(
             member_id=Subquery(
@@ -219,7 +183,7 @@ class HIDEvent(models.Model):
             ]
         ),
         output_field=models.BooleanField(),
-        db_persist=False,
+        db_persist=True,
     )
 
     objects = HIDEventQuerySet.as_manager()
@@ -302,13 +266,10 @@ class HIDEvent(models.Model):
         return event_types.get(self.event_type, f"Unknown Event Type {self.event_type}")
 
     def decoded_card_number(self) -> str | None:
-        """Requires annotations from `with_decoded_card_number`"""
         if self.raw_card_number is None:
             return None
-        elif self.card_is_26_bit:
+        try:
-            if self.card_is_valid_26_bit:
+            cred = Credential.from_hex(self.raw_card_number)
-                return f"{self.card_facility_code_26_bit} - {self.card_number_26_bit}"
+            return f"{cred.facility_code} - {cred.card_number}"
-            else:
+        except InvalidHexCode as e:
-                return "Invalid"
+            return f"Invalid: {e}"
-        else:
-            return "Not 26 bit card"

doorcontrol/tables.py

@@ -20,6 +20,7 @@ class UnitTimeTable(tables.Table):
 
 
 class DeniedAccessTable(tables.Table):
+    decoded_card_number = tables.Column(orderable=False)
     name = tables.TemplateColumn(
         "{{ record.forename|default:'' }} {{ record.surname|default:'' }}"
     )

doorcontrol/tasks/scrapehidevents.py

@@ -17,13 +17,15 @@ def get_cardholders(door: Door):
             member_id=cardholder.attrib.get("custom2"),
         )
 
+    cardholders = door.controller.get_cardholders()
     DoorCardholderMember.objects.bulk_create(
         (
             make_ch_member(cardholder)
-            for cardholder in door.controller.get_cardholders()
+            for cardholder in cardholders
             if "custom2" in cardholder.attrib
         ),
         update_conflicts=True,
+        unique_fields=("door", "cardholder_id"),
         update_fields=("member",),
     )
 

doorcontrol/tasks/update_doors.py

@@ -38,11 +38,9 @@ class DoorMember:
     def from_membershipworks_member(cls, member: Member, door: Door):
         if member.access_card_facility_code and member.access_card_number:
             credentials = {
-                Credential(
+                Credential.from_code(
-                    code=(
+                    member.access_card_facility_code,
-                        member.access_card_facility_code,
+                    member.access_card_number,
-                        member.access_card_number,
-                    )
                 )
             }
         else:
@@ -108,7 +106,7 @@ class DoorMember:
             },
             cardholderID=data.attrib["cardholderID"],
             credentials={
-                Credential(hex_code=(c.attrib["rawCardNumber"]))
+                Credential.from_hex(c.attrib["rawCardNumber"])
                 for c in data.findall("{*}Credential")
             },
             schedules={r.attrib["scheduleName"] for r in data.findall("{*}Role")},
@@ -172,9 +170,11 @@ class DoorMember:
                 xml_credentials = [
                     E.Credential(
                         {
-                            "formatName": str(credential.code[0]),
+                            "formatName": str(credential.facility_code),
-                            "cardNumber": str(credential.code[1]),
+                            "cardNumber": str(credential.card_number),
-                            "formatID": self.door.card_formats[str(credential.code[0])],
+                            "formatID": self.door.card_formats[
+                                str(credential.facility_code)
+                            ],
                             "isCard": "true",
                             "cardholderID": self.cardholderID,
                         }
@@ -222,7 +222,7 @@ def update_door(door: Door, dry_run: bool = False):
     }
 
     existing_door_credentials = {
-        Credential(hex_code=c.attrib["rawCardNumber"])
+        Credential.from_hex(c.attrib["rawCardNumber"])
         for c in door.controller.get_credentials()
     }
 

doorcontrol/views.py

@@ -2,9 +2,10 @@ import datetime
 from typing import TYPE_CHECKING
 
 from django.contrib.auth.mixins import PermissionRequiredMixin
+from django.contrib.postgres.aggregates import StringAgg
 from django.core.exceptions import BadRequest
-from django.db.models import Count, F, FloatField, Q, Window
+from django.db.models import Count, F, FloatField, Func, Q, Value, Window
-from django.db.models.functions import Lead, Trunc
+from django.db.models.functions import Lead, NullIf, Trunc
 from django.urls import path, reverse_lazy
 from django.utils.text import slugify
 from django.views.generic.list import ListView
@@ -12,8 +13,6 @@ from django.views.generic.list import ListView
 import django_filters
 import django_tables2 as tables
 from django_filters.views import BaseFilterView
-from django_mysql.models.aggregates import GroupConcat
-from django_mysql.models.functions import ConcatWS
 from django_tables2 import SingleTableMixin
 from django_tables2.export.views import ExportMixin
 
@@ -174,7 +173,7 @@ class AccessPerUnitTime(BaseAccessReport):
                 members_delta=(
                     F("members")
                     / Window(
-                        Lead("members"),
+                        Lead(NullIf("members", 0.0)),
                         order_by="-unit_time",
                         output_field=FloatField(),
                     )
@@ -185,7 +184,7 @@ class AccessPerUnitTime(BaseAccessReport):
                 access_count_delta=(
                     F("access_count")
                     / Window(
-                        Lead("access_count"),
+                        Lead(NullIf("access_count", 0.0)),
                         order_by="-unit_time",
                         output_field=FloatField(),
                     )
@@ -206,12 +205,7 @@ class DeniedAccess(BaseAccessReport):
         denied_event_types = [
             t for t in HIDEvent.EventType if t.name.startswith("DENIED_ACCESS")
         ]
-        return (
+        return super().get_table_data().filter(event_type__in=denied_event_types)
-            super()
-            .get_table_data()
-            .filter(event_type__in=denied_event_types)
-            .with_decoded_card_number()
-        )
 
 
 @register_report
@@ -228,8 +222,10 @@ class MostActiveMembers(BaseAccessReport):
             .values("member_id")
             .annotate(
                 access_count=Count("member_id"),
-                name=GroupConcat(
+                name=StringAgg(
-                    ConcatWS("forename", "surname", separator=" "), distinct=True
+                    Func(Value(" "), "forename", "surname", function="concat_ws"),
+                    ", ",
+                    distinct=True,
                 ),
             )
             .order_by("-access_count")
@@ -254,8 +250,10 @@ class DetailByDay(BaseAccessReport):
                     "member_id",
                     filter=Q(event_type__in=HIDEvent.EventType.any_granted_access()),
                 ),
-                name=GroupConcat(
+                name=StringAgg(
-                    ConcatWS("forename", "surname", separator=" "), distinct=True
+                    Func(Value(" "), "forename", "surname", function="concat_ws"),
+                    ", ",
+                    distinct=True,
                 ),
             )
             .order_by("-timestamp__date")

membershipworks/admin.py

@@ -10,6 +10,7 @@ from django_object_actions import (
 )
 from django_q.models import Task
 from django_q.tasks import async_task
+from simple_history.admin import SimpleHistoryAdmin
 
 from .models import (
     Event,
@@ -21,14 +22,40 @@ from .models import (
     Member,
     Transaction,
 )
-from .tasks.scrape import (
+from .tasks.scrape import scrape_event_details, scrape_events, scrape_membershipworks
-    scrape_event_details,
-    scrape_membershipworks,
-)
 from .tasks.ucsAccounts import sync_accounts
 
 
-class ReadOnlyAdmin(admin.ModelAdmin):
+class TaskLabel:
+    def __init__(self, label: str, task) -> None:
+        self.label = label
+        self.task = task
+
+    def __str__(self) -> str:
+        try:
+            last_run = naturaltime(
+                Task.objects.filter(group=self.task.q_task_group)
+                .values_list("started", flat=True)
+                .latest("started")
+            )
+        except Task.DoesNotExist:
+            last_run = "Never"
+        return f"{self.label} [Last Run {last_run}]"
+
+
+def run_task_action(admin: admin.ModelAdmin, label: str, task):
+    @action(label=TaskLabel(label, task))
+    def action_func(request, obj):
+        async_task(task, group=task.q_task_group)
+        admin.message_user(
+            request,
+            "Queued task, please wait a few seconds/minutes then refresh the page",
+        )
+
+    return action_func
+
+
+class ReadOnlyAdminMixin:
     def has_add_permission(self, request, obj=None):
         return False
 
@@ -39,39 +66,18 @@ class ReadOnlyAdmin(admin.ModelAdmin):
         return False
 
 
-class BaseMembershipWorksAdmin(DjangoObjectActions, ReadOnlyAdmin):
+class BaseMembershipWorksAdmin(
+    DjangoObjectActions, ReadOnlyAdminMixin, SimpleHistoryAdmin
+):
     changelist_actions = ("refresh_membershipworks_data", "sync_ucs_accounts")
 
-    # internal method from DjangoObjectActions
+    @property
-    def _get_tool_dict(self, tool_name):
+    def refresh_membershipworks_data(self):
-        tool = super(DjangoObjectActions, self)._get_tool_dict(tool_name)
+        return run_task_action(self, "Refresh Data", scrape_membershipworks)
-        if tool_name == "refresh_membershipworks_data":
-            try:
-                last_run_time = naturaltime(
-                    Task.objects.filter(group=scrape_membershipworks.q_task_group)
-                    .values_list("started", flat=True)
-                    .latest("started")
-                )
-            except Task.DoesNotExist:
-                last_run_time = "Never"
-            tool["label"] = f"Refresh Data [Last Run {last_run_time}]"
-        return tool
 
-    @action
+    @property
-    def refresh_membershipworks_data(self, request, obj):
+    def sync_ucs_accounts(self):
-        async_task(scrape_membershipworks, group=scrape_membershipworks.q_task_group)
+        return run_task_action(self, "Sync UCS Accounts", sync_accounts)
-        self.message_user(
-            request,
-            "Queued refresh, please wait a few seconds/minutes then refresh the page",
-        )
-
-    @action
-    def sync_ucs_accounts(self, request, obj):
-        async_task(sync_accounts, group=sync_accounts.q_task_group)
-        self.message_user(
-            request,
-            "Queued refresh, please wait a few seconds/minutes then refresh the page",
-        )
 
 
 class MemberFlagInline(admin.TabularInline):
@@ -172,10 +178,56 @@ class EventAdmin(DjangoObjectActions, admin.ModelAdmin):
     show_facets = admin.ShowFacets.ALWAYS
     search_fields = ["eid", "title", "url"]
     date_hierarchy = "start"
-    exclude = ["url", "details", "registrations"]
     autocomplete_fields = ["instructor"]
     change_actions = ["fetch_details"]
     actions = ["fetch_details"]
+    changelist_actions = ["refresh_membershipworks_data"]
+
+    fieldsets = [
+        (
+            None,
+            {
+                "fields": [
+                    "instructor",
+                    "materials_fee",
+                    "materials_fee_included_in_price",
+                    "instructor_percentage",
+                    "instructor_flat_rate",
+                    ("should_survey", "survey_email_sent"),
+                ]
+            },
+        ),
+        (
+            "Details",
+            {
+                "classes": ["collapse"],
+                "fields": [
+                    "eid",
+                    "_url",
+                    "start",
+                    "end",
+                    "duration",
+                    "count",
+                    "cap",
+                    "category",
+                    "calendar",
+                    "venue",
+                    "occurred",
+                ],
+            },
+        ),
+        (
+            "Advanced details",
+            {
+                "classes": ["collapse"],
+                "fields": ["details_timestamp", "details", "registrations"],
+            },
+        ),
+    ]
+
+    @property
+    def refresh_membershipworks_data(self):
+        return run_task_action(self, "Refresh Data", scrape_events)
 
     def get_readonly_fields(self, request: HttpRequest, obj: EventExt) -> list[str]:
         fields = []
@@ -187,7 +239,7 @@ class EventAdmin(DjangoObjectActions, admin.ModelAdmin):
             else:
                 fields.append(field.name)
         fields.insert(fields.index("end") + 1, "duration")
-        fields.append("details_timestamp")
+        fields += ["details_timestamp", "details", "registrations"]
         return fields
 
     @admin.display(ordering="title")

membershipworks/apps.py

@@ -9,6 +9,7 @@ def post_migrate_callback(sender, **kwargs):
 
     from .tasks.event_survey_emails import send_survey_emails
     from .tasks.scrape import scrape_events, scrape_membershipworks
+    from .tasks.simple_history import q_clean_duplicate_history
     from .tasks.ucsAccounts import sync_accounts
 
     ensure_scheduled(
@@ -32,6 +33,11 @@ def post_migrate_callback(sender, **kwargs):
         schedule_type=Schedule.HOURLY,
     )
 
+    ensure_scheduled(
+        q_clean_duplicate_history,
+        schedule_type=Schedule.HOURLY,
+    )
+
 
 class MembershipworksConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"

membershipworks/migrations/0001_initial.py

@@ -1,13 +1,21 @@
-# Generated by Django 5.0 on 2023-12-20 05:40
+# Generated by Django 5.1 on 2024-08-21 18:17
+
+import uuid
 
 import django.db.models.deletion
 from django.db import migrations, models
+from django.db.models.functions import Cast
+
+import django_db_views.migration_functions
+import django_db_views.operations
 
 
 class Migration(migrations.Migration):
     initial = True
 
-    dependencies = []
+    dependencies = [
+        ("reservations", "0001_initial"),
+    ]
 
     operations = [
         migrations.CreateModel(
@@ -115,18 +123,6 @@ class Migration(migrations.Migration):
                         blank=True, db_column="Parent Account ID", null=True
                     ),
                 ),
-                (
-                    "gift_membership_purchased_by",
-                    models.TextField(
-                        blank=True, db_column="Gift Membership purchased by", null=True
-                    ),
-                ),
-                (
-                    "purchased_gift_membership_for",
-                    models.TextField(
-                        blank=True, db_column="Purchased Gift Membership for", null=True
-                    ),
-                ),
                 (
                     "closet_storage",
                     models.TextField(
@@ -151,18 +147,6 @@ class Migration(migrations.Migration):
                         db_column="Access Permitted Shops During Extended Hours?"
                     ),
                 ),
-                (
-                    "normal_access_permitted_during_covid19_limited_operations",
-                    models.BooleanField(
-                        db_column="Normal Access Permitted During COVID-19 Limited Operations"
-                    ),
-                ),
-                (
-                    "access_permitted_during_covid19_staffed_period_only",
-                    models.BooleanField(
-                        db_column="Access Permitted During COVID-19 Staffed Period Only"
-                    ),
-                ),
                 (
                     "access_front_door_and_studio_space_during_extended_hours",
                     models.BooleanField(
@@ -349,18 +333,15 @@ class Migration(migrations.Migration):
                     "liability_form_filled_out",
                     models.BooleanField(db_column="Liability Form Filled Out"),
                 ),
-                (
-                    "self_certify_essential_business",
-                    models.BooleanField(db_column="selfCertifyEssentialBusiness"),
-                ),
-                (
-                    "accepted_covid19_policy",
-                    models.BooleanField(db_column="Accepted COVID-19 Policy"),
-                ),
             ],
             options={
                 "db_table": "members",
                 "ordering": ("first_name", "last_name"),
+                "indexes": [
+                    models.Index(fields=["account_name"], name="account_name_idx"),
+                    models.Index(fields=["first_name"], name="first_name_idx"),
+                    models.Index(fields=["last_name"], name="last_name_idx"),
+                ],
             },
         ),
         migrations.CreateModel(
@@ -386,6 +367,7 @@ class Migration(migrations.Migration):
                     "member",
                     models.ForeignKey(
                         db_column="uid",
+                        db_constraint=False,
                         on_delete=django.db.models.deletion.PROTECT,
                         to="membershipworks.member",
                     ),
@@ -393,6 +375,11 @@ class Migration(migrations.Migration):
             ],
             options={
                 "db_table": "memberflag",
+                "constraints": [
+                    models.UniqueConstraint(
+                        fields=("member", "flag"), name="unique_member_flag"
+                    )
+                ],
             },
         ),
         migrations.AddField(
@@ -416,7 +403,7 @@ class Migration(migrations.Migration):
                         verbose_name="ID",
                     ),
                 ),
-                ("sid", models.CharField(blank=True, max_length=27, null=True)),
+                ("sid", models.CharField(blank=True, max_length=256, null=True)),
                 ("timestamp", models.DateTimeField()),
                 ("type", models.TextField(blank=True, null=True)),
                 (
@@ -469,6 +456,7 @@ class Migration(migrations.Migration):
                     models.ForeignKey(
                         blank=True,
                         db_column="uid",
+                        db_constraint=False,
                         null=True,
                         on_delete=django.db.models.deletion.PROTECT,
                         related_name="transactions",
@@ -480,22 +468,350 @@ class Migration(migrations.Migration):
                 "db_table": "transactions",
             },
         ),
-        migrations.AddConstraint(
+        migrations.CreateModel(
-            model_name="memberflag",
+            name="EventCategory",
-            constraint=models.UniqueConstraint(
+            fields=[
-                fields=("member", "flag"), name="unique_member_flag"
+                ("id", models.IntegerField(primary_key=True, serialize=False)),
+                ("title", models.TextField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name="Event",
+            fields=[
+                (
+                    "eid",
+                    models.CharField(max_length=255, primary_key=True, serialize=False),
+                ),
+                ("url", models.TextField()),
+                ("title", models.TextField()),
+                ("start", models.DateTimeField()),
+                ("end", models.DateTimeField(blank=True, null=True)),
+                ("cap", models.IntegerField(blank=True, null=True)),
+                ("count", models.IntegerField()),
+                (
+                    "calendar",
+                    models.IntegerField(
+                        choices=[
+                            (0, "Hidden"),
+                            (1, "Green"),
+                            (2, "Red"),
+                            (3, "Yellow"),
+                            (4, "Blue"),
+                            (5, "Purple"),
+                            (6, "Magenta"),
+                            (7, "Grey"),
+                            (8, "Teal"),
+                        ]
+                    ),
+                ),
+                ("venue", models.TextField(blank=True, null=True)),
+                (
+                    "category",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="membershipworks.eventcategory",
+                    ),
+                ),
+                (
+                    "occurred",
+                    models.GeneratedField(
+                        db_persist=True,
+                        expression=models.Q(
+                            ("cap", 0),
+                            ("count", 0),
+                            ("calendar", 0),
+                            _connector="OR",
+                            _negated=True,
+                        ),
+                        output_field=models.BooleanField(),
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="EventInstructor",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.TextField(blank=True)),
+                (
+                    "member",
+                    models.OneToOneField(
+                        blank=True,
+                        db_constraint=False,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="membershipworks.member",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="EventExt",
+            fields=[
+                (
+                    "event_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="membershipworks.event",
+                    ),
+                ),
+                (
+                    "materials_fee",
+                    models.DecimalField(
+                        blank=True, decimal_places=4, max_digits=13, null=True
+                    ),
+                ),
+                (
+                    "instructor",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="membershipworks.eventinstructor",
+                    ),
+                ),
+                (
+                    "instructor_flat_rate",
+                    models.DecimalField(decimal_places=4, default=0, max_digits=13),
+                ),
+                (
+                    "instructor_percentage",
+                    models.DecimalField(decimal_places=4, default=0.5, max_digits=5),
+                ),
+                ("materials_fee_included_in_price", models.BooleanField(null=True)),
+                ("details", models.JSONField(blank=True, null=True)),
+                ("registrations", models.JSONField(blank=True, null=True)),
+                (
+                    "details_timestamp",
+                    models.GeneratedField(
+                        db_persist=True,
+                        expression=models.Func(
+                            Cast(models.F("details___ts"), models.IntegerField()),
+                            function="to_timestamp",
+                        ),
+                        output_field=models.DateTimeField(),
+                        verbose_name="Last details fetch",
+                    ),
+                ),
+                ("should_survey", models.BooleanField(default=False)),
+                ("survey_email_sent", models.BooleanField(default=False)),
+            ],
+            options={
+                "verbose_name": "event",
+                "ordering": ["-start"],
+            },
+            bases=("membershipworks.event",),
+        ),
+        migrations.CreateModel(
+            name="EventMeetingTime",
+            fields=[
+                (
+                    "event",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="meeting_times",
+                        to="membershipworks.eventext",
+                    ),
+                ),
+                (
+                    "reservation_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="reservations.reservation",
+                    ),
+                ),
+            ],
+            options={
+                "constraints": [],
+            },
+        ),
+        migrations.CreateModel(
+            name="EventInvoice",
+            fields=[
+                (
+                    "uuid",
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("date_submitted", models.DateField()),
+                ("date_paid", models.DateField(blank=True, null=True)),
+                ("pdf", models.FileField(upload_to="protected/invoices/%Y/%m/%d/")),
+                ("amount", models.DecimalField(decimal_places=4, max_digits=13)),
+                (
+                    "event",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="invoice",
+                        to="membershipworks.eventext",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="EventTicketType",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("label", models.TextField()),
+                ("restrict_to", models.TextField(blank=True, null=True)),
+                ("list_price", models.FloatField()),
+                ("quantity", models.IntegerField()),
+            ],
+            options={
+                "managed": False,
+                "base_manager_name": "objects",
+            },
+        ),
+        django_db_views.operations.ViewRunPython(
+            code=django_db_views.migration_functions.ForwardViewMigration(
+                "SELECT\n        row_number() over () as id,\n        eventext.event_ptr_id as event_id,\n        tkt.*,\n        jsonb_path_query_first(\n            eventext.details,\n            '$.tkt[*] ? (exists (@.dsp ? (@[*] == \"5771675edcdf126302a2f6b9\"))).amt'\n        )::numeric as members_price\n    FROM membershipworks_eventext AS eventext,\n        jsonb_to_recordset(eventext.details -> 'tkt') AS tkt (\n            lbl TEXT,\n            amt NUMERIC,\n            cnt INT,\n            dsp JSONB\n        )",
+                "membershipworks_eventtickettype",
+                engine="django.db.backends.postgresql",
             ),
+            reverse_code=django_db_views.migration_functions.BackwardViewMigration(
+                "",
+                "membershipworks_eventtickettype",
+                engine="django.db.backends.postgresql",
+            ),
+            atomic=False,
         ),
-        migrations.AddIndex(
+        migrations.CreateModel(
-            model_name="member",
+            name="EventAttendeeStats",
-            index=models.Index(fields=["account_name"], name="account_name_idx"),
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("gross_revenue", models.FloatField()),
+            ],
+            options={
+                "managed": False,
+            },
         ),
-        migrations.AddIndex(
+        django_db_views.operations.ViewRunPython(
-            model_name="member",
+            code=django_db_views.migration_functions.ForwardViewMigration(
-            index=models.Index(fields=["first_name"], name="first_name_idx"),
+                "SELECT eventext.event_ptr_id as event_id, SUM(usr.sum) as gross_revenue\n    FROM\n        membershipworks_eventext as eventext,\n        jsonb_to_recordset(eventext.details -> 'usr') AS usr (\n            sum NUMERIC\n        )\n    GROUP BY event_id",
+                "membershipworks_eventattendeestats",
+                engine="django.db.backends.postgresql",
+            ),
+            reverse_code=django_db_views.migration_functions.BackwardViewMigration(
+                "",
+                "membershipworks_eventattendeestats",
+                engine="django.db.backends.postgresql",
+            ),
+            atomic=False,
         ),
-        migrations.AddIndex(
+        migrations.CreateModel(
-            model_name="member",
+            name="EventAttendee",
-            index=models.Index(fields=["last_name"], name="last_name_idx"),
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(max_length=256)),
+                ("email", models.CharField(max_length=256)),
+                ("sum", models.FloatField()),
+            ],
+            options={
+                "managed": False,
+            },
+        ),
+        django_db_views.operations.ViewRunPython(
+            code=django_db_views.migration_functions.ForwardViewMigration(
+                "SELECT eventext.event_ptr_id as event_id, usr.*\n    FROM\n        membershipworks_eventext AS eventext,\n        jsonb_to_recordset(eventext.details -> 'usr') AS usr (\n            uid TEXT,\n            nam TEXT,\n            eml TEXT,\n            sum NUMERIC\n        )",
+                "membershipworks_eventattendee",
+                engine="django.db.backends.postgresql",
+            ),
+            reverse_code=django_db_views.migration_functions.BackwardViewMigration(
+                "",
+                "membershipworks_eventattendee",
+                engine="django.db.backends.postgresql",
+            ),
+            atomic=False,
+        ),
+        migrations.CreateModel(
+            name="EventTicketAggregate",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("quantity", models.IntegerField()),
+                ("amount", models.DecimalField(decimal_places=4, max_digits=13)),
+                ("materials", models.DecimalField(decimal_places=4, max_digits=13)),
+                (
+                    "amount_without_materials",
+                    models.DecimalField(decimal_places=4, max_digits=13),
+                ),
+                (
+                    "instructor_revenue",
+                    models.DecimalField(decimal_places=4, max_digits=13),
+                ),
+                (
+                    "instructor_amount",
+                    models.DecimalField(decimal_places=4, max_digits=13),
+                ),
+            ],
+            options={
+                "managed": False,
+            },
+        ),
+        django_db_views.operations.ViewRunPython(
+            code=django_db_views.migration_functions.ForwardViewMigration(
+                'SELECT "membershipworks_eventtickettype"."event_id", SUM("membershipworks_eventtickettype"."cnt") AS "quantity", SUM((CASE WHEN ("membershipworks_eventtickettype"."dsp" ? (SELECT U0."id" FROM "flag" U0 WHERE (U0."name" = \'Members\' AND U0."type" = \'folder\') ORDER BY U0."name" ASC LIMIT 1) OR ("membershipworks_eventtickettype"."dsp" IS NULL AND ("membershipworks_event"."start" < \'2024-07-01 00:00:00-04:00\'::timestamptz OR "membershipworks_eventtickettype"."members_price" = 0))) THEN "membershipworks_eventtickettype"."amt" ELSE "membershipworks_eventtickettype"."members_price" END * "membershipworks_eventtickettype"."cnt")) AS "amount", SUM(CASE WHEN ("membershipworks_eventext"."materials_fee_included_in_price" OR ("membershipworks_eventext"."materials_fee" = 0 AND "membershipworks_eventext"."materials_fee" IS NOT NULL)) THEN ("membershipworks_eventext"."materials_fee" * "membershipworks_eventtickettype"."cnt") WHEN "membershipworks_eventext"."materials_fee_included_in_price" IS NULL THEN NULL ELSE 0 END) AS "materials", SUM(((CASE WHEN ("membershipworks_eventtickettype"."dsp" ? (SELECT U0."id" FROM "flag" U0 WHERE (U0."name" = \'Members\' AND U0."type" = \'folder\') ORDER BY U0."name" ASC LIMIT 1) OR ("membershipworks_eventtickettype"."dsp" IS NULL AND ("membershipworks_event"."start" < \'2024-07-01 00:00:00-04:00\'::timestamptz OR "membershipworks_eventtickettype"."members_price" = 0))) THEN "membershipworks_eventtickettype"."amt" ELSE "membershipworks_eventtickettype"."members_price" END * "membershipworks_eventtickettype"."cnt") - CASE WHEN ("membershipworks_eventext"."materials_fee_included_in_price" OR ("membershipworks_eventext"."materials_fee" = 0 AND "membershipworks_eventext"."materials_fee" IS NOT NULL)) THEN ("membershipworks_eventext"."materials_fee" * "membershipworks_eventtickettype"."cnt") WHEN "membershipworks_eventext"."materials_fee_included_in_price" IS NULL THEN NULL ELSE 0 END)) AS "amount_without_materials", SUM((((CASE WHEN ("membershipworks_eventtickettype"."dsp" ? (SELECT U0."id" FROM "flag" U0 WHERE (U0."name" = \'Members\' AND U0."type" = \'folder\') ORDER BY U0."name" ASC LIMIT 1) OR ("membershipworks_eventtickettype"."dsp" IS NULL AND ("membershipworks_event"."start" < \'2024-07-01 00:00:00-04:00\'::timestamptz OR "membershipworks_eventtickettype"."members_price" = 0))) THEN "membershipworks_eventtickettype"."amt" ELSE "membershipworks_eventtickettype"."members_price" END * "membershipworks_eventtickettype"."cnt") - CASE WHEN ("membershipworks_eventext"."materials_fee_included_in_price" OR ("membershipworks_eventext"."materials_fee" = 0 AND "membershipworks_eventext"."materials_fee" IS NOT NULL)) THEN ("membershipworks_eventext"."materials_fee" * "membershipworks_eventtickettype"."cnt") WHEN "membershipworks_eventext"."materials_fee_included_in_price" IS NULL THEN NULL ELSE 0 END) * "membershipworks_eventext"."instructor_percentage")) AS "instructor_revenue", SUM(((((CASE WHEN ("membershipworks_eventtickettype"."dsp" ? (SELECT U0."id" FROM "flag" U0 WHERE (U0."name" = \'Members\' AND U0."type" = \'folder\') ORDER BY U0."name" ASC LIMIT 1) OR ("membershipworks_eventtickettype"."dsp" IS NULL AND ("membershipworks_event"."start" < \'2024-07-01 00:00:00-04:00\'::timestamptz OR "membershipworks_eventtickettype"."members_price" = 0))) THEN "membershipworks_eventtickettype"."amt" ELSE "membershipworks_eventtickettype"."members_price" END * "membershipworks_eventtickettype"."cnt") - CASE WHEN ("membershipworks_eventext"."materials_fee_included_in_price" OR ("membershipworks_eventext"."materials_fee" = 0 AND "membershipworks_eventext"."materials_fee" IS NOT NULL)) THEN ("membershipworks_eventext"."materials_fee" * "membershipworks_eventtickettype"."cnt") WHEN "membershipworks_eventext"."materials_fee_included_in_price" IS NULL THEN NULL ELSE 0 END) * "membershipworks_eventext"."instructor_percentage") + CASE WHEN ("membershipworks_eventext"."materials_fee_included_in_price" OR ("membershipworks_eventext"."materials_fee" = 0 AND "membershipworks_eventext"."materials_fee" IS NOT NULL)) THEN ("membershipworks_eventext"."materials_fee" * "membershipworks_eventtickettype"."cnt") WHEN "membershipworks_eventext"."materials_fee_included_in_price" IS NULL THEN NULL ELSE 0 END)) AS "instructor_amount" FROM "membershipworks_eventtickettype" INNER JOIN "membershipworks_eventext" ON ("membershipworks_eventtickettype"."event_id" = "membershipworks_eventext"."event_ptr_id") INNER JOIN "membershipworks_event" ON ("membershipworks_eventext"."event_ptr_id" = "membershipworks_event"."eid") GROUP BY "membershipworks_eventtickettype"."event_id"',
+                "membershipworks_eventticketaggregate",
+                engine="django.db.backends.postgresql",
+            ),
+            reverse_code=django_db_views.migration_functions.BackwardViewMigration(
+                "",
+                "membershipworks_eventticketaggregate",
+                engine="django.db.backends.postgresql",
+            ),
+            atomic=False,
         ),
     ]

membershipworks/migrations/0002_historical_member_and_flags.py

@@ -0,0 +1,453 @@
+# Generated by Django 5.1 on 2024-08-28 19:20
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+import simple_history.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("membershipworks", "0001_initial"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="HistoricalFlag",
+            fields=[
+                ("id", models.CharField(db_index=True, max_length=24)),
+                ("name", models.TextField(blank=True, null=True)),
+                ("type", models.CharField(max_length=6)),
+                ("history_id", models.AutoField(primary_key=True, serialize=False)),
+                ("history_date", models.DateTimeField(db_index=True)),
+                ("history_change_reason", models.CharField(max_length=100, null=True)),
+                (
+                    "history_type",
+                    models.CharField(
+                        choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")],
+                        max_length=1,
+                    ),
+                ),
+                (
+                    "history_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "historical flag",
+                "verbose_name_plural": "historical flags",
+                "ordering": ("-history_date", "-history_id"),
+                "get_latest_by": ("history_date", "history_id"),
+            },
+            bases=(simple_history.models.HistoricalChanges, models.Model),
+        ),
+        migrations.CreateModel(
+            name="HistoricalMember",
+            fields=[
+                ("uid", models.CharField(db_index=True, max_length=24)),
+                (
+                    "year_of_birth",
+                    models.TextField(blank=True, db_column="Year of Birth", null=True),
+                ),
+                (
+                    "account_name",
+                    models.TextField(blank=True, db_column="Account Name", null=True),
+                ),
+                (
+                    "first_name",
+                    models.TextField(blank=True, db_column="First Name", null=True),
+                ),
+                (
+                    "last_name",
+                    models.TextField(blank=True, db_column="Last Name", null=True),
+                ),
+                ("phone", models.TextField(blank=True, db_column="Phone", null=True)),
+                ("email", models.TextField(blank=True, db_column="Email", null=True)),
+                (
+                    "volunteer_email",
+                    models.TextField(
+                        blank=True, db_column="Volunteer Email", null=True
+                    ),
+                ),
+                (
+                    "address_street",
+                    models.TextField(
+                        blank=True, db_column="Address (Street)", null=True
+                    ),
+                ),
+                (
+                    "address_city",
+                    models.TextField(blank=True, db_column="Address (City)", null=True),
+                ),
+                (
+                    "address_state_province",
+                    models.TextField(
+                        blank=True, db_column="Address (State/Province)", null=True
+                    ),
+                ),
+                (
+                    "address_postal_code",
+                    models.TextField(
+                        blank=True, db_column="Address (Postal Code)", null=True
+                    ),
+                ),
+                (
+                    "address_country",
+                    models.TextField(
+                        blank=True, db_column="Address (Country)", null=True
+                    ),
+                ),
+                (
+                    "profile_description",
+                    models.TextField(
+                        blank=True, db_column="Profile description", null=True
+                    ),
+                ),
+                (
+                    "website",
+                    models.TextField(blank=True, db_column="Website", null=True),
+                ),
+                ("fax", models.TextField(blank=True, db_column="Fax", null=True)),
+                (
+                    "contact_person",
+                    models.TextField(blank=True, db_column="Contact Person", null=True),
+                ),
+                (
+                    "password",
+                    models.TextField(blank=True, db_column="Password", null=True),
+                ),
+                (
+                    "position_relation",
+                    models.TextField(
+                        blank=True, db_column="Position/relation", null=True
+                    ),
+                ),
+                (
+                    "parent_account_id",
+                    models.TextField(
+                        blank=True, db_column="Parent Account ID", null=True
+                    ),
+                ),
+                (
+                    "closet_storage",
+                    models.TextField(
+                        blank=True, db_column="Closet Storage #", null=True
+                    ),
+                ),
+                (
+                    "storage_shelf",
+                    models.TextField(
+                        blank=True, db_column="Storage Shelf #", null=True
+                    ),
+                ),
+                (
+                    "personal_studio_space",
+                    models.TextField(
+                        blank=True, db_column="Personal Studio Space #", null=True
+                    ),
+                ),
+                (
+                    "access_permitted_shops_during_extended_hours",
+                    models.BooleanField(
+                        db_column="Access Permitted Shops During Extended Hours?"
+                    ),
+                ),
+                (
+                    "access_front_door_and_studio_space_during_extended_hours",
+                    models.BooleanField(
+                        db_column="Access Front Door and Studio Space During Extended Hours?"
+                    ),
+                ),
+                (
+                    "access_wood_shop",
+                    models.BooleanField(db_column="Access Wood Shop?"),
+                ),
+                (
+                    "access_metal_shop",
+                    models.BooleanField(db_column="Access Metal Shop?"),
+                ),
+                (
+                    "access_storage_closet",
+                    models.BooleanField(db_column="Access Storage Closet?"),
+                ),
+                (
+                    "access_studio_space",
+                    models.BooleanField(db_column="Access Studio Space?"),
+                ),
+                (
+                    "access_front_door",
+                    models.BooleanField(db_column="Access Front Door?"),
+                ),
+                (
+                    "access_card_number",
+                    models.TextField(
+                        blank=True, db_column="Access Card Number", null=True
+                    ),
+                ),
+                (
+                    "access_card_facility_code",
+                    models.TextField(
+                        blank=True, db_column="Access Card Facility Code", null=True
+                    ),
+                ),
+                (
+                    "auto_billing_id",
+                    models.TextField(
+                        blank=True, db_column="Auto Billing ID", null=True
+                    ),
+                ),
+                (
+                    "billing_method",
+                    models.TextField(blank=True, db_column="Billing Method", null=True),
+                ),
+                (
+                    "renewal_date",
+                    models.DateField(blank=True, db_column="Renewal Date", null=True),
+                ),
+                (
+                    "join_date",
+                    models.DateField(blank=True, db_column="Join Date", null=True),
+                ),
+                (
+                    "admin_note",
+                    models.TextField(blank=True, db_column="Admin note", null=True),
+                ),
+                (
+                    "profile_gallery_image_url",
+                    models.TextField(
+                        blank=True, db_column="Profile gallery image URL", null=True
+                    ),
+                ),
+                (
+                    "business_card_image_url",
+                    models.TextField(
+                        blank=True, db_column="Business card image URL", null=True
+                    ),
+                ),
+                (
+                    "instagram",
+                    models.TextField(blank=True, db_column="Instagram", null=True),
+                ),
+                (
+                    "pinterest",
+                    models.TextField(blank=True, db_column="Pinterest", null=True),
+                ),
+                (
+                    "youtube",
+                    models.TextField(blank=True, db_column="Youtube", null=True),
+                ),
+                ("yelp", models.TextField(blank=True, db_column="Yelp", null=True)),
+                (
+                    "google",
+                    models.TextField(blank=True, db_column="Google+", null=True),
+                ),
+                ("bbb", models.TextField(blank=True, db_column="BBB", null=True)),
+                (
+                    "twitter",
+                    models.TextField(blank=True, db_column="Twitter", null=True),
+                ),
+                (
+                    "facebook",
+                    models.TextField(blank=True, db_column="Facebook", null=True),
+                ),
+                (
+                    "linked_in",
+                    models.TextField(blank=True, db_column="LinkedIn", null=True),
+                ),
+                (
+                    "do_not_show_street_address_in_profile",
+                    models.TextField(
+                        blank=True,
+                        db_column="Do not show street address in profile",
+                        null=True,
+                    ),
+                ),
+                (
+                    "do_not_list_in_directory",
+                    models.TextField(
+                        blank=True, db_column="Do not list in directory", null=True
+                    ),
+                ),
+                (
+                    "how_did_you_hear",
+                    models.TextField(blank=True, db_column="HowDidYouHear", null=True),
+                ),
+                (
+                    "authorize_charge",
+                    models.TextField(
+                        blank=True, db_column="authorizeCharge", null=True
+                    ),
+                ),
+                (
+                    "policy_agreement",
+                    models.TextField(
+                        blank=True, db_column="policyAgreement", null=True
+                    ),
+                ),
+                (
+                    "waiver_form_signed_and_on_file_date",
+                    models.DateField(
+                        blank=True,
+                        db_column="Waiver form signed and on file date.",
+                        null=True,
+                    ),
+                ),
+                (
+                    "membership_agreement_signed_and_on_file_date",
+                    models.DateField(
+                        blank=True,
+                        db_column="Membership Agreement signed and on file date.",
+                        null=True,
+                    ),
+                ),
+                (
+                    "ip_address",
+                    models.TextField(blank=True, db_column="IP Address", null=True),
+                ),
+                (
+                    "audit_date",
+                    models.DateField(blank=True, db_column="Audit Date", null=True),
+                ),
+                (
+                    "agreement_version",
+                    models.TextField(
+                        blank=True, db_column="Agreement Version", null=True
+                    ),
+                ),
+                (
+                    "paperwork_status",
+                    models.TextField(
+                        blank=True, db_column="Paperwork status", null=True
+                    ),
+                ),
+                (
+                    "membership_agreement_dated",
+                    models.BooleanField(db_column="Membership agreement dated"),
+                ),
+                (
+                    "membership_agreement_acknowledgement_page_filled_out",
+                    models.BooleanField(
+                        db_column="Membership Agreement Acknowledgement Page Filled Out"
+                    ),
+                ),
+                (
+                    "membership_agreement_signed",
+                    models.BooleanField(db_column="Membership Agreement Signed"),
+                ),
+                (
+                    "liability_form_filled_out",
+                    models.BooleanField(db_column="Liability Form Filled Out"),
+                ),
+                ("history_id", models.AutoField(primary_key=True, serialize=False)),
+                ("history_date", models.DateTimeField(db_index=True)),
+                ("history_change_reason", models.CharField(max_length=100, null=True)),
+                (
+                    "history_type",
+                    models.CharField(
+                        choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")],
+                        max_length=1,
+                    ),
+                ),
+                (
+                    "history_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "historical member",
+                "verbose_name_plural": "historical members",
+                "ordering": ("-history_date", "-history_id"),
+                "get_latest_by": ("history_date", "history_id"),
+            },
+            bases=(simple_history.models.HistoricalChanges, models.Model),
+        ),
+        migrations.CreateModel(
+            name="HistoricalMemberFlag",
+            fields=[
+                (
+                    "id",
+                    models.BigIntegerField(
+                        auto_created=True, blank=True, db_index=True, verbose_name="ID"
+                    ),
+                ),
+                ("history_id", models.AutoField(primary_key=True, serialize=False)),
+                ("history_date", models.DateTimeField(db_index=True)),
+                ("history_change_reason", models.CharField(max_length=100, null=True)),
+                (
+                    "history_type",
+                    models.CharField(
+                        choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")],
+                        max_length=1,
+                    ),
+                ),
+                (
+                    "flag",
+                    simple_history.models.HistoricForeignKey(
+                        blank=True,
+                        db_constraint=False,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        related_name="+",
+                        to="membershipworks.flag",
+                    ),
+                ),
+                (
+                    "history_user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="+",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+                (
+                    "member",
+                    simple_history.models.HistoricForeignKey(
+                        blank=True,
+                        db_column="uid",
+                        db_constraint=False,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        related_name="+",
+                        to="membershipworks.member",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "historical member flag",
+                "verbose_name_plural": "historical member flags",
+                "ordering": ("-history_date", "-history_id"),
+                "get_latest_by": ("history_date", "history_id"),
+            },
+            bases=(simple_history.models.HistoricalChanges, models.Model),
+        ),
+        migrations.AlterField(
+            model_name="memberflag",
+            name="flag",
+            field=simple_history.models.HistoricForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="membershipworks.flag"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="memberflag",
+            name="member",
+            field=simple_history.models.HistoricForeignKey(
+                db_column="uid",
+                db_constraint=False,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="membershipworks.member",
+            ),
+        ),
+    ]

membershipworks/migrations/0002_remove_member_accepted_covid19_policy_and_more.py

@@ -1,36 +0,0 @@
-# Generated by Django 4.0.2 on 2022-03-01 19:33
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name="member",
-            name="accepted_covid19_policy",
-        ),
-        migrations.RemoveField(
-            model_name="member",
-            name="access_permitted_during_covid19_staffed_period_only",
-        ),
-        migrations.RemoveField(
-            model_name="member",
-            name="gift_membership_purchased_by",
-        ),
-        migrations.RemoveField(
-            model_name="member",
-            name="purchased_gift_membership_for",
-        ),
-        migrations.RemoveField(
-            model_name="member",
-            name="normal_access_permitted_during_covid19_limited_operations",
-        ),
-        migrations.RemoveField(
-            model_name="member",
-            name="self_certify_essential_business",
-        ),
-    ]

membershipworks/migrations/0003_alter_transaction_sid.py

@@ -1,17 +0,0 @@
-# Generated by Django 5.0 on 2023-12-20 06:03
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0002_remove_member_accepted_covid19_policy_and_more"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="transaction",
-            name="sid",
-            field=models.CharField(blank=True, max_length=256, null=True),
-        ),
-    ]

membershipworks/migrations/0004_alter_memberflag_member_alter_transaction_member.py

@@ -1,36 +0,0 @@
-# Generated by Django 5.0 on 2023-12-26 17:46
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0003_alter_transaction_sid"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="memberflag",
-            name="member",
-            field=models.ForeignKey(
-                db_column="uid",
-                db_constraint=False,
-                on_delete=django.db.models.deletion.PROTECT,
-                to="membershipworks.member",
-            ),
-        ),
-        migrations.AlterField(
-            model_name="transaction",
-            name="member",
-            field=models.ForeignKey(
-                blank=True,
-                db_column="uid",
-                db_constraint=False,
-                null=True,
-                on_delete=django.db.models.deletion.PROTECT,
-                related_name="transactions",
-                to="membershipworks.member",
-            ),
-        ),
-    ]

membershipworks/migrations/0005_event_eventcategory_eventext_event_category_and_more.py

@@ -1,154 +0,0 @@
-# Generated by Django 5.0 on 2023-12-30 19:28
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0004_alter_memberflag_member_alter_transaction_member"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="Event",
-            fields=[
-                (
-                    "eid",
-                    models.CharField(max_length=255, primary_key=True, serialize=False),
-                ),
-                ("url", models.TextField()),
-                ("title", models.TextField()),
-                ("start", models.DateTimeField()),
-                ("end", models.DateTimeField(blank=True, null=True)),
-                ("cap", models.IntegerField(blank=True, null=True)),
-                ("count", models.IntegerField()),
-                (
-                    "calendar",
-                    models.IntegerField(
-                        choices=[
-                            (0, "Hidden"),
-                            (1, "Green"),
-                            (2, "Red"),
-                            (3, "Yellow"),
-                            (4, "Blue"),
-                            (5, "Purple"),
-                            (6, "Magenta"),
-                            (7, "Grey"),
-                            (8, "Teal"),
-                        ]
-                    ),
-                ),
-                ("venue", models.TextField(blank=True, null=True)),
-            ],
-            options={
-                "abstract": False,
-            },
-        ),
-        migrations.CreateModel(
-            name="EventCategory",
-            fields=[
-                ("id", models.IntegerField(primary_key=True, serialize=False)),
-                ("title", models.TextField()),
-            ],
-        ),
-        migrations.CreateModel(
-            name="EventExt",
-            fields=[
-                (
-                    "event_ptr",
-                    models.OneToOneField(
-                        auto_created=True,
-                        on_delete=django.db.models.deletion.CASCADE,
-                        parent_link=True,
-                        primary_key=True,
-                        serialize=False,
-                        to="membershipworks.event",
-                    ),
-                ),
-                (
-                    "materials_fee",
-                    models.DecimalField(
-                        blank=True, decimal_places=4, max_digits=13, null=True
-                    ),
-                ),
-            ],
-            options={
-                "verbose_name": "event",
-            },
-            bases=("membershipworks.event",),
-        ),
-        migrations.AddField(
-            model_name="event",
-            name="category",
-            field=models.ForeignKey(
-                on_delete=django.db.models.deletion.PROTECT,
-                to="membershipworks.eventcategory",
-            ),
-        ),
-        migrations.CreateModel(
-            name="EventInstructor",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("name", models.TextField(blank=True)),
-                (
-                    "member",
-                    models.OneToOneField(
-                        blank=True,
-                        db_constraint=False,
-                        null=True,
-                        on_delete=django.db.models.deletion.PROTECT,
-                        to="membershipworks.member",
-                    ),
-                ),
-            ],
-        ),
-        migrations.CreateModel(
-            name="EventMeetingTime",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("start", models.DateTimeField()),
-                ("end", models.DateTimeField()),
-                (
-                    "event",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="meeting_times",
-                        to="membershipworks.eventext",
-                    ),
-                ),
-            ],
-        ),
-        migrations.AddField(
-            model_name="eventext",
-            name="instructor",
-            field=models.ForeignKey(
-                blank=True,
-                null=True,
-                on_delete=django.db.models.deletion.PROTECT,
-                to="membershipworks.eventinstructor",
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name="eventmeetingtime",
-            constraint=models.UniqueConstraint(
-                fields=("event", "start", "end"), name="unique_event_start_end"
-            ),
-        ),
-    ]

membershipworks/migrations/0006_eventext_instructor_flat_rate_and_more.py

@@ -1,25 +0,0 @@
-# Generated by Django 5.0 on 2024-01-01 17:08
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        (
-            "membershipworks",
-            "0005_event_eventcategory_eventext_event_category_and_more",
-        ),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="eventext",
-            name="instructor_flat_rate",
-            field=models.DecimalField(decimal_places=4, default=0, max_digits=13),
-        ),
-        migrations.AddField(
-            model_name="eventext",
-            name="instructor_percentage",
-            field=models.DecimalField(decimal_places=4, default=0.5, max_digits=5),
-        ),
-    ]

membershipworks/migrations/0007_eventmeetingtime_duration.py

@@ -1,24 +0,0 @@
-# Generated by Django 5.0.1 on 2024-01-03 19:22
-
-import django.db.models.expressions
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0006_eventext_instructor_flat_rate_and_more"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="eventmeetingtime",
-            name="duration",
-            field=models.GeneratedField(
-                db_persist=False,
-                expression=django.db.models.expressions.CombinedExpression(
-                    models.F("end"), "-", models.F("start")
-                ),
-                output_field=models.DurationField(),
-            ),
-        ),
-    ]

membershipworks/migrations/0008_event_occurred.py

@@ -1,27 +0,0 @@
-# Generated by Django 5.0.1 on 2024-01-19 20:21
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0007_eventmeetingtime_duration"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="event",
-            name="occurred",
-            field=models.GeneratedField(
-                db_persist=False,
-                expression=models.Q(
-                    ("cap", 0),
-                    ("count", 0),
-                    ("calendar", 0),
-                    _connector="OR",
-                    _negated=True,
-                ),
-                output_field=models.BooleanField(),
-            ),
-        ),
-    ]

membershipworks/migrations/0009_eventext_materials_fee_included_in_price.py

@@ -1,17 +0,0 @@
-# Generated by Django 5.0.1 on 2024-01-25 02:43
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0008_event_occurred"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="eventext",
-            name="materials_fee_included_in_price",
-            field=models.BooleanField(null=True),
-        ),
-    ]

membershipworks/migrations/0010_alter_eventext_options.py

@@ -1,16 +0,0 @@
-# Generated by Django 5.0.1 on 2024-01-29 19:17
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0009_eventext_materials_fee_included_in_price"),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name="eventext",
-            options={"ordering": ["-start"], "verbose_name": "event"},
-        ),
-    ]

membershipworks/migrations/0011_eventext_details.py

@@ -1,29 +0,0 @@
-# Generated by Django 5.0.1 on 2024-01-29 19:19
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0010_alter_eventext_options"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="eventext",
-            name="details",
-            field=models.JSONField(blank=True, null=True),
-        ),
-        migrations.AddField(
-            model_name="eventext",
-            name="details_timestamp",
-            field=models.GeneratedField(
-                db_persist=False,
-                expression=models.Func(
-                    models.Func(models.F("details___ts"), function="FROM_UNIXTIME"),
-                    template="CONVERT_TZ(%(expressions)s, @@session.time_zone, 'UTC')",
-                ),
-                output_field=models.DateTimeField(),
-            ),
-        ),
-    ]

membershipworks/migrations/0012_eventattendeestats_eventtickettype.py

@@ -1,79 +0,0 @@
-# Generated by Django 5.0.1 on 2024-01-29 19:19
-
-from django.db import migrations, models
-
-import django_db_views.migration_functions
-import django_db_views.operations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0011_eventext_details"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="EventAttendeeStats",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("gross_revenue", models.FloatField()),
-            ],
-            options={
-                "managed": False,
-            },
-        ),
-        django_db_views.operations.ViewRunPython(
-            code=django_db_views.migration_functions.ForwardViewMigration(
-                "SELECT\n        row_number() over () as id,\n        eventext.event_ptr_id AS event_id,\n        tkt.label,\n        tkt.list_price,\n        tkt.quantity,\n        GROUP_CONCAT(tkt.restrict_to SEPARATOR ',') as restrict_to\n    FROM\n        membershipworks_eventext AS eventext,\n        JSON_TABLE (eventext.details, '$.tkt[*]' COLUMNS (\n            id FOR ORDINALITY,\n            label VARCHAR(256) PATH '$.lbl' ERROR ON ERROR,\n            list_price DOUBLE PATH '$.amt' ERROR ON ERROR,\n            quantity INTEGER PATH '$.cnt' DEFAULT 0 ON EMPTY ERROR ON ERROR,\n            NESTED PATH '$.dsp[*]' COLUMNS (\n                restrict_to VARCHAR(100) PATH '$' ERROR ON ERROR\n            )\n        )) AS tkt\n    GROUP BY event_id, id",
-                "membershipworks_eventtickettype",
-                engine="django.db.backends.mysql",
-            ),
-            reverse_code=django_db_views.migration_functions.BackwardViewMigration(
-                "", "membershipworks_eventtickettype", engine="django.db.backends.mysql"
-            ),
-            atomic=False,
-        ),
-        migrations.CreateModel(
-            name="EventTicketType",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("label", models.TextField()),
-                ("restrict_to", models.TextField(blank=True, null=True)),
-                ("list_price", models.FloatField()),
-                ("quantity", models.IntegerField()),
-            ],
-            options={
-                "managed": False,
-                "base_manager_name": "objects",
-            },
-        ),
-        django_db_views.operations.ViewRunPython(
-            code=django_db_views.migration_functions.ForwardViewMigration(
-                "SELECT eventext.event_ptr_id as event_id, SUM(tkt.s) as gross_revenue\n    FROM\n        membershipworks_eventext as eventext,\n        JSON_TABLE(eventext.details, '$.usr[*]' COLUMNS (\n            s DOUBLE PATH '$.sum' DEFAULT 0 ON EMPTY\n        )) as tkt\n    GROUP BY event_id",
-                "membershipworks_eventattendeestats",
-                engine="django.db.backends.mysql",
-            ),
-            reverse_code=django_db_views.migration_functions.BackwardViewMigration(
-                "",
-                "membershipworks_eventattendeestats",
-                engine="django.db.backends.mysql",
-            ),
-            atomic=False,
-        ),
-    ]

membershipworks/migrations/0013_eventattendee.py

@@ -1,46 +0,0 @@
-# Generated by Django 5.0.1 on 2024-02-02 22:07
-
-from django.db import migrations, models
-
-import django_db_views.migration_functions
-import django_db_views.operations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0012_eventattendeestats_eventtickettype"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="EventAttendee",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("name", models.CharField(max_length=256)),
-                ("email", models.CharField(max_length=256)),
-                ("sum", models.FloatField()),
-            ],
-            options={
-                "managed": False,
-            },
-        ),
-        django_db_views.operations.ViewRunPython(
-            code=django_db_views.migration_functions.ForwardViewMigration(
-                "SELECT eventext.event_ptr_id as event_id, tkt.uid, tkt.name, tkt.email, tkt.sum\n    FROM\n        membershipworks_eventext as eventext,\n        JSON_TABLE(eventext.details, '$.usr[*]' COLUMNS (\n            uid VARCHAR(24) PATH '$.uid',\n            name VARCHAR(256) PATH '$.nam',\n            email VARCHAR(256) PATH '$.eml',\n            sum DOUBLE PATH '$.sum'\n        )) as tkt",
-                "membershipworks_eventattendee",
-                engine="django.db.backends.mysql",
-            ),
-            reverse_code=django_db_views.migration_functions.BackwardViewMigration(
-                "", "membershipworks_eventattendee", engine="django.db.backends.mysql"
-            ),
-            atomic=False,
-        ),
-    ]

membershipworks/migrations/0014_remove_eventext_details_timestamp.py

@@ -1,16 +0,0 @@
-# Generated by Django 5.0.1 on 2024-02-05 03:32
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0013_eventattendee"),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name="eventext",
-            name="details_timestamp",
-        ),
-    ]

membershipworks/migrations/0015_eventmeetingtime_end_after_start.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.2 on 2024-02-12 21:24
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0014_remove_eventext_details_timestamp"),
-    ]
-
-    operations = [
-        migrations.AddConstraint(
-            model_name="eventmeetingtime",
-            constraint=models.CheckConstraint(
-                check=models.Q(("end__gt", models.F("start"))), name="end_after_start"
-            ),
-        ),
-    ]

membershipworks/migrations/0016_eventinvoice.py

@@ -1,41 +0,0 @@
-# Generated by Django 5.0.2 on 2024-03-08 21:30
-
-import uuid
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0015_eventmeetingtime_end_after_start"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="EventInvoice",
-            fields=[
-                (
-                    "uuid",
-                    models.UUIDField(
-                        default=uuid.uuid4,
-                        editable=False,
-                        primary_key=True,
-                        serialize=False,
-                    ),
-                ),
-                ("date_submitted", models.DateField()),
-                ("date_paid", models.DateField(blank=True, null=True)),
-                ("pdf", models.FileField(upload_to="invoices/%Y/%m/%d/")),
-                ("amount", models.DecimalField(decimal_places=4, max_digits=13)),
-                (
-                    "event",
-                    models.OneToOneField(
-                        on_delete=django.db.models.deletion.PROTECT,
-                        related_name="invoice",
-                        to="membershipworks.eventext",
-                    ),
-                ),
-            ],
-        ),
-    ]

membershipworks/migrations/0017_eventext_registrations_alter_eventinvoice_pdf.py

@@ -1,22 +0,0 @@
-# Generated by Django 5.0.4 on 2024-04-30 05:07
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0016_eventinvoice"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="eventext",
-            name="registrations",
-            field=models.JSONField(blank=True, null=True),
-        ),
-        migrations.AlterField(
-            model_name="eventinvoice",
-            name="pdf",
-            field=models.FileField(upload_to="protected/invoices/%Y/%m/%d/"),
-        ),
-    ]

membershipworks/migrations/0018_eventext_details_timestamp.py

@@ -1,25 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-08 16:44
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0017_eventext_registrations_alter_eventinvoice_pdf"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="eventext",
-            name="details_timestamp",
-            field=models.GeneratedField(
-                db_persist=False,
-                expression=models.Func(
-                    models.Func(models.F("details___ts"), function="FROM_UNIXTIME"),
-                    template="CONVERT_TZ(%(expressions)s, @@session.time_zone, 'UTC')",
-                ),
-                output_field=models.DateTimeField(),
-                verbose_name="Last details fetch",
-            ),
-        ),
-    ]

membershipworks/migrations/0019_eventext_should_survey_eventext_survey_email_sent.py

@@ -1,22 +0,0 @@
-# Generated by Django 5.0.6 on 2024-05-20 22:07
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0018_eventext_details_timestamp"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="eventext",
-            name="should_survey",
-            field=models.BooleanField(default=False),
-        ),
-        migrations.AddField(
-            model_name="eventext",
-            name="survey_email_sent",
-            field=models.BooleanField(default=False),
-        ),
-    ]

membershipworks/migrations/0020_remove_eventmeetingtime_unique_event_start_end_and_more.py

@@ -1,82 +0,0 @@
-# Generated by Django 5.0.7 on 2024-07-30 23:10
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-def convert_meetingtimes_to_reservations(apps, schema_editor):
-    Reservation = apps.get_model("reservations", "Reservation")
-    EventMeetingTime = apps.get_model("membershipworks", "EventMeetingTime")
-    for meeting_time in EventMeetingTime.objects.all():
-        reservation = Reservation.objects.create(
-            id=meeting_time.id,
-            start=meeting_time.start,
-            end=meeting_time.end,
-        )
-        meeting_time.reservation_ptr = reservation
-        meeting_time.save()
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("membershipworks", "0019_eventext_should_survey_eventext_survey_email_sent"),
-        ("reservations", "0001_initial"),
-    ]
-
-    operations = [
-        # add reservation field
-        migrations.AddField(
-            model_name="eventmeetingtime",
-            name="reservation_ptr",
-            field=models.OneToOneField(
-                auto_created=True,
-                null=True,
-                on_delete=django.db.models.deletion.CASCADE,
-                parent_link=True,
-                serialize=False,
-                to="reservations.reservation",
-            ),
-            preserve_default=False,
-        ),
-        migrations.RunPython(convert_meetingtimes_to_reservations, atomic=True),
-        # remove primary key
-        migrations.RemoveField(
-            model_name="eventmeetingtime",
-            name="id",
-        ),
-        # make reservation non-nullable
-        migrations.AlterField(
-            model_name="eventmeetingtime",
-            name="reservation_ptr",
-            field=models.OneToOneField(
-                auto_created=True,
-                on_delete=django.db.models.deletion.CASCADE,
-                parent_link=True,
-                primary_key=True,
-                serialize=False,
-                to="reservations.reservation",
-            ),
-            preserve_default=False,
-        ),
-        # delete old columns
-        migrations.RemoveConstraint(
-            model_name="eventmeetingtime",
-            name="unique_event_start_end",
-        ),
-        migrations.RemoveConstraint(
-            model_name="eventmeetingtime",
-            name="end_after_start",
-        ),
-        migrations.RemoveField(
-            model_name="eventmeetingtime",
-            name="duration",
-        ),
-        migrations.RemoveField(
-            model_name="eventmeetingtime",
-            name="end",
-        ),
-        migrations.RemoveField(
-            model_name="eventmeetingtime",
-            name="start",
-        ),
-    ]

membershipworks/models.py

@@ -1,18 +1,18 @@
 import uuid
 from datetime import datetime, timedelta
+from decimal import Decimal
 from typing import TYPE_CHECKING, TypedDict
 
 import django.core.mail.message
 from django.conf import settings
 from django.contrib.auth.models import AbstractBaseUser
-from django.db import models
+from django.db import connection, models
 from django.db.models import (
     Case,
     Count,
     Exists,
     ExpressionWrapper,
     F,
-    Func,
     OuterRef,
     Q,
     QuerySet,
@@ -21,13 +21,14 @@ from django.db.models import (
     Value,
     When,
 )
-from django.db.models.functions import Coalesce
+from django.db.models.functions import Cast, Coalesce
 from django.urls import reverse
 from django.utils import timezone
 
 import nh3
 from django_db_views.db_view import DBView
 from django_stubs_ext import WithAnnotations
+from simple_history.models import HistoricalRecords, HistoricForeignKey
 
 from reservations.models import Reservation
 
@@ -89,6 +90,8 @@ class Flag(BaseModel):
     name = models.TextField(null=True, blank=True)
     type = models.CharField(max_length=6)
 
+    history = HistoricalRecords()
+
     class Meta:
         db_table = "flag"
         ordering = ("name",)
@@ -121,7 +124,6 @@ class MemberQuerySet(models.QuerySet):
         )
 
 
-# TODO: is this still a temporal table?
 class Member(BaseModel):
     uid = models.CharField(max_length=24, primary_key=True)
     year_of_birth = models.TextField(db_column="Year of Birth", null=True, blank=True)
@@ -248,6 +250,8 @@ class Member(BaseModel):
     )
     flags = models.ManyToManyField(Flag, through="MemberFlag", related_name="members")
 
+    history = HistoricalRecords()
+
     _api_names_override = {
         "uid": "Account ID",
         "how_did_you_hear": "Please tell us how you heard about the Claremont MakerSpace and what tools or shops you are most excited to start using:",
@@ -300,10 +304,12 @@ class Member(BaseModel):
 
 
 class MemberFlag(BaseModel):
-    member = models.ForeignKey(
+    member = HistoricForeignKey(
         Member, on_delete=models.PROTECT, db_column="uid", db_constraint=False
     )
-    flag = models.ForeignKey(Flag, on_delete=models.PROTECT)
+    flag = HistoricForeignKey(Flag, on_delete=models.PROTECT)
+
+    history = HistoricalRecords()
 
     class Meta:
         db_table = "memberflag"
@@ -409,7 +415,7 @@ class Event(BaseModel):
     occurred = models.GeneratedField(
         expression=~(Q(cap=0) | Q(count=0) | Q(calendar=EventCalendar.HIDDEN)),
         output_field=models.BooleanField(),
-        db_persist=False,
+        db_persist=True,
     )
     # TODO:
     # "lgo": {
@@ -473,13 +479,7 @@ class EventExtQuerySet(models.QuerySet["EventExtAnnotated"]):
     def with_financials(self) -> "QuerySet[EventExtAnnotatedWithFinancials]":
         return self.annotate(
             **{
-                field: Subquery(
+                field: F(f"ticket_aggregates__{field}")
-                    EventTicketType.objects.filter(event=OuterRef("pk"))
-                    .values("event__pk")
-                    .annotate(d=Sum(field))
-                    .values("d"),
-                    output_field=models.DecimalField(),
-                )
                 for field in [
                     "quantity",
                     "amount",
@@ -492,11 +492,12 @@ class EventExtQuerySet(models.QuerySet["EventExtAnnotated"]):
             total_due_to_instructor=(
                 F("instructor_amount") + F("instructor_flat_rate")
             ),
-            gross_revenue=Coalesce(F("attendee_stats__gross_revenue"), 0.0),
+            gross_revenue=Coalesce(
-            net_revenue=ExpressionWrapper(
+                F("attendee_stats__gross_revenue"),
-                F("gross_revenue") - F("total_due_to_instructor"),
+                0,
                 output_field=models.DecimalField(),
             ),
+            net_revenue=F("gross_revenue") - F("total_due_to_instructor"),
         )
 
 
@@ -549,12 +550,12 @@ class EventExt(Event):
     )
     details = models.JSONField(null=True, blank=True)
     details_timestamp = models.GeneratedField(
-        expression=Func(
+        expression=models.Func(
-            Func(F("details___ts"), function="FROM_UNIXTIME"),
+            Cast(models.F("details___ts"), models.IntegerField()),
-            template="CONVERT_TZ(%(expressions)s, @@session.time_zone, 'UTC')",
+            function="to_timestamp",
         ),
         output_field=models.DateTimeField(),
-        db_persist=False,
+        db_persist=True,
         verbose_name="Last details fetch",
     )
 
@@ -663,7 +664,7 @@ class EventInvoice(models.Model):
 
 class EventTicketTypeQuerySet(models.QuerySet["EventTicketType"]):
     def group_by_ticket_type(self):
-        return self.values("is_members_ticket").annotate(
+        return self.values(is_members_ticket=Q(restrict_to__isnull=False)).annotate(
             label=Case(
                 When(Q(is_members_ticket=True), Value("Members")),
                 default=Value("Non-Members"),
@@ -685,17 +686,8 @@ class EventTicketTypeQuerySet(models.QuerySet["EventTicketType"]):
 
 class EventTicketTypeManager(models.Manager["EventTicketType"]):
     def get_queryset(self) -> models.QuerySet["EventTicketType"]:
-        members_folder = Subquery(
-            Flag.objects.filter(name="Members", type="folder").values("id")[:1]
-        )
         qs = super().get_queryset()
         return qs.annotate(
-            members_price=Subquery(
-                qs.filter(event=OuterRef("event"), restrict_to=members_folder).values(
-                    "list_price"
-                ),
-                output_field=models.FloatField(),
-            ),
             # Before 2024-07-01, use Members ticket price for any
             # restricted ticket, but list price for unrestricted
             # (Non-Members) ticket. After, use Members ticket price
@@ -703,7 +695,7 @@ class EventTicketTypeManager(models.Manager["EventTicketType"]):
             actual_price=Case(
                 When(
                     # member ticket
-                    Q(restrict_to=members_folder)
+                    Q(restrict_to__has_key=settings.MW_MEMBERS_FOLDER_ID)
                     | (
                         # non-member ticket
                         Q(restrict_to__isnull=True)
@@ -723,7 +715,6 @@ class EventTicketTypeManager(models.Manager["EventTicketType"]):
                 ),
                 default="members_price",
             ),
-            is_members_ticket=(Q(restrict_to__isnull=False)),
             materials=Case(
                 When(
                     (
@@ -764,37 +755,28 @@ class EventTicketType(DBView):
     event = models.ForeignKey(
         EventExt, on_delete=models.DO_NOTHING, related_name="ticket_types"
     )
-    label = models.TextField()
+    label = models.TextField(db_column="lbl")
-    restrict_to = models.TextField(null=True, blank=True)
+    list_price = models.DecimalField(db_column="amt", max_digits=13, decimal_places=4)
-    list_price = models.FloatField()
+    members_price = models.DecimalField(max_digits=13, decimal_places=4)
-    quantity = models.IntegerField()
+    quantity = models.IntegerField(db_column="cnt")
+    restrict_to = models.JSONField(db_column="dsp")
 
-    # Due to the presence of JSON_TABLE, this view must (as of MariaDB
+    view_definition = f"""
-    # 11.2.2) be created as the root user. See
-    # https://jira.mariadb.org/browse/MDEV-27898
-
-    # nested path/group_concat to workaround inability to create JSON columns using
-    # JSON_TABLE in views
-    view_definition = """
     SELECT
         row_number() over () as id,
-        eventext.event_ptr_id AS event_id,
+        eventext.event_ptr_id as event_id,
-        tkt.label,
+        tkt.*,
-        tkt.list_price,
+        jsonb_path_query_first(
-        tkt.quantity,
+            eventext.details,
-        GROUP_CONCAT(tkt.restrict_to SEPARATOR ',') as restrict_to
+            '$.tkt[*] ? (exists (@.dsp ? (@[*] == "{settings.MW_MEMBERS_FOLDER_ID}"))).amt'
-    FROM
+        )::numeric as members_price
-        membershipworks_eventext AS eventext,
+    FROM membershipworks_eventext AS eventext,
-        JSON_TABLE (eventext.details, '$.tkt[*]' COLUMNS (
+        jsonb_to_recordset(eventext.details -> 'tkt') AS tkt (
-            id FOR ORDINALITY,
+            lbl TEXT,
-            label VARCHAR(256) PATH '$.lbl' ERROR ON ERROR,
+            amt NUMERIC,
-            list_price DOUBLE PATH '$.amt' ERROR ON ERROR,
+            cnt INT,
-            quantity INTEGER PATH '$.cnt' DEFAULT 0 ON EMPTY ERROR ON ERROR,
+            dsp JSONB
-            NESTED PATH '$.dsp[*]' COLUMNS (
+        )
-                restrict_to VARCHAR(100) PATH '$' ERROR ON ERROR
-            )
-        )) AS tkt
-    GROUP BY event_id, id
     """
 
     def __str__(self) -> str:
@@ -805,19 +787,59 @@ class EventTicketType(DBView):
         base_manager_name = "objects"
 
 
+class EventTicketAggregate(DBView):
+    event = models.OneToOneField(
+        EventExt,
+        on_delete=models.DO_NOTHING,
+        related_name="ticket_aggregates",
+        primary_key=True,
+    )
+    quantity = models.IntegerField()
+    amount = models.DecimalField(max_digits=13, decimal_places=4)
+    materials = models.DecimalField(max_digits=13, decimal_places=4)
+    amount_without_materials = models.DecimalField(max_digits=13, decimal_places=4)
+    instructor_revenue = models.DecimalField(max_digits=13, decimal_places=4)
+    instructor_amount = models.DecimalField(max_digits=13, decimal_places=4)
+
+    @staticmethod
+    def view_definition():
+        qs = EventTicketType.objects.values("event").annotate(
+            **{
+                field: Sum(field)
+                for field in [
+                    "quantity",
+                    "amount",
+                    "materials",
+                    "amount_without_materials",
+                    "instructor_revenue",
+                    "instructor_amount",
+                ]
+            },
+        )
+
+        with connection.cursor() as cursor:
+            return cursor.mogrify(*qs.query.sql_with_params())
+
+    def __str__(self) -> str:
+        return f"{self.event}: {self.quantity}, {self.amount}"
+
+    class Meta:
+        managed = False
+
+
 class EventAttendeeStats(DBView):
     event = models.ForeignKey(
         EventExt, on_delete=models.DO_NOTHING, related_name="attendee_stats"
     )
-    gross_revenue = models.FloatField()
+    gross_revenue = models.DecimalField(max_digits=13, decimal_places=4)
 
     view_definition = """
-    SELECT eventext.event_ptr_id as event_id, SUM(tkt.s) as gross_revenue
+    SELECT eventext.event_ptr_id as event_id, SUM(usr.sum) as gross_revenue
     FROM
         membershipworks_eventext as eventext,
-        JSON_TABLE(eventext.details, '$.usr[*]' COLUMNS (
+        jsonb_to_recordset(eventext.details -> 'usr') AS usr (
-            s DOUBLE PATH '$.sum' DEFAULT 0 ON EMPTY
+            sum NUMERIC
-        )) as tkt
+        )
     GROUP BY event_id
     """
 
@@ -830,20 +852,20 @@ class EventAttendee(DBView):
         EventExt, on_delete=models.DO_NOTHING, related_name="attendees"
     )
     uid = models.ForeignKey(Member, on_delete=models.DO_NOTHING)
-    name = models.CharField(max_length=256)
+    name = models.CharField(max_length=256, db_column="nam")
-    email = models.CharField(max_length=256)
+    email = models.CharField(max_length=256, db_column="eml")
-    sum = models.FloatField()
+    sum = models.DecimalField(max_digits=13, decimal_places=4)
 
     view_definition = """
-    SELECT eventext.event_ptr_id as event_id, tkt.uid, tkt.name, tkt.email, tkt.sum
+    SELECT eventext.event_ptr_id as event_id, usr.*
     FROM
-        membershipworks_eventext as eventext,
+        membershipworks_eventext AS eventext,
-        JSON_TABLE(eventext.details, '$.usr[*]' COLUMNS (
+        jsonb_to_recordset(eventext.details -> 'usr') AS usr (
-            uid VARCHAR(24) PATH '$.uid',
+            uid TEXT,
-            name VARCHAR(256) PATH '$.nam',
+            nam TEXT,
-            email VARCHAR(256) PATH '$.eml',
+            eml TEXT,
-            sum DOUBLE PATH '$.sum'
+            sum NUMERIC
-        )) as tkt
+        )
     """
 
     class Meta:

membershipworks/tasks/scrape.py

@@ -137,6 +137,7 @@ def scrape_events():
     events = Event.objects.bulk_create(
         [Event.from_api_dict(event_data) for event_data in data["evt"]],
         update_conflicts=True,
+        unique_fields=["eid"],
         update_fields=[
             field.attname
             for field in Event._meta.get_fields()

membershipworks/tasks/simple_history.py

@@ -0,0 +1,9 @@
+import django.core.management
+
+from cmsmanage.django_q2_helper import q_task_group
+
+
+# TODO: this probably should be more global, instead of owned by membershipworks app
+@q_task_group("Clean Duplicate History")
+def q_clean_duplicate_history():
+    django.core.management.call_command("clean_duplicate_history", "--auto")

membershipworks/views.py

@@ -10,6 +10,7 @@ from django.contrib.auth.mixins import (
     AccessMixin,
     PermissionRequiredMixin,
 )
+from django.contrib.postgres.aggregates import StringAgg
 from django.core import mail
 from django.core.files.base import ContentFile
 from django.db.models import OuterRef, Q, Subquery
@@ -33,7 +34,6 @@ import django_tables2 as tables
 import weasyprint
 from dal import autocomplete
 from django_filters.views import BaseFilterView
-from django_mysql.models.aggregates import GroupConcat
 from django_sendfile import sendfile
 from django_tables2 import A, SingleTableMixin
 from django_tables2.export.views import ExportMixin
@@ -538,7 +538,7 @@ class MissingPaperworkReport(
                 membership=Subquery(
                     qs.filter(
                         pk=OuterRef("pk"), flags__type__in=("level", "addon")
-                    ).values(m=GroupConcat("flags__name"))
+                    ).values(m=StringAgg("flags__name", ", "))
                 ),
             )
         )

paperwork/tests.py

@@ -1,3 +1,4 @@
+import logging
 from collections.abc import Callable
 from itertools import chain
 from typing import TypedDict
@@ -60,10 +61,17 @@ class PermissionRequiredViewTestCaseMixin:
         cls.user_with_permission.user_permissions.add(*resolved_permissions)
 
     def test_missing_permission(self) -> None:
+        # suppress PermissionDenied messages
+        logger = logging.getLogger("django.request")
+        previous_log_level = logger.getEffectiveLevel()
+        logger.setLevel(logging.ERROR)
+
         self.client.force_login(self.user_without_permission)
         response = self.client.get(self.path)
         self.assertEqual(response.status_code, 403)
 
+        logger.setLevel(previous_log_level)
+
 
 class WaiverReportTestCase(PermissionRequiredViewTestCaseMixin, TestCase):
     permissions = [{"model": Waiver, "codename": "view_waiver"}]

paperwork/views.py

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.mixins import PermissionRequiredMixin
+from django.contrib.postgres.aggregates import StringAgg
 from django.contrib.staticfiles import finders as staticfiles_finders
 from django.db import models
 from django.db.models import (
@@ -16,14 +17,13 @@ from django.db.models import (
     Value,
     When,
 )
-from django.db.models.functions import Concat
+from django.db.models.functions import Cast, Concat
 from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound
 from django.shortcuts import get_object_or_404, render
 from django.views.generic import ListView
 
 import requests
 import weasyprint
-from django_mysql.models.aggregates import GroupConcat
 from django_tables2 import SingleTableMixin
 from django_tables2.export.views import ExportMixin
 
@@ -158,14 +158,16 @@ class InstructorOrVendorReport(
             .get_table_data()
             .values("name")
             .annotate(
-                instructor_agreement_date=GroupConcat(
+                instructor_agreement_date=StringAgg(
-                    "instructor_agreement_date", distinct=True, ordering="asc"
+                    Cast("instructor_agreement_date", models.TextField()),
+                    delimiter=", ",
+                    distinct=True,
                 ),
-                w9_date=GroupConcat("w9_date", distinct=True, ordering="asc"),
+                w9_date=StringAgg(
-                phone=GroupConcat("phone", distinct=True, ordering="asc"),
+                    Cast("w9_date", models.TextField()), ", ", distinct=True
-                email_address=GroupConcat(
-                    "email_address", distinct=True, ordering="asc"
                 ),
+                phone=StringAgg("phone", ", ", distinct=True),
+                email_address=StringAgg("email_address", ", ", distinct=True),
             )
         )
 

pdm.lock

@@ -5,7 +5,7 @@
 groups = ["default", "debug", "dev", "lint", "server", "typing"]
 strategy = ["inherit_metadata"]
 lock_version = "4.5.0"
-content_hash = "sha256:fbe86ed2e7a1ce164ed8c00ecc3c51fa6a2b8a14209f3c60663fcc12d0367444"
+content_hash = "sha256:5d6778ee41d2095320769ec21bd878f60d2bafbdcf9bc24ab484929311118978"
 
 [[metadata.targets]]
 requires_python = "==3.11.*"
@@ -192,7 +192,7 @@ name = "cachetools"
 version = "5.4.0"
 requires_python = ">=3.7"
 summary = "Extensible memoizing collections and decorators"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "cachetools-5.4.0-py3-none-any.whl", hash = "sha256:3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474"},
@@ -204,7 +204,7 @@ name = "certifi"
 version = "2024.7.4"
 requires_python = ">=3.6"
 summary = "Python package for providing Mozilla's CA Bundle."
-groups = ["default", "dev", "typing"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"},
@@ -231,7 +231,7 @@ name = "charset-normalizer"
 version = "3.3.2"
 requires_python = ">=3.7.0"
 summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
-groups = ["default", "dev", "typing"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"},
@@ -561,21 +561,6 @@ files = [
     {file = "django_model_utils-4.5.1.tar.gz", hash = "sha256:1220f22d9a467d53a1e0f4cda4857df0b2f757edf9a29955c42461988caa648a"},
 ]
 
-[[package]]
-name = "django-mysql"
-version = "4.14.0"
-requires_python = ">=3.8"
-summary = "Django-MySQL extends Django's built-in MySQL and MariaDB support their specific features not available on other databases."
-groups = ["default"]
-marker = "python_version == \"3.11\""
-dependencies = [
-    "django>=3.2",
-]
-files = [
-    {file = "django_mysql-4.14.0-py3-none-any.whl", hash = "sha256:c8ae4b8004bd2e1b74999f0254d255771043913273216a8514cf09aa4bd937bb"},
-    {file = "django_mysql-4.14.0.tar.gz", hash = "sha256:77cb615afb8f2a92636617d46dbe11b97b28e2b97d8373cf7752c3e1f2c619f1"},
-]
-
 [[package]]
 name = "django-nh3"
 version = "0.1.1"
@@ -618,6 +603,21 @@ files = [
     {file = "django_picklefield-3.2-py3-none-any.whl", hash = "sha256:e9a73539d110f69825d9320db18bcb82e5189ff48dbed41821c026a20497764c"},
 ]
 
+[[package]]
+name = "django-postgres-metrics"
+version = "0.15.0"
+requires_python = ">=3.7"
+summary = "A Django app that exposes a bunch of PostgreSQL database metrics."
+groups = ["default"]
+marker = "python_version == \"3.11\""
+dependencies = [
+    "django-rich>=1.0",
+]
+files = [
+    {file = "django-postgres-metrics-0.15.0.tar.gz", hash = "sha256:98127ef55cf9d435b4dd0fdf1d160ea5b97690197d4d8e6148fc2cc2bb57ae12"},
+    {file = "django_postgres_metrics-0.15.0-py3-none-any.whl", hash = "sha256:1710d44e910b230e3b7c7ce77304d68a40b1fa3711b18418387455b8e114d97a"},
+]
+
 [[package]]
 name = "django-q2"
 version = "1.6.2"
@@ -651,6 +651,22 @@ files = [
     {file = "django_recurrence-1.11.1-py3-none-any.whl", hash = "sha256:0c65f30872599b5813a9bab6952dada23c55894f28674490a753ada559f14bc5"},
 ]
 
+[[package]]
+name = "django-rich"
+version = "1.11.0"
+requires_python = ">=3.8"
+summary = "Extensions for using Rich with Django."
+groups = ["default"]
+marker = "python_version == \"3.11\""
+dependencies = [
+    "django>=3.2",
+    "rich>=10",
+]
+files = [
+    {file = "django_rich-1.11.0-py3-none-any.whl", hash = "sha256:7e25f97c3b072df96f58bb76bdc0a7c40001315c3644da33a390efc95cd6cbdc"},
+    {file = "django_rich-1.11.0.tar.gz", hash = "sha256:d07fa5c57921240673b84f8fe57940e6ade034c1f7b481517e72bec524a0e126"},
+]
+
 [[package]]
 name = "django-sendfile2"
 version = "0.7.1"
@@ -665,6 +681,21 @@ files = [
     {file = "django_sendfile2-0.7.1-py3-none-any.whl", hash = "sha256:81971df1db77f688c4834b8540930902d0d929c64cf374bd604b81f5ac52c04a"},
 ]
 
+[[package]]
+name = "django-simple-history"
+version = "3.7.0"
+requires_python = ">=3.8"
+summary = "Store model history and view/revert changes from admin site."
+groups = ["default"]
+marker = "python_version == \"3.11\""
+dependencies = [
+    "django>=4.2",
+]
+files = [
+    {file = "django_simple_history-3.7.0-py3-none-any.whl", hash = "sha256:282cb2c4aa63f51547f17da7f2130abaa81ba01694676d19b88d52c94a57a52c"},
+    {file = "django_simple_history-3.7.0.tar.gz", hash = "sha256:ac3b7ca8b0d33f7ea6be8fe7fc98cf43415efa500ff5dfe736fbd1ebc0cf39f9"},
+]
+
 [[package]]
 name = "django-stubs"
 version = "5.0.2"
@@ -971,7 +1002,7 @@ name = "google-api-core"
 version = "2.19.1"
 requires_python = ">=3.7"
 summary = "Google API client core library"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "google-auth<3.0.dev0,>=2.14.1",
@@ -987,10 +1018,10 @@ files = [
 
 [[package]]
 name = "google-api-python-client"
-version = "2.141.0"
+version = "2.142.0"
 requires_python = ">=3.7"
 summary = "Google API Client Library for Python"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5",
@@ -1000,8 +1031,8 @@ dependencies = [
     "uritemplate<5,>=3.0.1",
 ]
 files = [
-    {file = "google_api_python_client-2.141.0-py2.py3-none-any.whl", hash = "sha256:43c05322b91791204465291b3852718fae38d4f84b411d8be847c4f86882652a"},
+    {file = "google_api_python_client-2.142.0-py2.py3-none-any.whl", hash = "sha256:266799082bb8301f423ec204dffbffb470b502abbf29efd1f83e644d36eb5a8f"},
-    {file = "google_api_python_client-2.141.0.tar.gz", hash = "sha256:0f225b1f45d5a6f8c2a400f48729f5d6da9a81138e81e0478d61fdd8edf6563a"},
+    {file = "google_api_python_client-2.142.0.tar.gz", hash = "sha256:a1101ac9e24356557ca22f07ff48b7f61fa5d4b4e7feeef3bda16e5dcb86350e"},
 ]
 
 [[package]]
@@ -1009,7 +1040,7 @@ name = "google-api-python-client-stubs"
 version = "1.27.0"
 requires_python = "<4.0,>=3.7"
 summary = "Type stubs for google-api-python-client"
-groups = ["dev"]
+groups = ["typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "google-api-python-client>=2.141.0",
@@ -1026,7 +1057,7 @@ name = "google-auth"
 version = "2.32.0"
 requires_python = ">=3.7"
 summary = "Google Authentication Library"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "cachetools<6.0,>=2.0.0",
@@ -1042,7 +1073,7 @@ files = [
 name = "google-auth-httplib2"
 version = "0.2.0"
 summary = "Google Authentication Library: httplib2 transport"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "google-auth",
@@ -1074,7 +1105,7 @@ name = "googleapis-common-protos"
 version = "1.63.2"
 requires_python = ">=3.7"
 summary = "Common protobufs used in Google APIs"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "protobuf!=3.20.0,!=3.20.1,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0.dev0,>=3.20.2",
@@ -1144,7 +1175,7 @@ name = "httplib2"
 version = "0.22.0"
 requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 summary = "A comprehensive HTTP client library."
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2; python_version > \"3.0\"",
@@ -1206,7 +1237,7 @@ name = "idna"
 version = "3.7"
 requires_python = ">=3.5"
 summary = "Internationalized Domain Names in Applications (IDNA)"
-groups = ["default", "dev", "server", "typing"]
+groups = ["default", "server", "typing"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"},
@@ -1439,17 +1470,6 @@ files = [
     {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
 ]
 
-[[package]]
-name = "mysqlclient"
-version = "2.2.4"
-requires_python = ">=3.8"
-summary = "Python interface to MySQL"
-groups = ["default"]
-marker = "python_version == \"3.11\""
-files = [
-    {file = "mysqlclient-2.2.4.tar.gz", hash = "sha256:33bc9fb3464e7d7c10b1eaf7336c5ff8f2a3d3b88bab432116ad2490beb3bf41"},
-]
-
 [[package]]
 name = "nh3"
 version = "0.2.18"
@@ -1601,7 +1621,7 @@ name = "proto-plus"
 version = "1.24.0"
 requires_python = ">=3.7"
 summary = "Beautiful, Pythonic protocol buffers."
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "protobuf<6.0.0dev,>=3.19.0",
@@ -1616,7 +1636,7 @@ name = "protobuf"
 version = "5.27.3"
 requires_python = ">=3.8"
 summary = ""
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "protobuf-5.27.3-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:a55c48f2a2092d8e213bd143474df33a6ae751b781dd1d1f4d953c128a415b25"},
@@ -1624,6 +1644,67 @@ files = [
     {file = "protobuf-5.27.3.tar.gz", hash = "sha256:82460903e640f2b7e34ee81a947fdaad89de796d324bcbc38ff5430bcdead82c"},
 ]
 
+[[package]]
+name = "psycopg"
+version = "3.2.1"
+requires_python = ">=3.8"
+summary = "PostgreSQL database adapter for Python"
+groups = ["default"]
+marker = "python_version == \"3.11\""
+dependencies = [
+    "backports-zoneinfo>=0.2.0; python_version < \"3.9\"",
+    "typing-extensions>=4.4",
+    "tzdata; sys_platform == \"win32\"",
+]
+files = [
+    {file = "psycopg-3.2.1-py3-none-any.whl", hash = "sha256:ece385fb413a37db332f97c49208b36cf030ff02b199d7635ed2fbd378724175"},
+    {file = "psycopg-3.2.1.tar.gz", hash = "sha256:dc8da6dc8729dacacda3cc2f17d2c9397a70a66cf0d2b69c91065d60d5f00cb7"},
+]
+
+[[package]]
+name = "psycopg-binary"
+version = "3.2.1"
+requires_python = ">=3.8"
+summary = "PostgreSQL database adapter for Python -- C optimisation distribution"
+groups = ["default"]
+marker = "implementation_name != \"pypy\" and python_version == \"3.11\""
+files = [
+    {file = "psycopg_binary-3.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa3931f308ab4a479d0ee22dc04bea867a6365cac0172e5ddcba359da043854b"},
+]
+
+[[package]]
+name = "psycopg-pool"
+version = "3.2.2"
+requires_python = ">=3.8"
+summary = "Connection Pool for Psycopg"
+groups = ["default"]
+marker = "python_version == \"3.11\""
+dependencies = [
+    "typing-extensions>=4.4",
+]
+files = [
+    {file = "psycopg_pool-3.2.2-py3-none-any.whl", hash = "sha256:273081d0fbfaced4f35e69200c89cb8fbddfe277c38cc86c235b90a2ec2c8153"},
+    {file = "psycopg_pool-3.2.2.tar.gz", hash = "sha256:9e22c370045f6d7f2666a5ad1b0caf345f9f1912195b0b25d0d3bcc4f3a7389c"},
+]
+
+[[package]]
+name = "psycopg"
+version = "3.2.1"
+extras = ["binary", "pool"]
+requires_python = ">=3.8"
+summary = "PostgreSQL database adapter for Python"
+groups = ["default"]
+marker = "python_version == \"3.11\""
+dependencies = [
+    "psycopg-binary==3.2.1; implementation_name != \"pypy\"",
+    "psycopg-pool",
+    "psycopg==3.2.1",
+]
+files = [
+    {file = "psycopg-3.2.1-py3-none-any.whl", hash = "sha256:ece385fb413a37db332f97c49208b36cf030ff02b199d7635ed2fbd378724175"},
+    {file = "psycopg-3.2.1.tar.gz", hash = "sha256:dc8da6dc8729dacacda3cc2f17d2c9397a70a66cf0d2b69c91065d60d5f00cb7"},
+]
+
 [[package]]
 name = "ptyprocess"
 version = "0.7.0"
@@ -1651,7 +1732,7 @@ name = "pyasn1"
 version = "0.6.0"
 requires_python = ">=3.8"
 summary = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "pyasn1-0.6.0-py2.py3-none-any.whl", hash = "sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473"},
@@ -1663,7 +1744,7 @@ name = "pyasn1-modules"
 version = "0.4.0"
 requires_python = ">=3.8"
 summary = "A collection of ASN.1-based protocols modules"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "pyasn1<0.7.0,>=0.4.6",
@@ -1702,7 +1783,7 @@ name = "pygments"
 version = "2.18.0"
 requires_python = ">=3.8"
 summary = "Pygments is a syntax highlighting package written in Python."
-groups = ["dev"]
+groups = ["default", "dev"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"},
@@ -1714,7 +1795,7 @@ name = "pyparsing"
 version = "3.1.2"
 requires_python = ">=3.6.8"
 summary = "pyparsing module - Classes and methods to define and execute parsing grammars"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"},
@@ -1804,7 +1885,7 @@ name = "requests"
 version = "2.32.3"
 requires_python = ">=3.8"
 summary = "Python HTTP for Humans."
-groups = ["default", "dev", "typing"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "certifi>=2017.4.17",
@@ -1833,12 +1914,29 @@ files = [
     {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"},
 ]
 
+[[package]]
+name = "rich"
+version = "13.8.0"
+requires_python = ">=3.7.0"
+summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+groups = ["default"]
+marker = "python_version == \"3.11\""
+dependencies = [
+    "markdown-it-py>=2.2.0",
+    "pygments<3.0.0,>=2.13.0",
+    "typing-extensions<5.0,>=4.0.0; python_version < \"3.9\"",
+]
+files = [
+    {file = "rich-13.8.0-py3-none-any.whl", hash = "sha256:2e85306a063b9492dffc86278197a60cbece75bcb766022f3436f567cae11bdc"},
+    {file = "rich-13.8.0.tar.gz", hash = "sha256:a5ac1f1cd448ade0d59cc3356f7db7a7ccda2c8cbae9c7a90c28ff463d3e91f4"},
+]
+
 [[package]]
 name = "rsa"
 version = "4.9"
 requires_python = ">=3.6,<4"
 summary = "Pure-Python RSA implementation"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 dependencies = [
     "pyasn1>=0.1.3",
@@ -1874,14 +1972,14 @@ files = [
 
 [[package]]
 name = "setuptools"
-version = "72.2.0"
+version = "73.0.1"
 requires_python = ">=3.8"
 summary = "Easily download, build, install, upgrade, and uninstall Python packages"
 groups = ["server", "typing"]
 marker = "python_version == \"3.11\""
 files = [
-    {file = "setuptools-72.2.0-py3-none-any.whl", hash = "sha256:f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4"},
+    {file = "setuptools-73.0.1-py3-none-any.whl", hash = "sha256:b208925fcb9f7af924ed2dc04708ea89791e24bde0d3020b27df0e116088b34e"},
-    {file = "setuptools-72.2.0.tar.gz", hash = "sha256:80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9"},
+    {file = "setuptools-73.0.1.tar.gz", hash = "sha256:d59a3e788ab7e012ab2c4baed1b376da6366883ee20d7a5fc426816e3d7b1193"},
 ]
 
 [[package]]
@@ -2102,7 +2200,7 @@ name = "types-httplib2"
 version = "0.22.0.20240310"
 requires_python = ">=3.8"
 summary = "Typing stubs for httplib2"
-groups = ["dev"]
+groups = ["typing"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "types-httplib2-0.22.0.20240310.tar.gz", hash = "sha256:1eda99fea18ec8a1dc1a725ead35b889d0836fec1b11ae6f1fe05440724c1d15"},
@@ -2140,14 +2238,14 @@ files = [
 
 [[package]]
 name = "types-psycopg2"
-version = "2.9.21.20240417"
+version = "2.9.21.20240819"
 requires_python = ">=3.8"
 summary = "Typing stubs for psycopg2"
 groups = ["typing"]
 marker = "python_version == \"3.11\""
 files = [
-    {file = "types-psycopg2-2.9.21.20240417.tar.gz", hash = "sha256:05db256f4a459fb21a426b8e7fca0656c3539105ff0208eaf6bdaf406a387087"},
+    {file = "types-psycopg2-2.9.21.20240819.tar.gz", hash = "sha256:4ed6b47464d6374fa64e5e3b234cea0f710e72123a4596d67ab50b7415a84666"},
-    {file = "types_psycopg2-2.9.21.20240417-py3-none-any.whl", hash = "sha256:644d6644d64ebbe37203229b00771012fb3b3bddd507a129a2e136485990e4f8"},
+    {file = "types_psycopg2-2.9.21.20240819-py3-none-any.whl", hash = "sha256:c9192311c27d7ad561eef705f1b2df1074f2cdcf445a98a6a2fcaaaad43278cf"},
 ]
 
 [[package]]
@@ -2168,14 +2266,14 @@ files = [
 
 [[package]]
 name = "types-python-dateutil"
-version = "2.9.0.20240316"
+version = "2.9.0.20240821"
 requires_python = ">=3.8"
 summary = "Typing stubs for python-dateutil"
-groups = ["dev"]
+groups = ["typing"]
 marker = "python_version == \"3.11\""
 files = [
-    {file = "types-python-dateutil-2.9.0.20240316.tar.gz", hash = "sha256:5d2f2e240b86905e40944dd787db6da9263f0deabef1076ddaed797351ec0202"},
+    {file = "types-python-dateutil-2.9.0.20240821.tar.gz", hash = "sha256:9649d1dcb6fef1046fb18bebe9ea2aa0028b160918518c34589a46045f6ebd98"},
-    {file = "types_python_dateutil-2.9.0.20240316-py3-none-any.whl", hash = "sha256:6b8cb66d960771ce5ff974e9dd45e38facb81718cc1e208b10b1baccbfdbee3b"},
+    {file = "types_python_dateutil-2.9.0.20240821-py3-none-any.whl", hash = "sha256:f5889fcb4e63ed4aaa379b44f93c32593d50b9a94c9a60a0c854d8cc3511cd57"},
 ]
 
 [[package]]
@@ -2264,7 +2362,7 @@ name = "uritemplate"
 version = "4.1.1"
 requires_python = ">=3.6"
 summary = "Implementation of RFC 6570 URI Templates"
-groups = ["default", "dev"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"},
@@ -2276,7 +2374,7 @@ name = "urllib3"
 version = "2.2.2"
 requires_python = ">=3.8"
 summary = "HTTP library with thread-safe connection pooling, file post, and more."
-groups = ["default", "dev", "typing"]
+groups = ["default", "typing"]
 marker = "python_version == \"3.11\""
 files = [
     {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"},

pyproject.toml

@@ -16,7 +16,6 @@ dependencies = [
     "markdownify~=0.13",
     "mdformat~=0.7",
     "mdformat-tables~=0.4",
-    "mysqlclient~=2.2",
     "django-autocomplete-light~=3.11",
     "weasyprint~=62.3",
     "requests~=2.32",
@@ -34,23 +33,25 @@ dependencies = [
     "tablib[ods,xlsx]~=3.6",
     "django-filter~=24.3",
     "django-db-views~=0.1",
-    "django-mysql~=4.14",
     "django-weasyprint~=2.3",
     "django-sendfile2~=0.7",
     "django-bootstrap5~=24.2",
     "django-configurations[database,email]~=2.5",
     "django-vite~=3.0",
     "django-template-partials~=24.4",
-    "google-api-python-client~=2.141",
+    "google-api-python-client~=2.142",
     "google-auth-oauthlib~=1.2",
     "django-model-utils~=4.5",
+    "psycopg[binary,pool]~=3.2",
+    "django-simple-history~=3.7",
+    "django-postgres-metrics~=0.15",
 ]
 requires-python = ">=3.11"
 
 [project.optional-dependencies]
 server = [
     "uvicorn[standard]~=0.30",
-    "setuptools~=72.2",
+    "setuptools~=73.0",
 ]
 
 [project.entry-points."djangoq.errorreporters"]
@@ -150,7 +151,7 @@ lint = [
 typing = [
     "mypy~=1.10",
     "django-stubs~=5.0",
-    "setuptools~=72.2",
+    "setuptools~=73.0",
     "types-bleach~=6.1",
     "types-requests~=2.32",
     "types-urllib3~=1.26",
@@ -159,6 +160,8 @@ typing = [
     "types-Pygments~=2.18",
     "types-psycopg2~=2.9",
     "types-lxml~=2024.8",
+    "google-api-python-client-stubs~=1.27",
+    "types-python-dateutil~=2.9",
 ]
 debug = [
     "django-debug-toolbar~=4.4",
@@ -168,8 +171,6 @@ dev = [
     "ipython~=8.26",
     "hypothesis[django]~=6.111",
     "tblib~=3.0",
-    "google-api-python-client-stubs~=1.27",
-    "types-python-dateutil~=2.9",
 ]
 
 [tool.pdm.scripts]

reservations/migrations/0001_initial.py

@@ -65,7 +65,7 @@ class Migration(migrations.Migration):
                 (
                     "duration",
                     models.GeneratedField(
-                        db_persist=False,
+                        db_persist=True,
                         expression=django.db.models.expressions.CombinedExpression(
                             models.F("end"), "-", models.F("start")
                         ),

reservations/models.py

@@ -87,7 +87,7 @@ class Reservation(models.Model):
     duration = models.GeneratedField(
         expression=F("end") - F("start"),
         output_field=models.DurationField(),
-        db_persist=False,
+        db_persist=True,
     )
 
     objects = ReservationQuerySet.as_manager()

reservations/tasks/sync_google_calendar.py

@@ -1,6 +1,7 @@
 import logging
 from datetime import date, datetime
 from http import HTTPStatus
+from typing import TYPE_CHECKING
 
 from django.conf import settings
 from django.utils import timezone
@@ -9,6 +10,9 @@ from google.oauth2 import service_account
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 
+if TYPE_CHECKING:
+    from googleapiclient._apis.calendar.v3 import CalendarResource, Event
+
 from cmsmanage.django_q2_helper import q_task_group
 from reservations.models import ExternalReservation, Reservation, Resource
 
@@ -26,180 +30,195 @@ def parse_google_calendar_datetime(dt) -> date | datetime:
         raise Exception("Google Calendar event with out a start/end date/dateTime")
 
 
-def update_calendar_event(
+class GoogleCalendarSynchronizer:
-    service, resource: Resource, existing_event, reservation: Reservation
+    service: "CalendarResource"
-):
+
-    changes = reservation.make_google_calendar_event()
+    def __init__(self) -> None:
-    # skip update if no changes are needed
+        self.service = build(
-    if (
+            "calendar",
-        parse_google_calendar_datetime(existing_event["start"]) != reservation.start
+            "v3",
-        or parse_google_calendar_datetime(existing_event["end"]) != reservation.end
+            credentials=service_account.Credentials.from_service_account_file(
-        or any(
+                settings.GOOGLE_SERVICE_ACCOUNT_FILE,
-            existing_event[k] != v
+                scopes=SCOPES,
-            for k, v in changes.items()
+            ),
-            if k not in ("start", "end")
         )
+
+    def update_calendar_event(
+        self, resource: Resource, existing_event: "Event", reservation: Reservation
     ):
-        logger.debug("Updating event")
+        changes = reservation.make_google_calendar_event()
-        new_event = existing_event | changes
+        # skip update if no changes are needed
-        service.events().update(
-            calendarId=resource.google_calendar,
-            eventId=reservation.google_calendar_event_id,
-            body=new_event,
-        ).execute()
-
-
-def insert_calendar_event(service, resource: Resource, reservation: Reservation):
-    new_gcal_event = reservation.make_google_calendar_event()
-    created_event = (
-        service.events()
-        .insert(
-            calendarId=resource.google_calendar,
-            body=new_gcal_event,
-        )
-        .execute()
-    )
-    reservation.google_calendar_event_id = created_event["id"]
-    reservation.save()
-
-
-def sync_resource_from_google_calendar(
-    service, resource: Resource, now: datetime
-) -> set[str]:
-    request = (
-        service.events()
-        .list(
-            calendarId=resource.google_calendar,
-            timeMin=now.isoformat(timespec="seconds"),
-            maxResults=2500,
-        )
-        .execute()
-    )
-    if "nextPageToken" in request:
-        # TODO: implement pagination
-        raise Exception(
-            "More events than fit on a page, and pagination not implemented"
-        )
-    events = request["items"]
-
-    for event in events:
         if (
-            "extendedProperties" in event
+            parse_google_calendar_datetime(existing_event["start"]) != reservation.start
-            and "private" in event["extendedProperties"]
+            or parse_google_calendar_datetime(existing_event["end"]) != reservation.end
-            and event["extendedProperties"]["private"].get("cmsmanage") == "1"
+            or any(
+                existing_event[k] != v
+                for k, v in changes.items()
+                if k not in ("start", "end")
+            )
         ):
-            try:
+            logger.debug("Updating event")
-                reservation = resource.reservation_set.get_subclass(
+            new_event = existing_event | changes
-                    google_calendar_event_id=event["id"]
+            self.service.events().update(
-                )
+                calendarId=resource.google_calendar,
-                # event exists in both Google Calendar and database, check for update
+                eventId=reservation.google_calendar_event_id,
-                logger.debug(
+                body=new_event,
-                    "Event in Google Calendar found in database: checking for update | %s",
+            ).execute()
-                    event["id"],
+
-                )
+    def insert_calendar_event(self, resource: Resource, reservation: Reservation):
-                update_calendar_event(service, resource, event, reservation)
+        new_gcal_event = reservation.make_google_calendar_event()
-            except Reservation.DoesNotExist:
+        created_event = (
-                # reservation deleted in database, so remove from Google Calendar
+            self.service.events()
-                logger.info(
+            .insert(
-                    "Event in Google Calendar not found in database: deleting | %s",
+                calendarId=resource.google_calendar,
-                    event["id"],
+                body=new_gcal_event,
-                )
-                service.events().delete(
-                    calendarId=resource.google_calendar,
-                    eventId=event["id"],
-                    sendUpdates="none",
-                ).execute()
-        else:
-            logger.debug(
-                "Event in Google Calendar not originated by CMSManage: adding/updating as external reservation | %s",
-                event["id"],
             )
-            # TODO: this might cause issues if something external
+            .execute()
-            # creates events with matching IDs in different calendars
+        )
-            reservation, created = ExternalReservation.objects.update_or_create(
+        reservation.google_calendar_event_id = created_event["id"]
-                google_calendar_event_id=event["id"],
+        reservation.save()
-                defaults={
-                    "title": event["summary"],
-                    "start": parse_google_calendar_datetime(event["start"]),
-                    "end": parse_google_calendar_datetime(event["end"]),
-                },
-            )
-            reservation.resources.add(resource)
 
-    return {event["id"] for event in events}
+    def insert_or_update_calendar_event(
-
+        self, resource: Resource, reservation: Reservation
-
+    ):
-def sync_resource_from_database(
-    service, resource: Resource, now: datetime, existing_event_ids: set[str]
-):
-    reservations = resource.reservation_set.filter(end__gt=now).select_subclasses()
-    # TODO: this could probably be more efficient?
-    for reservation in reservations:
         if not reservation.google_calendar_event_id:
             logger.info(
                 "Event in database has no Google Calendar event ID: inserting | %s",
                 reservation.google_calendar_event_id,
             )
-            insert_calendar_event(service, resource, reservation)
+            self.insert_calendar_event(resource, reservation)
 
-        # reservation has an event id, so check if we already handled it earlier
+        else:
-        elif reservation.google_calendar_event_id not in existing_event_ids:
+            # this event was in Google Calendar at some point (possibly for a different
+            # resource/calendar), but did not appear in list(). Try to update it, then
+            # fall back to insert
+            logger.info(
+                "Reservation with event id not in Google Calendar: trying update | %s",
+                reservation.google_calendar_event_id,
+            )
+            try:
+                event = (
+                    self.service.events()
+                    .get(
+                        calendarId=resource.google_calendar,
+                        eventId=reservation.google_calendar_event_id,
+                    )
+                    .execute()
+                )
+                self.update_calendar_event(resource, event, reservation)
+            except HttpError as error:
+                if error.status_code == HTTPStatus.NOT_FOUND:
+                    logger.info(
+                        "Event in database not in Google Calendar: inserting | %s",
+                        reservation.google_calendar_event_id,
+                    )
+                    self.insert_calendar_event(resource, reservation)
+                else:
+                    raise
+
+    def sync_resource_from_google_calendar(
+        self, resource: Resource, now: datetime
+    ) -> set[str]:
+        request = (
+            self.service.events()
+            .list(
+                calendarId=resource.google_calendar,
+                timeMin=now.isoformat(timespec="seconds"),
+                maxResults=2500,
+            )
+            .execute()
+        )
+        if "nextPageToken" in request:
+            # TODO: implement pagination
+            raise Exception(
+                "More events than fit on a page, and pagination not implemented"
+            )
+        events = request["items"]
+
+        for event in events:
+            if (
+                "extendedProperties" in event
+                and "private" in event["extendedProperties"]
+                and event["extendedProperties"]["private"].get("cmsmanage") == "1"
+            ):
+                try:
+                    reservation = resource.reservation_set.get_subclass(
+                        google_calendar_event_id=event["id"]
+                    )
+                    # event exists in both Google Calendar and database, check for update
+                    logger.debug(
+                        "Event in Google Calendar found in database: checking for update | %s",
+                        event["id"],
+                    )
+                    self.update_calendar_event(resource, event, reservation)
+                except Reservation.DoesNotExist:
+                    # reservation deleted in database, so remove from Google Calendar
+                    logger.info(
+                        "Event in Google Calendar not found in database: deleting | %s",
+                        event["id"],
+                    )
+                    self.service.events().delete(
+                        calendarId=resource.google_calendar,
+                        eventId=event["id"],
+                        sendUpdates="none",
+                    ).execute()
+            else:
+                logger.debug(
+                    "Event in Google Calendar not originated by CMSManage: adding/updating as external reservation | %s",
+                    event["id"],
+                )
+                # TODO: this might cause issues if something external
+                # creates events with matching IDs in different calendars
+                reservation, created = ExternalReservation.objects.update_or_create(
+                    google_calendar_event_id=event["id"],
+                    defaults={
+                        "title": event["summary"],
+                        "start": parse_google_calendar_datetime(event["start"]),
+                        "end": parse_google_calendar_datetime(event["end"]),
+                    },
+                )
+                reservation.resources.add(resource)
+
+        return {event["id"] for event in events}
+
+    def sync_reservation_from_database(self, reservation: Reservation):
+        for resource in reservation.resources.all():
+            self.insert_or_update_calendar_event(resource, reservation)
+
+    def sync_resource_from_database(
+        self, resource: Resource, now: datetime, existing_event_ids: set[str]
+    ):
+        reservations = (
+            resource.reservation_set.filter(end__gt=now)
+            # skip events we already pulled from Google Calendar during this sync
+            .exclude(google_calendar_event_id__in=existing_event_ids)
+            .select_subclasses()
+        )
+        # TODO: this could probably be more efficient?
+        for reservation in reservations:
             if isinstance(reservation, ExternalReservation):
                 logger.info(
                     "External event in database did not exist in future of Google Calendar: deleting locally | %s",
                     reservation.google_calendar_event_id,
                 )
                 reservation.delete()
+
             else:
-                # this event was in Google Calendar at some point (possibly for a different
+                self.insert_or_update_calendar_event(resource, reservation)
-                # resource/calendar), but did not appear in list(). Try to update it, then
-                # fall back to insert
-                logger.info(
-                    "Reservation with event id not in Google Calendar: trying update | %s",
-                    reservation.google_calendar_event_id,
-                )
-                try:
-                    event = (
-                        service.events()
-                        .get(
-                            calendarId=resource.google_calendar,
-                            eventId=reservation.google_calendar_event_id,
-                        )
-                        .execute()
-                    )
-                    update_calendar_event(service, resource, event, reservation)
-                except HttpError as error:
-                    if error.status_code == HTTPStatus.NOT_FOUND:
-                        logger.info(
-                            "Event in database not in Google Calendar: inserting | %s",
-                            reservation.google_calendar_event_id,
-                        )
-                        insert_calendar_event(service, resource, reservation)
-                    else:
-                        raise
 
+    def sync_resource(self, resource: Resource, now: datetime):
+        logger.info(
+            "Checking calendar %s for resource %s", resource.google_calendar, resource
+        )
 
-def sync_resource(service, resource: Resource, now: datetime):
+        existing_event_ids = self.sync_resource_from_google_calendar(resource, now)
-    logger.info(
+        self.sync_resource_from_database(resource, now, existing_event_ids)
-        "Checking calendar %s for resource %s", resource.google_calendar, resource
-    )
-
-    existing_event_ids = sync_resource_from_google_calendar(service, resource, now)
-    sync_resource_from_database(service, resource, now, existing_event_ids)
 
 
 @q_task_group("Sync Reservations with Google Calendar")
 def sync_reservations_with_google_calendar():
-    service = build(
+    synchronizer = GoogleCalendarSynchronizer()
-        "calendar",
-        "v3",
-        credentials=service_account.Credentials.from_service_account_file(
-            settings.GOOGLE_SERVICE_ACCOUNT_FILE,
-            scopes=SCOPES,
-        ),
-    )
-
     now = timezone.now()
 
     for resource in Resource.objects.all():
-        sync_resource(service, resource, now)
+        synchronizer.sync_resource(resource, now)