membershipworks: Scrape event data, with extension model for extra data

membershipworks/admin.py

@@ -5,7 +5,15 @@ from django_object_actions import DjangoObjectActions, action
 from django_q.tasks import async_task
 from django_q.models import Task
 
-from .models import Member, Flag, Transaction
+from .models import (
+    Member,
+    Flag,
+    Transaction,
+    Event,
+    EventExt,
+    EventMeetingTime,
+    EventInstructor,
+)
 from .tasks.scrape import scrape_membershipworks
 
 
@@ -73,3 +81,55 @@ class TransactionAdmin(BaseMembershipWorksAdmin):
     show_facets = admin.ShowFacets.ALWAYS
     search_fields = ["member", "name", "type", "note"]
     date_hierarchy = "timestamp"
+
+
+class EventMeetingTimeInline(admin.TabularInline):
+    model = EventMeetingTime
+    extra = 0
+    min_num = 1
+
+    readonly_fields = ["duration"]
+
+    # TODO: remove when switched to GeneratedField
+    @admin.display()
+    def duration(self, obj):
+        return obj.duration
+
+
+@admin.register(EventInstructor)
+class EventInstructorAdmin(admin.ModelAdmin):
+    autocomplete_fields = ["member"]
+
+
+@admin.register(EventExt)
+class EventAdmin(admin.ModelAdmin):
+    inlines = [EventMeetingTimeInline]
+    list_display = [
+        "title",
+        "start",
+        "end",
+        "duration",
+        "count",
+        "cap",
+        "category",
+        "venue",
+    ]
+    list_filter = ["category", "calendar", "venue"]
+    show_facets = admin.ShowFacets.ALWAYS
+    search_fields = ["eid", "title", "url"]
+    date_hierarchy = "start"
+    readonly_fields = [
+        field.name
+        for field in Event._meta.get_fields()
+        if not (field.auto_created or field.many_to_many or not field.concrete)
+    ] + ["duration"]
+
+    @admin.display()
+    def duration(self, obj):
+        return obj.duration
+
+    def has_add_permission(self, request, obj=None):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False

membershipworks/apps.py

@@ -7,7 +7,7 @@ def post_migrate_callback(sender, **kwargs):
 
     from cmsmanage.django_q2_helper import ensure_scheduled
 
-    from .tasks.scrape import scrape_membershipworks
+    from .tasks.scrape import scrape_membershipworks, scrape_events
     from .tasks.ucsAccounts import sync_accounts
 
     ensure_scheduled(
@@ -16,6 +16,12 @@ def post_migrate_callback(sender, **kwargs):
         schedule_type=Schedule.HOURLY,
     )
 
+    ensure_scheduled(
+        "Scrape MembershipWorks Events",
+        scrape_events,
+        schedule_type=Schedule.HOURLY,
+    )
+
     ensure_scheduled(
         "Sync UCS Accounts",
         sync_accounts,

membershipworks/management/commands/scrape_events.py

@@ -0,0 +1,8 @@
+from django.core.management.base import BaseCommand
+
+from membershipworks.tasks.scrape import scrape_events
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        scrape_events()

membershipworks/migrations/0005_event_eventcategory_eventext_event_category_and_more.py

@@ -0,0 +1,154 @@
+# Generated by Django 5.0 on 2023-12-30 19:28
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("membershipworks", "0004_alter_memberflag_member_alter_transaction_member"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Event",
+            fields=[
+                (
+                    "eid",
+                    models.CharField(max_length=255, primary_key=True, serialize=False),
+                ),
+                ("url", models.TextField()),
+                ("title", models.TextField()),
+                ("start", models.DateTimeField()),
+                ("end", models.DateTimeField(blank=True, null=True)),
+                ("cap", models.IntegerField(blank=True, null=True)),
+                ("count", models.IntegerField()),
+                (
+                    "calendar",
+                    models.IntegerField(
+                        choices=[
+                            (0, "Hidden"),
+                            (1, "Green"),
+                            (2, "Red"),
+                            (3, "Yellow"),
+                            (4, "Blue"),
+                            (5, "Purple"),
+                            (6, "Magenta"),
+                            (7, "Grey"),
+                            (8, "Teal"),
+                        ]
+                    ),
+                ),
+                ("venue", models.TextField(blank=True, null=True)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="EventCategory",
+            fields=[
+                ("id", models.IntegerField(primary_key=True, serialize=False)),
+                ("title", models.TextField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name="EventExt",
+            fields=[
+                (
+                    "event_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="membershipworks.event",
+                    ),
+                ),
+                (
+                    "materials_fee",
+                    models.DecimalField(
+                        blank=True, decimal_places=4, max_digits=13, null=True
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "event",
+            },
+            bases=("membershipworks.event",),
+        ),
+        migrations.AddField(
+            model_name="event",
+            name="category",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT,
+                to="membershipworks.eventcategory",
+            ),
+        ),
+        migrations.CreateModel(
+            name="EventInstructor",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.TextField(blank=True)),
+                (
+                    "member",
+                    models.OneToOneField(
+                        blank=True,
+                        db_constraint=False,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        to="membershipworks.member",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="EventMeetingTime",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("start", models.DateTimeField()),
+                ("end", models.DateTimeField()),
+                (
+                    "event",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="meeting_times",
+                        to="membershipworks.eventext",
+                    ),
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name="eventext",
+            name="instructor",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                to="membershipworks.eventinstructor",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="eventmeetingtime",
+            constraint=models.UniqueConstraint(
+                fields=("event", "start", "end"), name="unique_event_start_end"
+            ),
+        ),
+    ]

membershipworks/models.py

@@ -4,7 +4,7 @@ from datetime import datetime
 import django.core.mail.message
 from django.conf import settings
 from django.db import models
-from django.db.models import Exists, OuterRef
+from django.db.models import Exists, F, OuterRef, Sum
 from django.utils import timezone
 
 
@@ -340,3 +340,133 @@ class Transaction(BaseModel):
 
     class Meta:
         db_table = "transactions"
+
+
+class EventCategory(models.Model):
+    id = models.IntegerField(primary_key=True)
+    title = models.TextField()
+
+    @classmethod
+    def from_api_dict(cls, id: int, data):
+        return cls(id=id, title=data["ttl"])
+
+    def __str__(self):
+        return self.title
+
+
+class Event(BaseModel):
+    class EventCalendar(models.IntegerChoices):
+        HIDDEN = 0
+        GREEN = 1
+        RED = 2
+        YELLOW = 3
+        BLUE = 4
+        PURPLE = 5
+        MAGENTA = 6
+        GREY = 7
+        TEAL = 8
+
+    eid = models.CharField(max_length=255, primary_key=True)
+    url = models.TextField()
+    title = models.TextField()
+    start = models.DateTimeField()
+    end = models.DateTimeField(null=True, blank=True)
+    cap = models.IntegerField(null=True, blank=True)
+    count = models.IntegerField()
+    category = models.ForeignKey(EventCategory, on_delete=models.PROTECT)
+    calendar = models.IntegerField(choices=EventCalendar)
+    venue = models.TextField(null=True, blank=True)
+    # TODO:
+    # "lgo": {
+    #   "l": "https://d1tif55lvfk8gc.cloudfront.net/656e3842ae3975908b05e304.jpg?1673405126",
+    #   "s": "https://d1tif55lvfk8gc.cloudfront.net/656e3842ae3975908b05e304s.jpg?1673405126"
+    # },
+
+    _api_names_override = {
+        "title": "ttl",
+        "category_id": "grp",
+        "start": "sdp",
+        "end": "edp",
+        "count": "cnt",
+        "calendar": "cal",
+        "venue": "adn",
+    }
+
+    _date_fields = {
+        "sdp": None,
+        "edp": None,
+    }
+
+    _allowed_missing_fields = ["cap", "edp", "adn"]
+
+    def __str__(self):
+        return self.title
+
+
+class EventInstructor(models.Model):
+    name = models.TextField(blank=True)
+    member = models.OneToOneField(
+        Member, on_delete=models.PROTECT, null=True, blank=True, db_constraint=False
+    )
+
+    def __str__(self):
+        return str(self.member) if self.member else self.name
+
+
+class EventExtManager(models.Manager["EventExt"]):
+    def get_queryset(self) -> models.QuerySet["EventExt"]:
+        return (
+            super()
+            .get_queryset()
+            .annotate(duration=Sum(F("meeting_times__end") - F("meeting_times__start")))
+        )
+        # TODO: use simpler expression when GeneratedField fixed
+        # return super().get_queryset().annotate(duration=Sum("meeting_times__duration"))
+
+
+class EventExt(Event):
+    """Extension of `Event` to capture some fields not supported in MembershipWorks"""
+
+    objects = EventExtManager()
+
+    instructor = models.ForeignKey(
+        EventInstructor, on_delete=models.PROTECT, null=True, blank=True
+    )
+    materials_fee = models.DecimalField(
+        max_digits=13, decimal_places=4, null=True, blank=True
+    )
+
+    class Meta:
+        verbose_name = "event"
+
+
+class EventMeetingTimeManager(models.Manager["EventMeetingTime"]):
+    def get_queryset(self) -> models.QuerySet["EventMeetingTime"]:
+        return super().get_queryset().annotate(duration=F("end") - F("start"))
+
+
+class EventMeetingTime(models.Model):
+    objects = EventMeetingTimeManager()
+
+    event = models.ForeignKey(
+        EventExt, on_delete=models.CASCADE, related_name="meeting_times"
+    )
+    start = models.DateTimeField()
+    end = models.DateTimeField()
+
+    # TODO: Should use generated field instead of manager, but this is
+    # broken due to current Django bug, pending next release (> 5.0)
+    # ref: https://code.djangoproject.com/ticket/35019
+
+    # duration = models.GeneratedField(
+    #     expression=F("end") - F("start"),
+    #     output_field=models.DurationField(),
+    #     db_persist=False,
+    # )
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["event", "start", "end"], name="unique_event_start_end"
+            )
+        ]

membershipworks/tasks/scrape.py

@@ -4,12 +4,20 @@ import logging
 from django.conf import settings
 from django.db import transaction
 
-from membershipworks.models import Member, Flag, Transaction
+from membershipworks.models import (
+    Member,
+    Flag,
+    Transaction,
+    Event,
+    EventExt,
+    EventCategory,
+)
 from membershipworks import MembershipWorks
 
-
 logger = logging.getLogger(__name__)
 
+MAX_MEETING_TIME = timedelta(hours=6)
+
 
 def flags_for_member(csv_member, all_flags, folders):
     for flag in all_flags:
@@ -91,3 +99,45 @@ def scrape_membershipworks(*args, **options):
 
     scrape_members(membershipworks)
     scrape_transactions(membershipworks)
+
+
+def scrape_events():
+    membershipworks = MembershipWorks()
+    membershipworks.login(
+        settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
+    )
+
+    data = membershipworks.get_events_list(
+        datetime.fromtimestamp(0), datetime.now() + timedelta(weeks=52), categories=True
+    )
+    logger.info(f"{len(data)} events retrieved!")
+
+    for category_id, category_data in enumerate(data["_st"]["evg"]):
+        category = EventCategory.from_api_dict(category_id, category_data)
+        category.clean_fields()
+        category.save()
+
+    for event_data in data["evt"]:
+        logger.debug(event_data)
+        event = Event.from_api_dict(event_data)
+        event.clean_fields()
+        event.save()
+
+        try:
+            event_ext = EventExt.objects.get(event_ptr=event)
+        except EventExt.DoesNotExist:
+            event_ext = EventExt(event_ptr=event)
+            # create extension model instance
+            event_ext.save_base(raw=True)
+            event_ext.refresh_from_db()
+
+        if (
+            event_ext.end is not None
+            and event_ext.end - event_ext.start < MAX_MEETING_TIME
+        ):
+            meeting_times_count = event_ext.meeting_times.count()
+            if meeting_times_count == 0:
+                event_ext.meeting_times.create(start=event_ext.start, end=event_ext.end)
+            # if there is exactly one meeting time, it should match the event start/end
+            elif meeting_times_count == 1:
+                event_ext.meeting_times.update(start=event_ext.start, end=event_ext.end)