From e10bcd5b49953da6ee54cf5b1268453cf5ffbfad Mon Sep 17 00:00:00 2001 From: Adam Goldsmith Date: Mon, 29 Jan 2024 21:48:19 -0500 Subject: [PATCH] membershipworks: Add storage and scraping for Event details --- membershipworks/admin.py | 22 +++++++++++--- .../migrations/0011_eventext_details.py | 29 +++++++++++++++++++ membershipworks/models.py | 10 +++++++ membershipworks/tasks/scrape.py | 19 ++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 membershipworks/migrations/0011_eventext_details.py diff --git a/membershipworks/admin.py b/membershipworks/admin.py index 11fe82f..a0e16a3 100644 --- a/membershipworks/admin.py +++ b/membershipworks/admin.py @@ -2,7 +2,11 @@ from django.contrib import admin from django.contrib.humanize.templatetags.humanize import naturaltime from django.utils.html import format_html -from django_object_actions import DjangoObjectActions, action +from django_object_actions import ( + DjangoObjectActions, + action, + takes_instance_or_queryset, +) from django_q.models import Task from django_q.tasks import async_task @@ -15,7 +19,10 @@ from .models import ( Member, Transaction, ) -from .tasks.scrape import scrape_membershipworks +from .tasks.scrape import ( + scrape_event_details, + scrape_membershipworks, +) class ReadOnlyAdmin(admin.ModelAdmin): @@ -104,7 +111,7 @@ class EventInstructorAdmin(admin.ModelAdmin): @admin.register(EventExt) -class EventAdmin(admin.ModelAdmin): +class EventAdmin(DjangoObjectActions, admin.ModelAdmin): inlines = [EventMeetingTimeInline] list_display = [ "title", @@ -123,8 +130,10 @@ class EventAdmin(admin.ModelAdmin): show_facets = admin.ShowFacets.ALWAYS search_fields = ["eid", "title", "url"] date_hierarchy = "start" - exclude = ["url"] + exclude = ["url", "details"] autocomplete_fields = ["instructor"] + change_actions = ["fetch_details"] + actions = ["fetch_details"] @property def readonly_fields(self): @@ -137,6 +146,7 @@ class EventAdmin(admin.ModelAdmin): else: fields.append(field.name) fields.insert(fields.index("end") + 1, "duration") + fields.append("details_timestamp") return fields @admin.display(ordering="duration") @@ -150,6 +160,10 @@ class EventAdmin(admin.ModelAdmin): obj.url, ) + @takes_instance_or_queryset + def fetch_details(self, request, queryset): + scrape_event_details(queryset) + def has_add_permission(self, request, obj=None): return False diff --git a/membershipworks/migrations/0011_eventext_details.py b/membershipworks/migrations/0011_eventext_details.py new file mode 100644 index 0000000..3e2f57d --- /dev/null +++ b/membershipworks/migrations/0011_eventext_details.py @@ -0,0 +1,29 @@ +# Generated by Django 5.0.1 on 2024-01-29 19:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("membershipworks", "0010_alter_eventext_options"), + ] + + operations = [ + migrations.AddField( + model_name="eventext", + name="details", + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name="eventext", + name="details_timestamp", + field=models.GeneratedField( + db_persist=False, + expression=models.Func( + models.Func(models.F("details___ts"), function="FROM_UNIXTIME"), + template="CONVERT_TZ(%(expressions)s, @@session.time_zone, 'UTC')", + ), + output_field=models.DateTimeField(), + ), + ), + ] diff --git a/membershipworks/models.py b/membershipworks/models.py index f7ca734..b03cf49 100644 --- a/membershipworks/models.py +++ b/membershipworks/models.py @@ -9,6 +9,7 @@ from django.db.models import ( Exists, ExpressionWrapper, F, + Func, OuterRef, Q, Subquery, @@ -493,6 +494,15 @@ class EventExt(Event): instructor_flat_rate = models.DecimalField( max_digits=13, decimal_places=4, default=0 ) + details = models.JSONField(null=True, blank=True) + details_timestamp = models.GeneratedField( + expression=Func( + Func(F("details___ts"), function="FROM_UNIXTIME"), + template="CONVERT_TZ(%(expressions)s, @@session.time_zone, 'UTC')", + ), + output_field=models.DateTimeField(), + db_persist=False, + ) class Meta: verbose_name = "event" diff --git a/membershipworks/tasks/scrape.py b/membershipworks/tasks/scrape.py index 7082644..1f343ea 100644 --- a/membershipworks/tasks/scrape.py +++ b/membershipworks/tasks/scrape.py @@ -3,6 +3,7 @@ from datetime import datetime, timedelta from django.conf import settings from django.db import transaction +from django.db.models import QuerySet from membershipworks.membershipworks_api import FieldType, MembershipWorks from membershipworks.models import ( @@ -99,6 +100,17 @@ def scrape_membershipworks(*args, **options): scrape_transactions(membershipworks) +def scrape_event_details(queryset: QuerySet[EventExt]): + membershipworks = MembershipWorks() + membershipworks.login( + settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD + ) + + for event in queryset: + event.details = membershipworks.get_event_by_eid(event.eid) + event.save() + + def scrape_events(): membershipworks = MembershipWorks() membershipworks.login( @@ -150,3 +162,10 @@ def scrape_events(): # if there is exactly one meeting time, it should match the event start/end elif meeting_times_count == 1: event_ext.meeting_times.update(start=event_ext.start, end=event_ext.end) + + # event has no details, or last retrieval was before the event happened + if event_ext.details is None or event_ext.details_timestamp < ( + event_ext.end or event_ext.start + ): + event_ext.details = membershipworks.get_event_by_eid(event.eid) + event_ext.save()