membershipworks: Scrape event data, with extension model for extra data

This commit is contained in:
Adam Goldsmith 2023-12-30 14:34:55 -05:00
parent 546b13428e
commit f5688e39c3
6 changed files with 413 additions and 5 deletions

View File

@ -5,7 +5,15 @@ from django_object_actions import DjangoObjectActions, action
from django_q.tasks import async_task from django_q.tasks import async_task
from django_q.models import Task from django_q.models import Task
from .models import Member, Flag, Transaction from .models import (
Member,
Flag,
Transaction,
Event,
EventExt,
EventMeetingTime,
EventInstructor,
)
from .tasks.scrape import scrape_membershipworks from .tasks.scrape import scrape_membershipworks
@ -73,3 +81,55 @@ class TransactionAdmin(BaseMembershipWorksAdmin):
show_facets = admin.ShowFacets.ALWAYS show_facets = admin.ShowFacets.ALWAYS
search_fields = ["member", "name", "type", "note"] search_fields = ["member", "name", "type", "note"]
date_hierarchy = "timestamp" date_hierarchy = "timestamp"
class EventMeetingTimeInline(admin.TabularInline):
model = EventMeetingTime
extra = 0
min_num = 1
readonly_fields = ["duration"]
# TODO: remove when switched to GeneratedField
@admin.display()
def duration(self, obj):
return obj.duration
@admin.register(EventInstructor)
class EventInstructorAdmin(admin.ModelAdmin):
autocomplete_fields = ["member"]
@admin.register(EventExt)
class EventAdmin(admin.ModelAdmin):
inlines = [EventMeetingTimeInline]
list_display = [
"title",
"start",
"end",
"duration",
"count",
"cap",
"category",
"venue",
]
list_filter = ["category", "calendar", "venue"]
show_facets = admin.ShowFacets.ALWAYS
search_fields = ["eid", "title", "url"]
date_hierarchy = "start"
readonly_fields = [
field.name
for field in Event._meta.get_fields()
if not (field.auto_created or field.many_to_many or not field.concrete)
] + ["duration"]
@admin.display()
def duration(self, obj):
return obj.duration
def has_add_permission(self, request, obj=None):
return False
def has_delete_permission(self, request, obj=None):
return False

View File

@ -7,7 +7,7 @@ def post_migrate_callback(sender, **kwargs):
from cmsmanage.django_q2_helper import ensure_scheduled from cmsmanage.django_q2_helper import ensure_scheduled
from .tasks.scrape import scrape_membershipworks from .tasks.scrape import scrape_membershipworks, scrape_events
from .tasks.ucsAccounts import sync_accounts from .tasks.ucsAccounts import sync_accounts
ensure_scheduled( ensure_scheduled(
@ -16,6 +16,12 @@ def post_migrate_callback(sender, **kwargs):
schedule_type=Schedule.HOURLY, schedule_type=Schedule.HOURLY,
) )
ensure_scheduled(
"Scrape MembershipWorks Events",
scrape_events,
schedule_type=Schedule.HOURLY,
)
ensure_scheduled( ensure_scheduled(
"Sync UCS Accounts", "Sync UCS Accounts",
sync_accounts, sync_accounts,

View File

@ -0,0 +1,8 @@
from django.core.management.base import BaseCommand
from membershipworks.tasks.scrape import scrape_events
class Command(BaseCommand):
def handle(self, *args, **options):
scrape_events()

View File

@ -0,0 +1,154 @@
# Generated by Django 5.0 on 2023-12-30 19:28
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("membershipworks", "0004_alter_memberflag_member_alter_transaction_member"),
]
operations = [
migrations.CreateModel(
name="Event",
fields=[
(
"eid",
models.CharField(max_length=255, primary_key=True, serialize=False),
),
("url", models.TextField()),
("title", models.TextField()),
("start", models.DateTimeField()),
("end", models.DateTimeField(blank=True, null=True)),
("cap", models.IntegerField(blank=True, null=True)),
("count", models.IntegerField()),
(
"calendar",
models.IntegerField(
choices=[
(0, "Hidden"),
(1, "Green"),
(2, "Red"),
(3, "Yellow"),
(4, "Blue"),
(5, "Purple"),
(6, "Magenta"),
(7, "Grey"),
(8, "Teal"),
]
),
),
("venue", models.TextField(blank=True, null=True)),
],
options={
"abstract": False,
},
),
migrations.CreateModel(
name="EventCategory",
fields=[
("id", models.IntegerField(primary_key=True, serialize=False)),
("title", models.TextField()),
],
),
migrations.CreateModel(
name="EventExt",
fields=[
(
"event_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="membershipworks.event",
),
),
(
"materials_fee",
models.DecimalField(
blank=True, decimal_places=4, max_digits=13, null=True
),
),
],
options={
"verbose_name": "event",
},
bases=("membershipworks.event",),
),
migrations.AddField(
model_name="event",
name="category",
field=models.ForeignKey(
on_delete=django.db.models.deletion.PROTECT,
to="membershipworks.eventcategory",
),
),
migrations.CreateModel(
name="EventInstructor",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.TextField(blank=True)),
(
"member",
models.OneToOneField(
blank=True,
db_constraint=False,
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="membershipworks.member",
),
),
],
),
migrations.CreateModel(
name="EventMeetingTime",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("start", models.DateTimeField()),
("end", models.DateTimeField()),
(
"event",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="meeting_times",
to="membershipworks.eventext",
),
),
],
),
migrations.AddField(
model_name="eventext",
name="instructor",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.PROTECT,
to="membershipworks.eventinstructor",
),
),
migrations.AddConstraint(
model_name="eventmeetingtime",
constraint=models.UniqueConstraint(
fields=("event", "start", "end"), name="unique_event_start_end"
),
),
]

View File

@ -4,7 +4,7 @@ from datetime import datetime
import django.core.mail.message import django.core.mail.message
from django.conf import settings from django.conf import settings
from django.db import models from django.db import models
from django.db.models import Exists, OuterRef from django.db.models import Exists, F, OuterRef, Sum
from django.utils import timezone from django.utils import timezone
@ -340,3 +340,133 @@ class Transaction(BaseModel):
class Meta: class Meta:
db_table = "transactions" db_table = "transactions"
class EventCategory(models.Model):
id = models.IntegerField(primary_key=True)
title = models.TextField()
@classmethod
def from_api_dict(cls, id: int, data):
return cls(id=id, title=data["ttl"])
def __str__(self):
return self.title
class Event(BaseModel):
class EventCalendar(models.IntegerChoices):
HIDDEN = 0
GREEN = 1
RED = 2
YELLOW = 3
BLUE = 4
PURPLE = 5
MAGENTA = 6
GREY = 7
TEAL = 8
eid = models.CharField(max_length=255, primary_key=True)
url = models.TextField()
title = models.TextField()
start = models.DateTimeField()
end = models.DateTimeField(null=True, blank=True)
cap = models.IntegerField(null=True, blank=True)
count = models.IntegerField()
category = models.ForeignKey(EventCategory, on_delete=models.PROTECT)
calendar = models.IntegerField(choices=EventCalendar)
venue = models.TextField(null=True, blank=True)
# TODO:
# "lgo": {
# "l": "https://d1tif55lvfk8gc.cloudfront.net/656e3842ae3975908b05e304.jpg?1673405126",
# "s": "https://d1tif55lvfk8gc.cloudfront.net/656e3842ae3975908b05e304s.jpg?1673405126"
# },
_api_names_override = {
"title": "ttl",
"category_id": "grp",
"start": "sdp",
"end": "edp",
"count": "cnt",
"calendar": "cal",
"venue": "adn",
}
_date_fields = {
"sdp": None,
"edp": None,
}
_allowed_missing_fields = ["cap", "edp", "adn"]
def __str__(self):
return self.title
class EventInstructor(models.Model):
name = models.TextField(blank=True)
member = models.OneToOneField(
Member, on_delete=models.PROTECT, null=True, blank=True, db_constraint=False
)
def __str__(self):
return str(self.member) if self.member else self.name
class EventExtManager(models.Manager["EventExt"]):
def get_queryset(self) -> models.QuerySet["EventExt"]:
return (
super()
.get_queryset()
.annotate(duration=Sum(F("meeting_times__end") - F("meeting_times__start")))
)
# TODO: use simpler expression when GeneratedField fixed
# return super().get_queryset().annotate(duration=Sum("meeting_times__duration"))
class EventExt(Event):
"""Extension of `Event` to capture some fields not supported in MembershipWorks"""
objects = EventExtManager()
instructor = models.ForeignKey(
EventInstructor, on_delete=models.PROTECT, null=True, blank=True
)
materials_fee = models.DecimalField(
max_digits=13, decimal_places=4, null=True, blank=True
)
class Meta:
verbose_name = "event"
class EventMeetingTimeManager(models.Manager["EventMeetingTime"]):
def get_queryset(self) -> models.QuerySet["EventMeetingTime"]:
return super().get_queryset().annotate(duration=F("end") - F("start"))
class EventMeetingTime(models.Model):
objects = EventMeetingTimeManager()
event = models.ForeignKey(
EventExt, on_delete=models.CASCADE, related_name="meeting_times"
)
start = models.DateTimeField()
end = models.DateTimeField()
# TODO: Should use generated field instead of manager, but this is
# broken due to current Django bug, pending next release (> 5.0)
# ref: https://code.djangoproject.com/ticket/35019
# duration = models.GeneratedField(
# expression=F("end") - F("start"),
# output_field=models.DurationField(),
# db_persist=False,
# )
class Meta:
constraints = [
models.UniqueConstraint(
fields=["event", "start", "end"], name="unique_event_start_end"
)
]

View File

@ -4,12 +4,20 @@ import logging
from django.conf import settings from django.conf import settings
from django.db import transaction from django.db import transaction
from membershipworks.models import Member, Flag, Transaction from membershipworks.models import (
Member,
Flag,
Transaction,
Event,
EventExt,
EventCategory,
)
from membershipworks import MembershipWorks from membershipworks import MembershipWorks
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
MAX_MEETING_TIME = timedelta(hours=6)
def flags_for_member(csv_member, all_flags, folders): def flags_for_member(csv_member, all_flags, folders):
for flag in all_flags: for flag in all_flags:
@ -91,3 +99,45 @@ def scrape_membershipworks(*args, **options):
scrape_members(membershipworks) scrape_members(membershipworks)
scrape_transactions(membershipworks) scrape_transactions(membershipworks)
def scrape_events():
membershipworks = MembershipWorks()
membershipworks.login(
settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
)
data = membershipworks.get_events_list(
datetime.fromtimestamp(0), datetime.now() + timedelta(weeks=52), categories=True
)
logger.info(f"{len(data)} events retrieved!")
for category_id, category_data in enumerate(data["_st"]["evg"]):
category = EventCategory.from_api_dict(category_id, category_data)
category.clean_fields()
category.save()
for event_data in data["evt"]:
logger.debug(event_data)
event = Event.from_api_dict(event_data)
event.clean_fields()
event.save()
try:
event_ext = EventExt.objects.get(event_ptr=event)
except EventExt.DoesNotExist:
event_ext = EventExt(event_ptr=event)
# create extension model instance
event_ext.save_base(raw=True)
event_ext.refresh_from_db()
if (
event_ext.end is not None
and event_ext.end - event_ext.start < MAX_MEETING_TIME
):
meeting_times_count = event_ext.meeting_times.count()
if meeting_times_count == 0:
event_ext.meeting_times.create(start=event_ext.start, end=event_ext.end)
# if there is exactly one meeting time, it should match the event start/end
elif meeting_times_count == 1:
event_ext.meeting_times.update(start=event_ext.start, end=event_ext.end)