membershipworks: Scrape event data, with extension model for extra data
This commit is contained in:
parent
546b13428e
commit
f5688e39c3
@ -5,7 +5,15 @@ from django_object_actions import DjangoObjectActions, action
|
|||||||
from django_q.tasks import async_task
|
from django_q.tasks import async_task
|
||||||
from django_q.models import Task
|
from django_q.models import Task
|
||||||
|
|
||||||
from .models import Member, Flag, Transaction
|
from .models import (
|
||||||
|
Member,
|
||||||
|
Flag,
|
||||||
|
Transaction,
|
||||||
|
Event,
|
||||||
|
EventExt,
|
||||||
|
EventMeetingTime,
|
||||||
|
EventInstructor,
|
||||||
|
)
|
||||||
from .tasks.scrape import scrape_membershipworks
|
from .tasks.scrape import scrape_membershipworks
|
||||||
|
|
||||||
|
|
||||||
@ -73,3 +81,55 @@ class TransactionAdmin(BaseMembershipWorksAdmin):
|
|||||||
show_facets = admin.ShowFacets.ALWAYS
|
show_facets = admin.ShowFacets.ALWAYS
|
||||||
search_fields = ["member", "name", "type", "note"]
|
search_fields = ["member", "name", "type", "note"]
|
||||||
date_hierarchy = "timestamp"
|
date_hierarchy = "timestamp"
|
||||||
|
|
||||||
|
|
||||||
|
class EventMeetingTimeInline(admin.TabularInline):
|
||||||
|
model = EventMeetingTime
|
||||||
|
extra = 0
|
||||||
|
min_num = 1
|
||||||
|
|
||||||
|
readonly_fields = ["duration"]
|
||||||
|
|
||||||
|
# TODO: remove when switched to GeneratedField
|
||||||
|
@admin.display()
|
||||||
|
def duration(self, obj):
|
||||||
|
return obj.duration
|
||||||
|
|
||||||
|
|
||||||
|
@admin.register(EventInstructor)
|
||||||
|
class EventInstructorAdmin(admin.ModelAdmin):
|
||||||
|
autocomplete_fields = ["member"]
|
||||||
|
|
||||||
|
|
||||||
|
@admin.register(EventExt)
|
||||||
|
class EventAdmin(admin.ModelAdmin):
|
||||||
|
inlines = [EventMeetingTimeInline]
|
||||||
|
list_display = [
|
||||||
|
"title",
|
||||||
|
"start",
|
||||||
|
"end",
|
||||||
|
"duration",
|
||||||
|
"count",
|
||||||
|
"cap",
|
||||||
|
"category",
|
||||||
|
"venue",
|
||||||
|
]
|
||||||
|
list_filter = ["category", "calendar", "venue"]
|
||||||
|
show_facets = admin.ShowFacets.ALWAYS
|
||||||
|
search_fields = ["eid", "title", "url"]
|
||||||
|
date_hierarchy = "start"
|
||||||
|
readonly_fields = [
|
||||||
|
field.name
|
||||||
|
for field in Event._meta.get_fields()
|
||||||
|
if not (field.auto_created or field.many_to_many or not field.concrete)
|
||||||
|
] + ["duration"]
|
||||||
|
|
||||||
|
@admin.display()
|
||||||
|
def duration(self, obj):
|
||||||
|
return obj.duration
|
||||||
|
|
||||||
|
def has_add_permission(self, request, obj=None):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def has_delete_permission(self, request, obj=None):
|
||||||
|
return False
|
||||||
|
@ -7,7 +7,7 @@ def post_migrate_callback(sender, **kwargs):
|
|||||||
|
|
||||||
from cmsmanage.django_q2_helper import ensure_scheduled
|
from cmsmanage.django_q2_helper import ensure_scheduled
|
||||||
|
|
||||||
from .tasks.scrape import scrape_membershipworks
|
from .tasks.scrape import scrape_membershipworks, scrape_events
|
||||||
from .tasks.ucsAccounts import sync_accounts
|
from .tasks.ucsAccounts import sync_accounts
|
||||||
|
|
||||||
ensure_scheduled(
|
ensure_scheduled(
|
||||||
@ -16,6 +16,12 @@ def post_migrate_callback(sender, **kwargs):
|
|||||||
schedule_type=Schedule.HOURLY,
|
schedule_type=Schedule.HOURLY,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
ensure_scheduled(
|
||||||
|
"Scrape MembershipWorks Events",
|
||||||
|
scrape_events,
|
||||||
|
schedule_type=Schedule.HOURLY,
|
||||||
|
)
|
||||||
|
|
||||||
ensure_scheduled(
|
ensure_scheduled(
|
||||||
"Sync UCS Accounts",
|
"Sync UCS Accounts",
|
||||||
sync_accounts,
|
sync_accounts,
|
||||||
|
8
membershipworks/management/commands/scrape_events.py
Normal file
8
membershipworks/management/commands/scrape_events.py
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
from django.core.management.base import BaseCommand
|
||||||
|
|
||||||
|
from membershipworks.tasks.scrape import scrape_events
|
||||||
|
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
def handle(self, *args, **options):
|
||||||
|
scrape_events()
|
@ -0,0 +1,154 @@
|
|||||||
|
# Generated by Django 5.0 on 2023-12-30 19:28
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies = [
|
||||||
|
("membershipworks", "0004_alter_memberflag_member_alter_transaction_member"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name="Event",
|
||||||
|
fields=[
|
||||||
|
(
|
||||||
|
"eid",
|
||||||
|
models.CharField(max_length=255, primary_key=True, serialize=False),
|
||||||
|
),
|
||||||
|
("url", models.TextField()),
|
||||||
|
("title", models.TextField()),
|
||||||
|
("start", models.DateTimeField()),
|
||||||
|
("end", models.DateTimeField(blank=True, null=True)),
|
||||||
|
("cap", models.IntegerField(blank=True, null=True)),
|
||||||
|
("count", models.IntegerField()),
|
||||||
|
(
|
||||||
|
"calendar",
|
||||||
|
models.IntegerField(
|
||||||
|
choices=[
|
||||||
|
(0, "Hidden"),
|
||||||
|
(1, "Green"),
|
||||||
|
(2, "Red"),
|
||||||
|
(3, "Yellow"),
|
||||||
|
(4, "Blue"),
|
||||||
|
(5, "Purple"),
|
||||||
|
(6, "Magenta"),
|
||||||
|
(7, "Grey"),
|
||||||
|
(8, "Teal"),
|
||||||
|
]
|
||||||
|
),
|
||||||
|
),
|
||||||
|
("venue", models.TextField(blank=True, null=True)),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
"abstract": False,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
migrations.CreateModel(
|
||||||
|
name="EventCategory",
|
||||||
|
fields=[
|
||||||
|
("id", models.IntegerField(primary_key=True, serialize=False)),
|
||||||
|
("title", models.TextField()),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
migrations.CreateModel(
|
||||||
|
name="EventExt",
|
||||||
|
fields=[
|
||||||
|
(
|
||||||
|
"event_ptr",
|
||||||
|
models.OneToOneField(
|
||||||
|
auto_created=True,
|
||||||
|
on_delete=django.db.models.deletion.CASCADE,
|
||||||
|
parent_link=True,
|
||||||
|
primary_key=True,
|
||||||
|
serialize=False,
|
||||||
|
to="membershipworks.event",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"materials_fee",
|
||||||
|
models.DecimalField(
|
||||||
|
blank=True, decimal_places=4, max_digits=13, null=True
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
"verbose_name": "event",
|
||||||
|
},
|
||||||
|
bases=("membershipworks.event",),
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="event",
|
||||||
|
name="category",
|
||||||
|
field=models.ForeignKey(
|
||||||
|
on_delete=django.db.models.deletion.PROTECT,
|
||||||
|
to="membershipworks.eventcategory",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
migrations.CreateModel(
|
||||||
|
name="EventInstructor",
|
||||||
|
fields=[
|
||||||
|
(
|
||||||
|
"id",
|
||||||
|
models.BigAutoField(
|
||||||
|
auto_created=True,
|
||||||
|
primary_key=True,
|
||||||
|
serialize=False,
|
||||||
|
verbose_name="ID",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
("name", models.TextField(blank=True)),
|
||||||
|
(
|
||||||
|
"member",
|
||||||
|
models.OneToOneField(
|
||||||
|
blank=True,
|
||||||
|
db_constraint=False,
|
||||||
|
null=True,
|
||||||
|
on_delete=django.db.models.deletion.PROTECT,
|
||||||
|
to="membershipworks.member",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
migrations.CreateModel(
|
||||||
|
name="EventMeetingTime",
|
||||||
|
fields=[
|
||||||
|
(
|
||||||
|
"id",
|
||||||
|
models.BigAutoField(
|
||||||
|
auto_created=True,
|
||||||
|
primary_key=True,
|
||||||
|
serialize=False,
|
||||||
|
verbose_name="ID",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
("start", models.DateTimeField()),
|
||||||
|
("end", models.DateTimeField()),
|
||||||
|
(
|
||||||
|
"event",
|
||||||
|
models.ForeignKey(
|
||||||
|
on_delete=django.db.models.deletion.CASCADE,
|
||||||
|
related_name="meeting_times",
|
||||||
|
to="membershipworks.eventext",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="eventext",
|
||||||
|
name="instructor",
|
||||||
|
field=models.ForeignKey(
|
||||||
|
blank=True,
|
||||||
|
null=True,
|
||||||
|
on_delete=django.db.models.deletion.PROTECT,
|
||||||
|
to="membershipworks.eventinstructor",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
migrations.AddConstraint(
|
||||||
|
model_name="eventmeetingtime",
|
||||||
|
constraint=models.UniqueConstraint(
|
||||||
|
fields=("event", "start", "end"), name="unique_event_start_end"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
@ -4,7 +4,7 @@ from datetime import datetime
|
|||||||
import django.core.mail.message
|
import django.core.mail.message
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.db.models import Exists, OuterRef
|
from django.db.models import Exists, F, OuterRef, Sum
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
|
|
||||||
@ -340,3 +340,133 @@ class Transaction(BaseModel):
|
|||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
db_table = "transactions"
|
db_table = "transactions"
|
||||||
|
|
||||||
|
|
||||||
|
class EventCategory(models.Model):
|
||||||
|
id = models.IntegerField(primary_key=True)
|
||||||
|
title = models.TextField()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_api_dict(cls, id: int, data):
|
||||||
|
return cls(id=id, title=data["ttl"])
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return self.title
|
||||||
|
|
||||||
|
|
||||||
|
class Event(BaseModel):
|
||||||
|
class EventCalendar(models.IntegerChoices):
|
||||||
|
HIDDEN = 0
|
||||||
|
GREEN = 1
|
||||||
|
RED = 2
|
||||||
|
YELLOW = 3
|
||||||
|
BLUE = 4
|
||||||
|
PURPLE = 5
|
||||||
|
MAGENTA = 6
|
||||||
|
GREY = 7
|
||||||
|
TEAL = 8
|
||||||
|
|
||||||
|
eid = models.CharField(max_length=255, primary_key=True)
|
||||||
|
url = models.TextField()
|
||||||
|
title = models.TextField()
|
||||||
|
start = models.DateTimeField()
|
||||||
|
end = models.DateTimeField(null=True, blank=True)
|
||||||
|
cap = models.IntegerField(null=True, blank=True)
|
||||||
|
count = models.IntegerField()
|
||||||
|
category = models.ForeignKey(EventCategory, on_delete=models.PROTECT)
|
||||||
|
calendar = models.IntegerField(choices=EventCalendar)
|
||||||
|
venue = models.TextField(null=True, blank=True)
|
||||||
|
# TODO:
|
||||||
|
# "lgo": {
|
||||||
|
# "l": "https://d1tif55lvfk8gc.cloudfront.net/656e3842ae3975908b05e304.jpg?1673405126",
|
||||||
|
# "s": "https://d1tif55lvfk8gc.cloudfront.net/656e3842ae3975908b05e304s.jpg?1673405126"
|
||||||
|
# },
|
||||||
|
|
||||||
|
_api_names_override = {
|
||||||
|
"title": "ttl",
|
||||||
|
"category_id": "grp",
|
||||||
|
"start": "sdp",
|
||||||
|
"end": "edp",
|
||||||
|
"count": "cnt",
|
||||||
|
"calendar": "cal",
|
||||||
|
"venue": "adn",
|
||||||
|
}
|
||||||
|
|
||||||
|
_date_fields = {
|
||||||
|
"sdp": None,
|
||||||
|
"edp": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
_allowed_missing_fields = ["cap", "edp", "adn"]
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return self.title
|
||||||
|
|
||||||
|
|
||||||
|
class EventInstructor(models.Model):
|
||||||
|
name = models.TextField(blank=True)
|
||||||
|
member = models.OneToOneField(
|
||||||
|
Member, on_delete=models.PROTECT, null=True, blank=True, db_constraint=False
|
||||||
|
)
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return str(self.member) if self.member else self.name
|
||||||
|
|
||||||
|
|
||||||
|
class EventExtManager(models.Manager["EventExt"]):
|
||||||
|
def get_queryset(self) -> models.QuerySet["EventExt"]:
|
||||||
|
return (
|
||||||
|
super()
|
||||||
|
.get_queryset()
|
||||||
|
.annotate(duration=Sum(F("meeting_times__end") - F("meeting_times__start")))
|
||||||
|
)
|
||||||
|
# TODO: use simpler expression when GeneratedField fixed
|
||||||
|
# return super().get_queryset().annotate(duration=Sum("meeting_times__duration"))
|
||||||
|
|
||||||
|
|
||||||
|
class EventExt(Event):
|
||||||
|
"""Extension of `Event` to capture some fields not supported in MembershipWorks"""
|
||||||
|
|
||||||
|
objects = EventExtManager()
|
||||||
|
|
||||||
|
instructor = models.ForeignKey(
|
||||||
|
EventInstructor, on_delete=models.PROTECT, null=True, blank=True
|
||||||
|
)
|
||||||
|
materials_fee = models.DecimalField(
|
||||||
|
max_digits=13, decimal_places=4, null=True, blank=True
|
||||||
|
)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
verbose_name = "event"
|
||||||
|
|
||||||
|
|
||||||
|
class EventMeetingTimeManager(models.Manager["EventMeetingTime"]):
|
||||||
|
def get_queryset(self) -> models.QuerySet["EventMeetingTime"]:
|
||||||
|
return super().get_queryset().annotate(duration=F("end") - F("start"))
|
||||||
|
|
||||||
|
|
||||||
|
class EventMeetingTime(models.Model):
|
||||||
|
objects = EventMeetingTimeManager()
|
||||||
|
|
||||||
|
event = models.ForeignKey(
|
||||||
|
EventExt, on_delete=models.CASCADE, related_name="meeting_times"
|
||||||
|
)
|
||||||
|
start = models.DateTimeField()
|
||||||
|
end = models.DateTimeField()
|
||||||
|
|
||||||
|
# TODO: Should use generated field instead of manager, but this is
|
||||||
|
# broken due to current Django bug, pending next release (> 5.0)
|
||||||
|
# ref: https://code.djangoproject.com/ticket/35019
|
||||||
|
|
||||||
|
# duration = models.GeneratedField(
|
||||||
|
# expression=F("end") - F("start"),
|
||||||
|
# output_field=models.DurationField(),
|
||||||
|
# db_persist=False,
|
||||||
|
# )
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
constraints = [
|
||||||
|
models.UniqueConstraint(
|
||||||
|
fields=["event", "start", "end"], name="unique_event_start_end"
|
||||||
|
)
|
||||||
|
]
|
||||||
|
@ -4,12 +4,20 @@ import logging
|
|||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.db import transaction
|
from django.db import transaction
|
||||||
|
|
||||||
from membershipworks.models import Member, Flag, Transaction
|
from membershipworks.models import (
|
||||||
|
Member,
|
||||||
|
Flag,
|
||||||
|
Transaction,
|
||||||
|
Event,
|
||||||
|
EventExt,
|
||||||
|
EventCategory,
|
||||||
|
)
|
||||||
from membershipworks import MembershipWorks
|
from membershipworks import MembershipWorks
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
MAX_MEETING_TIME = timedelta(hours=6)
|
||||||
|
|
||||||
|
|
||||||
def flags_for_member(csv_member, all_flags, folders):
|
def flags_for_member(csv_member, all_flags, folders):
|
||||||
for flag in all_flags:
|
for flag in all_flags:
|
||||||
@ -91,3 +99,45 @@ def scrape_membershipworks(*args, **options):
|
|||||||
|
|
||||||
scrape_members(membershipworks)
|
scrape_members(membershipworks)
|
||||||
scrape_transactions(membershipworks)
|
scrape_transactions(membershipworks)
|
||||||
|
|
||||||
|
|
||||||
|
def scrape_events():
|
||||||
|
membershipworks = MembershipWorks()
|
||||||
|
membershipworks.login(
|
||||||
|
settings.MEMBERSHIPWORKS_USERNAME, settings.MEMBERSHIPWORKS_PASSWORD
|
||||||
|
)
|
||||||
|
|
||||||
|
data = membershipworks.get_events_list(
|
||||||
|
datetime.fromtimestamp(0), datetime.now() + timedelta(weeks=52), categories=True
|
||||||
|
)
|
||||||
|
logger.info(f"{len(data)} events retrieved!")
|
||||||
|
|
||||||
|
for category_id, category_data in enumerate(data["_st"]["evg"]):
|
||||||
|
category = EventCategory.from_api_dict(category_id, category_data)
|
||||||
|
category.clean_fields()
|
||||||
|
category.save()
|
||||||
|
|
||||||
|
for event_data in data["evt"]:
|
||||||
|
logger.debug(event_data)
|
||||||
|
event = Event.from_api_dict(event_data)
|
||||||
|
event.clean_fields()
|
||||||
|
event.save()
|
||||||
|
|
||||||
|
try:
|
||||||
|
event_ext = EventExt.objects.get(event_ptr=event)
|
||||||
|
except EventExt.DoesNotExist:
|
||||||
|
event_ext = EventExt(event_ptr=event)
|
||||||
|
# create extension model instance
|
||||||
|
event_ext.save_base(raw=True)
|
||||||
|
event_ext.refresh_from_db()
|
||||||
|
|
||||||
|
if (
|
||||||
|
event_ext.end is not None
|
||||||
|
and event_ext.end - event_ext.start < MAX_MEETING_TIME
|
||||||
|
):
|
||||||
|
meeting_times_count = event_ext.meeting_times.count()
|
||||||
|
if meeting_times_count == 0:
|
||||||
|
event_ext.meeting_times.create(start=event_ext.start, end=event_ext.end)
|
||||||
|
# if there is exactly one meeting time, it should match the event start/end
|
||||||
|
elif meeting_times_count == 1:
|
||||||
|
event_ext.meeting_times.update(start=event_ext.start, end=event_ext.end)
|
||||||
|
Loading…
Reference in New Issue
Block a user