From 612c99918c50a72b9ae36a4b961e357c0f57b22e Mon Sep 17 00:00:00 2001 From: Adam Goldsmith Date: Fri, 1 Sep 2023 23:33:08 -0400 Subject: [PATCH] Update for Mailman 3 --- mailman_sync.py | 290 ++++++++++++++++------------------- systemd/mailman_sync.service | 9 +- 2 files changed, 137 insertions(+), 162 deletions(-) diff --git a/mailman_sync.py b/mailman_sync.py index 10e389d..c125b62 100755 --- a/mailman_sync.py +++ b/mailman_sync.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 """ -Update Mailman 2 lists via a json API of the form: +Update Mailman 3 lists via a json API of the form: { LIST: { "config"?: { "real_name": REAL_NAME, - "moderator": [ADDRESS, ...], "subject_prefix": PREFIX, "reply_to_address": REPLY_TO_ADDRESS, } + "moderators"?: [ADDRESS, ...], "members": [ADDRESS, ...] }, ... @@ -17,129 +17,115 @@ Update Mailman 2 lists via a json API of the form: """ import argparse -from dataclasses import dataclass import os -from pathlib import Path -import secrets -import string -import subprocess -import tempfile +import email.utils +import mailmanclient import requests -PASSWORD_CHARS = string.ascii_letters + string.digits + string.punctuation -PASSWORD_LEN = 18 + +def config_list( + list: mailmanclient.MailingList, + dry_run: bool, + real_name: str, + subject_prefix: str, + reply_to_address: str, +): + # TODO: some of this is announcement list specific, some not + config_changes = { + "display_name": real_name, + "subject_prefix": subject_prefix, + "reply_to_address": reply_to_address, + "reply_goes_to_list": "explicit_header_only", + "first_strip_reply_to": True, + # quiet member management + "send_welcome_message": False, + "send_goodbye_message": False, + "allow_list_posts": False, + "anonymous_list": True, + "archive_policy": "private", + "member_roster_visibility": "moderators", + "subscribe_policy": "confirm_then_moderate", + "unsubscribe_policy": "confirm", + "default_member_action": "hold", + "default_nonmember_action": "discard", + "advertised": False, + } + for setting, value in config_changes.items(): + old_value = list.settings.get(setting) + if old_value != value: + print(f"Changed {setting}: {old_value} -> {value}") + + if not dry_run: + list.settings.update(config_changes) + list.settings.save() -@dataclass -class ListManager: - mailman_bin: Path - list_name: str - dry_run: bool +# TODO: this will never update someone's display name +def diff_roster( + expected_members: list[str], existing_members: list[mailmanclient.Member] +) -> tuple[dict[str, str], list[mailmanclient.Member]]: + expected_members_dict = dict( + list(reversed(email.utils.parseaddr(member))) for member in expected_members + ) + existing_members_dict = {member.address: member for member in existing_members} + members_to_add = { + k: expected_members_dict[k] + for k in set(expected_members_dict) - set(existing_members_dict) + } + members_to_remove = [ + existing_members_dict[k] + for k in set(existing_members_dict) - set(expected_members_dict) + ] + return members_to_add, members_to_remove - def _call_script(self, script: str, args: list[str], **kwargs): - output = subprocess.run( - [self.mailman_bin / script, *args], - encoding="ascii", - capture_output=True, - check=True, - **kwargs, - ) - for line in output.stdout.splitlines(): - print(f"[{script} {self.list_name}] {line}") - def newlist(self, urlhost: str, emailhost: str, admin: str): - password = "".join(secrets.choice(PASSWORD_CHARS) for i in range(PASSWORD_LEN)) +def sync_members( + list: mailmanclient.MailingList, dry_run: bool, expected_members: list[str] +): + members_to_add, members_to_remove = diff_roster(expected_members, list.members) - self._call_script( - "newlist", - [ - "--quiet", - f"--urlhost={urlhost}", - f"--emailhost={emailhost}", - self.list_name, - admin, - password, - ], - ) + for address, display_name in members_to_add.items(): + print(f"Adding '{display_name} <{address}>' to list {list}") + if not dry_run: + list.subscribe( + address, + display_name, + pre_verified=True, + pre_confirmed=True, + pre_approved=True, + send_welcome_message=False, + ) - def config_list( - self, - emailhost: str, - real_name: str, - moderator: list[str], - subject_prefix: str, - reply_to_address: str, - ): - config_changes = "\n".join( - [ - # have to modify mlist directly to allow removing suffix - "mlist.real_name = " + repr(real_name), - "host_name = " + repr(emailhost), - "moderator = " + repr(moderator), - "subject_prefix = " + repr(subject_prefix), - "from_is_list = 1", - "anonymous_list = 1", - "first_strip_reply_to = 1", - "reply_to_address = " + repr(reply_to_address), - # Use explicit address for Reply-To - "reply_goes_to_list = 2", - # quiet member management - "send_reminders = 0", - "send_welcome_msg = 0", - "send_goodbye_msg = 0", - # ConcealSubscription | DontReceiveDuplicates - "new_member_options = 272", - "advertised = 0", - "subscribe_policy = 3", # require approval to join - "unsubscribe_policy = 1", # require approval to unsubscribe - "private_roster = 2", # only admins can view the roster - "default_member_moderation = 1", - "generic_nonmember_action = 3", # discard non-member emails - "forward_auto_discards = 0", # don't notify admin about discards - "archive_private = 1", - ] - ) + # TODO: could use `.mass_unsubscribe()` instead + for member in members_to_remove: + print(f"Removing {member} from list {list}") + if not dry_run: + list.unsubscribe(member.address, pre_approved=True, pre_confirmed=True) - with tempfile.NamedTemporaryFile("w", suffix=".py") as config_file: - config_file.write(config_changes) - config_file.flush() - args = [ - "--inputfile", - config_file.name, - self.list_name, - ] - if self.dry_run: - args.append("--checkonly") +def sync_moderators( + list: mailmanclient.MailingList, dry_run: bool, expected_members: list[str] +): + members_to_add, members_to_remove = diff_roster(expected_members, list.moderators) - self._call_script("config_list", args) + for address, display_name in members_to_add.items(): + print(f"Adding '{display_name} <{address}>' as moderator to list {list}") + if not dry_run: + list.subscribe(address, display_name) - def sync_members(self, members: list[str]): - args = [ - "--welcome-msg=no", - "--goodbye-msg=no", - "--notifyadmin=no", - "--file", - "-", - self.list_name, - ] - if self.dry_run: - args.append("--no-change") - - members_data = "\n".join(members) - self._call_script("sync_members", args, input=members_data) + for member in members_to_remove: + print(f"Removing {member} as moderator from list {list}") + if not dry_run: + list.remove_moderator(member.address) def main( - mailman_bin: Path, api: str, api_auth: str, - list_suffix: str, + mailman_client: mailmanclient.Client, dry_run: bool, - urlhost: str, - emailhost: str, - admin: str, + mail_host: str, ): r = requests.get(api, headers={"Authorization": api_auth}) if not r.ok: @@ -147,50 +133,40 @@ def main( return expected_lists = r.json() - existing_lists = subprocess.run( - [mailman_bin / "list_lists", "-b"], - encoding="ascii", - capture_output=True, - check=True, - ).stdout.split("\n") - for name, props in expected_lists.items(): - list_name = name + list_suffix - list_manager = ListManager(mailman_bin, list_name, dry_run) - if list_name.lower() not in existing_lists: - if dry_run: - print(f"Skipping non-existing list {list_name} in dry run mode") - continue - else: - list_manager.newlist(urlhost, emailhost, admin) + domain = mailman_client.get_domain(mail_host) + existing_lists = {list.list_name: list for list in domain.get_lists()} - if "config" in props: - print(f"Configuring/syncing {list_name}...") - list_manager.config_list(emailhost, **props["config"]) + for name, props in expected_lists.items(): + if name in existing_lists: + list = existing_lists[name] + elif dry_run: + print(f"Skipping non-existing list {name} in dry run mode") + continue else: - print("Not configuring {list_name}, as it has no config section") - list_manager.sync_members(props["members"]) + list = domain.create_list(name, props.get("style")) + + if "config" not in props: + print(f"Not configuring {name}, as it has no config section") + else: + print(f"Configuring/syncing {name}...") + config_list(list, dry_run, **props["config"]) + + if "moderators" in props: + sync_moderators(list, dry_run, props["moderators"]) + sync_members(list, dry_run, props["members"]) def parse_arguments(): argp = argparse.ArgumentParser(description=__doc__) - argp.add_argument( - "--bin", - default="/usr/local/mailman", - type=Path, - help="Path to Mailman site admin scripts (default %(default)s)", - ) argp.add_argument("--api", required=True, help="API endpoint to retrieve JSON from") - argp.add_argument("--list-suffix", help="Suffix for mailing lists", default="") + argp.add_argument("--mail-host", help="Base domain for all lists", required=True) argp.add_argument( - "--urlhost", help="Urlhost to use when creating new lists", required=True + "--mailman-url", + help="base URL for for Mailman3 REST API", + default="http://localhost:9001", ) argp.add_argument( - "--emailhost", help="Emailhost to use when creating new lists", required=True - ) - argp.add_argument( - "--admin", - help="Admin email address to use when creating new lists", - required=True, + "--mailman-user", help="Username for Mailman3 REST API", default="restadmin" ) argp.add_argument( "-n", @@ -204,23 +180,21 @@ def parse_arguments(): if __name__ == "__main__": args = parse_arguments() - if "API_AUTH" in os.environ: - api_auth = os.environ.get("API_AUTH") - else: - print("Missing API_AUTH environment variable") - exit(-1) + for env_var in ("API_AUTH", "MAILMAN_PASS"): + if env_var not in os.environ: + print(f"Missing {env_var} environment variable") + exit(-1) - try: - main( - args.bin, - args.api, - api_auth, - args.list_suffix, - args.dry_run, - args.urlhost, - args.emailhost, - args.admin, - ) - except subprocess.CalledProcessError as e: - print(e.stderr) - raise + mailman_client = mailmanclient.Client( + args.mailman_url + "/3.1", + args.mailman_user, + os.environ["MAILMAN_PASS"], + ) + + main( + args.api, + os.environ["API_AUTH"], + mailman_client, + args.dry_run, + args.mail_host, + ) diff --git a/systemd/mailman_sync.service b/systemd/mailman_sync.service index f2efbf2..912bc88 100644 --- a/systemd/mailman_sync.service +++ b/systemd/mailman_sync.service @@ -1,5 +1,5 @@ [Unit] -Description=Synchronize Mailman lists to CMSManage API +Description=Synchronize Mailman lists with CMSManage API [Service] User=mailman @@ -7,7 +7,8 @@ Group=mailman Type=oneshot TimeoutStartSec=600 EnvironmentFile=/opt/mailman-sync/env -ExecStart=/usr/bin/python3.9 /opt/mailman-sync/mailman_sync.py \ - --bin /usr/local/cpanel/3rdparty/mailman/bin/ \ +ExecStart=/usr/bin/python3 /opt/mailman-sync/mailman_sync.py \ --api https://paperwork.claremontmakerspace.org/api/v1/paperwork/certification_definition/mailing_lists.json \ - --list-suffix _claremontmakerspace.org + --mail-host claremontmakerspace.org \ + --mailman-url http://localhost:8001 \ + --mailman-user restadmin