Update for Mailman 3

mailman_sync.py

@@ -1,15 +1,15 @@
 #!/usr/bin/env python3
 
 """
-Update Mailman 2 lists via a json API of the form:
+Update Mailman 3 lists via a json API of the form:
 {
   LIST: {
     "config"?: {
       "real_name": REAL_NAME,
-      "moderator": [ADDRESS, ...],
       "subject_prefix": PREFIX,
       "reply_to_address": REPLY_TO_ADDRESS,
     }
+    "moderators"?: [ADDRESS, ...],
     "members": [ADDRESS, ...]
   },
   ...
@@ -17,129 +17,115 @@ Update Mailman 2 lists via a json API of the form:
 """
 
 import argparse
-from dataclasses import dataclass
 import os
-from pathlib import Path
-import secrets
-import string
-import subprocess
-import tempfile
+import email.utils
 
+import mailmanclient
 import requests
 
-PASSWORD_CHARS = string.ascii_letters + string.digits + string.punctuation
-PASSWORD_LEN = 18
+
+def config_list(
+    list: mailmanclient.MailingList,
+    dry_run: bool,
+    real_name: str,
+    subject_prefix: str,
+    reply_to_address: str,
+):
+    # TODO: some of this is announcement list specific, some not
+    config_changes = {
+        "display_name": real_name,
+        "subject_prefix": subject_prefix,
+        "reply_to_address": reply_to_address,
+        "reply_goes_to_list": "explicit_header_only",
+        "first_strip_reply_to": True,
+        # quiet member management
+        "send_welcome_message": False,
+        "send_goodbye_message": False,
+        "allow_list_posts": False,
+        "anonymous_list": True,
+        "archive_policy": "private",
+        "member_roster_visibility": "moderators",
+        "subscribe_policy": "confirm_then_moderate",
+        "unsubscribe_policy": "confirm",
+        "default_member_action": "hold",
+        "default_nonmember_action": "discard",
+        "advertised": False,
+    }
+    for setting, value in config_changes.items():
+        old_value = list.settings.get(setting)
+        if old_value != value:
+            print(f"Changed {setting}: {old_value} -> {value}")
+
+    if not dry_run:
+        list.settings.update(config_changes)
+        list.settings.save()
 
 
-@dataclass
-class ListManager:
-    mailman_bin: Path
-    list_name: str
-    dry_run: bool
+# TODO: this will never update someone's display name
+def diff_roster(
+    expected_members: list[str], existing_members: list[mailmanclient.Member]
+) -> tuple[dict[str, str], list[mailmanclient.Member]]:
+    expected_members_dict = dict(
+        list(reversed(email.utils.parseaddr(member))) for member in expected_members
+    )
+    existing_members_dict = {member.address: member for member in existing_members}
+    members_to_add = {
+        k: expected_members_dict[k]
+        for k in set(expected_members_dict) - set(existing_members_dict)
+    }
+    members_to_remove = [
+        existing_members_dict[k]
+        for k in set(existing_members_dict) - set(expected_members_dict)
+    ]
+    return members_to_add, members_to_remove
 
-    def _call_script(self, script: str, args: list[str], **kwargs):
-        output = subprocess.run(
-            [self.mailman_bin / script, *args],
-            encoding="ascii",
-            capture_output=True,
-            check=True,
-            **kwargs,
-        )
-        for line in output.stdout.splitlines():
-            print(f"[{script} {self.list_name}] {line}")
 
-    def newlist(self, urlhost: str, emailhost: str, admin: str):
-        password = "".join(secrets.choice(PASSWORD_CHARS) for i in range(PASSWORD_LEN))
+def sync_members(
+    list: mailmanclient.MailingList, dry_run: bool, expected_members: list[str]
+):
+    members_to_add, members_to_remove = diff_roster(expected_members, list.members)
 
-        self._call_script(
-            "newlist",
-            [
-                "--quiet",
-                f"--urlhost={urlhost}",
-                f"--emailhost={emailhost}",
-                self.list_name,
-                admin,
-                password,
-            ],
-        )
+    for address, display_name in members_to_add.items():
+        print(f"Adding '{display_name} <{address}>' to list {list}")
+        if not dry_run:
+            list.subscribe(
+                address,
+                display_name,
+                pre_verified=True,
+                pre_confirmed=True,
+                pre_approved=True,
+                send_welcome_message=False,
+            )
 
-    def config_list(
-        self,
-        emailhost: str,
-        real_name: str,
-        moderator: list[str],
-        subject_prefix: str,
-        reply_to_address: str,
-    ):
-        config_changes = "\n".join(
-            [
-                # have to modify mlist directly to allow removing suffix
-                "mlist.real_name = " + repr(real_name),
-                "host_name = " + repr(emailhost),
-                "moderator = " + repr(moderator),
-                "subject_prefix = " + repr(subject_prefix),
-                "from_is_list = 1",
-                "anonymous_list = 1",
-                "first_strip_reply_to = 1",
-                "reply_to_address = " + repr(reply_to_address),
-                # Use explicit address for Reply-To
-                "reply_goes_to_list = 2",
-                # quiet member management
-                "send_reminders = 0",
-                "send_welcome_msg = 0",
-                "send_goodbye_msg = 0",
-                # ConcealSubscription | DontReceiveDuplicates
-                "new_member_options = 272",
-                "advertised = 0",
-                "subscribe_policy = 3",  # require approval to join
-                "unsubscribe_policy = 1",  # require approval to unsubscribe
-                "private_roster = 2",  # only admins can view the roster
-                "default_member_moderation = 1",
-                "generic_nonmember_action = 3",  # discard non-member emails
-                "forward_auto_discards = 0",  # don't notify admin about discards
-                "archive_private = 1",
-            ]
-        )
+    # TODO: could use `.mass_unsubscribe()` instead
+    for member in members_to_remove:
+        print(f"Removing {member} from list {list}")
+        if not dry_run:
+            list.unsubscribe(member.address, pre_approved=True, pre_confirmed=True)
 
-        with tempfile.NamedTemporaryFile("w", suffix=".py") as config_file:
-            config_file.write(config_changes)
-            config_file.flush()
 
-            args = [
-                "--inputfile",
-                config_file.name,
-                self.list_name,
-            ]
-            if self.dry_run:
-                args.append("--checkonly")
+def sync_moderators(
+    list: mailmanclient.MailingList, dry_run: bool, expected_members: list[str]
+):
+    members_to_add, members_to_remove = diff_roster(expected_members, list.moderators)
 
-            self._call_script("config_list", args)
+    for address, display_name in members_to_add.items():
+        print(f"Adding '{display_name} <{address}>' as moderator to list {list}")
+        if not dry_run:
+            list.subscribe(address, display_name)
 
-    def sync_members(self, members: list[str]):
-        args = [
-            "--welcome-msg=no",
-            "--goodbye-msg=no",
-            "--notifyadmin=no",
-            "--file",
-            "-",
-            self.list_name,
-        ]
-        if self.dry_run:
-            args.append("--no-change")
-
-        members_data = "\n".join(members)
-        self._call_script("sync_members", args, input=members_data)
+    for member in members_to_remove:
+        print(f"Removing {member} as moderator from list {list}")
+        if not dry_run:
+            list.remove_moderator(member.address)
 
 
 def main(
-    mailman_bin: Path,
     api: str,
     api_auth: str,
-    list_suffix: str,
+    mailman_client: mailmanclient.Client,
     dry_run: bool,
-    urlhost: str,
-    emailhost: str,
-    admin: str,
+    mail_host: str,
 ):
     r = requests.get(api, headers={"Authorization": api_auth})
     if not r.ok:
@@ -147,50 +133,40 @@ def main(
         return
     expected_lists = r.json()
 
-    existing_lists = subprocess.run(
-        [mailman_bin / "list_lists", "-b"],
-        encoding="ascii",
-        capture_output=True,
-        check=True,
-    ).stdout.split("\n")
-    for name, props in expected_lists.items():
-        list_name = name + list_suffix
-        list_manager = ListManager(mailman_bin, list_name, dry_run)
-        if list_name.lower() not in existing_lists:
-            if dry_run:
-                print(f"Skipping non-existing list {list_name} in dry run mode")
-                continue
-            else:
-                list_manager.newlist(urlhost, emailhost, admin)
+    domain = mailman_client.get_domain(mail_host)
+    existing_lists = {list.list_name: list for list in domain.get_lists()}
 
-        if "config" in props:
-            print(f"Configuring/syncing {list_name}...")
-            list_manager.config_list(emailhost, **props["config"])
+    for name, props in expected_lists.items():
+        if name in existing_lists:
+            list = existing_lists[name]
+        elif dry_run:
+            print(f"Skipping non-existing list {name} in dry run mode")
+            continue
         else:
-            print("Not configuring {list_name}, as it has no config section")
-        list_manager.sync_members(props["members"])
+            list = domain.create_list(name, props.get("style"))
+
+        if "config" not in props:
+            print(f"Not configuring {name}, as it has no config section")
+        else:
+            print(f"Configuring/syncing {name}...")
+            config_list(list, dry_run, **props["config"])
+
+        if "moderators" in props:
+            sync_moderators(list, dry_run, props["moderators"])
+        sync_members(list, dry_run, props["members"])
 
 
 def parse_arguments():
     argp = argparse.ArgumentParser(description=__doc__)
-    argp.add_argument(
-        "--bin",
-        default="/usr/local/mailman",
-        type=Path,
-        help="Path to Mailman site admin scripts (default %(default)s)",
-    )
     argp.add_argument("--api", required=True, help="API endpoint to retrieve JSON from")
-    argp.add_argument("--list-suffix", help="Suffix for mailing lists", default="")
+    argp.add_argument("--mail-host", help="Base domain for all lists", required=True)
     argp.add_argument(
-        "--urlhost", help="Urlhost to use when creating new lists", required=True
+        "--mailman-url",
+        help="base URL for for Mailman3 REST API",
+        default="http://localhost:9001",
     )
     argp.add_argument(
-        "--emailhost", help="Emailhost to use when creating new lists", required=True
-    )
-    argp.add_argument(
-        "--admin",
-        help="Admin email address to use when creating new lists",
-        required=True,
+        "--mailman-user", help="Username for Mailman3 REST API", default="restadmin"
     )
     argp.add_argument(
         "-n",
@@ -204,23 +180,21 @@ def parse_arguments():
 if __name__ == "__main__":
     args = parse_arguments()
 
-    if "API_AUTH" in os.environ:
-        api_auth = os.environ.get("API_AUTH")
-    else:
-        print("Missing API_AUTH environment variable")
-        exit(-1)
+    for env_var in ("API_AUTH", "MAILMAN_PASS"):
+        if env_var not in os.environ:
+            print(f"Missing {env_var} environment variable")
+            exit(-1)
 
-    try:
-        main(
-            args.bin,
-            args.api,
-            api_auth,
-            args.list_suffix,
-            args.dry_run,
-            args.urlhost,
-            args.emailhost,
-            args.admin,
-        )
-    except subprocess.CalledProcessError as e:
-        print(e.stderr)
-        raise
+    mailman_client = mailmanclient.Client(
+        args.mailman_url + "/3.1",
+        args.mailman_user,
+        os.environ["MAILMAN_PASS"],
+    )
+
+    main(
+        args.api,
+        os.environ["API_AUTH"],
+        mailman_client,
+        args.dry_run,
+        args.mail_host,
+    )

systemd/mailman_sync.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Synchronize Mailman lists to CMSManage API
+Description=Synchronize Mailman lists with CMSManage API
 
 [Service]
 User=mailman
@@ -7,7 +7,8 @@ Group=mailman
 Type=oneshot
 TimeoutStartSec=600
 EnvironmentFile=/opt/mailman-sync/env
-ExecStart=/usr/bin/python3.9 /opt/mailman-sync/mailman_sync.py \
-    --bin /usr/local/cpanel/3rdparty/mailman/bin/ \
-    --api https://paperwork.claremontmakerspace.org/api/v1/paperwork/certification_definition/mailing_lists.json \
-    --list-suffix _claremontmakerspace.org
+ExecStart=/usr/bin/python3 /opt/mailman-sync/mailman_sync.py \
+    --api https://paperwork.claremontmakerspace.org/api/v1/paperwork/department/mailing_lists.json \
+    --mail-host claremontmakerspace.org \
+    --mailman-url http://localhost:8001 \
+    --mailman-user restadmin