Update for Mailman 3
This commit is contained in:
parent
2e612837c7
commit
612c99918c
272
mailman_sync.py
272
mailman_sync.py
@ -1,15 +1,15 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Update Mailman 2 lists via a json API of the form:
|
||||
Update Mailman 3 lists via a json API of the form:
|
||||
{
|
||||
LIST: {
|
||||
"config"?: {
|
||||
"real_name": REAL_NAME,
|
||||
"moderator": [ADDRESS, ...],
|
||||
"subject_prefix": PREFIX,
|
||||
"reply_to_address": REPLY_TO_ADDRESS,
|
||||
}
|
||||
"moderators"?: [ADDRESS, ...],
|
||||
"members": [ADDRESS, ...]
|
||||
},
|
||||
...
|
||||
@ -17,129 +17,115 @@ Update Mailman 2 lists via a json API of the form:
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import string
|
||||
import subprocess
|
||||
import tempfile
|
||||
import email.utils
|
||||
|
||||
import mailmanclient
|
||||
import requests
|
||||
|
||||
PASSWORD_CHARS = string.ascii_letters + string.digits + string.punctuation
|
||||
PASSWORD_LEN = 18
|
||||
|
||||
|
||||
@dataclass
|
||||
class ListManager:
|
||||
mailman_bin: Path
|
||||
list_name: str
|
||||
dry_run: bool
|
||||
|
||||
def _call_script(self, script: str, args: list[str], **kwargs):
|
||||
output = subprocess.run(
|
||||
[self.mailman_bin / script, *args],
|
||||
encoding="ascii",
|
||||
capture_output=True,
|
||||
check=True,
|
||||
**kwargs,
|
||||
)
|
||||
for line in output.stdout.splitlines():
|
||||
print(f"[{script} {self.list_name}] {line}")
|
||||
|
||||
def newlist(self, urlhost: str, emailhost: str, admin: str):
|
||||
password = "".join(secrets.choice(PASSWORD_CHARS) for i in range(PASSWORD_LEN))
|
||||
|
||||
self._call_script(
|
||||
"newlist",
|
||||
[
|
||||
"--quiet",
|
||||
f"--urlhost={urlhost}",
|
||||
f"--emailhost={emailhost}",
|
||||
self.list_name,
|
||||
admin,
|
||||
password,
|
||||
],
|
||||
)
|
||||
|
||||
def config_list(
|
||||
self,
|
||||
emailhost: str,
|
||||
list: mailmanclient.MailingList,
|
||||
dry_run: bool,
|
||||
real_name: str,
|
||||
moderator: list[str],
|
||||
subject_prefix: str,
|
||||
reply_to_address: str,
|
||||
):
|
||||
config_changes = "\n".join(
|
||||
[
|
||||
# have to modify mlist directly to allow removing suffix
|
||||
"mlist.real_name = " + repr(real_name),
|
||||
"host_name = " + repr(emailhost),
|
||||
"moderator = " + repr(moderator),
|
||||
"subject_prefix = " + repr(subject_prefix),
|
||||
"from_is_list = 1",
|
||||
"anonymous_list = 1",
|
||||
"first_strip_reply_to = 1",
|
||||
"reply_to_address = " + repr(reply_to_address),
|
||||
# Use explicit address for Reply-To
|
||||
"reply_goes_to_list = 2",
|
||||
# TODO: some of this is announcement list specific, some not
|
||||
config_changes = {
|
||||
"display_name": real_name,
|
||||
"subject_prefix": subject_prefix,
|
||||
"reply_to_address": reply_to_address,
|
||||
"reply_goes_to_list": "explicit_header_only",
|
||||
"first_strip_reply_to": True,
|
||||
# quiet member management
|
||||
"send_reminders = 0",
|
||||
"send_welcome_msg = 0",
|
||||
"send_goodbye_msg = 0",
|
||||
# ConcealSubscription | DontReceiveDuplicates
|
||||
"new_member_options = 272",
|
||||
"advertised = 0",
|
||||
"subscribe_policy = 3", # require approval to join
|
||||
"unsubscribe_policy = 1", # require approval to unsubscribe
|
||||
"private_roster = 2", # only admins can view the roster
|
||||
"default_member_moderation = 1",
|
||||
"generic_nonmember_action = 3", # discard non-member emails
|
||||
"forward_auto_discards = 0", # don't notify admin about discards
|
||||
"archive_private = 1",
|
||||
"send_welcome_message": False,
|
||||
"send_goodbye_message": False,
|
||||
"allow_list_posts": False,
|
||||
"anonymous_list": True,
|
||||
"archive_policy": "private",
|
||||
"member_roster_visibility": "moderators",
|
||||
"subscribe_policy": "confirm_then_moderate",
|
||||
"unsubscribe_policy": "confirm",
|
||||
"default_member_action": "hold",
|
||||
"default_nonmember_action": "discard",
|
||||
"advertised": False,
|
||||
}
|
||||
for setting, value in config_changes.items():
|
||||
old_value = list.settings.get(setting)
|
||||
if old_value != value:
|
||||
print(f"Changed {setting}: {old_value} -> {value}")
|
||||
|
||||
if not dry_run:
|
||||
list.settings.update(config_changes)
|
||||
list.settings.save()
|
||||
|
||||
|
||||
# TODO: this will never update someone's display name
|
||||
def diff_roster(
|
||||
expected_members: list[str], existing_members: list[mailmanclient.Member]
|
||||
) -> tuple[dict[str, str], list[mailmanclient.Member]]:
|
||||
expected_members_dict = dict(
|
||||
list(reversed(email.utils.parseaddr(member))) for member in expected_members
|
||||
)
|
||||
existing_members_dict = {member.address: member for member in existing_members}
|
||||
members_to_add = {
|
||||
k: expected_members_dict[k]
|
||||
for k in set(expected_members_dict) - set(existing_members_dict)
|
||||
}
|
||||
members_to_remove = [
|
||||
existing_members_dict[k]
|
||||
for k in set(existing_members_dict) - set(expected_members_dict)
|
||||
]
|
||||
return members_to_add, members_to_remove
|
||||
|
||||
|
||||
def sync_members(
|
||||
list: mailmanclient.MailingList, dry_run: bool, expected_members: list[str]
|
||||
):
|
||||
members_to_add, members_to_remove = diff_roster(expected_members, list.members)
|
||||
|
||||
for address, display_name in members_to_add.items():
|
||||
print(f"Adding '{display_name} <{address}>' to list {list}")
|
||||
if not dry_run:
|
||||
list.subscribe(
|
||||
address,
|
||||
display_name,
|
||||
pre_verified=True,
|
||||
pre_confirmed=True,
|
||||
pre_approved=True,
|
||||
send_welcome_message=False,
|
||||
)
|
||||
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".py") as config_file:
|
||||
config_file.write(config_changes)
|
||||
config_file.flush()
|
||||
# TODO: could use `.mass_unsubscribe()` instead
|
||||
for member in members_to_remove:
|
||||
print(f"Removing {member} from list {list}")
|
||||
if not dry_run:
|
||||
list.unsubscribe(member.address, pre_approved=True, pre_confirmed=True)
|
||||
|
||||
args = [
|
||||
"--inputfile",
|
||||
config_file.name,
|
||||
self.list_name,
|
||||
]
|
||||
if self.dry_run:
|
||||
args.append("--checkonly")
|
||||
|
||||
self._call_script("config_list", args)
|
||||
def sync_moderators(
|
||||
list: mailmanclient.MailingList, dry_run: bool, expected_members: list[str]
|
||||
):
|
||||
members_to_add, members_to_remove = diff_roster(expected_members, list.moderators)
|
||||
|
||||
def sync_members(self, members: list[str]):
|
||||
args = [
|
||||
"--welcome-msg=no",
|
||||
"--goodbye-msg=no",
|
||||
"--notifyadmin=no",
|
||||
"--file",
|
||||
"-",
|
||||
self.list_name,
|
||||
]
|
||||
if self.dry_run:
|
||||
args.append("--no-change")
|
||||
for address, display_name in members_to_add.items():
|
||||
print(f"Adding '{display_name} <{address}>' as moderator to list {list}")
|
||||
if not dry_run:
|
||||
list.subscribe(address, display_name)
|
||||
|
||||
members_data = "\n".join(members)
|
||||
self._call_script("sync_members", args, input=members_data)
|
||||
for member in members_to_remove:
|
||||
print(f"Removing {member} as moderator from list {list}")
|
||||
if not dry_run:
|
||||
list.remove_moderator(member.address)
|
||||
|
||||
|
||||
def main(
|
||||
mailman_bin: Path,
|
||||
api: str,
|
||||
api_auth: str,
|
||||
list_suffix: str,
|
||||
mailman_client: mailmanclient.Client,
|
||||
dry_run: bool,
|
||||
urlhost: str,
|
||||
emailhost: str,
|
||||
admin: str,
|
||||
mail_host: str,
|
||||
):
|
||||
r = requests.get(api, headers={"Authorization": api_auth})
|
||||
if not r.ok:
|
||||
@ -147,50 +133,40 @@ def main(
|
||||
return
|
||||
expected_lists = r.json()
|
||||
|
||||
existing_lists = subprocess.run(
|
||||
[mailman_bin / "list_lists", "-b"],
|
||||
encoding="ascii",
|
||||
capture_output=True,
|
||||
check=True,
|
||||
).stdout.split("\n")
|
||||
domain = mailman_client.get_domain(mail_host)
|
||||
existing_lists = {list.list_name: list for list in domain.get_lists()}
|
||||
|
||||
for name, props in expected_lists.items():
|
||||
list_name = name + list_suffix
|
||||
list_manager = ListManager(mailman_bin, list_name, dry_run)
|
||||
if list_name.lower() not in existing_lists:
|
||||
if dry_run:
|
||||
print(f"Skipping non-existing list {list_name} in dry run mode")
|
||||
if name in existing_lists:
|
||||
list = existing_lists[name]
|
||||
elif dry_run:
|
||||
print(f"Skipping non-existing list {name} in dry run mode")
|
||||
continue
|
||||
else:
|
||||
list_manager.newlist(urlhost, emailhost, admin)
|
||||
list = domain.create_list(name, props.get("style"))
|
||||
|
||||
if "config" in props:
|
||||
print(f"Configuring/syncing {list_name}...")
|
||||
list_manager.config_list(emailhost, **props["config"])
|
||||
if "config" not in props:
|
||||
print(f"Not configuring {name}, as it has no config section")
|
||||
else:
|
||||
print("Not configuring {list_name}, as it has no config section")
|
||||
list_manager.sync_members(props["members"])
|
||||
print(f"Configuring/syncing {name}...")
|
||||
config_list(list, dry_run, **props["config"])
|
||||
|
||||
if "moderators" in props:
|
||||
sync_moderators(list, dry_run, props["moderators"])
|
||||
sync_members(list, dry_run, props["members"])
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
argp = argparse.ArgumentParser(description=__doc__)
|
||||
argp.add_argument(
|
||||
"--bin",
|
||||
default="/usr/local/mailman",
|
||||
type=Path,
|
||||
help="Path to Mailman site admin scripts (default %(default)s)",
|
||||
)
|
||||
argp.add_argument("--api", required=True, help="API endpoint to retrieve JSON from")
|
||||
argp.add_argument("--list-suffix", help="Suffix for mailing lists", default="")
|
||||
argp.add_argument("--mail-host", help="Base domain for all lists", required=True)
|
||||
argp.add_argument(
|
||||
"--urlhost", help="Urlhost to use when creating new lists", required=True
|
||||
"--mailman-url",
|
||||
help="base URL for for Mailman3 REST API",
|
||||
default="http://localhost:9001",
|
||||
)
|
||||
argp.add_argument(
|
||||
"--emailhost", help="Emailhost to use when creating new lists", required=True
|
||||
)
|
||||
argp.add_argument(
|
||||
"--admin",
|
||||
help="Admin email address to use when creating new lists",
|
||||
required=True,
|
||||
"--mailman-user", help="Username for Mailman3 REST API", default="restadmin"
|
||||
)
|
||||
argp.add_argument(
|
||||
"-n",
|
||||
@ -204,23 +180,21 @@ def parse_arguments():
|
||||
if __name__ == "__main__":
|
||||
args = parse_arguments()
|
||||
|
||||
if "API_AUTH" in os.environ:
|
||||
api_auth = os.environ.get("API_AUTH")
|
||||
else:
|
||||
print("Missing API_AUTH environment variable")
|
||||
for env_var in ("API_AUTH", "MAILMAN_PASS"):
|
||||
if env_var not in os.environ:
|
||||
print(f"Missing {env_var} environment variable")
|
||||
exit(-1)
|
||||
|
||||
try:
|
||||
main(
|
||||
args.bin,
|
||||
args.api,
|
||||
api_auth,
|
||||
args.list_suffix,
|
||||
args.dry_run,
|
||||
args.urlhost,
|
||||
args.emailhost,
|
||||
args.admin,
|
||||
mailman_client = mailmanclient.Client(
|
||||
args.mailman_url + "/3.1",
|
||||
args.mailman_user,
|
||||
os.environ["MAILMAN_PASS"],
|
||||
)
|
||||
|
||||
main(
|
||||
args.api,
|
||||
os.environ["API_AUTH"],
|
||||
mailman_client,
|
||||
args.dry_run,
|
||||
args.mail_host,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(e.stderr)
|
||||
raise
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=Synchronize Mailman lists to CMSManage API
|
||||
Description=Synchronize Mailman lists with CMSManage API
|
||||
|
||||
[Service]
|
||||
User=mailman
|
||||
@ -7,7 +7,8 @@ Group=mailman
|
||||
Type=oneshot
|
||||
TimeoutStartSec=600
|
||||
EnvironmentFile=/opt/mailman-sync/env
|
||||
ExecStart=/usr/bin/python3.9 /opt/mailman-sync/mailman_sync.py \
|
||||
--bin /usr/local/cpanel/3rdparty/mailman/bin/ \
|
||||
ExecStart=/usr/bin/python3 /opt/mailman-sync/mailman_sync.py \
|
||||
--api https://paperwork.claremontmakerspace.org/api/v1/paperwork/certification_definition/mailing_lists.json \
|
||||
--list-suffix _claremontmakerspace.org
|
||||
--mail-host claremontmakerspace.org \
|
||||
--mailman-url http://localhost:8001 \
|
||||
--mailman-user restadmin
|
||||
|
||||
Loading…
Reference in New Issue
Block a user