Update for Mailman 3
This commit is contained in:
parent
2e612837c7
commit
a19fa0bbef
272
mailman_sync.py
272
mailman_sync.py
@ -1,15 +1,15 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
"""
|
"""
|
||||||
Update Mailman 2 lists via a json API of the form:
|
Update Mailman 3 lists via a json API of the form:
|
||||||
{
|
{
|
||||||
LIST: {
|
LIST: {
|
||||||
"config"?: {
|
"config"?: {
|
||||||
"real_name": REAL_NAME,
|
"real_name": REAL_NAME,
|
||||||
"moderator": [ADDRESS, ...],
|
|
||||||
"subject_prefix": PREFIX,
|
"subject_prefix": PREFIX,
|
||||||
"reply_to_address": REPLY_TO_ADDRESS,
|
"reply_to_address": REPLY_TO_ADDRESS,
|
||||||
}
|
}
|
||||||
|
"moderators"?: [ADDRESS, ...],
|
||||||
"members": [ADDRESS, ...]
|
"members": [ADDRESS, ...]
|
||||||
},
|
},
|
||||||
...
|
...
|
||||||
@ -17,129 +17,115 @@ Update Mailman 2 lists via a json API of the form:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
from dataclasses import dataclass
|
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
import email.utils
|
||||||
import secrets
|
|
||||||
import string
|
|
||||||
import subprocess
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
|
import mailmanclient
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
PASSWORD_CHARS = string.ascii_letters + string.digits + string.punctuation
|
|
||||||
PASSWORD_LEN = 18
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class ListManager:
|
|
||||||
mailman_bin: Path
|
|
||||||
list_name: str
|
|
||||||
dry_run: bool
|
|
||||||
|
|
||||||
def _call_script(self, script: str, args: list[str], **kwargs):
|
|
||||||
output = subprocess.run(
|
|
||||||
[self.mailman_bin / script, *args],
|
|
||||||
encoding="ascii",
|
|
||||||
capture_output=True,
|
|
||||||
check=True,
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
for line in output.stdout.splitlines():
|
|
||||||
print(f"[{script} {self.list_name}] {line}")
|
|
||||||
|
|
||||||
def newlist(self, urlhost: str, emailhost: str, admin: str):
|
|
||||||
password = "".join(secrets.choice(PASSWORD_CHARS) for i in range(PASSWORD_LEN))
|
|
||||||
|
|
||||||
self._call_script(
|
|
||||||
"newlist",
|
|
||||||
[
|
|
||||||
"--quiet",
|
|
||||||
f"--urlhost={urlhost}",
|
|
||||||
f"--emailhost={emailhost}",
|
|
||||||
self.list_name,
|
|
||||||
admin,
|
|
||||||
password,
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
def config_list(
|
def config_list(
|
||||||
self,
|
list: mailmanclient.MailingList,
|
||||||
emailhost: str,
|
dry_run: bool,
|
||||||
real_name: str,
|
real_name: str,
|
||||||
moderator: list[str],
|
|
||||||
subject_prefix: str,
|
subject_prefix: str,
|
||||||
reply_to_address: str,
|
reply_to_address: str,
|
||||||
):
|
):
|
||||||
config_changes = "\n".join(
|
# TODO: some of this is announcement list specific, some not
|
||||||
[
|
config_changes = {
|
||||||
# have to modify mlist directly to allow removing suffix
|
"display_name": real_name,
|
||||||
"mlist.real_name = " + repr(real_name),
|
"subject_prefix": subject_prefix,
|
||||||
"host_name = " + repr(emailhost),
|
"reply_to_address": reply_to_address,
|
||||||
"moderator = " + repr(moderator),
|
"reply_goes_to_list": "explicit_header_only",
|
||||||
"subject_prefix = " + repr(subject_prefix),
|
"first_strip_reply_to": True,
|
||||||
"from_is_list = 1",
|
|
||||||
"anonymous_list = 1",
|
|
||||||
"first_strip_reply_to = 1",
|
|
||||||
"reply_to_address = " + repr(reply_to_address),
|
|
||||||
# Use explicit address for Reply-To
|
|
||||||
"reply_goes_to_list = 2",
|
|
||||||
# quiet member management
|
# quiet member management
|
||||||
"send_reminders = 0",
|
"send_welcome_message": False,
|
||||||
"send_welcome_msg = 0",
|
"send_goodbye_message": False,
|
||||||
"send_goodbye_msg = 0",
|
"allow_list_posts": False,
|
||||||
# ConcealSubscription | DontReceiveDuplicates
|
"anonymous_list": True,
|
||||||
"new_member_options = 272",
|
"archive_policy": "private",
|
||||||
"advertised = 0",
|
"member_roster_visibility": "moderators",
|
||||||
"subscribe_policy = 3", # require approval to join
|
"subscribe_policy": "confirm_then_moderate",
|
||||||
"unsubscribe_policy = 1", # require approval to unsubscribe
|
"unsubscribe_policy": "confirm",
|
||||||
"private_roster = 2", # only admins can view the roster
|
"default_member_action": "hold",
|
||||||
"default_member_moderation = 1",
|
"default_nonmember_action": "discard",
|
||||||
"generic_nonmember_action = 3", # discard non-member emails
|
"advertised": False,
|
||||||
"forward_auto_discards = 0", # don't notify admin about discards
|
}
|
||||||
"archive_private = 1",
|
for setting, value in config_changes.items():
|
||||||
|
old_value = list.settings.get(setting)
|
||||||
|
if old_value != value:
|
||||||
|
print(f"Changed {setting}: {old_value} -> {value}")
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
list.settings.update(config_changes)
|
||||||
|
list.settings.save()
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: this will never update someone's display name
|
||||||
|
def diff_roster(
|
||||||
|
expected_members: list[str], existing_members: list[mailmanclient.Member]
|
||||||
|
) -> tuple[dict[str, str], list[mailmanclient.Member]]:
|
||||||
|
expected_members_dict = dict(
|
||||||
|
list(reversed(email.utils.parseaddr(member))) for member in expected_members
|
||||||
|
)
|
||||||
|
existing_members_dict = {member.address: member for member in existing_members}
|
||||||
|
members_to_add = {
|
||||||
|
k: expected_members_dict[k]
|
||||||
|
for k in set(expected_members_dict) - set(existing_members_dict)
|
||||||
|
}
|
||||||
|
members_to_remove = [
|
||||||
|
existing_members_dict[k]
|
||||||
|
for k in set(existing_members_dict) - set(expected_members_dict)
|
||||||
]
|
]
|
||||||
|
return members_to_add, members_to_remove
|
||||||
|
|
||||||
|
|
||||||
|
def sync_members(
|
||||||
|
list: mailmanclient.MailingList, dry_run: bool, expected_members: list[str]
|
||||||
|
):
|
||||||
|
members_to_add, members_to_remove = diff_roster(expected_members, list.members)
|
||||||
|
|
||||||
|
for address, display_name in members_to_add.items():
|
||||||
|
print(f"Adding '{display_name} <{address}>' to list {list}")
|
||||||
|
if not dry_run:
|
||||||
|
list.subscribe(
|
||||||
|
address,
|
||||||
|
display_name,
|
||||||
|
pre_verified=True,
|
||||||
|
pre_confirmed=True,
|
||||||
|
pre_approved=True,
|
||||||
|
send_welcome_message=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
with tempfile.NamedTemporaryFile("w", suffix=".py") as config_file:
|
# TODO: could use `.mass_unsubscribe()` instead
|
||||||
config_file.write(config_changes)
|
for member in members_to_remove:
|
||||||
config_file.flush()
|
print(f"Removing {member} from list {list}")
|
||||||
|
if not dry_run:
|
||||||
|
list.unsubscribe(member.address, pre_approved=True, pre_confirmed=True)
|
||||||
|
|
||||||
args = [
|
|
||||||
"--inputfile",
|
|
||||||
config_file.name,
|
|
||||||
self.list_name,
|
|
||||||
]
|
|
||||||
if self.dry_run:
|
|
||||||
args.append("--checkonly")
|
|
||||||
|
|
||||||
self._call_script("config_list", args)
|
def sync_moderators(
|
||||||
|
list: mailmanclient.MailingList, dry_run: bool, expected_members: list[str]
|
||||||
|
):
|
||||||
|
members_to_add, members_to_remove = diff_roster(expected_members, list.moderators)
|
||||||
|
|
||||||
def sync_members(self, members: list[str]):
|
for address, display_name in members_to_add.items():
|
||||||
args = [
|
print(f"Adding '{display_name} <{address}>' as moderator to list {list}")
|
||||||
"--welcome-msg=no",
|
if not dry_run:
|
||||||
"--goodbye-msg=no",
|
list.subscribe(address, display_name)
|
||||||
"--notifyadmin=no",
|
|
||||||
"--file",
|
|
||||||
"-",
|
|
||||||
self.list_name,
|
|
||||||
]
|
|
||||||
if self.dry_run:
|
|
||||||
args.append("--no-change")
|
|
||||||
|
|
||||||
members_data = "\n".join(members)
|
for member in members_to_remove:
|
||||||
self._call_script("sync_members", args, input=members_data)
|
print(f"Removing {member} as moderator from list {list}")
|
||||||
|
if not dry_run:
|
||||||
|
list.remove_moderator(member.address)
|
||||||
|
|
||||||
|
|
||||||
def main(
|
def main(
|
||||||
mailman_bin: Path,
|
|
||||||
api: str,
|
api: str,
|
||||||
api_auth: str,
|
api_auth: str,
|
||||||
list_suffix: str,
|
mailman_client: mailmanclient.Client,
|
||||||
dry_run: bool,
|
dry_run: bool,
|
||||||
urlhost: str,
|
mail_host: str,
|
||||||
emailhost: str,
|
|
||||||
admin: str,
|
|
||||||
):
|
):
|
||||||
r = requests.get(api, headers={"Authorization": api_auth})
|
r = requests.get(api, headers={"Authorization": api_auth})
|
||||||
if not r.ok:
|
if not r.ok:
|
||||||
@ -147,50 +133,40 @@ def main(
|
|||||||
return
|
return
|
||||||
expected_lists = r.json()
|
expected_lists = r.json()
|
||||||
|
|
||||||
existing_lists = subprocess.run(
|
domain = mailman_client.get_domain(mail_host)
|
||||||
[mailman_bin / "list_lists", "-b"],
|
existing_lists = {list.list_name: list for list in domain.get_lists()}
|
||||||
encoding="ascii",
|
|
||||||
capture_output=True,
|
|
||||||
check=True,
|
|
||||||
).stdout.split("\n")
|
|
||||||
for name, props in expected_lists.items():
|
for name, props in expected_lists.items():
|
||||||
list_name = name + list_suffix
|
if name in existing_lists:
|
||||||
list_manager = ListManager(mailman_bin, list_name, dry_run)
|
list = existing_lists[name]
|
||||||
if list_name.lower() not in existing_lists:
|
elif dry_run:
|
||||||
if dry_run:
|
print(f"Skipping non-existing list {name} in dry run mode")
|
||||||
print(f"Skipping non-existing list {list_name} in dry run mode")
|
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
list_manager.newlist(urlhost, emailhost, admin)
|
list = domain.create_list(name, props.get("style"))
|
||||||
|
|
||||||
if "config" in props:
|
if "config" not in props:
|
||||||
print(f"Configuring/syncing {list_name}...")
|
print(f"Not configuring {name}, as it has no config section")
|
||||||
list_manager.config_list(emailhost, **props["config"])
|
|
||||||
else:
|
else:
|
||||||
print("Not configuring {list_name}, as it has no config section")
|
print(f"Configuring/syncing {name}...")
|
||||||
list_manager.sync_members(props["members"])
|
config_list(list, dry_run, **props["config"])
|
||||||
|
|
||||||
|
if "moderators" in props:
|
||||||
|
sync_moderators(list, dry_run, props["moderators"])
|
||||||
|
sync_members(list, dry_run, props["members"])
|
||||||
|
|
||||||
|
|
||||||
def parse_arguments():
|
def parse_arguments():
|
||||||
argp = argparse.ArgumentParser(description=__doc__)
|
argp = argparse.ArgumentParser(description=__doc__)
|
||||||
argp.add_argument(
|
|
||||||
"--bin",
|
|
||||||
default="/usr/local/mailman",
|
|
||||||
type=Path,
|
|
||||||
help="Path to Mailman site admin scripts (default %(default)s)",
|
|
||||||
)
|
|
||||||
argp.add_argument("--api", required=True, help="API endpoint to retrieve JSON from")
|
argp.add_argument("--api", required=True, help="API endpoint to retrieve JSON from")
|
||||||
argp.add_argument("--list-suffix", help="Suffix for mailing lists", default="")
|
argp.add_argument("--mail-host", help="Base domain for all lists", required=True)
|
||||||
argp.add_argument(
|
argp.add_argument(
|
||||||
"--urlhost", help="Urlhost to use when creating new lists", required=True
|
"--mailman-url",
|
||||||
|
help="base URL for for Mailman3 REST API",
|
||||||
|
default="http://localhost:9001",
|
||||||
)
|
)
|
||||||
argp.add_argument(
|
argp.add_argument(
|
||||||
"--emailhost", help="Emailhost to use when creating new lists", required=True
|
"--mailman-user", help="Username for Mailman3 REST API", default="restadmin"
|
||||||
)
|
|
||||||
argp.add_argument(
|
|
||||||
"--admin",
|
|
||||||
help="Admin email address to use when creating new lists",
|
|
||||||
required=True,
|
|
||||||
)
|
)
|
||||||
argp.add_argument(
|
argp.add_argument(
|
||||||
"-n",
|
"-n",
|
||||||
@ -204,23 +180,21 @@ def parse_arguments():
|
|||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
args = parse_arguments()
|
args = parse_arguments()
|
||||||
|
|
||||||
if "API_AUTH" in os.environ:
|
for env_var in ("API_AUTH", "MAILMAN_PASS"):
|
||||||
api_auth = os.environ.get("API_AUTH")
|
if env_var not in os.environ:
|
||||||
else:
|
print(f"Missing {env_var} environment variable")
|
||||||
print("Missing API_AUTH environment variable")
|
|
||||||
exit(-1)
|
exit(-1)
|
||||||
|
|
||||||
try:
|
mailman_client = mailmanclient.Client(
|
||||||
main(
|
args.mailman_url + "/3.1",
|
||||||
args.bin,
|
args.mailman_user,
|
||||||
args.api,
|
os.environ["MAILMAN_PASS"],
|
||||||
api_auth,
|
)
|
||||||
args.list_suffix,
|
|
||||||
args.dry_run,
|
main(
|
||||||
args.urlhost,
|
args.api,
|
||||||
args.emailhost,
|
os.environ["API_AUTH"],
|
||||||
args.admin,
|
mailman_client,
|
||||||
|
args.dry_run,
|
||||||
|
args.mail_host,
|
||||||
)
|
)
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
print(e.stderr)
|
|
||||||
raise
|
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
[Unit]
|
[Unit]
|
||||||
Description=Synchronize Mailman lists to CMSManage API
|
Description=Synchronize Mailman lists with CMSManage API
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
User=mailman
|
User=mailman
|
||||||
@ -7,7 +7,8 @@ Group=mailman
|
|||||||
Type=oneshot
|
Type=oneshot
|
||||||
TimeoutStartSec=600
|
TimeoutStartSec=600
|
||||||
EnvironmentFile=/opt/mailman-sync/env
|
EnvironmentFile=/opt/mailman-sync/env
|
||||||
ExecStart=/usr/bin/python3.9 /opt/mailman-sync/mailman_sync.py \
|
ExecStart=/usr/bin/python3 /opt/mailman-sync/mailman_sync.py \
|
||||||
--bin /usr/local/cpanel/3rdparty/mailman/bin/ \
|
--api https://paperwork.claremontmakerspace.org/api/v1/paperwork/department/mailing_lists.json \
|
||||||
--api https://paperwork.claremontmakerspace.org/api/v1/paperwork/certification_definition/mailing_lists.json \
|
--mail-host claremontmakerspace.org \
|
||||||
--list-suffix _claremontmakerspace.org
|
--mailman-url http://localhost:8001 \
|
||||||
|
--mailman-user restadmin
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user