From e9a81dd70d87ce5a724fcfe88912371e4ccbca32 Mon Sep 17 00:00:00 2001 From: Jared Mauch Date: Thu, 2 Jul 2026 16:58:20 -0400 Subject: [PATCH 1/4] Process monthly membership reminder bounces against member lists. Site list bounces were previously forwarded to admins without scoring. Register reminder bounces on each subscribed list that sends reminders, while still forwarding owner moderation notices unchanged. --- Mailman/Defaults.py.in | 12 ++- Mailman/Queue/BounceRunner.py | 93 ++++++++++++++++++---- cron/mailpasswds | 1 + tests/test_bounce_reminders.py | 139 +++++++++++++++++++++++++++++++++ 4 files changed, 226 insertions(+), 19 deletions(-) create mode 100644 tests/test_bounce_reminders.py diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index 697af315..83f587be 100755 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -836,12 +836,16 @@ VERP_PROBES = No # A perfect opportunity for doing VERP is the password reminders, which are # already addressed individually to each recipient. Set this to Yes to enable -# VERPs on all password reminders. However, because password reminders are -# sent from the site list and site list bounces aren't processed but are just -# forwarded to the site list admins, this isn't too useful. See comments at -# lines 70-84 of Mailman/Queue/BounceRunner.py for why we don't process them. +# VERPs on all password reminders. This is useful when +# BOUNCE_PROCESS_REMINDER_BOUNCES is enabled. VERP_PASSWORD_REMINDERS = No +# When a monthly membership/password reminder bounces, register the bounce +# against every list the member is subscribed to that sends reminders and has +# bounce processing enabled. Other site list bounces (e.g. owner moderation +# notices) are still forwarded to the site list administrators. +BOUNCE_PROCESS_REMINDER_BOUNCES = Yes + # Another good opportunity is when regular delivery is personalized. Here # again, we're already incurring the performance hit for addressing each # individual recipient. Set this to Yes to enable VERPs on all personalized diff --git a/Mailman/Queue/BounceRunner.py b/Mailman/Queue/BounceRunner.py index 970d3236..cb23a1cd 100644 --- a/Mailman/Queue/BounceRunner.py +++ b/Mailman/Queue/BounceRunner.py @@ -23,12 +23,14 @@ import time import pickle +from email.iterators import typed_subpart_iterator from email.mime.text import MIMEText from email.mime.message import MIMEMessage from email.utils import parseaddr from Mailman import mm_cfg from Mailman import Utils +from Mailman.MailList import MailList from Mailman import LockFile from Mailman.Errors import NotAMemberError from Mailman.Message import UserNotification @@ -74,8 +76,9 @@ def __init__(self): # site list's bounce address. The bounce runner would then dutifully # register a bounce for all 4 lists that aperson@example.com was a # member of, and eventually that person would get disabled on all - # their lists. So now we ignore site list bounces. Ce La Vie for - # password reminder bounces. + # their lists. So now we ignore most site list bounces. Membership + # reminder bounces can optionally be processed; see + # BOUNCE_PROCESS_REMINDER_BOUNCES. self._bounce_events_file = os.path.join( mm_cfg.DATA_DIR, 'bounce-events-%05d.pck' % os.getpid()) self._bounce_events_fp = None @@ -97,6 +100,22 @@ def _queue_bounces(self, listname, addrs, msg): os.fsync(self._bounce_events_fp.fileno()) self._bouncecnt += len(addrs) + def _queue_reminder_bounces(self, addr, msg): + """Queue a reminder bounce for each subscribed list that sends reminders.""" + for listname in Utils.list_names(): + if listname.lower() == mm_cfg.MAILMAN_SITE_LIST.lower(): + continue + mlist = MailList(listname, lock=0) + if not mlist.bounce_processing: + continue + if not mlist.send_reminders: + continue + if not mlist.isMember(addr): + continue + if mlist.getMemberOption(addr, mm_cfg.SuppressPasswordReminder): + continue + self._queue_bounces(listname, [addr], msg) + def _register_bounces(self): syslog('bounce', '%s processing %s queued bounces', self, self._bouncecnt) @@ -206,18 +225,24 @@ def _dispose(self, mlist, msg, msgdata): # message sent directly to the -bounces address. We have to do these # cases separately, because sending to site-owner will reset the # envelope sender. + is_site_list = (mlist.internal_name().lower() == + mm_cfg.MAILMAN_SITE_LIST.lower()) + process_reminder_bounce = False # Is this a site list bounce? - if (mlist.internal_name().lower() == - mm_cfg.MAILMAN_SITE_LIST.lower()): - # Send it on to the site owners, but craft the envelope sender to - # be the -loop detection address, so if /they/ bounce, we won't - # get stuck in a bounce loop. - outq.enqueue(msg, msgdata, - recips=mlist.owner, - envsender=Utils.get_site_email(extra='loop'), - nodecorate=1, - ) - return + if is_site_list: + if (mm_cfg.BOUNCE_PROCESS_REMINDER_BOUNCES and + is_membership_reminder_bounce(msg)): + process_reminder_bounce = True + else: + # Send it on to the site owners, but craft the envelope sender + # to be the -loop detection address, so if /they/ bounce, we + # won't get stuck in a bounce loop. + outq.enqueue(msg, msgdata, + recips=mlist.owner, + envsender=Utils.get_site_email(extra='loop'), + nodecorate=1, + ) + return # Is this a possible looping message sent directly to a list-bounces # address other than the site list? # Check From: because unix_from might be VERP'd. @@ -233,7 +258,7 @@ def _dispose(self, mlist, msg, msgdata): ) return # List isn't doing bounce processing? - if not mlist.bounce_processing: + if not process_reminder_bounce and not mlist.bounce_processing: return # Try VERP detection first, since it's quick and easy addrs = verp_bounce(mlist, msg) @@ -268,7 +293,11 @@ def _dispose(self, mlist, msg, msgdata): # can let None's sneak through. In any event, this will kill them. # addrs = filter(None, addrs) # MAS above filter moved up so we don't try to queue an empty list. - self._queue_bounces(mlist.internal_name(), addrs, msg) + if process_reminder_bounce: + for addr in addrs: + self._queue_reminder_bounces(addr, msg) + else: + self._queue_bounces(mlist.internal_name(), addrs, msg) _doperiodic = BounceMixin._doperiodic @@ -277,6 +306,40 @@ def _cleanup(self): Runner._cleanup(self) + +def is_membership_reminder_bounce(msg): + """Return true if this bounce is for a monthly membership reminder.""" + for part in typed_subpart_iterator(msg, 'message', 'rfc822'): + orig = part.get_payload(0) + if orig is None: + continue + if orig.get('X-Mailman-Membership-Reminder', '').lower() == 'yes': + return True + if orig.get('x-list-administrivia', '').lower() != 'yes': + continue + sender = parseaddr(orig.get('from', ''))[1].lower() + if not sender: + continue + # Owner notifications are sent from the site -bounces address. + if _is_site_bounces_address(sender): + continue + subj = Utils.oneline(orig.get('subject', ''), 'us-ascii').lower() + if 'reminder' in subj: + return True + return False + + + +def _is_site_bounces_address(addr): + addr = addr.lower() + mailbox, domain = Utils.ParseEmail(addr) + if mailbox == '%s-bounces' % mm_cfg.MAILMAN_SITE_LIST: + return True + if mailbox.startswith('%s-bounces+' % mm_cfg.MAILMAN_SITE_LIST): + return True + return False + + def verp_bounce(mlist, msg): bmailbox, bdomain = Utils.ParseEmail(mlist.GetBouncesEmail()) diff --git a/cron/mailpasswds b/cron/mailpasswds index fdef0494..d7c2532d 100755 --- a/cron/mailpasswds +++ b/cron/mailpasswds @@ -225,6 +225,7 @@ def main(): finally: i18n.set_translation(otrans) msg['X-No-Archive'] = 'yes' + msg['X-Mailman-Membership-Reminder'] = 'yes' del msg['auto-submitted'] msg['Auto-Submitted'] = 'auto-generated' # We want to make this look like it's coming from the siteowner's diff --git a/tests/test_bounce_reminders.py b/tests/test_bounce_reminders.py new file mode 100644 index 00000000..c2455ff9 --- /dev/null +++ b/tests/test_bounce_reminders.py @@ -0,0 +1,139 @@ +# Copyright (C) 2026 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. + +"""Test membership reminder bounce handling.""" + +import os +import pickle +import unittest +import email + +_TESTDIR = os.path.dirname(os.path.abspath(__file__)) +try: + from Mailman import __init__ +except ImportError: + import paths + +from Mailman import Utils +from Mailman.Queue.BounceRunner import is_membership_reminder_bounce + + +SAMPLE = os.path.join( + _TESTDIR, 'bounces', 'samples_from_messages', + 'sample_01_undelivered_mail_returned_to_sender.eml') + + + +class ReminderBounceTest(unittest.TestCase): + def test_membership_reminder_sample(self): + if not os.path.exists(SAMPLE): + self.skipTest('sample bounce file not available') + with open(SAMPLE, 'rb') as fp: + msg = email.message_from_binary_file(fp) + self.assertTrue(is_membership_reminder_bounce(msg)) + + def test_owner_notification_is_not_reminder(self): + msg = email.message_from_string("""\ +Content-Type: multipart/report; boundary="BOUND" + +--BOUND +Content-Type: text/plain + +A moderation notice bounced. + +--BOUND +Content-Type: message/rfc822 + +From: mailman-bounces@dom.ain +To: mylist-owner@dom.ain +Subject: Your message awaits moderator approval +X-List-Administrivia: yes + +The held message. + +--BOUND-- +""") + self.assertFalse(is_membership_reminder_bounce(msg)) + + def test_explicit_header(self): + msg = email.message_from_string("""\ +Content-Type: multipart/report; boundary="BOUND" + +--BOUND +Content-Type: message/rfc822 + +From: mailman-owner@dom.ain +To: user@example.com +Subject: anything +X-Mailman-Membership-Reminder: yes + +Reminder body. + +--BOUND-- +""") + self.assertTrue(is_membership_reminder_bounce(msg)) + + + +class ReminderBounceQueueTest(unittest.TestCase): + def test_queue_reminder_bounces(self): + from Mailman.Queue.BounceRunner import BounceMixin + import Mailman.Queue.BounceRunner as br_mod + from Mailman import mm_cfg + + class FakeList(object): + bounce_processing = 1 + send_reminders = 1 + + def isMember(self, addr): + return addr == 'user@example.com' + + def getMemberOption(self, addr, option): + return 0 + + class TestMixin(BounceMixin): + pass + + runner = TestMixin() + msg = email.message_from_string('Subject: bounce\n\n') + old_list_names = Utils.list_names + old_mail_list = br_mod.MailList + try: + Utils.list_names = lambda: [mm_cfg.MAILMAN_SITE_LIST, 'mylist'] + br_mod.MailList = lambda name, lock=0: FakeList() + runner._queue_reminder_bounces('user@example.com', msg) + finally: + Utils.list_names = old_list_names + br_mod.MailList = old_mail_list + + runner._bounce_events_fp.seek(0) + events = [] + while True: + try: + events.append(pickle.load(runner._bounce_events_fp)) + except EOFError: + break + self.assertEqual(len(events), 1) + self.assertEqual(events[0][0], 'mylist') + self.assertEqual(events[0][1], 'user@example.com') + runner._bounce_events_fp.close() + os.unlink(runner._bounce_events_file) + runner._bounce_events_fp = None + runner._bouncecnt = 0 + + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(ReminderBounceTest)) + suite.addTest(unittest.makeSuite(ReminderBounceQueueTest)) + return suite + + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') From 24a73619da7c78a01798901bea2b32ce0d223ed2 Mon Sep 17 00:00:00 2001 From: Jared Mauch Date: Mon, 6 Jul 2026 12:20:16 -0400 Subject: [PATCH 2/4] Improve test Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/test_bounce_reminders.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/test_bounce_reminders.py b/tests/test_bounce_reminders.py index c2455ff9..9f9b5983 100644 --- a/tests/test_bounce_reminders.py +++ b/tests/test_bounce_reminders.py @@ -22,20 +22,25 @@ from Mailman.Queue.BounceRunner import is_membership_reminder_bounce -SAMPLE = os.path.join( - _TESTDIR, 'bounces', 'samples_from_messages', - 'sample_01_undelivered_mail_returned_to_sender.eml') - - class ReminderBounceTest(unittest.TestCase): - def test_membership_reminder_sample(self): - if not os.path.exists(SAMPLE): - self.skipTest('sample bounce file not available') - with open(SAMPLE, 'rb') as fp: - msg = email.message_from_binary_file(fp) - self.assertTrue(is_membership_reminder_bounce(msg)) + def test_membership_reminder_heuristic(self): + msg = email.message_from_string("""\ +Content-Type: multipart/report; boundary=\"BOUND\" +--BOUND +Content-Type: message/rfc822 + +From: mylist-bounces@dom.ain +To: user@example.com +Subject: Monthly membership reminder +X-List-Administrivia: yes + +Reminder body. + +--BOUND-- +""") + self.assertTrue(is_membership_reminder_bounce(msg)) def test_owner_notification_is_not_reminder(self): msg = email.message_from_string("""\ Content-Type: multipart/report; boundary="BOUND" From 3a46e66b89ea8ce8c2f93c7a220816281691e8a8 Mon Sep 17 00:00:00 2001 From: Jared Mauch Date: Mon, 6 Jul 2026 12:20:46 -0400 Subject: [PATCH 3/4] Improve comment Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Mailman/Queue/BounceRunner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Mailman/Queue/BounceRunner.py b/Mailman/Queue/BounceRunner.py index cb23a1cd..d24de2f7 100644 --- a/Mailman/Queue/BounceRunner.py +++ b/Mailman/Queue/BounceRunner.py @@ -77,8 +77,8 @@ def __init__(self): # register a bounce for all 4 lists that aperson@example.com was a # member of, and eventually that person would get disabled on all # their lists. So now we ignore most site list bounces. Membership - # reminder bounces can optionally be processed; see - # BOUNCE_PROCESS_REMINDER_BOUNCES. + # reminder bounces can be processed when BOUNCE_PROCESS_REMINDER_BOUNCES + # is enabled. self._bounce_events_file = os.path.join( mm_cfg.DATA_DIR, 'bounce-events-%05d.pck' % os.getpid()) self._bounce_events_fp = None From c1c820410274236742dd4bf93aa63dc8e9a79774 Mon Sep 17 00:00:00 2001 From: Jared Mauch Date: Mon, 6 Jul 2026 12:21:58 -0400 Subject: [PATCH 4/4] improvements Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Mailman/Queue/BounceRunner.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Mailman/Queue/BounceRunner.py b/Mailman/Queue/BounceRunner.py index d24de2f7..c7cb2aa4 100644 --- a/Mailman/Queue/BounceRunner.py +++ b/Mailman/Queue/BounceRunner.py @@ -102,6 +102,14 @@ def _queue_bounces(self, listname, addrs, msg): def _queue_reminder_bounces(self, addr, msg): """Queue a reminder bounce for each subscribed list that sends reminders.""" + today = time.localtime()[:3] + if self._bounce_events_fp is None: + omask = os.umask(0o006) + try: + self._bounce_events_fp = open(self._bounce_events_file, 'a+b') + finally: + os.umask(omask) + queued = 0 for listname in Utils.list_names(): if listname.lower() == mm_cfg.MAILMAN_SITE_LIST.lower(): continue @@ -114,7 +122,12 @@ def _queue_reminder_bounces(self, addr, msg): continue if mlist.getMemberOption(addr, mm_cfg.SuppressPasswordReminder): continue - self._queue_bounces(listname, [addr], msg) + pickle.dump((listname, addr, today, msg), self._bounce_events_fp, 1) + queued += 1 + if queued: + self._bounce_events_fp.flush() + os.fsync(self._bounce_events_fp.fileno()) + self._bouncecnt += queued def _register_bounces(self): syslog('bounce', '%s processing %s queued bounces',