Fix body_line_iterator(decode=True) returning no lines in Python 3.11

Mailman/Bouncers/AOL.py

@@ -29,7 +29,10 @@ def process(msg):
         return
     addrs = []
     found = False
-    for line in msg.get_payload(decode=True).splitlines():
+    payload = msg.get_payload(decode=True)
+    if isinstance(payload, bytes):
+        payload = payload.decode('us-ascii', errors='replace')
+    for line in payload.splitlines():
         if scre.search(line):
             found = True
             continue

Mailman/Bouncers/SimpleMatch.py

@@ -17,10 +17,30 @@
 
 """Recognizes simple heuristically delimited bounces."""
 
+import io
 import re
 import email.iterators
 
 
+def _body_line_iterator(msg):
+    """Iterate body lines with Content-Transfer-Encoding properly decoded.
+
+    Python 3's body_line_iterator only yields str payloads; get_payload(
+    decode=True) returns bytes (CTE-decoded). This helper bridges the gap so
+    quoted-printable and base64 bodies are decoded before pattern matching.
+    """
+    for subpart in email.iterators.typed_subpart_iterator(msg):
+        payload = subpart.get_payload(decode=True)
+        if not isinstance(payload, bytes):
+            continue
+        charset = subpart.get_content_charset('us-ascii') or 'us-ascii'
+        try:
+            text = payload.decode(charset, errors='replace')
+        except LookupError:
+            text = payload.decode('us-ascii', errors='replace')
+        yield from io.StringIO(text)
+
+
 
 def _c(pattern):
     return re.compile(pattern, re.IGNORECASE)
@@ -224,7 +244,7 @@ def process(msg, patterns=None):
     # we process the message multiple times anyway.
     for scre, ecre, acre in patterns:
         state = 0
-        for line in email.iterators.body_line_iterator(msg, decode=True):
+        for line in _body_line_iterator(msg):
             if state == 0:
                 if scre.search(line):
                     state = 1

Mailman/Bouncers/SimpleWarning.py

@@ -21,7 +21,7 @@
 import email.iterators
 
 from Mailman.Bouncers.BouncerAPI import Stop
-from Mailman.Bouncers.SimpleMatch import _c
+from Mailman.Bouncers.SimpleMatch import _c, _body_line_iterator
 
 
 
@@ -75,7 +75,7 @@ def process(msg):
     addrs = {}
     for scre, ecre, acre in patterns:
         state = 0
-        for line in email.iterators.body_line_iterator(msg, decode=True):
+        for line in _body_line_iterator(msg):
             if state == 0:
                 if scre.search(line):
                     state = 1

Mailman/Cgi/create.py

@@ -31,7 +31,7 @@
 from Mailman import i18n
 from Mailman.htmlformat import *
 from Mailman.Logging.Syslog import syslog
-from Mailman.Utils import sha_new
+from Mailman.Utils import sha_new, hash_password
 
 # Set up i18n
 _ = i18n._
@@ -199,7 +199,7 @@ def sigterm_handler(signum, frame, mlist=mlist):
         # Install the emergency shutdown signal handler
         signal.signal(signal.SIGTERM, sigterm_handler)
 
-        pw = sha_new(password).hexdigest()
+        pw = hash_password(password)
         # Guarantee that all newly created files have the proper permission.
         # proper group ownership should be assured by the autoconf script
         # enforcing that all directories have the group sticky bit set

Mailman/Handlers/Tagger.py

@@ -26,6 +26,7 @@
 from email.header import decode_header
 
 from Mailman import Utils
+from Mailman.Bouncers.SimpleMatch import _body_line_iterator
 from Mailman.Logging.Syslog import syslog
 from Mailman.Handlers.CookHeaders import change_header
 
@@ -97,7 +98,7 @@ def scanbody(msg, numlines=None):
     # the first numlines of body text.
     lines = []
     lineno = 0
-    reader = list(email.iterators.body_line_iterator(msg, decode=True))
+    reader = list(_body_line_iterator(msg))
     while numlines is None or lineno < numlines:
         try:
             line = reader.pop(0)

Mailman/MTA/Postfix.py

@@ -442,12 +442,12 @@ def checkperms(state):
         if stat and (stat[ST_MODE] & targetmode) != targetmode:
             state.ERRORS += 1
             octmode = oct(stat[ST_MODE])
-            print(C_('%(file)s permissions must be 0664 (got %(octmode)s)'),)
+            print(C_('%(file)s permissions must be 0664 (got %(octmode)s)') % locals())
             if state.FIX:
                 print(C_('(fixing)'))
                 os.chmod(file, stat[ST_MODE] | targetmode)
             else:
-                print
+                print()
         # Make sure the corresponding .db files are owned by the Mailman user.
         # We don't need to check the group ownership of the file, since
         # check_perms checks this itself.
@@ -481,7 +481,7 @@ def checkperms(state):
         if stat and (stat[ST_MODE] & targetmode) != targetmode:
             state.ERRORS += 1
             octmode = oct(stat[ST_MODE])
-            print(C_('%(dbfile)s permissions must be 0664 (got %(octmode)s)'),)
+            print(C_('%(dbfile)s permissions must be 0664 (got %(octmode)s)') % locals())
             if state.FIX:
                 print(C_('(fixing)'))
                 os.chmod(dbfile, stat[ST_MODE] | targetmode)

Mailman/Utils.py

@@ -727,6 +727,11 @@ def hash_password(plaintext):
         binascii.hexlify(dk).decode('ascii'))
 
 
+def is_old_password_format(pw):
+    """Return True if pw is a legacy password that needs upgrading to PBKDF2."""
+    return bool(pw) and not pw.startswith(_PBKDF2_SHA256_PREFIX)
+
+
 def verify_password(response, stored):
     """Verify password against stored hash.
 

bin/check_perms

@@ -141,7 +141,7 @@ def checkwalk(arg, dirname, names):
         octperms = oct(targetperms)
         if S_ISDIR(mode) and (mode & targetperms) != targetperms:
             arg.ERRORS += 1
-            print(C_('directory permissions must be %(octperms)s: %(path)s'),)
+            print(C_('directory permissions must be %(octperms)s: %(path)s') % locals())
             if STATE.FIX:
                 print(C_('(fixing)'))
                 os.chmod(path, mode | targetperms)
@@ -150,7 +150,7 @@ def checkwalk(arg, dirname, names):
         elif os.path.splitext(path)[1] in ('.py', '.pyc', '.pyo'):
             octperms = oct(PYFILEPERMS)
             if mode & PYFILEPERMS != PYFILEPERMS:
-                print(C_('source perms must be %(octperms)s: %(path)s'),)
+                print(C_('source perms must be %(octperms)s: %(path)s') % locals())
                 arg.ERRORS += 1
                 if STATE.FIX:
                     print(C_('(fixing)'))
@@ -161,7 +161,7 @@ def checkwalk(arg, dirname, names):
             # Article files must be group writeable
             octperms = oct(ARTICLEFILEPERMS)
             if mode & ARTICLEFILEPERMS != ARTICLEFILEPERMS:
-                print(C_('article db files must be %(octperms)s: %(path)s'),)
+                print(C_('article db files must be %(octperms)s: %(path)s') % locals())
                 arg.ERRORS += 1
                 if STATE.FIX:
                     print(C_('(fixing)'))
@@ -187,7 +187,7 @@ def checkall():
             continue
         if (mode & DIRPERMS) != DIRPERMS:
             STATE.ERRORS += 1
-            print(C_('directory must be at least 02775: %(d)s'),)
+            print(C_('directory must be at least 02775: %(d)s') % locals())
             if STATE.FIX:
                 print(C_('(fixing)'))
                 os.chmod(d, mode | DIRPERMS)
@@ -204,7 +204,7 @@ def checkarchives():
     mode = statmode(private)
     if mode & S_IROTH:
         STATE.ERRORS += 1
-        print(C_('%(private)s must not be other-readable'),)
+        print(C_('%(private)s must not be other-readable') % locals())
         if STATE.FIX:
             print(C_('(fixing)'))
             os.chmod(private, mode & ~S_IROTH)
@@ -253,7 +253,7 @@ def checkarchivedbs():
             continue
         if mode & S_IRWXO:
             STATE.ERRORS += 1
-            print(C_('%(dbdir)s "other" perms must be 000'),)
+            print(C_('%(dbdir)s "other" perms must be 000') % locals())
             if STATE.FIX:
                 print(C_('(fixing)'))
                 os.chmod(dbdir, mode & ~S_IRWXO)
@@ -272,7 +272,7 @@ def checkcgi():
         mode = statmode(path)
         if mode & S_IXGRP and not mode & S_ISGID:
             STATE.ERRORS += 1
-            print(C_('%(path)s must be set-gid'),)
+            print(C_('%(path)s must be set-gid') % locals())
             if STATE.FIX:
                 print(C_('(fixing)'))
                 os.chmod(path, mode | S_ISGID)
@@ -286,7 +286,7 @@ def checkmail():
     mode = statmode(wrapper)
     if not mode & S_ISGID:
         STATE.ERRORS += 1
-        print(C_('%(wrapper)s must be set-gid'),)
+        print(C_('%(wrapper)s must be set-gid') % locals())
         if STATE.FIX:
             print(C_('(fixing)'))
             os.chmod(wrapper, mode | S_ISGID)
@@ -346,7 +346,7 @@ def checkdata():
                 continue
             if (mode & targetmode) != targetmode:
                 STATE.ERRORS += 1
-                print(C_('file permissions must be at least 660: %(path)s'),)
+                print(C_('file permissions must be at least 660: %(path)s') % locals())
                 if STATE.FIX:
                     print(C_('(fixing)'))
                     os.chmod(path, mode | targetmode)

bin/newlist

@@ -241,8 +241,7 @@ def main():
 
     # And send the notice to the list owner
     if not quiet and not automate:
-        print('Hit enter to notify %(listname)s owner...'),
-        sys.stdin.readline()
+        input('Hit enter to notify %s owner...' % listname)
     if not quiet:
         siteowner = Utils.get_site_email(mlist.host_name, 'owner')
         text = Utils.maketext(

bin/update

@@ -814,27 +814,27 @@ def send_password_upgrade_notification(mlist):
         listaddr = mlist.GetListEmail()
         siteowner = Utils.get_site_email(mlist.host_name, 'owner')
 
-        text = _("""\
+        text = C_("""\
 This is an automated message from your Mailman installation.
 
-Your mailing list "%(listname)s" (%(listaddr)s) is using an older password 
-hashing format. For security reasons, we have upgraded Mailman to use a more 
+Your mailing list "%(listname)s" (%(listaddr)s) is using an older password
+hashing format. For security reasons, we have upgraded Mailman to use a more
 secure password hashing method (PBKDF2-SHA256 instead of SHA1).
 
 To complete the upgrade, please log in to your list administration page:
 
     %(admin_url)s
 
-Simply logging in with your current list administrator password will 
-automatically upgrade your password to the new secure format. You don't 
-need to change your password - just log in once and the upgrade will happen 
+Simply logging in with your current list administrator password will
+automatically upgrade your password to the new secure format. You don't
+need to change your password - just log in once and the upgrade will happen
 automatically.
 
-This is a one-time upgrade. After you log in, your password will be 
-automatically converted to the new format and you won't need to do anything 
+This is a one-time upgrade. After you log in, your password will be
+automatically converted to the new format and you won't need to do anything
 else.
 
-If you have any questions or concerns, please contact the site administrator 
+If you have any questions or concerns, please contact the site administrator
 at %(siteowner)s.
 
 Thank you for your attention to this important security upgrade.
@@ -844,8 +844,8 @@ Thank you for your attention to this important security upgrade.
             'admin_url': admin_url,
             'siteowner': siteowner,
         }
-        
-        subject = _('Action Required: Password Upgrade for %(listname)s') % {
+
+        subject = C_('Action Required: Password Upgrade for %(listname)s') % {
             'listname': listname
         }