test-dovecot.py: convert to py3 and fix compat with newer releasesHEADmaster

2 files changed, 167 insertions, 52 deletions

diff --git a/scripts/test-dovecot.py b/scripts/test-dovecot.py
index ab9ca715..41f8e512 100755
--- a/scripts/test-dovecot.py
+++ b/scripts/test-dovecot.py
@@ -1,7 +1,7 @@
-#!/usr/bin/python2
+#!/usr/bin/python3
 #
 #    dovecot.py quality assurance test script
-#    Copyright (C) 2008-2020 Canonical Ltd.
+#    Copyright (C) 2008-2026 Canonical Ltd.
 #    Modified by: Marc Deslauriers <marc.deslauriers@canonical.com>
 #
 #    This program is free software: you can redistribute it and/or modify
@@ -54,7 +54,46 @@ class DovecotBasics(testlib.TestlibCase):
         # CVE-2008-1199 but if not enabled, it fails due to
         # http://wiki2.dovecot.org/Errors/ChgrpNoPerm when
         # when using postfix/procmail for an MDA
-        config = '''
+
+        if self.lsb_release['Release'] >= 25.10:
+            config = '''
+dovecot_config_version = 2.4.0
+dovecot_storage_version = 2.4.0
+log_timestamp = "%Y-%m-%d %H:%M:%S "
+#mail_extra_groups = mail
+mail_privileged_group = mail
+protocols = imap pop3
+mail_access_groups = mail
+mail_driver = mbox
+mail_path = ~/mail
+mail_inbox_path = /var/mail/%{user}
+
+protocol imap {
+  mail_plugins = acl imap_acl
+}
+passdb mypassdb {
+  driver = pam
+}
+userdb myuserdb {
+  driver = passwd
+}
+service auth {
+  unix_listener auth-master {
+    mode = 0600
+  }
+  user = root
+}
+lda_mailbox_autocreate = yes
+protocol lda {
+  postmaster_address = root@localhost
+  mail_plugins = sieve
+  auth_socket_path = /var/run/dovecot/auth-master
+}
+
+acl_driver = vfile
+'''
+        else:
+            config = '''
 log_timestamp = "%Y-%m-%d %H:%M:%S "
 #mail_extra_groups = mail
 mail_privileged_group = mail
@@ -105,19 +144,19 @@ mmap_disable = yes
         '''Internal factorization of POP3 protocol checks with an established
         connection.'''
 
-        self.assertEqual(pop.user(self.user.login), '+OK')
-        self.assertEqual(pop.pass_(self.user.password), '+OK Logged in.')
+        self.assertEqual(pop.user(self.user.login), b'+OK')
+        self.assertEqual(pop.pass_(self.user.password), b'+OK Logged in.')
 
         # check messages
         self.assertEqual(pop.stat()[0], 2, '2 available messages')
-        self.assertEqual(pop.list()[1], ['1 163', '2 161'])
-        self.assertEqual('\n'.join(pop.retr(1)[1]), '''Date: Thu, 16 Nov 2006 17:12:23 -0800
+        self.assertEqual(pop.list()[1], [b'1 163', b'2 161'])
+        self.assertEqual(b'\n'.join(pop.retr(1)[1]), b'''Date: Thu, 16 Nov 2006 17:12:23 -0800
 From: Test User 1 <test1@test1.com>
 To: Dovecot tester <dovecot@test.com>
 Subject: Test 1
 
 Some really important news.''')
-        self.assertEqual('\n'.join(pop.retr(2)[1]), '''Date: Tue, 28 Nov 2006 11:29:34 +0100
+        self.assertEqual(b'\n'.join(pop.retr(2)[1]), b'''Date: Tue, 28 Nov 2006 11:29:34 +0100
 From: Test User 2 <test2@test2.com>
 To: Dovecot tester <dovecot@test.com>
 Subject: Test 2
@@ -126,7 +165,7 @@ More news.
 
 Get cracking!''')
 
-        self.assertEqual(pop.quit(), '+OK Logging out.')
+        self.assertEqual(pop.quit(), b'+OK Logging out.')
 
         if self.lsb_release['Release'] >= 18.10:
             expected_status = 'Status: NRO\nStatus: R\n'
@@ -145,24 +184,28 @@ Get cracking!''')
         connection.'''
 
         # check empty password
-        self.assertEqual(pop.user(self.user.login), '+OK')
+        self.assertEqual(pop.user(self.user.login), b'+OK')
         self.assertRaises(poplib.error_proto, pop.pass_, '')
 
         # check wrong password
-        self.assertEqual(pop.user(self.user.login), '+OK')
+        self.assertEqual(pop.user(self.user.login), b'+OK')
         self.assertRaises(poplib.error_proto, pop.pass_, '123')
 
     def test_pop3(self):
         '''Test POP3 protocol'''
 
         if (self.lsb_release['Release'] == 14.04 or
-           self.lsb_release['Release'] >= 17.10):
-            message = '+OK Dovecot (Ubuntu) ready.'
+            (self.lsb_release['Release'] >= 17.10 and
+             self.lsb_release['Release'] < 25.10)):
+            message = b'+OK Dovecot (Ubuntu) ready.'
         else:
-            message = '+OK Dovecot ready.'
+            message = b'+OK Dovecot ready.'
 
         pop = poplib.POP3('localhost')
-        self.assertEqual(pop.getwelcome(), message)
+        welcome = pop.getwelcome()
+        if isinstance(welcome, str):
+            welcome = welcome.decode('utf-8')
+        self.assertEqual(welcome, message)
 
         self._test_pop3_proto(pop)
 
@@ -170,13 +213,17 @@ Get cracking!''')
         '''Test POP3 protocol'''
 
         if (self.lsb_release['Release'] == 14.04 or
-           self.lsb_release['Release'] >= 17.10):
-            message = '+OK Dovecot (Ubuntu) ready.'
+            (self.lsb_release['Release'] >= 17.10 and
+             self.lsb_release['Release'] < 25.10)):
+            message = b'+OK Dovecot (Ubuntu) ready.'
         else:
-            message = '+OK Dovecot ready.'
+            message = b'+OK Dovecot ready.'
 
         pop = poplib.POP3('localhost')
-        self.assertEqual(pop.getwelcome(), message)
+        welcome = pop.getwelcome()
+        if isinstance(welcome, str):
+            welcome = welcome.decode('utf-8')
+        self.assertEqual(welcome, message)
 
         self._test_pop3_proto_badauth(pop)
 
@@ -184,13 +231,17 @@ Get cracking!''')
         '''Test POP3S protocol'''
 
         if (self.lsb_release['Release'] == 14.04 or
-           self.lsb_release['Release'] >= 17.10):
-            message = '+OK Dovecot (Ubuntu) ready.'
+            (self.lsb_release['Release'] >= 17.10 and
+             self.lsb_release['Release'] < 25.10)):
+            message = b'+OK Dovecot (Ubuntu) ready.'
         else:
-            message = '+OK Dovecot ready.'
+            message = b'+OK Dovecot ready.'
 
         pop = poplib.POP3_SSL('localhost')
-        self.assertEqual(pop.getwelcome(), message)
+        welcome = pop.getwelcome()
+        if isinstance(welcome, str):
+            welcome = welcome.decode('utf-8')
+        self.assertEqual(welcome, message)
 
         self._test_pop3_proto(pop)
 
@@ -278,43 +329,47 @@ Get cracking!''')
         imap.login(self.user.login, self.user.password)
 
         # list mailboxes
-        status, list = imap.list()
+        status, imap_list = imap.list()
+        if isinstance(imap_list, str):
+            imap_list = imap_list.decode('utf-8')
+
         self.assertEqual(status, 'OK')
         found_inbox = False
 
         if self.lsb_release['Release'] >= 14.04:
-            inbox_name = 'INBOX'
+            inbox_name = b'INBOX'
         else:
-            inbox_name = '"INBOX"'
+            inbox_name = b'"INBOX"'
 
-        for entry in list:
+        for entry in imap_list:
             if entry.endswith(inbox_name):
                 found_inbox = True
         self.assertTrue(found_inbox)
 
         # check mails
         self.assertEqual(imap.select('INBOX')[0], 'OK')
-        self.assertEqual(imap.search(None, 'ALL'), ('OK', ['1 2']))
+        self.assertEqual(imap.search(None, 'ALL'), ('OK', [b'1 2']))
         self.assertEqual(imap.fetch('1', '(FLAGS)'), 
-            ('OK', ['1 (FLAGS (\\Recent))']))
+            ('OK', [b'1 (FLAGS (\\Recent))']))
 
         report = imap.fetch('2', '(FLAGS)')
         self.assertTrue(len(report)>1, report)
         self.assertTrue(len(report[1])>0, report)
         checkflags = second_flags
-        if 'Recent' in report[1][0] and not 'Recent' in checkflags:
+        if b'Recent' in report[1][0] and not 'Recent' in checkflags:
             if len(checkflags):
                 checkflags += ' '
             checkflags += '\\Recent'
-        self.assertEqual(report,
-            ('OK', ['2 (FLAGS (%s))'%(checkflags)]))
+        result = '2 (FLAGS (%s))' % checkflags
+        result = result.encode('utf-8')
+        self.assertEqual(report, ('OK', [result]))
 
         self.assertEqual(imap.fetch('1', '(BODY[TEXT])')[1][0][1].rstrip(), 
-            'Some really important news.')
+            b'Some really important news.')
         self.assertEqual(imap.fetch('2', '(BODY[TEXT])')[1][0][1].rstrip(), 
-            'More news.\r\n\r\nGet cracking!')
+            b'More news.\r\n\r\nGet cracking!')
 
-        email1='''Date: Thu, 16 Nov 2006 17:12:23 -0800\r
+        email1=b'''Date: Thu, 16 Nov 2006 17:12:23 -0800\r
 From: Test User 1 <test1@test1.com>\r
 To: Dovecot tester <dovecot@test.com>\r
 Subject: Test 1\r
@@ -324,11 +379,11 @@ Some really important news.\r
 
         # HACK to deal with procmail redelivery into Maildir
         if os.path.exists(self.user.home+"/Maildir"):
-            email1 = email1.replace('\r\n\r\n', '\r\nStatus: N\r\n\r\n', 1)+'\r\n'
+            email1 = email1.replace(b'\r\n\r\n', b'\r\nStatus: N\r\n\r\n', 1)+b'\r\n'
 
         size1 = len(email1)
         self.assertEqual(imap.fetch('1', '(RFC822)')[1],
-            [('1 (RFC822 {%d}' % (size1), email1), ')'])
+            [(b'1 (RFC822 {%d}' % (size1), email1), b')'])
 
         # save email to local mail store
         self.assertEqual(imap.create('NewStorage')[0], 'OK')
@@ -337,10 +392,10 @@ Some really important news.\r
         # delete mail 1
         self.assertEqual(imap.store('1', '+FLAGS', '\\Deleted')[0], 'OK')
         self.assertEqual(imap.expunge()[0], 'OK')
-        self.assertEqual(imap.search(None, 'ALL'), ('OK', ['1']))
+        self.assertEqual(imap.search(None, 'ALL'), ('OK', [b'1']))
 
         # old mail 2 is mail 1 now
-        email2='''Date: Tue, 28 Nov 2006 11:29:34 +0100\r
+        email2=b'''Date: Tue, 28 Nov 2006 11:29:34 +0100\r
 From: Test User 2 <test2@test2.com>\r
 To: Dovecot tester <dovecot@test.com>\r
 Subject: Test 2\r
@@ -351,26 +406,26 @@ Get cracking!'''
 
         # HACK to deal with procmail redelivery into Maildir
         if os.path.exists(self.user.home+"/Maildir"):
-            email2 = email2.replace('\r\n\r\n', '\r\nStatus: R\r\n\r\n', 1)+'\r\n\r\n'
+            email2 = email2.replace(b'\r\n\r\n', b'\r\nStatus: R\r\n\r\n', 1)+b'\r\n\r\n'
 
         size2 = len(email2)
         self.assertEqual(imap.fetch('1', '(RFC822)')[1],
-            [('1 (RFC822 {%d}' % (size2), email2), ')'])
+            [(b'1 (RFC822 {%d}' % (size2), email2), b')'])
 
         # pull messages back out of local mail store
         self.assertEqual(imap.select('NewStorage')[0], 'OK')
-        self.assertEqual(imap.search(None, 'ALL'), ('OK', ['1']))
+        self.assertEqual(imap.search(None, 'ALL'), ('OK', [b'1']))
         report = imap.fetch('1', '(FLAGS)')
         self.assertTrue(len(report)>1, report)
         self.assertTrue(len(report[1])>0, report)
-        if 'Recent' in report[1][0]:
+        if b'Recent' in report[1][0]:
             self.assertEqual(report,
-                ('OK', ['1 (FLAGS (\\Seen \\Recent))']))
+                ('OK', [b'1 (FLAGS (\\Seen \\Recent))']))
         else:
             self.assertEqual(report,
-                ('OK', ['1 (FLAGS (\\Seen))']))
+                ('OK', [b'1 (FLAGS (\\Seen))']))
         self.assertEqual(imap.fetch('1', '(RFC822)')[1],
-            [('1 (RFC822 {%d}' % (size1), email1), ')'])
+            [(b'1 (RFC822 {%d}' % (size1), email1), b')'])
 
         imap.close()
         imap.logout()
@@ -396,7 +451,13 @@ Get cracking!'''
         os.rename(second, mailpath+'/cur/'+os.path.basename(second))
 
         # Reconfigure dovecot for maildir location
-        subprocess.call(['sed', '-i', 's#^mail_location = mbox:~/mail:INBOX=/var/mail/%u#mail_location = maildir:~/Maildir#', '/etc/dovecot/dovecot.conf'])
+        if self.lsb_release['Release'] >= 25.10:
+            subprocess.call(['sed', '-i', 's#^mail_driver = mbox#mail_driver = maildir#', '/etc/dovecot/dovecot.conf'])
+            subprocess.call(['sed', '-i', 's#^mail_path = ~/mail#mail_path = ~/Maildir#', '/etc/dovecot/dovecot.conf'])
+            subprocess.call(['sed', '-i', 's#^mail_inbox_path = /var/mail/%{user}##', '/etc/dovecot/dovecot.conf'])
+        else:
+            subprocess.call(['sed', '-i', 's#^mail_location = mbox:~/mail:INBOX=/var/mail/%u#mail_location = maildir:~/Maildir#', '/etc/dovecot/dovecot.conf'])
+
         self.dovecot.reload_conf(self)
 
     def test_imap(self):
diff --git a/scripts/testlib_dovecot.py b/scripts/testlib_dovecot.py
index e5c203c0..658cbb04 100755
--- a/scripts/testlib_dovecot.py
+++ b/scripts/testlib_dovecot.py
@@ -44,10 +44,14 @@ class Dovecot:
         user is set up, and /var/mail/$user contains an unread and a read mail.
         '''
 
+        # This is insane, we should just use lsb_release here
         self.new_version = False
         with open("/etc/dovecot/dovecot.conf", "r") as f:
-            if f.read().find('!include_try /usr/share/dovecot/protocols.d/*.protocol')>0:
+            contents = f.read()
+            if contents.find('!include_try /usr/share/dovecot/protocols.d/*.protocol')>0:
                 self.new_version = True
+            if contents.find('dovecot_config_version = 2.4.0')>0:
+                self.even_newer_version = True
 
         # Stop dovecot before we make any changes to the config file
         self.pidfile = "/var/run/dovecot/master.pid"
@@ -93,6 +97,38 @@ auth_passdb = pam
 auth_user = root
 
 '''
+                elif self.even_newer_version:
+                    # dovecot 2.4.x
+                    config='''
+
+dovecot_config_version = 2.4.0
+dovecot_storage_version = 2.4.0
+
+auth_mechanisms = plain cram-md5
+log_timestamp = "%Y-%m-%d %H:%M:%S "
+mail_path = ~/mail:INBOX=/var/mail/%u
+mail_driver = mbox
+mail_privileged_group = mail
+# Yuck, the following makes dovecot vulnerable to
+# CVE-2008-1199 but if not enabled, it fails due to
+# http://wiki2.dovecot.org/Errors/ChgrpNoPerm when
+# when using postfix/procmail for an MDA
+mail_access_groups = mail
+passdb {
+  args = /etc/dovecot/test.passwd
+  driver = passwd-file
+}
+passdb {
+  driver = pam
+}
+protocols = imap pop3
+service auth {
+  user = root
+}
+userdb {
+  driver = passwd
+}
+'''
                 elif self.new_version:
                     # dovecot 2.0.x
                     config='''
@@ -145,7 +181,23 @@ auth default {
   user = root
 }
 '''
-        if self.new_version:
+
+        if self.even_newer_version:
+            key_config_str = 'ssl_server_key_file = %s\n'
+            cert_config_str = 'ssl_server_cert_file = %s\n'
+            # ssl_key and ssl_cert need to be defined in dovecot 2.0.x
+            # Raring+ uses snakeoil by default
+            if cert_key == None:
+                if os.path.exists('/etc/ssl/private/dovecot.pem'):
+                    cert_key = '/etc/ssl/private/dovecot.pem'
+                else:
+                    cert_key = '/etc/ssl/private/ssl-cert-snakeoil.key'
+            if cert_pub == None:
+                if os.path.exists('/etc/ssl/certs/dovecot.pem'):
+                    cert_pub = '/etc/ssl/certs/dovecot.pem'
+                else:
+                    cert_pub = '/etc/ssl/certs/ssl-cert-snakeoil.pem'
+        elif self.new_version:
             key_config_str = 'ssl_key = <%s\n'
             cert_config_str = 'ssl_cert = <%s\n'
             # ssl_key and ssl_cert need to be defined in dovecot 2.0.x
@@ -166,10 +218,10 @@ auth default {
 
         self.cert_key = cert_key
         if self.cert_key:
-            config = key_config_str % (self.cert_key) + config
+            config = config + key_config_str % (self.cert_key)
         self.cert_pub = cert_pub
         if self.cert_pub:
-            config = cert_config_str % (self.cert_pub) + config
+            config = config + cert_config_str % (self.cert_pub)
 
         with open("/etc/dovecot/conf.d/10-ssl.conf", "r") as f:
             if f.read().find('ssl_dh = </usr/share/dovecot/dh.pem')>0:
@@ -277,7 +329,8 @@ Get cracking!
                 os.unlink(socket)
 
         # restore original configuration
-        os.rename('/etc/dovecot/dovecot.conf.autotest', '/etc/dovecot/dovecot.conf')
+        if os.path.exists('/etc/dovecot/dovecot.conf.autotest'):
+            os.rename('/etc/dovecot/dovecot.conf.autotest', '/etc/dovecot/dovecot.conf')
         if os.path.exists('/etc/dovecot/dovecot-postfix.conf.autotest'):
             os.rename('/etc/dovecot/dovecot-postfix.conf.autotest', '/etc/dovecot/dovecot-postfix.conf')
 
@@ -290,6 +343,7 @@ Get cracking!
         # reload the daemon
         if os.path.exists('/etc/init.d/dovecot'):
             subprocess.call(['/etc/init.d/dovecot', 'reload'], stdout=subprocess.PIPE)
+            time.sleep(1)
         else:
             subprocess.call(['reload', 'dovecot'], stdout=subprocess.PIPE)