email: Add support for Dovecot 2.4

Sunil:

- When dovecot package is upgrade from 2.3 to 2.4 during distribution upgrade,
automatically re-run setup.

- Upgrade existing setups to new scheme by re-running setup with incremented app
version.

- Don't query dovecot version during app initialization. Instead overwrite the
DropinConfigs component to query dovecot version during setup and enable
operations.

- Use apt.Cache() to retrieve the installed version of dovecot package. Use
plinth.utils.Version to parse the version and perform a comparison.

- Split even configuration files that have not changed for simplicity.

- Add/update links in Dovecot configuration files.

Tests:

- Install email app on a testing container. Ensure that all files in
/etc/dovecot/conf.d/ are linked properly to 2.4 versions. TLS configuration is
accurate. Use Sogo to test login and sending mails.

  - User with LDAP account and correct password is able to login.

  - User without LDAP account or incorrect password is unable to login.

  - Send mail with Sogo to another account on the server. Notice that mails are
  stored in /var/mail/{user}/mail/ with mail:mail ownership in mbox format.

  - Logging in with email such as user@example.com works. Capital letters are
  allowed.

  - "Archive", "Drafts", "Sent", "Junk", "Trash" folders are automatically
  created and are marked with special flags. Creating additional folders such
  as "Sent Items" also results in them having special flags.

  - Thunderbird is able to connect via SSL with a self-signed certificate
  exception.

  - When an example spam message is sent, it is automatically moved to "Junk"
  folder after getting marked by rspamd.

  - When a message is moved to Junk folder, it is learned as spam by rspamd as
  seen in its admin console.

  - When a message is moved out of Junk folder (to other than "Trash" folder),
  it is learned as not-spam by rspamd as seen in its admin console.

- Install email app on a stable container with patches. Ensure that all files in
/etc/dovecot/conf.d/ are linked properly to 2.3 versions. TLS configuration is
accurate. Use Sogo to test login and sending mails.

- Install email app on a stable container without patches. Apply patches. Ensure
that all files in /etc/dovecot/conf.d/ are linked properly to 2.3 versions. TLS
configuration is accurate for dovecot 2.3. Use Sogo to test login and sending
mails. Perform distribution upgrade to testing. Ensure that all files in
/etc/dovecot/conf.d/ are linked properly to 2.3 versions. TLS configuration is
accurate for dovecot 2.4. Use Sogo to test login and sending mails.

Signed-off-by: Benedek Nagy <contact@nbenedek.me>
Signed-off-by: Sunil Mohan Adapa <sunil@medhas.org>
Reviewed-by: Sunil Mohan Adapa <sunil@medhas.org>

plinth/modules/email/__init__.py

@@ -20,7 +20,7 @@ from plinth.privileged import service as service_privileged
 from plinth.signals import domain_added, domain_removed
 from plinth.utils import format_lazy, gettext_noop
 
-from . import aliases, manifest, privileged
+from . import aliases, dovecot, manifest, privileged
 
 _description = [
     _('This is a complete email server solution using Postfix, Dovecot, '
@@ -52,7 +52,7 @@ class EmailApp(plinth.app.App):
 
     app_id = 'email'
 
-    _version = 6
+    _version = 7
 
     def __init__(self) -> None:
         """Initialize the email app."""
@@ -95,21 +95,12 @@ class EmailApp(plinth.app.App):
                 'dovecot-lmtpd', 'dovecot-managesieved', 'dovecot-ldap',
                 'rspamd', 'redis-server', 'openssl'
             ], conflicts=['exim4-base', 'exim4-config', 'exim4-daemon-light'],
-            conflicts_action=Packages.ConflictsAction.REMOVE)
+            conflicts_action=Packages.ConflictsAction.REMOVE,
+            rerun_setup_on_upgrade=True)
         self.add(packages)
 
         dropin_configs = DropinConfigs('dropin-configs-email', [
             '/etc/apache2/conf-available/email-freedombox.conf',
-            '/etc/dovecot/conf.d/05-freedombox-passdb.conf',
-            '/etc/dovecot/conf.d/05-freedombox-userdb.conf',
-            '/etc/dovecot/conf.d/15-freedombox-auth.conf',
-            '/etc/dovecot/conf.d/15-freedombox-mail.conf',
-            '/etc/dovecot/conf.d/90-freedombox-imap.conf',
-            '/etc/dovecot/conf.d/90-freedombox-lmtp.conf',
-            '/etc/dovecot/conf.d/90-freedombox-mailboxes.conf',
-            '/etc/dovecot/conf.d/90-freedombox-master.conf',
-            '/etc/dovecot/conf.d/90-freedombox-tls.conf',
-            '/etc/dovecot/conf.d/freedombox-ldap.conf.ext',
             '/etc/fail2ban/jail.d/dovecot-freedombox.conf',
             '/etc/postfix/freedombox-aliases.cf',
             '/etc/rspamd/local.d/freedombox-logging.inc',
@@ -121,10 +112,24 @@ class EmailApp(plinth.app.App):
         dropin_configs_sieve = DropinConfigs('dropin-configs-email-sieve', [
             '/etc/dovecot/freedombox-sieve/learn-ham.sieve',
             '/etc/dovecot/freedombox-sieve/learn-spam.sieve',
-            '/etc/dovecot/freedombox-sieve-after/sort-spam.sieve',
-            '/etc/dovecot/conf.d/95-freedombox-sieve.conf'
+            '/etc/dovecot/freedombox-sieve-after/sort-spam.sieve'
         ])
         self.add(dropin_configs_sieve)
+        dropin_configs_dovecot = DovecotDropinConfigs(
+            'dropin-configs-email-dovecot', [
+                '/etc/dovecot/conf.d/05-freedombox-passdb.conf',
+                '/etc/dovecot/conf.d/05-freedombox-userdb.conf',
+                '/etc/dovecot/conf.d/15-freedombox-auth.conf',
+                '/etc/dovecot/conf.d/15-freedombox-mail.conf',
+                '/etc/dovecot/conf.d/90-freedombox-imap.conf',
+                '/etc/dovecot/conf.d/90-freedombox-lmtp.conf',
+                '/etc/dovecot/conf.d/90-freedombox-mailboxes.conf',
+                '/etc/dovecot/conf.d/90-freedombox-master.conf',
+                '/etc/dovecot/conf.d/90-freedombox-tls.conf',
+                '/etc/dovecot/conf.d/95-freedombox-sieve.conf',
+                '/etc/dovecot/conf.d/freedombox-ldap.conf.ext'
+            ])
+        self.add(dropin_configs_dovecot)
 
         listen_ports = [(25, 'tcp4'), (25, 'tcp6'), (465, 'tcp4'),
                         (465, 'tcp6'), (587, 'tcp4'), (587, 'tcp6')]
@@ -212,6 +217,7 @@ class EmailApp(plinth.app.App):
         # Enable drop-in configuration files component for sieve (temporarily)
         # to ensure that sievec can compile.
         self.get_component('dropin-configs-email-sieve').enable()
+        self.get_component('dropin-configs-email-dovecot').enable()
         service_privileged.try_restart('dovecot')
         privileged.setup_spam()
 
@@ -228,6 +234,20 @@ class EmailApp(plinth.app.App):
             service_privileged.try_restart('rspamd')
 
 
+class DovecotDropinConfigs(DropinConfigs):
+    """Configure dovecot based on its package version."""
+
+    def get_target_path(self, path):
+        """Return Path object for a target path."""
+        version = '2.3'
+        if dovecot.is_version_24():
+            version = '2.4'
+
+        target_path = super().get_target_path(path)
+        target_path = target_path.parent / version / target_path.name
+        return target_path
+
+
 def _get_first_admin():
     """Return an admin user in the system or None if non exist."""
     from django.contrib.auth.models import User

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/05-freedombox-passdb.conf

@@ -0,0 +1,20 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# See:
+# https://doc.dovecot.org/main/core/config/auth/passdb.html
+# https://doc.dovecot.org/main/howto/active_directory.html
+#
+# For passdb, the passwd driver looks up using NSS. In FreedomBox, NSS is
+# configured to lookup LDAP with the help of libnss-ldapd. Lookup using passdb
+# would have been sufficient if FreedomBox allowed all its users to login using
+# pam. However, by default, FreedomBox disallows all users but 'admin' group to
+# login. Hence, the need for LDAP lookup.
+#
+passdb freedombox-ldap {
+  driver = ldap
+  ldap_uris = ldapi:///
+  ldap_base = dc=thisbox
+  ldap_bind = yes
+  ldap_bind_userdn = uid=%{user},ou=users,dc=thisbox
+  ldap_filter = (&(objectClass=posixAccount)(uid=%{user}))
+}

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/05-freedombox-userdb.conf

@@ -0,0 +1,32 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# See:
+# https://doc.dovecot.org/main/core/config/auth/userdb.html
+#
+# Users in FreedomBox are not expected to access mail by logging into the
+# system. Storing the mail in single location instead of home directories and
+# with single UID/GID simplifies security reasoning and backup/restore
+# operations.
+#
+# When FreedomBox has multiple domains a user is expected to get a mailbox that
+# is same across the domains. Changing an domain name is not uncommon in
+# FreedomBox. So, authenticate and store mails based on username only instead of
+# including domain names in storage path.
+#
+# Directories are created under /var/mail as necessary by dovecot. Permissions
+# for newly created directories are inherited from parent directory. FreedomBox
+# will remove all permissions for 'others' from /var/mail to ensure that mail is
+# not read by non-root users.
+#
+# userdb provides lookup for three parameters after authentication of a user.
+# These parameters are uid, gid, and home directory of the user. If these do not
+# change from user to user, a 'static' database type with fixed values is
+# sufficient as userdb.
+userdb freedombox-static {
+    driver = static
+    fields {
+        uid=mail
+        gid=mail
+        home=/var/mail/%{user | username | lower}
+    }
+}

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/15-freedombox-auth.conf

@@ -0,0 +1,10 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# See:
+# https://doc.dovecot.org/main/core/config/auth/basic.html
+# https://doc.dovecot.org/main/core/config/auth/databases/ldap.html#username
+
+# Outlook and Windows Mail work only with LOGIN mechanism, not the standard PLAIN
+auth_mechanisms = plain login
+
+auth_username_format = %{user | lower}

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/15-freedombox-mail.conf

@@ -0,0 +1,18 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# See: https://doc.dovecot.org/main/core/config/mail_location.html
+
+# Use sdbox, a format specific to dovecot, for storing mails. The format allows
+# better performance with some IMAP queries. When this is combined with Full
+# Text Search (FTS), users will get optimal web and desktop mail experience.
+# Don't pick mdbox format because is requires regular expunge maintenance. We
+# have enabled btrfs filesystem compression by default.
+mail_driver = sdbox
+mail_path = ~/mail
+
+# We try to deliver all mail using a single UID 'mail' and a single GID 'mail'.
+# In Debian, UID of mail user is 8 and GID of mail user is 8 as set in
+# /usr/share/base-passwd/{passwd|group}.master. By default first valid UID in
+# dovecot is 500.
+first_valid_uid = 8
+last_valid_uid = 8

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/90-freedombox-imap.conf

@@ -0,0 +1,11 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# Make rspamd learn spam/ham when the user marks mails as junk or not junk.
+# https://doc.dovecot.org/main/core/config/sieve/overview.html
+# https://doc.dovecot.org/main/core/plugins/sieve.html
+
+protocol imap {
+  mail_plugins {
+    imap_sieve = yes
+  }
+}

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/90-freedombox-lmtp.conf

@@ -0,0 +1,12 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# See:
+# https://doc.dovecot.org/main/core/config/sieve/overview.html
+# https://doc.dovecot.org/main/core/plugins/sieve.html
+
+# Enable the sieve plugin to sort mail during delivery using sieve scripts.
+protocol lmtp {
+  mail_plugins {
+    sieve = yes
+  }
+}

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/90-freedombox-mailboxes.conf

@@ -0,0 +1,72 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# Mark various mailboxes with special use flags (RFC 6154). Various names used
+# in mail clients for mailboxes: https://www.imapwiki.org/SpecialUse
+# See:
+# https://doc.dovecot.org/main/core/config/mail_location.html#custom-namespace-location
+
+namespace inbox {
+  # Archive
+  mailbox Archive {
+    auto = subscribe
+    special_use = \Archive
+  }
+  mailbox Archives {  # Thunderbird
+    auto = no
+    special_use = \Archive
+  }
+
+  # Drafts
+  mailbox Drafts {
+    auto = subscribe
+    special_use = \Drafts
+  }
+
+  # Sent
+  mailbox Sent {
+    auto = subscribe
+    special_use = \Sent
+  }
+  mailbox "Sent Items" {  # Outlook 2010/2013
+    auto = no
+    special_use = \Sent
+  }
+  mailbox "Sent Messages" {  # iOS
+    auto = no
+    special_use = \Sent
+  }
+
+  # Junk
+  mailbox Junk {
+    auto = subscribe
+    autoexpunge = 60d
+    special_use = \Junk
+  }
+  mailbox Spam {  # KMail, K-9 Mail
+    auto = no
+    autoexpunge = 60d
+    special_use = \Junk
+  }
+  mailbox "Junk E-mail" {  # Outlook 2010
+    auto = no
+    autoexpunge = 60d
+    special_use = \Junk
+  }
+  mailbox INBOX.Junk {
+    auto = no
+    autoexpunge = 60d
+    special_use = \Junk
+  }
+
+  # Trash
+  mailbox Trash {
+    auto = subscribe
+    autoexpunge = 60d
+    special_use = \Trash
+  }
+  mailbox INBOX.Trash {
+    auto = no
+    autoexpunge = 60d
+    special_use = \Trash
+  }
+}

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/90-freedombox-master.conf

@@ -0,0 +1,21 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# Listen on Unix domain sockets for postfix to use dovecot SASL authentication
+# and for postfix to deliver mail using dovecot to local mailboxes. See:
+# https://doc.dovecot.org/main/howto/sasl/postfix.html#postfix-and-dovecot-sasl
+
+service auth {
+  unix_listener /var/spool/postfix/private/auth {
+      mode = 0600
+      user = postfix
+      group = postfix
+  }
+}
+
+service lmtp {
+  unix_listener /var/spool/postfix/private/dovecot-lmtp {
+    mode = 0600
+    user = postfix
+    group = postfix
+  }
+}

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/90-freedombox-tls.conf

@@ -0,0 +1,12 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# Mozilla Guideline v5.7, Dovecot 2.3.21, OpenSSL 3.4.0, intermediate.
+# Generated 2025-07-16: https://ssl-config.mozilla.org/
+# See: https://doc.dovecot.org/main/core/config/ssl.html
+ssl = required
+
+ssl_min_protocol = TLSv1.2
+ssl_server_prefer_ciphers = client
+
+ssl_curve_list = X25519:prime256v1:secp384r1
+ssl_cipher_list = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/95-freedombox-sieve.conf

@@ -0,0 +1,43 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# Default sieve scripts applied for delivery to all users. To move mail to Junk
+# folder based on classification headers set by rspamd. See:
+# https://doc.dovecot.org/main/core/plugins/sieve.html
+
+sieve_script freedombox-after {
+  type = after
+  driver = file
+  path = /etc/dovecot/freedombox-sieve-after
+}
+
+sieve_plugins {
+  sieve_imapsieve = yes
+  sieve_extprograms = yes
+}
+
+sieve_global_extensions {
+  vnd.dovecot.pipe = yes
+  vnd.dovecot.environment = yes
+}
+
+# Make rspamd learn spam/ham when the user marks mails as junk or not junk.
+# https://doc.dovecot.org/main/core/config/spam_reporting.html
+sieve_pipe_bin_dir = /usr/bin
+
+# When moving a mail from to Junk folder from elsewhere
+mailbox Junk {
+  sieve_script learn-spam {
+    type = before
+    cause = copy
+    path = /etc/dovecot/freedombox-sieve/learn-spam.sieve
+  }
+}
+
+# When moving a mail from from Junk folder to elsewhere
+imapsieve_from Junk {
+  sieve_script learn-ham {
+    type = before
+    cause = copy
+    path = /etc/dovecot/freedombox-sieve/learn-ham.sieve
+  }
+}

plinth/modules/email/data/usr/share/freedombox/etc/dovecot/conf.d/2.4/freedombox-ldap.conf.ext

@@ -0,0 +1,4 @@
+# Do not edit this file. Manage your settings on FreedomBox.
+
+# This file is not needed for Dovecot >= 2.4. It is only needed for simplifying
+# compatibility with Dovecot 2.3.

plinth/modules/email/dovecot.py

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Utilities to configure Dovecot."""
+
+import apt
+
+from plinth.utils import Version
+
+
+def is_version_24():
+    """Return the currently installed version of Dovecot."""
+    cache = apt.Cache()
+    try:
+        version = cache['dovecot-core'].installed.version
+    except KeyError:
+        return True
+
+    return Version(version) >= Version('1:2.4')

plinth/modules/email/privileged/__init__.py

@@ -2,8 +2,7 @@
 """Provides privileged actions that run as root."""
 
 from .aliases import setup_aliases
-from .dkim import (get_dkim_public_key, setup_dkim,
-                   fix_incorrect_key_ownership)
+from .dkim import fix_incorrect_key_ownership, get_dkim_public_key, setup_dkim
 from .domain import set_domains
 from .home import setup_home
 from .postfix import setup_postfix

plinth/modules/email/privileged/tls.py

@@ -10,6 +10,7 @@ See: https://doc.dovecot.org/configuration_manual/dovecot_ssl_configuration/
 import pathlib
 
 from .. import postfix
+from ..dovecot import is_version_24
 
 # Mozilla Guideline v5.6, Postfix 1.17.7, OpenSSL 1.1.1d, intermediate
 # Generated 2021-08
@@ -68,15 +69,27 @@ def set_postfix_config(primary_domain, all_domains):
 
 def set_dovecot_config(primary_domain, all_domains):
     """Set dovecot configuration for TLS certificates."""
+    is_new_version = is_version_24()
+
+    # Determine whether to prefix file paths with '<' based on version
+    prefix = ''
+    cert_naming = 'ssl_server_cert_file'
+    key_naming = 'ssl_server_key_file'
+    if not is_new_version:
+        prefix = '<'
+        cert_naming = 'ssl_cert'
+        key_naming = 'ssl_key'
+
     content = f'''# This file is managed by FreedomBox
-ssl_cert = </etc/dovecot/letsencrypt/{primary_domain}/cert.pem
-ssl_key = </etc/dovecot/letsencrypt/{primary_domain}/privkey.pem
+{cert_naming} = {prefix}/etc/dovecot/letsencrypt/{primary_domain}/cert.pem
+{key_naming} = {prefix}/etc/dovecot/letsencrypt/{primary_domain}/privkey.pem
 '''
+
     for domain in all_domains:
         content += f'''
 local_name {domain} {{
-  ssl_cert = </etc/dovecot/letsencrypt/{domain}/cert.pem
-  ssl_key = </etc/dovecot/letsencrypt/{domain}/privkey.pem
+  {cert_naming} = {prefix}/etc/dovecot/letsencrypt/{domain}/cert.pem
+  {key_naming} = {prefix}/etc/dovecot/letsencrypt/{domain}/privkey.pem
 }}
 '''
     cert_config = pathlib.Path('/etc/dovecot/conf.d/91-freedombox-tls.conf')