email: Implement outbound mail filtering

- Make extensive use of the milter protocol
- Milter: add X-Fbx- headers to emails
- All submitted mails go to smtp:localhost:10025 for filtering
- Header privacy for submitted mails
- Rspamd: be able to inject and replace FreedomBox-managed config
- Reserve special addresses for future use

Known issue: internal emails do not go through spam filtering

plinth/modules/email_server/audit/spam.py

@@ -3,12 +3,13 @@
 
 import glob
 import logging
+import re
 import subprocess
 
 from plinth import actions
 
-import plinth.modules.email_server.postconf as postconf
 from . import models
+from plinth.modules.email_server import interproc, lock, postconf
 
 milter_config = {
     'milter_mail_macros': 'i ' + ' '.join([
@@ -19,9 +20,56 @@ milter_config = {
     # XXX In postconf this field is a list
     'smtpd_milters': 'inet:127.0.0.1:11332',
     # XXX In postconf this field is a list
-    'non_smtpd_milters': 'inet:127.0.0.1:11332'
+    'non_smtpd_milters': 'inet:127.0.0.1:11332',
+    'milter_header_checks': 'regexp:fbx-managed/pre-queue-milter-headers',
+
+    # Last-resort internal header cleanup at smtp client
+    'smtp_header_checks': 'regexp:/etc/postfix/freedombox-internal-cleanup',
+    # Reserved mail transports
+    # XXX This field is a list
+    'transport_maps': 'regexp:/etc/postfix/freedombox-transport-to',
+    # XXX This field is a list
+    'sender_dependent_default_transport_maps': \
+    'regexp:/etc/postfix/freedombox-transport-from',
 }
 
+# FreedomBox egress filtering
+
+egress_filter = postconf.ServiceFlags(
+    service='127.0.0.1:10025', type='inet', private='n', unpriv='-',
+    chroot='y', wakeup='-', maxproc='-', command_args='smtpd'
+)
+
+egress_filter_options = {
+    'syslog_name': 'postfix/fbxout',
+    'cleanup_service_name': 'fbxcleanup',
+    'content_filter': '',
+    'receive_override_options': 'no_unknown_recipient_checks',
+    'smtpd_helo_restrictions': '',
+    'smtpd_client_restrictions': '',
+    'smtpd_relay_restrictions': '',
+    'smtpd_recipient_restrictions': 'permit_mynetworks,reject',
+    'mynetworks': '127.0.0.0/8,[::1]/128'
+}
+
+egress_filter_cleanup = postconf.ServiceFlags(
+    service='fbxcleanup', type='unix', private='n', unpriv='-',
+    chroot='y', wakeup='-', maxproc='0', command_args='cleanup'
+)
+
+egress_filter_cleanup_options = {
+    'syslog_name': 'postfix/fbxout',
+    'header_checks': 'regexp:/etc/postfix/freedombox-header-cleanup',
+    'nested_header_checks': ''
+}
+
+# Rspamd config
+
+rspamd_boundary = re.compile('#[ ]*--[ ]*([A-Z]{3,5})[ ]+FREEDOMBOX CONFIG$')
+rspamd_header = '#-- BEGIN FREEDOMBOX CONFIG\n'
+rspamd_footer = '#-- END FREEDOMBOX CONFIG\n'
+
+rspamd_mutex = lock.Mutex('rspamd-config')
 logger = logging.getLogger(__name__)
 
 
@@ -37,7 +85,7 @@ def repair():
 
 
 def check_filter():
-    diagnosis = models.MainCfDiagnosis('Postfix milter')
+    diagnosis = models.MainCfDiagnosis('Inbound and outbound mail filters')
     current = postconf.get_many_unsafe(milter_config.keys())
     diagnosis.compare_and_advise(current=current, default=milter_config)
     return diagnosis
@@ -50,9 +98,69 @@ def fix_filter(diagnosis):
 
 
 def action_set_filter():
+    _compile_sieve()
+    postconf.set_master_cf_options(egress_filter, egress_filter_options)
+    postconf.set_master_cf_options(egress_filter_cleanup,
+                                   egress_filter_cleanup_options)
+
     with postconf.mutex.lock_all():
         fix_filter(check_filter())
-    _compile_sieve()
+
+    with rspamd_mutex.lock_all():
+        # XXX Maybe use globbing?
+        _inject_rspamd_config('override', 'options.inc')
+        _inject_rspamd_config('local', 'milter_headers.conf')
+
+
+def _inject_rspamd_config(type, name):
+    template_path = '/etc/plinth/rspamd-config/%s_%s' % (type, name)
+    config_path = '/etc/rspamd/%s.d/%s' % (type, name)
+
+    logger.info('Opening Rspamd config file %s', config_path)
+
+    template = None
+    config = None
+    try:
+        template = open(template_path, 'r')
+        config = open(config_path, 'a+')
+        with interproc.atomically_rewrite(config_path) as scratch:
+            config.seek(0)
+            inject_rspamd_config3(template, config, scratch)
+    finally:
+        if config is not None:
+            config.close()
+        if template is not None:
+            template.close()
+
+
+def inject_rspamd_config3(template, config, scratch):
+    """Write modified rspamd config to the `scratch` stream"""
+    # Copy the original up to the config header line
+    for line in config:
+        match = rspamd_boundary.match(line.strip())
+        if match and match.group(1) == 'BEGIN':
+            break
+        scratch.write(line)
+        if not line.endswith('\n'):  # in case no new line was at the eof
+            scratch.write('\n')
+
+    # Inject template data
+    scratch.write(rspamd_header)
+    for line in template:
+        scratch.write(line)
+        if not line.endswith('\n'):  # in case no new line was at the eof
+            scratch.write('\n')
+    scratch.write(rspamd_footer)
+
+    # Find the config trailer line
+    for line in config:
+        match = rspamd_boundary.match(line.strip())
+        if match and match.group(1) == 'END':
+            break
+
+    # Copy the original
+    for line in config:
+        scratch.write(line)  # keep original file ending style
 
 
 def _compile_sieve():
@@ -66,7 +174,5 @@ def _run_sievec(sieve_file):
     args = ['sievec', '--', sieve_file]
     completed = subprocess.run(args, capture_output=True)
     if completed.returncode != 0:
-        logger.critical('Subprocess returned %d', completed.returncode)
-        logger.critical('Stdout: %r', completed.stdout)
-        logger.critical('Stderr: %r', completed.stderr)
+        interproc.log_subprocess(completed)
         raise OSError('Sieve compilation failed: ' + sieve_file)

plinth/modules/email_server/data/etc/plinth/rspamd-config/local_milter_headers.conf

@@ -0,0 +1,39 @@
+# The following section is managed by FreedomBox
+# Be careful not to edit
+
+use = ["x-fbx-mail-type", "authentication-results", "x-spam-level",
+       "x-spam-status", "x-spamd-bar", "x-spamd-result"];
+
+routines {
+  authentication-results {
+    add_smtp_user = false;
+  }
+}
+
+# Custom headers required by FreedomBox filtering system
+custom {
+x-fbx-mail-type = <<EOD
+  return function(task, common_meta)
+    -- parameters are task and metadata from previous functions
+    -- add headers
+    local add_table = {["x-fbx-mail-type"] = "unknown;"}
+    local auth_user = task:get_user()
+    if auth_user == nil or auth_user == "" then
+      add_table["x-fbx-mail-type"] = "smtpd-anon;"
+    else
+      add_table["x-fbx-mail-type"] = "smtpd-submission;"
+      -- add_table["x-fbx-sasl-user"] = auth_user
+    end
+    -- remove foreign x-fbx- headers
+    local remove_table = {}
+    local function callback(header_name, header_value)
+      local prefix = "x-fbx-"
+      if header_name:lower():sub(1, #prefix) == prefix then
+        remove_table[header_name] = 0
+      end
+    end
+    task:headers_foreach(callback, {full = 'true'})
+    return nil, add_table, remove_table, {}
+  end
+EOD;
+}

plinth/modules/email_server/data/etc/plinth/rspamd-config/override_options.inc

@@ -0,0 +1,6 @@
+# The following section is managed by FreedomBox
+# Be careful not to edit
+
+# Do not trust 127.0.0.1
+local_addrs = [];
+local_networks = [];

plinth/modules/email_server/data/etc/postfix/freedombox-header-cleanup

@@ -0,0 +1,14 @@
+# -*- mode: conf-space -*-
+# This file is managed by FreedomBox. Direct edits will be lost!
+
+# Outbound header privacy
+/^Delivered-To:/ IGNORE
+/^Received[:-]/ IGNORE
+/^User-Agent:/ IGNORE
+/^X-EIP:/ IGNORE
+/^X-Mailer:/ IGNORE
+/^X-Originating-IP:/ IGNORE
+/^X-Sender:/ IGNORE
+# Currently localhost:10025 does not have a milter
+# so it is fine to delete internal headers now
+/^X-Fbx-/ IGNORE

plinth/modules/email_server/data/etc/postfix/freedombox-internal-cleanup

@@ -0,0 +1,5 @@
+# -*- mode: conf-space -*-
+# This file is managed by FreedomBox. Direct edits will be lost!
+
+# Scrub all internal headers
+/^X-Fbx-/ IGNORE

plinth/modules/email_server/data/etc/postfix/freedombox-transport-from

@@ -0,0 +1,12 @@
+# -*- mode: conf-space -*-
+# This file is managed by FreedomBox. Direct edits will be lost!
+
+# Sender dependent transport map format
+# Overrides default_transport
+
+# Reserved domain names
+/\.onion$/i error: not implemented: mail delivery from .onion
+/\.i2p$/i error: not implemented: mail delivery from .i2p
+/\.fm\.localhost$/i error: not implemented: mail delivery from .fm.localhost
+# Caret addresses
+/^[a-z0-9\-\.]+\^/i error: not implemented: caret addresses

plinth/modules/email_server/data/etc/postfix/freedombox-transport-to

@@ -0,0 +1,9 @@
+# -*- mode: conf-space -*-
+# This file is managed by FreedomBox. Direct edits will be lost!
+
+# Transport map format
+
+# Reserved domain names
+/\.onion$/i error: not implemented: mail delivery to .onion
+/\.i2p$/i error: not implemented: mail delivery to .i2p
+/\.fm\.localhost$/i error: not implemented: mail delivery to .fm.localhost

plinth/modules/email_server/data/var/spool/postfix/fbx-managed/README.txt

@@ -0,0 +1,5 @@
+DO NOT PUT PERSONAL ITEMS HERE!
+This folder in its entirety is managed by FreedomBox.
+
+/var/spool/postfix/fbx-managed/ contains config files read by chrooted Postfix
+services.

plinth/modules/email_server/data/var/spool/postfix/fbx-managed/pre-queue-milter-headers

@@ -0,0 +1,7 @@
+# -*- mode: conf-space -*-
+# This file is managed by FreedomBox. Direct edits will be lost!
+
+# Outbound
+/^X-Fbx-Mail-Type: smtpd-submission;/ FILTER smtp:[127.0.0.1]:10025
+# Inbound
+/^X-Fbx-Mail-Type: smtpd-anon;/ IGNORE

plinth/modules/email_server/manifest.py

@@ -55,9 +55,12 @@ backup = {
     'data': {
         'directories': [
             '/var/lib/plinth/mailsrv',
+            '/var/spool/postfix/fbx-managed',
             '/etc/postfix',
-            '/etc/dovecot'
+            '/etc/dovecot',
+            '/etc/rspamd',
+            '/var/lib/rspamd',
         ]
     },
-    'services': ['postfix', 'dovecot']
+    'services': ['postfix', 'dovecot', 'rspamd']
 }